Refactor: pems_streamlit package

.devcontainer/Dockerfile

@@ -50,10 +50,9 @@ ENV PATH="$PATH:/$USER/.local/bin" \
     PYTHONUSERBASE="/$USER/.local"
 
 # install devcontainer requirements
-RUN pip install -e .[dev,test]
-
+RUN pip install .[dev,test]
 # install docs requirements
 RUN pip install --no-cache-dir -r docs/requirements.txt
 
-# install streamlit requirements
-RUN pip install --no-cache-dir -r streamlit_app/requirements.txt
+# install source packages as editable
+RUN pip install -e ./pems_streamlit -e ./pems_web

.github/workflows/tests-pytest.yml

@@ -25,8 +25,7 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-          pip install -e .[test]
-          pip install -r streamlit_app/requirements.txt
+          pip install .[test] ./pems_streamlit ./pems_web
 
       - name: Run tests
         run: ./tests/pytest/run.sh

.vscode/launch.json

@@ -33,7 +33,7 @@
       "type": "debugpy",
       "request": "launch",
       "module": "streamlit",
-      "args": ["run", "streamlit_app/main.py"],
+      "args": ["run", "pems_streamlit/src/pems_streamlit/main.py"],
       "env": {
         "PYTHONBUFFERED": "1",
         "PYTHONWARNINGS": "default",
@@ -45,7 +45,7 @@
       "type": "debugpy",
       "request": "launch",
       "module": "streamlit",
-      "args": ["run", "streamlit_app/main.py"],
+      "args": ["run", "pems_streamlit/src/pems_streamlit/main.py"],
       "env": {
         "PYTHONBUFFERED": "1",
         "PYTHONWARNINGS": "default",

compose.yml

@@ -69,13 +69,11 @@ services:
   streamlit:
     build:
       context: .
-      dockerfile: streamlit_app/Dockerfile
+      dockerfile: pems_streamlit/container/Dockerfile
     image: caltrans/pems:streamlit
     env_file: .env
     ports:
       - "${STREAMLIT_LOCAL_PORT:-8501}:8501"
-    volumes:
-      - ./:/caltrans/app
 
 volumes:
   pgdata:

infra/copilot/streamlit/manifest.yml

@@ -17,7 +17,7 @@ http:
 image:
   # Docker build arguments. For additional overrides: https://aws.github.io/copilot-cli/docs/manifest/backend-service/#image-build
   build:
-    dockerfile: ../streamlit_app/Dockerfile
+    dockerfile: ../pems_streamlit/container/Dockerfile
     context: ../
   # Port exposed through your container to route traffic to it.
   port: 8501

pems_streamlit/container/Dockerfile

@@ -0,0 +1,61 @@
+ARG PYTHON_VERSION=3.12
+
+# multi-stage build
+#
+# stage 1: builds the package wheel from source
+#          using the git metadata for version info
+FROM python:${PYTHON_VERSION} AS build_wheel
+WORKDIR /build
+
+# upgrade pip, install build dependencies
+RUN python -m pip install --upgrade pip build setuptools_scm
+
+# copy source files
+COPY . .
+RUN git config --global --add safe.directory /build
+
+# Move into directory to run the build
+WORKDIR /build/pems_streamlit
+
+# build package
+RUN python -m build
+
+# multi-stage build
+#
+# stage 2: installs the wheel in a fresh base container
+#          using the pre-built package, and copying only needed source
+FROM python:${PYTHON_VERSION} AS app_container
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    USER=caltrans
+
+EXPOSE 8501
+
+# create non-root $USER and home directory
+RUN useradd --create-home --shell /bin/bash $USER && \
+    python -m pip install --upgrade pip
+
+COPY LICENSE LICENSE
+
+# switch to non-root $USER
+USER $USER
+
+# update env for local pip installs
+# see https://docs.python.org/3/using/cmdline.html#envvar-PYTHONUSERBASE
+# since all `pip install` commands are in the context of $USER
+# $PYTHONUSERBASE is the location used by default
+ENV PATH="$PATH:/$USER/.local/bin" \
+    PYTHONUSERBASE="/$USER/.local" \
+    PYTHONPATH="$PYTHONPATH:/$USER/app"
+
+WORKDIR /$USER/app
+
+COPY .streamlit .streamlit
+COPY pems_streamlit/container/entrypoint.sh entrypoint.sh
+COPY pems_streamlit/container/run.py run.py
+COPY --from=build_wheel /build/pems_streamlit/dist /wheels
+
+RUN pip install $(find /wheels -name pems_streamlit*.whl)
+
+ENTRYPOINT ["./entrypoint.sh"]

streamlit_app/entrypoint.sh → pems_streamlit/container/entrypoint.sh

@@ -7,7 +7,8 @@ BASE_URL_PATH=${STREAMLIT_BASE_URL:-""}
 echo "Starting Streamlit with base URL path: ${BASE_URL_PATH}"
 
 # Execute the Streamlit command, passing the base URL path
-exec streamlit run streamlit_app/main.py \
+# Use the run.py helper since streamlit run wants the path to a python file
+exec streamlit run run.py \
   --server.port=8501 \
   --server.address=0.0.0.0 \
   --server.baseUrlPath="${BASE_URL_PATH}"

pems_streamlit/container/run.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+import runpy
+
+# since we're installing pems_streamlit as a package now, the main.py file won't exist
+# in the running container, so we can't point `streamlit run` at it
+
+# instead, use runpy to run the pems_streamlit.main module directly.
+
+if __name__ == "__main__":
+    runpy.run_module("pems_streamlit.main", run_name="__main__", alter_sys=True)

pems_streamlit/pyproject.toml

@@ -0,0 +1,27 @@
+[project]
+name = "pems_streamlit"
+description = "The Streamlit application for PeMS data visualizations."
+dynamic = ["version"]
+requires-python = ">=3.12"
+dependencies = [
+    "boto3==1.39.7",
+    "django==5.2.3",
+    "pandas==2.3.0",
+    "streamlit==1.45.1",
+]
+
+[build-system]
+requires = ["setuptools>=75", "setuptools_scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["pems_streamlit*"]
+namespaces = false
+
+[tool.setuptools.package-data]
+pems_streamlit = ["*.txt"]
+
+[tool.setuptools_scm]
+# Tell scm to look one directory up for the .git folder
+root = ".."

streamlit_app/main.py → pems_streamlit/src/pems_streamlit/main.py

@@ -1,10 +1,9 @@
 import logging
 import os
 
-
 import streamlit as st
 
-from streamlit_app.utils import discover_apps
+from pems_streamlit.utils import discover_apps
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)

streamlit_app/utils.py → pems_streamlit/src/pems_streamlit/utils.py

@@ -15,6 +15,7 @@ def _convert_to_pages(apps: list[Path]) -> list[StreamlitPage]:
 
 
 def _default_app_page() -> StreamlitPage:
+    """Creates a no-op default page."""
     return st.Page(APP_DIR / "__init__.py", default=True, title="PeMS Streamlit apps")
 
 
@@ -54,5 +55,6 @@ def discover_apps() -> list[StreamlitPage]:
     default_app_page = _default_app_page()
     apps = _discover_apps()
     pages = _convert_to_pages(apps)
+    # Streamlit needs at least one page marked as default
     pages.insert(0, default_app_page)
     return pages

pyproject.toml

@@ -8,8 +8,9 @@ license-files = ["LICENSE"]
 classifiers = ["Programming Language :: Python :: 3 :: Only"]
 requires-python = ">=3.12"
 maintainers = [{ name = "Compiler LLC", email = "dev@compiler.la" }]
-# Link all the local packages so they are installed together
-dependencies = ["pems_web @ file:./pems_web"]
+# local packages are installed as part of devcontainer build
+# to ensure they are all installed in editable mode
+dependencies = []
 
 [project.optional-dependencies]
 dev = [
@@ -41,8 +42,8 @@ include = '\.pyi?$'
 [tool.coverage.run]
 branch = true
 relative_files = true
-source = ["pems_web", "streamlit_app"]
-omit = ["streamlit_app/apps/*"]
+source = ["pems_streamlit", "pems_web"]
+omit = ["pems_streamlit/src/pems_streamlit/apps/*"]
 
 [tool.djlint]
 ignore = "H006,H017,H031"
@@ -56,4 +57,4 @@ use_gitignore = true
 [tool.pytest.ini_options]
 DJANGO_SETTINGS_MODULE = "tests.pytest.pems_web.settings"
 # Explicitly add src directories to pytest's python path.
-pythonpath = ["pems_web/src"]
+pythonpath = ["pems_streamlit/src", "pems_web/src"]

tests/pytest/streamlit_app/test_main.py → tests/pytest/pems_streamlit/test_main.py

@@ -1,5 +1,6 @@
 import pytest
-from streamlit_app import main
+
+from pems_streamlit import main
 
 
 @pytest.mark.parametrize(
@@ -19,8 +20,8 @@ def test_main(mocker, monkeypatch, nav_option, expected_nav):
     else:
         monkeypatch.delenv(nav_key, False)
 
-    mock_discover = mocker.patch("streamlit_app.main.discover_apps", return_value=apps)
-    mock_navigate = mocker.patch("streamlit_app.main.st.navigation")
+    mock_discover = mocker.patch("pems_streamlit.main.discover_apps", return_value=apps)
+    mock_navigate = mocker.patch("pems_streamlit.main.st.navigation")
 
     main.main()
 

tests/pytest/streamlit_app/test_utils.py → tests/pytest/pems_streamlit/test_utils.py

@@ -3,7 +3,7 @@
 import pytest
 from streamlit.navigation.page import StreamlitPage
 
-from streamlit_app import utils
+from pems_streamlit import utils
 
 
 @pytest.fixture
@@ -25,7 +25,7 @@ def app_paths(test_apps_path):
 
 
 def test_convert_to_pages(mocker, app_paths):
-    page_factory = mocker.patch("streamlit_app.utils._make_app_page", return_value="page")
+    page_factory = mocker.patch("pems_streamlit.utils._make_app_page", return_value="page")
 
     pages = utils._convert_to_pages(app_paths)
 
@@ -67,8 +67,8 @@ def test_make_app_page(app_paths):
 
 
 def test_discover_apps(mocker):
-    mock_discover = mocker.patch("streamlit_app.utils._discover_apps", return_value=[])
-    mock_convert = mocker.patch("streamlit_app.utils._convert_to_pages", return_value=[])
+    mock_discover = mocker.patch("pems_streamlit.utils._discover_apps", return_value=[])
+    mock_convert = mocker.patch("pems_streamlit.utils._convert_to_pages", return_value=[])
 
     result = utils.discover_apps()