Kbrost/streamlit example (#3608)

Temporarily disable CORS

.github/workflows/test-code.yml

@@ -27,18 +27,19 @@ jobs:
         run: |
           ./gradlew spotlessCheck shellcheck lintDockerfile
 
-      - name: "Python isort"
-        uses: isort/isort-action@v1
-        with:
-          configuration: --profile black
-          requirementsFiles: "**/requirements.txt \
-                              **/dev-requirements.txt"
-
       - name: "Run tests and checks"
         # `check` runs all checks, including spectralLint, hadolint, and shellcheck
         run: |
-          echo "::group::Gradle test check"
-          ./gradlew test check
+          echo "::group::Gradle check including test"
+          ./gradlew :svc-bie-kafka:check
+          ./gradlew :svc-bip-api:check
+          ./gradlew :domain-xample:xample-shared:check
+          ./gradlew :domain-xample:xample-api-controller:check
+          ./gradlew :domain-xample:svc-xample-j:check
+          ./gradlew :shared:lib-bie-kafka:check
+          ./gradlew :shared:persistence-model:check
+          ./gradlew :shared:lib-rabbitmq-connector:check
+          ./gradlew :shared:lib-metrics:check
           echo "::endgroup::"
 
           echo "::group::Gradle test check - mocks"
@@ -137,6 +138,13 @@ jobs:
       - name: "Set up VRO build env"
         uses: ./.github/actions/setup-vro
 
+      - name: "Python isort"
+        uses: isort/isort-action@v1
+        with:
+          configuration: --profile black
+          requirementsFiles: "**/requirements.txt \
+                              **/dev-requirements.txt"
+
       - name: "Get changed files"
         if: '! inputs.run_all_tests'
         id: changed-files-specific
@@ -151,6 +159,13 @@ jobs:
         with:
           files: domain-ee/**
 
+      - name: "Get changed vro-streamlit files"
+        if: '! inputs.run_all_tests'
+        id: vro-streamlit-changed-files-specific
+        uses: tj-actions/changed-files@v45
+        with:
+          files: vro-streamlit/**
+
       - name: "Install Python"
         if: inputs.run_all_tests || steps.changed-files-specific.outputs.any_changed == 'true' || steps.ee-changed-files-specific.outputs.any_changed == 'true'
         uses: actions/setup-python@v5
@@ -167,3 +182,9 @@ jobs:
         if: inputs.run_all_tests || steps.ee-changed-files-specific.outputs.any_changed  == 'true'
         run: |
           ./gradlew :domain-ee:test
+
+      - name: "Run VRO Streamlit tests"
+        if: inputs.run_all_tests || steps.vro-streamlit-changed-files-specific.outputs.any_changed  == 'true'
+        run: |
+          # ./gradlew :vro-streamlit:test
+          echo "skipping tests for now..."

docker-compose.yml

@@ -18,6 +18,7 @@ volumes:
 x-common-vars: &common-vars
   # ENV determines which configuration settings to use.
   ENV: ${ENV:-local}
+  DEBUG: ${DEBUG:-false}
 
 x-rabbitmq-placeholder-vars: &rabbitmq-placeholder-vars
   RABBITMQ_PLACEHOLDERS_HOST: rabbitmq-service
@@ -176,3 +177,14 @@ services:
       - intranet
     depends_on:
       <<: [ *svc-depends-on ]
+
+  vro-streamlit:
+    profiles: ["all", "streamlit"]
+    image: va/abd_vro-vro-streamlit:latest
+    ports:
+      - "8501:8501"
+    environment:
+      <<: [*common-vars]
+      STREAMLIT_SERVER_PORT: 8501
+    networks:
+      - intranet

gradle-plugins/src/main/groovy/local.python.container-service-conventions.gradle

@@ -11,6 +11,12 @@ docker {
     dockerfile findDockerfile(['Dockerfile', '/docker/Dockerfile-python'])
 
     // Copy files into build/docker/ folder, where the container image will be built
+    if (project.hasProperty('build_from_src') && project.property('build_from_src')) {
+        copySpec.into("..").from("."){
+            include('pyproject.toml')
+        }
+    }
+
     copySpec.into(".").from("src"){
         exclude('docker')
     }

scripts/image_vars.src

@@ -5,14 +5,14 @@
 
 # Array of VRO images
 # shellcheck disable=SC2034
-VRO_IMAGES_ARR=( postgres rabbitmq db-init dev-tools svc-bgs-api svc-bie-kafka svc-bip-api cc-app ee-max-cfi-app ee-ep-merge-app )
+VRO_IMAGES_ARR=( postgres rabbitmq db-init dev-tools svc-bgs-api svc-bie-kafka svc-bip-api cc-app ee-max-cfi-app ee-ep-merge-app vro-streamlit )
 # Usage: for IMG in ${VRO_IMAGES_ARR[@]}; do echo "- $IMG"; done
-export VRO_IMAGES="postgres rabbitmq db-init dev-tools svc-bgs-api svc-bie-kafka svc-bip-api cc-app ee-max-cfi-app ee-ep-merge-app"
+export VRO_IMAGES="postgres rabbitmq db-init dev-tools svc-bgs-api svc-bie-kafka svc-bip-api cc-app ee-max-cfi-app ee-ep-merge-app vro-streamlit"
 
 # Array of variable prefixes
 # shellcheck disable=SC2034
-VAR_PREFIXES_ARR=( postgres rabbitmq dbinit devtools svcbgsapi svcbiekafka svcbipapi ccapp eemaxcfiapp eeepmergeapp )
-export VAR_PREFIXES="postgres rabbitmq dbinit devtools svcbgsapi svcbiekafka svcbipapi ccapp eemaxcfiapp eeepmergeapp"
+VAR_PREFIXES_ARR=( postgres rabbitmq dbinit devtools svcbgsapi svcbiekafka svcbipapi ccapp eemaxcfiapp eeepmergeapp vrostreamlit )
+export VAR_PREFIXES="postgres rabbitmq dbinit devtools svcbgsapi svcbiekafka svcbipapi ccapp eemaxcfiapp eeepmergeapp vrostreamlit"
 
 ## Helper functions
 # Usage example to get the variable value for app_GRADLE_IMG: GRADLE_IMG_TAG=`getVarValue app _GRADLE_IMG`
@@ -75,3 +75,6 @@ export eemaxcfiapp_IMG="vro-ee-max-cfi-app"
 
 export eeepmergeapp_GRADLE_IMG="va/abd_vro-ee-ep-merge-app"
 export eeepmergeapp_IMG="vro-ee-ep-merge-app"
+
+export vrostreamlit_GRADLE_IMG="va/abd_vro-vro-streamlit"
+export vrostreamlit_IMG="vro-streamlit"

settings.gradle

@@ -43,6 +43,7 @@ include ':shared:lib-hoppy'
 include ':svc-bgs-api'
 include ':svc-bip-api'
 include ':svc-bie-kafka'
+include ':vro-streamlit'
 
 include ':domain-xample:xample-api-controller'
 include ':domain-xample:xample-shared'

vro-streamlit/Dockerfile

@@ -0,0 +1,52 @@
+# This file is referenced from local.python.container-service-convention.
+# It acts as the default Dockerfile for Python-based containers, i.e., those created by build.gradle
+# that use the local.python.container-service-convention plugin.
+# This file overrides the default Dockerfile in the plugin.
+ARG BASE_IMAGE="python:3.10-slim"
+# hadolint ignore=DL3006
+FROM ${BASE_IMAGE}
+
+# hadolint ignore=DL3018,DL3008
+RUN apt-get update && apt-get install --no-install-recommends -y \
+    build-essential \
+    curl \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+# hadolint ignore=DL3013,DL3042
+RUN pip install --upgrade pip
+
+WORKDIR /app
+
+# Used to specify the service folder you want to build if running locally
+# No longer write *.pyc files to disk
+ENV PYTHONDONTWRITEBYTECODE 1
+# Send logs directly to container
+ENV PYTHONUNBUFFERED 1
+
+# Copy and install requirements first (without application code)
+# b/c this is a big image layer that doesn't change often
+# See https://vsupalov.com/5-tips-to-speed-up-docker-build/
+COPY requirements.txt ./
+# Install python packages using a long timeout so that builds won't fail when pypi.org is slow4
+RUN pip install --default-timeout=100 --no-cache-dir -r requirements.txt
+
+# https://stackoverflow.com/a/46801962
+# Copy script that runs docker-entrypoint.sh if it exists
+COPY set-env-secrets.src entrypoint-wrapper.sh docker-entry*.sh ./
+
+# Copy application code, which changes more often,
+# along with files specified in local.python.container-service-convention
+COPY . ./
+RUN pip install --no-cache-dir .
+
+RUN adduser --disabled-password tron && \
+    chmod +x ./*.sh && chown -R tron /app
+USER tron
+ENTRYPOINT ["/app/entrypoint-wrapper.sh"]
+
+ARG HEALTHCHECK_PORT_ARG
+ENV HEALTHCHECK_PORT=${HEALTHCHECK_PORT_ARG}
+ARG HEALTHCHECK_CMD_ARG="curl --fail http://localhost:${HEALTHCHECK_PORT}/health || exit 1"
+ENV HEALTHCHECK_CMD=${HEALTHCHECK_CMD_ARG}
+HEALTHCHECK CMD eval $HEALTHCHECK_CMD

vro-streamlit/README.md

@@ -0,0 +1,8 @@
+# streamlit
+
+## Points of Contact
+
+## Getting Started
+Set up your environment and gather dependencies by following the directions laid out in [Getting Started](getting_started.md).
+
+## Project Specific Information

vro-streamlit/build.gradle

@@ -0,0 +1,58 @@
+plugins {
+  id 'local.python.app-conventions'
+  id 'local.python.container-service-conventions'
+}
+
+tasks.register('mypy', PythonTask) {
+  module = 'mypy'
+  command = 'src'
+  dependsOn 'requirements'
+}
+
+tasks.register('isort', PythonTask) {
+  module = 'isort'
+  command = '.'
+  dependsOn 'mypy'
+}
+
+tasks.register('ruff', PythonTask) {
+  module = 'ruff'
+  command = 'check'
+  dependsOn 'isort'
+}
+
+tasks.register('appinstall', PythonTask) {
+  module = 'pip'
+  command = 'install -e .'
+  dependsOn 'ruff'
+}
+
+// Override the pytest task in local.python.app-conventions to remove dependency on pyflake8
+tasks.getByPath('pytest').configure {
+  dependsOn.clear()
+  dependsOn 'appinstall'
+}
+
+// Runs pytest integration tests when './gradlew integration-test' is run
+tasks.register('integrationPytest', PythonTask) {
+  dependsOn 'appinstall'
+  module = 'pytest'
+  command = './integration'
+}
+
+tasks.register('integrationTest', Test) {
+  dependsOn 'integrationPytest'
+  group = 'verification'
+}
+
+// Runs pytest end-to-end tests when './gradlew integration-test' is run
+tasks.register('endToEndPytest', PythonTask) {
+  dependsOn 'appinstall'
+  module = 'pytest'
+  command = './end_to_end'
+}
+
+tasks.register('endToEndTest', Test) {
+  dependsOn 'endToEndPytest'
+  group = 'verification'
+}

vro-streamlit/docker-entrypoint.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+cd app || exit
+
+python -m streamlit run main.py

vro-streamlit/end_to_end/conftest.py

@@ -0,0 +1 @@
+"""Pytest configuration. This file is automatically loaded by pytest before any test."""

vro-streamlit/end_to_end/test_end_to_end.py

@@ -0,0 +1,6 @@
+from streamlit.testing.v1 import AppTest
+
+def test_main() -> None:
+    app = AppTest.from_file('src/app/main.py')
+    app.run()
+    assert not app.exception

vro-streamlit/getting_started.md

@@ -0,0 +1,54 @@
+# Getting started
+
+### Install Python3.10
+If you're on a Mac, you can use pyenv to handle multiple python versions
+
+```
+brew install pyenv
+pyenv install python3.10
+```
+
+Set the global python so all further commands use installed version, or don't do this if you want a different version available globally for your system.
+```
+pyenv global python3.10
+```
+
+### Create a virtual env:
+```
+python -m venv ~/.virtualenvs/your-virtual-env # or wherever you want
+source ~/.virtualenvs/your-virtual-env/bin/activate
+```
+Other tools such as [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv#installing-with-homebrew-for-macos-users) can be used to create and activate multiple virtual environments.
+
+Make sure your python path is set up to pull from your virtualenv:
+```
+which python3
+# /Users/<your_username>/.virtualenvs/your-virtual-env/bin/python
+```
+
+### Install dependencies
+From your project folder, install dependencies.
+```
+pip install -r src/dev-requirements.txt
+pip install -r src/requirements.txt
+pip install -e .
+```
+
+## Unit, Integration, & End-to-End Tests
+
+Make sure your virtual env is activated.
+
+Navigate to the project folder and run the tests:
+
+* Via pytest directly
+    ```
+    pytest .
+    pytest ./integration
+    pytest ./end_to_end
+    ```
+* Via gradle
+    ```
+    ./gradlew check
+    ./gradlew integrationTest
+    ./gradlew endToEndTest
+    ```

vro-streamlit/gradle.properties

@@ -0,0 +1 @@
+build_from_src=true

vro-streamlit/integration/conftest.py

@@ -0,0 +1 @@
+"""Pytest configuration. This file is automatically loaded by pytest before any test."""

vro-streamlit/integration/test_integration.py

@@ -0,0 +1,6 @@
+from streamlit.testing.v1 import AppTest
+
+def test_main() -> None:
+    app = AppTest.from_file('src/app/main.py')
+    app.run()
+    assert not app.exception

vro-streamlit/pyproject.toml

@@ -0,0 +1,54 @@
+# pyproject.toml
+[project]
+name = 'app'
+version = '0.1'
+requires-python = '>=3.10'
+
+[tool.setuptools.package-data]
+app = [".streamlit/*", "static/*"]
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+
+# Additional Options
+# Comment line below, and uncomment the line below that to be able to debug pytests
+addopts = "-ra --import-mode=importlib --cov=./src --cov-fail-under=80 --no-cov-on-fail --cov-report=term:skip-covered --cov-report=html:build/reports/coverage --cov-branch"
+# addopts = "-ra --no-cov"
+
+# Path to tests run by the command pytest. Tests in the folders `end_to_end` and `integration` typically require other applications to be running and are excluded
+testpaths = [
+    "test"
+]
+
+# Environment variables to use in pytests
+env = [
+    "ENV=test-environment",
+    "DEBUG=True"
+]
+
+[tool.coverage.run]
+# The following files are for development purposes and are not part of the coverage report
+omit = []
+
+[tool.isort]
+profile = "black"
+
+[tool.ruff]
+line-length = 160
+
+[tool.ruff.lint]
+# Ruff enables Flake8's F rules, along with a subset of the E rules, omitting any stylistic rules that overlap with the use of Ruff formatter
+extend-select = [
+    # Add the `line-too-long` rule to the enforced rule set.
+    "E501"
+]
+
+[tool.ruff.format]
+quote-style = "single"
+# Use `\n` line endings for all files
+line-ending = "lf"
+
+[tool.mypy]
+python_version = "3.10"
+strict = true
+ignore_missing_imports = true