Make deployment on cloud.gov easy

.github/workflows/build_docker_images.yml

@@ -30,6 +30,7 @@ on:
   push:
     branches:
       - main
+      - deploy-to-cloud-gov
       - keycloak-realm-with-groups
     tags: [v*]
 
@@ -38,7 +39,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       REGISTRY: ghcr.io
-      IMAGE_NAME: sartography/spiffworkflow-frontend
+      IMAGE_NAME: gsa-tts/spiffworkflow-frontend
       BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
     permissions:
       contents: read
@@ -94,7 +95,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       REGISTRY: ghcr.io
-      IMAGE_NAME: sartography/spiffworkflow-backend
+      IMAGE_NAME: gsa-tts/spiffworkflow-backend
       BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
     permissions:
       contents: read
@@ -150,7 +151,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       REGISTRY: ghcr.io
-      IMAGE_NAME: sartography/connector-proxy-demo
+      IMAGE_NAME: gsa-tts/connector-proxy-demo
       BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
 
     permissions:

.github/workflows/trivy.yml

@@ -0,0 +1,49 @@
+---
+name: Trivy Scan
+on:
+  workflow_dispatch:
+  workflow_call:
+  push:
+    branches:
+      - main
+      - deploy-to-cloud-gov
+
+permissions:
+  contents: read
+
+jobs:
+  scan-third-party:
+    permissions:
+      contents: read
+      security-events: write
+      actions: read
+    name: Trivy Scan Third Party Images
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - name: ghcr.io/gsa-tts/spiffworkflow-backend:deploy-to-cloud-gov-latest
+          - name: ghcr.io/gsa-tts/spiffworkflow-frontend:deploy-to-cloud-gov-latest
+          - name: ghcr.io/gsa-tts/connector-proxy-demo:deploy-to-cloud-gov-latest
+    steps:
+      - name: Pull Third Party Docker Images
+        run: docker pull ${{ matrix.image.name }}
+
+      - name: Run Trivy vulnerability scanner on Third Party Images
+        uses: aquasecurity/[email protected]
+        with:
+          image-ref: '${{ matrix.image.name }}'
+          scan-type: 'image'
+          hide-progress: false
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+          exit-code: 0 # Setting the exit-code to 1 will fail the action, without publishing to Github Security Tab (> aquasecurity/[email protected])
+          severity: 'CRITICAL,HIGH'
+          timeout: 15m0s
+          ignore-unfixed: true
+
+      - name: Upload Trivy scan results to GitHub Security tab for Third Party Images
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: 'trivy-results.sarif'

.gitignore

@@ -13,3 +13,4 @@ process_models/
 .mypy_cache
 .aider*
 /spiffworkflow_docker_compose
+vars.yml

README-cloudgov.md

@@ -0,0 +1,26 @@
+### Cloudgov Sandbox Necessities
+Create a rds instance for use with the system called `spiffworkflow-db`:
+`cf create-service aws-rds micro-psql spiffworkflow-db`
+
+In order to utilize the newly created db, the following security groups will also need to be added to your sandbox:
+`cf bind-security-group trusted_local_networks ORGNAME --lifecycle running --space SPACENAME`
+`cf bind-security-group trusted_local_networks_egress ORGNAME --lifecycle running --space SPACENAME`
+
+In order to successfully get the front end to redirect in the sandbox space, run the following command:
+`cf bind-security-group public_networks_egress ORGNAME --lifecycle running --space SPACENAME`
+
+In order to enable the backend to connect to the connector proxy, run the following command, using your value for slug in vars.yml:
+`cf add-network-policy spiffworkflow((slug))-backend spiffworkflow((slug))-connector --port 61443 --protocol tcp`
+
+For example, if your value for `slug` in `vars.yml` was `-abc123` then this command would be:
+`cf add-network-policy spiffworkflow-abc123-backend spiffworkflow-abc123-connector --port 61443 --protocol tcp`
+
+### Generating a Github SSH Key
+In order to utilize some of the processes in this application, we utilize a forked process model repo, accessible via ssh endpoint.
+`[email protected]:GSA-TTS/gsa-process-models.git`.
+Generate a new ssh key pairing for use with git, using `ssh-keygen -t rsa -b 4096 -C "[email protected]"`. Once this has been created, navigate to the location where the keys are stored, usually `~/.ssh/` and copy the `<my_key_name>.pub` to [Github SSH Keys](https://github.com/settings/keys). Then, copy the plaintext `<my_key_name>` private key to `vars.yml` under the variable `github_ssh_key`. Alternatively, if you wish to use the actual keyfile and copy it onto the deployed instance for use, you can use the environment var `SPIFFWORKFLOW_BACKEND_GIT_SSH_PRIVATE_KEY_PATH:`
+
+### Deploying
+- Create a database service
+- Copy vars.yml-template to `vars.yml`
+- `cf push --vars-file vars.yml`

manifest.yml

@@ -0,0 +1,117 @@
+---
+defaults: &defaults
+  disk_quota: 3G
+  instances: 1
+  default-route: false
+applications:
+
+#################################
+- name: spiffworkflow((slug))-connector
+  <<: *defaults
+  memory: 256M
+  disk_quota: 3G
+  health-check-type: http
+  health-check-http-endpoint: /liveness
+  routes:
+    - route: spiffworkflow((slug))-connector.apps.internal
+  docker:
+    image: ((connector-image))
+  command: |
+    # Make sure the Cloud Foundry-provided CA is recognized when making TLS connections
+    cat /etc/cf-system-certificates/* > /usr/local/share/ca-certificates/cf-system-certificates.crt
+    /usr/sbin/update-ca-certificates
+    /app/bin/boot_server_in_docker
+  env:
+    FLASK_ENV: "${FLASK_ENV:-development}"
+    FLASK_DEBUG: "0"
+    FLASK_SESSION_SECRET_KEY: ((connector-flask-secret-key))
+    CONNECTOR_PROXY_PORT: "8080"
+    REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
+
+#################################
+- name: spiffworkflow((slug))-backend
+  <<: *defaults
+  disk_quota: 3G
+  health-check-http-endpoint: /api/v1.0/status
+  health-check-type: http
+  routes:
+    - route: spiffworkflow((slug)).app.cloud.gov/api
+  services:
+    - ((db-instance))
+  docker:
+    image: ((backend-image))
+  memory: 512M
+  command: |
+    # Get the postgres URI from the service binding. (SQL Alchemy insists on "postgresql://".🙄)
+    export SPIFFWORKFLOW_BACKEND_DATABASE_URI=$( echo $VCAP_SERVICES | jq -r '.["aws-rds"][].credentials.uri' | sed -e s/postgres/postgresql/ )
+
+    # Make sure the Cloud Foundry-provided CA is recognized when making TLS connections
+    cat /etc/cf-system-certificates/* > /usr/local/share/ca-certificates/cf-system-certificates.crt
+    /usr/sbin/update-ca-certificates
+
+    # Verify that this is working. It should return '{"ok": true}'
+    # curl https://spiffworkflow((slug))-connector.apps.internal:61443/liveness
+
+    /app/bin/clone_process_models
+    /app/bin/boot_server_in_docker
+  env:
+    APPLICATION_ROOT: "/"
+    FLASK_SESSION_SECRET_KEY: ((backend-flask-session-key))
+    FLASK_DEBUG: "0"
+    # This is necessary for Python to pick up custom CAs
+    REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt"
+
+    # All of the configuration variables are documented here:
+    # spiffworkflow-backend/src/spiffworkflow_backend/config/default.py
+    SPIFFWORKFLOW_BACKEND_BPMN_SPEC_ABSOLUTE_DIR: "/app/process_models"
+    SPIFFWORKFLOW_BACKEND_CHECK_FRONTEND_AND_BACKEND_URL_COMPATIBILITY: "false"
+    SPIFFWORKFLOW_BACKEND_CONNECTOR_PROXY_URL: "https://spiffworkflow((slug))-connector.apps.internal:61443"
+    SPIFFWORKFLOW_BACKEND_DATABASE_TYPE: "postgres"
+    SPIFFWORKFLOW_BACKEND_ENV: "local_docker"
+    SPIFFWORKFLOW_BACKEND_EXTENSIONS_API_ENABLED: "true"
+    SPIFFWORKFLOW_BACKEND_GIT_COMMIT_ON_SAVE: "true"
+    SPIFFWORKFLOW_BACKEND_GIT_PUBLISH_CLONE_URL: ((git-process-models-repo))
+    SPIFFWORKFLOW_BACKEND_GIT_PUBLISH_TARGET_BRANCH: ((target-branch-for-publish))
+
+    # This branch needs to exist, otherwise we can't clone it at startup and startup fails
+    SPIFFWORKFLOW_BACKEND_GIT_SOURCE_BRANCH: ((source-branch))
+    SPIFFWORKFLOW_BACKEND_GIT_SSH_PRIVATE_KEY: ((github-ssh-key))
+    SPIFFWORKFLOW_BACKEND_LOAD_FIXTURE_DATA: "false"
+    SPIFFWORKFLOW_BACKEND_LOG_LEVEL: "INFO"
+    SPIFFWORKFLOW_BACKEND_OPEN_ID_CLIENT_ID: "spiffworkflow-backend"
+    SPIFFWORKFLOW_BACKEND_OPEN_ID_CLIENT_SECRET_KEY: "((openid-secret))"
+    SPIFFWORKFLOW_BACKEND_OPEN_ID_SERVER_URL: "https://spiffworkflow((slug)).app.cloud.gov/api/openid"
+    SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME: "example.yml"
+    SPIFFWORKFLOW_BACKEND_PORT: "8080"
+    SPIFFWORKFLOW_BACKEND_RUN_BACKGROUND_SCHEDULER_IN_CREATE_APP: "true"
+    SPIFFWORKFLOW_BACKEND_UPGRADE_DB: "true"
+    SPIFFWORKFLOW_BACKEND_URL: "https://spiffworkflow((slug)).app.cloud.gov/api"
+    SPIFFWORKFLOW_BACKEND_URL_FOR_FRONTEND: "https://spiffworkflow((slug)).app.cloud.gov"
+    SPIFFWORKFLOW_BACKEND_USE_WERKZEUG_MIDDLEWARE_PROXY_FIX: "true"
+    SPIFFWORKFLOW_BACKEND_WSGI_PATH_PREFIX: "/api"
+
+################################
+- name: spiffworkflow((slug))-frontend
+  <<: *defaults
+  routes:
+    - route: spiffworkflow((slug)).app.cloud.gov
+  docker:
+    image: ((frontend-image))
+  memory: 256M
+  health-check-type: port
+  env:
+    APPLICATION_ROOT: "/"
+    PORT0: "80"
+    SPIFFWORKFLOW_FRONTEND_RUNTIME_CONFIG_APP_ROUTING_STRATEGY: "path_based"
+    SPIFFWORKFLOW_FRONTEND_RUNTIME_CONFIG_BACKEND_BASE_URL: "https://spiffworkflow((slug)).app.cloud.gov/api"
+    BACKEND_BASE_URL: "https://spiffworkflow((slug)).app.cloud.gov/api"
+    # We may need to set BACKEND_URL; see spiffworkflow-frontend/src/config/tsx:15-72
+
+    # Other vars this image understands:
+    # CYPRESS_RECORD_KEY
+    # SPIFFWORKFLOW_FRONTEND_PORT
+    # SPIFFWORKFLOW_FRONTEND_URL
+    # CYPRESS_RECORD_KEY
+    # REACT_APP_BACKEND_BASE_URL
+    # PUBLIC_URL
+    # NODE_ENV

spiffworkflow-backend/Dockerfile

@@ -27,6 +27,9 @@ RUN apt-get update \
   && apt-get install -y -q git-core curl procps gunicorn3 default-mysql-client vim-tiny jq libpq5 \
   && rm -rf /var/lib/apt/lists/*
 
+# install jq so we can get values from vcap
+RUN pip install jq
+
 # keep pip up to date
 RUN pip install --upgrade pip
 RUN pip install poetry==1.8.1

spiffworkflow-backend/bin/boot_server_in_docker

@@ -85,7 +85,9 @@ fi
 
 # ensure that the Process Models Directory is initialized as a git repo
 log_info "Running git init"
+git config --global init.defaultBranch main
 git init "${SPIFFWORKFLOW_BACKEND_BPMN_SPEC_ABSOLUTE_DIR}"
+git config --global --add safe.directory "${SPIFFWORKFLOW_BACKEND_BPMN_SPEC_ABSOLUTE_DIR}"
 
 if [[ -z "${SPIFFWORKFLOW_BACKEND_THREADS_PER_WORKER:-}" ]]; then
   # default to 3 * 2 = 6 threads per worker

spiffworkflow-backend/pyproject.toml

@@ -96,7 +96,7 @@ mypy = ">=0.961"
 typeguard = "^4"
 xdoctest = {extras = ["colors"], version = "^1.2.0"}
 pre-commit = "^4.0.1"
-ruff = "^0.5.5"
+ruff = "^0.8.1"
 
 pytest-random-order = "^1.1.0"
 pytest-flask = "^1.2.0"

spiffworkflow-backend/src/spiffworkflow_backend/config/permissions/example.yml

@@ -19,6 +19,12 @@ users:
     email: [email protected]
     password: oskar
     preferred_username: Oskar
+  asteel:
+    service: local_open_id
+    email: [email protected]
+    password: asteel
+    preferred_username: asteel
+
 
 groups:
   admin:

vars.yml-template

@@ -0,0 +1,28 @@
+backend-image: ghcr.io/gsa-tts/spiffworkflow-backend:deploy-to-cloud-gov-latest
+frontend-image: ghcr.io/gsa-tts/spiffworkflow-frontend:deploy-to-cloud-gov-latest
+connector-image: ghcr.io/gsa-tts/connector-proxy-demo:deploy-to-cloud-gov-latest
+
+db-instance: spiffworkflow-db
+backend-flask-session-key: 66eef9e98a3f4e6f85258154e4a1bdce
+connector-flask-secret-key: 66eef9e99a3f4e6f85258154e4a1bdce
+git-process-models-repo: [email protected]:GSA-TTS/gsa-process-models.git
+openid-secret: flarblegarble
+source-branch: process-models-playground
+target-branch-for-publish: publish-staging-branch
+
+# Generate a Key to be used for pulling and pushing commits to the bpmn process models repo (refer to git-process-models-repo var above):
+# ssh-keygen -t rsa -b 4096 -C "[email protected]"
+# Add the public key to:
+# https://github.com/settings/keys
+# And then put the private key here.
+github-ssh-key: |
+  -----BEGIN OPENSSH PRIVATE KEY-----
+  ...
+  ...
+  ...
+  -----END OPENSSH PRIVATE KEY-----
+
+# The "slug" is a URL-friendly string that can be used to distinguish between
+# deployments. It will be including in app names and generated URLs. (You can leave
+# this empty for an "official" deployment.)
+slug: -flamingo-stardust