Skip to content
This repository has been archived by the owner on May 8, 2024. It is now read-only.

Commit

Permalink
Airflow deployment updates
Browse files Browse the repository at this point in the history
  • Loading branch information
Drew Meyers committed Feb 11, 2024
1 parent ecb044c commit 7ccc942
Show file tree
Hide file tree
Showing 11 changed files with 193 additions and 522 deletions.
296 changes: 0 additions & 296 deletions .github/workflows/build_docker_images.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,305 +11,9 @@ on:
env:
REGISTRY: ghcr.io
TAG: ${{ github.event.inputs.tag }}
HYSDS_CORE_IMAGE_NAME: ${{ github.repository }}/hysds-core
MOZART_IMAGE_NAME: ${{ github.repository }}/hysds-mozart
GRQ_IMAGE_NAME: ${{ github.repository }}/hysds-grq2
VERDI_IMAGE_NAME: ${{ github.repository }}/hysds-verdi
REMOTE_UI_IMAGE_NAME: ${{ github.repository }}/hysds-ui-remote
REMOTE_UI_AUTH_IMAGE_NAME: ${{ github.repository }}/hysds-ui-remote-auth
ADES_WPST_API_IMAGE_NAME: ${{ github.repository }}/ades-wpst-api
ADES_WPST_API_DEVELOP_IMAGE_NAME: ${{ github.repository }}/ades-wpst-api-develop
SPS_API_IMAGE_NAME: ${{ github.repository }}/sps-api
SPS_HYSDS_PGE_BASE_IMAGE_NAME: ${{ github.repository }}/sps-hysds-pge-base
SPS_JOB_PUBLISHER_IMAGE_NAME: ${{ github.repository }}/sps-job-publisher
SPS_AIRFLOW: ${{ github.repository }}/sps-airflow

jobs:
build-hysds-core:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Log in to the Container registry
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for HySDS Core Docker image
id: metacore
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
with:
images: ${{ env.REGISTRY }}/${{ env.HYSDS_CORE_IMAGE_NAME }}
- name: Build and push HySDS Core Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: hysds
push: true
tags: ${{ env.REGISTRY }}/${{ env.HYSDS_CORE_IMAGE_NAME }}:${{ env.TAG }}
labels: ${{ steps.metacore.outputs.labels }}

build-hysds-mozart:
runs-on: ubuntu-latest
needs: build-hysds-core
steps:
- uses: actions/checkout@v2
- name: Log in to the Container registry
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for HyDSDS Mozart Docker image
id: metamozart
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
with:
images: ${{ env.REGISTRY }}/${{ env.MOZART_IMAGE_NAME }}
- name: Build and push HySDS Mozart Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: hysds/mozart/rest_api/
push: true
tags: ${{ env.REGISTRY }}/${{ env.MOZART_IMAGE_NAME }}:${{ env.TAG }}
labels: ${{ steps.metamozart.outputs.labels }}
build-args: |
PARENT_DOCKER_IMAGE=${{ env.REGISTRY }}/${{ env.HYSDS_CORE_IMAGE_NAME }}:${{ env.TAG }}
build-hysds-grq:
runs-on: ubuntu-latest
needs: build-hysds-core
steps:
- uses: actions/checkout@v2
- name: Log in to the Container registry
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for HyDSDS GRQ Docker image
id: metagrq
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
with:
images: ${{ env.REGISTRY }}/${{ env.GRQ_IMAGE_NAME }}
- name: Build and push HySDS GRQ Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: hysds/grq/rest_api/
push: true
tags: ${{ env.REGISTRY }}/${{ env.GRQ_IMAGE_NAME }}:${{ env.TAG }}
labels: ${{ steps.metagrq.outputs.labels }}
build-args: |
PARENT_DOCKER_IMAGE=${{ env.REGISTRY }}/${{ env.HYSDS_CORE_IMAGE_NAME }}:${{ env.TAG }}
build-hysds-verdi:
runs-on: ubuntu-latest
needs: build-hysds-core
steps:
- uses: actions/checkout@v2
- name: Log in to the Container registry
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for HyDSDS Verdi Docker image
id: metaverdi
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
with:
images: ${{ env.REGISTRY }}/${{ env.VERDI_IMAGE_NAME }}
- name: Build and push HySDS Verdi Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: hysds/verdi/
push: true
tags: ${{ env.REGISTRY }}/${{ env.VERDI_IMAGE_NAME }}:${{ env.TAG }}
labels: ${{ steps.metaverdi.outputs.labels }}
build-args: |
PARENT_DOCKER_IMAGE=${{ env.REGISTRY }}/${{ env.HYSDS_CORE_IMAGE_NAME }}:${{ env.TAG }}
build-hysds-ui-remote:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Log in to the Container registry
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for HyDSDS Remote UI Docker image
id: metauiremote
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
with:
images: ${{ env.REGISTRY }}/${{ env.REMOTE_UI_IMAGE_NAME }}
- name: Build and push HySDS Remote UI Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: hysds/ui/
file: hysds/ui/remote.Dockerfile
push: true
tags: ${{ env.REGISTRY }}/${{ env.REMOTE_UI_IMAGE_NAME }}:${{ env.TAG }}
labels: ${{ steps.metauiremote.outputs.labels }}

build-hysds-ui-remote-auth:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Log in to the Container registry
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for HyDSDS Remote UI Docker image
id: metauiremote
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
with:
images: ${{ env.REGISTRY }}/${{ env.REMOTE_UI_AUTH_IMAGE_NAME }}
- name: Build and push HySDS Remote UI with Auth Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: hysds/ui/
file: hysds/ui/remote_auth.Dockerfile
push: true
tags: ${{ env.REGISTRY }}/${{ env.REMOTE_UI_AUTH_IMAGE_NAME }}:${{ env.TAG }}
labels: ${{ steps.metauiremote.outputs.labels }}

build-ades-wpst-api:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
repository: unity-sds/ades_wpst
- name: Log in to the container registry
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for ADES WPST API Docker image
id: metaades
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
with:
images: ${{ env.REGISTRY }}/${{ env.ADES_WPST_API_IMAGE_NAME }}
- name: Build and push ADES WPST API Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: .
file: docker/Dockerfile
push: true
tags: ${{ env.REGISTRY }}/${{ env.ADES_WPST_API_IMAGE_NAME }}:${{ env.TAG }}
labels: ${{ steps.metaades.outputs.labels }}

build-sps-api:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
repository: unity-sds/unity-sps-api
- name: Log in to the container registry
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for SPS API Docker image
id: metasps
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
with:
images: ${{ env.REGISTRY }}/${{ env.SPS_API_IMAGE_NAME }}
- name: Build and push SPS API Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: .
file: docker/Dockerfile
push: true
tags: ${{ env.REGISTRY }}/${{ env.SPS_API_IMAGE_NAME }}:${{ env.TAG }}
labels: ${{ steps.metasps.outputs.labels }}

build-ades-wpst-api-develop:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
repository: unity-sds/ades_wpst
ref: docker-build-args
- name: Log in to the container registry
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for ADES WPST API Develop Docker image
id: metaades
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
with:
images: ${{ env.REGISTRY }}/${{ env.ADES_WPST_API_DEVELOP_IMAGE_NAME }}
- name: Build and push ADES WPST API Develop Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: .
file: docker/Dockerfile
push: true
tags: ${{ env.REGISTRY }}/${{ env.ADES_WPST_API_DEVELOP_IMAGE_NAME }}:${{ env.TAG }}
labels: ${{ steps.metaades.outputs.labels }}

build-sps-hysds-pge-base:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
repository: unity-sds/unity-sps-register_job
- name: Log in to the container registry
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for SPS HySDS base PGE Docker image
id: metasps
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
with:
images: ${{ env.REGISTRY }}/${{ env.SPS_HYSDS_PGE_BASE_IMAGE_NAME }}
- name: Build and push SPS HySDS base PGE Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: .
file: docker/Dockerfile_sps-hysds-pge-base
push: true
tags: ${{ env.REGISTRY }}/${{ env.SPS_HYSDS_PGE_BASE_IMAGE_NAME }}:${{ env.TAG }}
labels: ${{ steps.metasps.outputs.labels }}

build-sps-job-publisher:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
repository: unity-sds/unity-sps-register_job
- name: Log in to the container registry
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for SPS Job Publisher Docker image
id: metasps
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
with:
images: ${{ env.REGISTRY }}/${{ env.SPS_JOB_PUBLISHER_IMAGE_NAME }}
- name: Build and push SPS Job Publisher Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: .
file: docker/Dockerfile_publish_job
push: true
tags: ${{ env.REGISTRY }}/${{ env.SPS_JOB_PUBLISHER_IMAGE_NAME }}:${{ env.TAG }}
labels: ${{ steps.metasps.outputs.labels }}

build-sps-airflow:
runs-on: ubuntu-latest
steps:
Expand Down
51 changes: 35 additions & 16 deletions terraform-unity/.terraform.lock.hcl
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

19 changes: 11 additions & 8 deletions terraform-unity/main.tf
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
module "unity-sps-airflow" {
source = "./modules/terraform-unity-sps-airflow"
venue = var.venue
counter = var.counter
release = var.release
eks_cluster_name = var.eks_cluster_name
kubeconfig_filepath = var.kubeconfig_filepath
docker_images = var.docker_images
elb_subnets = var.elb_subnets
source = "./modules/terraform-unity-sps-airflow"
project = var.project
venue = var.venue
service_area = var.service_area
counter = var.counter
release = var.release
eks_cluster_name = var.eks_cluster_name
kubeconfig_filepath = var.kubeconfig_filepath
airflow_webserver_password = var.airflow_webserver_password
custom_airflow_docker_image = var.custom_airflow_docker_image
helm_charts = var.helm_charts
}
Loading

0 comments on commit 7ccc942

Please sign in to comment.