From 04c6ea059edf91232dc116e635422b539ef3e64b Mon Sep 17 00:00:00 2001 From: owenlittlejohns Date: Wed, 24 Jan 2024 15:43:38 -0700 Subject: [PATCH] DAS-2070 - Implement GitHub workflows CI/CD. --- .github/pull_request_template.md | 17 ++++ .github/workflows/publish_docker_image.yml | 83 +++++++++++++++++++ .github/workflows/run_tests.yml | 42 ++++++++++ .../workflows/run_tests_on_pull_requests.yml | 13 +++ CHANGELOG.md | 4 +- CONTRIBUTING.md | 47 +++++++++++ README.md | 51 ++++++++++-- bin/build-image | 2 +- bin/build-test | 6 +- bin/clean-images | 4 +- bin/extract-release-notes.sh | 22 +++++ bin/run-test | 7 +- docker/service.Dockerfile | 6 +- docker/tests.Dockerfile | 7 +- harmony_browse_image_generator/__main__.py | 3 +- harmony_browse_image_generator/exceptions.py | 10 ++- 16 files changed, 297 insertions(+), 27 deletions(-) create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/publish_docker_image.yml create mode 100644 .github/workflows/run_tests.yml create mode 100644 .github/workflows/run_tests_on_pull_requests.yml create mode 100644 CONTRIBUTING.md create mode 100755 bin/extract-release-notes.sh diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..aec749e --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,17 @@ +## Description + +A short description of the changes in this PR. + +## Jira Issue ID + +[DAS-XXXX](https://bugs.earthdata.nasa.gov/browse/DAS-XXXX) + +## Local Test Steps + + +## PR Acceptance Checklist +* [ ] Jira ticket acceptance criteria met. +* [ ] `CHANGELOG.md` updated to include high level summary of PR changes. +* [ ] `docker/service_version.txt` updated if publishing a release. +* [ ] Tests added/updated and passing. +* [ ] Documentation updated (if needed). diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml new file mode 100644 index 0000000..772f259 --- /dev/null +++ b/.github/workflows/publish_docker_image.yml @@ -0,0 +1,83 @@ +# This workflow will run when changes are detected in the `main` branch, which +# must include an update to the `docker/service_version.txt` file. The workflow +# can also be manually triggered by a repository maintainer. This workflow will +# first trigger the reusable workflow in `.github/workflows/run_tests.yml`, +# which runs the `unittest` suite. If that workflow is successful, the latest +# version of the service Docker image is pushed to ghcr.io, a tag is added to +# the latest git commit, and a GitHub release is created with the release notes +# from the latest version of HyBIG. +name: Publish Harmony Browse Image Generator (HyBIG) Docker image + +on: + push: + branches: [ main ] + paths: docker/service_version.txt + workflow_dispatch: + +env: + IMAGE_NAME: ${{ github.repository }} + REGISTRY: ghcr.io + +jobs: + run_tests: + uses: ./.github/workflows/run_tests.yml + + build_and_publish_image: + needs: run_tests + runs-on: ubuntu-latest + environment: release + permissions: + # write permission is required to create a GitHub release + contents: write + id-token: write + packages: write + strategy: + fail-fast: false + + steps: + - name: Checkout harmony-browse-image-generator repository + uses: actions/checkout@v3 + with: + lfs: true + + - name: Extract semantic version number + run: echo "semantic_version=$(cat docker/service_version.txt)" >> $GITHUB_ENV + + - name: Extract release version notes + run: | + version_release_notes=$(./bin/extract-release-notes.sh) + echo "RELEASE_NOTES<> $GITHUB_ENV + echo "${version_release_notes}" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Log-in to ghcr.io registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Add tags to the Docker image + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=semver,pattern={{version}},value=${{ env.semantic_version }} + + - name: Push Docker image + uses: docker/build-push-action@v3 + with: + context: . + file: docker/service.Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Publish GitHub release + uses: ncipollo/release-action@v1 + with: + body: ${{ env.RELEASE_NOTES }} + commit: main + name: Version ${{ env.semantic_version }} + tag: ${{ env.semantic_version }} diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml new file mode 100644 index 0000000..78a72f6 --- /dev/null +++ b/.github/workflows/run_tests.yml @@ -0,0 +1,42 @@ +# This workflow will build the service and test Docker images for HyBIG, then +# run the Python `unittest` suite within a test Docker container, reporting +# test results and code coverage as artefacts. It will be called by the +# workflow that run tests against new PRs and as a first step in the workflow +# that publishes new Docker images. +name: Run Python unit tests + +on: + workflow_call + +jobs: + build_and_test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + + steps: + - name: Checkout harmony-browse-image-generator repository + uses: actions/checkout@v3 + with: + lfs: true + + - name: Build service image + run: ./bin/build-image + + - name: Build test image + run: ./bin/build-test + + - name: Run test image + run: ./bin/run-test + + - name: Archive test results + uses: actions/upload-artifact@v3 + with: + name: Test results + path: test-reports/ + + - name: Archive coverage report + uses: actions/upload-artifact@v3 + with: + name: Coverage report + path: coverage/* diff --git a/.github/workflows/run_tests_on_pull_requests.yml b/.github/workflows/run_tests_on_pull_requests.yml new file mode 100644 index 0000000..a948fd1 --- /dev/null +++ b/.github/workflows/run_tests_on_pull_requests.yml @@ -0,0 +1,13 @@ +# This workflow will run when a PR is opened against the `main` branch. It will +# trigger the reusable workflow in `.github/workflows/run_tests.yml`, which +# builds the service and test Docker images, and runs the `unittest` suite in a +# Docker container built from the test image. +name: Run Python unit tests for pull requests against main + +on: + pull_request: + branches: [ main ] + +jobs: + build_and_test: + uses: ./.github/workflows/run_tests.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 59e9b96..2fbeb6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ ## v1.0.0 ### 2024-01-22 This version of the Harmony Browse Image Generator (HyBIG) contains all -functionality previously released internally to EOSDIS as sds/HyBIG:0.0.11. +functionality previously released internally to EOSDIS as +sds/harmony-browse-image-generator:0.0.11. + Additional contents to the repository include updated documentation and files outlined by the NASA open-source guidelines. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..3ac9347 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,47 @@ +# Contributing to HyBIG + +Thanks for contributing! + +## Making Changes + +To allow us to incorporate your changes, please use the +[Fork-and-Pull](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/getting-started/about-collaborative-development-models#fork-and-pull-model) +development model: + +1. Fork this repository to your personal account. +2. Create a branch and make your changes. +3. Test the changes locally/in your personal fork. +4. Submit a pull request to open a discussion about your proposed changes. +5. The maintainers will talk with you about it and decide to merge or request + additional changes. + +For larger items, consider contacting the maintainers first to coordinate +development efforts. + +## Commits + +Our ticketing and CI/CD tools are configured to sync statuses amongst each +other. Commits play an important role in this process. Please start all commits +with the Jira ticket number associated with your feature, task, or bug. All +commit messages should follow the format +"[Jira Project]-XXXX - [Your commit message here]" + +## General coding practices: + +This repository adheres to Python coding style recommendations from +[PEP8](https://peps.python.org/pep-0008/). Additionally, type hints are +encouraged in all function signatures. + +When adding or updating functionality, please ensure unit tests are added to +the existing `unittest` suite in the `tests` directory, which cover each branch +of the code. + +## Disclaimer + +HyBIG maintainers will review all pull requests submitted. Only requests that +meet the standard of quality set forth by existing code, following the patterns +set forth by existing code, and adhering to existing design patterns will be +considered and/or accepted. + +For general tips on open source contributions, see +[Contributing to Open Source on GitHub](https://guides.github.com/activities/contributing-to-open-source/). diff --git a/README.md b/README.md index 3bcec18..792dc5b 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,15 @@ # Harmony Browse Image Generator (HyBIG) backend service. This Harmony backend service is designed to produce browse imagery, with -default behaviour to produce browse imagery that is compatible with Global -Image Browse Services (GIBS). +default behaviour to produce browse imagery that is compatible with the NASA +Global Image Browse Services ([GIBS](https://www.earthdata.nasa.gov/eosdis/science-system-description/eosdis-components/gibs)). ## Repository structure: ``` |- .pre-commit-config.yaml |- CHANGELOG.md +|- CONTRIBUTING.md |- README.md |- bin |- conda_requirements.txt @@ -25,6 +26,8 @@ Image Browse Services (GIBS). * `CHANGELOG.md` - This file contains a record of changes applied to each new release of a service Docker image. Any release of a new service version should have a record of what was changed in this file. +* `CONTRIBUTING.md` - This file contains guidance for making contributions to + HyBIG, including recommended git best practices. * `README.md` - This file, containing guidance on developing the service. * `bin` - A directory containing utility scripts to build the service and test images. This also includes scripts that Bamboo uses to deploy new service @@ -40,6 +43,9 @@ Image Browse Services (GIBS). * `harmony_browse_image_generator` - The directory containing Python source code for the HyBIG. `adapter.py` contains the `BrowseImageGeneratorAdapter` class that is invoked by calls to the service. +* `legacy-CHANGELOG.md` - Notes for each version that was previously released + internally to EOSDIS, prior to open-source publication of the code and Docker + image. * `pip_requirements.txt` - A list of service Python package dependencies. * `tests` - A directory containing the service unit test suite. @@ -99,11 +105,36 @@ updated: * CHANGELOG.md - Notes should be added to capture the changes to the service. * docker/service_version.txt - The semantic version number should be updated. -## Docker image publication: +## CI/CD: -Initially service Docker images will be hosted in AWS Elastic Container -Registry (ECR). When this repository is migrated to the NASA GitHub -organisation, service images will be published to ghcr.io, instead. +The CI/CD for HyBIG is contained in GitHub workflows in the +`.github/workflows` directory: + +* `run_tests.yml` - A reusable workflow that builds the service and test Docker + images, then runs the Python unit test suite in an instance of the test + Docker container. +* `run_tests_on_pull_requests.yml` - Triggered for all PRs against the `main` + branch. It runs the workflow in `run_tests.yml` to ensure all tests pass for + the new code. +* `publish_docker_image.yml` - Triggered either manually or for commits to the + `main` branch that contain changes to the `docker/service_version.txt` file. + +The `publish_docker_image.yml` workflow will: + +* Run the full unit test suite, to prevent publication of broken code. +* Extract the semantic version number from `docker/service_version.txt`. +* Extract the released notes for the most recent version from `CHANGELOG.md`. +* Create a GitHub release that will also tag the related git commit with the + semantic version number. + +Before triggering a release, ensure both the `docker/service_version.txt` and +`CHANGELOG.md` files are updated. The `CHANGELOG.md` file requires a specific +format for a new release, as it looks for the following string to define the +newest relate of the code (starting at the top of the file). + +``` +## vX.Y.Z +``` ## Releasing a new version of the service: @@ -112,3 +143,11 @@ that service version can be released to a Harmony environment by updating the main Harmony Bamboo deployment project. Find the environment you wish to release the service version to and update the associated environment variable to update the semantic version tag at the end of the full Docker image name. + +## Get in touch: + +You can reach out to the maintainers of this repository via email: + +* david.p.auty@nasa.gov +* matthew.savoie@colorado.edu +* owen.m.littlejohns@nasa.gov diff --git a/bin/build-image b/bin/build-image index 3ac4592..a2d4284 100755 --- a/bin/build-image +++ b/bin/build-image @@ -19,7 +19,7 @@ image="ghcr.io/nasa/harmony-browse-image-generator" tag=${1:-latest} -# Remove old versions of: sds/harmony-browse-image-generator and +# Remove old versions of: ghcr.io/nasa/harmony-browse-image-generator and # ghcr.io/nasa/harmony-browse-image-generator-test ./bin/clean-images diff --git a/bin/build-test b/bin/build-test index f9fbe5f..519304c 100755 --- a/bin/build-test +++ b/bin/build-test @@ -4,9 +4,9 @@ # Build a Docker container to run the test suite for the Harmony Browse Image # Generator (HyBIG). # -# To build the sds/harmony-browse-image-generator-test image, the -# sds/harmony-browse-image-generator image must also be present locally, as it -# is used as the base image in `docker/tests.Dockerfile`. +# To build the ghcr.io/nasa/harmony-browse-image-generator-test image, the +# ghcr.io/nasa/harmony-browse-image-generator image must also be present +# locally, as it is used as the base image in `docker/tests.Dockerfile`. # # 2020-05-07: Adapted from SwotRepr. # 2022-01-03: Updated Dockerfile path. diff --git a/bin/clean-images b/bin/clean-images index 3be53e7..7e1cd03 100755 --- a/bin/clean-images +++ b/bin/clean-images @@ -2,8 +2,8 @@ ############################################################################### # # Look for all versions of images containing the string -# "sds/harmony-browse-image-generator" and remove them. This is primarily for -# clean-up after development. +# "ghcr.io/nasa/harmony-browse-image-generator" and remove them. This is +# primarily for clean-up after development. # # 2020-05-07: Adapted from SwotRepr. # 2023-04-04: Updated for the Harmony Browse Image Generator (HyBIG). diff --git a/bin/extract-release-notes.sh b/bin/extract-release-notes.sh new file mode 100755 index 0000000..3da532c --- /dev/null +++ b/bin/extract-release-notes.sh @@ -0,0 +1,22 @@ +#!/bin/bash +############################################################################### +# +# A bash script to extract only the notes related to the most recent version of +# HyBIG from CHANGELOG.md +# +# 2023-06-16: Created. +# 2023-10-10: Copied from earthdata-varinfo repository to HOSS. +# 2024-01-03: Copied from HOSS repository to the Swath Projector. +# 2024-01-23: Copied and modified from Swath Projector repository to HyBIG. +# +############################################################################### + +CHANGELOG_FILE="CHANGELOG.md" +VERSION_PATTERN="^## v" + +# Read the file and extract text between the first two occurrences of the +# VERSION_PATTERN +result=$(awk "/$VERSION_PATTERN/{c++; if(c==2) exit;} c==1" "$CHANGELOG_FILE") + +# Print the result +echo "$result" | grep -v "^#" diff --git a/bin/run-test b/bin/run-test index a0cdb1e..72035c2 100755 --- a/bin/run-test +++ b/bin/run-test @@ -1,12 +1,13 @@ #!/bin/bash ############################################################################### # -# Execute the sds/harmony-browse-image-generator-test Docker image as a -# container, within Bamboo. +# Execute the ghcr.io/nasa/harmony-browse-image-generator-test Docker image as +# a container, within Bamboo. # # 2020-05-07: Adapted from SwotRepr. # 2021-01-05: Updated for JUnit output. # 2023-04-04: Updated for the Harmony Browse Image Generator (HyBIG). +# 2024-01-23: Updated to use new GitHub-related images. # ############################################################################### @@ -26,4 +27,4 @@ mkdir -p coverage docker run --rm \ -v $(pwd)/test-reports:/home/tests/reports \ -v $(pwd)/coverage:/home/tests/coverage \ - sds/harmony-browse-image-generator-test "$@" + ghcr.io/nasa/harmony-browse-image-generator-test "$@" diff --git a/docker/service.Dockerfile b/docker/service.Dockerfile index ce67da6..d727a9f 100644 --- a/docker/service.Dockerfile +++ b/docker/service.Dockerfile @@ -1,8 +1,8 @@ ############################################################################### # -# Service image for sds/harmony-browse-image-generator, HyBIG, a Harmony -# backend service that creates JPEG or PNG browse images from input GeoTIFF -# imagery. +# Service image for ghcr.io/nasa/harmony-browse-image-generator, HyBIG, a +# Harmony backend service that creates JPEG or PNG browse images from input +# GeoTIFF imagery. # # This image installs dependencies via Pip. The service code is then copied # into the Docker image. diff --git a/docker/tests.Dockerfile b/docker/tests.Dockerfile index 2176be3..915a479 100644 --- a/docker/tests.Dockerfile +++ b/docker/tests.Dockerfile @@ -1,9 +1,10 @@ ############################################################################### # # Test image for the Harmony Browse Image Generator (HyBIG). This test image -# uses the main service image, sds/harmony-browse-image-generator, as a base -# layer for the tests. This ensures that the contents of the service image are -# tested, preventing discrepancies between the service and test environments. +# uses the main service image, ghcr.io/nasa/harmony-browse-image-generator, as +# a base layer for the tests. This ensures that the contents of the service +# image are tested, preventing discrepancies between the service and test +# environments. # # 2023-04-04: Updated for HyBIG. # 2023-04-17: Added --no-cache-dir to keep Docker images slim. diff --git a/harmony_browse_image_generator/__main__.py b/harmony_browse_image_generator/__main__.py index d07013d..f6d6295 100644 --- a/harmony_browse_image_generator/__main__.py +++ b/harmony_browse_image_generator/__main__.py @@ -5,6 +5,7 @@ from harmony import is_harmony_cli, run_cli, setup_cli from harmony_browse_image_generator.adapter import BrowseImageGeneratorAdapter +from harmony_browse_image_generator.exceptions import SERVICE_NAME def main(arguments: list[str]): @@ -12,7 +13,7 @@ def main(arguments: list[str]): respond to them """ - parser = ArgumentParser(prog='harmony-browse-image-generator', + parser = ArgumentParser(prog=SERVICE_NAME, description='Run Harmony Browse Image Generator.') setup_cli(parser) diff --git a/harmony_browse_image_generator/exceptions.py b/harmony_browse_image_generator/exceptions.py index 68760e3..55c7810 100644 --- a/harmony_browse_image_generator/exceptions.py +++ b/harmony_browse_image_generator/exceptions.py @@ -4,26 +4,28 @@ """ from harmony.util import HarmonyException +SERVICE_NAME = 'harmony-browse-image-generator' + class HyBIGError(HarmonyException): """Base service exception.""" def __init__(self, message=None): - super().__init__(message, 'sds/harmony-browse-image-generator') + super().__init__(message, SERVICE_NAME) class HyBIGNoColorInformation(HarmonyException): """Used to describe missing color information.""" def __init__(self, message=None): - super().__init__(message, 'sds/harmony-browse-image-generator') + super().__init__(message, SERVICE_NAME) class HyBIGInvalidMessageError(HarmonyException): """Input Harmony Message could not be used as presented.""" def __init__(self, message=None): - super().__init__(message, 'sds/harmony-browse-image-generator') + super().__init__(message, SERVICE_NAME) class HyBIGValueError(HarmonyException): """Input was incorrect for the routine.""" def __init__(self, message=None): - super().__init__(message, 'sds/harmony-browse-image-generator') + super().__init__(message, SERVICE_NAME)