2023 update (#87)

* Update notebooks

* 2023 update

* Fix pep

* Clean notebook

* Update docker image base

* Update vector.py

* Automatically update integration test validation results

* Add experiment database access

* Update

* Update after permissions

* Update dockerfile and CI

* Update docker

* Remove AWS CLI

* Add test env

* Simplify by installing pytest in setup.py

* Add file copying to workflow

* Try new paths

* Check everything works up to copy step

* Try ls

* Fix

* Try combined approach

* Try smaller tests

* Fix paths

* Automatically update integration test validation results

* Try changing locations files are written to

* Fix

* Update .gitignore

* Automatically update integration test validation results

* Update stats_tests.csv

* Automatically update integration test validation results

* Clean repo

* Update gitignore

* Update test_coastline.py

* Update test_coastline.py

* Update test_coastline.py

* Debug

* Debug

* Finally fix validation bug

* Automatically update integration test validation results

* Fix formatting

* Fix pep8

* Add image build and artifacts

* Automatically update integration test validation results

* Update vector.py

* Automatically update integration test validation results

---------

Co-authored-by: robbibt <[email protected]>

.github/workflows/dea-coastlines-image.yaml

@@ -0,0 +1,180 @@
+name: Image build and test
+
+env:
+  IMAGE_NAME: geoscienceaustralia/dea-coastlines
+
+on:
+  push:
+    branches:
+      - develop
+    paths:
+      - 'coastlines/**'
+      - 'data/**'
+      - 'tests/**'
+      - '.github/workflows/dea-coastlines-image.yml'
+      - 'Dockerfile'
+      - 'requirements.in'
+      - 'requirements.txt'
+      - 'setup.py'
+      - 'codecov.yaml'
+
+  pull_request:
+    branches:
+      - develop
+    paths:
+      - 'coastlines/**'
+      - 'data/**'
+      - 'tests/**'
+      - '.github/workflows/dea-coastlines-image.yml'
+      - 'Dockerfile'
+      - 'requirements.in'
+      - 'requirements.txt'
+      - 'setup.py'
+      - 'codecov.yaml'
+
+  release:
+    types: [edited, published]
+
+permissions:
+  id-token: write  # Required for requesting Json web token
+  contents: write  # Required for actions/checkout
+  pull-requests: write  # Required for validation results comment bot
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::060378307146:role/github-actions-dea-notebooks--sandbox
+          aws-region: ap-southeast-2
+          role-duration-seconds: 7200  # 2 hours
+
+      - name: Get database credentials
+        run: |
+          username_password=$(aws ssm get-parameter --name /dea-sandbox-eks/sandbox_reader/db.creds --with-decryption  --query Parameter.Value --output text)
+          echo DATACUBE_DB_URL=postgresql://${username_password}@localhost:5432/odc >> $GITHUB_ENV
+
+      - name: Open port forward to RDS
+        run: |
+          npx basti connect \
+            --custom-target-vpc vpc-086904199e505c1f6 \
+            --custom-target-host db-aurora-dea-sandbox-eks-1.cos5zfpkso9m.ap-southeast-2.rds.amazonaws.com \
+            --custom-target-port 5432 \
+            --local-port 5432 &
+          # Wait until the connection comes up, but, if it doesn't, don't hang forever.
+          npx wait-on --timeout 120000 --interval 1000 tcp:127.0.0.1:5432
+          echo "PGPORT=5432" >> $GITHUB_ENV
+          echo "PGHOST=localhost" >> $GITHUB_ENV
+
+      - name: Build DEA Coastlines docker image 
+        timeout-minutes: 30
+        shell: bash
+        run: |
+          docker build -t dea_coastlines .
+
+      - name: Run integration tests
+        run: |
+          # Download tide modelling files and unzip
+          wget --no-verbose https://www.dropbox.com/s/uemd8ib2vfw5nad/tide_models.zip?dl=1 -O tide_models.zip
+          unzip -q tide_models.zip
+
+          # Run integration tests using Docker
+          docker run \
+            --net=host \
+            --env DATACUBE_DB_URL \
+            --env AWS_SESSION_TOKEN \
+            --env AWS_REGION \
+            --env AWS_ACCESS_KEY_ID \
+            --env AWS_SECRET_ACCESS_KEY \
+            --env AWS_SESSION_TOKEN \
+            --volume ${GITHUB_WORKSPACE}:/code \
+            --volume ${GITHUB_WORKSPACE}/tide_models:/var/share/tide_models \
+            dea_coastlines pytest -v --cov=coastlines --cov-report=xml tests
+
+      # Commit validation results produced by integration tests back into repo
+      - name: Commit validation results into repository
+        uses: stefanzweifel/git-auto-commit-action@v4
+        if: github.event_name == 'pull_request'
+        continue-on-error: true
+        with:
+          commit_message: Automatically update integration test validation results
+          file_pattern: 'tests/stats_tests.png tests/stats_tests.csv tests/README.md'
+
+      # Post validation tesults as comment on PR
+      - name: Post validation results as comment
+        uses: mshick/add-pr-comment@v2
+        if: github.event_name == 'pull_request'
+        with:
+          message: |
+            ![](https://github.com/GeoscienceAustralia/dea-coastlines/blob/${{ github.head_ref }}/tests/stats_tests.png?raw=true")
+            
+            For full integration test results, refer to the [Tests directory README](https://github.com/GeoscienceAustralia/dea-coastlines/tree/${{ github.head_ref }}/tests).
+
+      # Upload code coverage results
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+      # Add output GeoPackage data to Actions artifact so it can be downloaded
+      - uses: actions/upload-artifact@v3
+        with:
+          name: output_geopackage
+          path: ./data/processed/tests/coastlines_tests.gpkg
+
+      # Add output ESRI Shapefile data to Actions artifact so it can be downloaded
+      - uses: actions/upload-artifact@v3
+        with:
+          name: output_shps
+          path: ./data/processed/tests/coastlines_tests.shp.zip
+
+  push_ecr:
+    needs: [test]
+    runs-on: ubuntu-latest
+
+    # Only run on a push to the main branch OR a release
+    if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'release')
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: arn:aws:iam::538673716275:role/github-actions-role
+          aws-region: ap-southeast-2
+
+      - name: Get tag for this build if it exists
+        if: github.event_name == 'release'
+        run: |
+          echo "RELEASE=${GITHUB_REF/refs\/tags\/}" >> $GITHUB_ENV
+
+      - name: Push release image to ECR
+        uses: whoan/docker-build-with-cache-action@master
+        if: github.event_name == 'release'
+        with:
+          registry: 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com
+          image_name: ${{ env.IMAGE_NAME }}
+          image_tag: ${{ env.RELEASE }}
+
+      - name: Get git commit hash for push to main
+        if: github.event_name != 'release'
+        run: |
+          echo "TAG=dev$(git rev-parse --short HEAD)" \
+          >> $GITHUB_ENV
+
+      - name: Push unstable image to ECR
+        uses: whoan/docker-build-with-cache-action@master
+        if: github.event_name != 'release'
+        with:
+          registry: 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com
+          image_name: ${{ env.IMAGE_NAME }}
+          image_tag: latest,${{ env.TAG }}

.gitignore

@@ -11,4 +11,6 @@
 !*.sql
 !**.github
 !*.md
+!requirements.txt
 !tests/*.png
+!tests/*.csv