post git pull merge conflict resolutions

.github/workflows/dea-intertidal-image.yml

@@ -16,7 +16,6 @@ on:
       - 'Dockerfile'
       - 'requirements.in'
       - 'setup.py'
-      - 'docker-compose.yml'
       - 'codecov.yaml'
 
   pull_request:
@@ -30,16 +29,15 @@ on:
       - 'Dockerfile'
       - 'requirements.in'
       - 'setup.py'
-      - 'docker-compose.yml'
       - 'codecov.yaml'
 
   release:
-    types: [created, edited, published]
-
+    types: [edited, published]
 
 permissions:
-  id-token: write   # This is required for requesting the JWT
-  contents: write    # This is required for actions/checkout
+  id-token: write  # Required for requesting Json web token
+  contents: write  # Required for actions/checkout
+  pull-requests: write  # Required for validation results comment bot
 
 jobs:
   test:
@@ -50,38 +48,84 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Build DEA Intertidal image
-        timeout-minutes: 20
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::060378307146:role/github-actions-dea-notebooks--sandbox
+          aws-region: ap-southeast-2
+          role-duration-seconds: 7200  # 2 hours
+
+      - name: Get database credentials
+        run: |
+          username_password=$(aws ssm get-parameter --name /dea-sandbox-eks/sandbox_reader/db.creds --with-decryption  --query Parameter.Value --output text)
+          echo DATACUBE_DB_URL=postgresql://${username_password}@localhost:5432/odc >> $GITHUB_ENV
+
+      - name: Open port forward to RDS
+        run: |
+          npx basti connect \
+            --custom-target-vpc vpc-086904199e505c1f6 \
+            --custom-target-host db-aurora-dea-sandbox-eks-1.cos5zfpkso9m.ap-southeast-2.rds.amazonaws.com \
+            --custom-target-port 5432 \
+            --local-port 5432 &
+          # Wait until the connection comes up, but, if it doesn't, don't hang forever.
+          npx wait-on --timeout 120000 --interval 1000 tcp:127.0.0.1:5432
+          echo "PGPORT=5432" >> $GITHUB_ENV
+          echo "PGHOST=localhost" >> $GITHUB_ENV
+
+      - name: Build DEA Intertidal docker image 
+        timeout-minutes: 30
         shell: bash
         run: |
-          docker-compose build
+          docker build -t dea_intertidal .
 
-      - name: Run tests
+      - name: Run integration tests
         run: |
           # Download tide modelling files and unzip
           # TODO: Replace with S3 sync from dea-non-public-data
           wget --no-verbose https://www.dropbox.com/s/uemd8ib2vfw5nad/tide_models.zip?dl=1 -O tide_models.zip
           unzip -q tide_models.zip
 
-          # Run integration tests using Docker
-          docker compose run dea_intertidal pytest -v --cov=intertidal --cov-report=xml tests
+          # Run integration tests using Docker, setting up datacube access, AWS configuration and
+          # adding volumes that provide access to tide model data and allow us to export artifacts
+          # from the run
+          docker run \
+            --net=host \
+            --env DATACUBE_DB_URL \
+            --env AWS_SESSION_TOKEN \
+            --env AWS_REGION \
+            --env AWS_ACCESS_KEY_ID \
+            --env AWS_SECRET_ACCESS_KEY \
+            --env AWS_SESSION_TOKEN \
+            --volume ${GITHUB_WORKSPACE}:/code \
+            --volume ${GITHUB_WORKSPACE}/tide_models:/var/share/tide_models \
+            --volume ${GITHUB_WORKSPACE}/artifacts:/mnt/artifacts \
+            dea_intertidal pytest -v --cov=intertidal --cov-report=xml tests
 
           # Copy out validation outputs produced by the integration tests and place them
           # in correct output locations so they can be committed back into the repository
           cp ./artifacts/validation.jpg ./tests/validation.jpg
           cp ./artifacts/validation.csv ./tests/validation.csv
           cp ./artifacts/README.md ./tests/README.md
 
-      # - name: Setup upterm session
-      #   uses: lhotari/action-upterm@v1
-
       # Commit validation results produced by integration tests back into repo
       - name: Commit validation results into repository
         uses: stefanzweifel/git-auto-commit-action@v4
+        if: github.event_name == 'pull_request'
+        continue-on-error: true
         with:
           commit_message: Automatically update integration test validation results
           file_pattern: 'tests/validation.jpg tests/validation.csv tests/README.md'
 
+      # Post validation tesults as comment on PR
+      - name: Post validation results as comment
+        uses: mshick/add-pr-comment@v2
+        if: github.event_name == 'pull_request'
+        with:
+          message: |
+            ![](https://github.com/GeoscienceAustralia/dea-intertidal/blob/${{ github.head_ref }}/tests/validation.jpg?raw=true")
+            
+            For full integration test results, refer to the [Tests directory README](https://github.com/GeoscienceAustralia/dea-intertidal/tree/${{ github.head_ref }}/tests).
+
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
         env:
@@ -91,8 +135,8 @@ jobs:
     needs: [test]
     runs-on: ubuntu-latest
 
-    # Only run on a push to the main branch
-    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    # Only run on a push to the main branch OR a release
+    if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'release')
 
     steps:
       - name: Checkout code

.gitignore

@@ -9,6 +9,7 @@
 !*.yaml
 !*.yml
 !*.in
+!*.txt
 !**.github/workflows
 !*.gitignore
 !*.dockerignore

Dockerfile

@@ -1,4 +1,8 @@
-FROM osgeo/gdal:ubuntu-small-3.4.1
+# Base image with:
+# - Ubuntu 22.04
+# - Python 3.10.12
+# - GDAL 3.7.3, released 2023/10/30
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.7.3
 
 ENV DEBIAN_FRONTEND=noninteractive \
     LC_ALL=C.UTF-8 \
@@ -15,7 +19,7 @@ RUN apt-get update && \
       wget \
       unzip \
       python3-pip \
-      libpq-dev python-dev \
+      libpq-dev \
     && apt-get autoclean && \
     apt-get autoremove && \
     rm -rf /var/lib/{apt,dpkg,cache,log}
@@ -25,9 +29,11 @@ RUN pip install pip-tools
 
 # Pip installation
 RUN mkdir -p /conf
-COPY requirements.in /conf/
-RUN pip-compile --extra-index-url=https://packages.dea.ga.gov.au/ --output-file=/conf/requirements.txt /conf/requirements.in
-RUN pip install -r /conf/requirements.txt
+# COPY requirements.in /conf/
+# RUN pip-compile --extra-index-url=https://packages.dea.ga.gov.au/ --output-file=/conf/requirements.txt /conf/requirements.in
+COPY requirements.txt /conf/
+RUN pip install -r /conf/requirements.txt \
+    && pip install --no-cache-dir awscli
 
 # Copy source code and install it
 RUN mkdir -p /code

README.md

@@ -9,7 +9,7 @@
 
 **License:** The code in this repository is licensed under the [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0). Digital Earth Australia data is licensed under the [Creative Commons by Attribution 4.0 license](https://creativecommons.org/licenses/by/4.0/).
 
-**Contact:** For assistance with any of the Python code or Jupyter Notebooks in this repository, please post a [Github issue](https://github.com/GeoscienceAustralia/dea-intertidal/issues). For questions or more information about this workflow, email Robbi.BishopTaylor@ga.gov.au.
+**Contact:** For assistance with any of the Python code or Jupyter Notebooks in this repository, please post a [Github issue](https://github.com/GeoscienceAustralia/dea-intertidal/issues). For questions or more information about DEA Intertidal, email earth.observation@ga.gov.au.
 
 **To cite:** 
 > Bishop-Taylor, R., Sagar, S., Lymburner, L., Beaman, R.L., 2019. Between the tides: modelling the elevation of Australia's exposed intertidal zone at continental scale. Estuarine, Coastal and Shelf Science. https://doi.org/10.1016/j.ecss.2019.03.006
@@ -19,3 +19,20 @@
 > Sagar, S., Roberts, D., Bala, B., Lymburner, L., 2017. Extracting the intertidal extent and topography of the Australian coastline from a 28 year time series of Landsat observations. Remote Sensing of Environment 195, 153-169. https://doi.org/10.1016/j.rse.2017.04.009
 
 ---
+
+The DEA Intertidal product suite maps the changing elevation, exposure and tidal characteristics of Australia's exposed intertidal zone, the complex zone that defines the interface between land and sea. It is the next generation of DEA's intertidal products that have been used across government and industry to help better characterise and understand this complex zone that defines the interface between land and sea.
+
+Incorporating both Sentinel-2 and Landsat data, the product suite provides an annual 10 m resolution elevation product for the intertidal zone, enabling users to better monitor and understand some of the most dynamic regions of Australia's coastlines. Utilising an improved tidal modelling capability, the product suite includes a continental scale mapping of intertidal exposure over time, enabling scientists and managers to integrate the data into ecological and migratory species applications and modelling.
+
+## Repository structure
+
+The DEA Intertidal Github repository contains the following important sections:
+* `intertidal`: The DEA Intertidal Python package, containing modules required for loading data, tide modelling, intertidal elevation, and exposure calculations
+* `notebooks`: Jupyter Notebooks providing workflows for generating key DEA Intertidal outputs. Importantly:
+  * `notebooks/Intertidal_CLI.ipynb`: For running the entire DEA Intertidal workflow via the Command Line Interface
+  * `notebooks/Intertidal_workflow.ipynb`: For running the entire DEA Intertidal workflow via interactive notebook cells
+  * `notebooks/Intertidal_elevation.ipynb`: For customising and running the DEA Intertidal Elevation portion of the workflow
+  * `notebooks/Intertidal_elevation_stac.ipynb`: For running DEA Intertidal Elevation on global satellite data loaded from Microsoft Planetary Computer using STAC metadata
+* `data`: Contains required `raw` input data files and output `interim` and `processed` outputs
+* `metadata`: Open Data Cube (ODC) metadata required for indexing DEA Intertidal into an ODC datacube
+* `tests`: Unit and integration tests, including automatically generated validation test results