From 35a7b3e836d062d224c4a1c59320c36f905f9c05 Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Wed, 27 Nov 2024 12:21:18 -0900 Subject: [PATCH] ADS-342 - Add regression test for opera-rtc-s1-browse * update gitattributes for opera-rtc-s1-browse regression test * add regression test for opera-rtc-s1-browse * upgrade opera-rtc-s1-browse test to harmony-py 0.5.0 * add opera-rtc-s1-browse to build-all-images.yml * add opera-rtc-s1-browse to notebook-test-suite.yml * remove unused import * apply black line length recommendations * [pre-commit.ci] auto fixes from pre-commit.com hooks --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .gitattributes | 1 + .github/workflows/build-all-images.yml | 4 + .github/workflows/notebook-test-suite.yml | 3 +- config/services_tests_config_prod.json | 5 +- config/services_tests_config_uat.json | 5 +- script/test-in-bamboo.sh | 2 +- test/Makefile | 7 +- test/opera-rtc-s1-browse/environment.yaml | 10 ++ .../opera-rtc-s1-browse_Regression.ipynb | 91 +++++++++++++++++++ .../reference_data/rgb.pgw | 6 ++ .../reference_data/rgb.png | 3 + .../reference_data/rgb.png.aux.xml | 4 + test/opera-rtc-s1-browse/version.txt | 1 + test/run_notebooks.sh | 2 +- 14 files changed, 136 insertions(+), 8 deletions(-) create mode 100644 test/opera-rtc-s1-browse/environment.yaml create mode 100644 test/opera-rtc-s1-browse/opera-rtc-s1-browse_Regression.ipynb create mode 100644 test/opera-rtc-s1-browse/reference_data/rgb.pgw create mode 100644 test/opera-rtc-s1-browse/reference_data/rgb.png create mode 100644 test/opera-rtc-s1-browse/reference_data/rgb.png.aux.xml create mode 100644 test/opera-rtc-s1-browse/version.txt diff --git a/.gitattributes b/.gitattributes index e64c0f27..f950688c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ test/nsidc-icesat2/reference_files/*.h5 filter=lfs diff=lfs merge=lfs -text test/subset-band-name/reference_data/*.hdf filter=lfs diff=lfs merge=lfs -text +test/opera-rtc-s1-browse/reference_data/*.png filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/build-all-images.yml b/.github/workflows/build-all-images.yml index 8659b779..8cddfb40 100644 --- a/.github/workflows/build-all-images.yml +++ b/.github/workflows/build-all-images.yml @@ -58,6 +58,10 @@ jobs: - image: "net2cog" notebook: "net2cog_Regression.ipynb" + - + image: "opera-rtc-s1-browse" + notebook: "opera-rtc-s1-browse_Regression.ipynb" + lfs: "true" uses: ./.github/workflows/build-target-image.yml with: diff --git a/.github/workflows/notebook-test-suite.yml b/.github/workflows/notebook-test-suite.yml index eb1ec5ad..a555c85c 100644 --- a/.github/workflows/notebook-test-suite.yml +++ b/.github/workflows/notebook-test-suite.yml @@ -60,6 +60,7 @@ on: - subset-band-name - swath-projector - trajectory-subsetter + - opera-rtc-s1-browse docker-image-tag: description: "Optional Docker image tag to use for the tests" required: false @@ -67,7 +68,7 @@ on: default: 'latest' repository_dispatch: - types: [all,geoloco,harmony-gdal-adapter,harmony-netcdf-to-zarr,harmony-service-example,hoss,hybig,net2cog,swath-projector,trajectory-subsetter] + types: [all,geoloco,harmony-gdal-adapter,harmony-netcdf-to-zarr,harmony-service-example,hoss,hybig,net2cog,swath-projector,trajectory-subsetter,opera-rtc-s1-browse] inputs: harmony-environment: required: true diff --git a/config/services_tests_config_prod.json b/config/services_tests_config_prod.json index e4afcbfe..7772476f 100644 --- a/config/services_tests_config_prod.json +++ b/config/services_tests_config_prod.json @@ -1,5 +1,5 @@ { - "all": "harmony,harmony-regression,geoloco,hga,n2z,regridder,hoss,hybig,net2cog,subset-band-name,swath-projector,nsidc-icesat2,trajectory-subsetter", + "all": "harmony,harmony-regression,geoloco,hga,n2z,regridder,hoss,hybig,net2cog,subset-band-name,swath-projector,nsidc-icesat2,trajectory-subsetter,opera-rtc-s1-browse", "batchee": "TBD", "geoloco": "geoloco", "giovanni-adapter": "TBD", @@ -17,5 +17,6 @@ "stitchee": "TBD", "subset-band-name": "subset-band-name", "swath-projector": "swath-projector", - "trajectory-subsetter": "nsidc-icesat2,trajectory-subsetter" + "trajectory-subsetter": "nsidc-icesat2,trajectory-subsetter", + "opera-rtc-s1-browse": "opera-rtc-s1-browse" } diff --git a/config/services_tests_config_uat.json b/config/services_tests_config_uat.json index 75692fd5..9cac553e 100644 --- a/config/services_tests_config_uat.json +++ b/config/services_tests_config_uat.json @@ -1,5 +1,5 @@ { - "all": "harmony,harmony-regression,geoloco,hga,n2z,regridder,hoss,variable-subsetter,hybig,net2cog,subset-band-name,swath-projector,nsidc-icesat2,trajectory-subsetter", + "all": "harmony,harmony-regression,geoloco,hga,n2z,regridder,hoss,variable-subsetter,hybig,net2cog,subset-band-name,swath-projector,nsidc-icesat2,trajectory-subsetter,opera-rtc-s1-browse", "batchee": "TBD", "geoloco": "geoloco", "giovanni-adapter": "TBD", @@ -17,5 +17,6 @@ "stitchee": "TBD", "subset-band-name": "subset-band-name", "swath-projector": "swath-projector", - "trajectory-subsetter": "nsidc-icesat2,trajectory-subsetter" + "trajectory-subsetter": "nsidc-icesat2,trajectory-subsetter", + "opera-rtc-s1-browse": "opera-rtc-s1-browse" } diff --git a/script/test-in-bamboo.sh b/script/test-in-bamboo.sh index 2fa432d7..ec47bfb5 100755 --- a/script/test-in-bamboo.sh +++ b/script/test-in-bamboo.sh @@ -42,7 +42,7 @@ echo "harmony host url: ${harmony_host_url}" ## e.g. if REGRESSION_TESTS_N2Z_IMAGE environment was set, the value would be used instead of the default. image_names=() -all_tests=(harmony harmony-regression hoss hga n2z nsidc-icesat2 swath-projector trajectory-subsetter variable-subsetter regridder hybig geoloco net2cog subset-band-name) +all_tests=(harmony harmony-regression hoss hga n2z nsidc-icesat2 swath-projector trajectory-subsetter variable-subsetter regridder hybig geoloco net2cog subset-band-name opera-rtc-s1-browse) for image in "${all_tests[@]}"; do image_names+=($(image_name "$image" true)) done diff --git a/test/Makefile b/test/Makefile index ea1a3898..26727c45 100644 --- a/test/Makefile +++ b/test/Makefile @@ -57,6 +57,10 @@ net2cog-image: Dockerfile net2cog/environment.yaml docker build -t ghcr.io/nasa/regression-tests-net2cog:latest -f ./Dockerfile \ --build-arg notebook=net2cog_Regression.ipynb --build-arg sub_dir=net2cog . +opera-rtc-s1-browse-image: Dockerfile opera-rtc-s1-browse/environment.yaml + docker build -t ghcr.io/nasa/regression-tests-opera-rtc-s1-browse:latest -f ./Dockerfile \ + --build-arg notebook=opera-rtc-s1-browse_Regression.ipynb --build-arg sub_dir=opera-rtc-s1-browse . + images: harmony-image \ harmony-regression-image \ hga-image \ @@ -70,4 +74,5 @@ images: harmony-image \ trajectory-subsetter-image \ variable-subsetter-image \ geoloco-image \ - net2cog-image + net2cog-image \ + opera-rtc-s1-browse-image diff --git a/test/opera-rtc-s1-browse/environment.yaml b/test/opera-rtc-s1-browse/environment.yaml new file mode 100644 index 00000000..54235cab --- /dev/null +++ b/test/opera-rtc-s1-browse/environment.yaml @@ -0,0 +1,10 @@ +name: papermill-opera-rtc-s1-browse +channels: + - conda-forge + - nodefaults +dependencies: + - python=3.12 + - pip=24.3.1 + - notebook=7.2.2 + - papermill=2.6.0 + - harmony-py==0.5.0 diff --git a/test/opera-rtc-s1-browse/opera-rtc-s1-browse_Regression.ipynb b/test/opera-rtc-s1-browse/opera-rtc-s1-browse_Regression.ipynb new file mode 100644 index 00000000..d12827b5 --- /dev/null +++ b/test/opera-rtc-s1-browse/opera-rtc-s1-browse_Regression.ipynb @@ -0,0 +1,91 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "24ad02a3-f52b-4e10-8654-e28d3e998c59", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9692ec8e-89e4-4742-9b11-57a8644cd770", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "from tempfile import TemporaryDirectory\n", + "\n", + "import harmony\n", + "\n", + "\n", + "environments = {\n", + " 'http://localhost:3000': harmony.Environment.LOCAL,\n", + " 'https://harmony.sit.earthdata.nasa.gov': harmony.Environment.SIT,\n", + " 'https://harmony.uat.earthdata.nasa.gov': harmony.Environment.UAT,\n", + " 'https://harmony.earthdata.nasa.gov': harmony.Environment.PROD,\n", + "}\n", + "assert harmony_host_url in environments\n", + "harmony_client = harmony.Client(env=environments[harmony_host_url])\n", + "\n", + "request = harmony.Request(\n", + " collection=harmony.Collection(id='OPERA_L2_RTC-S1_V1'),\n", + " granule_name=[\n", + " 'OPERA_L2_RTC-S1_T035-073251-IW2_20240512T020817Z_20240512T122756Z_S1A_30_v1.0'\n", + " ],\n", + " format='image/png',\n", + ")\n", + "job_id = harmony_client.submit(request)\n", + "harmony_client.wait_for_processing(job_id)\n", + "\n", + "with TemporaryDirectory() as temp_dir:\n", + " output_files = [\n", + " Path(future.result())\n", + " for future in harmony_client.download_all(job_id, directory=temp_dir)\n", + " ]\n", + "\n", + " assert len(output_files) == 3\n", + " assert output_files[0].suffixes == ['.png']\n", + " assert output_files[1].suffixes == ['.pgw']\n", + " assert output_files[2].suffixes == ['.png', '.aux', '.xml']\n", + "\n", + " reference_data = Path('reference_data')\n", + " assert output_files[0].read_bytes() == (reference_data / 'rgb.png').read_bytes()\n", + " assert output_files[1].read_bytes() == (reference_data / 'rgb.pgw').read_bytes()\n", + " assert (\n", + " output_files[2].read_bytes()\n", + " == (reference_data / 'rgb.png.aux.xml').read_bytes()\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/test/opera-rtc-s1-browse/reference_data/rgb.pgw b/test/opera-rtc-s1-browse/reference_data/rgb.pgw new file mode 100644 index 00000000..828c89aa --- /dev/null +++ b/test/opera-rtc-s1-browse/reference_data/rgb.pgw @@ -0,0 +1,6 @@ +0.0002746658541715035 +0.0 +0.0 +-0.00027463466178079705 +-122.96676772672274 +38.05992846197979 diff --git a/test/opera-rtc-s1-browse/reference_data/rgb.png b/test/opera-rtc-s1-browse/reference_data/rgb.png new file mode 100644 index 00000000..f8c734a5 --- /dev/null +++ b/test/opera-rtc-s1-browse/reference_data/rgb.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dc4fde05d3d95ea0308316eb744f7705add083cdf8c93ca8a9ff7da252eb92d +size 2102667 diff --git a/test/opera-rtc-s1-browse/reference_data/rgb.png.aux.xml b/test/opera-rtc-s1-browse/reference_data/rgb.png.aux.xml new file mode 100644 index 00000000..34bcba3e --- /dev/null +++ b/test/opera-rtc-s1-browse/reference_data/rgb.png.aux.xml @@ -0,0 +1,4 @@ + + GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AXIS["Latitude",NORTH],AXIS["Longitude",EAST],AUTHORITY["EPSG","4326"]] + -1.2296690505964982e+02, 2.7466585417150349e-04, 0.0000000000000000e+00, 3.8060065779310676e+01, 0.0000000000000000e+00, -2.7463466178079705e-04 + diff --git a/test/opera-rtc-s1-browse/version.txt b/test/opera-rtc-s1-browse/version.txt new file mode 100644 index 00000000..8acdd82b --- /dev/null +++ b/test/opera-rtc-s1-browse/version.txt @@ -0,0 +1 @@ +0.0.1 diff --git a/test/run_notebooks.sh b/test/run_notebooks.sh index 62c1b55a..5e7455af 100755 --- a/test/run_notebooks.sh +++ b/test/run_notebooks.sh @@ -15,7 +15,7 @@ echo "Running regression tests" # Specify the test images to run, by default all built by the Makefile. If # the script is invoked with a list of images, only run those. -all_images=(harmony harmony-regression hoss hga n2z nsidc-icesat2 swath-projector trajectory-subsetter variable-subsetter regridder hybig geoloco net2cog subset-band-name) +all_images=(harmony harmony-regression hoss hga n2z nsidc-icesat2 swath-projector trajectory-subsetter variable-subsetter regridder hybig geoloco net2cog subset-band-name opera-rtc-s1-browse) specified_images=() # Parse command line arguments while [[ $# -gt 0 ]]; do