From c974fbab285e9b57cf837455f0fc7ccb89ca6e07 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 25 Jun 2024 09:16:17 -0600 Subject: [PATCH 01/39] DAS-2180: allows run_tests.sh to run from command line. Removes the /home path from the coverage dir so that you can run the test script locally outside of docker. --- tests/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run_tests.sh b/tests/run_tests.sh index a821522..75e8f85 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -28,7 +28,7 @@ fi echo "\n" echo "Test Coverage Estimates" coverage report --omit="tests/*" -coverage html --omit="tests/*" -d /home/tests/coverage +coverage html --omit="tests/*" -d tests/coverage # Run pylint # Ignored errors/warnings: From 0d4305fcd60361c14dd322b9be0d48ce32fe1152 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 25 Jun 2024 13:54:43 -0600 Subject: [PATCH 02/39] DAS-2180: Extracts harmony_service_entry. First stab at separating concerns. This now has a harmony_browse_image_generator as well as a harmony_service_entry. --- README.md | 9 +++++++-- docker/service.Dockerfile | 3 ++- docker/service_version.txt | 2 +- .../__main__.py | 2 +- .../adapter.py | 0 tests/test_adapter.py | 20 +++++++++---------- tests/unit/test_adapter.py | 4 ++-- 7 files changed, 23 insertions(+), 17 deletions(-) rename {harmony_browse_image_generator => harmony_service_entry}/__main__.py (91%) rename {harmony_browse_image_generator => harmony_service_entry}/adapter.py (100%) diff --git a/README.md b/README.md index 962faa8..ef39323 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,7 @@ also with units of degrees. |- 📂 docker |- 📂 docs |- 📂 harmony_browse_image_generator +|- 📂 harmony_service_entry |- 📂 tests |- CHANGELOG.md |- CONTRIBUTING.md @@ -156,8 +157,12 @@ also with units of degrees. * `docs` - A directory with example usage notebooks. * `harmony_browse_image_generator` - A directory containing Python source code - for the HyBIG. `adapter.py` contains the `BrowseImageGeneratorAdapter` - class that is invoked by calls to the service. + for the HyBIG library. This directory contains the business logic for + generating GIBS compatible browse images. + +* `harmony_service_entry` - A directory containing the Harmony Service specific + python code. `adapter.py` contains the `BrowseImageGeneratorAdapter` class + that is invoked by calls to the service. * `tests` - A directory containing the service unit test suite. diff --git a/docker/service.Dockerfile b/docker/service.Dockerfile index cad007d..6e51aea 100644 --- a/docker/service.Dockerfile +++ b/docker/service.Dockerfile @@ -29,9 +29,10 @@ RUN pip install --no-input --no-cache-dir \ # Copy service code. COPY ./harmony_browse_image_generator harmony_browse_image_generator +COPY ./harmony_service_entry harmony_service_entry # Set GDAL related environment variables. ENV CPL_ZIP_ENCODING=UTF-8 # Configure a container to be executable via the `docker run` command. -ENTRYPOINT ["python", "-m", "harmony_browse_image_generator"] +ENTRYPOINT ["python", "-m", "harmony_service_entry"] diff --git a/docker/service_version.txt b/docker/service_version.txt index 23aa839..f0bb29e 100644 --- a/docker/service_version.txt +++ b/docker/service_version.txt @@ -1 +1 @@ -1.2.2 +1.3.0 diff --git a/harmony_browse_image_generator/__main__.py b/harmony_service_entry/__main__.py similarity index 91% rename from harmony_browse_image_generator/__main__.py rename to harmony_service_entry/__main__.py index 60ad288..bbc0764 100644 --- a/harmony_browse_image_generator/__main__.py +++ b/harmony_service_entry/__main__.py @@ -5,8 +5,8 @@ from harmony import is_harmony_cli, run_cli, setup_cli -from harmony_browse_image_generator.adapter import BrowseImageGeneratorAdapter from harmony_browse_image_generator.exceptions import SERVICE_NAME +from harmony_service_entry.adapter import BrowseImageGeneratorAdapter def main(arguments: list[str]): diff --git a/harmony_browse_image_generator/adapter.py b/harmony_service_entry/adapter.py similarity index 100% rename from harmony_browse_image_generator/adapter.py rename to harmony_service_entry/adapter.py diff --git a/tests/test_adapter.py b/tests/test_adapter.py index 454e23d..0e74226 100644 --- a/tests/test_adapter.py +++ b/tests/test_adapter.py @@ -14,16 +14,16 @@ from rasterio.warp import Resampling from rioxarray import open_rasterio -from harmony_browse_image_generator.adapter import BrowseImageGeneratorAdapter from harmony_browse_image_generator.browse import ( convert_mulitband_to_raster, prepare_raster_for_writing, ) +from harmony_service_entry.adapter import BrowseImageGeneratorAdapter from tests.utilities import Granule, create_stac class TestAdapter(TestCase): - """A class testing the harmony_browse_image_generator.adapter module.""" + """A class testing the harmony_service_entry.adapter module.""" @classmethod def setUpClass(cls): @@ -99,10 +99,10 @@ def assert_expected_output_catalog( ) @patch('harmony_browse_image_generator.browse.reproject') - @patch('harmony_browse_image_generator.adapter.rmtree') - @patch('harmony_browse_image_generator.adapter.mkdtemp') - @patch('harmony_browse_image_generator.adapter.download') - @patch('harmony_browse_image_generator.adapter.stage') + @patch('harmony_service_entry.adapter.rmtree') + @patch('harmony_service_entry.adapter.mkdtemp') + @patch('harmony_service_entry.adapter.download') + @patch('harmony_service_entry.adapter.stage') def test_valid_request_jpeg( self, mock_stage, mock_download, mock_mkdtemp, mock_rmtree, mock_reproject ): @@ -334,10 +334,10 @@ def move_tif(*args, **kwargs): mock_rmtree.assert_called_once_with(self.temp_dir) @patch('harmony_browse_image_generator.browse.reproject') - @patch('harmony_browse_image_generator.adapter.rmtree') - @patch('harmony_browse_image_generator.adapter.mkdtemp') - @patch('harmony_browse_image_generator.adapter.download') - @patch('harmony_browse_image_generator.adapter.stage') + @patch('harmony_service_entry.adapter.rmtree') + @patch('harmony_service_entry.adapter.mkdtemp') + @patch('harmony_service_entry.adapter.download') + @patch('harmony_service_entry.adapter.stage') def test_valid_request_png( self, mock_stage, mock_download, mock_mkdtemp, mock_rmtree, mock_reproject ): diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index 2a34b84..1cf3715 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -6,13 +6,13 @@ from harmony.util import config from pystac import Asset, Item -from harmony_browse_image_generator.adapter import BrowseImageGeneratorAdapter from harmony_browse_image_generator.exceptions import HyBIGInvalidMessageError +from harmony_service_entry.adapter import BrowseImageGeneratorAdapter from tests.utilities import Granule, create_stac class TestAdapter(TestCase): - """A class testing the harmony_browse_image_generator.adapter module.""" + """A class testing the harmony_service_entry.adapter module.""" @classmethod def setUpClass(cls): From 0e62f898cd2e4d26c838ab2abff02d337bbbb050 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 26 Jun 2024 09:02:29 -0600 Subject: [PATCH 03/39] DAS-2180: Adds pyproject.toml skeleton --- pyproject.toml | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..22dd44b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,68 @@ +[project] +name = "browse-image-generator" +dynamic = ["dependencies", "version"] + + +authors = [ + {name="Owen Littlejohns", email="owen.m.littlejohns@nasa.gov"}, + {name="Matt Savoie", email="savoie@colorado.edu"}, +] +description = "Python package designed to produce browse imagery compatible with NASA's Global Image Browse Services (GIBS)." + +readme = "README.md" +requires-python = ">=3.9" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] + +[project.urls] +Homepage = "https://github.com/nasa/harmony-browse-image-generator" +Issues = "https://github.com/nasa/harmony-browse-image-generator/issues" + +[build-system] +requires = ["hatchling", "hatch-requirements-txt"] +build-backend = "hatchling.build" + +[tool.hatch.metadata.hooks.requirements_txt] +files = [ + "pip_requirements.txt", + "pip_requirements_skip_snyk.txt" +] +[tool.hatch.version] +path = "docker/service_version.txt" +pattern= '^v?(?P.*)$' # + + +[tool.hatch.build.targets.sdist] +include = [ + "harmony_browse_image_generator/*.py" +] +exclude = [ + ".*", +] +[tool.hatch.build.targets.wheel] +packages=["harmony_browse_image_generator"] + +[tool.black] +skip-string-normalization = 1 + +[tool.isort] +profile = "black" + +[tool.ruff] +lint.select = [ + "E", # pycodestyle + "F", # pyflakes + "UP", # pyupgrade + "I", # organize imports + "D", # docstyle +] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[[tool.mypy.overrides]] +module = "harmony.*,matplotlib.*,rasterio.*,osgeo_utils.*,pystac.*,affine.*,pycodestyle.*,imagequant.*,PIL.*,rioxarray.*, xarray.*" +ignore_missing_imports = true From fb3a1a7e747f2306a5851f7e9fd320110f76b95f Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 26 Jun 2024 10:31:43 -0600 Subject: [PATCH 04/39] DAS-2180: Move service utilities to service entry. --- harmony_browse_image_generator/__init__.py | 7 +++++++ harmony_service_entry/__main__.py | 9 +++------ harmony_service_entry/adapter.py | 14 ++++++-------- .../utilities.py | 2 +- tests/unit/test_utilities.py | 2 +- 5 files changed, 18 insertions(+), 16 deletions(-) rename {harmony_browse_image_generator => harmony_service_entry}/utilities.py (96%) diff --git a/harmony_browse_image_generator/__init__.py b/harmony_browse_image_generator/__init__.py index e69de29..a88970b 100644 --- a/harmony_browse_image_generator/__init__.py +++ b/harmony_browse_image_generator/__init__.py @@ -0,0 +1,7 @@ +"""Package containing core functionality for browse image generation.""" + +from .browse import create_browse_imagery +from .color_utility import get_color_palette_from_item +from .exceptions import SERVICE_NAME + +__all__ = ['create_browse_imagery', 'SERVICE_NAME', 'get_color_palette_from_item'] diff --git a/harmony_service_entry/__main__.py b/harmony_service_entry/__main__.py index bbc0764..f7c025d 100644 --- a/harmony_service_entry/__main__.py +++ b/harmony_service_entry/__main__.py @@ -1,19 +1,16 @@ -""" Run the Harmony Browse Image Generator Adapter via the Harmony CLI. """ +"""Run the Harmony Browse Image Generator Adapter via the Harmony CLI.""" from argparse import ArgumentParser from sys import argv from harmony import is_harmony_cli, run_cli, setup_cli -from harmony_browse_image_generator.exceptions import SERVICE_NAME +from harmony_browse_image_generator import SERVICE_NAME from harmony_service_entry.adapter import BrowseImageGeneratorAdapter def main(arguments: list[str]): - """Parse command line arguments and invoke the appropriate method to - respond to them - - """ + """Parse command line arguments and invoke the appropriate method.""" parser = ArgumentParser( prog=SERVICE_NAME, description='Run Harmony Browse Image Generator.' ) diff --git a/harmony_service_entry/adapter.py b/harmony_service_entry/adapter.py index 5d6bdf3..681f874 100644 --- a/harmony_service_entry/adapter.py +++ b/harmony_service_entry/adapter.py @@ -23,10 +23,12 @@ from harmony.util import bbox_to_geometry, download, generate_output_filename, stage from pystac import Asset, Catalog, Item -from harmony_browse_image_generator.browse import create_browse_imagery -from harmony_browse_image_generator.color_utility import get_color_palette_from_item +from harmony_browse_image_generator import ( + create_browse_imagery, + get_color_palette_from_item, +) from harmony_browse_image_generator.exceptions import HyBIGInvalidMessageError -from harmony_browse_image_generator.utilities import ( +from harmony_service_entry.utilities import ( get_asset_name, get_file_mime_type, get_tiled_file_extension, @@ -34,10 +36,7 @@ class BrowseImageGeneratorAdapter(BaseHarmonyAdapter): - """This class extends the BaseHarmonyAdapter class from the - harmony-service-lib package to implement HyBIG operations. - - """ + """HyBIG extention to the harmony-service-lib BaseHarmonyAdapter.""" def invoke(self) -> Catalog: """Adds validation to process_item based invocations.""" @@ -91,7 +90,6 @@ def get_asset_from_item(self, item: Item) -> Asset: def process_item(self, item: Item, source: HarmonySource) -> Item: """Processes a single input STAC item.""" - try: working_directory = mkdtemp() results = item.clone() diff --git a/harmony_browse_image_generator/utilities.py b/harmony_service_entry/utilities.py similarity index 96% rename from harmony_browse_image_generator/utilities.py rename to harmony_service_entry/utilities.py index f8640d8..892c5f3 100644 --- a/harmony_browse_image_generator/utilities.py +++ b/harmony_service_entry/utilities.py @@ -1,4 +1,4 @@ -""" Module containing utility functionality. """ +"""Module containing service utility functionality.""" import re from mimetypes import guess_type as guess_mime_type diff --git a/tests/unit/test_utilities.py b/tests/unit/test_utilities.py index 1a0800f..8548fc7 100644 --- a/tests/unit/test_utilities.py +++ b/tests/unit/test_utilities.py @@ -1,7 +1,7 @@ from pathlib import Path from unittest import TestCase -from harmony_browse_image_generator.utilities import ( +from harmony_service_entry.utilities import ( get_asset_name, get_file_mime_type, get_tiled_file_extension, From bf4a010325a758f2055dabcf274688d8fe0c36e6 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 26 Jun 2024 10:43:33 -0600 Subject: [PATCH 05/39] DAS-2180: Renames directory before fixing imports --- .../__init__.py | 0 .../browse.py | 21 +++++++++---------- .../color_utility.py | 7 +++---- .../crs.py | 3 +-- .../exceptions.py | 0 .../sizes.py | 15 +++++++------ pyproject.toml | 2 +- 7 files changed, 22 insertions(+), 26 deletions(-) rename {harmony_browse_image_generator => hybig}/__init__.py (100%) rename {harmony_browse_image_generator => hybig}/browse.py (99%) rename {harmony_browse_image_generator => hybig}/color_utility.py (99%) rename {harmony_browse_image_generator => hybig}/crs.py (99%) rename {harmony_browse_image_generator => hybig}/exceptions.py (100%) rename {harmony_browse_image_generator => hybig}/sizes.py (99%) diff --git a/harmony_browse_image_generator/__init__.py b/hybig/__init__.py similarity index 100% rename from harmony_browse_image_generator/__init__.py rename to hybig/__init__.py diff --git a/harmony_browse_image_generator/browse.py b/hybig/browse.py similarity index 99% rename from harmony_browse_image_generator/browse.py rename to hybig/browse.py index a867a60..8b5d272 100644 --- a/harmony_browse_image_generator/browse.py +++ b/hybig/browse.py @@ -11,17 +11,6 @@ from affine import dumpsw from harmony.message import Message as HarmonyMessage from harmony.message import Source as HarmonySource -from matplotlib.cm import ScalarMappable -from matplotlib.colors import Normalize -from numpy import ndarray -from osgeo_utils.auxiliary.color_palette import ColorPalette -from PIL import Image -from rasterio.io import DatasetReader -from rasterio.plot import reshape_as_image, reshape_as_raster -from rasterio.warp import Resampling, reproject -from rioxarray import open_rasterio -from xarray import DataArray - from harmony_browse_image_generator.color_utility import ( NODATA_IDX, NODATA_RGBA, @@ -39,6 +28,16 @@ create_tiled_output_parameters, get_target_grid_parameters, ) +from matplotlib.cm import ScalarMappable +from matplotlib.colors import Normalize +from numpy import ndarray +from osgeo_utils.auxiliary.color_palette import ColorPalette +from PIL import Image +from rasterio.io import DatasetReader +from rasterio.plot import reshape_as_image, reshape_as_raster +from rasterio.warp import Resampling, reproject +from rioxarray import open_rasterio +from xarray import DataArray def create_browse_imagery( diff --git a/harmony_browse_image_generator/color_utility.py b/hybig/color_utility.py similarity index 99% rename from harmony_browse_image_generator/color_utility.py rename to hybig/color_utility.py index ce62d07..c794aa7 100644 --- a/harmony_browse_image_generator/color_utility.py +++ b/hybig/color_utility.py @@ -10,14 +10,13 @@ import numpy as np import requests from harmony.message import Source as HarmonySource -from osgeo_utils.auxiliary.color_palette import ColorPalette -from pystac import Item -from rasterio.io import DatasetReader - from harmony_browse_image_generator.exceptions import ( HyBIGError, HyBIGNoColorInformation, ) +from osgeo_utils.auxiliary.color_palette import ColorPalette +from pystac import Item +from rasterio.io import DatasetReader # Constants for output PNG images # Applied to transparent pixels where alpha < 255 diff --git a/harmony_browse_image_generator/crs.py b/hybig/crs.py similarity index 99% rename from harmony_browse_image_generator/crs.py rename to hybig/crs.py index efd8686..5ef768b 100644 --- a/harmony_browse_image_generator/crs.py +++ b/hybig/crs.py @@ -11,14 +11,13 @@ """ from harmony.message import SRS +from harmony_browse_image_generator.exceptions import HyBIGInvalidMessageError from pyproj.crs import CRS as pyCRS # pylint: disable-next=no-name-in-module from rasterio.crs import CRS from xarray import DataArray -from harmony_browse_image_generator.exceptions import HyBIGInvalidMessageError - # These are the CRSs that GIBS will accept as input. When the user hasn't # directly specified an output CRS, the code will attempt to choose the best # one of these. diff --git a/harmony_browse_image_generator/exceptions.py b/hybig/exceptions.py similarity index 100% rename from harmony_browse_image_generator/exceptions.py rename to hybig/exceptions.py diff --git a/harmony_browse_image_generator/sizes.py b/hybig/sizes.py similarity index 99% rename from harmony_browse_image_generator/sizes.py rename to hybig/sizes.py index f6e9473..7ef253a 100644 --- a/harmony_browse_image_generator/sizes.py +++ b/hybig/sizes.py @@ -15,14 +15,6 @@ from affine import Affine from harmony.message import Message from harmony.message_utility import has_dimensions, has_scale_extents, has_scale_sizes -from pyproj import Transformer -from pyproj.crs import CRS as pyCRS - -# pylint: disable-next=no-name-in-module -from rasterio.crs import CRS -from rasterio.transform import AffineTransformer, from_bounds, from_origin -from xarray import DataArray - from harmony_browse_image_generator.crs import ( PREFERRED_CRS, choose_best_crs_from_metadata, @@ -30,6 +22,13 @@ is_preferred_crs, ) from harmony_browse_image_generator.exceptions import HyBIGValueError +from pyproj import Transformer +from pyproj.crs import CRS as pyCRS + +# pylint: disable-next=no-name-in-module +from rasterio.crs import CRS +from rasterio.transform import AffineTransformer, from_bounds, from_origin +from xarray import DataArray class GridParams(TypedDict): diff --git a/pyproject.toml b/pyproject.toml index 22dd44b..79dd0a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "browse-image-generator" +name = "hybig" dynamic = ["dependencies", "version"] From f885f45eea9435729f832708dfe61171e6f672d1 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 26 Jun 2024 10:44:36 -0600 Subject: [PATCH 06/39] DAS-2180: renames harmony_browse_image_generator -> hybig --- README.md | 4 ++-- docker/service.Dockerfile | 2 +- harmony_service_entry/__main__.py | 2 +- harmony_service_entry/adapter.py | 10 ++++----- hybig/browse.py | 27 ++++++++++++----------- hybig/color_utility.py | 9 ++++---- hybig/crs.py | 3 ++- hybig/sizes.py | 15 +++++++------ pyproject.toml | 8 +++---- tests/run_tests.sh | 2 +- tests/test_adapter.py | 8 +++---- tests/test_code_format.py | 4 ++-- tests/unit/test_adapter.py | 2 +- tests/unit/test_browse.py | 24 ++++++++++----------- tests/unit/test_color_utility.py | 36 ++++++++++--------------------- tests/unit/test_crs.py | 6 +++--- tests/unit/test_sizes.py | 14 ++++++------ tests/unit/test_utilities.py | 2 +- 18 files changed, 84 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index ef39323..26499cd 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ also with units of degrees. |- 📂 bin |- 📂 docker |- 📂 docs -|- 📂 harmony_browse_image_generator +|- 📂 hybig |- 📂 harmony_service_entry |- 📂 tests |- CHANGELOG.md @@ -156,7 +156,7 @@ also with units of degrees. * `docs` - A directory with example usage notebooks. -* `harmony_browse_image_generator` - A directory containing Python source code +* `hybig` - A directory containing Python source code for the HyBIG library. This directory contains the business logic for generating GIBS compatible browse images. diff --git a/docker/service.Dockerfile b/docker/service.Dockerfile index 6e51aea..b6ddf38 100644 --- a/docker/service.Dockerfile +++ b/docker/service.Dockerfile @@ -28,7 +28,7 @@ RUN pip install --no-input --no-cache-dir \ -r pip_requirements_skip_snyk.txt # Copy service code. -COPY ./harmony_browse_image_generator harmony_browse_image_generator +COPY ./hybig hybig COPY ./harmony_service_entry harmony_service_entry # Set GDAL related environment variables. diff --git a/harmony_service_entry/__main__.py b/harmony_service_entry/__main__.py index f7c025d..ae852a7 100644 --- a/harmony_service_entry/__main__.py +++ b/harmony_service_entry/__main__.py @@ -5,8 +5,8 @@ from harmony import is_harmony_cli, run_cli, setup_cli -from harmony_browse_image_generator import SERVICE_NAME from harmony_service_entry.adapter import BrowseImageGeneratorAdapter +from hybig import SERVICE_NAME def main(arguments: list[str]): diff --git a/harmony_service_entry/adapter.py b/harmony_service_entry/adapter.py index 681f874..1665cf3 100644 --- a/harmony_service_entry/adapter.py +++ b/harmony_service_entry/adapter.py @@ -23,16 +23,16 @@ from harmony.util import bbox_to_geometry, download, generate_output_filename, stage from pystac import Asset, Catalog, Item -from harmony_browse_image_generator import ( - create_browse_imagery, - get_color_palette_from_item, -) -from harmony_browse_image_generator.exceptions import HyBIGInvalidMessageError from harmony_service_entry.utilities import ( get_asset_name, get_file_mime_type, get_tiled_file_extension, ) +from hybig import ( + create_browse_imagery, + get_color_palette_from_item, +) +from hybig.exceptions import HyBIGInvalidMessageError class BrowseImageGeneratorAdapter(BaseHarmonyAdapter): diff --git a/hybig/browse.py b/hybig/browse.py index 8b5d272..41e04ef 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -11,7 +11,18 @@ from affine import dumpsw from harmony.message import Message as HarmonyMessage from harmony.message import Source as HarmonySource -from harmony_browse_image_generator.color_utility import ( +from matplotlib.cm import ScalarMappable +from matplotlib.colors import Normalize +from numpy import ndarray +from osgeo_utils.auxiliary.color_palette import ColorPalette +from PIL import Image +from rasterio.io import DatasetReader +from rasterio.plot import reshape_as_image, reshape_as_raster +from rasterio.warp import Resampling, reproject +from rioxarray import open_rasterio +from xarray import DataArray + +from hybig.color_utility import ( NODATA_IDX, NODATA_RGBA, OPAQUE, @@ -22,22 +33,12 @@ get_color_palette, remove_alpha, ) -from harmony_browse_image_generator.exceptions import HyBIGError -from harmony_browse_image_generator.sizes import ( +from hybig.exceptions import HyBIGError +from hybig.sizes import ( GridParams, create_tiled_output_parameters, get_target_grid_parameters, ) -from matplotlib.cm import ScalarMappable -from matplotlib.colors import Normalize -from numpy import ndarray -from osgeo_utils.auxiliary.color_palette import ColorPalette -from PIL import Image -from rasterio.io import DatasetReader -from rasterio.plot import reshape_as_image, reshape_as_raster -from rasterio.warp import Resampling, reproject -from rioxarray import open_rasterio -from xarray import DataArray def create_browse_imagery( diff --git a/hybig/color_utility.py b/hybig/color_utility.py index c794aa7..2daef7a 100644 --- a/hybig/color_utility.py +++ b/hybig/color_utility.py @@ -10,14 +10,15 @@ import numpy as np import requests from harmony.message import Source as HarmonySource -from harmony_browse_image_generator.exceptions import ( - HyBIGError, - HyBIGNoColorInformation, -) from osgeo_utils.auxiliary.color_palette import ColorPalette from pystac import Item from rasterio.io import DatasetReader +from hybig.exceptions import ( + HyBIGError, + HyBIGNoColorInformation, +) + # Constants for output PNG images # Applied to transparent pixels where alpha < 255 TRANSPARENT = np.uint8(0) diff --git a/hybig/crs.py b/hybig/crs.py index 5ef768b..4f86fff 100644 --- a/hybig/crs.py +++ b/hybig/crs.py @@ -11,13 +11,14 @@ """ from harmony.message import SRS -from harmony_browse_image_generator.exceptions import HyBIGInvalidMessageError from pyproj.crs import CRS as pyCRS # pylint: disable-next=no-name-in-module from rasterio.crs import CRS from xarray import DataArray +from hybig.exceptions import HyBIGInvalidMessageError + # These are the CRSs that GIBS will accept as input. When the user hasn't # directly specified an output CRS, the code will attempt to choose the best # one of these. diff --git a/hybig/sizes.py b/hybig/sizes.py index 7ef253a..cf92672 100644 --- a/hybig/sizes.py +++ b/hybig/sizes.py @@ -15,13 +15,6 @@ from affine import Affine from harmony.message import Message from harmony.message_utility import has_dimensions, has_scale_extents, has_scale_sizes -from harmony_browse_image_generator.crs import ( - PREFERRED_CRS, - choose_best_crs_from_metadata, - choose_target_crs, - is_preferred_crs, -) -from harmony_browse_image_generator.exceptions import HyBIGValueError from pyproj import Transformer from pyproj.crs import CRS as pyCRS @@ -30,6 +23,14 @@ from rasterio.transform import AffineTransformer, from_bounds, from_origin from xarray import DataArray +from hybig.crs import ( + PREFERRED_CRS, + choose_best_crs_from_metadata, + choose_target_crs, + is_preferred_crs, +) +from hybig.exceptions import HyBIGValueError + class GridParams(TypedDict): """Convenience to describe a grid parameters dictionary.""" diff --git a/pyproject.toml b/pyproject.toml index 79dd0a1..044feaa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,7 @@ [project] -name = "hybig" +name = "hybig-py" dynamic = ["dependencies", "version"] - authors = [ {name="Owen Littlejohns", email="owen.m.littlejohns@nasa.gov"}, {name="Matt Savoie", email="savoie@colorado.edu"}, @@ -37,13 +36,14 @@ pattern= '^v?(?P.*)$' # [tool.hatch.build.targets.sdist] include = [ - "harmony_browse_image_generator/*.py" + "hybig/*.py" ] exclude = [ ".*", ] + [tool.hatch.build.targets.wheel] -packages=["harmony_browse_image_generator"] +packages=["hybig"] [tool.black] skip-string-normalization = 1 diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 75e8f85..7d60861 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -35,7 +35,7 @@ coverage html --omit="tests/*" -d tests/coverage # W1203 - use of f-strings in log statements. This warning is leftover from # using ''.format() vs % notation. For more information, see: # https://github.com/PyCQA/pylint/issues/2354#issuecomment-414526879 -pylint harmony_browse_image_generator --disable=W1203 +pylint hybig --disable=W1203 RESULT=$? RESULT=$((3 & $RESULT)) diff --git a/tests/test_adapter.py b/tests/test_adapter.py index 0e74226..b62393d 100644 --- a/tests/test_adapter.py +++ b/tests/test_adapter.py @@ -14,11 +14,11 @@ from rasterio.warp import Resampling from rioxarray import open_rasterio -from harmony_browse_image_generator.browse import ( +from harmony_service_entry.adapter import BrowseImageGeneratorAdapter +from hybig.browse import ( convert_mulitband_to_raster, prepare_raster_for_writing, ) -from harmony_service_entry.adapter import BrowseImageGeneratorAdapter from tests.utilities import Granule, create_stac @@ -98,7 +98,7 @@ def assert_expected_output_catalog( }, ) - @patch('harmony_browse_image_generator.browse.reproject') + @patch('hybig.browse.reproject') @patch('harmony_service_entry.adapter.rmtree') @patch('harmony_service_entry.adapter.mkdtemp') @patch('harmony_service_entry.adapter.download') @@ -333,7 +333,7 @@ def move_tif(*args, **kwargs): # Ensure container clean-up was requested: mock_rmtree.assert_called_once_with(self.temp_dir) - @patch('harmony_browse_image_generator.browse.reproject') + @patch('hybig.browse.reproject') @patch('harmony_service_entry.adapter.rmtree') @patch('harmony_service_entry.adapter.mkdtemp') @patch('harmony_service_entry.adapter.download') diff --git a/tests/test_code_format.py b/tests/test_code_format.py index 27f825b..f015a57 100644 --- a/tests/test_code_format.py +++ b/tests/test_code_format.py @@ -21,10 +21,10 @@ class TestCodeFormat(TestCase): @classmethod def setUpClass(cls): - cls.python_files = Path('harmony_browse_image_generator').rglob('*.py') + cls.python_files = Path('hybig').rglob('*.py') def test_pycodestyle_adherence(self): - """Ensure all code in the `harmony_browse_image_generator` directory + """Ensure all code in the `hybig` directory adheres to PEP8 defined standard. """ diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index 1cf3715..1ea8e46 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -6,8 +6,8 @@ from harmony.util import config from pystac import Asset, Item -from harmony_browse_image_generator.exceptions import HyBIGInvalidMessageError from harmony_service_entry.adapter import BrowseImageGeneratorAdapter +from hybig.exceptions import HyBIGInvalidMessageError from tests.utilities import Granule, create_stac diff --git a/tests/unit/test_browse.py b/tests/unit/test_browse.py index 9ea1293..f3359f8 100644 --- a/tests/unit/test_browse.py +++ b/tests/unit/test_browse.py @@ -20,7 +20,7 @@ from rasterio.warp import Resampling from xarray import DataArray -from harmony_browse_image_generator.browse import ( +from hybig.browse import ( convert_mulitband_to_raster, convert_singleband_to_raster, create_browse_imagery, @@ -33,19 +33,19 @@ validate_file_crs, validate_file_type, ) -from harmony_browse_image_generator.color_utility import ( +from hybig.color_utility import ( OPAQUE, TRANSPARENT, convert_colormap_to_palette, get_color_palette, palette_from_remote_colortable, ) -from harmony_browse_image_generator.exceptions import HyBIGError +from hybig.exceptions import HyBIGError from tests.unit.utility import rasterio_test_file class TestBrowse(TestCase): - """A class testing the harmony_browse_image_generator.browse module.""" + """A class testing the hybig.browse module.""" @classmethod def setUpClass(cls): @@ -146,9 +146,9 @@ def test_create_browse_imagery_with_single_band_raster(self): message, test_tif_filename, HarmonySource({}), None, None ) - @patch('harmony_browse_image_generator.browse.reproject') + @patch('hybig.browse.reproject') @patch('rasterio.open') - @patch('harmony_browse_image_generator.browse.open_rasterio') + @patch('hybig.browse.open_rasterio') def test_create_browse_imagery_with_mocks( self, rioxarray_open_mock, rasterio_open_mock, reproject_mock ): @@ -548,7 +548,7 @@ def test_prepare_raster_for_writing_jpeg_4band(self): self.assertEqual(expected_color_map, actual_color_map) np.testing.assert_array_equal(expected_raster, actual_raster) - @patch('harmony_browse_image_generator.browse.palettize_raster') + @patch('hybig.browse.palettize_raster') def test_prepare_raster_for_writing_png_4band(self, palettize_mock): raster = self.random.integers(255, size=(4, 7, 8)) driver = 'PNG' @@ -557,8 +557,8 @@ def test_prepare_raster_for_writing_png_4band(self, palettize_mock): palettize_mock.assert_called_once_with(raster) - @patch('harmony_browse_image_generator.browse.Image') - @patch('harmony_browse_image_generator.browse.get_color_map_from_image') + @patch('hybig.browse.Image') + @patch('hybig.browse.get_color_map_from_image') def test_palettize_raster_no_alpha_layer(self, get_color_map_mock, image_mock): """Test that the quantize function is called by a correct image.""" @@ -580,8 +580,8 @@ def test_palettize_raster_no_alpha_layer(self, get_color_map_mock, image_mock): np.testing.assert_array_equal(expected_out_raster, out_raster) - @patch('harmony_browse_image_generator.browse.Image') - @patch('harmony_browse_image_generator.browse.get_color_map_from_image') + @patch('hybig.browse.Image') + @patch('hybig.browse.get_color_map_from_image') def test_palettize_raster_with_alpha_layer(self, get_color_map_mock, image_mock): """Test that the quantize function is called by a correct image.""" @@ -747,7 +747,7 @@ def test_validate_file_type_invalid(self): ): validate_file_type(ds) - @patch('harmony_browse_image_generator.color_utility.requests.get') + @patch('hybig.color_utility.requests.get') def test_palette_from_remote_colortable(self, mock_get): with self.subTest('successful retrieval of colortable'): returned_colortable = ( diff --git a/tests/unit/test_color_utility.py b/tests/unit/test_color_utility.py index 9de7db6..2f65870 100644 --- a/tests/unit/test_color_utility.py +++ b/tests/unit/test_color_utility.py @@ -8,13 +8,13 @@ from rasterio import DatasetReader from requests import Response -from harmony_browse_image_generator.color_utility import ( +from hybig.color_utility import ( convert_colormap_to_palette, get_color_palette, get_color_palette_from_item, get_remote_palette_from_source, ) -from harmony_browse_image_generator.exceptions import ( +from hybig.exceptions import ( HyBIGError, HyBIGNoColorInformation, ) @@ -44,9 +44,7 @@ def setUp(self): props = {} self.item = Item('id', geometry, bbox, date, props) - @patch( - 'harmony_browse_image_generator.color_utility.palette_from_remote_colortable' - ) + @patch('hybig.color_utility.palette_from_remote_colortable') def test_get_color_palette_from_item_with_no_assets( self, palette_from_remote_colortable_mock ): @@ -54,9 +52,7 @@ def test_get_color_palette_from_item_with_no_assets( self.assertIsNone(actual) palette_from_remote_colortable_mock.assert_not_called() - @patch( - 'harmony_browse_image_generator.color_utility.palette_from_remote_colortable' - ) + @patch('hybig.color_utility.palette_from_remote_colortable') def test_get_color_palette_from_item_no_palette_asset( self, palette_from_remote_colortable_mock ): @@ -67,9 +63,7 @@ def test_get_color_palette_from_item_no_palette_asset( self.assertIsNone(actual) palette_from_remote_colortable_mock.assert_not_called() - @patch( - 'harmony_browse_image_generator.color_utility.palette_from_remote_colortable' - ) + @patch('hybig.color_utility.palette_from_remote_colortable') def test_get_color_palette_from_item_palette_asset(self, palette_from_remote_mock): asset = Asset('data href', roles=['data']) palette_asset = Asset('palette href', roles=['palette']) @@ -85,7 +79,7 @@ def test_get_color_palette_from_item_palette_asset(self, palette_from_remote_moc palette_from_remote_mock.assert_called_once_with('palette href') self.assertEqual(expected_palette, actual) - @patch('harmony_browse_image_generator.color_utility.requests.get') + @patch('hybig.color_utility.requests.get') def test_get_color_palette_from_item_palette_asset_fails(self, get_mock): """Raise exception if there is a colortable, but it cannot be retrieved.""" asset = Asset('data href', roles=['data']) @@ -104,9 +98,7 @@ def test_get_color_palette_from_item_palette_asset_fails(self, get_mock): ): get_color_palette_from_item(self.item) - @patch( - 'harmony_browse_image_generator.color_utility.palette_from_remote_colortable' - ) + @patch('hybig.color_utility.palette_from_remote_colortable') def test_get_remote_palette_from_source(self, palette_from_remote_mock): with self.subTest('No variables in source'): test_source = HarmonySource({}) @@ -212,9 +204,7 @@ def test_get_remote_palette_from_source(self, palette_from_remote_mock): get_remote_palette_from_source(test_source) palette_from_remote_mock.reset_mock() - @patch( - 'harmony_browse_image_generator.color_utility.get_remote_palette_from_source' - ) + @patch('hybig.color_utility.get_remote_palette_from_source') def test_get_color_palette_with_item_palette( self, get_remote_palette_from_source_mock ): @@ -228,7 +218,7 @@ def test_get_color_palette_with_item_palette( get_remote_palette_from_source_mock.assert_not_called() ds.colormap.assert_not_called() - @patch('harmony_browse_image_generator.color_utility.requests.get') + @patch('hybig.color_utility.requests.get') def test_get_color_palette_request_fails(self, get_mock): failed_response = Mock(Response) failed_response.ok = False @@ -258,9 +248,7 @@ def test_get_color_palette_request_fails(self, get_mock): get_color_palette(ds, source, None) ds.colormap.assert_not_called() - @patch( - 'harmony_browse_image_generator.color_utility.palette_from_remote_colortable' - ) + @patch('hybig.color_utility.palette_from_remote_colortable') def test_get_color_palette_finds_no_url(self, palette_from_remote_mock): palette_from_remote_mock.side_effect = HyBIGError('mocked exception') ds = Mock(DatasetReader) @@ -287,9 +275,7 @@ def test_get_color_palette_finds_no_url(self, palette_from_remote_mock): get_color_palette(ds, source, None) palette_from_remote_mock.assert_called_once_with('url:of:colortable') - @patch( - 'harmony_browse_image_generator.color_utility.palette_from_remote_colortable' - ) + @patch('hybig.color_utility.palette_from_remote_colortable') def test_get_color_palette_source_remote_exists(self, palette_from_remote_mock): ds = Mock(DatasetReader) ds.colormap.return_value = self.colormap diff --git a/tests/unit/test_crs.py b/tests/unit/test_crs.py index 3fa14e2..68a52fe 100644 --- a/tests/unit/test_crs.py +++ b/tests/unit/test_crs.py @@ -12,12 +12,12 @@ from rasterio.crs import CRS from rioxarray import open_rasterio -from harmony_browse_image_generator.crs import ( +from hybig.crs import ( PREFERRED_CRS, choose_best_crs_from_metadata, choose_target_crs, ) -from harmony_browse_image_generator.exceptions import HyBIGInvalidMessageError +from hybig.exceptions import HyBIGInvalidMessageError from tests.unit.utility import rasterio_test_file ## Test constants @@ -128,7 +128,7 @@ def test_choose_target_crs_with_invalid_SRS_from_harmony_message(self): with self.assertRaisesRegex(HyBIGInvalidMessageError, 'Bad input SRS'): choose_target_crs(test_srs_is_json, None) - @patch('harmony_browse_image_generator.crs.choose_crs_from_metadata') + @patch('hybig.crs.choose_crs_from_metadata') def test_choose_target_harmony_message_has_crs_but_no_srs(self, mock_choose_fxn): """Explicitly show we do not support format.crs only. diff --git a/tests/unit/test_sizes.py b/tests/unit/test_sizes.py index 4899850..462bc42 100644 --- a/tests/unit/test_sizes.py +++ b/tests/unit/test_sizes.py @@ -10,9 +10,9 @@ from rasterio.crs import CRS from rioxarray import open_rasterio -from harmony_browse_image_generator.crs import PREFERRED_CRS -from harmony_browse_image_generator.exceptions import HyBIGValueError -from harmony_browse_image_generator.sizes import ( +from hybig.crs import PREFERRED_CRS +from hybig.exceptions import HyBIGValueError +from hybig.sizes import ( METERS_PER_DEGREE, ScaleExtent, best_guess_target_dimensions, @@ -299,8 +299,8 @@ def test_compute_tile_dimensions_nonuniform(self): self.assertEqual(expected_dimensions, actual_dimensions) - @patch('harmony_browse_image_generator.sizes.get_cells_per_tile') - @patch('harmony_browse_image_generator.sizes.needs_tiling') + @patch('hybig.sizes.get_cells_per_tile') + @patch('hybig.sizes.needs_tiling') def test_create_tile_output_parameters( self, needs_tiling_mock, cells_per_tile_mock ): @@ -475,7 +475,7 @@ def test_message_has_scale_sizes(self): actual_dimensions = choose_target_dimensions(message, None, scale_extent, None) self.assertDictEqual(expected_dimensions, actual_dimensions) - @patch('harmony_browse_image_generator.sizes.best_guess_target_dimensions') + @patch('hybig.sizes.best_guess_target_dimensions') def test_message_has_no_information(self, mock_best_guess_target_dimensions): """Test message with no information gets sent to best guess.""" message = Message({}) @@ -489,7 +489,7 @@ def test_message_has_no_information(self, mock_best_guess_target_dimensions): dataset, scale_extent, target_crs ) - @patch('harmony_browse_image_generator.sizes.best_guess_target_dimensions') + @patch('hybig.sizes.best_guess_target_dimensions') def test_message_has_just_one_dimension(self, mock_best_guess_target_dimensions): """Message with only one dimension. diff --git a/tests/unit/test_utilities.py b/tests/unit/test_utilities.py index 8548fc7..e439ed3 100644 --- a/tests/unit/test_utilities.py +++ b/tests/unit/test_utilities.py @@ -9,7 +9,7 @@ class TestUtilities(TestCase): - """A class testing the harmony_browse_image_generator.utilities module.""" + """A class testing the hybig.utilities module.""" def test_get_file_mime_type(self): """Ensure a MIME type can be retrieved from an input file path.""" From 55eb7f0ee12c54d029efe5ee39435a305da4580f Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 26 Jun 2024 11:02:15 -0600 Subject: [PATCH 07/39] DAS-2180: Adds empty init for mypy --- harmony_service_entry/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 harmony_service_entry/__init__.py diff --git a/harmony_service_entry/__init__.py b/harmony_service_entry/__init__.py new file mode 100644 index 0000000..e69de29 From daec460188ff6c9bb7e560b06c40bde1f45c48a6 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 26 Jun 2024 13:53:40 -0600 Subject: [PATCH 08/39] DAS-2180: Separate exceptions into hybig and hybig-service --- harmony_service_entry/__main__.py | 2 +- harmony_service_entry/adapter.py | 4 ++-- harmony_service_entry/exceptions.py | 18 ++++++++++++++++ hybig/__init__.py | 3 +-- hybig/crs.py | 8 +++---- hybig/exceptions.py | 33 ++++++----------------------- tests/unit/test_adapter.py | 2 +- tests/unit/test_crs.py | 4 ++-- 8 files changed, 35 insertions(+), 39 deletions(-) create mode 100644 harmony_service_entry/exceptions.py diff --git a/harmony_service_entry/__main__.py b/harmony_service_entry/__main__.py index ae852a7..6423c6f 100644 --- a/harmony_service_entry/__main__.py +++ b/harmony_service_entry/__main__.py @@ -6,7 +6,7 @@ from harmony import is_harmony_cli, run_cli, setup_cli from harmony_service_entry.adapter import BrowseImageGeneratorAdapter -from hybig import SERVICE_NAME +from harmony_service_entry.exception import SERVICE_NAME def main(arguments: list[str]): diff --git a/harmony_service_entry/adapter.py b/harmony_service_entry/adapter.py index 1665cf3..11ef592 100644 --- a/harmony_service_entry/adapter.py +++ b/harmony_service_entry/adapter.py @@ -23,6 +23,7 @@ from harmony.util import bbox_to_geometry, download, generate_output_filename, stage from pystac import Asset, Catalog, Item +from harmony_service_entry.exceptions import HyBIGInvalidMessageError, HyBIGServiceError from harmony_service_entry.utilities import ( get_asset_name, get_file_mime_type, @@ -32,7 +33,6 @@ create_browse_imagery, get_color_palette_from_item, ) -from hybig.exceptions import HyBIGInvalidMessageError class BrowseImageGeneratorAdapter(BaseHarmonyAdapter): @@ -142,7 +142,7 @@ def process_item(self, item: Item, source: HarmonySource) -> Item: except Exception as exception: self.logger.exception(exception) - raise exception + raise HyBIGServiceError from exception finally: rmtree(working_directory) diff --git a/harmony_service_entry/exceptions.py b/harmony_service_entry/exceptions.py new file mode 100644 index 0000000..51ccfa5 --- /dev/null +++ b/harmony_service_entry/exceptions.py @@ -0,0 +1,18 @@ +"""Module defining harmony service errors raised by HyBIG service.""" + +from harmony.util import HarmonyException + +SERVICE_NAME = 'harmony-browse-image-generator' + + +# TODO: rename as hybigserviceerror. +class HyBIGServiceError(HarmonyException): + """Base service exception.""" + + def __init__(self, message=None): + """All service errors are assocated with SERVICE_NAME.""" + super().__init__(message=message, category=SERVICE_NAME) + + +class HyBIGInvalidMessageError(HyBIGServiceError): + """Input Harmony Message could not be used as presented.""" diff --git a/hybig/__init__.py b/hybig/__init__.py index a88970b..851e5ce 100644 --- a/hybig/__init__.py +++ b/hybig/__init__.py @@ -2,6 +2,5 @@ from .browse import create_browse_imagery from .color_utility import get_color_palette_from_item -from .exceptions import SERVICE_NAME -__all__ = ['create_browse_imagery', 'SERVICE_NAME', 'get_color_palette_from_item'] +__all__ = ['create_browse_imagery', 'get_color_palette_from_item'] diff --git a/hybig/crs.py b/hybig/crs.py index 4f86fff..42ef015 100644 --- a/hybig/crs.py +++ b/hybig/crs.py @@ -17,7 +17,7 @@ from rasterio.crs import CRS from xarray import DataArray -from hybig.exceptions import HyBIGInvalidMessageError +from hybig.exceptions import HyBIGValueError # These are the CRSs that GIBS will accept as input. When the user hasn't # directly specified an output CRS, the code will attempt to choose the best @@ -49,7 +49,7 @@ def choose_crs_from_srs(srs: SRS): prefer epsg to wkt prefer wkt to proj4 - Raise an InvalidMessage error if the harmony SRS cannot be converted to a + Raise HyBIGValueError if the harmony SRS cannot be converted to a rasterio CRS for any reason. """ @@ -60,9 +60,7 @@ def choose_crs_from_srs(srs: SRS): return CRS.from_string(srs.wkt) return CRS.from_string(srs.proj4) except Exception as exception: - raise HyBIGInvalidMessageError( - f'Bad input SRS: {str(exception)}' - ) from exception + raise HyBIGValueError(f'Bad input SRS: {str(exception)}') from exception def is_preferred_crs(crs: CRS) -> bool: diff --git a/hybig/exceptions.py b/hybig/exceptions.py index 4e9170f..dd277ea 100644 --- a/hybig/exceptions.py +++ b/hybig/exceptions.py @@ -1,36 +1,17 @@ -""" Module defining custom exceptions, designed to return user-friendly error - messaging to the end-user. +"""Module defining custom exceptions.""" -""" -from harmony.util import HarmonyException - -SERVICE_NAME = 'harmony-browse-image-generator' - - -class HyBIGError(HarmonyException): - """Base service exception.""" +class HyBIGError(Exception): + """Base error class for exceptions rasied by HyBIG library.""" def __init__(self, message=None): - super().__init__(message, SERVICE_NAME) + """All HyBIG errors have a message field.""" + self.message = message -class HyBIGNoColorInformation(HarmonyException): +class HyBIGNoColorInformation(HyBIGError): """Used to describe missing color information.""" - def __init__(self, message=None): - super().__init__(message, SERVICE_NAME) - - -class HyBIGInvalidMessageError(HarmonyException): - """Input Harmony Message could not be used as presented.""" - def __init__(self, message=None): - super().__init__(message, SERVICE_NAME) - - -class HyBIGValueError(HarmonyException): +class HyBIGValueError(HyBIGError): """Input was incorrect for the routine.""" - - def __init__(self, message=None): - super().__init__(message, SERVICE_NAME) diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index 1ea8e46..512c428 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -7,7 +7,7 @@ from pystac import Asset, Item from harmony_service_entry.adapter import BrowseImageGeneratorAdapter -from hybig.exceptions import HyBIGInvalidMessageError +from harmony_service_entry.exceptions import HyBIGInvalidMessageError from tests.utilities import Granule, create_stac diff --git a/tests/unit/test_crs.py b/tests/unit/test_crs.py index 68a52fe..9eab2eb 100644 --- a/tests/unit/test_crs.py +++ b/tests/unit/test_crs.py @@ -17,7 +17,7 @@ choose_best_crs_from_metadata, choose_target_crs, ) -from hybig.exceptions import HyBIGInvalidMessageError +from hybig.exceptions import HyBIGValueError from tests.unit.utility import rasterio_test_file ## Test constants @@ -125,7 +125,7 @@ def test_choose_target_crs_with_proj4_from_harmony_message_and_empty_epsg(self): def test_choose_target_crs_with_invalid_SRS_from_harmony_message(self): """Test SRS does not have epsg, wkt or proj4 string.""" test_srs_is_json = {'how': 'did this happen?'} - with self.assertRaisesRegex(HyBIGInvalidMessageError, 'Bad input SRS'): + with self.assertRaisesRegex(HyBIGValueError, 'Bad input SRS'): choose_target_crs(test_srs_is_json, None) @patch('hybig.crs.choose_crs_from_metadata') From 7125cfdce10fbcef795d5247d3177e4185d0238f Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 26 Jun 2024 14:00:00 -0600 Subject: [PATCH 09/39] DAS-2180: Bump version on github actions This may upgrade to node 20 --- .github/workflows/run_tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 78a72f6..0afce2f 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout harmony-browse-image-generator repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: lfs: true @@ -30,13 +30,13 @@ jobs: run: ./bin/run-test - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: Test results path: test-reports/ - name: Archive coverage report - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: Coverage report path: coverage/* From 1758deefab5d14b10104a1ddd3e561c6cceac79e Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 26 Jun 2024 14:16:00 -0600 Subject: [PATCH 10/39] DAS-2180: includes harmony_service_entry in pylint. --- harmony_service_entry/__main__.py | 4 ++-- harmony_service_entry/exceptions.py | 1 - tests/run_tests.sh | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/harmony_service_entry/__main__.py b/harmony_service_entry/__main__.py index 6423c6f..5188729 100644 --- a/harmony_service_entry/__main__.py +++ b/harmony_service_entry/__main__.py @@ -5,8 +5,8 @@ from harmony import is_harmony_cli, run_cli, setup_cli -from harmony_service_entry.adapter import BrowseImageGeneratorAdapter -from harmony_service_entry.exception import SERVICE_NAME +from .adapter import BrowseImageGeneratorAdapter +from .exceptions import SERVICE_NAME def main(arguments: list[str]): diff --git a/harmony_service_entry/exceptions.py b/harmony_service_entry/exceptions.py index 51ccfa5..245f177 100644 --- a/harmony_service_entry/exceptions.py +++ b/harmony_service_entry/exceptions.py @@ -5,7 +5,6 @@ SERVICE_NAME = 'harmony-browse-image-generator' -# TODO: rename as hybigserviceerror. class HyBIGServiceError(HarmonyException): """Base service exception.""" diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 7d60861..eee9102 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -35,7 +35,7 @@ coverage html --omit="tests/*" -d tests/coverage # W1203 - use of f-strings in log statements. This warning is leftover from # using ''.format() vs % notation. For more information, see: # https://github.com/PyCQA/pylint/issues/2354#issuecomment-414526879 -pylint hybig --disable=W1203 +pylint hybig harmony_service_entry --disable=W1203 RESULT=$? RESULT=$((3 & $RESULT)) From ccec6a0184203b1823f610ab3acaa1ed75fd092f Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 26 Jun 2024 14:16:50 -0600 Subject: [PATCH 11/39] DAS-2180: Ruff Auto-fixes. Mostly reformating/capitalization and removing unused imports --- harmony_service_entry/adapter.py | 11 +++++------ harmony_service_entry/utilities.py | 1 - hybig/browse.py | 7 ++++--- hybig/color_utility.py | 4 +--- hybig/sizes.py | 12 +++--------- tests/test_adapter.py | 6 +++--- tests/unit/test_browse.py | 11 ++++------- tests/unit/test_sizes.py | 14 ++++++-------- tests/unit/test_utilities.py | 6 ++---- 9 files changed, 28 insertions(+), 44 deletions(-) diff --git a/harmony_service_entry/adapter.py b/harmony_service_entry/adapter.py index 11ef592..6e21764 100644 --- a/harmony_service_entry/adapter.py +++ b/harmony_service_entry/adapter.py @@ -1,9 +1,9 @@ -""" `HarmonyAdapter` for Harmony Browse Image Generator (HyBIG). +"""`HarmonyAdapter` for Harmony Browse Image Generator (HyBIG). - The class in this file is the top level of abstraction for a service that - will accept a GeoTIFF input and create a browse image (PNG/JPEG) and - accompanying ESRI world file. By default, this service will aim to create - Global Imagery Browse Services (GIBS) compatible browse imagery. +The class in this file is the top level of abstraction for a service that +will accept a GeoTIFF input and create a browse image (PNG/JPEG) and +accompanying ESRI world file. By default, this service will aim to create +Global Imagery Browse Services (GIBS) compatible browse imagery. """ @@ -153,7 +153,6 @@ def stage_output(self, transformed_file: Path, input_file: str) -> str: message. """ - ext = get_tiled_file_extension(transformed_file) output_file_name = generate_output_filename(input_file, ext=ext) diff --git a/harmony_service_entry/utilities.py b/harmony_service_entry/utilities.py index 892c5f3..d049f25 100644 --- a/harmony_service_entry/utilities.py +++ b/harmony_service_entry/utilities.py @@ -34,7 +34,6 @@ def get_asset_name(name: str, url: str) -> str: dictionary. """ - tiled_pattern = r"\.(r\d+c\d+)\." tile_id = re.search(tiled_pattern, url) if tile_id is not None: diff --git a/hybig/browse.py b/hybig/browse.py index 41e04ef..56cd3f7 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -240,7 +240,7 @@ def prepare_raster_for_writing( def palettize_raster(raster: ndarray) -> tuple[ndarray, dict]: - """convert an RGB or RGBA image into a 1band image and palette. + """Convert an RGB or RGBA image into a 1band image and palette. Converts a 3 or 4 band np raster into a PIL image. Quantizes the image into a 1band raster with palette @@ -271,7 +271,8 @@ def add_alpha( alpha: ndarray | None, quantized_array: ndarray, color_map: dict ) -> tuple[ndarray, dict]: """If the input data had alpha values, manually set the quantized_image - index to the transparent index in those places.""" + index to the transparent index in those places. + """ if alpha is not None and np.any(alpha != OPAQUE): # Set any alpha to the transparent index value quantized_array = np.where(alpha != OPAQUE, TRANSPARENT_IDX, quantized_array) @@ -294,7 +295,7 @@ def get_color_map_from_image(image: Image) -> dict: def get_aux_xml_filename(image_filename: Path) -> Path: - """get aux.xml filenames.""" + """Get aux.xml filenames.""" return image_filename.with_suffix(image_filename.suffix + '.aux.xml') diff --git a/hybig/color_utility.py b/hybig/color_utility.py index 2daef7a..9b81c29 100644 --- a/hybig/color_utility.py +++ b/hybig/color_utility.py @@ -5,8 +5,6 @@ """ -from typing import TYPE_CHECKING - import numpy as np import requests from harmony.message import Source as HarmonySource @@ -32,7 +30,7 @@ def remove_alpha(raster: np.ndarray) -> tuple[np.ndarray, np.ndarray, None]: - """remove alpha layer when it exists.""" + """Remove alpha layer when it exists.""" if raster.shape[0] == 4: return raster[0:3, :, :], raster[3, :, :] return raster, None diff --git a/hybig/sizes.py b/hybig/sizes.py index cf92672..6464f9a 100644 --- a/hybig/sizes.py +++ b/hybig/sizes.py @@ -15,8 +15,6 @@ from affine import Affine from harmony.message import Message from harmony.message_utility import has_dimensions, has_scale_extents, has_scale_sizes -from pyproj import Transformer -from pyproj.crs import CRS as pyCRS # pylint: disable-next=no-name-in-module from rasterio.crs import CRS @@ -24,12 +22,8 @@ from xarray import DataArray from hybig.crs import ( - PREFERRED_CRS, - choose_best_crs_from_metadata, choose_target_crs, - is_preferred_crs, ) -from hybig.exceptions import HyBIGValueError class GridParams(TypedDict): @@ -281,7 +275,7 @@ def create_tiled_output_parameters( def compute_tile_dimensions(origins: list[int]) -> list[int]: - """return a list of tile dimensions. + """Return a list of tile dimensions. From a list of origin locations, return the dimension for each tile. @@ -290,7 +284,7 @@ def compute_tile_dimensions(origins: list[int]) -> list[int]: def compute_tile_boundaries(target_size: int, full_size: int) -> list[int]: - """returns a list of boundary cells. + """Returns a list of boundary cells. The returned boundary cells are the column [or row] values for each of the output tiles. They should always start at 0, and end at the full_size @@ -308,7 +302,7 @@ def compute_tile_boundaries(target_size: int, full_size: int) -> list[int]: def get_cells_per_tile() -> int: - """optimum cells per tile. + """Optimum cells per tile. From discussions this is chosen to be 4096, so that any image that is tiled will end up with 4096x4096 gridcell tiles. diff --git a/tests/test_adapter.py b/tests/test_adapter.py index b62393d..01cdd00 100644 --- a/tests/test_adapter.py +++ b/tests/test_adapter.py @@ -1,4 +1,4 @@ -""" End-to-end tests of the Harmony Browse Image Generator (HyBIG). """ +"""End-to-end tests of the Harmony Browse Image Generator (HyBIG).""" from pathlib import Path from shutil import copy, rmtree @@ -137,7 +137,7 @@ def test_valid_request_jpeg( mock_mkdtemp.return_value = self.temp_dir def move_tif(*args, **kwargs): - """copy fixture tiff to download location.""" + """Copy fixture tiff to download location.""" copy(self.red_tif_fixture, expected_downloaded_file) return expected_downloaded_file @@ -372,7 +372,7 @@ def test_valid_request_png( mock_mkdtemp.return_value = self.temp_dir def move_tif(*args, **kwargs): - """copy fixture tiff to download location.""" + """Copy fixture tiff to download location.""" copy(self.red_tif_fixture, expected_downloaded_file) return expected_downloaded_file diff --git a/tests/unit/test_browse.py b/tests/unit/test_browse.py index f3359f8..9f014bf 100644 --- a/tests/unit/test_browse.py +++ b/tests/unit/test_browse.py @@ -1,4 +1,4 @@ -""" Unit tests for browse module. """ +"""Unit tests for browse module.""" import shutil import tempfile @@ -304,7 +304,6 @@ def test_create_browse_imagery_with_mocks( def test_convert_singleband_to_raster_without_colortable(self): """Tests convert_gray_1band_to_raster.""" - return_data = np.copy(self.data).astype('float64') return_data[0][1] = np.nan ds = DataArray(return_data).expand_dims('band') @@ -561,7 +560,6 @@ def test_prepare_raster_for_writing_png_4band(self, palettize_mock): @patch('hybig.browse.get_color_map_from_image') def test_palettize_raster_no_alpha_layer(self, get_color_map_mock, image_mock): """Test that the quantize function is called by a correct image.""" - raster = self.random.integers(255, dtype='uint8', size=(3, 10, 11)) quantized_output = Image.fromarray( @@ -584,7 +582,6 @@ def test_palettize_raster_no_alpha_layer(self, get_color_map_mock, image_mock): @patch('hybig.browse.get_color_map_from_image') def test_palettize_raster_with_alpha_layer(self, get_color_map_mock, image_mock): """Test that the quantize function is called by a correct image.""" - raster = self.random.integers(255, dtype='uint8', size=(4, 10, 11)) # No transparent pixels raster[3, :, :] = 255 @@ -714,7 +711,7 @@ def test_get_tiled_filename(self): self.assertEqual(expected_filename, actual_filename) def test_validate_file_crs_valid(self): - """valid file should return None.""" + """Valid file should return None.""" da = Mock(DataArray) da.rio.crs = CRS.from_epsg(4326) try: @@ -723,14 +720,14 @@ def test_validate_file_crs_valid(self): self.fail('Valid file threw unexpected exception.') def test_validate_file_crs_missing(self): - """invalid file should raise exception.""" + """Invalid file should raise exception.""" da = Mock(DataArray) da.rio.crs = None with self.assertRaisesRegex(HyBIGError, 'Input geotiff must have defined CRS.'): validate_file_crs(da) def test_validate_file_type_valid(self): - """validation should not raise exception.""" + """Validation should not raise exception.""" ds = Mock(DatasetReader) ds.driver = 'GTiff' try: diff --git a/tests/unit/test_sizes.py b/tests/unit/test_sizes.py index 462bc42..dcfca1b 100644 --- a/tests/unit/test_sizes.py +++ b/tests/unit/test_sizes.py @@ -11,7 +11,6 @@ from rioxarray import open_rasterio from hybig.crs import PREFERRED_CRS -from hybig.exceptions import HyBIGValueError from hybig.sizes import ( METERS_PER_DEGREE, ScaleExtent, @@ -120,7 +119,7 @@ def test_grid_parameters_from_harmony_message_has_complete_information(self): self.assertDictEqual(expected_parameters, actual_parameters) def test_grid_parameters_from_harmony_no_message_information(self): - """input granule is in preferred_crs on a 25km grid""" + """Input granule is in preferred_crs on a 25km grid""" crs = CRS.from_epsg(sp_seaice_grid['epsg']) height = sp_seaice_grid['height'] width = sp_seaice_grid['width'] @@ -255,14 +254,14 @@ def test_needs_tiling(self): self.assertFalse(needs_tiling(grid_parameters)) def test_get_cells_per_tile(self): - """test how tiles sizes are generated.""" + """Test how tiles sizes are generated.""" expected_cells_per_tile = self.CELLS_PER_TILE actual_cells_per_tile = get_cells_per_tile() self.assertEqual(expected_cells_per_tile, actual_cells_per_tile) self.assertIsInstance(actual_cells_per_tile, int) def test_compute_tile_boundaries_exact(self): - """tests subdivision of output image.""" + """Tests subdivision of output image.""" cells_per_tile = 10 full_width = 10 * 4 expected_origins = [0.0, 10.0, 20.0, 30.0, 40.0] @@ -272,7 +271,7 @@ def test_compute_tile_boundaries_exact(self): self.assertEqual(expected_origins, actual_origins) def test_compute_tile_boundaries_with_leftovers(self): - """tests subdivision of output image.""" + """Tests subdivision of output image.""" cells_per_tile = 10 full_width = 10 * 4 + 3 expected_origins = [0.0, 10.0, 20.0, 30.0, 40.0, 43.0] @@ -282,7 +281,7 @@ def test_compute_tile_boundaries_with_leftovers(self): self.assertEqual(expected_origins, actual_origins) def test_compute_tile_dimensions_uniform(self): - """test tile dimensions.""" + """Test tile dimensions.""" tile_origins = [0.0, 10.0, 20.0, 30.0, 40.0, 43.0] expected_dimensions = [10.0, 10.0, 10.0, 10.0, 3.0, 0.0] @@ -291,7 +290,7 @@ def test_compute_tile_dimensions_uniform(self): self.assertEqual(expected_dimensions, actual_dimensions) def test_compute_tile_dimensions_nonuniform(self): - """test tile dimensions.""" + """Test tile dimensions.""" tile_origins = [0.0, 20.0, 35.0, 40.0, 43.0] expected_dimensions = [20.0, 15.0, 5.0, 3.0, 0.0] @@ -425,7 +424,6 @@ def test_scale_extent_in_harmony_message(self): def test_scale_extent_from_input_image_and_no_crs_transformation(self): """Ensure no change of output extent when src_crs == target_crs""" - with open_rasterio( self.fixtures / 'RGB.byte.small.tif', mode='r', mask_and_scale=True ) as in_array: diff --git a/tests/unit/test_utilities.py b/tests/unit/test_utilities.py index e439ed3..fb25e2c 100644 --- a/tests/unit/test_utilities.py +++ b/tests/unit/test_utilities.py @@ -26,8 +26,7 @@ def test_get_file_mime_type(self): self.assertIsNone(get_file_mime_type('file.xyzzyx')) def test_get_tiled_file_extension(self): - """ensure correct extensions are extracted""" - + """Ensure correct extensions are extracted""" test_params = [ (Path('/tmp/tmp4w/14316c44a.r00c02.png.aux.xml'), '.r00c02.png.aux.xml'), (Path('/tmp/tmp4w/14316c44a.png.aux.xml'), '.png.aux.xml'), @@ -45,8 +44,7 @@ def test_get_tiled_file_extension(self): self.assertEqual(expected_extension, actual_extension) def test_get_asset_name(self): - """ensure correct asset names are generated""" - + """Ensure correct asset names are generated""" test_params = [ ( ('name', 'https://tmp_bucket/tmp4w/14316c44a.r00c02.png.aux.xml'), From 87b5d52c1e62dbfefe426b65926e6bd5612cf1f0 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 26 Jun 2024 14:44:34 -0600 Subject: [PATCH 12/39] DAS-2180: Ensure all py files adhere to PEP8 --- tests/test_code_format.py | 18 ++++++++++-------- tests/unit/test_crs.py | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/test_code_format.py b/tests/test_code_format.py index f015a57..fcc11b6 100644 --- a/tests/test_code_format.py +++ b/tests/test_code_format.py @@ -1,3 +1,6 @@ +"""Ensure code formatting.""" + +from itertools import chain from pathlib import Path from unittest import TestCase @@ -19,15 +22,14 @@ class TestCodeFormat(TestCase): from PEP8 for these errors. """ - @classmethod - def setUpClass(cls): - cls.python_files = Path('hybig').rglob('*.py') - def test_pycodestyle_adherence(self): - """Ensure all code in the `hybig` directory - adheres to PEP8 defined standard. + """Check files for PEP8 compliance.""" + python_files = chain( + Path('hybig').rglob('*.py'), + Path('harmony_service_entry').rglob('*.py'), + Path('tests').rglob('*.py'), + ) - """ style_guide = StyleGuide(ignore=['E501', 'W503', 'E203', 'E701']) - results = style_guide.check_files(self.python_files) + results = style_guide.check_files(python_files) self.assertEqual(results.total_errors, 0, 'Found code style issues.') diff --git a/tests/unit/test_crs.py b/tests/unit/test_crs.py index 9eab2eb..baf4dc8 100644 --- a/tests/unit/test_crs.py +++ b/tests/unit/test_crs.py @@ -20,7 +20,7 @@ from hybig.exceptions import HyBIGValueError from tests.unit.utility import rasterio_test_file -## Test constants +# Test constants WKT_EPSG_3031 = ( 'PROJCS["WGS 84 / Antarctic Polar Stereographic",' 'GEOGCS["WGS 84",DATUM["WGS_1984",' From a9169e021967cde2299c7e58b893ebe6b25141e0 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 26 Jun 2024 16:06:03 -0600 Subject: [PATCH 13/39] DAS-2180: This is definitely a v2 of this library. --- docker/service_version.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/service_version.txt b/docker/service_version.txt index f0bb29e..227cea2 100644 --- a/docker/service_version.txt +++ b/docker/service_version.txt @@ -1 +1 @@ -1.3.0 +2.0.0 diff --git a/pyproject.toml b/pyproject.toml index 044feaa..100f12e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ files = [ ] [tool.hatch.version] path = "docker/service_version.txt" -pattern= '^v?(?P.*)$' # +pattern= '^v?(?P.*)$' [tool.hatch.build.targets.sdist] From 40385f6ba877828f9979c0cdf1da0b3bf0b06757 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 26 Jun 2024 17:04:15 -0600 Subject: [PATCH 14/39] DAS-2180: update README.md for service and lib. --- README.md | 84 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 26499cd..465e9b4 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@ # Harmony Browse Image Generator (HyBIG). -This Harmony backend service is designed to produce browse imagery, with -default behaviour to produce browse imagery that is compatible with the NASA -Global Image Browse Services ([GIBS](https://www.earthdata.nasa.gov/eosdis/science-system-description/eosdis-components/gibs)). +This library is designed to produce browse imagery, with default behaviours to +produce browse imagery that is compatible with the NASA Global Image Browse +Services +([GIBS](https://www.earthdata.nasa.gov/eosdis/science-system-description/eosdis-components/gibs)). This means that defaults for images are selected to match the visualization generation requirements and recommendations put forth in the GIBS Interface @@ -143,22 +144,25 @@ also with units of degrees. |- legacy-CHANGELOG.md |- pip_requirements.txt |- pip_requirements_skip_snyk.txt +|- pyproject.toml + ``` * `bin` - A directory containing utility scripts to build the service and test - images. A script to extract the release notes for the most recent service - version, as contained in `CHANGELOG.md` is also in this directory. + images. A script to extract the release notes for the most recent version, as + contained in `CHANGELOG.md` is also in this directory. * `docker` - A directory containing the Dockerfiles for the service and test images. It also contains `service_version.txt`, which contains the semantic - version number of the service image. Any time an update is made that should - have an accompanying service image release, this file should be updated. + version number of the library and service image. Any time an update is made + that should have an accompanying library and service image release, this file + should be updated. * `docs` - A directory with example usage notebooks. -* `hybig` - A directory containing Python source code - for the HyBIG library. This directory contains the business logic for - generating GIBS compatible browse images. +* `hybig` - A directory containing Python source code for the HyBIG library. + This directory contains the business logic for generating GIBS compatible + browse images. * `harmony_service_entry` - A directory containing the Harmony Service specific python code. `adapter.py` contains the `BrowseImageGeneratorAdapter` class @@ -167,8 +171,8 @@ also with units of degrees. * `tests` - A directory containing the service unit test suite. * `CHANGELOG.md` - This file contains a record of changes applied to each new - release of a service Docker image. Any release of a new service version - should have a record of what was changed in this file. + release of HyBIG. Any release of a new service version should have a record + of what was changed in this file. * `CONTRIBUTING.md` - This file contains guidance for making contributions to HyBIG, including recommended git best practices. @@ -176,9 +180,11 @@ also with units of degrees. * `LICENSE` - Required for distribution under NASA open-source approval. Details conditions for use, reproduction and distribution. -* `README.md` - This file, containing guidance on developing the service. +* `README.md` - This file, containing guidance on developing the library and + service. -* `dev-requirements.txt` - list of packages required for service development. +* `dev-requirements.txt` - list of packages required for library and service + development. * `legacy-CHANGELOG.md` - Notes for each version that was previously released internally to EOSDIS, prior to open-source publication of the code and Docker @@ -192,17 +198,22 @@ also with units of degrees. naive and cannot pre-install required libraries so that `pip install GDAL` fails and we have no work around. +* `pyproject.toml` - Configuration file used by packaging tools, as well as + other tools such as linters, type checkers, etc. + ## Local development: -Local testing of service functionality is best achieved via a local instance of +Local testing of service functionality is achieved via a local instance of [Harmony](https://github.com/nasa/harmony). Please see instructions there regarding creation of a local Harmony instance. -If testing small functions locally that do not require inputs from the main -Harmony application, it is recommended that you create a Python virtual -environment, and then install the necessary dependencies for the -service within that environment via conda and pip then install the pre-commit hooks. +If developing changes to the library, or testing small functions locally that +do not require inputs from the main Harmony application, it is recommended that +you create a Python virtual environment, and then install the necessary +dependencies for the service within that environment via conda and pip then +install the pre-commit hooks. Note that you will need the gdal libraries +available to your virtual environment to install the `gdal` package with pip. ``` > conda create --name hybig-env python==3.11 @@ -232,22 +243,21 @@ Currently, the `unittest` suite is run automatically within a GitHub workflow as part of a CI/CD pipeline. These tests are run for all changes made in a PR against the `main` branch. The tests must pass in order to merge the PR. -The unit tests are also run prior to publication of a new Docker image, when -commits including changes to `docker/service_version.txt` are merged into the -`main` branch. If these unit tests fail, the new version of the Docker image -will not be published. +The unit tests are also run prior to publication of new library packages and +Docker image, when commits including changes to `docker/service_version.txt` +are merged into the `main` branch. If these unit tests fail, the new version of +the Docker image and library package will not be published. ## Versioning: -Service Docker images for HyBIG adhere to semantic version numbers: -major.minor.patch. +Service Docker images and the package library for HyBIG adhere to semantic +version numbers: major.minor.patch. * Major increments: These are non-backwards compatible API changes. * Minor increments: These are backwards compatible API changes. * Patch increments: These updates do not affect the API to the service. -When publishing a new Docker image for the service, two files need to be -updated: +When publishing, two files need to be updated: * `CHANGELOG.md` - Notes should be added to capture the changes to the service. * `docker/service_version.txt` - The semantic version number should be updated. @@ -266,14 +276,20 @@ The CI/CD for HyBIG is contained in GitHub workflows in the * `publish_docker_image.yml` - Triggered either manually or for commits to the `main` branch that contain changes to the `docker/service_version.txt` file. -The `publish_docker_image.yml` workflow will: +* `publish_to_pypi.yml` - Triggered either manually or for commits to the + `main` branch that contain changes to the `docker/service_version.txt`file. + +The `publish_release.yml` workflow will: * Run the full unit test suite, to prevent publication of broken code. * Extract the semantic version number from `docker/service_version.txt`. * Extract the released notes for the most recent version from `CHANGELOG.md`. -* Create a GitHub release that will also tag the related git commit with the - semantic version number. * Build and deploy a this service's docker image to `ghcr.io`. +* Build the library package to be published to PyPI. +* Publish the package to PyPI. +* Publish a GitHub release under the semantic version number, with associated + git tag. + Before triggering a release, ensure both the `docker/service_version.txt` and `CHANGELOG.md` files are updated. The `CHANGELOG.md` file requires a specific @@ -281,7 +297,13 @@ format for a new release, as it looks for the following string to define the newest release of the code (starting at the top of the file). ``` -## vX.Y.Z - YYYY-MM-DD +## [vX.Y.Z] - YYYY-MM-DD +``` + +Where the markdown reference needs to be updated at the bottom of the file following the existing pattern. +``` +[unreleased]:https://github.com/nasa/harmony-browse-image-generator/compare/X.Y.Z..HEAD +[vX.Y.Z]:https://github.com/nasa/harmony-browse-image-generator/compare/X.Y.Y..X.Y.Z ``` ### pre-commit hooks: From 1c182f7c40042d0f46d14734b426466bd44d3e00 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 26 Jun 2024 18:00:20 -0600 Subject: [PATCH 15/39] DAS-2180: Stab at updating workflow to publish to testpypi. I already had done one by hand so also updates version. We can go back to v2 on PyPI --- ...h_docker_image.yml => publish_release.yml} | 26 ++++++++++++++----- README.md | 9 +++---- docker/service_version.txt | 2 +- hybig/__init__.py | 4 +-- hybig/browse.py | 11 ++++++++ 5 files changed, 38 insertions(+), 14 deletions(-) rename .github/workflows/{publish_docker_image.yml => publish_release.yml} (77%) diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_release.yml similarity index 77% rename from .github/workflows/publish_docker_image.yml rename to .github/workflows/publish_release.yml index 772f259..dd0428c 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_release.yml @@ -3,10 +3,11 @@ # can also be manually triggered by a repository maintainer. This workflow will # first trigger the reusable workflow in `.github/workflows/run_tests.yml`, # which runs the `unittest` suite. If that workflow is successful, the latest -# version of the service Docker image is pushed to ghcr.io, a tag is added to -# the latest git commit, and a GitHub release is created with the release notes -# from the latest version of HyBIG. -name: Publish Harmony Browse Image Generator (HyBIG) Docker image +# version of the service Docker image is pushed to ghcr.io, a library package +# is built and published to PyPI, a tag is added to the latest git commit, and +# a GitHub release is created with the release notes from the latest version of +# HyBIG. +name: Publish Harmony Browse Image Generator (HyBIG) on: push: @@ -22,7 +23,7 @@ jobs: run_tests: uses: ./.github/workflows/run_tests.yml - build_and_publish_image: + build_and_publish: needs: run_tests runs-on: ubuntu-latest environment: release @@ -36,7 +37,7 @@ jobs: steps: - name: Checkout harmony-browse-image-generator repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: lfs: true @@ -74,6 +75,19 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Build hybig-py package + run: python -m build + + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + - name: Publish GitHub release uses: ncipollo/release-action@v1 with: diff --git a/README.md b/README.md index 465e9b4..bc534b3 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,7 @@ This library is designed to produce browse imagery, with default behaviours to produce browse imagery that is compatible with the NASA Global Image Browse -Services -([GIBS](https://www.earthdata.nasa.gov/eosdis/science-system-description/eosdis-components/gibs)). +Services ([GIBS](https://www.earthdata.nasa.gov/eosdis/science-system-description/eosdis-components/gibs)). This means that defaults for images are selected to match the visualization generation requirements and recommendations put forth in the GIBS Interface @@ -171,7 +170,7 @@ also with units of degrees. * `tests` - A directory containing the service unit test suite. * `CHANGELOG.md` - This file contains a record of changes applied to each new - release of HyBIG. Any release of a new service version should have a record + release of HyBIG. Any release of a new version should have a record of what was changed in this file. * `CONTRIBUTING.md` - This file contains guidance for making contributions to @@ -204,7 +203,7 @@ also with units of degrees. ## Local development: -Local testing of service functionality is achieved via a local instance of +Local testing of service functionality can be achieved via a local instance of [Harmony](https://github.com/nasa/harmony). Please see instructions there regarding creation of a local Harmony instance. @@ -257,7 +256,7 @@ version numbers: major.minor.patch. * Minor increments: These are backwards compatible API changes. * Patch increments: These updates do not affect the API to the service. -When publishing, two files need to be updated: +When publishing a new release, two files must be updated: * `CHANGELOG.md` - Notes should be added to capture the changes to the service. * `docker/service_version.txt` - The semantic version number should be updated. diff --git a/docker/service_version.txt b/docker/service_version.txt index 227cea2..38f77a6 100644 --- a/docker/service_version.txt +++ b/docker/service_version.txt @@ -1 +1 @@ -2.0.0 +2.0.1 diff --git a/hybig/__init__.py b/hybig/__init__.py index 851e5ce..645fdbd 100644 --- a/hybig/__init__.py +++ b/hybig/__init__.py @@ -1,6 +1,6 @@ """Package containing core functionality for browse image generation.""" -from .browse import create_browse_imagery +from .browse import create_browse, create_browse_imagery from .color_utility import get_color_palette_from_item -__all__ = ['create_browse_imagery', 'get_color_palette_from_item'] +__all__ = ['create_browse', 'create_browse_imagery', 'get_color_palette_from_item'] diff --git a/hybig/browse.py b/hybig/browse.py index 56cd3f7..f4e8dfb 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -41,6 +41,17 @@ ) +def create_browse(): + """Create browse image from geotiff. + + Exposed library function to allow users to create browse images from the + hybig library. This function parses the input params and builds out the + correct Harmony input structures [Message and Source] to call the service's + entry point create_browse_imagery. + """ + pass + + def create_browse_imagery( message: HarmonyMessage, input_file_path: str, From 6cf3d6bae63eda3d520e470d264f20a1e429847b Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 3 Jul 2024 13:44:26 -0600 Subject: [PATCH 16/39] DAS-2180: renames harmony_service_entry -> harmony_service This makes more sense since entry is tied to Docker. --- README.md | 6 +++--- docker/service.Dockerfile | 4 ++-- .../__init__.py | 0 .../__main__.py | 0 .../adapter.py | 4 ++-- .../exceptions.py | 0 .../utilities.py | 0 pyproject.toml | 6 ++++++ tests/run_tests.sh | 2 +- tests/test_adapter.py | 20 +++++++++---------- tests/test_code_format.py | 2 +- tests/unit/test_adapter.py | 6 +++--- tests/unit/test_utilities.py | 2 +- 13 files changed, 29 insertions(+), 23 deletions(-) rename {harmony_service_entry => harmony_service}/__init__.py (100%) rename {harmony_service_entry => harmony_service}/__main__.py (100%) rename {harmony_service_entry => harmony_service}/adapter.py (98%) rename {harmony_service_entry => harmony_service}/exceptions.py (100%) rename {harmony_service_entry => harmony_service}/utilities.py (100%) diff --git a/README.md b/README.md index bc534b3..c841749 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ also with units of degrees. |- 📂 docker |- 📂 docs |- 📂 hybig -|- 📂 harmony_service_entry +|- 📂 harmony_service |- 📂 tests |- CHANGELOG.md |- CONTRIBUTING.md @@ -163,9 +163,9 @@ also with units of degrees. This directory contains the business logic for generating GIBS compatible browse images. -* `harmony_service_entry` - A directory containing the Harmony Service specific +* `harmony_service` - A directory containing the Harmony Service specific python code. `adapter.py` contains the `BrowseImageGeneratorAdapter` class - that is invoked by calls to the service. + that is invoked by calls to the Harmony service. * `tests` - A directory containing the service unit test suite. diff --git a/docker/service.Dockerfile b/docker/service.Dockerfile index b6ddf38..695b920 100644 --- a/docker/service.Dockerfile +++ b/docker/service.Dockerfile @@ -29,10 +29,10 @@ RUN pip install --no-input --no-cache-dir \ # Copy service code. COPY ./hybig hybig -COPY ./harmony_service_entry harmony_service_entry +COPY ./harmony_service harmony_service # Set GDAL related environment variables. ENV CPL_ZIP_ENCODING=UTF-8 # Configure a container to be executable via the `docker run` command. -ENTRYPOINT ["python", "-m", "harmony_service_entry"] +ENTRYPOINT ["python", "-m", "harmony_service"] diff --git a/harmony_service_entry/__init__.py b/harmony_service/__init__.py similarity index 100% rename from harmony_service_entry/__init__.py rename to harmony_service/__init__.py diff --git a/harmony_service_entry/__main__.py b/harmony_service/__main__.py similarity index 100% rename from harmony_service_entry/__main__.py rename to harmony_service/__main__.py diff --git a/harmony_service_entry/adapter.py b/harmony_service/adapter.py similarity index 98% rename from harmony_service_entry/adapter.py rename to harmony_service/adapter.py index 6e21764..7d2ceb5 100644 --- a/harmony_service_entry/adapter.py +++ b/harmony_service/adapter.py @@ -23,8 +23,8 @@ from harmony.util import bbox_to_geometry, download, generate_output_filename, stage from pystac import Asset, Catalog, Item -from harmony_service_entry.exceptions import HyBIGInvalidMessageError, HyBIGServiceError -from harmony_service_entry.utilities import ( +from harmony_service.exceptions import HyBIGInvalidMessageError, HyBIGServiceError +from harmony_service.utilities import ( get_asset_name, get_file_mime_type, get_tiled_file_extension, diff --git a/harmony_service_entry/exceptions.py b/harmony_service/exceptions.py similarity index 100% rename from harmony_service_entry/exceptions.py rename to harmony_service/exceptions.py diff --git a/harmony_service_entry/utilities.py b/harmony_service/utilities.py similarity index 100% rename from harmony_service_entry/utilities.py rename to harmony_service/utilities.py diff --git a/pyproject.toml b/pyproject.toml index 100f12e..f22682a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,9 +3,15 @@ name = "hybig-py" dynamic = ["dependencies", "version"] authors = [ + {name="Matt Savoie", email="savoie@colorado.edu"}, {name="Owen Littlejohns", email="owen.m.littlejohns@nasa.gov"}, +] + +maintainers = [ {name="Matt Savoie", email="savoie@colorado.edu"}, + {name="Owen Littlejohns", email="owen.m.littlejohns@nasa.gov"}, ] + description = "Python package designed to produce browse imagery compatible with NASA's Global Image Browse Services (GIBS)." readme = "README.md" diff --git a/tests/run_tests.sh b/tests/run_tests.sh index eee9102..db2b1da 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -35,7 +35,7 @@ coverage html --omit="tests/*" -d tests/coverage # W1203 - use of f-strings in log statements. This warning is leftover from # using ''.format() vs % notation. For more information, see: # https://github.com/PyCQA/pylint/issues/2354#issuecomment-414526879 -pylint hybig harmony_service_entry --disable=W1203 +pylint hybig harmony_service --disable=W1203 RESULT=$? RESULT=$((3 & $RESULT)) diff --git a/tests/test_adapter.py b/tests/test_adapter.py index 01cdd00..c8f9b78 100644 --- a/tests/test_adapter.py +++ b/tests/test_adapter.py @@ -14,7 +14,7 @@ from rasterio.warp import Resampling from rioxarray import open_rasterio -from harmony_service_entry.adapter import BrowseImageGeneratorAdapter +from harmony_service.adapter import BrowseImageGeneratorAdapter from hybig.browse import ( convert_mulitband_to_raster, prepare_raster_for_writing, @@ -23,7 +23,7 @@ class TestAdapter(TestCase): - """A class testing the harmony_service_entry.adapter module.""" + """A class testing the harmony_service.adapter module.""" @classmethod def setUpClass(cls): @@ -99,10 +99,10 @@ def assert_expected_output_catalog( ) @patch('hybig.browse.reproject') - @patch('harmony_service_entry.adapter.rmtree') - @patch('harmony_service_entry.adapter.mkdtemp') - @patch('harmony_service_entry.adapter.download') - @patch('harmony_service_entry.adapter.stage') + @patch('harmony_service.adapter.rmtree') + @patch('harmony_service.adapter.mkdtemp') + @patch('harmony_service.adapter.download') + @patch('harmony_service.adapter.stage') def test_valid_request_jpeg( self, mock_stage, mock_download, mock_mkdtemp, mock_rmtree, mock_reproject ): @@ -334,10 +334,10 @@ def move_tif(*args, **kwargs): mock_rmtree.assert_called_once_with(self.temp_dir) @patch('hybig.browse.reproject') - @patch('harmony_service_entry.adapter.rmtree') - @patch('harmony_service_entry.adapter.mkdtemp') - @patch('harmony_service_entry.adapter.download') - @patch('harmony_service_entry.adapter.stage') + @patch('harmony_service.adapter.rmtree') + @patch('harmony_service.adapter.mkdtemp') + @patch('harmony_service.adapter.download') + @patch('harmony_service.adapter.stage') def test_valid_request_png( self, mock_stage, mock_download, mock_mkdtemp, mock_rmtree, mock_reproject ): diff --git a/tests/test_code_format.py b/tests/test_code_format.py index fcc11b6..0c2a3c8 100644 --- a/tests/test_code_format.py +++ b/tests/test_code_format.py @@ -26,7 +26,7 @@ def test_pycodestyle_adherence(self): """Check files for PEP8 compliance.""" python_files = chain( Path('hybig').rglob('*.py'), - Path('harmony_service_entry').rglob('*.py'), + Path('harmony_service').rglob('*.py'), Path('tests').rglob('*.py'), ) diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index 512c428..39e358b 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -6,13 +6,13 @@ from harmony.util import config from pystac import Asset, Item -from harmony_service_entry.adapter import BrowseImageGeneratorAdapter -from harmony_service_entry.exceptions import HyBIGInvalidMessageError +from harmony_service.adapter import BrowseImageGeneratorAdapter +from harmony_service.exceptions import HyBIGInvalidMessageError from tests.utilities import Granule, create_stac class TestAdapter(TestCase): - """A class testing the harmony_service_entry.adapter module.""" + """A class testing the harmony_service.adapter module.""" @classmethod def setUpClass(cls): diff --git a/tests/unit/test_utilities.py b/tests/unit/test_utilities.py index fb25e2c..8ea96c1 100644 --- a/tests/unit/test_utilities.py +++ b/tests/unit/test_utilities.py @@ -1,7 +1,7 @@ from pathlib import Path from unittest import TestCase -from harmony_service_entry.utilities import ( +from harmony_service.utilities import ( get_asset_name, get_file_mime_type, get_tiled_file_extension, From 2ecae0e9eccf9c3466858fa77576da2e5abf5a01 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 3 Jul 2024 16:12:19 -0600 Subject: [PATCH 17/39] DAS-2180: Adds create_browse library function Also adds first stab at documenation but I will need to re-review. --- README.md | 106 ++++++++++++++++++++++++++++++++++++++-- hybig/browse.py | 127 ++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 219 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index c841749..d7eb8b3 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # Harmony Browse Image Generator (HyBIG). -This library is designed to produce browse imagery, with default behaviours to -produce browse imagery that is compatible with the NASA Global Image Browse +This repository contains code designed to produce browse imagery. Its default behaviour +produces images compatible with the NASA Global Image Browse Services ([GIBS](https://www.earthdata.nasa.gov/eosdis/science-system-description/eosdis-components/gibs)). -This means that defaults for images are selected to match the visualization -generation requirements and recommendations put forth in the GIBS Interface -Control Document (ICD), which can be found on [Earthdata +This means that default parameters for images are selected to match the +visualization generation requirements and recommendations put forth in the GIBS +Interface Control Document (ICD), which can be found on [Earthdata Wiki](https://wiki.earthdata.nasa.gov/display/GITC/Ingest+Delivery+Methods) along with [additional GIBS documentation](https://nasa-gibs.github.io/gibs-api-docs/). @@ -16,6 +16,102 @@ images. Scientific parameter raster data as well as RGB[A] raster images can be converted to browse PNGs. These browse images undergo transformation by reprojection, tiling and coloring to seamlessly integrate with GIBS. +The repository contains code and infrastructure to support both the HyBIG +Service as well as `hybig-py`. The HyBIG Service is packaged as a Docker +container that is deployed to [NASA's +Harmony](https://harmony.earthdata.nasa.gov/) system. The business logic is +contained in the [`hybig-py` library](https://pypi.org/project/hybig-py/) which +exposes functions to generate browse images in python scripts. + +### hybig-py + +The browse image generation logic is packaged in the hybig-py +library. Currently, a single function, `create_browse` is exposed to the user. + +``` python +def create_browse( + source_tiff: str, + params: dict = {}, + palette: str | ColorPalette | None = None, + logger: Logger = None, +) -> list[tuple[Path, Path, Path]]: + """Create browse imagery from an input geotiff. + + This is the exposed library function to allow users to create browse images + from the hybig library. It parses the input params and constructs the + correct Harmony input structure [Message.Format] to call the service's + entry point create_browse_imagery. + + Output images are created and deposited into the input GeoTIFF's directory. + + Args: + source_tiff: str, location of the input geotiff to process. + + params: Optional[dict], A dictionary with the following keys: + + mime: Optional[str], MIME type of the output image (default: 'image/png'). + any string that contains 'jpeg' will return a jpeg image, + otherwise create a png. + + crs: Optional[dict], Target image's Coordinate Reference System. + A dictionary with 'epsg', 'proj4' or 'wkt' key. + + scale_extent: Optional[dict], Scale Extents for the image. The dictionary + contains "x" and "y" keys each with value which is dictionary + of "min", "max" values in the same units as the crs. + e.g.: { "x": { "min": 0.5, "max": 125 }, + "y": { "min": 52, "max": 75.22 } } + + scale_size: Optional[dict], Scale sizes for the image. The dictionary + contains "x" and "y" keys with the horizontal and veritcal + resolution in the same units as the crs. + e.g.: { "x": 10, "y": 5 } + + height: Optional[int], height of the output image in gridcells. + + width: Optional[int], width of the output image in gridcells. + + palette: Optional[str | ColorPalette], either a URL to a remote color palette + that is fetched and loaded or a ColorPalette object used to color + the output browse image. If not provided, a grayscale image is + generated. + + logger: Optional[Logger], a configured Logger object. If None a default + logger will be used. + + Note: + if supplied, scale_size, scale_extent, height and width must be + internally consistent. To define a valid output grid: + * Specify scale_extent and 1 of: + * height and width + * scale_sizes (in the x and y horizontal spatial dimensions) + * Specify all three of the above, but ensure values must be consistent + with one another. + + Returns: + List of 3-element tuples. These are the file paths of: + - The output browse image + - Its associated ESRI world file (containing georeferencing information) + - The auxiliary XML file (containing duplicative georeferencing information) + + + Example Usage: + results = create_browse( + "/path/to/geotiff", + { + "mime": "image/png", + "crs": {"epsg": "EPSG:4326"}, + "scale_extent": { + "x": {"min": -180, "max": 180}, + "y": {"min": -90, "max": 90}, + }, + "scale_size": {"x": 10, "y": 10}, + }, + "https://remote-colortable", + logger, + ) +``` + ### Reprojection GIBS expects to receive images in one of three Coordinate Reference System (CRS) projections. diff --git a/hybig/browse.py b/hybig/browse.py index f4e8dfb..602c654 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -2,7 +2,7 @@ import re from itertools import zip_longest -from logging import Logger +from logging import Logger, getLogger from pathlib import Path import matplotlib @@ -31,6 +31,7 @@ TRANSPARENT_RGBA, all_black_color_map, get_color_palette, + palette_from_remote_colortable, remove_alpha, ) from hybig.exceptions import HyBIGError @@ -41,15 +42,122 @@ ) -def create_browse(): - """Create browse image from geotiff. +def create_browse( + source_tiff: str, + params: dict = {}, + palette: str | ColorPalette | None = None, + logger: Logger = None, +) -> list[tuple[Path, Path, Path]]: + """Create browse imagery from an input geotiff. - Exposed library function to allow users to create browse images from the - hybig library. This function parses the input params and builds out the - correct Harmony input structures [Message and Source] to call the service's + This is the exposed library function to allow users to create browse images + from the hybig library. It parses the input params and constructs the + correct Harmony input structure [Message.Format] to call the service's entry point create_browse_imagery. + + Output images are created and deposited into the input GeoTIFF's directory. + + Args: + source_tiff: str, location of the input geotiff to process. + + params: Optional[dict], A dictionary with the following keys: + + mime: [str], MIME type of the output image (default: 'image/png'). + any string that contains 'jpeg' will return a jpeg image, + otherwise create a png. + + crs: Optional[dict], Target image's Coordinate Reference System. + A dictionary with 'epsg', 'proj4' or 'wkt' key. + + scale_extent: Optional[dict], Scale Extents for the image. The dictionary + contains "x" and "y" keys each with value which is dictionary + of "min", "max" values in the same units as the crs. + e.g.: { "x": { "min": 0.5, "max": 125 }, + "y": { "min": 52, "max": 75.22 } } + + scale_size: Optional[dict], Scale sizes for the image. The dictionary + contains "x" and "y" keys with the horizontal and veritcal + resolution in the same units as the crs. + e.g.: { "x": 10, "y": 5 } + + height: Optional[int], height of the output image in gridcells. + + width: Optional[int], width of the output image in gridcells. + + palette: Optional[str | ColorPalette], either a URL to a remote color palette + that is fetched and loaded or a ColorPalette object used to color + the output browse image. If not provided, a grayscale image is + generated. + + logger: Optional[Logger], a configured Logger object. If None a default + logger will be used. + + Note: + if supplied, scale_size, scale_extent, height and width must be + internally consistent. To define a valid output grid: + * Specify scale_extent and 1 of: + * height and width + * scale_sizes (in the x and y horizontal spatial dimensions) + * Specify all three of the above, but ensure values must be consistent + with one another. + + Returns: + List of 3-element tuples. These are the file paths of: + - The output browse image + - Its associated ESRI world file (containing georeferencing information) + - The auxiliary XML file (containing duplicative georeferencing information) + + + Example Usage: + results = create_browse( + "/path/to/geotiff", + { + "mime": "image/png", + "crs": {"epsg": "EPSG:4326"}, + "scale_extent": { + "x": {"min": -180, "max": 180}, + "y": {"min": -90, "max": 90}, + }, + "scale_size": {"x": 10, "y": 10}, + }, + "https://remote-colortable", + logger, + ) + """ - pass + + mime = params.get('mime', 'image/png') + crs = params.get('crs', None) + scale_extent = params.get('scale_extent', None) + scale_size = params.get('scale_size', None) + height = params.get('height', None) + width = params.get('width', None) + + harmony_message = HarmonyMessage( + { + "format": { + "mime": mime, + "crs": crs, + "srs": crs, + "scaleExtent": scale_extent, + "scaleSize": scale_size, + "height": height, + "width": width, + }, + } + ) + + if logger is None: + logger = getLogger('hybig-py') + + if isinstance(palette, str): + color_palette = palette_from_remote_colortable(palette) + else: + color_palette = palette + + return create_browse_imagery( + harmony_message, source_tiff, HarmonySource({}), color_palette, logger + ) def create_browse_imagery( @@ -61,8 +169,9 @@ def create_browse_imagery( ) -> list[tuple[Path, Path, Path]]: """Create browse image from input geotiff. - Take input browse image and return a 2-element tuple for the file paths - of the output browse image and its associated ESRI world file. + Take input browse image and return a 3-element tuple for the file paths of + the output browse image, its associated ESRI world file and the auxilary + xml file. """ output_driver = image_driver(message.format.mime) From ab3f7308dac53f00b5f86d6315124c95889bac2c Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Mon, 8 Jul 2024 13:16:04 -0600 Subject: [PATCH 18/39] DAS-2180: tweaks the README.md for clarity. --- README.md | 12 +++++++----- hybig/browse.py | 8 ++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d7eb8b3..af67e69 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ The browse image generation logic is packaged in the hybig-py library. Currently, a single function, `create_browse` is exposed to the user. ``` python + def create_browse( source_tiff: str, params: dict = {}, @@ -49,15 +50,15 @@ def create_browse( params: Optional[dict], A dictionary with the following keys: - mime: Optional[str], MIME type of the output image (default: 'image/png'). + mime: [str], MIME type of the output image (default: 'image/png'). any string that contains 'jpeg' will return a jpeg image, otherwise create a png. crs: Optional[dict], Target image's Coordinate Reference System. A dictionary with 'epsg', 'proj4' or 'wkt' key. - scale_extent: Optional[dict], Scale Extents for the image. The dictionary - contains "x" and "y" keys each with value which is dictionary + scale_extent: Optional[dict], Scale Extents for the image. This dictionary + contains "x" and "y" keys each whose value which is a dictionary of "min", "max" values in the same units as the crs. e.g.: { "x": { "min": 0.5, "max": 125 }, "y": { "min": 52, "max": 75.22 } } @@ -65,7 +66,7 @@ def create_browse( scale_size: Optional[dict], Scale sizes for the image. The dictionary contains "x" and "y" keys with the horizontal and veritcal resolution in the same units as the crs. - e.g.: { "x": 10, "y": 5 } + e.g.: { "x": 10, "y": 10 } height: Optional[int], height of the output image in gridcells. @@ -85,7 +86,7 @@ def create_browse( * Specify scale_extent and 1 of: * height and width * scale_sizes (in the x and y horizontal spatial dimensions) - * Specify all three of the above, but ensure values must be consistent + * Specify all three of the above, but ensure values are consistent with one another. Returns: @@ -110,6 +111,7 @@ def create_browse( "https://remote-colortable", logger, ) + ``` ### Reprojection diff --git a/hybig/browse.py b/hybig/browse.py index 602c654..09d61d0 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -69,8 +69,8 @@ def create_browse( crs: Optional[dict], Target image's Coordinate Reference System. A dictionary with 'epsg', 'proj4' or 'wkt' key. - scale_extent: Optional[dict], Scale Extents for the image. The dictionary - contains "x" and "y" keys each with value which is dictionary + scale_extent: Optional[dict], Scale Extents for the image. This dictionary + contains "x" and "y" keys each whose value which is a dictionary of "min", "max" values in the same units as the crs. e.g.: { "x": { "min": 0.5, "max": 125 }, "y": { "min": 52, "max": 75.22 } } @@ -78,7 +78,7 @@ def create_browse( scale_size: Optional[dict], Scale sizes for the image. The dictionary contains "x" and "y" keys with the horizontal and veritcal resolution in the same units as the crs. - e.g.: { "x": 10, "y": 5 } + e.g.: { "x": 10, "y": 10 } height: Optional[int], height of the output image in gridcells. @@ -98,7 +98,7 @@ def create_browse( * Specify scale_extent and 1 of: * height and width * scale_sizes (in the x and y horizontal spatial dimensions) - * Specify all three of the above, but ensure values must be consistent + * Specify all three of the above, but ensure values are consistent with one another. Returns: From 23d8aac91f5a7c7e7c64b8d468c627bec4c595c3 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Mon, 8 Jul 2024 17:50:42 -0600 Subject: [PATCH 19/39] DAS-2180: Adds first lib test --- tests/unit/test_browse.py | 55 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_browse.py b/tests/unit/test_browse.py index 9f014bf..0438d1f 100644 --- a/tests/unit/test_browse.py +++ b/tests/unit/test_browse.py @@ -2,12 +2,13 @@ import shutil import tempfile -from logging import getLogger +from logging import Logger, getLogger from pathlib import Path from unittest import TestCase from unittest.mock import MagicMock, Mock, call, patch import numpy as np +from harmony.message import SRS from harmony.message import Message as HarmonyMessage from harmony.message import Source as HarmonySource from numpy.testing import assert_array_equal @@ -23,6 +24,7 @@ from hybig.browse import ( convert_mulitband_to_raster, convert_singleband_to_raster, + create_browse, create_browse_imagery, get_color_map_from_image, get_tiled_filename, @@ -781,3 +783,54 @@ def test_palette_from_remote_colortable(self, mock_get): ' http://this-domain-does-not-exist.com/bad-url' ), ) + + +class TestCreateBrowse(TestCase): + """A class testing the create_browse function call. + + Ensure library calls the `create_browse_imagery` function the same as the + service. + + """ + + @patch('hybig.browse.create_browse_imagery') + def test_one(self, mock_create_browse_imagery): + source_tiff = '/Path/to/source.tiff' + params = { + 'mime': 'image/png', + 'crs': {'epsg': 'EPSG:4326'}, + 'scale_extent': { + 'x': {'min': -180, 'max': 180}, + 'y': {'min': -90, 'max': 90}, + }, + 'scale_size': {'x': 10, 'y': 10}, + } + mock_logger = MagicMock(spec=Logger) + mock_palette = MagicMock(spec=ColorPalette) + + results = create_browse(source_tiff, params, mock_palette, mock_logger) + + call_args = mock_create_browse_imagery.call_args + self.assertIsInstance(call_args[0][0], HarmonyMessage) + self.assertEqual(call_args[0][1], source_tiff) + self.assertIsInstance(call_args[0][2], HarmonySource) + self.assertEqual(call_args[0][3], mock_palette) + self.assertEqual(call_args[0][4], mock_logger) + + # verify message params. + harmony_message = call_args[0][0] + harmony_format = harmony_message.format + + self.assertEqual(harmony_format.mime, "image/png") + self.assertEqual(harmony_format['crs'], {"epsg": "EPSG:4326"}) + self.assertEqual(harmony_format['srs'], {"epsg": "EPSG:4326"}) + self.assertEqual( + harmony_format['scaleExtent'], + { + "x": {"min": -180, "max": 180}, + "y": {"min": -90, "max": 90}, + }, + ) + self.assertEqual(harmony_format['scaleSize'], {"x": 10, "y": 10}) + self.assertIsNone(harmony_message['format']['height']) + self.assertIsNone(harmony_message['format']['width']) From 2852316b51924a7cdcdac7ac5294b8b43f91155d Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Mon, 15 Jul 2024 16:43:31 -0600 Subject: [PATCH 20/39] DAS-2180: Add test to get remote color palette. --- tests/unit/test_browse.py | 58 +++++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/tests/unit/test_browse.py b/tests/unit/test_browse.py index 0438d1f..cbebb5a 100644 --- a/tests/unit/test_browse.py +++ b/tests/unit/test_browse.py @@ -794,7 +794,8 @@ class TestCreateBrowse(TestCase): """ @patch('hybig.browse.create_browse_imagery') - def test_one(self, mock_create_browse_imagery): + def test_calls_create_browse_with_correct_params(self, mock_create_browse_imagery): + """Ensure correct harmony message is created from inputs.""" source_tiff = '/Path/to/source.tiff' params = { 'mime': 'image/png', @@ -808,19 +809,22 @@ def test_one(self, mock_create_browse_imagery): mock_logger = MagicMock(spec=Logger) mock_palette = MagicMock(spec=ColorPalette) - results = create_browse(source_tiff, params, mock_palette, mock_logger) + create_browse(source_tiff, params, mock_palette, mock_logger) - call_args = mock_create_browse_imagery.call_args - self.assertIsInstance(call_args[0][0], HarmonyMessage) - self.assertEqual(call_args[0][1], source_tiff) - self.assertIsInstance(call_args[0][2], HarmonySource) - self.assertEqual(call_args[0][3], mock_palette) - self.assertEqual(call_args[0][4], mock_logger) + mock_create_browse_imagery.assert_called_once() + call_args = mock_create_browse_imagery.call_args[0] + self.assertIsInstance(call_args[0], HarmonyMessage) + self.assertEqual(call_args[1], source_tiff) + self.assertIsInstance(call_args[2], HarmonySource) + self.assertEqual(call_args[3], mock_palette) + self.assertEqual(call_args[4], mock_logger) # verify message params. - harmony_message = call_args[0][0] + harmony_message = call_args[0] harmony_format = harmony_message.format + # HarmonyMessage.Format does not have a json representation to compare + # to so compare the pieces individually. self.assertEqual(harmony_format.mime, "image/png") self.assertEqual(harmony_format['crs'], {"epsg": "EPSG:4326"}) self.assertEqual(harmony_format['srs'], {"epsg": "EPSG:4326"}) @@ -834,3 +838,39 @@ def test_one(self, mock_create_browse_imagery): self.assertEqual(harmony_format['scaleSize'], {"x": 10, "y": 10}) self.assertIsNone(harmony_message['format']['height']) self.assertIsNone(harmony_message['format']['width']) + + @patch('hybig.browse.palette_from_remote_colortable') + @patch('hybig.browse.create_browse_imagery') + def test_calls_create_browse_with_remote_palette( + self, mock_create_browse_imagery, mock_palette_from_remote_color_table + ): + """Ensure remote palette is used.""" + mock_palette = MagicMock(sepc=ColorPalette) + mock_palette_from_remote_color_table.return_value = mock_palette + remote_color_url = 'https://path/to/colormap.txt' + source_tiff = '/Path/to/source.tiff' + mock_logger = MagicMock(spec=Logger) + + # Act + create_browse(source_tiff, {}, remote_color_url, mock_logger) + + # Assert a remote colortable was fetched. + mock_palette_from_remote_color_table.assert_called_once_with(remote_color_url) + + mock_create_browse_imagery.assert_called_once() + ( + call_harmony_message, + call_source_tiff, + call_harmony_source, + call_color_palette, + call_logger, + ) = mock_create_browse_imagery.call_args[0] + + # create_browse_imagery called with the color palette returned from + # palette_from_remote_colortable + self.assertEqual(call_color_palette, mock_palette) + + self.assertIsInstance(call_harmony_message, HarmonyMessage) + self.assertIsInstance(call_harmony_source, HarmonySource) + self.assertEqual(call_source_tiff, source_tiff) + self.assertEqual(call_logger, mock_logger) From f0548ba47b5d9e0cca3bf3088b39c3de796afb76 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Fri, 19 Jul 2024 10:23:08 -0600 Subject: [PATCH 21/39] DAS-2180: Point to real PyPI, choose major version. --- .github/workflows/publish_release.yml | 6 ++---- docker/service_version.txt | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index dd0428c..2175689 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -75,7 +75,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: python-version: '3.11' @@ -83,10 +83,8 @@ jobs: - name: Build hybig-py package run: python -m build - - name: Publish to TestPyPI + - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: https://test.pypi.org/legacy/ - name: Publish GitHub release uses: ncipollo/release-action@v1 diff --git a/docker/service_version.txt b/docker/service_version.txt index 38f77a6..227cea2 100644 --- a/docker/service_version.txt +++ b/docker/service_version.txt @@ -1 +1 @@ -2.0.1 +2.0.0 From 489d85268b1f7d08bdb2867fedf1eb7a8b676e7a Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Fri, 19 Jul 2024 11:08:02 -0600 Subject: [PATCH 22/39] DAS-2180: updates CHANGELOG.md --- CHANGELOG.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index baf9a82..bf43837 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ HyBIG follows semantic versioning. All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). +## [v2.0.0] - 2024-07-19 + +**DAS-2180** - Adds pip installable library. + +This release is a refactor that extracts browse image generation logic from the +harmony service code. There are no user visible changes to the existing +functionality. The new library, +[hybig-py](https://pypi.org/project/hybig-py/), provides the `create_browse` +function to generate browse images, see the README.md for details. + ## [v1.2.2] - 2024-06-18 ### Changed @@ -52,7 +62,8 @@ outlined by the NASA open-source guidelines. For more information on internal releases prior to NASA open-source approval, see legacy-CHANGELOG.md. -[unreleased]:https://github.com/nasa/harmony-browse-image-generator/compare/1.2.2..HEAD +[unreleased]:https://github.com/nasa/harmony-browse-image-generator/compare/2.0.0..HEAD +[v2.0.0]:https://github.com/nasa/harmony-browse-image-generator/compare/1.2.2..2.0.0 [v1.2.2]: https://github.com/nasa/harmony-browse-image-generator/compare/1.2.1..1.2.2 [v1.2.1]: https://github.com/nasa/harmony-browse-image-generator/compare/1.2.0..1.2.1 [v1.2.0]: https://github.com/nasa/harmony-browse-image-generator/compare/1.1.0..1.2.0 From f27bcea6467ec02b743185278e7af21978853e9d Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Fri, 19 Jul 2024 13:55:11 -0600 Subject: [PATCH 23/39] DAS-2180: Adds matrix of library tests to CI renames run_tests -> run_service_tests Adds run_lib_tests and runs the unit tests against a python matrix. --- .github/workflows/publish_release.yml | 11 +++-- .github/workflows/run_lib_tests.yml | 43 +++++++++++++++++++ .../{run_tests.yml => run_service_tests.yml} | 4 +- .../workflows/run_tests_on_pull_requests.yml | 9 ++-- README.md | 13 +++--- pyproject.toml | 2 +- 6 files changed, 66 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/run_lib_tests.yml rename .github/workflows/{run_tests.yml => run_service_tests.yml} (95%) diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index 2175689..16901f1 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -1,7 +1,7 @@ # This workflow will run when changes are detected in the `main` branch, which # must include an update to the `docker/service_version.txt` file. The workflow # can also be manually triggered by a repository maintainer. This workflow will -# first trigger the reusable workflow in `.github/workflows/run_tests.yml`, +# first trigger the reusable workflow in `.github/workflows/run_service_tests.yml`, # which runs the `unittest` suite. If that workflow is successful, the latest # version of the service Docker image is pushed to ghcr.io, a library package # is built and published to PyPI, a tag is added to the latest git commit, and @@ -20,11 +20,14 @@ env: REGISTRY: ghcr.io jobs: - run_tests: - uses: ./.github/workflows/run_tests.yml + run_service_tests: + uses: ./.github/workflows/run_service_tests.yml + + run_lib_tests: + uses: ./.github/workflows/run_lib_tests.yml build_and_publish: - needs: run_tests + needs: [run_service_tests, run_lib_tests] runs-on: ubuntu-latest environment: release permissions: diff --git a/.github/workflows/run_lib_tests.yml b/.github/workflows/run_lib_tests.yml new file mode 100644 index 0000000..c38b8bd --- /dev/null +++ b/.github/workflows/run_lib_tests.yml @@ -0,0 +1,43 @@ +# This workflow will run the appropriate library tests across a python matrix of versions. +name: Run Python library tests + +on: + workflow_call + +jobs: + build_and_test_lib: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.10', '3.11'] + + steps: + - name: Checkout harmony-browse-image-generator repository + uses: actions/checkout@v4 + with: + lfs: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install GDAL + run: | + # Install packaged version of GDAL. + sudo apt-get update + sudo apt-get install -y libgdal-dev + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest + pip install -r pip_requirements.txt + # Use the gdal version that was installed + pip install GDAL==$(gdal-config --version) + pip install -r tests/pip_test_requirements.txt + + - name: Run tests without the adapters. + run: | + pytest $(find tests -name "test_*.py" ! -name "*adapter*") diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_service_tests.yml similarity index 95% rename from .github/workflows/run_tests.yml rename to .github/workflows/run_service_tests.yml index 0afce2f..1568825 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_service_tests.yml @@ -3,13 +3,13 @@ # test results and code coverage as artefacts. It will be called by the # workflow that run tests against new PRs and as a first step in the workflow # that publishes new Docker images. -name: Run Python unit tests +name: Run Python tests on: workflow_call jobs: - build_and_test: + build_and_test_service: runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/run_tests_on_pull_requests.yml b/.github/workflows/run_tests_on_pull_requests.yml index a948fd1..279a41e 100644 --- a/.github/workflows/run_tests_on_pull_requests.yml +++ b/.github/workflows/run_tests_on_pull_requests.yml @@ -1,5 +1,5 @@ # This workflow will run when a PR is opened against the `main` branch. It will -# trigger the reusable workflow in `.github/workflows/run_tests.yml`, which +# trigger the reusable workflow in `.github/workflows/run_service_tests.yml`, which # builds the service and test Docker images, and runs the `unittest` suite in a # Docker container built from the test image. name: Run Python unit tests for pull requests against main @@ -9,5 +9,8 @@ on: branches: [ main ] jobs: - build_and_test: - uses: ./.github/workflows/run_tests.yml + build_and_test_service: + uses: ./.github/workflows/run_service_tests.yml + + run_lib_tests: + uses: ./.github/workflows/run_lib_tests.yml diff --git a/README.md b/README.md index af67e69..4a7bf52 100644 --- a/README.md +++ b/README.md @@ -364,15 +364,16 @@ When publishing a new release, two files must be updated: The CI/CD for HyBIG is contained in GitHub workflows in the `.github/workflows` directory: -* `run_tests.yml` - A reusable workflow that builds the service and test Docker - images, then runs the Python unit test suite in an instance of the test - Docker container. +* `run_lib_tests.yml` - A reusable workflow that tests the library functions + against the supported python versions. +* `run_service_tests.yml` - A reusable workflow that builds the service and + test Docker images, then runs the Python unit test suite in an instance of + the test Docker container. * `run_tests_on_pull_requests.yml` - Triggered for all PRs against the `main` - branch. It runs the workflow in `run_tests.yml` to ensure all tests pass for - the new code. + branch. It runs the workflow in `run_service_tests.yml` and + `run_lib_tests.yml` to ensure all tests pass for the new code. * `publish_docker_image.yml` - Triggered either manually or for commits to the `main` branch that contain changes to the `docker/service_version.txt` file. - * `publish_to_pypi.yml` - Triggered either manually or for commits to the `main` branch that contain changes to the `docker/service_version.txt`file. diff --git a/pyproject.toml b/pyproject.toml index f22682a..71ab6e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ maintainers = [ description = "Python package designed to produce browse imagery compatible with NASA's Global Image Browse Services (GIBS)." readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.10" classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", From d1e37501c1f616cbeb25615de5a8d7fec3fe6fcc Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Fri, 19 Jul 2024 14:22:18 -0600 Subject: [PATCH 24/39] DAS-2180: Don't use an empty dict as default value. --- README.md | 22 ++++++++++------------ hybig/browse.py | 21 +++++++++++---------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 4a7bf52..9619c8f 100644 --- a/README.md +++ b/README.md @@ -28,11 +28,10 @@ exposes functions to generate browse images in python scripts. The browse image generation logic is packaged in the hybig-py library. Currently, a single function, `create_browse` is exposed to the user. -``` python - +```python def create_browse( source_tiff: str, - params: dict = {}, + params: dict = None, palette: str | ColorPalette | None = None, logger: Logger = None, ) -> list[tuple[Path, Path, Path]]: @@ -48,36 +47,36 @@ def create_browse( Args: source_tiff: str, location of the input geotiff to process. - params: Optional[dict], A dictionary with the following keys: + params: [dict | None], A dictionary with the following keys: mime: [str], MIME type of the output image (default: 'image/png'). any string that contains 'jpeg' will return a jpeg image, otherwise create a png. - crs: Optional[dict], Target image's Coordinate Reference System. + crs: [dict | None], Target image's Coordinate Reference System. A dictionary with 'epsg', 'proj4' or 'wkt' key. - scale_extent: Optional[dict], Scale Extents for the image. This dictionary + scale_extent: [dict | None], Scale Extents for the image. This dictionary contains "x" and "y" keys each whose value which is a dictionary of "min", "max" values in the same units as the crs. e.g.: { "x": { "min": 0.5, "max": 125 }, "y": { "min": 52, "max": 75.22 } } - scale_size: Optional[dict], Scale sizes for the image. The dictionary + scale_size: [dict | None], Scale sizes for the image. The dictionary contains "x" and "y" keys with the horizontal and veritcal resolution in the same units as the crs. e.g.: { "x": 10, "y": 10 } - height: Optional[int], height of the output image in gridcells. + height: [int | None], height of the output image in gridcells. - width: Optional[int], width of the output image in gridcells. + width: [int | none], width of the output image in gridcells. - palette: Optional[str | ColorPalette], either a URL to a remote color palette + palette: [str | ColorPalette | none], either a URL to a remote color palette that is fetched and loaded or a ColorPalette object used to color the output browse image. If not provided, a grayscale image is generated. - logger: Optional[Logger], a configured Logger object. If None a default + logger: [Logger | None], a configured Logger object. If None a default logger will be used. Note: @@ -111,7 +110,6 @@ def create_browse( "https://remote-colortable", logger, ) - ``` ### Reprojection diff --git a/hybig/browse.py b/hybig/browse.py index 09d61d0..3e513b6 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -44,7 +44,7 @@ def create_browse( source_tiff: str, - params: dict = {}, + params: dict = None, palette: str | ColorPalette | None = None, logger: Logger = None, ) -> list[tuple[Path, Path, Path]]: @@ -60,36 +60,36 @@ def create_browse( Args: source_tiff: str, location of the input geotiff to process. - params: Optional[dict], A dictionary with the following keys: + params: [dict | None], A dictionary with the following keys: mime: [str], MIME type of the output image (default: 'image/png'). any string that contains 'jpeg' will return a jpeg image, otherwise create a png. - crs: Optional[dict], Target image's Coordinate Reference System. + crs: [dict | None], Target image's Coordinate Reference System. A dictionary with 'epsg', 'proj4' or 'wkt' key. - scale_extent: Optional[dict], Scale Extents for the image. This dictionary + scale_extent: [dict | None], Scale Extents for the image. This dictionary contains "x" and "y" keys each whose value which is a dictionary of "min", "max" values in the same units as the crs. e.g.: { "x": { "min": 0.5, "max": 125 }, "y": { "min": 52, "max": 75.22 } } - scale_size: Optional[dict], Scale sizes for the image. The dictionary + scale_size: [dict | None], Scale sizes for the image. The dictionary contains "x" and "y" keys with the horizontal and veritcal resolution in the same units as the crs. e.g.: { "x": 10, "y": 10 } - height: Optional[int], height of the output image in gridcells. + height: [int | None], height of the output image in gridcells. - width: Optional[int], width of the output image in gridcells. + width: [int | none], width of the output image in gridcells. - palette: Optional[str | ColorPalette], either a URL to a remote color palette + palette: [str | ColorPalette | none], either a URL to a remote color palette that is fetched and loaded or a ColorPalette object used to color the output browse image. If not provided, a grayscale image is generated. - logger: Optional[Logger], a configured Logger object. If None a default + logger: [Logger | None], a configured Logger object. If None a default logger will be used. Note: @@ -125,7 +125,8 @@ def create_browse( ) """ - + if params is None: + params = {} mime = params.get('mime', 'image/png') crs = params.get('crs', None) scale_extent = params.get('scale_extent', None) From 88bea96aea22bd3e463d3cecc20bdbca700098b8 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Mon, 22 Jul 2024 17:41:47 -0600 Subject: [PATCH 25/39] DAS-2180: Cleans up science and service tests. Updates pip_test_requirements.txt Switches the coverage call. --- .github/workflows/run_lib_tests.yml | 4 ++-- tests/pip_test_requirements.txt | 7 ++++--- tests/run_tests.sh | 2 +- tests/{ => test_service}/test_adapter.py | 2 +- .../unit/test_adapter_unit.py} | 0 tests/unit/__init__.py | 0 6 files changed, 8 insertions(+), 7 deletions(-) rename tests/{ => test_service}/test_adapter.py (99%) rename tests/{unit/test_adapter.py => test_service/unit/test_adapter_unit.py} (100%) delete mode 100644 tests/unit/__init__.py diff --git a/.github/workflows/run_lib_tests.yml b/.github/workflows/run_lib_tests.yml index c38b8bd..26c20b2 100644 --- a/.github/workflows/run_lib_tests.yml +++ b/.github/workflows/run_lib_tests.yml @@ -38,6 +38,6 @@ jobs: pip install GDAL==$(gdal-config --version) pip install -r tests/pip_test_requirements.txt - - name: Run tests without the adapters. + - name: Run science tests while excluding the service tests. run: | - pytest $(find tests -name "test_*.py" ! -name "*adapter*") + pytest tests --ignore tests/test_service diff --git a/tests/pip_test_requirements.txt b/tests/pip_test_requirements.txt index 0cf95be..b0203ab 100644 --- a/tests/pip_test_requirements.txt +++ b/tests/pip_test_requirements.txt @@ -1,4 +1,5 @@ -coverage~=7.2.2 -pycodestyle~=2.10.0 -pylint~=2.17.2 +coverage~=7.6.0 +pycodestyle~=2.12.0 +pylint~=3.2.6 unittest-xml-reporting~=3.2.0 +pytest~=8.3.1 diff --git a/tests/run_tests.sh b/tests/run_tests.sh index db2b1da..dccbe4e 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -17,7 +17,7 @@ STATUS=0 export HDF5_DISABLE_VERSION_CHECK=1 # Run the standard set of unit tests, producing JUnit compatible output -coverage run -m xmlrunner discover tests -o tests/reports +coverage run -m pytest tests --junitxml=tests/reports/test-results-"$(date +'%Y%m%d%H%M%S')".xml RESULT=$? if [ "$RESULT" -ne "0" ]; then diff --git a/tests/test_adapter.py b/tests/test_service/test_adapter.py similarity index 99% rename from tests/test_adapter.py rename to tests/test_service/test_adapter.py index c8f9b78..e078259 100644 --- a/tests/test_adapter.py +++ b/tests/test_service/test_adapter.py @@ -32,7 +32,7 @@ def setUpClass(cls): cls.granule_url = 'https://www.example.com/input.tiff' cls.input_stac = create_stac(Granule(cls.granule_url, 'image/tiff', ['data'])) cls.staging_location = 's3://example-bucket' - cls.fixtures = Path(__file__).resolve().parent / 'fixtures' + cls.fixtures = Path(__file__).resolve().parent.parent / 'fixtures' cls.red_tif_fixture = cls.fixtures / 'red.tif' cls.user = 'blightyear' diff --git a/tests/unit/test_adapter.py b/tests/test_service/unit/test_adapter_unit.py similarity index 100% rename from tests/unit/test_adapter.py rename to tests/test_service/unit/test_adapter_unit.py diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py deleted file mode 100644 index e69de29..0000000 From d721a57370ecf8d6547a72ffe8e2c7c301cb40e7 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 30 Jul 2024 10:33:29 -0600 Subject: [PATCH 26/39] DAS-2180: Clarify why the test gdal is different from the docker one --- .github/workflows/run_lib_tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run_lib_tests.yml b/.github/workflows/run_lib_tests.yml index 26c20b2..74ac0d8 100644 --- a/.github/workflows/run_lib_tests.yml +++ b/.github/workflows/run_lib_tests.yml @@ -34,7 +34,9 @@ jobs: python -m pip install --upgrade pip pip install pytest pip install -r pip_requirements.txt - # Use the gdal version that was installed + # Use the gdal version that was installed in the previous step. This + # is not the same GDAL as installed in the docker images but in the + # end we're only using osgeo's ColorPalette pip install GDAL==$(gdal-config --version) pip install -r tests/pip_test_requirements.txt From aff9d3587c2639f73590af32d47c363caaf9f074 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 30 Jul 2024 11:40:41 -0600 Subject: [PATCH 27/39] DAS-2180: Break up run on sentence. --- README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9619c8f..1d33677 100644 --- a/README.md +++ b/README.md @@ -303,12 +303,14 @@ Local testing of service functionality can be achieved via a local instance of [Harmony](https://github.com/nasa/harmony). Please see instructions there regarding creation of a local Harmony instance. -If developing changes to the library, or testing small functions locally that -do not require inputs from the main Harmony application, it is recommended that -you create a Python virtual environment, and then install the necessary -dependencies for the service within that environment via conda and pip then -install the pre-commit hooks. Note that you will need the gdal libraries -available to your virtual environment to install the `gdal` package with pip. +For local development and testing of library modifications or small functions +independent of the main Harmony application: + +1. Create a Python virtual environment +1. Ensure GDAL libraries are accessable in the virtual environment. +1. Install the dependencies in `pip_requirements.txt`, `pip_requirements_skip_snyk.txt` and `dev-requirements.txt` +1. Install the pre-commit hooks. + ``` > conda create --name hybig-env python==3.11 From e383bf4f480c2440d8918c638664ae6ee7e9216f Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 30 Jul 2024 12:05:40 -0600 Subject: [PATCH 28/39] DAS-2180: Thanks Claude --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1d33677..cdae1f8 100644 --- a/README.md +++ b/README.md @@ -340,10 +340,10 @@ Currently, the `unittest` suite is run automatically within a GitHub workflow as part of a CI/CD pipeline. These tests are run for all changes made in a PR against the `main` branch. The tests must pass in order to merge the PR. -The unit tests are also run prior to publication of new library packages and -Docker image, when commits including changes to `docker/service_version.txt` -are merged into the `main` branch. If these unit tests fail, the new version of -the Docker image and library package will not be published. +Unit tests are executed automatically before publishing new library packages +and Docker images. This occurs when commits containing changes to +`docker/service_version.txt` are merged into the `main` branch. Failed unit +tests prevent the publication of new Docker images and library packages. ## Versioning: From 21a8643cc2e37a3f639b2efc45cabdb75db1c8d2 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 30 Jul 2024 12:24:30 -0600 Subject: [PATCH 29/39] DAS-2180: Clarify releases. --- README.md | 57 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index cdae1f8..5a0fd0d 100644 --- a/README.md +++ b/README.md @@ -249,9 +249,8 @@ also with units of degrees. * `docker` - A directory containing the Dockerfiles for the service and test images. It also contains `service_version.txt`, which contains the semantic - version number of the library and service image. Any time an update is made - that should have an accompanying library and service image release, this file - should be updated. + version number of the library and service image. Update this file with a new + version to trigger a release. * `docs` - A directory with example usage notebooks. @@ -340,28 +339,21 @@ Currently, the `unittest` suite is run automatically within a GitHub workflow as part of a CI/CD pipeline. These tests are run for all changes made in a PR against the `main` branch. The tests must pass in order to merge the PR. -Unit tests are executed automatically before publishing new library packages -and Docker images. This occurs when commits containing changes to -`docker/service_version.txt` are merged into the `main` branch. Failed unit -tests prevent the publication of new Docker images and library packages. +Unit tests are executed automatically by github actions on each Pull Request. + ## Versioning: -Service Docker images and the package library for HyBIG adhere to semantic +Docker service images and the hybig-py package library adhere to semantic version numbers: major.minor.patch. * Major increments: These are non-backwards compatible API changes. * Minor increments: These are backwards compatible API changes. * Patch increments: These updates do not affect the API to the service. -When publishing a new release, two files must be updated: - -* `CHANGELOG.md` - Notes should be added to capture the changes to the service. -* `docker/service_version.txt` - The semantic version number should be updated. - ## CI/CD: -The CI/CD for HyBIG is contained in GitHub workflows in the +The CI/CD for HyBIG is run on github actions with the workflows in the `.github/workflows` directory: * `run_lib_tests.yml` - A reusable workflow that tests the library functions @@ -376,23 +368,34 @@ The CI/CD for HyBIG is contained in GitHub workflows in the `main` branch that contain changes to the `docker/service_version.txt` file. * `publish_to_pypi.yml` - Triggered either manually or for commits to the `main` branch that contain changes to the `docker/service_version.txt`file. +* `publish_release.yml` - workflow runs automatically when there is a change to + the `docker/service_version.txt` file on the main branch. This workflow will: + * Run the full unit test suite, to prevent publication of broken code. + * Extract the semantic version number from `docker/service_version.txt`. + * Extract the released notes for the most recent version from `CHANGELOG.md`. + * Build and deploy a this service's docker image to `ghcr.io`. + * Build the library package to be published to PyPI. + * Publish the package to PyPI. + * Publish a GitHub release under the semantic version number, with associated + git tag. -The `publish_release.yml` workflow will: -* Run the full unit test suite, to prevent publication of broken code. -* Extract the semantic version number from `docker/service_version.txt`. -* Extract the released notes for the most recent version from `CHANGELOG.md`. -* Build and deploy a this service's docker image to `ghcr.io`. -* Build the library package to be published to PyPI. -* Publish the package to PyPI. -* Publish a GitHub release under the semantic version number, with associated - git tag. +## Releasing +A release consists of a new version hybig-py library published to PyPI and a +new Docker service image published to github's container repository. + +A release is made automatically when a commit to the main branch contains a +changes in the `docker/service_version.txt` file, see the [publish_release](#release-workflow) workflow in the CI/CD section above. + +Before merging a PR that will trigger a release, ensure these two files are updated: + +* `CHANGELOG.md` - Notes should be added to capture the changes to the service. +* `docker/service_version.txt` - The semantic version number should be updated. -Before triggering a release, ensure both the `docker/service_version.txt` and -`CHANGELOG.md` files are updated. The `CHANGELOG.md` file requires a specific -format for a new release, as it looks for the following string to define the -newest release of the code (starting at the top of the file). +The `CHANGELOG.md` file requires a specific format for a new release, as it +looks for the following string to define the newest release of the code +(starting at the top of the file). ``` ## [vX.Y.Z] - YYYY-MM-DD From 2e126b3f793b97eacb52e45bdfd5447f9e014a18 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 30 Jul 2024 13:03:50 -0600 Subject: [PATCH 30/39] DAS-2180: extention -> extension --- harmony_service/adapter.py | 2 +- harmony_service/utilities.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/harmony_service/adapter.py b/harmony_service/adapter.py index 7d2ceb5..e0ed1f8 100644 --- a/harmony_service/adapter.py +++ b/harmony_service/adapter.py @@ -36,7 +36,7 @@ class BrowseImageGeneratorAdapter(BaseHarmonyAdapter): - """HyBIG extention to the harmony-service-lib BaseHarmonyAdapter.""" + """HyBIG extension to the harmony-service-lib BaseHarmonyAdapter.""" def invoke(self) -> Catalog: """Adds validation to process_item based invocations.""" diff --git a/harmony_service/utilities.py b/harmony_service/utilities.py index d049f25..cb7e49f 100644 --- a/harmony_service/utilities.py +++ b/harmony_service/utilities.py @@ -15,9 +15,9 @@ def get_tiled_file_extension(file_name: Path) -> str: - """Return the correct extention to add to a staged file. + """Return the correct extension to add to a staged file. - Harmony's generate output filename can drop an extention incorrectly, so we + Harmony's generate output filename can drop an extension incorrectly, so we generate the correct one to pass in. """ From 7b2c903cdefc05544bdc26df0613a7095543a92d Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 30 Jul 2024 13:17:16 -0600 Subject: [PATCH 31/39] DAS-2180: use correct library name --- hybig/browse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hybig/browse.py b/hybig/browse.py index 3e513b6..298aa8d 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -51,7 +51,7 @@ def create_browse( """Create browse imagery from an input geotiff. This is the exposed library function to allow users to create browse images - from the hybig library. It parses the input params and constructs the + from the hybig-py library. It parses the input params and constructs the correct Harmony input structure [Message.Format] to call the service's entry point create_browse_imagery. From bdde01ec3b214d5931d4529d6ee14b0ef939b26f Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 30 Jul 2024 14:35:35 -0600 Subject: [PATCH 32/39] DAS-2180: Explain self-consistent grids. --- README.md | 8 ++++++-- hybig/browse.py | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5a0fd0d..9796733 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ def create_browse( """Create browse imagery from an input geotiff. This is the exposed library function to allow users to create browse images - from the hybig library. It parses the input params and constructs the + from the hybig-py library. It parses the input params and constructs the correct Harmony input structure [Message.Format] to call the service's entry point create_browse_imagery. @@ -86,7 +86,9 @@ def create_browse( * height and width * scale_sizes (in the x and y horizontal spatial dimensions) * Specify all three of the above, but ensure values are consistent - with one another. + with one another, noting that: + scale_size.x = (scale_extent.x.max - scale_extent.x.min) / width + scale_size.y = (scale_extent.y.max - scale_extent.y.min) / height Returns: List of 3-element tuples. These are the file paths of: @@ -110,6 +112,8 @@ def create_browse( "https://remote-colortable", logger, ) + + """ ``` ### Reprojection diff --git a/hybig/browse.py b/hybig/browse.py index 298aa8d..d18adc8 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -99,7 +99,9 @@ def create_browse( * height and width * scale_sizes (in the x and y horizontal spatial dimensions) * Specify all three of the above, but ensure values are consistent - with one another. + with one another, noting that: + scale_size.x = (scale_extent.x.max - scale_extent.x.min) / width + scale_size.y = (scale_extent.y.max - scale_extent.y.min) / height Returns: List of 3-element tuples. These are the file paths of: From 257a46243e20c5684a93e2559d811dcc6f06d26b Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 30 Jul 2024 14:59:50 -0600 Subject: [PATCH 33/39] DAS-2180: Extracts get_harmony_message_from_params Also adds back __init__.py to get tests running locally. (outside of docker and workflows) --- hybig/browse.py | 24 ++---------------------- hybig/browse_utility.py | 34 ++++++++++++++++++++++++++++++++++ tests/unit/__init__.py | 0 3 files changed, 36 insertions(+), 22 deletions(-) create mode 100644 hybig/browse_utility.py create mode 100644 tests/unit/__init__.py diff --git a/hybig/browse.py b/hybig/browse.py index d18adc8..dedac30 100644 --- a/hybig/browse.py +++ b/hybig/browse.py @@ -22,6 +22,7 @@ from rioxarray import open_rasterio from xarray import DataArray +from hybig.browse_utility import get_harmony_message_from_params from hybig.color_utility import ( NODATA_IDX, NODATA_RGBA, @@ -127,28 +128,7 @@ def create_browse( ) """ - if params is None: - params = {} - mime = params.get('mime', 'image/png') - crs = params.get('crs', None) - scale_extent = params.get('scale_extent', None) - scale_size = params.get('scale_size', None) - height = params.get('height', None) - width = params.get('width', None) - - harmony_message = HarmonyMessage( - { - "format": { - "mime": mime, - "crs": crs, - "srs": crs, - "scaleExtent": scale_extent, - "scaleSize": scale_size, - "height": height, - "width": width, - }, - } - ) + harmony_message = get_harmony_message_from_params(params) if logger is None: logger = getLogger('hybig-py') diff --git a/hybig/browse_utility.py b/hybig/browse_utility.py new file mode 100644 index 0000000..6dbd8cd --- /dev/null +++ b/hybig/browse_utility.py @@ -0,0 +1,34 @@ +"""Module containing utility functionality for browse generation.""" + +from harmony.message import Message as HarmonyMessage + + +def get_harmony_message_from_params(params: dict | None) -> HarmonyMessage: + """Constructs a harmony message from the input parms. + + We have to create a harmony message to pass to the create_browse_imagery + function so that both the library and service calls are identical. + + """ + if params is None: + params = {} + mime = params.get('mime', 'image/png') + crs = params.get('crs', None) + scale_extent = params.get('scale_extent', None) + scale_size = params.get('scale_size', None) + height = params.get('height', None) + width = params.get('width', None) + + return HarmonyMessage( + { + "format": { + "mime": mime, + "crs": crs, + "srs": crs, + "scaleExtent": scale_extent, + "scaleSize": scale_size, + "height": height, + "width": width, + }, + } + ) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 From fe1f55783b51ce1a8593c4c3985ad6a7f6c3aeb8 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 30 Jul 2024 15:08:27 -0600 Subject: [PATCH 34/39] DAS-2180: Sorts imports, cleans pyproject.toml --- pip_requirements.txt | 6 +++--- pyproject.toml | 22 ---------------------- tests/pip_test_requirements.txt | 2 +- 3 files changed, 4 insertions(+), 26 deletions(-) diff --git a/pip_requirements.txt b/pip_requirements.txt index cbaf9cd..2642ee3 100644 --- a/pip_requirements.txt +++ b/pip_requirements.txt @@ -1,8 +1,8 @@ harmony-service-lib~=1.0.27 -pystac~=0.5.6 matplotlib==3.9.0 -rasterio==1.3.10 -rioxarray==0.15.5 numpy==1.26.4 pillow==10.3.0 pyproj==3.6.1 +pystac~=0.5.6 +rasterio==1.3.10 +rioxarray==0.15.5 diff --git a/pyproject.toml b/pyproject.toml index 71ab6e3..e1ea909 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,25 +50,3 @@ exclude = [ [tool.hatch.build.targets.wheel] packages=["hybig"] - -[tool.black] -skip-string-normalization = 1 - -[tool.isort] -profile = "black" - -[tool.ruff] -lint.select = [ - "E", # pycodestyle - "F", # pyflakes - "UP", # pyupgrade - "I", # organize imports - "D", # docstyle -] - -[tool.ruff.lint.pydocstyle] -convention = "google" - -[[tool.mypy.overrides]] -module = "harmony.*,matplotlib.*,rasterio.*,osgeo_utils.*,pystac.*,affine.*,pycodestyle.*,imagequant.*,PIL.*,rioxarray.*, xarray.*" -ignore_missing_imports = true diff --git a/tests/pip_test_requirements.txt b/tests/pip_test_requirements.txt index b0203ab..f9d53bd 100644 --- a/tests/pip_test_requirements.txt +++ b/tests/pip_test_requirements.txt @@ -1,5 +1,5 @@ coverage~=7.6.0 pycodestyle~=2.12.0 pylint~=3.2.6 -unittest-xml-reporting~=3.2.0 pytest~=8.3.1 +unittest-xml-reporting~=3.2.0 From 12c3df3b9ca4f7bd2e3926f3bdefa94dd4f9346f Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 30 Jul 2024 15:15:02 -0600 Subject: [PATCH 35/39] DAS-2180: updates comments on the run_tests.sh script --- tests/run_tests.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/run_tests.sh b/tests/run_tests.sh index dccbe4e..ecc70c5 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -8,6 +8,10 @@ # 2020-05-07: Adapted from SwotRepr project. # 2022-01-03: Removed safety checks, as these are now run in Snyk. # 2023-04-04: Updated for use with the Harmony Browse Image Generator (HyBIG). +# 2024-07-30: Changes coverage to use pytest and output a unified +# result. xmlrunner was unable to handle finding the tests in the separate +# locations. Also use a relative path to the html output so that coverages can be +# run outside of docker. # ############################################################################### From 8a70fe3a9670536fe90733d578508aa0d139b614 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 30 Jul 2024 15:20:16 -0600 Subject: [PATCH 36/39] DAS-2180: really only export create_browse --- harmony_service/adapter.py | 6 ++---- hybig/__init__.py | 5 ++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/harmony_service/adapter.py b/harmony_service/adapter.py index e0ed1f8..0f4700a 100644 --- a/harmony_service/adapter.py +++ b/harmony_service/adapter.py @@ -29,10 +29,8 @@ get_file_mime_type, get_tiled_file_extension, ) -from hybig import ( - create_browse_imagery, - get_color_palette_from_item, -) +from hybig.browse import create_browse_imagery +from hybig.color_utility import get_color_palette_from_item class BrowseImageGeneratorAdapter(BaseHarmonyAdapter): diff --git a/hybig/__init__.py b/hybig/__init__.py index 645fdbd..85c0b32 100644 --- a/hybig/__init__.py +++ b/hybig/__init__.py @@ -1,6 +1,5 @@ """Package containing core functionality for browse image generation.""" -from .browse import create_browse, create_browse_imagery -from .color_utility import get_color_palette_from_item +from .browse import create_browse -__all__ = ['create_browse', 'create_browse_imagery', 'get_color_palette_from_item'] +__all__ = ['create_browse'] From 8c18fb8b1f6225ad2d9a72915ed8777c858cd737 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Tue, 30 Jul 2024 15:24:45 -0600 Subject: [PATCH 37/39] DAS-2180: update notes on service.Dockerfile --- docker/service.Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/service.Dockerfile b/docker/service.Dockerfile index 695b920..660c591 100644 --- a/docker/service.Dockerfile +++ b/docker/service.Dockerfile @@ -11,6 +11,8 @@ # 2023-04-04: Updated for HyBIG. # 2023-04-23: Updated conda clean and pip install to keep Docker image slim. # 2024-06-18: Updates to remove conda dependency. +# 2024-07-30: Updates to handle separate service an science code directories +# and updates the entrypoint of the new service container # ############################################################################### FROM python:3.11 @@ -28,8 +30,8 @@ RUN pip install --no-input --no-cache-dir \ -r pip_requirements_skip_snyk.txt # Copy service code. -COPY ./hybig hybig COPY ./harmony_service harmony_service +COPY ./hybig hybig # Set GDAL related environment variables. ENV CPL_ZIP_ENCODING=UTF-8 From c455ebf002dfad23da2916b90b45faf78ae65813 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 31 Jul 2024 17:08:00 -0600 Subject: [PATCH 38/39] DAS-2180: Add notice of gdal requirements. --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index 9796733..a278782 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,25 @@ def create_browse( """ ``` +### library installation + +The hybig-py library can be installed from PyPI but has a prerequisite +dependency requirement on the GDAL libraries. Ensure you have an environment with the libraries available. You can check on Linux/macOS: +```bash +gdal-config --version +``` +on windows (if GDAL is in your PATH): +```bash +gdalinfo --version +``` + +Once verified, you can simply install the libary: + +```bash +pip install hybig-py +``` + + ### Reprojection GIBS expects to receive images in one of three Coordinate Reference System (CRS) projections. From e8c8c4c77ee8765cc7142ac972e57cbcd34fa9c2 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 31 Jul 2024 17:19:53 -0600 Subject: [PATCH 39/39] DAS-2180: kick off tests again. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a278782..2747d52 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,8 @@ def create_browse( ### library installation The hybig-py library can be installed from PyPI but has a prerequisite -dependency requirement on the GDAL libraries. Ensure you have an environment with the libraries available. You can check on Linux/macOS: +dependency requirement on the GDAL libraries. Ensure you have an environment +with the libraries available. You can check on Linux/macOS: ```bash gdal-config --version ```