Update service to Python 3.11 and earthdata-varinfo. Update notebook …

…dependencies. Squashed commit of the following: commit 6b66a2592458df96ddc45bdba7c24e50927ff1d4 Author: Owen Littlejohns <[email protected]> Date: Thu Jul 27 12:41:10 2023 -0400 Update recommended notebook Python version to 3.11 commit 07f59b596c3357c65be2d23f24c4ea12e74e1737 Author: Owen Littlejohns <[email protected]> Date: Thu Jul 20 22:57:38 2023 -0400 Update service dependencies and Python environment to 3.11. commit 6edde6dd7d4508d60c86645dca4a7f8c62666fcb Author: Owen Littlejohns <[email protected]> Date: Wed Apr 5 20:59:22 2023 -0400 Update documentation notebook dependencies to mitigate Snyk vulnerability.
nasa · Jul 28, 2023 · 0b7d614 · 0b7d614
1 parent 02d2708
commit 0b7d614
Show file tree

Hide file tree

Showing 15 changed files with 59 additions and 48 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,11 @@
+## v0.0.4
+### 2023-07-20
+
+This version of the Swath Projector updates dependencies used by the service,
+including converting to the use of `earthdata-varinfo` in place of
+`sds-varinfo`. The Python version of the conda environment is also updated to
+Python 3.11.
+
 ## v0.0.3
 ### 2022-12-20
 

diff --git a/doc/Swath Projector User Guide.ipynb b/doc/Swath Projector User Guide.ipynb
@@ -379,7 +379,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.13"
+   "version": "3.11.4"
   }
  },
  "nbformat": 4,

diff --git a/doc/requirements.txt b/doc/requirements.txt
@@ -1,7 +1,7 @@
 #
 # Requirements file for running documentation Jupyter notebook.
 # 1) Create a conda environment:
-#    $ conda create --name=swotrepr-jupyter python=3.10 --channel conda-forge --channel defaults
+#    $ conda create --name=swotrepr-jupyter python=3.11 --channel conda-forge --channel defaults
 # 2) Activate the environment:
 #    $ conda activate swotrepr-jupyter
 # 3) Install these Python packages:
@@ -10,7 +10,7 @@
 #    $ jupyter notebook.
 # 5) Click on the Swath Projector User Guide notebook.
 #
-harmony-py ~= 0.4.6
-matplotlib ~= 3.7.1
-netCDF4 ~= 1.6.3
-notebook ~= 6.5.3
+harmony-py ~= 0.4.9
+matplotlib ~= 3.7.2
+netCDF4 ~= 1.6.4
+notebook ~= 7.0.0
diff --git a/docker/service.Dockerfile b/docker/service.Dockerfile
@@ -10,6 +10,7 @@
 # created conda environment.
 #
 # Updated: 2021-07-15 - Change Python version from 3.7 to 3.9
+# Updated: 2023-07-20 - Update Python version to 3.11.
 #
 FROM continuumio/miniconda3
 
@@ -19,7 +20,7 @@ WORKDIR "/home"
 COPY pip_requirements.txt .
 
 # Create Conda environment
-RUN conda create -y --name swotrepr python=3.10 -q --channel conda-forge \
+RUN conda create -y --name swotrepr python=3.11 -q --channel conda-forge \
 	--channel defaults && conda clean --all --quiet --yes
 
 # Install additional Pip dependencies

diff --git a/docker/service_version.txt b/docker/service_version.txt
@@ -1 +1 @@
-0.0.3
+0.0.4
diff --git a/pip_requirements.txt b/pip_requirements.txt
@@ -1,11 +1,8 @@
---extra-index-url https://maven.earthdata.nasa.gov/repository/python-repo/simple
-# Packages available only from Maven
-sds-varinfo ~= 3.0.0
-
 # Open source packages available from PyPI:
-harmony-service-lib~=1.0.20
-netCDF4 ~= 1.5.8
-numpy ~= 1.22.4
-pyproj ~= 3.3.1
-pyresample ~= 1.24.1
+earthdata-varinfo ~= 1.0.0
+harmony-service-lib~=1.0.22
+netCDF4 ~= 1.6.4
+numpy ~= 1.24.2
+pyproj ~= 3.6.0
+pyresample ~= 1.27.1
 pystac ~= 0.5.6
diff --git a/pymods/reproject.py b/pymods/reproject.py
@@ -18,8 +18,9 @@
 CF_CONFIG_FILE = 'pymods/cf_config.json'
 
 
-def reproject(message: Message, granule_url: str, local_filename: str,
-              temp_dir: str, logger: logging.Logger) -> str:
+def reproject(message: Message, collection_short_name: str, granule_url: str,
+              local_filename: str, temp_dir: str,
+              logger: logging.Logger) -> str:
     """ Derive reprojection parameters from the input Harmony message. Then
         extract listing of science variables and coordinate variables from the
         source granule. Then reproject all science variables. Finally merge all
@@ -38,8 +39,9 @@ def reproject(message: Message, granule_url: str, local_filename: str,
                 f'Interpolation: {parameters.get("interpolation")}')
 
     try:
-        var_info = VarInfoFromNetCDF4(parameters['input_file'], logger,
-                                      CF_CONFIG_FILE)
+        var_info = VarInfoFromNetCDF4(parameters['input_file'],
+                                      short_name=collection_short_name,
+                                      config_file=CF_CONFIG_FILE)
     except Exception as err:
         logger.error(f'Unable to parse input file variables: {str(err)}')
         raise Exception('Unable to parse input file variables') from err

diff --git a/pymods/utilities.py b/pymods/utilities.py
@@ -83,7 +83,7 @@ def get_variable_numeric_fill_value(variable: Variable) -> FillValueType:
         fill_value = None
 
     if not isinstance(fill_value,
-                      (np.integer, np.long, np.floating, int, float)):
+                      (np.integer, np.longlong, np.floating, int, float)):
         fill_value = None
 
     if fill_value is not None:

diff --git a/swotrepr.py b/swotrepr.py
@@ -73,8 +73,9 @@ def process_item(self, item: pystac.Item, source: harmony.message.Source):
             logger.info('Granule data copied')
 
             # Call Reprojection utility
-            working_filename = reproject(self.message, granule_url,
-                                         input_filename, workdir, logger)
+            working_filename = reproject(self.message, source.shortName,
+                                         granule_url, input_filename, workdir,
+                                         logger)
 
             # Stage the output file with a conventional filename
             output_filename = generate_output_filename(asset.href, is_regridded=True)

diff --git a/test/test_input_file_download.py b/test/test_input_file_download.py
@@ -34,7 +34,9 @@ def test_local_file_does_not_exist(self):
         )
 
         with self.assertRaises(Exception) as context:
-            reproject(reprojector.message, 'https://example.com/no_such_file.nc4',
-                      'test/data/no_such_file', '/no/such/dir', reprojector.logger)
+            reproject(reprojector.message, 'short name',
+                      'https://example.com/no_such_file.nc4',
+                      'test/data/no_such_file', '/no/such/dir',
+                      reprojector.logger)
 
         self.assertEqual(str(context.exception), 'Input file does not exist')
diff --git a/test/unit/test_interpolation.py b/test/unit/test_interpolation.py
@@ -44,8 +44,9 @@ def setUp(self):
         }
         self.temp_directory = '/tmp/01234'
         self.logger = Logger('test')
-        self.var_info = VarInfoFromNetCDF4(self.message_parameters['input_file'], self.logger,
-                                           CF_CONFIG_FILE)
+        self.var_info = VarInfoFromNetCDF4(self.message_parameters['input_file'],
+                                           short_name='harmony_example_l2',
+                                           config_file=CF_CONFIG_FILE)
         self.mock_target_area = MagicMock(spec=AreaDefinition,
                                           shape='ta_shape',
                                           area_id='/lon, /lat')

diff --git a/test/unit/test_nc_merge.py b/test/unit/test_nc_merge.py
@@ -34,7 +34,8 @@ def setUpClass(cls):
 
         cls.metadata_variables = set()
         cls.var_info = VarInfoFromNetCDF4(cls.properties['input_file'],
-                                          cls.logger, CF_CONFIG_FILE)
+                                          short_name='VIIRS_NPP-NAVO-L2P-v3.0',
+                                          config_file=CF_CONFIG_FILE)
         create_output(cls.properties, cls.output_file, cls.tmp_dir,
                       cls.science_variables, cls.metadata_variables,
                       cls.logger, cls.var_info)

diff --git a/test/unit/test_nc_single_band.py b/test/unit/test_nc_single_band.py
@@ -40,6 +40,7 @@ def setUpClass(cls):
             'geographic_crs_name': 'unknown',
             'grid_mapping_name': 'latitude_longitude',
             'towgs84': (0, 0, 0.0, 0.0),
+            'horizontal_datum_name': 'World Geodetic System 1984',
         }
         cls.non_geographic_area = AreaDefinition.from_extent(
             'cea', '+proj=cea', (2, 4),

diff --git a/test/unit/test_swotrepr_geometry.py b/test/unit/test_swotrepr_geometry.py
@@ -43,10 +43,10 @@ def setUpClass(cls):
         cls.test_file = Dataset(cls.test_path, 'w')
         cls.test_file.createDimension('nj', size=3)
         cls.test_file.createDimension('ni', size=4)
-        cls.test_file.createVariable('lat', np.float, dimensions=('nj', 'ni'))
-        cls.test_file.createVariable('lon', np.float, dimensions=('nj', 'ni'))
-        cls.test_file.createVariable('lat_1d', np.float, dimensions=('ni',))
-        cls.test_file.createVariable('lon_1d', np.float, dimensions=('ni',))
+        cls.test_file.createVariable('lat', float, dimensions=('nj', 'ni'))
+        cls.test_file.createVariable('lon', float, dimensions=('nj', 'ni'))
+        cls.test_file.createVariable('lat_1d', float, dimensions=('ni',))
+        cls.test_file.createVariable('lon_1d', float, dimensions=('ni',))
         cls.test_file['lat'][:] = cls.lat_data
         cls.test_file['lon'][:] = cls.lon_data
         cls.test_file['lat_1d'][:] = np.array([0.0, 3.0, 6.0, 9.0])
@@ -286,9 +286,9 @@ def test_get_valid_coordinates_mask(self):
                 test_file = Dataset(test_path, 'w')
                 test_file.createDimension('nj', size=2)
                 test_file.createDimension('ni', size=2)
-                test_file.createVariable('lat', np.float, dimensions=('nj', 'ni'),
+                test_file.createVariable('lat', float, dimensions=('nj', 'ni'),
                                          fill_value=fill_value)
-                test_file.createVariable('lon', np.float, dimensions=('nj', 'ni'),
+                test_file.createVariable('lon', float, dimensions=('nj', 'ni'),
                                          fill_value=fill_value)
                 test_file['lat'][:] = lat_data
                 test_file['lon'][:] = lon_data

diff --git a/test/unit/test_utilities.py b/test/unit/test_utilities.py
@@ -20,12 +20,10 @@ class TestUtilities(TestBase):
     def test_create_coordinates_key(self):
         """ Extract the coordinates from a `VariableFromNetCDF4` instance and
             return an alphabetically sorted tuple. The ordering prevents any
-            shuffling due to sds-varinfo storing CF-Convention attribute
+            shuffling due to earthdata-varinfo storing CF-Convention attribute
             references as a Python Set.
 
         """
-        logger = getLogger('test')
-
         data = np.ones((2, 4))
         dimensions = ('lat', 'lon')
         expected_output = ('/lat', '/lon')
@@ -49,7 +47,7 @@ def test_create_coordinates_key(self):
 
                     nc4_variable.setncattr('coordinates', coordinates)
 
-                varinfo = VarInfoFromNetCDF4('test.nc', logger)
+                varinfo = VarInfoFromNetCDF4('test.nc')
                 varinfo_variable = varinfo.get_variable('/group/variable')
                 self.assertEqual(create_coordinates_key(varinfo_variable),
                                  expected_output)
@@ -153,18 +151,17 @@ def test_get_variable_numeric_fill_value(self):
         """
         variable = Mock(spec=Variable)
 
-        test_args = [['np.float', np.float, 4.0, 4.0],
-                     ['np.float128', np.float128, 4.0, 4.0],
+        test_args = [['np.float128', np.float128, 4.0, 4.0],
                      ['np.float16', np.float16, 4.0, 4.0],
                      ['np.float32', np.float32, 4.0, 4.0],
                      ['np.float64', np.float64, 4.0, 4.0],
                      ['np.float_', np.float_, 4.0, 4.0],
-                     ['np.int', np.int, 5, 5],
-                     ['np.int0', np.int, 5, 5],
-                     ['np.int16', np.int, 5, 5],
-                     ['np.int32', np.int, 5, 5],
-                     ['np.int64', np.int, 5, 5],
-                     ['np.int8', np.int, 5, 5],
+                     ['int', int, 5, 5],
+                     ['np.int0', np.int0, 5, 5],
+                     ['np.int16', np.int16, 5, 5],
+                     ['np.int32', np.int32, 5, 5],
+                     ['np.int64', np.int64, 5, 5],
+                     ['np.int8', np.int8, 5, 5],
                      ['np.uint', np.uint, 5, 5],
                      ['np.uint0', np.uint0, 5, 5],
                      ['np.uint16', np.uint16, 5, 5],
@@ -173,7 +170,7 @@ def test_get_variable_numeric_fill_value(self):
                      ['np.uint8', np.uint8, 5, 5],
                      ['np.uintc', np.uintc, 5, 5],
                      ['np.uintp', np.uintp, 5, 5],
-                     ['np.long', np.long, 5, 5],
+                     ['np.longlong', np.longlong, 5, 5],
                      ['float', float, 4.0, 4.0],
                      ['int', int, 5, 5],
                      ['str', str, '1235', None]]