diff --git a/CHANGELOG.md b/CHANGELOG.md index 037358d..d2fe83c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +## v1.0.2 +### 2024-2-26 + +This version of HOSS correctly handles edge-aligned geographic collections by +adding the attribute `cell_alignment` with the value `edge` to `hoss_config.json` +for edge-aligned collections (namely, ATL16), and by adding functions that +create pseudo bounds for edge-aligned collections to make HOSS use the +`dimension_utilities.py` function, `get_dimension_indices_from_bounds`. The +pseudo bounds are only used internally and are not returned in the HOSS subset. + +This change also includes an addition of a CF override that addresses an +issue with the ATL16 metadata for the variables `/spolar_asr_obs_grid` and +`/spolar_lorate_blowing_snow_freq` where their `grid_mapping` attribute points +to north polar variables instead of south polar variables. This CF Override +can be removed if/when the metadata is corrected. + ## v1.0.1 ### 2023-12-19 diff --git a/docker/service_version.txt b/docker/service_version.txt index 7dea76e..6d7de6e 100644 --- a/docker/service_version.txt +++ b/docker/service_version.txt @@ -1 +1 @@ -1.0.1 +1.0.2 diff --git a/hoss/dimension_utilities.py b/hoss/dimension_utilities.py index 2db6397..ab6b10a 100644 --- a/hoss/dimension_utilities.py +++ b/hoss/dimension_utilities.py @@ -12,6 +12,7 @@ from logging import Logger from typing import Dict, Set, Tuple +from pathlib import PurePosixPath from netCDF4 import Dataset from numpy.ma.core import MaskedArray import numpy as np @@ -19,7 +20,7 @@ from harmony.message import Message from harmony.message_utility import rgetattr from harmony.util import Config -from varinfo import VarInfoFromDmr +from varinfo import VarInfoFromDmr, VariableFromDmr from hoss.bbox_utilities import flatten_list from hoss.exceptions import InvalidNamedDimension, InvalidRequestedRange @@ -75,8 +76,148 @@ def prefetch_dimension_variables(opendap_url: str, varinfo: VarInfoFromDmr, logger.info('Variables being retrieved in prefetch request: ' f'{format_variable_set_string(required_dimensions)}') - return get_opendap_nc4(opendap_url, required_dimensions, output_dir, - logger, access_token, config) + + required_dimensions_nc4 = get_opendap_nc4(opendap_url, + required_dimensions, output_dir, + logger, access_token, config) + + # Create bounds variables if necessary. + add_bounds_variables(required_dimensions_nc4, required_dimensions, + varinfo, logger) + + return required_dimensions_nc4 + + +def add_bounds_variables(dimensions_nc4: str, + required_dimensions: Set[str], + varinfo: VarInfoFromDmr, + logger: Logger) -> None: + """ Augment a NetCDF4 file with artificial bounds variables for each + dimension variable that has been identified by the earthdata-varinfo + configuration file to have an edge-aligned attribute" + + For each dimension variable: + (1) Check if the variable needs a bounds variable. + (2) If so, create a bounds array from within the `write_bounds` + function. + (3) Then write the bounds variable to the NetCDF4 URL. + + """ + with Dataset(dimensions_nc4, 'r+') as prefetch_dataset: + for dimension_name in required_dimensions: + dimension_variable = varinfo.get_variable(dimension_name) + if needs_bounds(dimension_variable): + write_bounds(prefetch_dataset, dimension_variable) + + logger.info('Artificial bounds added for dimension variable: ' + f'{dimension_name}') + + +def needs_bounds(dimension: VariableFromDmr) -> bool: + """ Check if a dimension variable needs a bounds variable. + This will be the case when dimension cells are edge-aligned + and bounds for that dimension do not already exist. + + """ + return ( + dimension.attributes.get('cell_alignment') == 'edge' + and dimension.references.get('bounds') is None + ) + + +def get_bounds_array(prefetch_dataset: Dataset, + dimension_path: str) -> np.ndarray: + """ Create an array containing the minimum and maximum bounds + for each pixel in a given dimension. + + The minimum and maximum values are determined under the assumption + that the dimension data is monotonically increasing and contiguous. + So for every bounds but the last, the bounds are simply extracted + from the dimension dataset. + + The final bounds must be calculated with the assumption that + the last data cell is edge-aligned and thus has a value the does + not account for the cell length. So, the final bound is determined + by taking the median of all the resolutions in the dataset to obtain + a resolution that can be added to the final data value. + + Ex: Input dataset with resolution of 3 degrees: [ ... , 81, 84, 87] + + Minimum | Maximum + <...> <...> + 81 84 + 84 87 + 87 ? -> 87 + median resolution -> 87 + 3 -> 90 + + """ + # Access the dimension variable's data using the variable's full path. + dimension_array = prefetch_dataset[dimension_path][:] + + median_resolution = np.median(np.diff(dimension_array)) + + # This array is the transpose of what is required, just for easier assignment + # of values (indices are [row, column]) during the bounds calculations: + cell_bounds = np.zeros(shape=(2, dimension_array.size), dtype=dimension_array.dtype) + + # Minimum values are equal to the dimension pixel values (for lower left pixel alignment): + cell_bounds[0] = dimension_array[:] + + # Maximum values are the next dimension pixel values (for lower left pixel alignment), + # so these values almost mirror the minimum values but start at the second pixel + # instead of the first. Here we calculate each bound except for the very last one. + cell_bounds[1][:-1] = dimension_array[1:] + + # Last maximum value is the last pixel value (minimum) plus the median resolution: + cell_bounds[1][-1] = dimension_array[-1] + median_resolution + + # Return transpose of array to get correct shape: + return cell_bounds.T + + +def write_bounds(prefetch_dataset: Dataset, + dimension_variable: VariableFromDmr) -> None: + """ Write the input bounds array to a given dimension dataset. + + First a new dimension is created for the new bounds variable + to allow the variable to be two-dimensional. + + Then the new bounds variable is created using two dimensions: + (1) the existing dimension of the dimension dataset, and + (2) the new bounds variable dimension. + + """ + bounds_array = get_bounds_array(prefetch_dataset, + dimension_variable.full_name_path) + + # Create the second bounds dimension. + dimension_name = str(PurePosixPath(dimension_variable.full_name_path).name) + dimension_group = str(PurePosixPath(dimension_variable.full_name_path).parent) + bounds_name = dimension_name + '_bnds' + bounds_full_path_name = dimension_variable.full_name_path + '_bnds' + bounds_dimension_name = dimension_name + 'v' + + if dimension_group == '/': + # The root group must be explicitly referenced here. + bounds_dim = prefetch_dataset.createDimension(bounds_dimension_name, 2) + else: + bounds_dim = prefetch_dataset[dimension_group].createDimension(bounds_dimension_name, 2) + + # Dimension variables only have one dimension - themselves. + variable_dimension = prefetch_dataset[dimension_variable.full_name_path].dimensions[0] + + bounds_data_type = str(dimension_variable.data_type) + bounds = prefetch_dataset.createVariable(bounds_full_path_name, + bounds_data_type, + (variable_dimension, + bounds_dim,)) + + # Write data to the new variable in the prefetch dataset. + bounds[:] = bounds_array[:] + + # Update varinfo attributes and references. + prefetch_dataset[dimension_variable.full_name_path].setncatts({'bounds': bounds_name}) + dimension_variable.references['bounds'] = {bounds_name, } + dimension_variable.attributes['bounds'] = bounds_name def is_dimension_ascending(dimension: MaskedArray) -> bool: diff --git a/hoss/hoss_config.json b/hoss/hoss_config.json index d960a8f..cefe364 100644 --- a/hoss/hoss_config.json +++ b/hoss/hoss_config.json @@ -245,6 +245,20 @@ ], "_Description": "Ensure variables in /Soil_Moisture_Retrieval_Data_Polar_PM group point to correct coordinate variables." }, + { + "Applicability": { + "Mission": "ICESat2", + "ShortNamePath": "ATL16", + "Variable_Pattern": ".*_grid_(lat|lon)" + }, + "Attributes": [ + { + "Name": "cell_alignment", + "Value": "edge" + } + ], + "_Description": "ATL16 has edge-aligned grid cells." + }, { "Applicability": { "Mission": "ICESat2", @@ -357,6 +371,19 @@ } ], "_Description": "Ensure the latitude and longitude dimension variables know their associated grid_mapping variable." + }, + { + "Applicability": { + "Mission": "ICESat2", + "ShortNamePath": "ATL16", + "Variable_Pattern": "/spolar_(asr_obs_grid|lorate_blowing_snow_freq)" + }, + "Attributes": [ + { + "Name": "grid_mapping", + "Value": "crs_latlon: spolar_grid_lat crs_latlon: spolar_grid_lon" + } + ] } ], "CF_Supplements": [ diff --git a/tests/data/ATL16_prefetch.dmr b/tests/data/ATL16_prefetch.dmr new file mode 100644 index 0000000..7ddecd8 --- /dev/null +++ b/tests/data/ATL16_prefetch.dmr @@ -0,0 +1,225 @@ + + + + + + + + + + + + north polar grid longitude + + + longitude + + + degrees_east + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + -180. + + + 180. + + + referenceInformation + + + The north polar grid longitude one-dimensional array parameter, with the longitudes applicable to each north polar grid cell. ATL09 ATM histogram top longitude (longitude ()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: East=+ values. Range of longitude values: -180.0 to +180.0 degrees, with a step size of polar_grid_lon_scale. Grid array organization: npolar_grid_lon (i), where i=longitude index, as per ATBD Section 2.0. Given weekly polar_grid_lon_scale = 3.0 degrees, the weekly array dimension is: npolar_grid_lon (120), type: double precision. Comprises the x-grid axis for north polar gridded parameters. Reference: npolar_grid_lon (1) = -180.0 degrees, at the upper left corner of a north polar gridded parameter cell location (i,j) = (1,1). + + + X + + + + + + south polar grid latitude + + + latitude + + + degrees_north + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + -90. + + + -60. + + + referenceInformation + + + The south polar grid latitude one-dimensional array parameter, with the latitudes applicable to each south polar grid cell. ATL09 ATM histogram top latitude (latitude ()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: South=- values. Range of latitude values: -90.0 up to -60.0 degrees, with a step size of polar_grid_lat_scale. Grid array organization: spolar_grid_lat (j), where j=latitude index, as per ATBD Section 2.0. Given weekly polar_grid_lat_scale = 1.0 degrees, the weekly array dimension is: spolar_grid_lat (30), type: double precision. Comprises the y-grid axis for south polar gridded parameters. Reference: spolar_grid_lat (1) = -90.0 degrees, at the lower left corner of a south polar gridded parameter cell location (i,j) = (1,1). + + + Y + + + + + + global grid longitude + + + longitude + + + degrees_east + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + -180. + + + 180. + + + referenceInformation + + + The global grid longitude one-dimensional array parameter, with the longitudes applicable to each global grid cell. The ATL09 ATM histogram top longitude (longitude()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: East=+ values. Range of longitude values: -180.0 to +180.0 degrees, with a step size of global_grid_lon_scale. Grid array organization: global_grid_lon (i), where i=longitude index, as per ATBD Section 2.0. Given weekly global_grid_lon_scale = 3.0 degrees, array dimension: global_grid_lon (120), type: double precision. Comprises the x-grid axis for global gridded parameters. Reference: global_grid_lon (1) = -180.0 degrees, at the lower left corner of a global gridded parameter cell location (i,j) = (1,1). + + + X + + + + + + global grid latitude + + + latitude + + + degrees_north + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + -90. + + + 90. + + + referenceInformation + + + The global grid latitude one-dimensional array parameter, with the latitudes applicable to each global grid cell. ATL09 ATM histogram top latitude (latitude ()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: North=+ values. Range of latitude values: -90.0 to +90.0 degrees, with a step size of global_grid_lat_scale. Grid array organization: global_grid_lat (j), where j=latitude index, as per ATBD Section 2.0. Given weekly global_grid_lat_scale = 3.0 degrees, array dimension: global_grid_lat (60), type: double precision. Comprises the y-grid axis for global gridded parameters. Reference: global_grid_lat (1) = -90.0 degrees, at the lower left corner of a global gridded parameter cell location (i,j) = (1,1). + + + Y + + + + + + south polar grid longitude + + + longitude + + + degrees_east + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + -180. + + + 180. + + + referenceInformation + + + The south polar grid longitude one-dimensional array parameter, with the longitudes applicable to each south polar grid cell. ATL09 ATM histogram top longitude (longitude ()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: East=+ values. Range of longitude values: -180.0 to +180.0 degrees, with a step size of polar_grid_lon_scale. Grid array organization: spolar_grid_lon (i), where i=longitude index, as per ATBD Section 2.0. Given weekly polar_grid_lon_scale = 3.0 degrees, the weekly array dimension is: spolar_grid_lon (120), type: double precision. Comprises the x-grid axis for south polar gridded parameters. Reference: spolar_grid_lon (1) = -180.0 degrees, at the lower left corner of a south polar gridded parameter cell location (i,j) = (1,1). + + + X + + + + + + north polar grid latitude + + + latitude + + + degrees_north + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + 60. + + + 90. + + + referenceInformation + + + The north polar grid latitude one-dimensional array parameter, with the latitudes applicable to each north polar grid cell. ATL09 ATM histogram top latitude (latitude ()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: North=+ values. Range of latitude values: +90.0 down to +60.0 degrees, with a step size of polar_grid_lat_scale. Grid array organization: npolar_grid_lat (j), where j=latitude index, as per ATBD Section 2.0. Given weekly polar_grid_lat_scale = 1.0 degrees, the weekly array dimension is: npolar_grid_lat (30), type: double precision. Comprises the y-grid axis for north polar gridded parameters. Reference: npolar_grid_lat (1) = +90.0 degrees, at the upper left corner of a north polar gridded parameter cell location (i,j) = (1,1). + + + Y + + + + + + + + global apparent surface reflectance observation grid + + + + 1 + + + + L3B ATM ATBD, Section 3.9, Table 4., Section 5.0, Table 5. + + + + modelResult + + + + The global apparent surface reflectance (ASR) observation count two-dimensional gridded parameter. The number of observations used to compute the average global apparent surface reflectance (ASR). Only surface signal detected ASR 25 Hz (high-rate profile) observations (apparent_surf_reflec () > 0.0) in each grid cell are used to compute the cell global ASR (global_asr (i,j)). Grid array organization: global_asr_obs_grid (i,j), where i=longitude index, j=latitude index, as per ATBD Section 2.0. Given the weekly global_grid_lon_scale = 3.0 degrees and the global_grid_lat_scale = 3.0 degrees, the weekly global gridded array dimension is: global_asr_obs_grid (120,60), type: floating point . Reference: global_asr_obs_grid (1,1) = (-180.0 degrees longitude, -90.0 degrees latitude), representing the lower left corner of the gridded parameter cell in the global projection array. The global observation counts are applicable to the atmosphere gridded parameter: global_asr (i,j). + + + + global_grid_lat global_grid_lon + + + + crs_latlon: global_grid_lat crs_latlon: global_grid_lon + + + + + ATL16 + + \ No newline at end of file diff --git a/tests/data/ATL16_prefetch.nc4 b/tests/data/ATL16_prefetch.nc4 new file mode 100644 index 0000000..28b1054 Binary files /dev/null and b/tests/data/ATL16_prefetch.nc4 differ diff --git a/tests/data/ATL16_prefetch_bnds.dmr b/tests/data/ATL16_prefetch_bnds.dmr new file mode 100644 index 0000000..d48b6a5 --- /dev/null +++ b/tests/data/ATL16_prefetch_bnds.dmr @@ -0,0 +1,220 @@ + + + + + + + + + + + + + north polar grid longitude + + + longitude + + + degrees_east + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + -180. + + + 180. + + + referenceInformation + + + The north polar grid longitude one-dimensional array parameter, with the longitudes applicable to each north polar grid cell. ATL09 ATM histogram top longitude (longitude ()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: East=+ values. Range of longitude values: -180.0 to +180.0 degrees, with a step size of polar_grid_lon_scale. Grid array organization: npolar_grid_lon (i), where i=longitude index, as per ATBD Section 2.0. Given weekly polar_grid_lon_scale = 3.0 degrees, the weekly array dimension is: npolar_grid_lon (120), type: double precision. Comprises the x-grid axis for north polar gridded parameters. Reference: npolar_grid_lon (1) = -180.0 degrees, at the upper left corner of a north polar gridded parameter cell location (i,j) = (1,1). + + + X + + + + + + south polar grid latitude + + + latitude + + + degrees_north + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + -90. + + + -60. + + + referenceInformation + + + The south polar grid latitude one-dimensional array parameter, with the latitudes applicable to each south polar grid cell. ATL09 ATM histogram top latitude (latitude ()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: South=- values. Range of latitude values: -90.0 up to -60.0 degrees, with a step size of polar_grid_lat_scale. Grid array organization: spolar_grid_lat (j), where j=latitude index, as per ATBD Section 2.0. Given weekly polar_grid_lat_scale = 1.0 degrees, the weekly array dimension is: spolar_grid_lat (30), type: double precision. Comprises the y-grid axis for south polar gridded parameters. Reference: spolar_grid_lat (1) = -90.0 degrees, at the lower left corner of a south polar gridded parameter cell location (i,j) = (1,1). + + + Y + + + + + + global grid longitude + + + longitude + + + degrees_east + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + -180. + + + 180. + + + referenceInformation + + + The global grid longitude one-dimensional array parameter, with the longitudes applicable to each global grid cell. The ATL09 ATM histogram top longitude (longitude()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: East=+ values. Range of longitude values: -180.0 to +180.0 degrees, with a step size of global_grid_lon_scale. Grid array organization: global_grid_lon (i), where i=longitude index, as per ATBD Section 2.0. Given weekly global_grid_lon_scale = 3.0 degrees, array dimension: global_grid_lon (120), type: double precision. Comprises the x-grid axis for global gridded parameters. Reference: global_grid_lon (1) = -180.0 degrees, at the lower left corner of a global gridded parameter cell location (i,j) = (1,1). + + + X + + + + + + global grid latitude + + + latitude + + + degrees_north + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + -90. + + + 90. + + + referenceInformation + + + The global grid latitude one-dimensional array parameter, with the latitudes applicable to each global grid cell. ATL09 ATM histogram top latitude (latitude ()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: North=+ values. Range of latitude values: -90.0 to +90.0 degrees, with a step size of global_grid_lat_scale. Grid array organization: global_grid_lat (j), where j=latitude index, as per ATBD Section 2.0. Given weekly global_grid_lat_scale = 3.0 degrees, array dimension: global_grid_lat (60), type: double precision. Comprises the y-grid axis for global gridded parameters. Reference: global_grid_lat (1) = -90.0 degrees, at the lower left corner of a global gridded parameter cell location (i,j) = (1,1). + + + Y + + + + + + south polar grid longitude + + + longitude + + + degrees_east + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + -180. + + + 180. + + + referenceInformation + + + The south polar grid longitude one-dimensional array parameter, with the longitudes applicable to each south polar grid cell. ATL09 ATM histogram top longitude (longitude ()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: East=+ values. Range of longitude values: -180.0 to +180.0 degrees, with a step size of polar_grid_lon_scale. Grid array organization: spolar_grid_lon (i), where i=longitude index, as per ATBD Section 2.0. Given weekly polar_grid_lon_scale = 3.0 degrees, the weekly array dimension is: spolar_grid_lon (120), type: double precision. Comprises the x-grid axis for south polar gridded parameters. Reference: spolar_grid_lon (1) = -180.0 degrees, at the lower left corner of a south polar gridded parameter cell location (i,j) = (1,1). + + + X + + + + + + north polar grid latitude + + + latitude + + + degrees_north + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + 60. + + + 90. + + + referenceInformation + + + The north polar grid latitude one-dimensional array parameter, with the latitudes applicable to each north polar grid cell. ATL09 ATM histogram top latitude (latitude ()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: North=+ values. Range of latitude values: +90.0 down to +60.0 degrees, with a step size of polar_grid_lat_scale. Grid array organization: npolar_grid_lat (j), where j=latitude index, as per ATBD Section 2.0. Given weekly polar_grid_lat_scale = 1.0 degrees, the weekly array dimension is: npolar_grid_lat (30), type: double precision. Comprises the y-grid axis for north polar gridded parameters. Reference: npolar_grid_lat (1) = +90.0 degrees, at the upper left corner of a north polar gridded parameter cell location (i,j) = (1,1). + + + Y + + + + + + + + + bnds_variable + + + edge + + + + + + bnds_variable + + + + + + edge + + + + + + + ATL16 + + \ No newline at end of file diff --git a/tests/data/ATL16_prefetch_group.dmr b/tests/data/ATL16_prefetch_group.dmr new file mode 100644 index 0000000..c200956 --- /dev/null +++ b/tests/data/ATL16_prefetch_group.dmr @@ -0,0 +1,219 @@ + + + + + + + + zelda the good dog + + + zelda dog + + + Unknown. + + + 10 inches + + + 9 pounds + + + 917-years-old + + + Zelda the Good Dog hails from unknown regions of the world. While her age was once estimated to be around 7-years-old by a veterinary professional, it is believed that this estimate was made under intimidation by the dog herself in order to conceal the true nature of her being: an ancient omniscient sage. + + + + + + + + + + + + + north polar grid longitude + + + longitude + + + degrees_east + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + -180. + + + 180. + + + referenceInformation + + + The north polar grid longitude one-dimensional array parameter, with the longitudes applicable to each north polar grid cell. ATL09 ATM histogram top longitude (longitude ()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: East=+ values. Range of longitude values: -180.0 to +180.0 degrees, with a step size of polar_grid_lon_scale. Grid array organization: npolar_grid_lon (i), where i=longitude index, as per ATBD Section 2.0. Given weekly polar_grid_lon_scale = 3.0 degrees, the weekly array dimension is: npolar_grid_lon (120), type: double precision. Comprises the x-grid axis for north polar gridded parameters. Reference: npolar_grid_lon (1) = -180.0 degrees, at the upper left corner of a north polar gridded parameter cell location (i,j) = (1,1). + + + X + + + + + + south polar grid latitude + + + latitude + + + degrees_north + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + -90. + + + -60. + + + referenceInformation + + + The south polar grid latitude one-dimensional array parameter, with the latitudes applicable to each south polar grid cell. ATL09 ATM histogram top latitude (latitude ()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: South=- values. Range of latitude values: -90.0 up to -60.0 degrees, with a step size of polar_grid_lat_scale. Grid array organization: spolar_grid_lat (j), where j=latitude index, as per ATBD Section 2.0. Given weekly polar_grid_lat_scale = 1.0 degrees, the weekly array dimension is: spolar_grid_lat (30), type: double precision. Comprises the y-grid axis for south polar gridded parameters. Reference: spolar_grid_lat (1) = -90.0 degrees, at the lower left corner of a south polar gridded parameter cell location (i,j) = (1,1). + + + Y + + + + + + global grid longitude + + + longitude + + + degrees_east + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + -180. + + + 180. + + + referenceInformation + + + The global grid longitude one-dimensional array parameter, with the longitudes applicable to each global grid cell. The ATL09 ATM histogram top longitude (longitude()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: East=+ values. Range of longitude values: -180.0 to +180.0 degrees, with a step size of global_grid_lon_scale. Grid array organization: global_grid_lon (i), where i=longitude index, as per ATBD Section 2.0. Given weekly global_grid_lon_scale = 3.0 degrees, array dimension: global_grid_lon (120), type: double precision. Comprises the x-grid axis for global gridded parameters. Reference: global_grid_lon (1) = -180.0 degrees, at the lower left corner of a global gridded parameter cell location (i,j) = (1,1). + + + X + + + + + + global grid latitude + + + latitude + + + degrees_north + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + -90. + + + 90. + + + referenceInformation + + + The global grid latitude one-dimensional array parameter, with the latitudes applicable to each global grid cell. ATL09 ATM histogram top latitude (latitude ()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: North=+ values. Range of latitude values: -90.0 to +90.0 degrees, with a step size of global_grid_lat_scale. Grid array organization: global_grid_lat (j), where j=latitude index, as per ATBD Section 2.0. Given weekly global_grid_lat_scale = 3.0 degrees, array dimension: global_grid_lat (60), type: double precision. Comprises the y-grid axis for global gridded parameters. Reference: global_grid_lat (1) = -90.0 degrees, at the lower left corner of a global gridded parameter cell location (i,j) = (1,1). + + + Y + + + + + + south polar grid longitude + + + longitude + + + degrees_east + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + -180. + + + 180. + + + referenceInformation + + + The south polar grid longitude one-dimensional array parameter, with the longitudes applicable to each south polar grid cell. ATL09 ATM histogram top longitude (longitude ()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: East=+ values. Range of longitude values: -180.0 to +180.0 degrees, with a step size of polar_grid_lon_scale. Grid array organization: spolar_grid_lon (i), where i=longitude index, as per ATBD Section 2.0. Given weekly polar_grid_lon_scale = 3.0 degrees, the weekly array dimension is: spolar_grid_lon (120), type: double precision. Comprises the x-grid axis for south polar gridded parameters. Reference: spolar_grid_lon (1) = -180.0 degrees, at the lower left corner of a south polar gridded parameter cell location (i,j) = (1,1). + + + X + + + + + + north polar grid latitude + + + latitude + + + degrees_north + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + 60. + + + 90. + + + referenceInformation + + + The north polar grid latitude one-dimensional array parameter, with the latitudes applicable to each north polar grid cell. ATL09 ATM histogram top latitude (latitude ()) derived from the ATM range window geolocation. Based on WGS84 Earth-centered, Earth-fixed terrestrial reference system and geodetic data. Direction: North=+ values. Range of latitude values: +90.0 down to +60.0 degrees, with a step size of polar_grid_lat_scale. Grid array organization: npolar_grid_lat (j), where j=latitude index, as per ATBD Section 2.0. Given weekly polar_grid_lat_scale = 1.0 degrees, the weekly array dimension is: npolar_grid_lat (30), type: double precision. Comprises the y-grid axis for north polar gridded parameters. Reference: npolar_grid_lat (1) = +90.0 degrees, at the upper left corner of a north polar gridded parameter cell location (i,j) = (1,1). + + + Y + + + + ATL16 + + \ No newline at end of file diff --git a/tests/data/ATL16_prefetch_group.nc4 b/tests/data/ATL16_prefetch_group.nc4 new file mode 100644 index 0000000..83772d8 Binary files /dev/null and b/tests/data/ATL16_prefetch_group.nc4 differ diff --git a/tests/data/ATL16_variables.nc4 b/tests/data/ATL16_variables.nc4 new file mode 100644 index 0000000..e6eee6e Binary files /dev/null and b/tests/data/ATL16_variables.nc4 differ diff --git a/tests/data/README.md b/tests/data/README.md index db2610e..d06a301 100644 --- a/tests/data/README.md +++ b/tests/data/README.md @@ -1,65 +1,94 @@ # Listing of test files -* ABoVE_TVPRM_bbox.nc4 - This is an example output from a bounding box request - to the ABoVE TVPRM collection, where -160 ≤ longitude (degrees east) ≤ -145, - 68 ≤ latitude (degrees north) ≤ 70. Note, ABoVE TVPRM has 8784 time slices. - To minimise the size of the stored artefact in Git, this example output only - contains the first 10 time slices. ABoVE TVPRM has a projected grid that uses - an Albers Conical Equal Area CRS. -* ABoVE_TVPRM_example.dmr - An example `.dmr` file for the ABoVE TVPRM - collection, as obtained from OPeNDAP. -* ABoVE_TVPRM_prefetch.nc4 - An example dimension prefetch output from OPeNDAP - for the ABoVE TVPRM collection. This contains the `time`, `x` and `y` - variables. -* ATL03_example.dmr - An example `.dmr` file from the ICESat-2/ATL03 collection, - as obtained from OPeNDAP. ATL03 is a trajectory data set and should only be - used (currently) with the variable subsetting operations of the service. -* GPM_3IMERGHH_bounds.nc4 - An example output from a bounding box and temporal - subset request for a collection containing bounds variables. The bounding box - is -30 ≤ longitude (degrees east) ≤ -15, 45 ≤ latitude (degrees north) ≤ 60. -* GPM_3IMERGHH_example.dmr - An example `.dmr` file for the GPM/3IMERGHH - collection, as obtained from OPeNDAP. GPM/3IMERGHH has a half-hourly time - dimension to the grid. It also contains bounds variable references. -* GPM_3IMERGHH_prefetch.nc4 - An example dimension prefetch output from OPeNDAP - for the GPM/IMERGHH colleciton. This contains the `/Grid/time`, `/Grid/lat`, - and `/Grid/lon` dimension variables along with their associated bounds - variables. -* M2T1NXSLV_example.dmr` - An example `.dmr` file for a MERRA-2 collection. - Granules in this collection are geographically gridded and contain a time - dimension that has half-hour resolution. -* M2T1NXSLV_geo_temporal.nc4 - Example output for a MERRA-2 collection. This - output is for both a spatial (bounding box) and temporal subset. -* M2T1NXSLV_prefetch.nc4 - An example dimension prefetch output from OPeNDAP - for a MERRA-2 collection. This contains a longitude, latitude and temporal - dimension. -* M2T1NXSLV_temporal.nc4 - An example output for a MERRA-2 collection, with a - request to OPeNDAP for a temporal subset. -* f16_ssmis_20200102v7.nc - An input granule for the RSSMIF16D collection. The - variables in this collection a 3-dimensional, gridded with a latitude and - longitude dimension and a single element time dimension. RSSMIF16D also has - 0 ≤ longitude (degrees east) ≤ 360. -* f16_ssmis_filled.nc - The output from a spatial subset request when the - requested bounding box crosses the Prime Meridian (and therefore the edge of - the grid). -* f16_ssmis_geo.nc - The output from a spatial and variable subset request with - a bounding box. -* f16_ssmis_geo_desc.nc - The output from a spatial and variable subset request - with a bounding box input. The latitude dimension is also descending in this - example, unlike the native ascending ordering. -* f16_ssmis_geo_no_vars.nc - The results of a spatial subset only with a - bounding box. -* f16_ssmis_lat_lon.nc - The output for a prefetch request that retrieves only - the latitude and longitude variables for the RSSMIF16D collection. This - sample predates the temporal subsetting work, and so does not also include - the temporal dimension variable. -* f16_ssmis_lat_lon_desc.nc - The output for a prefetch request that retrieves - only the latitude and longitude variables for the RSSMIF16D collection. This - differs from the previous sample file, as the latitude dimension is descending - in this file. -* f16_ssmis_unfilled.nc - The sample output for a request to OPeNDAP when the - longitude range crosses the edge of the bounding box. In this case, the data - retrieved from OPeNDAP will be a band in latitude, but cover the full - longitudinal range. HOSS will later fill the required region of that band - before returning the result to the end-user. -* rssmif16d_example.dmr - An example `.dmr` file as retrieved from OPeNDAP for - the RSSMIF16D collection. +* ABoVE_TVPRM_bbox.nc4 + - This is an example output from a bounding box request to the ABoVE + TVPRM collection, where -160 ≤ longitude (degrees east) ≤ -145, 68 ≤ latitude + (degrees north) ≤ 70. Note, ABoVE TVPRM has 8784 time slices. To minimise the + size of the stored artefact in Git, this example output only contains the first + 10 time slices. ABoVE TVPRM has a projected grid that uses an Albers Conical + Equal Area CRS. +* ABoVE_TVPRM_example.dmr + - An example `.dmr` file for the ABoVE TVPRM collection, as obtained from OPeNDAP +* ABoVE_TVPRM_prefetch.nc4 + - An example dimension prefetch output from OPeNDAP for the ABoVE TVPRM collection. + This contains the `time`, `x` and `y` variables +* ATL03_example.dmr + - An example `.dmr` file from the ICESat-2/ATL03 collection, as obtained from + OPeNDAP. ATL03 is a trajectory data set and should only be used (currently) + with the variable subsetting operations of the service +* GPM_3IMERGHH_bounds.nc4 + - An example output from a bounding box and temporal subset request for + a collection containing bounds variables. The bounding box is -30 ≤ longitude + (degrees east) ≤ -15, 45 ≤ latitude (degrees north) ≤ 60 +* GPM_3IMERGHH_example.dmr + - An example `.dmr` file for the GPM/3IMERGHH collection, as obtained from + OPeNDAP. GPM/3IMERGHH has a half-hourly time dimension to the grid. It also + contains bounds variable references +* GPM_3IMERGHH_prefetch.nc4 + - An example dimension prefetch output from OPeNDAP for the GPM/IMERGHH + collection. This contains the `/Grid/time`, `/Grid/lat`, and `/Grid/lon` + dimension variables along with their associated bounds variables +* M2T1NXSLV_example.dmr + - An example `.dmr` file for a MERRA-2 collection. Granules in this collection + are geographically gridded and contain a time dimension that has half-hour + resolution +* M2T1NXSLV_geo_temporal.nc4 + - Example output for a MERRA-2 collection. This output is for both a spatial + (bounding box) and temporal subset +* M2T1NXSLV_prefetch.nc4 + - An example dimension prefetch output from OPeNDAP for a MERRA-2 collection. + This contains a longitude, latitude and temporal dimension +* M2T1NXSLV_temporal.nc4 + - An example output for a MERRA-2 collection, with a request to OPeNDAP for + a temporal subset +* f16_ssmis_20200102v7.nc + - An input granule for the RSSMIF16D collection. The variables in this + collection a 3-dimensional, gridded with a latitude and longitude dimension + and a single element time dimension. RSSMIF16D also has 0 ≤ longitude + (degrees east) ≤ 360 +* f16_ssmis_filled.nc + - The output from a spatial subset request when the requested bounding box + crosses the Prime Meridian (and therefore the edge of the grid) +* f16_ssmis_geo.nc + - The output from a spatial and variable subset request with a bounding box +* f16_ssmis_geo_desc.nc + - The output from a spatial and variable subset request with a bounding box + input. The latitude dimension is also descending in this example, unlike + the native ascending ordering +* f16_ssmis_geo_no_vars.nc + - The results of a spatial subset only with a bounding box +* f16_ssmis_lat_lon.nc + - The output for a prefetch request that retrieves only the latitude and + longitude variables for the RSSMIF16D collection. This sample predates + the temporal subsetting work, and so does not also include the temporal + dimension variable +* f16_ssmis_lat_lon_desc.nc + - The output for a prefetch request that retrieves only the latitude and + longitude variables for the RSSMIF16D collection. This differs from the + previous sample file, as the latitude dimension is descending in this file +* f16_ssmis_unfilled.nc + - The sample output for a request to OPeNDAP when the longitude range crosses + the edge of the bounding box. In this case, the data retrieved from OPeNDAP + will be a band in latitude, but cover the full longitudinal range. HOSS + will later fill the required region of that band before returning the result + to the end-user +* rssmif16d_example.dmr + - An example `.dmr` file as retrieved from OPeNDAP for the RSSMIF16D collection +* ATL16_prefetch.dmr + - An example `.dmr` file retrieved from OPeNDAP for the ATL16 collection, but whittled + down to only contain the six required dimension variables. +* ATL16_prefetch.nc4 + - A sample output file that contains the six required dimension variables and one + requested science variable in the ATL16 collection. +* ATL16_prefetch_group.dmr + - An example `.dmr` file that is identical to the `ATL16_prefetch.dmr` file + except for an additional fabricated nested group variable, whereas all the variables in the + ATL16 collection are in the root directory. +* ATL16_prefetch_group.nc4 + - A sample output file that is nearly identical to the `ATL16_prefetch.nc4` file except + for an additional fabricated nested group variable (the same one in the + ATL16_prefetch_group.dmr file). +* ATL16_prefetch_bnds.dmr + - An example `.dmr` file that is nearly identical to the `ATL16_prefetch.dmr` file + except for four additional fabricated variables that represented the four + possible cases of combining bounds variable existence and cell alignment. \ No newline at end of file diff --git a/tests/test_adapter.py b/tests/test_adapter.py index fed0da0..6ceff4c 100755 --- a/tests/test_adapter.py +++ b/tests/test_adapter.py @@ -32,6 +32,7 @@ def setUpClass(cls): cls.atl03_variable = '/gt1r/geophys_corr/geoid' cls.gpm_variable = '/Grid/precipitationCal' cls.rssmif16d_variable = '/wind_speed' + cls.atl16_variable = '/global_asr_obs_grid' cls.staging_location = 's3://example-bucket/' with open('tests/data/ATL03_example.dmr', 'r') as file_handler: @@ -46,6 +47,9 @@ def setUpClass(cls): with open('tests/data/GPM_3IMERGHH_example.dmr', 'r') as file_handler: cls.gpm_imerghh_dmr = file_handler.read() + with open('tests/data/ATL16_prefetch.dmr', 'r') as file_handler: + cls.atl16_dmr = file_handler.read() + def setUp(self): """ Have to mock mkdtemp, to know where to put mock .dmr content. """ self.tmp_dir = mkdtemp() @@ -84,7 +88,7 @@ def assert_expected_output_catalog(self, catalog: Catalog, self.assertListEqual(list(items[0].assets.keys()), ['data']) self.assertDictEqual( items[0].assets['data'].to_dict(), - {'href': expected_href, + {'href': expected_href, 'title': expected_title, 'type': 'application/x-netcdf4', 'roles': ['data']} @@ -2008,3 +2012,112 @@ def test_exception_handling(self, mock_stage, mock_download_subset, mock_stage.assert_not_called() mock_rmtree.assert_called_once_with(self.tmp_dir) + + @patch('hoss.dimension_utilities.get_fill_slice') + @patch('hoss.utilities.uuid4') + @patch('hoss.adapter.mkdtemp') + @patch('shutil.rmtree') + @patch('hoss.utilities.util_download') + @patch('hoss.adapter.stage') + def test_edge_aligned_no_bounds_end_to_end(self, mock_stage, + mock_util_download, + mock_rmtree, mock_mkdtemp, + mock_uuid, + mock_get_fill_slice): + """ Ensure a request for a collection that contains dimension variables + with edge-aligned grid cells is correctly processed regardless of + whether or not a bounds variable associated with that dimension + variable exists. + + """ + expected_output_basename = 'opendap_url_global_asr_obs_grid_subsetted.nc4' + expected_staged_url = f'{self.staging_location}{expected_output_basename}' + + mock_uuid.side_effect = [Mock(hex='uuid'), Mock(hex='uuid2')] + mock_mkdtemp.return_value = self.tmp_dir + mock_stage.return_value = expected_staged_url + + dmr_path = write_dmr(self.tmp_dir, self.atl16_dmr) + + dimensions_path = f'{self.tmp_dir}/dimensions.nc4' + copy('tests/data/ATL16_prefetch.nc4', dimensions_path) + + all_variables_path = f'{self.tmp_dir}/variables.nc4' + copy('tests/data/ATL16_variables.nc4', all_variables_path) + + mock_util_download.side_effect = [dmr_path, dimensions_path, + all_variables_path] + + message = Message({ + 'accessToken': 'fake-token', + 'callback': 'https://example.com/', + 'sources': [{ + 'collection': 'C1238589498-EEDTEST', + 'shortName': 'ATL16', + 'variables': [{'id': '', + 'name': self.atl16_variable, + 'fullPath': self.atl16_variable}]}], + 'stagingLocation': self.staging_location, + 'subset': {'bbox': [77, 71.25, 88, 74.75]}, + 'user': 'sride', + }) + + hoss = HossAdapter(message, config=config(False), catalog=self.input_stac) + _, output_catalog = hoss.invoke() + + # Ensure that there is a single item in the output catalog with the + # expected asset: + self.assert_expected_output_catalog(output_catalog, + expected_staged_url, + expected_output_basename) + + # Ensure the expected requests were made against OPeNDAP. + self.assertEqual(mock_util_download.call_count, 3) + mock_util_download.assert_has_calls([ + call(f'{self.granule_url}.dmr.xml', self.tmp_dir, hoss.logger, + access_token=message.accessToken, data=None, cfg=hoss.config), + call(f'{self.granule_url}.dap.nc4', self.tmp_dir, hoss.logger, + access_token=message.accessToken, data=ANY, cfg=hoss.config), + call(f'{self.granule_url}.dap.nc4', self.tmp_dir, hoss.logger, + access_token=message.accessToken, data=ANY, cfg=hoss.config), + ]) + + # Ensure the constraint expression for dimensions data included only + # dimension variables and their associated bounds variables. + dimensions_data = mock_util_download.call_args_list[1][1].get('data', {}) + self.assert_valid_request_data( + dimensions_data, + {'%2Fglobal_grid_lat', + '%2Fglobal_grid_lon'} + ) + + # Ensure the constraint expression contains all the required variables. + # The latitude and longitude index ranges here depend on whether + # the cells have centre-alignment or edge-alignment. + # Previously, the incorrect index ranges assuming centre-alignment: + # latitude [54:55] with values (72,75) + # longitude [86:89] with values (78,81,84,87) + # + # Now, the correct index ranges with edge-alignment: + # latitude: [53:54] for values (69,72). + # longitude:[85:89] for values (75,78,81,84,87) + # + index_range_data = mock_util_download.call_args_list[2][1].get('data', {}) + self.assert_valid_request_data( + index_range_data, + {'%2Fglobal_asr_obs_grid%5B53%3A54%5D%5B85%3A89%5D', + '%2Fglobal_grid_lat%5B53%3A54%5D', + '%2Fglobal_grid_lon%5B85%3A89%5D'} + ) + + # Ensure the output was staged with the expected file name + mock_stage.assert_called_once_with(f'{self.tmp_dir}/uuid2.nc4', + expected_output_basename, + 'application/x-netcdf4', + location=self.staging_location, + logger=hoss.logger) + + mock_rmtree.assert_called_once_with(self.tmp_dir) + + # Ensure no variables were filled + mock_get_fill_slice.assert_not_called() diff --git a/tests/unit/test_dimension_utilities.py b/tests/unit/test_dimension_utilities.py index 370fb74..e4fda16 100644 --- a/tests/unit/test_dimension_utilities.py +++ b/tests/unit/test_dimension_utilities.py @@ -7,6 +7,7 @@ from harmony.util import config from harmony.message import Message +from pathlib import PurePosixPath from netCDF4 import Dataset from numpy.ma import masked_array from numpy.testing import assert_array_equal @@ -22,7 +23,11 @@ get_requested_index_ranges, is_almost_in, is_dimension_ascending, is_index_subset, - prefetch_dimension_variables) + prefetch_dimension_variables, + add_bounds_variables, + needs_bounds, + get_bounds_array, + write_bounds) from hoss.exceptions import InvalidNamedDimension, InvalidRequestedRange @@ -45,6 +50,16 @@ def setUpClass(cls): cls.varinfo_with_bounds = VarInfoFromDmr( 'tests/data/GPM_3IMERGHH_example.dmr' ) + cls.bounds_array = np.array([ + [90.0, 89.0], [89.0, 88.0], [88.0, 87.0], [87.0, 86.0], + [86.0, 85.0], [85.0, 84.0], [84.0, 83.0], [83.0, 82.0], + [82.0, 81.0], [81.0, 80.0], [80.0, 79.0], [79.0, 78.0], + [78.0, 77.0], [77.0, 76.0], [76.0, 75.0], [75.0, 74.0], + [74.0, 73.0], [73.0, 72.0], [72.0, 71.0], [71.0, 70.0], + [70.0, 69.0], [69.0, 68.0], [68.0, 67.0], [67.0, 66.0], + [66.0, 65.0], [65.0, 64.0], [64.0, 63.0], [63.0, 62.0], + [62.0, 61.0], [61.0, 60.0] + ]) def setUp(self): """ Create fixtures that should be unique per test. """ @@ -225,8 +240,7 @@ def test_get_dimension_indices_from_indices(self): def test_add_index_range(self): """ Ensure the correct combinations of index ranges are added as - suffixes to the input variable based upon that variable's - dimensions. + suffixes to the input variable based upon that variable's dimensions. If a dimension range has the lower index > upper index, that indicates the bounding box crosses the edge of the grid. In this @@ -272,8 +286,10 @@ def test_get_fill_slice(self): slice(16, 200) ) + @patch('hoss.dimension_utilities.add_bounds_variables') @patch('hoss.dimension_utilities.get_opendap_nc4') - def test_prefetch_dimension_variables(self, mock_get_opendap_nc4): + def test_prefetch_dimension_variables(self, mock_get_opendap_nc4, + mock_add_bounds_variables): """ Ensure that when a list of required variables is specified, a request to OPeNDAP will be sent requesting only those that are grid-dimension variables (both spatial and temporal). @@ -304,6 +320,163 @@ def test_prefetch_dimension_variables(self, mock_get_opendap_nc4): output_dir, self.logger, access_token, self.config) + mock_add_bounds_variables.assert_called_once_with(prefetch_path, + required_dimensions, + self.varinfo, self.logger) + + @patch('hoss.dimension_utilities.needs_bounds') + @patch('hoss.dimension_utilities.write_bounds') + def test_add_bounds_variables(self, mock_write_bounds, mock_needs_bounds): + """ Ensure that `write_bounds` is called when it's needed, + and that it's not called when it's not needed. + + """ + prefetch_dataset_name = 'tests/data/ATL16_prefetch.nc4' + varinfo_prefetch = VarInfoFromDmr( + 'tests/data/ATL16_prefetch.dmr' + ) + required_dimensions = {'/npolar_grid_lat', '/npolar_grid_lon', + '/spolar_grid_lat', '/spolar_grid_lon', + '/global_grid_lat', '/global_grid_lon'} + + with self.subTest('Bounds need to be written'): + mock_needs_bounds.return_value = True + add_bounds_variables(prefetch_dataset_name, + required_dimensions, + varinfo_prefetch, + self.logger) + self.assertEqual(mock_write_bounds.call_count, 6) + + mock_needs_bounds.reset_mock() + mock_write_bounds.reset_mock() + + with self.subTest('Bounds should not be written'): + mock_needs_bounds.return_value = False + add_bounds_variables(prefetch_dataset_name, + required_dimensions, + varinfo_prefetch, + self.logger) + mock_write_bounds.assert_not_called() + + def test_needs_bounds(self): + """ Ensure that the correct boolean value is returned for four + different cases: + + 1) False - cell_alignment[edge] attribute exists and + bounds variable already exists. + 2) False - cell_alignment[edge] attribute does not exist and + bounds variable already exists. + 3) True - cell_alignment[edge] attribute exists and + bounds variable does not exist. + 4) False - cell_alignment[edge] attribute does not exist and + bounds variable does not exist. + + """ + varinfo_bounds = VarInfoFromDmr( + 'tests/data/ATL16_prefetch_bnds.dmr' + ) + + with self.subTest('Variable has cell alignment and bounds'): + self.assertFalse(needs_bounds(varinfo_bounds.get_variable( + '/variable_edge_has_bnds'))) + + with self.subTest('Variable has no cell alignment and has bounds'): + self.assertFalse(needs_bounds(varinfo_bounds.get_variable( + '/variable_no_edge_has_bnds'))) + + with self.subTest('Variable has cell alignment and no bounds'): + self.assertTrue(needs_bounds(varinfo_bounds.get_variable( + '/variable_edge_no_bnds'))) + + with self.subTest('Variable has no cell alignment and no bounds'): + self.assertFalse(needs_bounds(varinfo_bounds.get_variable( + '/variable_no_edge_no_bnds'))) + + def test_get_bounds_array(self): + """ Ensure that the expected bounds array is created given + the input dimension variable values. + + """ + prefetch_dataset = Dataset('tests/data/ATL16_prefetch.nc4', 'r') + dimension_path = '/npolar_grid_lat' + + expected_bounds_array = self.bounds_array + + assert_array_equal(get_bounds_array(prefetch_dataset, + dimension_path), + expected_bounds_array) + + def test_write_bounds(self): + """ Ensure that bounds data array is written to the dimension + dataset, both when the dimension variable is in the root group + and in a nested group. + + """ + varinfo_prefetch = VarInfoFromDmr('tests/data/ATL16_prefetch_group.dmr') + prefetch_dataset = Dataset('tests/data/ATL16_prefetch_group.nc4', 'r+') + + # Expected variable contents in file. + expected_bounds_data = self.bounds_array + + with self.subTest('Dimension variable is in the root group'): + root_variable_full_path = '/npolar_grid_lat' + root_varinfo_variable = varinfo_prefetch.get_variable( + root_variable_full_path) + root_variable_name = 'npolar_grid_lat' + root_bounds_name = root_variable_name + '_bnds' + + write_bounds(prefetch_dataset, root_varinfo_variable) + + # Check that bounds variable was written to the root group. + self.assertTrue(prefetch_dataset.variables[root_bounds_name]) + + resulting_bounds_root_data = prefetch_dataset.variables[ + root_bounds_name][:] + + assert_array_equal(resulting_bounds_root_data, + expected_bounds_data) + # Check that varinfo variable has 'bounds' attribute. + self.assertEqual(root_varinfo_variable.attributes['bounds'], + root_bounds_name) + # Check that NetCDF4 dimension variable has 'bounds' attribute. + self.assertEqual(prefetch_dataset.variables[ + root_variable_name].__dict__.get('bounds'), + root_bounds_name) + # Check that VariableFromDmr has 'bounds' reference in + # the references dictionary. + self.assertEqual(root_varinfo_variable.references['bounds'], + {root_bounds_name, }) + + with self.subTest('Dimension variable is in a nested group'): + nested_variable_full_path = '/group1/group2/zelda' + nested_varinfo_variable = varinfo_prefetch.get_variable( + nested_variable_full_path) + nested_variable_name = 'zelda' + nested_group_path = '/group1/group2' + nested_group = prefetch_dataset[nested_group_path] + nested_bounds_name = nested_variable_name + '_bnds' + + write_bounds(prefetch_dataset, nested_varinfo_variable) + + # Check that bounds variable exists in the nested group. + self.assertTrue(nested_group.variables[nested_bounds_name]) + + resulting_bounds_nested_data = nested_group.variables[ + nested_bounds_name][:] + assert_array_equal(resulting_bounds_nested_data, + expected_bounds_data) + # Check that varinfo variable has 'bounds' attribute. + self.assertEqual(nested_varinfo_variable.attributes['bounds'], + nested_bounds_name) + # Check that NetCDF4 dimension variable has 'bounds' attribute. + self.assertEqual(nested_group.variables[ + nested_variable_name].__dict__.get('bounds'), + nested_bounds_name) + # Check that VariableFromDmr 'has bounds' reference in + # the references dictionary. + self.assertEqual(nested_varinfo_variable.references['bounds'], + {nested_bounds_name, }) + @patch('hoss.dimension_utilities.get_opendap_nc4') def test_prefetch_dimensions_with_bounds(self, mock_get_opendap_nc4): """ Ensure that a variable which has dimensions with `bounds` metadata