0.4.0 release (#85)

Co-authored-by: Anil Natha <[email protected]>

* Added support for custom metadata definition and search (#81)

* Added support for custom metadata definition and search

* Updated CHANGELOG.md

* Update collection.py (#82)

remove print statement that was annoying me.

* Stac updates (#84)

* added stac out put options to data_services call, added properties parsing to datasets

* aded test dataset for properties

* updated changelog

* bump version

* Update CHANGELOG.md

Updated 0.4.0 release and date.

---------

Co-authored-by: Anil Natha <[email protected]>
Co-authored-by: Rishi Verma <[email protected]>
Co-authored-by: Anil Natha <[email protected]>
Co-authored-by: Rob Tapella <[email protected]>
Co-authored-by: Nga Chung <[email protected]>

Author: 5 people

CHANGELOG.md

@@ -5,9 +5,25 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.0] - 2024-03-29
 
+### Added
+* We've added the ability to override settings in the default config file by passing in a config file with the settings needing to be overridden when instantiating a Unity object. [56](https://github.com/unity-sds/unity-py/issues/56)
+* Collection creation (create_collection) through dataService library
+* Added support for defining custom metadata for project and venue.
+* Added methods to return STAC content instead of unity domain objects if requested
+* Added properties parsing of stac metadata to dataset objects
+### Fixed
+### Changed
+* Updated get_collections and get_collection_data to support limit parameter.
+* Updated get_collection_data to support filter parameter which takes CQL string.
+### Removed
+### Security
+### Deprecated
+
+
+## [0.3.0] - 2024-02-12
 
-## Unreleased [0.3.0]
 ### Added
 ### Fixed
 * fixed an issue with encoding a json deploy request twice [71](https://github.com/unity-sds/unity-py/issues/71)

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "unity-sds-client"
-version = "0.3.0"
+version = "0.4.0"
 description = "Unity-Py is a Python client to simplify interactions with NASA's Unity Platform."
 authors = ["Anil Natha, Mike Gangl"]
 readme = "README.md"

pyproject.toml

@@ -0,0 +1,2 @@
+[TEST]
+client_id = THIS_IS_A_TEST_CLIENT_ID

tests/test_files/config_override.cfg

@@ -0,0 +1 @@
+{"numberMatched":{"total_size":2},"numberReturned":2,"stac_version":"1.0.0","type":"FeatureCollection","links":[{"rel":"self","href":"https://dxebrgu0bc9w7.cloudfront.net/am-uds-dapa/collections/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1/items?limit=10"},{"rel":"root","href":"https://dxebrgu0bc9w7.cloudfront.net"}],"features":[{"type":"Feature","stac_version":"1.0.0","id":"urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1:SISTER_EMIT_L2B_FRCOV_20230807T182755_001","properties":{"Description":"This catalog contains the output data products of the SISTER fractional cover PGE, including a fractional cover cloud-optimized GeoTIFF. Execution artifacts including the runconfig file and execution log file are included with the fractional cover data.","soil":{"0.0":0.0,"0.1":0.0,"0.2":0.0,"0.3":0.0,"0.4":0.0,"0.5":0.0,"0.6":0.0,"0.7":0.0,"0.8":0.0,"0.9":0.0,"1.0":0.0},"vegetation":{"0.0":0.0,"0.1":0.0,"0.2":0.0,"0.3":0.0,"0.4":0.0,"0.5":0.0,"0.6":0.0,"0.7":0.0,"0.8":0.0,"0.9":0.0,"1.0":0.0},"water":{"0.0":0.0,"0.1":0.0,"0.2":0.0,"0.3":0.0,"0.4":0.0,"0.5":0.0,"0.6":0.0,"0.7":0.0,"0.8":0.0,"0.9":0.0,"1.0":0.0},"snow_ice":{"0.0":1862737.0,"0.1":0.0,"0.2":0.0,"0.3":0.0,"0.4":0.0,"0.5":0.0,"0.6":0.0,"0.7":0.0,"0.8":0.0,"0.9":0.0,"1.0":0.0},"datetime":"2024-03-19T04:54:33.703000Z","start_datetime":"2023-08-07T18:27:55Z","end_datetime":"2023-08-07T18:28:07Z","created":"2023-08-07T18:27:55Z","updated":"2024-03-19T04:55:14.767000Z","status":"completed","provider":"unity"},"geometry":{"type":"Point","coordinates":[0.0,0.0]},"links":[{"rel":"collection","href":"."},{"rel":"self","href":"https://dxebrgu0bc9w7.cloudfront.net/am-uds-dapa/collections/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1/items/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1:SISTER_EMIT_L2B_FRCOV_20230807T182755_001","type":"application/json","title":"urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1:SISTER_EMIT_L2B_FRCOV_20230807T182755_001"}],"assets":{"SISTER_EMIT_L2B_FRCOV_20230807T182755_001.cmr.xml":{"href":"s3://sps-test-ds-storage/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1:SISTER_EMIT_L2B_FRCOV_20230807T182755_001/SISTER_EMIT_L2B_FRCOV_20230807T182755_001.cmr.xml","title":"SISTER_EMIT_L2B_FRCOV_20230807T182755_001.cmr.xml","description":"size=1766;checksumType=md5;checksum=bd1eec65627f63e998113952c3778f6d;","roles":["metadata"]},"SISTER_EMIT_L2B_FRCOV_20230807T182755_001.json":{"href":"s3://sps-test-ds-storage/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1:SISTER_EMIT_L2B_FRCOV_20230807T182755_001/SISTER_EMIT_L2B_FRCOV_20230807T182755_001.json","title":"SISTER_EMIT_L2B_FRCOV_20230807T182755_001.json","description":"size=-1;checksumType=md5;checksum=unknown;","roles":["metadata"]},"SISTER_EMIT_L2B_FRCOV_20230807T182755_001.tif":{"href":"s3://sps-test-ds-storage/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1:SISTER_EMIT_L2B_FRCOV_20230807T182755_001/SISTER_EMIT_L2B_FRCOV_20230807T182755_001.tif","title":"SISTER_EMIT_L2B_FRCOV_20230807T182755_001.tif","description":"size=-1;checksumType=md5;checksum=unknown;","roles":["data"]},"SISTER_EMIT_L2B_FRCOV_20230807T182755_001.log":{"href":"s3://sps-test-ds-storage/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1:SISTER_EMIT_L2B_FRCOV_20230807T182755_001/SISTER_EMIT_L2B_FRCOV_20230807T182755_001.log","title":"SISTER_EMIT_L2B_FRCOV_20230807T182755_001.log","description":"size=-1;checksumType=md5;checksum=unknown;","roles":["metadata"]},"SISTER_EMIT_L2B_FRCOV_20230807T182755_001.png":{"href":"s3://sps-test-ds-storage/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1:SISTER_EMIT_L2B_FRCOV_20230807T182755_001/SISTER_EMIT_L2B_FRCOV_20230807T182755_001.png","title":"SISTER_EMIT_L2B_FRCOV_20230807T182755_001.png","description":"size=-1;checksumType=md5;checksum=unknown;","roles":["browse"]}},"bbox":[-180.0,-90.0,180.0,90.0],"stac_extensions":[],"collection":"urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1"},{"type":"Feature","stac_version":"1.0.0","id":"urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1:SISTER_EMIT_L2B_FRCOV_20240103T131936_001","properties":{"Description":"This catalog contains the output data products of the SISTER fractional cover PGE, including a fractional cover cloud-optimized GeoTIFF. Execution artifacts including the runconfig file and execution log file are included with the fractional cover data.","soil":{"0.0":0.0,"0.1":0.0,"0.2":0.0,"0.3":0.0,"0.4":0.0,"0.5":0.0,"0.6":0.0,"0.7":0.0,"0.8":0.0,"0.9":0.0,"1.0":0.0},"vegetation":{"0.0":0.0,"0.1":0.0,"0.2":0.0,"0.3":0.0,"0.4":0.0,"0.5":0.0,"0.6":0.0,"0.7":0.0,"0.8":0.0,"0.9":0.0,"1.0":0.0},"water":{"0.0":0.0,"0.1":0.0,"0.2":0.0,"0.3":0.0,"0.4":0.0,"0.5":0.0,"0.6":0.0,"0.7":0.0,"0.8":0.0,"0.9":0.0,"1.0":0.0},"snow_ice":{"0.0":93871.0,"0.1":0.0,"0.2":0.0,"0.3":0.0,"0.4":0.0,"0.5":0.0,"0.6":0.0,"0.7":0.0,"0.8":0.0,"0.9":0.0,"1.0":0.0},"datetime":"2024-03-18T18:04:15.591000Z","start_datetime":"2024-01-03T13:19:36Z","end_datetime":"2024-01-03T13:19:48Z","created":"2024-01-03T13:19:36Z","updated":"2024-03-18T18:04:27.773000Z","status":"completed","provider":"unity"},"geometry":{"type":"Point","coordinates":[0.0,0.0]},"links":[{"rel":"collection","href":"."},{"rel":"self","href":"https://dxebrgu0bc9w7.cloudfront.net/am-uds-dapa/collections/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1/items/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1:SISTER_EMIT_L2B_FRCOV_20240103T131936_001","type":"application/json","title":"urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1:SISTER_EMIT_L2B_FRCOV_20240103T131936_001"}],"assets":{"SISTER_EMIT_L2B_FRCOV_20240103T131936_001.cmr.xml":{"href":"s3://sps-test-ds-storage/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1:SISTER_EMIT_L2B_FRCOV_20240103T131936_001/SISTER_EMIT_L2B_FRCOV_20240103T131936_001.cmr.xml","title":"SISTER_EMIT_L2B_FRCOV_20240103T131936_001.cmr.xml","description":"size=1766;checksumType=md5;checksum=a8c0e55579d2f1b47eb6e99188082559;","roles":["metadata"]},"SISTER_EMIT_L2B_FRCOV_20240103T131936_001.json":{"href":"s3://sps-test-ds-storage/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1:SISTER_EMIT_L2B_FRCOV_20240103T131936_001/SISTER_EMIT_L2B_FRCOV_20240103T131936_001.json","title":"SISTER_EMIT_L2B_FRCOV_20240103T131936_001.json","description":"size=-1;checksumType=md5;checksum=unknown;","roles":["metadata"]},"SISTER_EMIT_L2B_FRCOV_20240103T131936_001.tif":{"href":"s3://sps-test-ds-storage/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1:SISTER_EMIT_L2B_FRCOV_20240103T131936_001/SISTER_EMIT_L2B_FRCOV_20240103T131936_001.tif","title":"SISTER_EMIT_L2B_FRCOV_20240103T131936_001.tif","description":"size=-1;checksumType=md5;checksum=unknown;","roles":["data"]},"SISTER_EMIT_L2B_FRCOV_20240103T131936_001.png":{"href":"s3://sps-test-ds-storage/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1:SISTER_EMIT_L2B_FRCOV_20240103T131936_001/SISTER_EMIT_L2B_FRCOV_20240103T131936_001.png","title":"SISTER_EMIT_L2B_FRCOV_20240103T131936_001.png","description":"size=-1;checksumType=md5;checksum=unknown;","roles":["browse"]},"SISTER_EMIT_L2B_FRCOV_20240103T131936_001.log":{"href":"s3://sps-test-ds-storage/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1/urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1:SISTER_EMIT_L2B_FRCOV_20240103T131936_001/SISTER_EMIT_L2B_FRCOV_20240103T131936_001.log","title":"SISTER_EMIT_L2B_FRCOV_20240103T131936_001.log","description":"size=-1;checksumType=md5;checksum=unknown;","roles":["metadata"]}},"bbox":[-180.0,-90.0,180.0,90.0],"stac_extensions":[],"collection":"urn:nasa:unity:unity:test:SBG-L2B_FRCOV___1"}]}

tests/test_files/dataset_with_properties.json

@@ -0,0 +1,50 @@
+from unity_sds_client.unity import Unity
+from unity_sds_client.unity_exception import UnityException
+from unity_sds_client.unity_session import UnitySession
+from unity_sds_client.resources.collection import Collection
+from unity_sds_client.unity_services import UnityServices as Services
+
+import pytest
+
+
+@pytest.fixture
+def cleanup_update_test():
+    yield None
+    print("Cleanup...")
+
+
+def test_collection_creation(cleanup_update_test):
+    #no venue, project
+    with pytest.raises(UnityException):
+        s = Unity()
+        ds = s.client(Services.DATA_SERVICE)
+        ds.create_collection(Collection("urn:nasa:unity:myproject:myvenue:identifier"), True)
+    #no Venue
+    with pytest.raises(UnityException):
+        s = Unity()
+        s.set_project("unity")
+        ds = s.client(Services.DATA_SERVICE)
+        ds.create_collection(Collection("urn:nasa:unity:myproject:myvenue:identifier"), True)
+    #null collections
+    # does not match name
+    with pytest.raises(UnityException):
+        s = Unity()
+        s.set_project("unity")
+        s.set_venue("test")
+        ds = s.client(Services.DATA_SERVICE)
+        ds.create_collection(None, True)
+
+    #does not match name
+    with pytest.raises(UnityException):
+        s = Unity()
+        s.set_project("unity")
+        s.set_venue("test")
+        ds = s.client(Services.DATA_SERVICE)
+        ds.create_collection( Collection("urn:nasa:unity:myproject:myvenue:identifier"), True)
+
+        # does not match name
+    s = Unity()
+    s.set_project("unity")
+    s.set_venue("test")
+    ds = s.client(Services.DATA_SERVICE)
+    ds.create_collection(Collection("urn:nasa:unity:unity:test:my_collection_id"), True)

tests/test_unity_data_service.py

@@ -16,6 +16,10 @@ def test_default_unity_client():
     client = Unity()
     assert True == True
 
+def test_unity_client_config_override():
+    client = Unity(config_file_override="tests/test_files/config_override.cfg")
+    setting = client._session._config.get("TEST","client_id")
+    assert setting == "THIS_IS_A_TEST_CLIENT_ID"
 
 @pytest.mark.regression
 def test_example_regression_test(cleanup_update_test):
@@ -25,6 +29,14 @@ def test_example_regression_test(cleanup_update_test):
     collections = dataManager.get_collections()
     assert len(collections) > 0
 
+@pytest.mark.regression
+def test_example_regression_test(cleanup_update_test):
+    print("Example collection test")
+    s = Unity()
+    dataManager = s.client(Services.DATA_SERVICE)
+    collections_json = dataManager.get_collections(output_stac=True)
+    assert collections_json is not None
+
 
 @pytest.mark.regression
 def test_example_process(cleanup_update_test):

tests/test_unity_py.py

@@ -56,6 +56,15 @@ def test_read_stac():
     assert len(data_files) == 1
     assert data_files[0].roles == ["metadata"]
 
+    # Try a "classic" catalog + item files stac catalog
+    collection = Collection.from_stac("tests/test_files/dataset_with_properties.json")
+    datasets = collection.datasets
+    # Added 3/26/24 to check the properties being added
+    assert datasets[0].properties is not None
+    assert datasets[0].properties['soil']
+    print("Properties is not null")
+
+
 
 def test_write_stac():
     collection = Collection.from_stac("tests/test_files/cmr_granules.json")

tests/test_unity_stac.py

@@ -28,6 +28,7 @@ def __init__(self, id):
         self._beginning_time = None
         self._ending_time = None
 
+
     def add_dataset(self, dataset: Dataset):
         self._datasets.append(dataset)
 
@@ -199,7 +200,6 @@ def from_stac(stac_file):
                 ds = Dataset(item.id, id, item.properties.get("start_datetime", None),
                              item.properties.get("end_datetime", None), item.properties.get("created", None))
                 if item.collection_id is not None and item.collection_id != id:
-                     print ("if2")
                      ds = Dataset(item.id, item.collection_id, item.properties.get("start_datetime", None),
                                  item.properties.get("end_datetime", None), item.properties.get("created", None))
 

unity_sds_client/resources/collection.py

@@ -12,7 +12,7 @@ def __str__(self):
     def __repr__(self):
         return self.__str__()
 
-    def __init__(self, name, collection_id, start_time, end_time, creation_time ):
+    def __init__(self, name, collection_id, start_time, end_time, creation_time, properties=None ):
         """Dataset object construction)
 
         Parameters
@@ -44,6 +44,12 @@ def __init__(self, name, collection_id, start_time, end_time, creation_time ):
         self.geometry = None
         self.bbox = None
 
+        #add non-reserved properties
+        if properties is not None:
+            for key, value in properties.items():
+                if key not in ['start_datetime','created', 'end_datetime']:
+                    self.properties[key] = value
+
     def add_data_file(self, datafile: type=DataFile):
         """adds a data file to a dataset