Merge pull request #145 from unity-sds/develop

release/3.6.1

Author: ngachung

CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.6.1] - 2023-04-24
+### Changed
+- [#144](https://github.com/unity-sds/unity-data-services/pull/144) fix: downloaded stac to return local absolute path
+
 ## [3.6.0] - 2023-04-24
 ### Added
 - [#142](https://github.com/unity-sds/unity-data-services/pull/142) feat: Support DAAC download files stac file, not just direct json text

cumulus_lambda_functions/stage_in_out/download_granules_daac.py

@@ -63,31 +63,37 @@ def __download_one_granule(self, assets: dict):
         headers = {
             'Authorization': f'Bearer {self.__edl_token}'
         }
+        local_item = {}
         for k, v in assets.items():
+            local_item[k] = v
             try:
                 LOGGER.debug(f'downloading: {v["href"]}')
                 r = requests.get(v['href'], headers=headers)
                 if r.status_code >= 400:
                     raise RuntimeError(f'wrong response status: {r.status_code}. details: {r.content}')
                 # TODO. how to correctly check redirecting to login page
-                with open(os.path.join(self._download_dir, os.path.basename(v["href"])), 'wb') as fd:
+                local_file_path = os.path.join(self._download_dir, os.path.basename(v["href"]))
+                with open(local_file_path, 'wb') as fd:
                     fd.write(r.content)
+                local_item[k]['href'] = local_file_path
             except Exception as e:
                 LOGGER.exception(f'failed to download {v}')
-                v['cause'] = str(e)
+                local_item[k]['description'] = f'download failed. {str(e)}'
                 error_log.append(v)
-        return error_log
+        return local_item, error_log
 
     def download(self, **kwargs) -> list:
         self.__set_props_from_env()
         LOGGER.debug(f'creating download dir: {self._download_dir}')
         downloading_urls = self.__get_downloading_urls(self._granules_json)
         error_list = []
+        local_items = []
         for each in downloading_urls:
             LOGGER.debug(f'working on {each}')
-            current_error_list = self.__download_one_granule(each)
+            local_item, current_error_list = self.__download_one_granule(each)
             error_list.extend(current_error_list)
+            local_items.append({'assets': local_item})
         if len(error_list) > 0:
             with open(f'{self._download_dir}/error.log', 'w') as error_file:
                 error_file.write(json.dumps(error_list, indent=4))
-        return downloading_urls
+        return local_items

cumulus_lambda_functions/stage_in_out/download_granules_s3.py

@@ -58,25 +58,30 @@ def __download_one_granule(self, assets: dict):
         :return:
         """
         error_log = []
+        local_item = {}
         for k, v in assets.items():
+            local_item[k] = v
             try:
                 LOGGER.debug(f'downloading: {v["href"]}')
-                self.__s3.set_s3_url(v['href']).download(self._download_dir)
+                local_file_path = self.__s3.set_s3_url(v['href']).download(self._download_dir)
+                local_item[k]['href'] = local_file_path
             except Exception as e:
                 LOGGER.exception(f'failed to download {v}')
-                v['cause'] = str(e)
+                local_item[k]['description'] = f'download failed. {str(e)}'
                 error_log.append(v)
-        return error_log
+        return local_item, error_log
 
     def download(self, **kwargs) -> list:
         self.__set_props_from_env()
         downloading_urls = self.__get_downloading_urls(self._granules_json)
         error_list = []
+        local_items = []
         for each in downloading_urls:
             LOGGER.debug(f'working on {each}')
-            current_error_list = self.__download_one_granule(each)
+            local_item, current_error_list = self.__download_one_granule(each)
+            local_items.append({'assets': local_item})
             error_list.extend(current_error_list)
         if len(error_list) > 0:
             with open(f'{self._download_dir}/error.log', 'w') as error_file:
                 error_file.write(json.dumps(error_list, indent=4))
-        return downloading_urls
+        return local_items

setup.py

@@ -18,7 +18,7 @@
 
 setup(
     name="cumulus_lambda_functions",
-    version="3.6.0",
+    version="3.6.1",
     packages=find_packages(),
     install_requires=install_requires,
     tests_require=['mock', 'nose', 'sphinx', 'sphinx_rtd_theme', 'coverage', 'pystac', 'python-dotenv', 'jsonschema'],

tests/integration_tests/test_docker_entry.py

@@ -224,6 +224,10 @@ def test_02_download(self):
             download_result = choose_process()
             self.assertTrue(isinstance(download_result, list), f'download_result is not list: {download_result}')
             self.assertEqual(sum([len(k) for k in download_result]), len(glob(os.path.join(tmp_dir_name, '*'))), f'downloaded file does not match')
+            self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}')
+            for each_granule in zip(granule_json, download_result):
+                remote_filename = os.path.basename(each_granule[0]['assets']['data']['href'])
+                self.assertEqual(each_granule[1]['assets']['data']['href'], os.path.join(tmp_dir_name, remote_filename), f"mismatched: {each_granule[0]['assets']['data']['href']}")
         return
 
     def test_02_download__daac(self):
@@ -245,6 +249,11 @@ def test_02_download__daac(self):
             error_file = os.path.join(tmp_dir_name, 'error.log')
             if FileUtils.file_exist(error_file):
                 self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}')
+            self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}')
+            for each_granule in zip(granule_json, download_result):
+                remote_filename = os.path.basename(each_granule[0]['assets']['data']['href'])
+                self.assertEqual(each_granule[1]['assets']['data']['href'], os.path.join(tmp_dir_name, remote_filename),
+                                 f"mismatched: {each_granule[0]['assets']['data']['href']}")
         return
 
     def test_02_download__daac__from_file(self):
@@ -271,6 +280,11 @@ def test_02_download__daac__from_file(self):
             error_file = os.path.join(downloading_dir, 'error.log')
             if FileUtils.file_exist(error_file):
                 self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}')
+            self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}')
+            for each_granule in zip(granule_json, download_result):
+                remote_filename = os.path.basename(each_granule[0]['assets']['data']['href'])
+                self.assertEqual(each_granule[1]['assets']['data']['href'], os.path.join(downloading_dir, remote_filename),
+                                 f"mismatched: {each_granule[0]['assets']['data']['href']}")
         return
 
     def test_02_download__daac_error(self):
@@ -314,6 +328,11 @@ def test_02_download__from_file(self):
             error_file = os.path.join(downloading_dir, 'error.log')
             if FileUtils.file_exist(error_file):
                 self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}')
+            self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}')
+            for each_granule in zip(granule_json, download_result):
+                remote_filename = os.path.basename(each_granule[0]['assets']['data']['href'])
+                self.assertEqual(each_granule[1]['assets']['data']['href'], os.path.join(downloading_dir, remote_filename),
+                                 f"mismatched: {each_granule[0]['assets']['data']['href']}")
         return
 
     def test_03_upload(self):