From d3ac1d1e3bae09ddb2285fcdc1440126ae26399a Mon Sep 17 00:00:00 2001
From: Michael Cayanan <42812746+mcayanan@users.noreply.github.com>
Date: Fri, 9 Dec 2022 13:56:06 -0800
Subject: [PATCH] HC-444: Add capability to purge job to not delete from object
 store (#27)

* HC-444: Purge Job update - Add capability to not delete from object store

* test using exec wrapper

* Revert "test using exec wrapper"

This reverts commit bb5bccd607584c8b0c2a2bff1545777d0b40460b.

* update message

* add newline

Co-authored-by: Mike Cayanan <michael.d.cayanan@jpl.nasa.gov>
---
 docker/hysds-io.json.lw-tosca-purge |  8 +++++++-
 docker/job-spec.json.lw-tosca-purge |  4 ++++
 purge.py                            | 13 +++++++++----
 3 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/docker/hysds-io.json.lw-tosca-purge b/docker/hysds-io.json.lw-tosca-purge
index e9bae17..d7228de 100644
--- a/docker/hysds-io.json.lw-tosca-purge
+++ b/docker/hysds-io.json.lw-tosca-purge
@@ -20,6 +20,12 @@
       "from": "value",
       "type": "text",
       "value": "purge"
+    },
+    {
+      "name": "delete_from_object_store",
+      "from": "submitter",
+      "type": "boolean",
+      "default": "true"
     }
   ]
-}
\ No newline at end of file
+}
diff --git a/docker/job-spec.json.lw-tosca-purge b/docker/job-spec.json.lw-tosca-purge
index 31d1c8e..f55fe17 100644
--- a/docker/job-spec.json.lw-tosca-purge
+++ b/docker/job-spec.json.lw-tosca-purge
@@ -20,6 +20,10 @@
     {
       "name": "operation",
       "destination": "context"
+    },
+    {
+      "name": "delete_from_object_store",
+      "destination": "context"
     }
    ]
 }
diff --git a/purge.py b/purge.py
index 1748934..6434ffa 100644
--- a/purge.py
+++ b/purge.py
@@ -41,12 +41,13 @@ def delete_from_object_store(es_result):
     return dataset
 
 
-def purge_products(query, component, operation):
+def purge_products(query, component, operation, delete_from_obj_store=True):
     """
     Iterator used to iterate across a query result and submit jobs for every hit
     :param query: query to post to ElasticSearch and whose result will be iterated, JSON sting enc
     :param component: tosca || figaro
     :param operation: purge or something else
+    :param delete_from_obj_store: Flag to indicate whether to purge the dataset from the object store
     """
     logger.debug("action: %s for %s", operation, component)
     logger.debug("query: %s" % json.dumps(query, indent=2))
@@ -80,8 +81,11 @@ def purge_products(query, component, operation):
         bulk_res = es.es.bulk(index=es_index, body=body, filter_path=filter_path)
         logger.info(json.dumps(bulk_res, indent=2))
 
-        logger.info("purging datasets from object store: ")
-        p.map(delete_from_object_store, results)  # deleting objects from storage (s3, etc.)
+        if delete_from_obj_store is True:
+            logger.info("purging datasets from object store: ")
+            p.map(delete_from_object_store, results)  # deleting objects from storage (s3, etc.)
+        else:
+            logger.info("skip purging datasets from object store. delete_from_object_store=False")
 
         dataset_purge_stats = {}
         deleted_docs_count = 0
@@ -153,6 +157,7 @@ def purge_products(query, component, operation):
 
     component_val = context['component']
     operation_val = context['operation']
+    delete_from_object_store_val = context.get("delete_from_object_store", True)
 
     query_obj = context['query']
     try:
@@ -160,4 +165,4 @@ def purge_products(query, component, operation):
     except TypeError as e:
         logger.warning(e)
 
-    purge_products(query_obj, component_val, operation_val)
+    purge_products(query_obj, component_val, operation_val, delete_from_object_store_val)