HC-302: Add job detail information on number of datasets purged (#24)

* HC-302: Report number of datasets purged per type

* update

* Support for possible list

* only formulate the message if we really did find stuff that was purged.

Co-authored-by: Mike Cayanan <[email protected]>

purge.py

@@ -5,7 +5,7 @@
 import osaka.main
 from hysds.celery import app
 from hysds.es_util import get_mozart_es, get_grq_es
-from utils import revoke
+from utils import revoke, create_info_message_files
 
 
 LOG_FILE_NAME = 'purge.log'
@@ -37,12 +37,12 @@ def purge_products(query, component, operation):
         es_index = app.conf["DATASET_ALIAS"]
 
     results = es.query(index=es_index, body=query)  # Querying for products
-
     if component == 'tosca':
+        deleted_datasets = dict()
         for result in results:
             ident = result["_id"]
             index = result["_index"]
-
+            dataset = result["_source"]["dataset"]
             # find the Best URL first
             best = None
             for url in result["_source"]["urls"]:
@@ -55,7 +55,18 @@ def purge_products(query, component, operation):
 
             es.delete_by_id(index=index, id=ident, ignore=404)  # removing the metadata
             logger.info('Purged %s' % ident)
-
+            if dataset in deleted_datasets:
+                count = deleted_datasets[dataset]
+                deleted_datasets[dataset] = count + 1
+            else:
+                deleted_datasets[dataset] = 1
+
+        if len(deleted_datasets) != 0:
+            msg_details = "Datasets purged by type:\n\n"
+            for ds in deleted_datasets.keys():
+                msg_details += "{} - {}\n".format(deleted_datasets[ds], ds)
+
+            create_info_message_files(msg_details=msg_details)
     else:
         purge = True if operation == 'purge' else False  # purge job from index
 

utils.py

@@ -37,3 +37,27 @@ def revoke(task_id, state):
 
     # revoke task
     app.control.revoke(task_id, terminate=True)
+
+
+def create_info_message_files(msg=None, msg_details=None):
+    """
+    Creates the _alt_msg.txt and _alt_msg_details.txt
+    files for population into the job status json.
+
+    :param msg: The short info message. Can be a list or a string.
+     Should be shorter than 35 characters.
+    :param msg_details: The message details.
+    :return:
+    """
+
+    if msg:
+        with open('_alt_msg.txt', 'w') as f:
+            if isinstance(msg, list):
+                for m in msg:
+                    f.write("%s\n" % str(m))
+            else:
+                f.write("%s\n" % str(msg))
+
+    if msg_details:
+        with open('_alt_msg_details.txt', 'w') as f:
+            f.write("%s\n" % msg_details)