-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* HC-210 fixed imports and black formatted the notify_by_email.py * HC-210 changing notify_by_email.sh so it takes 0 arguments. * HC-210 updated import for MIMEMultipart updated job-spec to consistantly use the /home/ops/lightweight-jobs/ directory * HC-210 fixed encoders implementation in notify_by_email.py * HC-210 changed directory to reference file path vs. getcdw() * HC-210 fixed path * NSDS-210 changed context["objectid"] to context["id"]. Don't know if this was supposed to have changed. * HC-210 Changed "rule_name" to "name" * fixed get_by_id on alias bug * HC-210 encoding string into a bytes object before b64 encoding it. * HC-210 Forgot to include cc_recipients in send_email function. * HC-210 specifiying ascii when encoding the query * HC-210 decoded ascii encoded addresses. * HC-210 fixed doc source issues. * HC-210 fixed facetview link creation * HC-210 Fixed facetview_url again Co-authored-by: Poreh <[email protected]> Co-authored-by: DustinKLo <[email protected]>
- Loading branch information
3 people
authored
Jun 3, 2020
1 parent
9d7add4
commit bb5877c
Showing
9 changed files
with
108 additions
and
94 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -7,20 +7,20 @@ | |
| import socket | ||
|
|
||
| from smtplib import SMTP | ||
| from email.MIMEMultipart import MIMEMultipart | ||
| from email.MIMEText import MIMEText | ||
| from email.MIMEBase import MIMEBase | ||
| from email.Header import Header | ||
| from email.Utils import parseaddr, formataddr, COMMASPACE, formatdate | ||
| from email import Encoders | ||
| from email.mime.multipart import MIMEMultipart | ||
| from email.mime.text import MIMEText | ||
| from email.mime.base import MIMEBase | ||
| from email.header import Header | ||
| from email.utils import parseaddr, formataddr, COMMASPACE | ||
| from email import encoders | ||
|
|
||
| from hysds.celery import app | ||
| from hysds.es_util import get_mozart_es, get_grq_es | ||
| from hysds_commons.net_utils import get_container_host_ip | ||
|
|
||
|
|
||
| def read_context(): | ||
| with open('_context.json', 'r') as f: | ||
| with open("_context.json", "r") as f: | ||
| cxt = json.load(f) | ||
| return cxt | ||
|
|
||
|
|
@@ -31,13 +31,15 @@ def get_hostname(): | |
| return socket.getfqdn() | ||
| except Exception as e: | ||
| print(e) | ||
| print('socket.getfqdn() failed, passing...') | ||
| print("socket.getfqdn() failed, passing...") | ||
| pass | ||
| try: | ||
| return socket.gethostbyname(socket.gethostname()) | ||
| except Exception as e: | ||
| print(e) | ||
| raise RuntimeError("Failed to resolve hostname for full email address. Check system.") | ||
| raise RuntimeError( | ||
| "Failed to resolve hostname for full email address. Check system." | ||
| ) | ||
|
|
||
|
|
||
| def send_email(sender, cc, bcc, subject, body, attachments=None): | ||
|
|
@@ -60,10 +62,10 @@ def send_email(sender, cc, bcc, subject, body, attachments=None): | |
|
|
||
| # Header class is smart enough to try US-ASCII, then the charset we | ||
| # provide, then fall back to UTF-8. | ||
| header_charset = 'ISO-8859-1' | ||
| header_charset = "ISO-8859-1" | ||
|
|
||
| # We must choose the body charset manually | ||
| for body_charset in 'US-ASCII', 'ISO-8859-1', 'UTF-8': | ||
| for body_charset in "US-ASCII", "ISO-8859-1", "UTF-8": | ||
| try: | ||
| body.encode(body_charset) | ||
| except UnicodeError: | ||
|
|
@@ -83,35 +85,44 @@ def send_email(sender, cc, bcc, subject, body, attachments=None): | |
| recipient_name = str(Header(str(recipient_name), header_charset)) | ||
|
|
||
| # Make sure email addresses do not contain non-ASCII characters | ||
| recipient_addr = recipient_addr.encode('ascii') | ||
| recipient_addr = recipient_addr.encode("ascii") | ||
| recipient_addr = recipient_addr.decode() | ||
| unicode_parsed_cc.append((recipient_name, recipient_addr)) | ||
|
|
||
| unicode_parsed_bcc = [] | ||
| for recipient_name, recipient_addr in parsed_bcc: | ||
| recipient_name = str(Header(str(recipient_name), header_charset)) | ||
|
|
||
| # Make sure email addresses do not contain non-ASCII characters | ||
| recipient_addr = recipient_addr.encode('ascii') | ||
| recipient_addr = recipient_addr.encode("ascii") | ||
| recipient_addr = recipient_addr.decode() | ||
| unicode_parsed_bcc.append((recipient_name, recipient_addr)) | ||
|
|
||
| # Create the message ('plain' stands for Content-Type: text/plain) | ||
| msg = MIMEMultipart() | ||
| msg['CC'] = COMMASPACE.join([formataddr((recipient_name, recipient_addr)) | ||
| for recipient_name, recipient_addr in unicode_parsed_cc]) | ||
| msg['BCC'] = COMMASPACE.join([formataddr((recipient_name, recipient_addr)) | ||
| for recipient_name, recipient_addr in unicode_parsed_bcc]) | ||
| msg['Subject'] = Header(str(subject), header_charset) | ||
| msg['FROM'] = "[email protected]" | ||
| msg.attach(MIMEText(body.encode(body_charset), 'plain', body_charset)) | ||
| msg["CC"] = COMMASPACE.join( | ||
| [ | ||
| formataddr((recipient_name, recipient_addr)) | ||
| for recipient_name, recipient_addr in unicode_parsed_cc | ||
| ] | ||
| ) | ||
| msg["BCC"] = COMMASPACE.join( | ||
| [ | ||
| formataddr((recipient_name, recipient_addr)) | ||
| for recipient_name, recipient_addr in unicode_parsed_bcc | ||
| ] | ||
| ) | ||
| msg["Subject"] = Header(str(subject), header_charset) | ||
| msg["FROM"] = "[email protected]" | ||
| msg.attach(MIMEText(body.encode(body_charset), "plain", body_charset)) | ||
|
|
||
| # Add attachments | ||
| if isinstance(attachments, dict): | ||
| for fname in attachments: | ||
| part = MIMEBase('application', "octet-stream") | ||
| part = MIMEBase("application", "octet-stream") | ||
| part.set_payload(attachments[fname]) | ||
| Encoders.encode_base64(part) | ||
| part.add_header('Content-Disposition', | ||
| 'attachment; filename="%s"' % fname) | ||
| encoders.encode_base64(part) | ||
| part.add_header("Content-Disposition", 'attachment; filename="%s"' % fname) | ||
| msg.attach(part) | ||
|
|
||
| # Send the message via SMTP to docker host | ||
|
|
@@ -126,22 +137,21 @@ def get_cities(src): | |
| """Return list of cities.""" | ||
|
|
||
| cities = [] | ||
| for city in src.get('city', []): | ||
| cities.append("%s, %s" % | ||
| (city.get('name', ''), city.get('admin1_name', ''))) | ||
| for city in src.get("city", []): | ||
| cities.append("%s, %s" % (city.get("name", ""), city.get("admin1_name", ""))) | ||
| return cities | ||
|
|
||
|
|
||
| def get_value(d, key): | ||
| """Return value from source based on key.""" | ||
|
|
||
| for k in key.split('.'): | ||
| for k in key.split("."): | ||
| if k in d: | ||
| d = d[k] | ||
| else: | ||
| return None | ||
| if isinstance(d, list): | ||
| return ', '.join([str(i) for i in d]) | ||
| return ", ".join([str(i) for i in d]) | ||
| else: | ||
| return d | ||
|
|
||
|
|
@@ -154,8 +164,10 @@ def get_metadata_snippet(src, snippet_cfg): | |
| val = get_value(src, k) | ||
| if val is not None: | ||
| body += "%s: %s\n" % (label, val) | ||
| body += "location type: %s\n" % src.get('location', {}).get('type', None) | ||
| body += "location coordinates: %s\n" % src.get('location', {}).get('coordinates', []) | ||
| body += "location type: %s\n" % src.get("location", {}).get("type", None) | ||
| body += "location coordinates: %s\n" % src.get("location", {}).get( | ||
| "coordinates", [] | ||
| ) | ||
| cities = get_cities(src) | ||
| body += "Closest cities: %s" % "\n\t\t".join(cities) | ||
| return body | ||
|
|
@@ -169,91 +181,107 @@ def get_facetview_link(link, _id, version=None): | |
| :param version: str | ||
| :return: constructed URL for facetview | ||
| """ | ||
| if link.endswith("/"): | ||
| link = link[:-1] | ||
| origin = link.split("/")[-1:] | ||
| print(origin) | ||
| if "figaro" in origin: | ||
| term = "job_id" | ||
| else: | ||
| term = "_id" | ||
| if version is None: | ||
| query = { | ||
| 'query': { | ||
| 'query_string': { | ||
| 'query': '_id:%s' % _id | ||
| } | ||
| } | ||
| } | ||
| b64 = base64.urlsafe_b64encode(json.dumps(query)) | ||
| query_string = 'query_string="' + term + '%3A%5C"' + _id + '%5C""' | ||
| else: | ||
| query = { | ||
| 'query': { | ||
| 'query_string': { | ||
| 'query': '_id:%s AND system_versions:%s' % (_id, version) | ||
| } | ||
| } | ||
| } | ||
| b64 = base64.urlsafe_b64encode(json.dumps(query)) | ||
| if link.endswith('/'): | ||
| link = link[:-1] | ||
| return '%s/?base64=%s' % (link, b64) | ||
| query_string = 'query_string="' + term + '%3A%5C"' + _id + '%5C""&system_version="' + version + '"' | ||
| print(_id) | ||
| return "%s/?%s" % (link, query_string) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| cwd = os.getcwd() | ||
| settings_file = os.path.join(cwd, 'settings.json') | ||
| path = "/".join(__file__.split("/")[0:-1]) | ||
| settings_file = os.path.join(path, "settings.json") | ||
| settings_file = os.path.normpath(settings_file) # normalizing the path | ||
| settings = json.load(open(settings_file)) | ||
|
|
||
| context = read_context() | ||
|
|
||
| object_id = context['objectid'] | ||
| url = context['url'] | ||
| emails = context['emails'] | ||
| rule_name = context['rule_name'] | ||
| component = context['component'] | ||
| object_id = context["id"] | ||
| url = context["url"] | ||
| emails = context["emails"] | ||
| rule_name = context["name"] | ||
| component = context["component"] | ||
|
|
||
| if component == "mozart" or component == "figaro": | ||
| es = get_mozart_es() | ||
| index = app.conf["STATUS_ALIAS"] | ||
| facetview_url = app.conf["MOZART_URL"] | ||
| facetview_url = "/".join(facetview_url.split("/")[0:-2]) + "/hysds_ui/figaro" | ||
| else: # "tosca" | ||
| es = get_grq_es() | ||
| index = app.conf["DATASET_ALIAS"] | ||
| facetview_url = "https://aria-search-beta.jpl.nasa.gov/search" # TODO: why is it hard coded | ||
| facetview_url = app.conf["MOZART_URL"] | ||
| facetview_url = "/".join(facetview_url.split("/")[0:-2]) + "/hysds_ui/tosca" | ||
|
|
||
| cc_recipients = [i.strip() for i in emails.split(',')] | ||
| cc_recipients = [i.strip() for i in emails.split(",")] | ||
| bcc_recipients = [] | ||
| email_subject = "[monitor] (notify_by_email:%s) %s" % (rule_name, object_id) | ||
| email_body = "Product with id %s was ingested." % object_id | ||
| email_attachments = None | ||
|
|
||
| doc = es.get_by_id(index=index, id=object_id, ignore=404) | ||
| query = { | ||
| "query": { | ||
| "term": { | ||
| "_id": object_id | ||
| } | ||
| } | ||
| } | ||
| result = es.search(index=index, body=query) # can't use get_by_id on alias | ||
|
|
||
| if doc['found'] is True: | ||
| email_body += "\n\n%s" % get_metadata_snippet(doc, settings['SNIPPET_CFG']) | ||
| if result["hits"]["total"]["value"] > 0: | ||
| doc = result["hits"]["hits"][0] | ||
| email_body += "\n\n%s" % get_metadata_snippet(doc, settings["SNIPPET_CFG"]) | ||
| email_body += "\n\nThe entire metadata json for this product has been attached for your convenience.\n\n" | ||
| email_attachments = { | ||
| 'metadata.json': json.dumps(doc, indent=2) # attach metadata json | ||
| "metadata.json": json.dumps(doc, indent=2) # attach metadata json | ||
| } | ||
|
|
||
| # attach browse images | ||
| if len(doc['browse_urls']) > 0: | ||
| browse_url = doc['browse_urls'][0] | ||
| if len(doc['images']) > 0: | ||
| doc_source = doc["_source"] | ||
| if len(doc_source["browse_urls"]) > 0: | ||
| browse_url = doc_source["browse_urls"][0] | ||
| if len(doc_source["images"]) > 0: | ||
| email_body += "Browse images have been attached as well.\n\n" | ||
| for i in doc['images']: | ||
| small_img = i['small_img'] | ||
| for i in doc_source["images"]: | ||
| small_img = i["small_img"] | ||
| small_img_url = os.path.join(browse_url, small_img) | ||
| r = requests.get(small_img_url) | ||
| if r.status_code != 200: | ||
| continue | ||
| email_attachments[small_img] = r.content | ||
| else: | ||
| doc = None | ||
| email_body += "\n\n" | ||
|
|
||
| email_body += "You may access the product here:\n\n%s" % url | ||
|
|
||
| system_version = None if doc is None else doc.get('system_version') | ||
| system_version = None if doc is None else doc.get("system_version") | ||
| facet_url = get_facetview_link(facetview_url, object_id, system_version) | ||
|
|
||
| if facet_url is not None: | ||
| email_body += "\n\nYou may view this product in FacetView here:\n\n%s" % facet_url | ||
| email_body += "\n\nNOTE: You may have to cut and paste the FacetView link into your " | ||
| email_body += ( | ||
| "\n\nYou may view this product in FacetView here:\n\n%s" % facet_url | ||
| ) | ||
| email_body += ( | ||
| "\n\nNOTE: You may have to cut and paste the FacetView link into your " | ||
| ) | ||
| email_body += "browser's address bar to prevent your email client from escaping the curly brackets." | ||
|
|
||
| username_email = "%s@%s" % (getpass.getuser(), get_hostname()) | ||
| send_email(username_email, bcc_recipients, email_subject, email_body, attachments=email_attachments) | ||
| send_email( | ||
| username_email, | ||
| cc_recipients, | ||
| bcc_recipients, | ||
| email_subject, | ||
| email_body, | ||
| attachments=email_attachments, | ||
| ) | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters