From 9ff9f9a903a07636b241261d790c6c506dca33a9 Mon Sep 17 00:00:00 2001 From: Gerald Manipon Date: Wed, 20 Mar 2019 09:07:45 -0700 Subject: [PATCH] python3 port (#5) * port to python3 (#4) * initial 2to3 run * remove supervisor from required (not python3 compatible yet) * format to PEP8 spec * fix location of views/services imports; fix encoding issues * pasteurized * remove imports not yet installed via install_requires * require future * bump version --- db_create.py | 6 ++ run.py | 9 +- run_debug.py | 6 ++ scripts/create_user_rules_index.py | 21 ++++- setup.py | 11 ++- tosca/__init__.py | 17 +++- tosca/lib/forms.py | 7 ++ tosca/lib/grq_utils.py | 23 ++++- tosca/lib/ldap.py | 28 ++++-- tosca/lib/query.py | 33 +++++-- tosca/lib/urs.py | 21 +++-- tosca/lib/zipstream.py | 141 +++++++++++++++------------- tosca/models/user.py | 11 ++- tosca/services/dataset.py | 29 ++++-- tosca/services/download.py | 55 ++++++++--- tosca/services/kml.py | 50 ++++++---- tosca/services/query.py | 9 +- tosca/services/user_rules.py | 144 ++++++++++++++++++----------- tosca/services/user_tags.py | 65 ++++++++----- tosca/services/wget.py | 123 +++++++++++++----------- tosca/views/js.py | 6 ++ tosca/views/main.py | 58 +++++++----- 22 files changed, 574 insertions(+), 299 deletions(-) diff --git a/db_create.py b/db_create.py index 6c6cf429..ab2b1bad 100755 --- a/db_create.py +++ b/db_create.py @@ -1,3 +1,9 @@ #!/usr/bin/env python +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() from tosca import db db.create_all() diff --git a/run.py b/run.py index 74298d97..6d8f8111 100644 --- a/run.py +++ b/run.py @@ -1,6 +1,11 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() from tosca import app if __name__ == '__main__': - context = ('server.pem', 'server.key') - app.run(host="0.0.0.0", port=8879, debug=True, ssl_context=context) + app.run(host="0.0.0.0", port=8879, debug=True) diff --git a/run_debug.py b/run_debug.py index 7ff2a595..ce5e1a71 100644 --- a/run_debug.py +++ b/run_debug.py @@ -1,3 +1,9 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() from tosca import app diff --git a/scripts/create_user_rules_index.py b/scripts/create_user_rules_index.py index b941a102..cb4d1244 100755 --- a/scripts/create_user_rules_index.py +++ b/scripts/create_user_rules_index.py @@ -1,16 +1,27 @@ #!/usr/bin/env python +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() from requests.exceptions import HTTPError from tosca import app from tosca.services.user_rules import create_user_rules_index, add_grq_mappings try: - create_user_rules_index(app.config['ES_URL'], app.config['USER_RULES_INDEX']) + create_user_rules_index( + app.config['ES_URL'], app.config['USER_RULES_INDEX']) except HTTPError as e: - if e.response.status_code == 400: pass - else: raise + if e.response.status_code == 400: + pass + else: + raise try: add_grq_mappings(app.config['ES_URL'], app.config['USER_RULES_INDEX']) except HTTPError as e: - if e.response.status_code == 404: pass - else: raise + if e.response.status_code == 404: + pass + else: + raise diff --git a/setup.py b/setup.py index 7e6c119c..d67f9a92 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,18 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import from setuptools import setup, find_packages setup( name='tosca', - version='0.2.3', + version='0.3.0', long_description='Advanced FacetView User Interface', packages=find_packages(), include_package_data=True, zip_safe=False, - install_requires=['Flask', 'gunicorn', 'gevent', 'supervisor', 'requests', + install_requires=['Flask', 'gunicorn', 'gevent', 'requests', 'Flask-SQLAlchemy', 'Flask-WTF', 'Flask-DebugToolbar', - 'Flask-Login', 'simpleldap', 'simplekml'] + 'Flask-Login', 'simpleldap', 'simplekml', + 'future>=0.17.1'] ) diff --git a/tosca/__init__.py b/tosca/__init__.py index 7ee1a811..c025523e 100644 --- a/tosca/__init__.py +++ b/tosca/__init__.py @@ -1,3 +1,9 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() import os from flask import Flask from flask_sqlalchemy import SQLAlchemy @@ -44,6 +50,7 @@ class ReverseProxied(object): :param app: the WSGI application ''' + def __init__(self, app): self.app = app @@ -70,7 +77,8 @@ def __call__(self, environ, start_response): # set database config dbdir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'data')) -app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + os.path.join(dbdir, 'app.db') +app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + \ + os.path.join(dbdir, 'app.db') db = SQLAlchemy(app) # debug toolbar @@ -84,22 +92,29 @@ def __call__(self, environ, start_response): # views blueprints from tosca.views.main import mod as viewsModule app.register_blueprint(viewsModule) + from tosca.views.js import mod as jsModule app.register_blueprint(jsModule) # services blueprints from tosca.services.query import mod as queryModule app.register_blueprint(queryModule) + from tosca.services.wget import mod as wgetModule app.register_blueprint(wgetModule) + from tosca.services.download import mod as downloadModule app.register_blueprint(downloadModule) + from tosca.services.user_tags import mod as userTagsModule app.register_blueprint(userTagsModule) + from tosca.services.user_rules import mod as userRulesModule app.register_blueprint(userRulesModule) + from tosca.services.kml import mod as kmlModule app.register_blueprint(kmlModule) + from tosca.services.dataset import mod as datasetModule app.register_blueprint(datasetModule) diff --git a/tosca/lib/forms.py b/tosca/lib/forms.py index 6bccf6d5..5ee48aaa 100644 --- a/tosca/lib/forms.py +++ b/tosca/lib/forms.py @@ -1,7 +1,14 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() from flask_wtf import Form from wtforms import TextField, BooleanField, PasswordField from wtforms.validators import Required + class LoginForm(Form): username = TextField('username', validators=[Required()]) password = PasswordField('password', validators=[Required()]) diff --git a/tosca/lib/grq_utils.py b/tosca/lib/grq_utils.py index 63ad8c96..50cdaebd 100644 --- a/tosca/lib/grq_utils.py +++ b/tosca/lib/grq_utils.py @@ -1,26 +1,39 @@ -import json, os +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import os from tosca import app import hysds_commons.request_utils -def mozart_call(method,data={}): + +def mozart_call(method, data={}): ''' Call mozart method with data @param method - method to call @param data - data to supply to call ''' - url = os.path.join(app.config['GRQ_URL'],"api/v0.1",method) + url = os.path.join(app.config['GRQ_URL'], "api/v0.1", method) getpost = "GET" - res = hysds.lib.request_utils.requests_json_response(getpost,url,data=data,verify=False,logger=app.logger) + res = hysds.lib.request_utils.requests_json_response( + getpost, url, data=data, verify=False, logger=app.logger) return res["result"] + + def get_hysds_io_list(): ''' Queries GRQ to get HySDS IOs ''' return mozart_call("hysds_io/list") + + def get_hysds_io(ident): ''' Queries GRQ to get HySDS Metadata object @param ident - identity to get ''' - return mozart_call("hysds_io/type",{"id":ident}) + return mozart_call("hysds_io/type", {"id": ident}) diff --git a/tosca/lib/ldap.py b/tosca/lib/ldap.py index 5c8598fc..029ac7c5 100644 --- a/tosca/lib/ldap.py +++ b/tosca/lib/ldap.py @@ -1,4 +1,13 @@ -import simpleldap, traceback +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import dict +from builtins import str +from future import standard_library +standard_library.install_aliases() +import simpleldap +import traceback from tosca import app @@ -10,20 +19,21 @@ def ldap_user_verified(username, password): host = app.config['LDAP_HOST'] base_dn = app.config['LDAP_BASEDN'] groups = app.config['LDAP_GROUPS'] - opts = { 'OPT_NETWORK_TIMEOUT': 10 } + opts = {'OPT_NETWORK_TIMEOUT': 10} try: l = simpleldap.Connection(host, dn='uid=%s,%s' % (username, base_dn), - encryption='ssl', password=password, options=opts) - except Exception, e: + encryption='ssl', password=password, options=opts) + except Exception as e: app.logger.info("Got error trying to verify LDAP user %s:" % username) app.logger.info("%s:\n\n%s" % (str(e), traceback.format_exc())) return None - + # validate user r = l.search('uid=%s' % username, base_dn=base_dn) if len(r) != 1: - app.logger.info("Got invalid number of entries for %s: %s" % (username, len(r))) + app.logger.info("Got invalid number of entries for %s: %s" % + (username, len(r))) app.logger.info("r: %s" % str(r)) return None @@ -32,7 +42,9 @@ def ldap_user_verified(username, password): for group in groups: g = l.search('cn=%s' % group, base_dn=base_dn) for this_g in g: - if uid in this_g['uniqueMember']: return dict(r[0]) + if uid in this_g['uniqueMember']: + return dict(r[0]) - app.logger.info("User %s is not part of any approved LDAP groups." % username) + app.logger.info( + "User %s is not part of any approved LDAP groups." % username) return None diff --git a/tosca/lib/query.py b/tosca/lib/query.py index 7b918b3b..ef4a49c6 100644 --- a/tosca/lib/query.py +++ b/tosca/lib/query.py @@ -1,7 +1,15 @@ -import json, requests, re +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import requests +import re from pprint import pformat from flask import url_for -from urllib import quote_plus +from urllib.parse import quote_plus from tosca import app @@ -18,7 +26,7 @@ def query(index, query_str): r = requests.post('%s/%s/_search' % (es_url, index), data=query_str) result = r.json() if r.status_code != 200: - app.logger.debug("Failed to query ES. Got status code %d:\n%s" % + app.logger.debug("Failed to query ES. Got status code %d:\n%s" % (r.status_code, json.dumps(result, indent=2))) r.raise_for_status() #app.logger.debug("result: %s" % pformat(r.json())) @@ -27,7 +35,8 @@ def query(index, query_str): for hit in result['hits']['hits']: # emulate result format from ElasticSearch <1.0 #app.logger.debug("hit: %s" % pformat(hit)) - if '_source' in hit: hit.setdefault('fields', {}).update(hit['_source']) + if '_source' in hit: + hit.setdefault('fields', {}).update(hit['_source']) hit['fields']['es_index'] = hit['_index'] # set dap url @@ -39,8 +48,9 @@ def query(index, query_str): # set redirector url if len(hit['fields']['urls']) == 0: hit['fields']['urls'] = None - else: hit['fields']['urls'] = [ url_for('services/dataset.resolve_url', - index=hit['_index'], id=hit['_id']) ] + else: + hit['fields']['urls'] = [url_for('services/dataset.resolve_url', + index=hit['_index'], id=hit['_id'])] # set closest city if len(hit['fields'].get('city', [])) > 0: @@ -60,7 +70,7 @@ def query_dataset_url(index, objectid): #app.logger.debug("url: %s" % r.url) result = r.json() if r.status_code != 200: - app.logger.debug("Failed to query ES. Got status code %d:\n%s" % + app.logger.debug("Failed to query ES. Got status code %d:\n%s" % (r.status_code, json.dumps(result, indent=2))) r.raise_for_status() #app.logger.info("result: %s" % json.dumps(result, indent=2)) @@ -97,7 +107,10 @@ def get_dataset_url(hit): ret_url = None for url in hit['_source'].get('urls', []): - if url.startswith('http'): ret_url = url - if url.startswith('http') and 'amazonaws.com' in url: ret_url = url - if url.startswith('http') and 'googleapis.com' in url: ret_url = url + if url.startswith('http'): + ret_url = url + if url.startswith('http') and 'amazonaws.com' in url: + ret_url = url + if url.startswith('http') and 'googleapis.com' in url: + ret_url = url return ret_url diff --git a/tosca/lib/urs.py b/tosca/lib/urs.py index 51ea3ea5..b2ebe23b 100644 --- a/tosca/lib/urs.py +++ b/tosca/lib/urs.py @@ -1,3 +1,10 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import str +from future import standard_library +standard_library.install_aliases() import traceback import json from flask import redirect, abort, request @@ -14,7 +21,7 @@ def urs_user_verified(code): # urs auth url urs_url = "{}/oauth/token".format(app.config['URS_URL']) - # set authorization value + # set authorization value auth_val = base64.b64encode("{}:{}".format(app.config['URS_APP_ID'], app.config['URS_APP_PASSWORD'])) @@ -26,9 +33,11 @@ def urs_user_verified(code): } # exchange - r = requests.post(urs_url, data=payload, headers={ "Authorization": "Basic {}".format(auth_val) }) - try: r.raise_for_status() - except Exception, e: + r = requests.post(urs_url, data=payload, headers={ + "Authorization": "Basic {}".format(auth_val)}) + try: + r.raise_for_status() + except Exception as e: app.logger.error("Got error trying to verify URS user.") app.logger.error("%s:\n\n%s" % (str(e), traceback.format_exc())) return None @@ -38,8 +47,8 @@ def urs_user_verified(code): # get user profile user_url = "{}{}".format(app.config['URS_URL'], packet['endpoint']) #app.logger.info("user_url: {}".format(user_url)) - r = requests.get(user_url, headers={ "Authorization": "{} {}".format(packet['token_type'], - packet['access_token']) }) + r = requests.get(user_url, headers={"Authorization": "{} {}".format(packet['token_type'], + packet['access_token'])}) r.raise_for_status() user_info = r.json() #app.logger.info("user_info: {}".format(json.dumps(user_info, indent=2, sort_keys=True))) diff --git a/tosca/lib/zipstream.py b/tosca/lib/zipstream.py index a2858977..1f222188 100644 --- a/tosca/lib/zipstream.py +++ b/tosca/lib/zipstream.py @@ -5,18 +5,30 @@ Derived directly from zipfile.py """ -import struct, os, time, sys +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import open +from builtins import chr +from future import standard_library +standard_library.install_aliases() +from builtins import object +import struct +import os +import time +import sys import binascii try: - import zlib # We may need its compression method + import zlib # We may need its compression method except ImportError: zlib = None __all__ = ["ZIP_STORED", "ZIP_DEFLATED", "ZipStream"] -ZIP64_LIMIT= (1 << 31) - 1 +ZIP64_LIMIT = (1 << 31) - 1 # constants for Zip file compression methods ZIP_STORED = 0 @@ -26,15 +38,16 @@ # Here are some struct module formats for reading headers structEndArchive = "<4s4H2lH" # 9 items, end of archive, 22 bytes stringEndArchive = "PK\005\006" # magic number for end of archive record -structCentralDir = "<4s4B4HlLL5HLl"# 19 items, central directory, 46 bytes +structCentralDir = "<4s4B4HlLL5HLl" # 19 items, central directory, 46 bytes stringCentralDir = "PK\001\002" # magic number for central directory structFileHeader = "<4s2B4HlLL2H" # 12 items, file header record, 30 bytes stringFileHeader = "PK\003\004" # magic number for file header -structEndArchive64Locator = "<4slql" # 4 items, locate Zip64 header, 20 bytes -stringEndArchive64Locator = "PK\x06\x07" # magic token for locator header -structEndArchive64 = "<4sqhhllqqqq" # 10 items, end of archive (Zip64), 56 bytes -stringEndArchive64 = "PK\x06\x06" # magic token for Zip64 header -stringDataDescriptor = "PK\x07\x08" # magic number for data descriptor +structEndArchive64Locator = "<4slql" # 4 items, locate Zip64 header, 20 bytes +stringEndArchive64Locator = "PK\x06\x07" # magic token for locator header +# 10 items, end of archive (Zip64), 56 bytes +structEndArchive64 = "<4sqhhllqqqq" +stringEndArchive64 = "PK\x06\x06" # magic token for Zip64 header +stringDataDescriptor = "PK\x07\x08" # magic number for data descriptor # indexes of entries in the central directory structure _CD_SIGNATURE = 0 @@ -76,27 +89,27 @@ class ZipInfo (object): """Class with attributes describing each file in the ZIP archive.""" __slots__ = ( - 'orig_filename', - 'filename', - 'date_time', - 'compress_type', - 'comment', - 'extra', - 'create_system', - 'create_version', - 'extract_version', - 'reserved', - 'flag_bits', - 'volume', - 'internal_attr', - 'external_attr', - 'header_offset', - 'CRC', - 'compress_size', - 'file_size', - ) - - def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): + 'orig_filename', + 'filename', + 'date_time', + 'compress_type', + 'comment', + 'extra', + 'create_system', + 'create_version', + 'extract_version', + 'reserved', + 'flag_bits', + 'volume', + 'internal_attr', + 'external_attr', + 'header_offset', + 'CRC', + 'compress_size', + 'file_size', + ) + + def __init__(self, filename="NoName", date_time=(1980, 1, 1, 0, 0, 0)): self.orig_filename = filename # Original file name in archive # Terminate the file name at the first null byte. Null bytes in file @@ -113,7 +126,7 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): self.filename = filename # Normalized file name self.date_time = date_time # year, month, day, hour, min, sec # Standard values: - self.compress_type = ZIP_STORED # Type of compression for the file + self.compress_type = ZIP_STORED # Type of compression for the file self.comment = "" # Comment for each file self.extra = "" # ZIP extra data if sys.platform == 'win32': @@ -124,7 +137,8 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): self.create_version = 20 # Version which created ZIP archive self.extract_version = 20 # Version needed to extract archive self.reserved = 0 # Must be zero - self.flag_bits = 0x08 # ZIP flag bits, bit 3 indicates presence of data descriptor + # ZIP flag bits, bit 3 indicates presence of data descriptor + self.flag_bits = 0x08 self.volume = 0 # Volume number of file header self.internal_attr = 0 # Internal attributes self.external_attr = 0 # External file attributes @@ -161,22 +175,21 @@ def FileHeader(self): # fall back to the ZIP64 extension fmt = ' ZIP64_LIMIT or zinfo.compress_size > ZIP64_LIMIT: extra.append(zinfo.file_size) extra.append(zinfo.compress_size) - file_size = 0xffffffff #-1 - compress_size = 0xffffffff #-1 + file_size = 0xffffffff # -1 + compress_size = 0xffffffff # -1 else: file_size = zinfo.file_size compress_size = zinfo.compress_size @@ -352,7 +364,8 @@ def archive_footer(self): extra_data = zinfo.extra if extra: # Append a ZIP64 field to the extra's - extra_data = struct.pack('' % (self.id) diff --git a/tosca/services/dataset.py b/tosca/services/dataset.py index c0679337..8d3da2b8 100644 --- a/tosca/services/dataset.py +++ b/tosca/services/dataset.py @@ -1,3 +1,9 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() import json from requests import HTTPError from flask import Blueprint, redirect, abort, request @@ -23,16 +29,23 @@ def resolve_url(index=None, id=None, path=None): # query err = "Unable to find dataset %s in index %s." % (id, index) - try: url = query_dataset_url(index, id) - except HTTPError, e: - if e.response.status_code == 404: abort(404, err) - else: raise(e) - if url is None: abort(500, err) + try: + url = query_dataset_url(index, id) + except HTTPError as e: + if e.response.status_code == 404: + abort(404, err) + else: + raise(e) + if url is None: + abort(500, err) # check if noredirect is set; if so just return url noredirect = request.args.get('noredirect', "false") - if noredirect in ('true', 'True', 'TRUE'): return url + if noredirect in ('true', 'True', 'TRUE'): + return url # redirect - if path is None: return redirect(url) - else: return redirect("%s/%s" % (url, path)) + if path is None: + return redirect(url) + else: + return redirect("%s/%s" % (url, path)) diff --git a/tosca/services/download.py b/tosca/services/download.py index a8c728c4..3e851ca7 100644 --- a/tosca/services/download.py +++ b/tosca/services/download.py @@ -1,8 +1,21 @@ -import os, json, requests, types, re, hashlib, shutil, subprocess +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import os +import json +import requests +import types +import re +import hashlib +import shutil +import subprocess from flask import jsonify, Blueprint, request, url_for from flask_login import login_required from pprint import pformat -from urlparse import urlparse +from urllib.parse import urlparse from tosca import app @@ -27,9 +40,10 @@ def download(dataset=None): es_url = app.config['ES_URL'] index = dataset app.logger.debug("ES query for download(): %s" % source) - r = requests.post('%s/%s/_search?search_type=scan&scroll=10m&size=100' % (es_url, index), data=source) + r = requests.post('%s/%s/_search?search_type=scan&scroll=10m&size=100' % + (es_url, index), data=source) if r.status_code != 200: - app.logger.debug("Failed to query ES. Got status code %d:\n%s" % + app.logger.debug("Failed to query ES. Got status code %d:\n%s" % (r.status_code, json.dumps(result, indent=2))) r.raise_for_status() #app.logger.debug("result: %s" % pformat(r.json())) @@ -41,14 +55,18 @@ def download(dataset=None): # get list of urls urls = [] while True: - r = requests.post('%s/_search/scroll?scroll=10m' % es_url, data=scroll_id) + r = requests.post('%s/_search/scroll?scroll=10m' % + es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: # emulate result format from ElasticSearch <1.0 - if '_source' in hit: hit['fields'].update(hit['_source']) - if len(hit['fields']['urls']) > 0: urls.append(hit['fields']['urls'][0]) + if '_source' in hit: + hit['fields'].update(hit['_source']) + if len(hit['fields']['urls']) > 0: + urls.append(hit['fields']['urls'][0]) # generate tarball urls.sort() @@ -56,10 +74,14 @@ def download(dataset=None): m.update(json.dumps(urls)) digest = m.hexdigest() sync_dir = os.path.join(app.config['SCRATCH_DIR'], 'data') - if not os.path.exists(sync_dir): os.makedirs(sync_dir) - dl_dir = os.path.join(app.config['SCRATCH_DIR'], 'downloads', digest, 'download') - if not os.path.exists(dl_dir): os.makedirs(dl_dir) - tar_file_path = os.path.join(app.config['SCRATCH_DIR'], 'downloads', digest, '%s.tbz2' % digest) + if not os.path.exists(sync_dir): + os.makedirs(sync_dir) + dl_dir = os.path.join( + app.config['SCRATCH_DIR'], 'downloads', digest, 'download') + if not os.path.exists(dl_dir): + os.makedirs(dl_dir) + tar_file_path = os.path.join( + app.config['SCRATCH_DIR'], 'downloads', digest, '%s.tbz2' % digest) if not os.path.exists(tar_file_path): for i, f in enumerate(urls): p = urlparse(f) @@ -69,12 +91,15 @@ def download(dataset=None): if not os.path.exists(src_dir): cmd = 'wget --no-check-certificate -P %s ' % sync_dir cmd += '--mirror -nv -np -nH --reject "index.html*" ' - cmd += '--user=pucops --password=puc_0ps --cut-dirs=%s %s' % (cut_dirs, f) + cmd += '--user=pucops --password=puc_0ps --cut-dirs=%s %s' % ( + cut_dirs, f) sts = subprocess.call(cmd, shell=True) src_dir = os.path.join(sync_dir, prod_name) dest_dir = os.path.join(dl_dir, prod_name) - if not os.path.exists(dest_dir): os.symlink(src_dir, dest_dir) - sts = subprocess.call('tar cfhj %s %s' % (tar_file_path, dl_dir), shell=True) + if not os.path.exists(dest_dir): + os.symlink(src_dir, dest_dir) + sts = subprocess.call('tar cfhj %s %s' % + (tar_file_path, dl_dir), shell=True) return jsonify({ 'success': True, diff --git a/tosca/services/kml.py b/tosca/services/kml.py index 631be16e..e927166c 100644 --- a/tosca/services/kml.py +++ b/tosca/services/kml.py @@ -1,4 +1,12 @@ -import os, json, requests +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import os +import json +import requests from flask import jsonify, Blueprint, request, url_for, Response from flask_login import login_required from pprint import pformat @@ -26,9 +34,10 @@ def get_kml(dataset=None): # query es_url = app.config['ES_URL'] index = dataset - r = requests.post('%s/%s/_search?search_type=scan&scroll=10m&size=100' % (es_url, index), data=source) + r = requests.post('%s/%s/_search?search_type=scan&scroll=10m&size=100' % + (es_url, index), data=source) if r.status_code != 200: - app.logger.debug("Failed to query ES. Got status code %d:\n%s" % + app.logger.debug("Failed to query ES. Got status code %d:\n%s" % (r.status_code, json.dumps(result, indent=2))) r.raise_for_status() #app.logger.debug("result: %s" % pformat(r.json())) @@ -40,10 +49,12 @@ def get_kml(dataset=None): # get list of results results = [] while True: - r = requests.post('%s/_search/scroll?scroll=10m' % es_url, data=scroll_id) + r = requests.post('%s/_search/scroll?scroll=10m' % + es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: del hit['_source']['city'] results.append(hit['_source']) @@ -55,7 +66,8 @@ def get_kml(dataset=None): for res in results: id = res['id'] url = res['urls'][0] if len(res['urls']) > 0 else None - browse_url = res['browse_urls'][0] if len(res['browse_urls']) > 0 else None + browse_url = res['browse_urls'][0] if len( + res['browse_urls']) > 0 else None location = res['location'] bbox = res['metadata']['bbox'] #app.logger.debug("%s" % json.dumps(res['metadata'], indent=2)) @@ -63,14 +75,14 @@ def get_kml(dataset=None): if len(img_coords) == 0: if 'dfdn' in res['metadata']: if res['metadata']['direction'] == 'asc': - lowerleft = res['metadata']['dfdn']['GeoCoordTopLeft'] + lowerleft = res['metadata']['dfdn']['GeoCoordTopLeft'] lowerright = res['metadata']['dfdn']['GeoCoordTopRight'] - upperleft = res['metadata']['dfdn']['GeoCoordBottomLeft'] + upperleft = res['metadata']['dfdn']['GeoCoordBottomLeft'] upperright = res['metadata']['dfdn']['GeoCoordBottomRight'] else: - upperright = res['metadata']['dfdn']['GeoCoordTopLeft'] + upperright = res['metadata']['dfdn']['GeoCoordTopLeft'] upperleft = res['metadata']['dfdn']['GeoCoordTopRight'] - lowerright = res['metadata']['dfdn']['GeoCoordBottomLeft'] + lowerright = res['metadata']['dfdn']['GeoCoordBottomLeft'] lowerleft = res['metadata']['dfdn']['GeoCoordBottomRight'] else: #img_coords['minLon'] = bbox[2][1] @@ -81,18 +93,18 @@ def get_kml(dataset=None): #lowerright = [img_coords['minLat'], img_coords['maxLon'], 0.] #upperright = [img_coords['maxLat'], img_coords['maxLon'], 0.] #upperleft = [img_coords['maxLat'], img_coords['minLon'], 0.] - lowerleft = [bbox[0][0], bbox[0][1], 0.] - lowerright = [bbox[1][0], bbox[1][1], 0.] - upperright = [bbox[2][0], bbox[2][1], 0.] - upperleft = [bbox[3][0], bbox[3][1], 0.] + lowerleft = [bbox[0][0], bbox[0][1], 0.] + lowerright = [bbox[1][0], bbox[1][1], 0.] + upperright = [bbox[2][0], bbox[2][1], 0.] + upperleft = [bbox[3][0], bbox[3][1], 0.] else: - lowerleft = [img_coords['minLat'], img_coords['minLon'], 0.] + lowerleft = [img_coords['minLat'], img_coords['minLon'], 0.] lowerright = [img_coords['minLat'], img_coords['maxLon'], 0.] upperright = [img_coords['maxLat'], img_coords['maxLon'], 0.] - upperleft = [img_coords['maxLat'], img_coords['minLon'], 0.] + upperleft = [img_coords['maxLat'], img_coords['minLon'], 0.] ground = kml.newgroundoverlay(name=id) ground.description = id - #if location['type'] == 'polygon': + # if location['type'] == 'polygon': # ground.gxlatlonquad.coords = location['coordinates'][0] ground.gxlatlonquad.coords = [ (lowerleft[1], lowerleft[0]), @@ -107,7 +119,7 @@ def get_kml(dataset=None): if browse_url: ground.icon.href = '%s/browse_small.png' % browse_url - + return Response(kml.kml(), headers={'Content-Type': 'application/vnd.google-earth.kml+xml', 'Content-Disposition': 'attachment; filename=%s.kml' % dataset}) - #return jsonify({'results': results}) + # return jsonify({'results': results}) diff --git a/tosca/services/query.py b/tosca/services/query.py index 6d1188d9..2dde14e7 100644 --- a/tosca/services/query.py +++ b/tosca/services/query.py @@ -1,4 +1,11 @@ -import json, requests +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import requests from flask import jsonify, Blueprint, request, Response, render_template, make_response from flask_login import login_required diff --git a/tosca/services/user_rules.py b/tosca/services/user_rules.py index 957672dc..f3fc4650 100644 --- a/tosca/services/user_rules.py +++ b/tosca/services/user_rules.py @@ -1,12 +1,28 @@ -import os, json, requests, types, re, time, copy, traceback +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import open +from builtins import int +from future import standard_library +standard_library.install_aliases() +import os +import json +import requests +import types +import re +import time +import copy +import traceback from flask import (jsonify, Blueprint, request, Response, render_template, -make_response, g, url_for, redirect) + make_response, g, url_for, redirect) from flask_login import login_required from pprint import pformat from string import Template -from urlparse import urljoin +from urllib.parse import urljoin from datetime import datetime -import functools, operator +import functools +import operator from hysds.celery import app as celery_app from hysds.task_worker import do_submit_task @@ -60,30 +76,36 @@ def add_grq_mappings(es_url, es_index): r = requests.put("%s/%s/_mapping/%s" % (es_url, es_index, doc_type), data=json.dumps(mappings[idx]['mappings'][doc_type])) r.raise_for_status() + + @mod.route('/user_rules/get_jobspec_names', methods=['GET']) def get_jobspecs(): """Get the list of jobspecs""" try: jspecs = hysds_commons.job_spec_utils.get_job_spec_types(app.config["MOZART_ES_URL"], logger=app.logger) - except requests.exceptions.HTTPError, e: + except requests.exceptions.HTTPError as e: if e.response.status_code == 404: jspecs = [] - else: raise + else: + raise return jsonify({"jobspecs": jspecs}) + @mod.route('/user_rules/get_container_names', methods=['GET']) def get_containers(): """Get the list of containers""" try: cspecs = hysds_commons.container_utils.get_container_types(app.config["MOZART_ES_URL"], logger=app.logger) - except requests.exceptions.HTTPError, e: + except requests.exceptions.HTTPError as e: if e.response.status_code == 404: cspecs = [] - else: raise + else: + raise return jsonify({"containers": cspecs}) + @mod.route('/user_rules/actions_config', methods=['GET']) @login_required def get_actions_config(): @@ -103,13 +125,15 @@ def get_actions_config(): app.config["MOZART_ES_URL"], app.config["OPS_USER"]), key=lambda s: s['label'].lower()) - except requests.exceptions.HTTPError, e: + except requests.exceptions.HTTPError as e: if e.response.status_code == 404: action_specs = [] - else: raise + else: + raise actions = [] for action in action_specs: - if not action[ifc_filter]: continue + if not action[ifc_filter]: + continue if action['public'] is False: if g.user.id in action.get('allowed_accounts', []): action['public'] = True @@ -124,7 +148,8 @@ def get_job_queues(): """Return list of job queues.""" job_type = request.args.get("job_type") - queues = hysds_commons.mozart_utils.get_queue_list(app.config["MOZART_REST_API"], job_type) + queues = hysds_commons.mozart_utils.get_queue_list( + app.config["MOZART_REST_API"], job_type) return jsonify(queues) @@ -172,20 +197,22 @@ def add_user_rule(): "query": { "bool": { "must": [ - { "term": { "username": g.user.id } }, - { "term": { "rule_name": rule_name } }, + {"term": {"username": g.user.id}}, + {"term": {"rule_name": rule_name}}, ] } } } - r = requests.post('%s/%s/.percolator/_search' % (es_url, es_index), data=json.dumps(query)) + r = requests.post('%s/%s/.percolator/_search' % + (es_url, es_index), data=json.dumps(query)) result = r.json() if r.status_code != 200: - app.logger.debug("Failed to query ES. Got status code %d:\n%s" % + app.logger.debug("Failed to query ES. Got status code %d:\n%s" % (r.status_code, json.dumps(result, indent=2))) r.raise_for_status() if result['hits']['total'] == 1: - app.logger.debug("Found a rule using that name already: %s" % rule_name) + app.logger.debug( + "Found a rule using that name already: %s" % rule_name) return jsonify({ 'success': False, 'message': "Found a rule using that name already: %s" % rule_name, @@ -204,7 +231,7 @@ def add_user_rule(): job_type = action['job_type'] passthru_query = action.get('passthru_query', False) query_all = action.get('query_all', False) - if job_type is None: + if job_type is None: app.logger.debug("No job_type find for '%s'." % workflow) return jsonify({ 'success': False, @@ -225,17 +252,18 @@ def add_user_rule(): "query": json.loads(query_string), "passthru_query": passthru_query, "query_all": query_all, - "queue":queue, + "queue": queue, "modified_time": mtime, "creation_time": mtime, } - r = requests.post('%s/%s/.percolator/' % (es_url, es_index), data=json.dumps(new_doc)) + r = requests.post('%s/%s/.percolator/' % + (es_url, es_index), data=json.dumps(new_doc)) result = r.json() if r.status_code != 201: - app.logger.debug("Failed to insert new rule for %s. Got status code %d:\n%s" % + app.logger.debug("Failed to insert new rule for %s. Got status code %d:\n%s" % (g.user.id, r.status_code, json.dumps(result, indent=2))) r.raise_for_status() - + return jsonify({ 'success': True, 'message': "", @@ -262,13 +290,13 @@ def get_user_rules(): # query if g.user.id == app.config['OPS_USER']: - query = {"query":{"match_all": {}}} + query = {"query": {"match_all": {}}} else: query = { "query": { "bool": { "must": [ - { "term": { "username": g.user.id } } + {"term": {"username": g.user.id}} ] } } @@ -276,7 +304,7 @@ def get_user_rules(): r = requests.post('%s/%s/.percolator/_search?search_type=scan&scroll=10m&size=100' % (es_url, es_index), data=json.dumps(query)) if r.status_code != 200: - app.logger.debug("Failed to query ES. Got status code %d:\n%s" % + app.logger.debug("Failed to query ES. Got status code %d:\n%s" % (r.status_code, json.dumps(r.content, indent=2))) r.raise_for_status() #app.logger.debug("result: %s" % pformat(r.json())) @@ -289,18 +317,20 @@ def get_user_rules(): rules = [] rule_count = 0 while True: - r = requests.post('%s/_search/scroll?scroll=10m' % es_url, data=scroll_id) + r = requests.post('%s/_search/scroll?scroll=10m' % + es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: rule_count += 1 rule = hit['_source'] - rule['#'] = rule_count + rule['#'] = rule_count rule['id'] = hit['_id'] rules.append(rule) - return jsonify({ "success": True, "rules": rules}) + return jsonify({"success": True, "rules": rules}) @mod.route('/user_rules/remove', methods=['POST']) @@ -327,7 +357,7 @@ def remove_user_rule(): app.logger.debug("Failed to delete rule with ID %s. Got status code %d" % (id, r.status_code)) r.raise_for_status() - + return jsonify({ 'success': True, 'message': "", @@ -352,8 +382,10 @@ def toggle_status(): app.logger.debug("Setting enabled to '%s' to id '%s'." % (enabled, id)) # update enabled - if enabled == "true": enabled = True - else: enabled = False + if enabled == "true": + enabled = True + else: + enabled = False new_doc = { "doc": { "enabled": enabled, @@ -363,13 +395,14 @@ def toggle_status(): } es_url = app.config['ES_URL'] es_index = app.config['USER_RULES_INDEX'] - r = requests.post('%s/%s/.percolator/%s/_update' % (es_url, es_index, id), data=json.dumps(new_doc)) + r = requests.post('%s/%s/.percolator/%s/_update' % + (es_url, es_index, id), data=json.dumps(new_doc)) result = r.json() if r.status_code != 200: - app.logger.debug("Failed to update enabled field for %s. Got status code %d:\n%s" % + app.logger.debug("Failed to update enabled field for %s. Got status code %d:\n%s" % (id, r.status_code, json.dumps(result, indent=2))) r.raise_for_status() - + return jsonify({ 'success': True, 'message': "" @@ -416,7 +449,7 @@ def edit_user_rule(): job_type = action['job_type'] passthru_query = action.get('passthru_query', False) query_all = action.get('query_all', False) - if job_type is None: + if job_type is None: app.logger.debug("No job_type find for '%s'." % workflow) return jsonify({ 'success': False, @@ -437,7 +470,7 @@ def edit_user_rule(): "query": json.loads(query_string), "passthru_query": passthru_query, "query_all": query_all, - "queue":queue, + "queue": queue, "modified_time": get_utc_time(), }, "doc_as_upsert": True @@ -447,13 +480,14 @@ def edit_user_rule(): #url = '%s/%s/%s/%s/_update' % (es_url, es_index, 'user_rule', id) #app.logger.debug("url: %s" % url) #app.logger.debug("data: %s" % json.dumps(new_doc, indent=2)) - r = requests.post('%s/%s/.percolator/%s/_update' % (es_url, es_index, id), data=json.dumps(new_doc)) + r = requests.post('%s/%s/.percolator/%s/_update' % + (es_url, es_index, id), data=json.dumps(new_doc)) result = r.json() if r.status_code != 200: - app.logger.debug("Failed to update rule for %s. Got status code %d:\n%s" % + app.logger.debug("Failed to update rule for %s. Got status code %d:\n%s" % (id, r.status_code, json.dumps(result, indent=2))) r.raise_for_status() - + return jsonify({ 'success': True, 'message': "", @@ -470,7 +504,7 @@ def process_this(): name = request.form['name'] workflow = request.form['workflow'] priority = int(request.form.get('priority', 0)) - queue = request.form.get('queue',None) + queue = request.form.get('queue', None) query_string = request.form['query_string'] kwargs = request.form['kwargs'] if workflow is None: @@ -525,7 +559,7 @@ def process_this(): 'query': json.loads(query_string), 'passthru_query': matched_action.get('passthru_query', False), 'query_all': matched_action.get('query_all', False), - 'queue' : queue + 'queue': queue } # if job_type is a url type, pass it on @@ -537,7 +571,7 @@ def process_this(): # html result html = "Retrieve the result of your job " html += "here." % result_link - + return jsonify({ 'success': True, 'message': "", @@ -548,7 +582,7 @@ def process_this(): payload = { 'type': 'job_iterator', 'function': 'hysds_commons.job_iterator.iterate', - 'args': [ "tosca", rule ], + 'args': ["tosca", rule], } do_submit_task(payload, celery_app.conf['ON_DEMAND_DATASET_QUEUE']) @@ -566,10 +600,12 @@ def process_this(): # monitor jobs by tag and username try: - monitor_url = url_for('my_jobs.index', path="#/tags/{0}".format(rule['rule_name'])) - monitor_url = monitor_url.replace("%23","#") #Remove url encoding + monitor_url = url_for( + 'my_jobs.index', path="#/tags/{0}".format(rule['rule_name'])) + monitor_url = monitor_url.replace("%23", "#") # Remove url encoding except werkzeug.routing.BuildError: - monitor_url = url_for('.monitor_jobs', tag=rule['rule_name'], username=rule['username']) + monitor_url = url_for( + '.monitor_jobs', tag=rule['rule_name'], username=rule['username']) html = "Your HySDS jobs were submitted. Monitor their execution " html += 'here.' % monitor_url @@ -595,10 +631,12 @@ def monitor_job(): app.logger.debug("orch_task_id: %s" % orch_task_id) # get orchestrator task ID of job - r = requests.get('%s/api/task/info/%s' % (app.config['FLOWER_URL'], orch_task_id)) + r = requests.get('%s/api/task/info/%s' % + (app.config['FLOWER_URL'], orch_task_id)) orch_task_info = r.json() if r.status_code != 200: - app.logger.debug("Failed to retrieve orchestrator task info for %s." % orch_task_id) + app.logger.debug( + "Failed to retrieve orchestrator task info for %s." % orch_task_id) r.raise_for_status() # check state and get task id @@ -606,15 +644,17 @@ def monitor_job(): task_id = eval(eval(orch_task_info['result']))[0] app.logger.debug("task_id: %s" % task_id) else: - app.logger.debug("Failed to retrieve orchestrator task state of 'SUCCESS'.") + app.logger.debug( + "Failed to retrieve orchestrator task state of 'SUCCESS'.") return jsonify({ 'success': False, 'message': "Failed to retrieve orchestrator task with state 'SUCCESS'.", 'task_info': orch_task_info, }) - + # get Mozart link to job - moz_url = '%s/?source={"query":{"query_string":{"query":"task_id:%s"}}}' % (app.config['MOZART_URL'], task_id) + moz_url = '%s/?source={"query":{"query_string":{"query":"task_id:%s"}}}' % ( + app.config['MOZART_URL'], task_id) # hand craft redirect so that { and } chars are not escaped return redirect(moz_url) diff --git a/tosca/services/user_tags.py b/tosca/services/user_tags.py index 03f21692..73ac983d 100644 --- a/tosca/services/user_tags.py +++ b/tosca/services/user_tags.py @@ -1,4 +1,13 @@ -import json, requests, types, re +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import requests +import types +import re from flask import jsonify, Blueprint, request, Response, render_template, make_response from flask_login import login_required from pprint import pformat @@ -24,18 +33,20 @@ def add_user_tag(): 'message': "Product ID not specified." }), 500 - app.logger.debug("Adding tag '%s' to id '%s' in index '%s'." % (tag, id, es_index)) + app.logger.debug("Adding tag '%s' to id '%s' in index '%s'." % + (tag, id, es_index)) # query es_url = app.config['ES_URL'] query = { - "fields": [ "metadata.user_tags" ], - "query": { "term": { "_id": id } } + "fields": ["metadata.user_tags"], + "query": {"term": {"_id": id}} } - r = requests.post('%s/%s/_search' % (es_url, es_index), data=json.dumps(query)) + r = requests.post('%s/%s/_search' % + (es_url, es_index), data=json.dumps(query)) result = r.json() if r.status_code != 200: - app.logger.debug("Failed to query ES. Got status code %d:\n%s" % + app.logger.debug("Failed to query ES. Got status code %d:\n%s" % (r.status_code, json.dumps(result, indent=2))) r.raise_for_status() if result['hits']['total'] != 1: @@ -48,23 +59,26 @@ def add_user_tag(): # get actual index (no aliases), doctype and user tags actual_index = result['hits']['hits'][0]['_index'] doctype = result['hits']['hits'][0]['_type'] - user_tags = result['hits']['hits'][0].get('fields', {}).get('metadata.user_tags', []) + user_tags = result['hits']['hits'][0].get( + 'fields', {}).get('metadata.user_tags', []) # add tag if not already there - if tag not in user_tags: user_tags.append(tag) + if tag not in user_tags: + user_tags.append(tag) # upsert new document new_doc = { - "doc": { "metadata": { "user_tags": user_tags } }, + "doc": {"metadata": {"user_tags": user_tags}}, "doc_as_upsert": True } - r = requests.post('%s/%s/%s/%s/_update' % (es_url, actual_index, doctype, id), data=json.dumps(new_doc)) + r = requests.post('%s/%s/%s/%s/_update' % + (es_url, actual_index, doctype, id), data=json.dumps(new_doc)) result = r.json() if r.status_code != 200: - app.logger.debug("Failed to update user_tags for %s. Got status code %d:\n%s" % + app.logger.debug("Failed to update user_tags for %s. Got status code %d:\n%s" % (id, r.status_code, json.dumps(result, indent=2))) r.raise_for_status() - + return jsonify({ 'success': True, 'message': "" @@ -86,18 +100,20 @@ def remove_user_tag(): 'message': "Product ID not specified." }), 500 - app.logger.debug("Removing tag '%s' from id '%s' in '%s'." % (tag, id, es_index)) + app.logger.debug("Removing tag '%s' from id '%s' in '%s'." % + (tag, id, es_index)) # query es_url = app.config['ES_URL'] query = { - "fields": [ "metadata.user_tags" ], - "query": { "term": { "_id": id } } + "fields": ["metadata.user_tags"], + "query": {"term": {"_id": id}} } - r = requests.post('%s/%s/_search' % (es_url, es_index), data=json.dumps(query)) + r = requests.post('%s/%s/_search' % + (es_url, es_index), data=json.dumps(query)) result = r.json() if r.status_code != 200: - app.logger.debug("Failed to query ES. Got status code %d:\n%s" % + app.logger.debug("Failed to query ES. Got status code %d:\n%s" % (r.status_code, json.dumps(result, indent=2))) r.raise_for_status() if result['hits']['total'] != 1: @@ -110,23 +126,26 @@ def remove_user_tag(): # get actual index (no aliases), doctype and user tags actual_index = result['hits']['hits'][0]['_index'] doctype = result['hits']['hits'][0]['_type'] - user_tags = result['hits']['hits'][0].get('fields', {}).get('metadata.user_tags', []) + user_tags = result['hits']['hits'][0].get( + 'fields', {}).get('metadata.user_tags', []) # add tag if not already there - if tag in user_tags: user_tags.remove(tag) + if tag in user_tags: + user_tags.remove(tag) # upsert new document new_doc = { - "doc": { "metadata": { "user_tags": user_tags } }, + "doc": {"metadata": {"user_tags": user_tags}}, "doc_as_upsert": True } - r = requests.post('%s/%s/%s/%s/_update' % (es_url, actual_index, doctype, id), data=json.dumps(new_doc)) + r = requests.post('%s/%s/%s/%s/_update' % + (es_url, actual_index, doctype, id), data=json.dumps(new_doc)) result = r.json() if r.status_code != 200: - app.logger.debug("Failed to update user_tags for %s. Got status code %d:\n%s" % + app.logger.debug("Failed to update user_tags for %s. Got status code %d:\n%s" % (id, r.status_code, json.dumps(result, indent=2))) r.raise_for_status() - + return jsonify({ 'success': True, 'message': "" diff --git a/tosca/services/wget.py b/tosca/services/wget.py index 7c803465..1e52f932 100644 --- a/tosca/services/wget.py +++ b/tosca/services/wget.py @@ -1,4 +1,13 @@ -import json, requests, types, re +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import requests +import types +import re from flask import jsonify, Blueprint, request, Response, render_template, make_response from flask_login import login_required from pprint import pformat @@ -6,7 +15,7 @@ from tosca import app import boto3 -from urlparse import urlparse +from urllib.parse import urlparse mod = Blueprint('services/wget', __name__) @@ -40,15 +49,17 @@ def wget_script(dataset=None): #app.logger.debug("ES src for wget_script(): %s" % json.dumps(src, indent=2)) new_src = {} for k in src: - if k != "facets": new_src[k] = src[k] + if k != "facets": + new_src[k] = src[k] #app.logger.debug("ES new_src for wget_script(): %s" % json.dumps(new_src, indent=2)) # query es_url = app.config['ES_URL'] index = dataset - r = requests.post('%s/%s/_search?search_type=scan&scroll=10m&size=100' % (es_url, index), data=json.dumps(new_src)) + r = requests.post('%s/%s/_search?search_type=scan&scroll=10m&size=100' % + (es_url, index), data=json.dumps(new_src)) if r.status_code != 200: - app.logger.debug("Failed to query ES. Got status code %d:\n%s" % + app.logger.debug("Failed to query ES. Got status code %d:\n%s" % (r.status_code, json.dumps(r.json(), indent=2))) r.raise_for_status() #app.logger.debug("result: %s" % pformat(r.json())) @@ -72,62 +83,68 @@ def stream_wget(scroll_id, source): wget_cmd_password = wget_cmd + ' --user=$user --password=$password' while True: - r = requests.post('%s/_search/scroll?scroll=10m' % es_url, data=scroll_id) + r = requests.post('%s/_search/scroll?scroll=10m' % + es_url, data=scroll_id) res = r.json() #app.logger.debug("res: %s" % pformat(res)) scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break - # Elastic Search seems like it's returning duplicate urls. Remove duplicates - unique_urls=[] + if len(res['hits']['hits']) == 0: + break + # Elastic Search seems like it's returning duplicate urls. Remove duplicates + unique_urls = [] for hit in res['hits']['hits']: - [unique_urls.append(url) for url in hit['_source']['urls'] if url not in unique_urls] - - for url in unique_urls: - if 'hysds-aria-products.s3-website' in url: - parsed_url = urlparse(url) - cut_dirs = len(parsed_url.path[1:].split('/')) - 1 - else: - if 's1a_ifg' in url: - cut_dirs = 3 - else: - cut_dirs = 6 - if 'hysds-aria-products.s3-website' in url: - files = get_s3_files(url) - for file in files: - yield 'echo "downloading %s"\n' % file - if 's1a_ifg' in url: - yield "%s --cut-dirs=%d %s\n" % (wget_cmd, cut_dirs, file) - else: - yield "%s --cut-dirs=%d %s\n" % (wget_cmd, cut_dirs, file) + [unique_urls.append(url) for url in hit['_source'] + ['urls'] if url not in unique_urls] + + for url in unique_urls: + if 'hysds-aria-products.s3-website' in url: + parsed_url = urlparse(url) + cut_dirs = len(parsed_url.path[1:].split('/')) - 1 + else: + if 's1a_ifg' in url: + cut_dirs = 3 + else: + cut_dirs = 6 + if 'hysds-aria-products.s3-website' in url: + files = get_s3_files(url) + for file in files: + yield 'echo "downloading %s"\n' % file + if 's1a_ifg' in url: + yield "%s --cut-dirs=%d %s\n" % (wget_cmd, cut_dirs, file) + else: + yield "%s --cut-dirs=%d %s\n" % (wget_cmd, cut_dirs, file) if 'aria-dav.jpl.nasa.gov' in url: - yield 'echo "downloading %s"\n' % url - yield "%s --cut-dirs=%d %s/\n" % (wget_cmd_password, (cut_dirs+1), url) + yield 'echo "downloading %s"\n' % url + yield "%s --cut-dirs=%d %s/\n" % (wget_cmd_password, (cut_dirs+1), url) if 'aria-csk-dav.jpl.nasa.gov' in url: - yield 'echo "downloading %s"\n' % url - yield "%s --cut-dirs=%d %s/\n" % (wget_cmd_password, (cut_dirs+1), url) + yield 'echo "downloading %s"\n' % url + yield "%s --cut-dirs=%d %s/\n" % (wget_cmd_password, (cut_dirs+1), url) if 'aria-dst-dav.jpl.nasa.gov' in url: - yield 'echo "downloading %s"\n' % url - yield "%s --cut-dirs=%d %s/\n" % (wget_cmd, cut_dirs, url) - break + yield 'echo "downloading %s"\n' % url + yield "%s --cut-dirs=%d %s/\n" % (wget_cmd, cut_dirs, url) + break headers = {'Content-Disposition': 'attachment; filename=wget.sh'} - return Response(stream_wget(scroll_id, source), headers=headers, mimetype="text/plain") + return Response(stream_wget(scroll_id, source), headers=headers, mimetype="text/plain") + def get_s3_files(url): - files = [] - parsed_url = urlparse(url) - bucket = parsed_url.hostname.split('.', 1)[0] - client = boto3.client('s3') - results = client.list_objects(Bucket=bucket, Delimiter='/', Prefix=parsed_url.path[1:] + '/') - - if results.get('Contents'): - for result in results.get('Contents'): - files.append(parsed_url.scheme + "://" + parsed_url.hostname + '/' + result.get('Key')) - - if results.get('CommonPrefixes'): - for result in results.get('CommonPrefixes'): - # Prefix values have a trailing '/'. Let's remove it to be consistent with our dir urls - folder = parsed_url.scheme + "://" + parsed_url.hostname + '/' + result.get('Prefix')[:-1] - files.extend(get_s3_files(folder)) - return files - + files = [] + parsed_url = urlparse(url) + bucket = parsed_url.hostname.split('.', 1)[0] + client = boto3.client('s3') + results = client.list_objects( + Bucket=bucket, Delimiter='/', Prefix=parsed_url.path[1:] + '/') + + if results.get('Contents'): + for result in results.get('Contents'): + files.append(parsed_url.scheme + "://" + + parsed_url.hostname + '/' + result.get('Key')) + + if results.get('CommonPrefixes'): + for result in results.get('CommonPrefixes'): + # Prefix values have a trailing '/'. Let's remove it to be consistent with our dir urls + folder = parsed_url.scheme + "://" + \ + parsed_url.hostname + '/' + result.get('Prefix')[:-1] + files.extend(get_s3_files(folder)) + return files diff --git a/tosca/views/js.py b/tosca/views/js.py index 5c7df423..3003daed 100644 --- a/tosca/views/js.py +++ b/tosca/views/js.py @@ -1,3 +1,9 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() from datetime import datetime from flask import render_template, Blueprint, Response diff --git a/tosca/views/main.py b/tosca/views/main.py index 0c5f938f..eaa92a72 100644 --- a/tosca/views/main.py +++ b/tosca/views/main.py @@ -1,7 +1,15 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import str +from future import standard_library +standard_library.install_aliases() from datetime import datetime -import hashlib, simpleldap +import hashlib +import simpleldap from flask import (render_template, Blueprint, g, redirect, session, request, -url_for, flash, abort, make_response) + url_for, flash, abort, make_response) from flask_login import login_required, login_user, logout_user, current_user from tosca import app, db, lm @@ -54,13 +62,13 @@ def unimplemented(e): @mod.route('/sitemap.xml') def sitemap(): #pages = [] - #for rule in app.url_map.iter_rules(): + # for rule in app.url_map.iter_rules(): # if "GET" in rule.methods and len(rule.arguments) == 0: # pages.append([rule.rule, datetime.utcnow()]) dt = datetime.utcnow() - pages = [ [ url_for('views/main.index', _external=True), dt ], - [ url_for('views/main.login', _external=True), dt ], - [ url_for('views/main.logout', _external=True), dt] ] + pages = [[url_for('views/main.index', _external=True), dt], + [url_for('views/main.login', _external=True), dt], + [url_for('views/main.logout', _external=True), dt]] sitemap_xml = render_template('sitemap.xml', pages=pages) resp = make_response(sitemap_xml) resp.headers['Content-Type'] = "application/xml" @@ -103,27 +111,30 @@ def login(): #app.logger.info('user: %s' % user) login_user(user) flash("Successfully authenticated.") - if 'TERMS_OF_USE' in app.config: flash(app.config['TERMS_OF_USE'], 'toc') + if 'TERMS_OF_USE' in app.config: + flash(app.config['TERMS_OF_USE'], 'toc') return redirect(request.args.get('next') or url_for('views/main.index')) raise RuntimeError("Error trying to authenticate.") # authenticate using LDAP and local db form = LoginForm() if form.validate_on_submit(): - #session['remember_me'] = form.remember_me.data + #session['remember_me'] = form.remember_me.data username = form.username.data password = form.password.data # authenticate ops user account if username == app.config['OPS_USER']: - ops_passwd_hex = hashlib.sha224(password).hexdigest() + ops_passwd_hex = hashlib.sha224(password.encode()).hexdigest() if app.config['OPS_PASSWORD_HASH'] == ops_passwd_hex: info = {} - else: info = None + else: + info = None elif username in app.config.get('OUTSIDE_ACCOUNTS', {}): - passwd_hex = hashlib.sha224(password).hexdigest() + passwd_hex = hashlib.sha224(password.encode()).hexdigest() if app.config['OUTSIDE_ACCOUNTS'][username] == passwd_hex: info = {} - else: info = None + else: + info = None else: # for everyone else authenticate via LDAP info = ldap_user_verified(username, password) @@ -139,7 +150,8 @@ def login(): #app.logger.info('user: %s' % user) login_user(user) flash("Successfully authenticated.") - if 'TERMS_OF_USE' in app.config: flash(app.config['TERMS_OF_USE'], 'toc') + if 'TERMS_OF_USE' in app.config: + flash(app.config['TERMS_OF_USE'], 'toc') return redirect(request.args.get('next') or url_for('views/main.index')) flash("Error trying to authenticate.") else: @@ -168,12 +180,14 @@ def index(): #app.logger.debug("g.user: %s" % g.user) #app.logger.debug("g.user.info: %s" % g.user.info) emails = g.user.info.get('mail', []) - if len(emails) > 0: email = emails[0] - else: email = "" + if len(emails) > 0: + email = emails[0] + else: + email = "" g.blueprints = app.blueprints return render_template('facetview.html', title='TOSCA: Advanced FacetView User Interface', - email= email, + email=email, current_year=datetime.now().year) @@ -196,11 +210,13 @@ def mapview(): #app.logger.debug("g.user: %s" % g.user) #app.logger.debug("g.user.info: %s" % g.user.info) emails = g.user.info.get('mail', []) - if len(emails) > 0: email = emails[0] - else: email = "" + if len(emails) > 0: + email = emails[0] + else: + email = "" return render_template('facetview_gibs.html', title='TOSCA: Advanced FacetView User Interface', - email= email, + email=email, current_year=datetime.now().year) @@ -218,7 +234,7 @@ def jpl(): def jpl_login(): form = LoginForm() if form.validate_on_submit(): - #session['remember_me'] = form.remember_me.data + #session['remember_me'] = form.remember_me.data username = form.username.data password = form.password.data else: @@ -227,5 +243,3 @@ def jpl_login(): return render_template('login_jpl.html', title='TOSCA: Advanced FacetView User Interface', form=form, current_year=datetime.now().year) - -