python3 port (#5)

* port to python3 (#4)
* initial 2to3 run
* remove supervisor from required (not python3 compatible yet)
* format to PEP8 spec
* fix location of views/services imports; fix encoding issues
* pasteurized
* remove imports not yet installed via install_requires
* require future
* bump version

db_create.py

@@ -1,3 +1,9 @@
 #!/usr/bin/env python
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
 from tosca import db
 db.create_all()

run.py

@@ -1,6 +1,11 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
 from tosca import app
 
 
 if __name__ == '__main__':
-    context = ('server.pem', 'server.key')
-    app.run(host="0.0.0.0", port=8879, debug=True, ssl_context=context)
+    app.run(host="0.0.0.0", port=8879, debug=True)

run_debug.py

@@ -1,3 +1,9 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
 from tosca import app
 
 

scripts/create_user_rules_index.py

@@ -1,16 +1,27 @@
 #!/usr/bin/env python
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
 from requests.exceptions import HTTPError
 
 from tosca import app
 from tosca.services.user_rules import create_user_rules_index, add_grq_mappings
 
 try:
-    create_user_rules_index(app.config['ES_URL'], app.config['USER_RULES_INDEX'])
+    create_user_rules_index(
+        app.config['ES_URL'], app.config['USER_RULES_INDEX'])
 except HTTPError as e:
-    if e.response.status_code == 400: pass
-    else: raise
+    if e.response.status_code == 400:
+        pass
+    else:
+        raise
 try:
     add_grq_mappings(app.config['ES_URL'], app.config['USER_RULES_INDEX'])
 except HTTPError as e:
-    if e.response.status_code == 404: pass
-    else: raise
+    if e.response.status_code == 404:
+        pass
+    else:
+        raise

setup.py

@@ -1,13 +1,18 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
 from setuptools import setup, find_packages
 
 setup(
     name='tosca',
-    version='0.2.3',
+    version='0.3.0',
     long_description='Advanced FacetView User Interface',
     packages=find_packages(),
     include_package_data=True,
     zip_safe=False,
-    install_requires=['Flask', 'gunicorn', 'gevent', 'supervisor', 'requests',
+    install_requires=['Flask', 'gunicorn', 'gevent', 'requests',
                       'Flask-SQLAlchemy', 'Flask-WTF', 'Flask-DebugToolbar',
-                      'Flask-Login', 'simpleldap', 'simplekml']
+                      'Flask-Login', 'simpleldap', 'simplekml',
+                      'future>=0.17.1']
 )

tosca/__init__.py

@@ -1,3 +1,9 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
 import os
 from flask import Flask
 from flask_sqlalchemy import SQLAlchemy
@@ -44,6 +50,7 @@ class ReverseProxied(object):
 
     :param app: the WSGI application
     '''
+
     def __init__(self, app):
         self.app = app
 
@@ -70,7 +77,8 @@ def __call__(self, environ, start_response):
 
 # set database config
 dbdir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'data'))
-app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + os.path.join(dbdir, 'app.db')
+app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + \
+    os.path.join(dbdir, 'app.db')
 db = SQLAlchemy(app)
 
 # debug toolbar
@@ -84,22 +92,29 @@ def __call__(self, environ, start_response):
 # views blueprints
 from tosca.views.main import mod as viewsModule
 app.register_blueprint(viewsModule)
+
 from tosca.views.js import mod as jsModule
 app.register_blueprint(jsModule)
 
 # services blueprints
 from tosca.services.query import mod as queryModule
 app.register_blueprint(queryModule)
+
 from tosca.services.wget import mod as wgetModule
 app.register_blueprint(wgetModule)
+
 from tosca.services.download import mod as downloadModule
 app.register_blueprint(downloadModule)
+
 from tosca.services.user_tags import mod as userTagsModule
 app.register_blueprint(userTagsModule)
+
 from tosca.services.user_rules import mod as userRulesModule
 app.register_blueprint(userRulesModule)
+
 from tosca.services.kml import mod as kmlModule
 app.register_blueprint(kmlModule)
+
 from tosca.services.dataset import mod as datasetModule
 app.register_blueprint(datasetModule)
 

tosca/lib/forms.py

@@ -1,7 +1,14 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
 from flask_wtf import Form
 from wtforms import TextField, BooleanField, PasswordField
 from wtforms.validators import Required
 
+
 class LoginForm(Form):
     username = TextField('username', validators=[Required()])
     password = PasswordField('password', validators=[Required()])

tosca/lib/grq_utils.py

@@ -1,26 +1,39 @@
-import json, os
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+import json
+import os
 from tosca import app
 
 import hysds_commons.request_utils
 
-def mozart_call(method,data={}):
+
+def mozart_call(method, data={}):
     '''
     Call mozart method with data
     @param method - method to call
     @param data - data to supply to call
     '''
-    url = os.path.join(app.config['GRQ_URL'],"api/v0.1",method)
+    url = os.path.join(app.config['GRQ_URL'], "api/v0.1", method)
     getpost = "GET"
-    res = hysds.lib.request_utils.requests_json_response(getpost,url,data=data,verify=False,logger=app.logger)
+    res = hysds.lib.request_utils.requests_json_response(
+        getpost, url, data=data, verify=False, logger=app.logger)
     return res["result"]
+
+
 def get_hysds_io_list():
     '''
     Queries GRQ to get HySDS IOs
     '''
     return mozart_call("hysds_io/list")
+
+
 def get_hysds_io(ident):
     '''
     Queries GRQ to get HySDS Metadata object
     @param ident - identity to get
     '''
-    return mozart_call("hysds_io/type",{"id":ident})
+    return mozart_call("hysds_io/type", {"id": ident})

tosca/lib/ldap.py

@@ -1,4 +1,13 @@
-import simpleldap, traceback
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from builtins import dict
+from builtins import str
+from future import standard_library
+standard_library.install_aliases()
+import simpleldap
+import traceback
 
 
 from tosca import app
@@ -10,20 +19,21 @@ def ldap_user_verified(username, password):
     host = app.config['LDAP_HOST']
     base_dn = app.config['LDAP_BASEDN']
     groups = app.config['LDAP_GROUPS']
-    opts = { 'OPT_NETWORK_TIMEOUT': 10 }
+    opts = {'OPT_NETWORK_TIMEOUT': 10}
 
     try:
         l = simpleldap.Connection(host, dn='uid=%s,%s' % (username, base_dn),
-                                  encryption='ssl', password=password, options=opts) 
-    except Exception, e:
+                                  encryption='ssl', password=password, options=opts)
+    except Exception as e:
         app.logger.info("Got error trying to verify LDAP user %s:" % username)
         app.logger.info("%s:\n\n%s" % (str(e), traceback.format_exc()))
         return None
-   
+
     # validate user
     r = l.search('uid=%s' % username, base_dn=base_dn)
     if len(r) != 1:
-        app.logger.info("Got invalid number of entries for %s: %s" % (username, len(r)))
+        app.logger.info("Got invalid number of entries for %s: %s" %
+                        (username, len(r)))
         app.logger.info("r: %s" % str(r))
         return None
 
@@ -32,7 +42,9 @@ def ldap_user_verified(username, password):
     for group in groups:
         g = l.search('cn=%s' % group, base_dn=base_dn)
         for this_g in g:
-            if uid in this_g['uniqueMember']: return dict(r[0])
+            if uid in this_g['uniqueMember']:
+                return dict(r[0])
 
-    app.logger.info("User %s is not part of any approved LDAP groups." % username)
+    app.logger.info(
+        "User %s is not part of any approved LDAP groups." % username)
     return None

tosca/lib/query.py

@@ -1,7 +1,15 @@
-import json, requests, re
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+import json
+import requests
+import re
 from pprint import pformat
 from flask import url_for
-from urllib import quote_plus
+from urllib.parse import quote_plus
 
 from tosca import app
 
@@ -18,7 +26,7 @@ def query(index, query_str):
     r = requests.post('%s/%s/_search' % (es_url, index), data=query_str)
     result = r.json()
     if r.status_code != 200:
-        app.logger.debug("Failed to query ES. Got status code %d:\n%s" % 
+        app.logger.debug("Failed to query ES. Got status code %d:\n%s" %
                          (r.status_code, json.dumps(result, indent=2)))
     r.raise_for_status()
     #app.logger.debug("result: %s" % pformat(r.json()))
@@ -27,7 +35,8 @@ def query(index, query_str):
     for hit in result['hits']['hits']:
         # emulate result format from ElasticSearch <1.0
         #app.logger.debug("hit: %s" % pformat(hit))
-        if '_source' in hit: hit.setdefault('fields', {}).update(hit['_source'])
+        if '_source' in hit:
+            hit.setdefault('fields', {}).update(hit['_source'])
         hit['fields']['es_index'] = hit['_index']
 
         # set dap url
@@ -39,8 +48,9 @@ def query(index, query_str):
         # set redirector url
         if len(hit['fields']['urls']) == 0:
             hit['fields']['urls'] = None
-        else: hit['fields']['urls'] = [ url_for('services/dataset.resolve_url',
-                                                index=hit['_index'], id=hit['_id']) ]
+        else:
+            hit['fields']['urls'] = [url_for('services/dataset.resolve_url',
+                                             index=hit['_index'], id=hit['_id'])]
 
         # set closest city
         if len(hit['fields'].get('city', [])) > 0:
@@ -60,7 +70,7 @@ def query_dataset_url(index, objectid):
     #app.logger.debug("url: %s" % r.url)
     result = r.json()
     if r.status_code != 200:
-        app.logger.debug("Failed to query ES. Got status code %d:\n%s" % 
+        app.logger.debug("Failed to query ES. Got status code %d:\n%s" %
                          (r.status_code, json.dumps(result, indent=2)))
     r.raise_for_status()
     #app.logger.info("result: %s" % json.dumps(result, indent=2))
@@ -97,7 +107,10 @@ def get_dataset_url(hit):
 
     ret_url = None
     for url in hit['_source'].get('urls', []):
-        if url.startswith('http'): ret_url = url
-        if url.startswith('http') and 'amazonaws.com' in url: ret_url = url
-        if url.startswith('http') and 'googleapis.com' in url: ret_url = url
+        if url.startswith('http'):
+            ret_url = url
+        if url.startswith('http') and 'amazonaws.com' in url:
+            ret_url = url
+        if url.startswith('http') and 'googleapis.com' in url:
+            ret_url = url
     return ret_url

tosca/lib/urs.py

@@ -1,3 +1,10 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from builtins import str
+from future import standard_library
+standard_library.install_aliases()
 import traceback
 import json
 from flask import redirect, abort, request
@@ -14,7 +21,7 @@ def urs_user_verified(code):
     # urs auth url
     urs_url = "{}/oauth/token".format(app.config['URS_URL'])
 
-    # set authorization value 
+    # set authorization value
     auth_val = base64.b64encode("{}:{}".format(app.config['URS_APP_ID'],
                                                app.config['URS_APP_PASSWORD']))
 
@@ -26,9 +33,11 @@ def urs_user_verified(code):
     }
 
     # exchange
-    r = requests.post(urs_url, data=payload, headers={ "Authorization": "Basic {}".format(auth_val) })
-    try: r.raise_for_status()
-    except Exception, e:
+    r = requests.post(urs_url, data=payload, headers={
+                      "Authorization": "Basic {}".format(auth_val)})
+    try:
+        r.raise_for_status()
+    except Exception as e:
         app.logger.error("Got error trying to verify URS user.")
         app.logger.error("%s:\n\n%s" % (str(e), traceback.format_exc()))
         return None
@@ -38,8 +47,8 @@ def urs_user_verified(code):
     # get user profile
     user_url = "{}{}".format(app.config['URS_URL'], packet['endpoint'])
     #app.logger.info("user_url: {}".format(user_url))
-    r = requests.get(user_url, headers={ "Authorization": "{} {}".format(packet['token_type'],
-                                                                         packet['access_token']) })
+    r = requests.get(user_url, headers={"Authorization": "{} {}".format(packet['token_type'],
+                                                                        packet['access_token'])})
     r.raise_for_status()
     user_info = r.json()
     #app.logger.info("user_info: {}".format(json.dumps(user_info, indent=2, sort_keys=True)))