Merge branch 'release/1.18.0'

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,21 @@
+## Overview
+
+Brief description of what this PR does, and why it is needed.
+
+Connects #XXX
+
+### Demo
+
+Optional. Screenshots, `curl` examples, etc.
+
+### Notes
+
+Optional. Ancillary topics, caveats, alternative strategies that didn't work out, anything else.
+
+## Testing Instructions
+
+ * How to test this PR
+ * Prefer bulleted description
+ * Start after checking out this branch
+ * Include any setup required, such as bundling scripts, restarting services, etc.
+ * Include test case, and expected output

Vagrantfile

@@ -90,9 +90,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
     worker.vm.synced_folder "src/mmw", "/opt/app/"
 
-    if ENV["VAGRANT_ENV"].nil? || ENV["VAGRANT_ENV"] != "TEST"
-      worker.vm.synced_folder "/opt/rwd-data", "/opt/rwd-data"
-    end
+    # Path to RWD data (ex. /media/passport/rwd-nhd)
+    worker.vm.synced_folder ENV.fetch("RWD_DATA", "/tmp"), "/opt/rwd-data"
 
     # Docker
     worker.vm.network "forwarded_port", {

deployment/ansible/roles/model-my-watershed.rwd/defaults/main.yml

@@ -2,4 +2,8 @@
 rwd_data_path: "/opt/rwd-data"
 rwd_host: "localhost"
 rwd_port: 5000
-rwd_docker_image: "quay.io/wikiwatershed/rwd:1.1.1"
+rwd_docker_image: "quay.io/wikiwatershed/rwd:1.2.1"
+
+app_config:
+    RWD_HOST: "{{ rwd_host }}"
+    RWD_PORT: "{{ rwd_port }}"

deployment/ansible/roles/model-my-watershed.rwd/tasks/app.yml

@@ -0,0 +1,16 @@
+---
+- name: Create RWD data directory
+  file: path="{{ rwd_data_path }}"
+        state=directory
+
+- name: Pull RWD docker container image
+  command: /usr/bin/docker pull {{ rwd_docker_image }}
+
+- name: Configure RWD service definition
+  template: src=upstart-mmw-rwd.conf.j2
+            dest=/etc/init/mmw-rwd.conf
+  notify:
+    - Restart mmw-rwd
+
+- name: Ensure service is running
+  service: name=mmw-rwd state=started

deployment/ansible/roles/model-my-watershed.rwd/tasks/configuration.yml

@@ -0,0 +1,10 @@
+---
+- name: Configure application
+  copy: content="{{ item.value }}"
+        dest="{{ envdir_home }}/{{ item.key }}"
+        owner=root
+        group=mmw
+        mode=0750
+  with_dict: "{{ app_config }}"
+  notify:
+    - Restart Celery

deployment/ansible/roles/model-my-watershed.rwd/tasks/main.yml

@@ -1,16 +1,3 @@
 ---
-- name: Create RWD data directory
-  file: path="{{ rwd_data_path }}"
-        state=directory
-
-- name: Pull RWD docker container image
-  command: /usr/bin/docker pull {{ rwd_docker_image }}
-
-- name: Configure RWD service definition
-  template: src=upstart-mmw-rwd.conf.j2
-            dest=/etc/init/mmw-rwd.conf
-  notify:
-    - Restart mmw-rwd
-
-- name: Ensure service is running
-  service: name=mmw-rwd state=started
+- { include: configuration.yml }
+- { include: app.yml }

deployment/cfn/application.py

@@ -173,7 +173,7 @@ def set_up_stack(self):
         self.app_server_auto_scaling_schedule_end_recurrence = self.add_parameter(  # NOQA
             Parameter(
                 'AppServerAutoScalingScheduleEndRecurrence', Type='String',
-                Default='0 23 * * *',
+                Default='0 1 * * *',
                 Description='Application server ASG schedule end recurrence'
             ), 'AppServerAutoScalingScheduleEndRecurrence')
 

deployment/cfn/tiler.py

@@ -164,7 +164,7 @@ def set_up_stack(self):
         self.tile_server_auto_scaling_schedule_end_recurrence = self.add_parameter(  # NOQA
             Parameter(
                 'TileServerAutoScalingScheduleEndRecurrence', Type='String',
-                Default='0 23 * * *',
+                Default='0 1 * * *',
                 Description='Tile server ASG schedule end recurrence'
             ), 'TileServerAutoScalingScheduleEndRecurrence')
 

deployment/cfn/utils/constants.py

@@ -2,7 +2,8 @@
     't2.micro',
     't2.small',
     't2.medium',
-    't2.large'
+    't2.large',
+    'r4.large'
 ]
 
 RDS_INSTANCE_TYPES = [

deployment/cfn/worker.py

@@ -170,7 +170,7 @@ def set_up_stack(self):
         self.worker_auto_scaling_schedule_end_recurrence = self.add_parameter(  # NOQA
             Parameter(
                 'WorkerAutoScalingScheduleEndRecurrence', Type='String',
-                Default='0 23 * * *',
+                Default='0 1 * * *',
                 Description='Worker ASG schedule end recurrence'
             ), 'WorkerAutoScalingScheduleEndRecurrence')
 
@@ -331,6 +331,7 @@ def create_auto_scaling_resources(self, worker_security_group, worker_lb):
         worker_launch_config = self.add_resource(
             asg.LaunchConfiguration(
                 worker_launch_config_name,
+                EbsOptimized=True,
                 ImageId=Ref(self.worker_ami),
                 IamInstanceProfile=Ref(self.worker_instance_profile),
                 InstanceType=Ref(self.worker_instance_type),
@@ -416,7 +417,10 @@ def get_cloud_config(self):
                 '  - path: /etc/mmw.d/env/ROLLBAR_SERVER_SIDE_ACCESS_TOKEN\n',
                 '    permissions: 0750\n',
                 '    owner: root:mmw\n',
-                '    content: ', self.get_input('RollbarServerSideAccessToken')]  # NOQA
+                '    content: ', self.get_input('RollbarServerSideAccessToken'),  # NOQA
+                '\n',
+                'runcmd:\n',
+                '  - /opt/model-my-watershed/scripts/aws/ebs-warmer.sh']
 
     def create_cloud_watch_resources(self, worker_auto_scaling_group):
         self.add_resource(cw.Alarm(

deployment/default.yml.example

@@ -29,8 +29,8 @@ AppServerAutoScalingScheduleStartCapacity: '1'
 # 8AM ET
 AppServerAutoScalingScheduleStartRecurrence: '0 13 * * 1-5'
 AppServerAutoScalingScheduleEndCapacity: '0'
-# 6PM ET
-AppServerAutoScalingScheduleEndRecurrence: '0 23 * * *'
+# 8PM ET
+AppServerAutoScalingScheduleEndRecurrence: '0 1 * * *'
 SSLCertificateARN: 'arn:aws:iam...'
 BackwardCompatSSLCertificateARN: 'arn:aws:iam...'
 TileServerInstanceType: 't2.micro'
@@ -45,8 +45,8 @@ TileServerAutoScalingScheduleStartCapacity: '1'
 # 8AM ET
 TileServerAutoScalingScheduleStartRecurrence: '0 13 * * 1-5'
 TileServerAutoScalingScheduleEndCapacity: '0'
-# 6PM ET
-TileServerAutoScalingScheduleEndRecurrence: '0 23 * * *'
+# 8PM ET
+TileServerAutoScalingScheduleEndRecurrence: '0 1 * * *'
 WorkerInstanceType: 't2.micro'
 # Leaving this commented dynamically looks up the
 # most recent AMI for this type.
@@ -59,8 +59,8 @@ WorkerAutoScalingScheduleStartCapacity: '2'
 # 8AM ET
 WorkerAutoScalingScheduleStartRecurrence: '0 13 * * 1-5'
 WorkerAutoScalingScheduleEndCapacity: '0'
-# 6PM ET
-WorkerAutoScalingScheduleEndRecurrence: '0 23 * * *'
+# 8PM ET
+WorkerAutoScalingScheduleEndRecurrence: '0 1 * * *'
 ITSIBaseURL: ''
 ITSISecretKey: ''
 RollbarServerSideAccessToken: ''

deployment/packer/template.js

@@ -63,7 +63,7 @@
             "ami_block_device_mappings": [
                 {
                     "device_name": "/dev/sdf",
-                    "snapshot_id": "snap-090ac799996dba0a4",
+                    "snapshot_id": "snap-0211cbbff8a81266f",
                     "volume_type": "gp2",
                     "delete_on_termination": true
                 }

scripts/aws/ebs-warmer.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Ref: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-initialize.html
+
+function log() {
+    echo "[$(date --rfc-3339=seconds)] $1"
+}
+
+log "Warming up RWD EBS volume..."
+
+find /opt/rwd-data/nhd/Main_Watershed/* -type f -print0 \
+    | xargs -0 -P0 -L1 -t cat >/dev/null
+
+find /opt/rwd-data/drb/Main_Watershed/* -type f -print0 \
+    | xargs -0 -P0 -L1 -t cat >/dev/null
+
+log "Done"

scripts/aws/setupdb.sh

@@ -15,6 +15,7 @@ where: \n
     -m  load/reload mapshed data\n
     -p  load/reload DEP data\n
     -q  load/reload water quality data\n
+    -x  purge s3 cache for given path\n
 "
 
 # HTTP accessible storage for initial app data
@@ -25,7 +26,7 @@ load_stream=false
 load_mapshed=false
 load_water_quality=false
 
-while getopts ":hbsdpmqf:" opt; do
+while getopts ":hbsdpmqf:x:" opt; do
     case $opt in
         h)
             echo -e $usage
@@ -44,6 +45,8 @@ while getopts ":hbsdpmqf:" opt; do
             load_water_quality=true ;;
         f)
             file_to_load=$OPTARG ;;
+        x)
+            path_to_purge=$OPTARG ;;
         \?)
             echo "invalid option: -$OPTARG"
             exit ;;
@@ -81,6 +84,11 @@ if [ ! -z "$file_to_load" ] ; then
     download_and_load $FILES
 fi
 
+if [ ! -z "$path_to_purge" ] ; then
+    PATHS=("$path_to_purge")
+    purge_tile_cache $PATHS
+fi
+
 if [ "$load_dep" = "true" ] ; then
     # Fetch DEP layers
     FILES=("dep_urban_areas.sql.gz" "dep_municipalities.sql.gz")

src/mmw/apps/core/templates/base.html

@@ -1,31 +1,6 @@
 {% include 'head.html' %}
 {% load staticfiles %}
 
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    {% block metatitle %}
-    <title>Model My Watershed</title>
-    {% endblock metatitle %}
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <link rel="icon" type="image/png" href="{% static 'favicon.png' %}" sizes="16x16">
-    <link rel="icon" type="image/png" href="{% static '[email protected]' %}" sizes="32x32">
-    <link rel="stylesheet" href="{% static 'css/vendor.css' %}" />
-    <link rel="stylesheet" href="{% static 'css/main.css' %}" />
-    <!-- Google Analytics -->
-    <script>
-        (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
-        (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
-        m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
-        })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
-
-        ga('create', '{{ GOOGLE_ANALYTICS_ACCOUNT }}', 'auto');
-        ga('send', 'pageview');
-        window.ga = ga;
-    </script>
-</head>
-
 <body>
     {% block header %}
     {% endblock header %}

src/mmw/apps/home/views.py

@@ -135,6 +135,8 @@ def get_client_settings(request):
             'draw_tools': settings.DRAW_TOOLS,
             'map_controls': settings.MAP_CONTROLS,
             'vizer_urls': settings.VIZER_URLS,
+            'vizer_ignore': settings.VIZER_IGNORE,
+            'vizer_names': settings.VIZER_NAMES,
             'model_packages': get_model_packages(),
             'mapshed_max_area': settings.GWLFE_CONFIG['MaxAoIArea']
         }),

src/mmw/apps/modeling/tasks.py

@@ -3,6 +3,7 @@
 from __future__ import unicode_literals
 from __future__ import absolute_import
 
+import os
 import logging
 import json
 import requests
@@ -30,6 +31,9 @@
 
 DRB = 'drb'
 
+RWD_HOST = os.environ.get('RWD_HOST', 'localhost')
+RWD_PORT = os.environ.get('RWD_PORT', '5000')
+
 
 @shared_task
 def start_rwd_job(location, snapping, data_source):
@@ -42,7 +46,8 @@ def start_rwd_job(location, snapping, data_source):
     location = json.loads(location)
     lat, lng = location
     end_point = 'rwd' if data_source == DRB else 'rwd-nhd'
-    rwd_url = 'http://localhost:5000/%s/%f/%f' % (end_point, lat, lng)
+    rwd_url = 'http://%s:%s/%s/%f/%f' % (RWD_HOST, RWD_PORT, end_point,
+                                         lat, lng)
 
     # The Webserver defaults to enable snapping, uses 1 (true) 0 (false)
     if not snapping:

src/mmw/apps/user/middleware.py

@@ -1,34 +1,18 @@
 # -*- coding: utf-8 -*-
 
 from django.conf import settings
-from django.core.urlresolvers import reverse
-from django.shortcuts import redirect
 
-from apps.user.views import itsi_login
 
 EMBED_FLAG = settings.ITSI['embed_flag']
-LOGIN_URL = reverse(itsi_login)
 
 
 class ItsiAuthenticationMiddleware(object):
     """
-    Middleware for automatically logging in ITSI users
-    and setting relevant flags.
+    Middleware for setting relevant ITSI flags.
     """
 
     def process_request(self, request):
-        """
-        Check if ITSI EMBED FLAG is set, and if so attempt to log
-        the user in with their ITSI credentials and redirect them
-        to current page.
-        """
-
-        # If flag is not set then return None so Django can proceed
-        # with the request as-is
-        if request.GET.get(EMBED_FLAG, 'false') != 'true':
-            return None
-
-        # Flag is set. Assume user is not logged in. Set session flag and
-        # Redirect them to ITSI LOGIN URL, with the current URL to return to
-        request.session[EMBED_FLAG] = True
-        return redirect('{0}?next={1}'.format(LOGIN_URL, request.path))
+        # If flag is set then set a session variable so it can be passed to
+        # front-end
+        if request.GET.get(EMBED_FLAG, 'false') == 'true':
+            request.session[EMBED_FLAG] = True

src/mmw/js/src/analyze/templates/catchmentWaterQualityTable.html

@@ -11,7 +11,7 @@
                   Total P (kg/ha)
               </th>
               <th class="text-right" data-sortable="true" data-sorter="window.noDataSort">
-                  Total TSS (kg/ha)
+                  Total SS (kg/ha)
               </th>
               <th class="text-right" data-sortable="true" data-sorter="window.noDataSort">
                   Avg TN (mg/l)

src/mmw/js/src/analyze/templates/resultsWindow.html

@@ -9,7 +9,7 @@
         <li>
             <a class="back-button" aria-controls="home" aria-expanded="false">
                 <button type="button" class="btn btn-lg" data-url="/">
-                    Draw
+                    Back
                 </button>
             </a>
 

src/mmw/js/src/analyze/tests.js

@@ -172,7 +172,7 @@ var tableHeaders = {
     animals: ['Animal', 'Count'],
     pointsource: ['NPDES Code', 'City', 'Discharge (MGD)', 'TN Load (kg/yr)', 'TP Load (kg/yr)'],
     catchment_water_quality: ['Id', 'Area (ha)', 'Total N (kg/ha)', 'Total P (kg/ha)',
-        'Total TSS (kg/ha)', 'Avg TN (mg/l)', 'Avg TP (mg/l)', 'Avg TSS (mg/l)'],
+        'Total SS (kg/ha)', 'Avg TN (mg/l)', 'Avg TP (mg/l)', 'Avg TSS (mg/l)'],
 };
 
 function tableRows(type, result) {

src/mmw/js/src/analyze/views.js

@@ -746,7 +746,9 @@ var PointSourceResultView = AnalyzeResultView.extend({
 
 var CatchmentWaterQualityResultView = AnalyzeResultView.extend({
     onShow: function() {
-        var desc = 'Delaware River Basin only: Stream Reach Assessment Tool model estimates',
+        var desc = 'Delaware River Basin only: <a target="_blank" ' +
+                   'href="https://www.arcgis.com/home/item.html?id=260d7e17039d48a6beee0f0b640eb754"​>' +
+                   'Stream Reach Assessment Tool</a> model estimates',
             chart = null;
         this.showAnalyzeResults(coreModels.CatchmentWaterQualityCensusCollection,
             CatchmentWaterQualityTableView, chart, desc);