Merge branch 'release/1.26.0'

README.md

@@ -50,13 +50,15 @@ $ ./scripts/bundle.sh
 To load or reload boundary data, from an `app` server, run (`scripts` is not mounted by default to the VM, you may need to copy the file over):
 
 ```bash
-$ ./scripts/setupdb.sh -b
+$ vagrant upload ./scripts/ app
+$ vagrant ssh app
+$ ./scripts/aws/setupdb.sh -b
 ```
 
 The same script can be used to load the stream network data:
 
 ```bash
-$ ./scripts/setupdb.sh -s
+$ ./scripts/aws/setupdb.sh -s
 ```
 
 Note that if you receive out of memory errors while loading the data, you may want to increase the RAM on your `services` VM (1512 MB may be all that is necessary).
@@ -164,7 +166,7 @@ $ ./scripts/manage.sh test_mapshed
 To check for JavaScript lint:
 
 ```bash
-$ ./scripts/npm.sh run lint
+$ ./scripts/yarn.sh run lint
 ```
 
 When creating new tests or debugging old tests, it may be easier to open the testem page, which polls for changes to the test bundle and updates the test state dynamically.
@@ -180,7 +182,7 @@ To enable livereload, [download the browser extension](http://livereload.com/ext
 and start the livereload server with the following command:
 
 ```bash
-./scripts/npm.sh run livereload
+./scripts/yarn.sh run livereload
 ```
 
 #### Bundling static assets
@@ -219,7 +221,14 @@ This flag is for troubleshooting purposes only.
 
 #### Adding JS dependencies
 
-To add a new JS dependency, update the `JS_DEPS` array in `bundle.sh`, and `package.json` accordingly.
-Because our dependencies are shrinkwrapped, follow the [instructions](https://docs.npmjs.com/cli/shrinkwrap#building-shrinkwrapped-packages) for adding a dependency to a shrinkwrapped package.
+To add a new JS dependency, use
+
+```console
+$ ./scripts/yarn.sh add --exact <dependency>
+```
+
+this will download the dependency to `node_modules`, add to the `package.json`, and to `yarn.lock`.
+Furthermore, it will be pinned to the current version.
+Then, update the `JS_DEPS` array in `bundle.sh`.
 Rebuild the vendor bundle using `./scripts/bundle.sh --vendor`.
-`npm` commands can be run using `./scripts/npm.sh`.
+`yarn` commands can be run using `./scripts/yarn.sh`.

deployment/ansible/group_vars/all

@@ -28,8 +28,9 @@ daemontools_version: "1:0.76-6ubuntu1"
 
 python_version: "2.7.12-1~16.04"
 
-app_nodejs_version: "0.10.32"
-app_nodejs_npm_version: "2.1.17"
+yarn_version: "1.19.*"
+app_nodejs_version: "12.11.1"
+app_nodejs_npm_version: "6.9.0"
 tiler_nodejs_version: "10.16.0"
 tiler_nodejs_npm_version: "6.9.0"
 

deployment/ansible/roles.yml

@@ -4,6 +4,8 @@
   version: 2.0.0
 - src: azavea.nodejs
   version: 0.3.0
+- src: azavea.yarn
+  version: 0.1.0
 - src: azavea.git
   version: 0.1.0
 - src: azavea.nginx

deployment/ansible/roles/model-my-watershed.app/meta/main.yml

@@ -3,6 +3,7 @@ dependencies:
   - { role: "model-my-watershed.base" }
   - { role: "azavea.python", python_development: True }
   - { role: "azavea.pip" }
+  - { role: "azavea.yarn" }
   - { role: "azavea.nodejs", nodejs_version: "{{ app_nodejs_version }}", nodejs_npm_version: "{{ app_nodejs_npm_version }}" }
   - { role: "azavea.phantomjs" }
   - { role: "azavea.build-essential" }

deployment/ansible/roles/model-my-watershed.app/tasks/static-files.yml

@@ -11,7 +11,7 @@
     - "{{ app_static_cache }}"
 
 - name: Install application JavaScript dependencies
-  command: "npm install --unsafe-perm"
+  command: "yarn"
   args:
     chdir: "{{ app_home }}"
   become: False
@@ -42,3 +42,15 @@
             dest="{{ app_static_root }}/test.html"
   become: False
   when: "['development', 'test'] | some_are_in(group_names)"
+
+- name: Copy JavaScript test dependencies - mocha.css
+  copy: src="{{ app_home }}/node_modules/mocha/mocha.css"
+        dest="{{ app_static_root }}"
+        remote_src=True
+  when: "['development', 'test'] | some_are_in(group_names)"
+
+- name: Copy JavaScript test dependencies - mocha.js
+  copy: src="{{ app_home }}/node_modules/mocha/mocha.js"
+        dest="{{ app_static_root }}"
+        remote_src=True
+  when: "['development', 'test'] | some_are_in(group_names)"

deployment/ansible/roles/model-my-watershed.app/templates/testem-harness.html.j2

@@ -5,10 +5,10 @@
 <style>
 #sandbox { display: none; }
 </style>
-<link rel="stylesheet" href="/testem/mocha.css">
-<script src="/testem/mocha.js"></script>
+<link rel="stylesheet" href="mocha.css">
+<script src="mocha.js"></script>
 <script src="/testem.js"></script>
-<script>mocha.setup('tdd')</script>
+<script>mocha.setup('bdd')</script>
 </head>
 <body>
 <div id="mocha"></div>

doc/arch/adr-007-pa-dep-client-app.md

@@ -0,0 +1,136 @@
+# Integrating a New Client App for PA DEP
+
+## Context
+
+The Pennsylvania Department of Environmental Protection (PA DEP) has been using desktop MapShed and MMW for their workflows for some time.
+This workflow involves running the MapShed model in MMW for a HUC-12,
+and also for an area of interest within the HUC-12,
+and copying the output tables into specific parts of the [MMW BMP Spreadsheet Tool][bmp-tool].
+This is cumbersome and error prone,
+and would be much better be done in an automated fashion,
+where when users download the spreadsheet it is pre-populated with the right values in the right places.
+
+Since this requires having two effective areas of interest at once,
+this workflow does not fit neatly in MMW,
+which always expects a single area of interest.
+Rather than change MMW to accomodate this unusual and rare use case,
+it would be much simpler to create a new client app that is focused on this workflow.
+It will use the MMW API for all calculations.
+
+Given that we'll have a new client app,
+this is an opportunity to introduce a more modern tech stack on the front-end.
+This was recently done for [a similar project][beekeepers],
+where a new front-end stack was added within a Django app to sandbox it from the rest of the project.
+This document aims to discuss the pros and cons of different arrangements for the new client app,
+and pick one that satisfies all project constraints.
+
+The one major directive from the clients is that,
+from a user's perspective,
+the site should be deeply integrated with MMW.
+That is to say,
+it should use the same colors and branding,
+and not feel like its own project.
+
+From the development perspective,
+we should aim for using the latest front-end stacks,
+clean communication with the back-end API,
+and having a fast deployment pipeline.
+
+### Considerations
+
+To find the right balance of coupling between the two apps,
+we must consider how the code is stored and written (same vs different repositories),
+how the apps are deployed (paired or independently),
+and how the apps are accessed by users (via subdomain or subdirectory).
+These considerations are not entirely independent of each other,
+e.g. paired deployments are easier to do with the same repository.
+
+We must also consider non-technical factors,
+such as the explicit directive to have a consistent user experience between the apps.
+This requirement rules out subdomain access,
+which would break the immersion of using the same app.
+
+#### Repository: Same vs Different
+
+Advantages of having the **same repository**:
+
+  * Familiarity, as it was done in the [Beekeepers][beekeepers] project
+  * All issues can be in one repo, simplifying project management
+  * Tasks that affect both front-end and back-end can be tracked in one commit or PR
+  * Simpler deployment, as front-end and back-end will always be in sync
+  * Styles can be easily reused between the two apps
+  * Easier to simulate production environments in development,
+    as we only need to start a single development environment.
+
+Advantages of having **different repositories**:
+
+  * Cleaner logical separation in the codebase
+  * Fewer cohabiting developer entry points, i.e. `package.json`, or multiple `node_modules`
+  * Does not require changing the `node` or `npm` version in MMW
+  * Does not require adding a Docker environment in MMW for building the front-end
+    - In case we do not change the `node` version in MMW,
+      we can isolate the new build toolchain in a Docker environment
+  * Faster deployment, as the client app can be deployed independently of the main app
+
+#### Deployment: Paired vs Independent
+
+Advantages of **paired deployment**:
+
+  * Front-end and back-end will always be in sync
+  * Does not require extra configuration of AWS
+  * Does not need extra steps during deployment
+  * Easier to write end-to-end tests
+
+Advantages of **independent deployment**:
+
+  * Front-end can be iterated much faster, without the heavy deployment cycle of MMW
+
+### Architecture Options
+
+#### Option A: Same Repository, Paired Deployment
+
+Similar to Beekeepers,
+we will create a Django app to house all files for the new client app.
+We will evaluate if the `node` version in the App VM can be upgraded,
+and if not we will use add a Docker container for building the front-end.
+Django will be configured to handle routing,
+which will render a different home page for the new routes.
+Deployments will not be affected at all and will proceed as usual.
+Some basic styles and colors will be shared between the two apps:
+they will have independent stylesheets, but some common resources,
+such as colors,
+may be shared between the two.
+
+#### Option B: Different Repositories, Independent Deployments
+
+The new app gets its own repository,
+which better isolates the new front-end code from legacy code.
+During development, the MMW VM would have to be running in addition to the client app.
+The app will be deployed to a static host like S3 or Netlify.
+nginx will be configured to proxy the static host at the subdirectory route.
+Deployments of cosmetic changes to the new client app can be made independently,
+but functional changes to how the API is consumed will have to be coordinated with MMW.
+Project management will be split across the two repos,
+with front-end tasks listed on the new repo and back-end tasks listed on MMW.
+
+## Decision
+
+We will go with Option A above,
+keeping one repository for the codebase and doing paired deployments.
+This has the advantages of being a familiar approach,
+keeping the back- and front-ends in sync,
+and not needing any changes to the deployment pipeline.
+
+## Consequences
+
+Since the more modern stack of the new app will require a more modern version of `node`,
+we will have to investigate if the one in the App VM can be safely upgraded without affecting the current `bundle` pipeline,
+or if it will require isolation within Docker as done for Beekeepers.
+
+As the new app will be deeply integrated with MMW,
+any changes,
+no matter how small,
+will require a full MMW deployment.
+
+[bmp-tool]: https://github.com/WikiWatershed/MMW-BMP-spreadsheet-tool
+[beekeepers]: https://github.com/project-icp/bee-pollinator-app/tree/develop/src/icp/apps/beekeepers

scripts/aws/setupdb.sh

@@ -68,6 +68,9 @@ export PUBLIC_HOSTED_ZONE_NAME=$(cat /etc/mmw.d/env/MMW_PUBLIC_HOSTED_ZONE_NAME)
 psql -c "CREATE EXTENSION IF NOT EXISTS postgis;"
 psql -c "ALTER TABLE spatial_ref_sys OWNER TO ${PGUSER};"
 
+# Create pg_trgm extension for faster LIKE matches
+psql -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;"
+
 # Run migrations
 envdir /etc/mmw.d/env /opt/app/manage.py migrate
 
@@ -83,6 +86,14 @@ function purge_tile_cache {
     done
 }
 
+function create_trgm_indexes {
+    # Creates a trgm index on the 'name' column for each table.
+    for table in "${TRGM_TABLES[@]}"; do
+        psql -c "CREATE INDEX IF NOT EXISTS trgm_idx_${table}_name ON ${table}"`
+                `" USING gin (name gin_trgm_ops);"
+    done
+}
+
 if [ ! -z "$file_to_load" ] ; then
     FILES=("$file_to_load")
     download_and_load $FILES
@@ -107,8 +118,10 @@ if [ "$load_boundary" = "true" ] ; then
     # Fetch boundary layer sql files
     FILES=("boundary_county.sql.gz" "boundary_school_district.sql.gz" "boundary_district.sql.gz" "boundary_huc12_deduped.sql.gz" "boundary_huc10.sql.gz" "boundary_huc08.sql.gz")
     PATHS=("county" "district" "huc8" "huc10" "huc12" "school")
+    TRGM_TABLES=("boundary_huc08" "boundary_huc10" "boundary_huc12")
 
     download_and_load $FILES
+    create_trgm_indexes $TRGM_TABLES
     purge_tile_cache $PATHS
 fi
 

scripts/data/climate/requirements.txt

@@ -1,3 +1,3 @@
 rasterio==1.0a9
-Pillow==4.2.1
+Pillow==6.2.0
 matplotlib==2.0.2

scripts/test.sh

@@ -23,7 +23,7 @@ vagrant ssh app -c "cd /opt/app && envdir /etc/mmw.d/env ./manage.py test --noin
 
 if [[ -z "${MMW_SKIP_JS_TESTS}" ]]; then
     vagrant ssh app -c "cd /var/www/mmw/static &&
-        xvfb-run /opt/app/node_modules/.bin/testem -f /opt/app/testem.json ci Firefox $*"
+        xvfb-run /opt/app/node_modules/.bin/testem -f /opt/app/testem.json ci $*"
 else
     echo "SKIPPING JS TESTS"
 fi

scripts/yarn.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Pass arguments to yarn the project root
+
+set -e
+set -x
+
+ARGS=$*
+
+vagrant ssh app -c "cd /opt/app && envdir /etc/mmw.d/env yarn $ARGS"