From 8674b6becf0aa1087152a945f00ad24619a44306 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Thu, 31 Mar 2016 14:11:55 -0400 Subject: [PATCH 001/136] On backend, change naming convention to GWLFE instead of Mapshed This is to be consistent with frontend. Also, change input to model_input in the POST to GWLFE to be consistent with TR55. --- src/mmw/apps/home/urls.py | 1 + src/mmw/apps/home/views.py | 10 +++++++++- .../modeling/migrations/0017_add_gwlfe.py | 19 +++++++++++++++++++ src/mmw/apps/modeling/models.py | 6 +++++- src/mmw/apps/modeling/tasks.py | 7 +++---- src/mmw/apps/modeling/urls.py | 2 +- src/mmw/apps/modeling/views.py | 14 +++++++------- src/mmw/mmw/settings/base.py | 18 ++++++++++++++++++ src/mmw/mmw/settings/production.py | 2 ++ 9 files changed, 65 insertions(+), 14 deletions(-) create mode 100644 src/mmw/apps/modeling/migrations/0017_add_gwlfe.py diff --git a/src/mmw/apps/home/urls.py b/src/mmw/apps/home/urls.py index 28fec9190..223e5aebc 100644 --- a/src/mmw/apps/home/urls.py +++ b/src/mmw/apps/home/urls.py @@ -12,6 +12,7 @@ url(r'^$', home_page, name='home_page'), url(r'^projects/$', projects, name='projects'), url(r'^project/$', project, name='project'), + url(r'^project/new/', project, name='project'), url(r'^project/(?P[0-9]+)/$', project, name='project'), url(r'^project/(?P[0-9]+)/clone/?$', project_clone, name='project_clone'), diff --git a/src/mmw/apps/home/views.py b/src/mmw/apps/home/views.py index caf22146a..e5babb093 100644 --- a/src/mmw/apps/home/views.py +++ b/src/mmw/apps/home/views.py @@ -113,6 +113,13 @@ def get_layer_url(layer): return urljoin(tiler_base, layer['code'] + tiler_postfix) +def get_model_packages(): + for model_package in settings.MODEL_PACKAGES: + if model_package['name'] in settings.DISABLED_MODEL_PACKAGES: + model_package['disabled'] = True + return settings.MODEL_PACKAGES + + def get_client_settings(request): EMBED_FLAG = settings.ITSI['embed_flag'] @@ -127,7 +134,8 @@ def get_client_settings(request): 'draw_tools': settings.DRAW_TOOLS, 'map_controls': settings.MAP_CONTROLS, 'google_maps_api_key': settings.GOOGLE_MAPS_API_KEY, - 'vizer_urls': settings.VIZER_URLS + 'vizer_urls': settings.VIZER_URLS, + 'model_packages': get_model_packages(), }) } diff --git a/src/mmw/apps/modeling/migrations/0017_add_gwlfe.py b/src/mmw/apps/modeling/migrations/0017_add_gwlfe.py new file mode 100644 index 000000000..14f604c1d --- /dev/null +++ b/src/mmw/apps/modeling/migrations/0017_add_gwlfe.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('modeling', '0016_old_scenarios'), + ] + + operations = [ + migrations.AlterField( + model_name='project', + name='model_package', + field=models.CharField(help_text='Which model pack was chosen for this project', max_length=255, choices=[('tr-55', 'Site Storm Model'), ('gwlfe', 'Watershed Multi-Year Model')]), + ), + ] diff --git a/src/mmw/apps/modeling/models.py b/src/mmw/apps/modeling/models.py index 2774cbff5..4939890ff 100644 --- a/src/mmw/apps/modeling/models.py +++ b/src/mmw/apps/modeling/models.py @@ -9,7 +9,11 @@ class Project(models.Model): TR55 = 'tr-55' - MODEL_PACKAGES = ((TR55, 'Simple Model'),) + GWLFE = 'gwlfe' + MODEL_PACKAGES = ( + (TR55, 'Site Storm Model'), + (GWLFE, 'Watershed Multi-Year Model'), + ) user = models.ForeignKey(User) name = models.CharField( diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index 472be85d3..8fcc00f03 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -22,14 +22,13 @@ @shared_task -def start_mapshed_job(input): +def start_gwlfe_job(model_input): """ - Runs a Mapshed job. For now, just wait 3s and return the input. + Runs a GWLFE job. For now, just wait 3s and return the input. """ - # TODO remove sleep and call actual mapshed code + # TODO remove sleep and call actual GWLFE code time.sleep(3) response_json = { - 'input': input } return response_json diff --git a/src/mmw/apps/modeling/urls.py b/src/mmw/apps/modeling/urls.py index 7a36c77e0..07f669911 100644 --- a/src/mmw/apps/modeling/urls.py +++ b/src/mmw/apps/modeling/urls.py @@ -25,7 +25,7 @@ url(r'jobs/' + uuid_regex, views.get_job, name='get_job'), url(r'start/tr55/$', views.start_tr55, name='start_tr55'), url(r'start/rwd/$', views.start_rwd, name='start_rwd'), - url(r'start/mapshed/$', views.start_mapshed, name='start_mapshed'), + url(r'start/gwlfe/$', views.start_gwlfe, name='start_gwlfe'), url(r'boundary-layers/(?P\w+)/(?P[0-9]+)/$', views.boundary_layer_detail, name='boundary_layer_detail'), ) diff --git a/src/mmw/apps/modeling/views.py b/src/mmw/apps/modeling/views.py index 0c9738b53..0218c8f9d 100644 --- a/src/mmw/apps/modeling/views.py +++ b/src/mmw/apps/modeling/views.py @@ -189,17 +189,17 @@ def start_rwd(request, format=None): @decorators.api_view(['POST']) @decorators.permission_classes((AllowAny, )) -def start_mapshed(request, format=None): +def start_gwlfe(request, format=None): """ - Starts a job to run Mapshed. + Starts a job to run GWLF-E. """ user = request.user if request.user.is_authenticated() else None created = now() - input = request.POST['input'] + model_input = request.POST['model_input'] job = Job.objects.create(created_at=created, result='', error='', traceback='', user=user, status='started') - task_list = _initiate_mapshed_job_chain(input, job.id) + task_list = _initiate_gwlfe_job_chain(model_input, job.id) job.uuid = task_list.id job.save() @@ -212,13 +212,13 @@ def start_mapshed(request, format=None): ) -def _initiate_mapshed_job_chain(input, job_id): +def _initiate_gwlfe_job_chain(model_input, job_id): exchange = MAGIC_EXCHANGE routing_key = choose_worker() - return chain(tasks.start_mapshed_job.s(input) + return chain(tasks.start_gwlfe_job.s(model_input) .set(exchange=exchange, routing_key=routing_key), - save_job_result.s(job_id, input)) \ + save_job_result.s(job_id, model_input)) \ .apply_async(link_error=save_job_error.s(job_id)) diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index 53ca5541c..7197c7037 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -394,4 +394,22 @@ def get_env_setting(setting): 'ZoomControl', ] +GWLFE = 'gwlfe' +TR55_PACKAGE = 'tr-55' + +MODEL_PACKAGES = [ + { + 'name': TR55_PACKAGE, + 'display_name': 'Site Storm Model', + 'description': '', + }, + { + 'name': GWLFE, + 'display_name': 'Watershed Multi-Year Model', + 'description': '', + }, +] + +DISABLED_MODEL_PACKAGES = [] + # END UI CONFIGURATION diff --git a/src/mmw/mmw/settings/production.py b/src/mmw/mmw/settings/production.py index 467a023cc..7308ee2f2 100644 --- a/src/mmw/mmw/settings/production.py +++ b/src/mmw/mmw/settings/production.py @@ -108,6 +108,8 @@ 'ZoomControl', ] +DISABLED_MODEL_PACKAGES = [GWLFE] + # END UI CONFIGURATION # Google API key for production deployment From e1daac5ccaaefcb27632f27981b64e1685b2a2f8 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Thu, 31 Mar 2016 14:20:09 -0400 Subject: [PATCH 002/136] Add route/controller for creating models with a specific model package * Add constants for model package strings (unfortunately there are two for TR-55) * Handle the GWLF-E package, leaving some details TBD * Add route for creating projects with a specific model package. This required some refactoring of the project controller. --- src/mmw/js/src/core/settings.js | 3 +- src/mmw/js/src/modeling/controllers.js | 81 ++++++++++++------- src/mmw/js/src/modeling/models.js | 107 +++++++++++++++++-------- src/mmw/js/src/routes.js | 1 + 4 files changed, 128 insertions(+), 64 deletions(-) diff --git a/src/mmw/js/src/core/settings.js b/src/mmw/js/src/core/settings.js index 1d1a2a540..73c6cd9ae 100644 --- a/src/mmw/js/src/core/settings.js +++ b/src/mmw/js/src/core/settings.js @@ -8,7 +8,8 @@ var defaultSettings = { boundary_layers: {}, draw_tools: [], map_controls: [], - vizer_urls: {} + vizer_urls: {}, + model_packages: [] }; var settings = (function() { diff --git a/src/mmw/js/src/modeling/controllers.js b/src/mmw/js/src/modeling/controllers.js index 423fc220a..b6bdd09e8 100644 --- a/src/mmw/js/src/modeling/controllers.js +++ b/src/mmw/js/src/modeling/controllers.js @@ -121,13 +121,7 @@ var ModelingController = { var lock = $.Deferred(); if (!App.currentProject) { - // Only make new project if this is the first time - // hitting the modeling views. - if (!App.projectNumber) { - project = makeProject(lock); - } else { - project = reinstateProject(App.projectNumber, lock); - } + project = reinstateProject(App.projectNumber, lock); App.currentProject = project; if (!App.projectNumber) { @@ -139,37 +133,31 @@ var ModelingController = { lock.resolve(); } - lock.done(function() { - project.on('change:id', updateUrl); - initScenarioEvents(project); - initViews(project); - if (App.projectNumber) { - updateUrl(); - } - }); + finishProjectSetup(project, lock); } } App.state.set('current_page_title', 'Model'); }, + makeNewProject: function(modelPackage) { + var project; + var lock = $.Deferred(); + project = makeProject(modelPackage, lock); + + App.currentProject = project; + + setupNewProjectScenarios(project); + finishProjectSetup(project, lock); + updateUrl(); + }, + projectCleanUp: function() { - App.rootView.hideCollapsable(); - if (App.currentProject) { - var scenarios = App.currentProject.get('scenarios'); - - App.currentProject.off('change:id', updateUrl); - scenarios.off('change:activeScenario change:id', updateScenario); - // App.projectNumber holds the number of the project that was - // in use when the user left the `/project` page. The intent - // is to allow the same project to be returned-to via the UI - // arrow buttons (see issue #690). - App.projectNumber = scenarios.at(0).get('project'); - } + projectCleanUp(); + }, - App.getMapView().updateModifications(null); - App.rootView.subHeaderRegion.empty(); - App.rootView.sidebarRegion.empty(); + makeNewProjectCleanUp: function() { + projectCleanUp(); }, // Since we handle redirects after ITSI sign-up within Backbone, @@ -214,6 +202,36 @@ var ModelingController = { } }; +function finishProjectSetup(project, lock) { + lock.done(function() { + project.on('change:id', updateUrl); + initScenarioEvents(project); + initViews(project); + if (App.projectNumber) { + updateUrl(); + } + }); +} + +function projectCleanUp() { + App.rootView.hideCollapsable(); + if (App.currentProject) { + var scenarios = App.currentProject.get('scenarios'); + + App.currentProject.off('change:id', updateUrl); + scenarios.off('change:activeScenario change:id', updateScenario); + // App.projectNumber holds the number of the project that was + // in use when the user left the `/project` page. The intent + // is to allow the same project to be returned-to via the UI + // arrow buttons (see issue #690). + App.projectNumber = scenarios.at(0).get('project'); + } + + App.getMapView().updateModifications(null); + App.rootView.subHeaderRegion.empty(); + App.rootView.sidebarRegion.empty(); +} + function updateItsiFromEmbedMode() { if (settings.get('itsi_embed')) { App.itsi.setLearnerUrl(Backbone.history.getFragment()); @@ -266,12 +284,13 @@ function initViews(project, lock) { App.rootView.sidebarRegion.show(resultsView); } -function makeProject(lock) { +function makeProject(modelPackage, lock) { var project = new models.ProjectModel({ name: 'Untitled Project', created_at: Date.now(), area_of_interest: App.map.get('areaOfInterest'), area_of_interest_name: App.map.get('areaOfInterestName'), + model_package: modelPackage, scenarios: new models.ScenariosCollection() }); lock.resolve(); diff --git a/src/mmw/js/src/modeling/models.js b/src/mmw/js/src/modeling/models.js index 125897859..23ed5e8b6 100644 --- a/src/mmw/js/src/modeling/models.js +++ b/src/mmw/js/src/modeling/models.js @@ -10,6 +10,10 @@ var Backbone = require('../../shim/backbone'), turfErase = require('turf-erase'), turfIntersect = require('turf-intersect'); +var GWLFE = 'gwlfe'; +var TR55_TASK = 'tr55'; +var TR55_PACKAGE = 'tr-55'; + var ModelPackageControlModel = Backbone.Model.extend({ defaults: { name: '' @@ -33,7 +37,17 @@ var ModelPackageModel = Backbone.Model.extend(); var Tr55TaskModel = coreModels.TaskModel.extend({ defaults: _.extend( { - taskName: 'tr55', + taskName: TR55_TASK, + taskType: 'modeling' + }, + coreModels.TaskModel.prototype.defaults + ) +}); + +var GwlfeTaskModel = coreModels.TaskModel.extend({ + defaults: _.extend( + { + taskName: GWLFE, taskType: 'modeling' }, coreModels.TaskModel.prototype.defaults @@ -93,7 +107,7 @@ var ProjectModel = Backbone.Model.extend({ created_at: null, // Date area_of_interest: null, // GeoJSON area_of_interest_name: null, // Human readable string for AOI. - model_package: '', // Package name + model_package: TR55_PACKAGE, // Package name scenarios: null, // ScenariosCollection user_id: 0, // User that created the project is_activity: false, // Project that persists across routes @@ -106,12 +120,6 @@ var ProjectModel = Backbone.Model.extend({ if (scenarios === null || typeof scenarios === 'string') { this.set('scenarios', new ScenariosCollection()); } - // TODO: For a new project, users will eventually - // be able to choose which modeling package - // they want to use in their project. For - // now, the only option is TR55, so it is - // hard-coded here. - this.set('model_package', 'tr-55'); this.set('user_id', App.user.get('id')); @@ -122,33 +130,16 @@ var ProjectModel = Backbone.Model.extend({ this.listenTo(this.get('scenarios'), 'add', this.addIdsToScenarios, this); }, + setProjectModel: function(modelPackage) { + this.set('model_package', modelPackage); + }, + createTaskModel: function() { - var packageName = this.get('model_package'); - switch (packageName) { - case 'tr-55': - return new Tr55TaskModel(); - } - throw 'Model package not supported: ' + packageName; + return createTaskModel(this.get('model_package')); }, createTaskResultCollection: function() { - var packageName = this.get('model_package'); - switch (packageName) { - case 'tr-55': - return new ResultCollection([ - { - name: 'runoff', - displayName: 'Runoff', - result: null - }, - { - name: 'quality', - displayName: 'Water Quality', - result: null - } - ]); - } - throw 'Model package not supported: ' + packageName; + return createTaskResultCollection(this.get('model_package')); }, fetchResultsIfNeeded: function() { @@ -241,6 +232,8 @@ var ProjectModel = Backbone.Model.extend({ scenarios = _.map(response.scenarios, function(scenario) { var scenarioModel = new ScenarioModel(scenario); scenarioModel.set('user_id', user_id); + scenarioModel.set('taskModel', createTaskModel(response.model_package)); + scenarioModel.set('results', createTaskResultCollection(response.model_package)); scenarioModel.get('modifications').reset(scenario.modifications); if (!_.isEmpty(scenario.results)) { scenarioModel.get('results').reset(scenario.results); @@ -824,7 +817,7 @@ var ScenariosCollection = Backbone.Collection.extend({ }); function getControlsForModelPackage(modelPackageName, options) { - if (modelPackageName === 'tr-55') { + if (modelPackageName === TR55_PACKAGE) { if (options && (options.compareMode || options.is_current_conditions)) { return new ModelPackageControlsCollection([ @@ -837,11 +830,57 @@ function getControlsForModelPackage(modelPackageName, options) { new ModelPackageControlModel({ name: 'precipitation' }) ]); } + } else if (modelPackageName === GWLFE) { + // TODO add controls for GWLFE + return new ModelPackageControlsCollection([]); } throw 'Model package not supported ' + modelPackageName; } +function createTaskModel(modelPackage) { + switch (modelPackage) { + case TR55_PACKAGE: + return new Tr55TaskModel(); + case GWLFE: + return new GwlfeTaskModel(); + } + throw 'Model package not supported: ' + modelPackage; +} + +function createTaskResultCollection(modelPackage) { + switch (modelPackage) { + case TR55_PACKAGE: + return new ResultCollection([ + { + name: 'runoff', + displayName: 'Runoff', + result: null + }, + { + name: 'quality', + displayName: 'Water Quality', + result: null + } + ]); + case GWLFE: + return new ResultCollection([ + { + name: 'runoff', + displayName: 'Runoff', + result: null + }, + { + name: 'quality', + displayName: 'Water Quality', + result: null + } + ]); + } + throw 'Model package not supported: ' + modelPackage; +} + + module.exports = { getControlsForModelPackage: getControlsForModelPackage, ResultModel: ResultModel, @@ -850,6 +889,10 @@ module.exports = { ModelPackageControlsCollection: ModelPackageControlsCollection, ModelPackageControlModel: ModelPackageControlModel, Tr55TaskModel: Tr55TaskModel, + GwlfeTaskModel: GwlfeTaskModel, + TR55_TASK: TR55_TASK, + TR55_PACKAGE: TR55_PACKAGE, + GWLFE: GWLFE, ProjectModel: ProjectModel, ProjectCollection: ProjectCollection, ModificationModel: ModificationModel, diff --git a/src/mmw/js/src/routes.js b/src/mmw/js/src/routes.js index 2a3565e21..f74b0d6a8 100644 --- a/src/mmw/js/src/routes.js +++ b/src/mmw/js/src/routes.js @@ -11,6 +11,7 @@ var router = require('./router').router, router.addRoute(/^/, DrawController, 'draw'); router.addRoute(/^analyze/, AnalyzeController, 'analyze'); +router.addRoute('project/new/:modelPackage(/)', ModelingController, 'makeNewProject'); router.addRoute('project(/:projectId)(/scenario/:scenarioId)(/)', ModelingController, 'project'); router.addRoute('project/:projectId/clone(/)', ModelingController, 'projectClone'); router.addRoute('project/:projectId/draw(/)', ModelingController, 'projectDraw'); From 199908f8aa2cb5c35077aa9a8022b4c44440335d Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Thu, 31 Mar 2016 14:23:41 -0400 Subject: [PATCH 003/136] Add model package dropdown in Analyze view There is now a dropdown to select the model package in the Analyze view. This is just a placeholder for a nicer dropdown that will be added after the prototype is done. The views shown for GWLF-E are just the TR-55 views for now, since it's not clear how they will be different. The backend does not return any results, so the views are blank. --- .../src/analyze/templates/resultsWindow.html | 25 ++++++++++++++++--- src/mmw/js/src/analyze/views.js | 7 ++++++ src/mmw/js/src/modeling/views.js | 16 ++++++++++-- src/mmw/sass/pages/_model.scss | 8 ++++++ 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/src/mmw/js/src/analyze/templates/resultsWindow.html b/src/mmw/js/src/analyze/templates/resultsWindow.html index e59fe7bdc..9dc0b3643 100644 --- a/src/mmw/js/src/analyze/templates/resultsWindow.html +++ b/src/mmw/js/src/analyze/templates/resultsWindow.html @@ -7,14 +7,31 @@
  • - + +
    + + +
    • diff --git a/src/mmw/js/src/analyze/views.js b/src/mmw/js/src/analyze/views.js index 62bb9aed2..f53b59fcf 100644 --- a/src/mmw/js/src/analyze/views.js +++ b/src/mmw/js/src/analyze/views.js @@ -5,6 +5,7 @@ var $ = require('jquery'), Marionette = require('../../shim/backbone.marionette'), App = require('../app'), models = require('./models'), + settings = require('../core/settings'), coreModels = require('../core/models'), chart = require('../core/chart'), utils = require('../core/utils'), @@ -43,6 +44,12 @@ var ResultsView = Marionette.LayoutView.extend({ })); }, + templateHelpers: function() { + return { + modelPackages: settings.get('model_packages') + }; + }, + transitionInCss: { height: '0%' }, diff --git a/src/mmw/js/src/modeling/views.js b/src/mmw/js/src/modeling/views.js index 1d0f99d09..1dfade654 100644 --- a/src/mmw/js/src/modeling/views.js +++ b/src/mmw/js/src/modeling/views.js @@ -24,7 +24,9 @@ projectMenuTmpl = require('./templates/projectMenu.html'), scenarioToolbarTabContentTmpl = require('./templates/scenarioToolbarTabContent.html'), tr55RunoffViews = require('./tr55/runoff/views.js'), - tr55QualityViews = require('./tr55/quality/views.js'); + tr55QualityViews = require('./tr55/quality/views.js'), + gwlfeRunoffViews = tr55RunoffViews, // TODO actually make views for gwlfe + gwlfeQualityViews = tr55QualityViews; var ENTER_KEYCODE = 13, ESCAPE_KEYCODE = 27; @@ -838,7 +840,7 @@ function triggerBarChartRefresh() { function getResultView(modelPackage, resultName) { switch (modelPackage) { - case 'tr-55': + case models.TR55_PACKAGE: switch(resultName) { case 'runoff': return tr55RunoffViews.ResultView; @@ -848,6 +850,16 @@ function getResultView(modelPackage, resultName) { console.log('Result not supported.'); } break; + case models.GWLFE: + switch(resultName) { + case 'runoff': + return gwlfeRunoffViews.ResultView; + case 'quality': + return gwlfeQualityViews.ResultView; + default: + console.log('Result not supported.'); + } + break; default: console.log('Model package ' + modelPackage + ' not supported.'); } diff --git a/src/mmw/sass/pages/_model.scss b/src/mmw/sass/pages/_model.scss index 4f3b37886..38dc9bd76 100644 --- a/src/mmw/sass/pages/_model.scss +++ b/src/mmw/sass/pages/_model.scss @@ -18,6 +18,14 @@ transform: translateX(100%); } + .back-button { + display: inline-block; + } + + .disabled { + text-decoration: none; + } + &.analyze { top: $height-lg; } From 5cb98b6c06685205cc985d4bb76b66fc5460c13f Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 8 Apr 2016 12:38:33 -0400 Subject: [PATCH 004/136] Add GWLF-E dependency Eventually this will be published to PyPI, but for now we pull it directly from GitHub. Also add numpy. --- src/mmw/requirements/base.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mmw/requirements/base.txt b/src/mmw/requirements/base.txt index f455b8034..a0ceea90f 100644 --- a/src/mmw/requirements/base.txt +++ b/src/mmw/requirements/base.txt @@ -13,3 +13,5 @@ tr55==1.1.3 requests==2.9.1 rollbar>=0.11.0,<=0.12.0 retry==0.9.1 +numpy==1.11.0 +-e git+https://github.com/WikiWatershed/gwlf-e.git@develop#egg=gwlf-e From 39ed651982bae1d4e1ecf23febf961eb0a25a89f Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 8 Apr 2016 16:38:15 -0400 Subject: [PATCH 005/136] Add static GWLF-E values --- src/mmw/mmw/settings/base.py | 1 + src/mmw/mmw/settings/gwlfe_settings.py | 394 +++++++++++++++++++++++++ 2 files changed, 395 insertions(+) create mode 100644 src/mmw/mmw/settings/gwlfe_settings.py diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index 7197c7037..b93fa7f32 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -13,6 +13,7 @@ from sys import path from layer_settings import LAYERS, VIZER_URLS # NOQA +from gwlfe_settings import GWLFE_DEFAULTS # NOQA # Normally you should not import ANYTHING from Django directly # into your settings, but ImproperlyConfigured is an exception. diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py new file mode 100644 index 000000000..5182c8552 --- /dev/null +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -0,0 +1,394 @@ +import numpy as np + +GWLFE_DEFAULTS = { + 'NRur': 10, # Number of Rural Land Use Categories + 'NUrb': 6, # Number of Urban Land Use Categories + 'TranVersionNo': '1.4.0', # GWLF-E Version + 'RecessionCoef': 0.06, # Recession Coefficient + 'SeepCoef': 0, # Seepage Coefficient + 'UnsatStor': 10, # Unsaturated Storage + 'SatStor': 0, # Saturated Storage + 'InitSnow': 0, # Initial Snow Days + 'TileDrainRatio': 0, # Tile Drain Ratio + 'TileDrainDensity': 0, # Tile Drain Density + 'ETFlag': 0, # ET Flag: 0 for Hammon method, 1 for Blainy-Criddle method + 'AntMoist': np.zeros(5), # Antecedent Rain + Melt Moisture Conditions for Days 1 to 5 # NOQA + 'StreamWithdrawal': np.zeros(12), # Surface Water Withdrawal/Extraction + 'GroundWithdrawal': np.zeros(12), # Groundwater Withdrawal/Extraction + 'PcntET': np.ones(12), # Percent monthly adjustment for ET calculation + 'PhysFlag': 0, # Flag: Physiographic Province Layer Detected (0 No; 1 Yes) + 'PointFlag': 1, # Flag: Point Source Layer Detected (0 No; 1 Yes) + 'SeptSysFlag': 0, # Flag: Septic System Layer Detected (0 No; 1 Yes) + 'CountyFlag': 0, # Flag: County Layer Detected (0 No; 1 Yes) + 'SoilPFlag': 1, # Flag: Soil P Layer Detected (0 No; 1 Yes) + 'GWNFlag': 1, # Flag: Groundwater N Layer Detected (0 No; 1 Yes) + 'SedAAdjust': 1, # Default Percent ET + 'SedNitr': 2000, # Soil Concentration: N (mg/l) + 'BankNFrac': 0.25, # % Bank N Fraction (0 - 1) + 'BankPFrac': 0.25, # % Bank P Fraction (0 - 1) + 'ManuredAreas': 2, # Manure Spreading Periods (Default = 2) + 'FirstManureMonth': 0, # MS Period 1: First Month + 'LastManureMonth': 0, # MS Period 1: Last Month + 'FirstManureMonth2': 0, # MS Period 2: First Month + 'LastManureMonth2': 0, # MS Period 2: Last Month + 'Nqual': 3, # Number of Contaminants (Default = 3) + 'Contaminant': ['Nitrogen', 'Phosphorus', 'Sediment'], + 'UrbBMPRed': np.zeros((16, 3)), # Urban BMP Reduction + 'SepticFlag': 1, # Flag: Septic Systems Layer Detected (0 No; 1 Yes) + 'NumPondSys': np.zeros(12), # Number of People on Pond Systems + 'NumShortSys': np.zeros(12), # Number of People on Short Circuit Systems + 'NumDischargeSys': np.zeros(12), # Number of People on Discharge Systems + 'NumSewerSys': np.zeros(12), # Number of People on Public Sewer Systems + 'NitrSepticLoad': 12, # Per Capita Tank Load: N (g/d) + 'PhosSepticLoad': 2.5, # Per Capita Tank Load: P (g/d) + 'NitrPlantUptake': 1.6, # Growing System Uptake: N (g/d) + 'PhosPlantUptake': 0.4, # Growing System Uptake: P (g/d) + 'TileNconc': 15, # Tile Drainage Concentration: N (mg/L) + 'TilePConc': 0.1, # Tile Drainage Concentration: P (mg/L) + 'TileSedConc': 50, # Tile Drainage Concentration: Sediment (mg/L) + 'n1': 0, # Row Crops + 'n2': 0, # Hay/Pasture + 'n2b': 0, # High Density Urban + 'n2c': 0, # Low Density Urban + 'n2d': 0, # Unpaved Roads + 'n3': 0, # Other + 'n4': 0, # Streambank Erosion + 'n5': 0, # Row Crops + 'n6': 0, # Hay/Pasture + 'n6b': 0, # High Density Urban + 'n6c': 0, # Low Density Urban + 'n6d': 0, # Unpaved Roads + 'n7': 0, # Other + 'n7b': 0, # Farm Animals + 'n8': 0, # Streambank Erosion + 'n9': 0, # Groundwater/Subsurface + 'n10': 0, # Point Source Discharges + 'n11': 0, # Septic Systems + 'n12': 0, # Row Crops + 'n13': 0, # Hay/Pasture + 'n13b': 0, # High Density Urban + 'n13c': 0, # Low Density Urban + 'n13d': 0, # Unpaved Roads + 'n14': 0, # Other + 'n14b': 0, # Farm Animals + 'n15': 0, # Streambank Erosion + 'n16': 0, # Groundwater/Subsurface + 'n17': 0, # Point Source Discharges + 'n18': 0, # Septic Systems + 'n19': 0, # Total Sediment Load (kg x 1000) + 'n20': 0, # Total Nitrogen Load (kg) + 'n21': 0, # Total Phosphorus Load (kg) + 'n22': 0, # Basin Area (Ha) + 'n23c': 5, # High Density Urban (Constructed Wetlands): % Drainage Used + 'n24c': 3, # Low Density Urban (Constructed Wetlands): % Drainage Used + 'n24d': 6, # High Density Urban (Bioretention Areas): % Drainage Used + 'n24e': 6, # Low Density Urban (Bioretention Areas): % Drainage Used + 'n25': 0, # Row Crops (BMP 1): Existing (%) + 'n25b': 0, # High Density Urban (Constructed Wetlands): Existing (%) + 'n25c': 0, # Low Density Urban (Constructed Wetlands): Existing (%) + 'n25d': 0, # High Density Urban (Bioretention Areas): Existing (%) + 'n25e': 0, # Low Density Urban (Bioretention Areas): Existing (%) + 'n26': 0, # Row Crops (BMP 2): Existing (%) + 'n26b': 0, # High Density Urban (Detention Basin): Existing (%) + 'n26c': 0, # Low Density Urban (Detention Basin): Existing (%) + 'n27': 0, # Row Crops (BMP 3): Existing (%) + 'n27b': 0, # Row Crops (BMP 4): Existing (%) + 'n28': 0, # Row Crops (BMP 5): Existing (%) + 'n28b': 0, # Row Crops (BMP 6): Existing (%) + 'n29': 0, # Row Crops (BMP 8): Existing (%) + 'n30': 0, # Row Crops (BMP 1): Future (%) + 'n30b': 0, # High Density Urban (Constructed Wetlands): Future (%) + 'n30c': 0, # Low Density Urban (Constructed Wetlands): Future (%) + 'n30d': 0, # High Density Urban (Bioretention Areas): Future (%) + 'n30e': 0, # Low Density Urban (Bioretention Areas): Future (%) + 'n31': 0, # Row Crops (BMP 2): Future (%) + 'n31b': 0, # High Density Urban (Detention Basin): Future (%) + 'n31c': 0, # Low Density Urban (Detention Basin): Future (%) + 'n32': 0, # Row Crops (BMP 3): Future (%) + 'n32b': 0, # Row Crops (BMP 4): Future (%) + 'n32c': 0, # Hay/Pasture (BMP 3): Existing (%) + 'n32d': 0, # Hay/Pasture (BMP 3): Future (%) + 'n33': 0, # Row Crops (BMP 5): Future (%) + 'n33b': 0, # Row Crops (BMP 6): Future (%) + 'n33c': 0, # Hay/Pasture (BMP 4): Existing (%) + 'n33d': 0, # Hay/Pasture (BMP 4): Future (%) + 'n34': 0, # Row Crops (BMP 8): Future (%) + 'n35': 0, # Hay/Pasture (BMP 5): Existing (%) + 'n35b': 0, # Hay/Pasture (BMP 6): Existing (%) + 'n36': 0, # Hay/Pasture (BMP 7): Existing (%) + 'n37': 0, # Hay/Pasture (BMP 8): Existing (%) + 'n38': 0, # Hay/Pasture (BMP 5): Future (%) + 'n38b': 0, # Hay/Pasture (BMP 6): Future (%) + 'n39': 0, # Hay/Pasture (BMP 7): Future (%) + 'n40': 0, # Hay/Pasture (BMP 8): Future (%) + 'n41b': 0, # AWMS (Livestock): Existing (%) + 'n41c': 0, # AWMS (Livestock): Future (%) + 'n41d': 0, # AWMS (Poultry): Existing (%) + 'n41e': 0, # AWMS (Poultry): Future (%) + 'n41f': 0, # Runoff Control: Existing (%) + 'n41g': 0, # Runoff Control: Future (%) + 'n41h': 0, # Phytase in Feed: Existing (%) + 'n41i': 0, # Phytase in Feed: Future (%) + 'n43': 0, # Stream Km with Vegetated Buffer Strips: Existing + 'GRLBN': 0, # Average Grazing Animal Loss Rate (Barnyard/Confined Area): Nitrogen # NOQA + 'NGLBN': 0, # Average Non-Grazing Animal Loss Rate (Barnyard/Confined Area): Nitrogen # NOQA + 'GRLBP': 0, # Average Grazing Animal Loss Rate (Barnyard/Confined Area): Phosphorus # NOQA + 'NGLBP': 0, # Average Non-Grazing Animal Loss Rate (Barnyard/Confined Area): Phosphorus # NOQA + 'NGLManP': 0, # Average Non-Grazing Animal Loss Rate (Manure Spreading): Phosphorus # NOQA + 'NGLBFC': 0, # Average Non-Grazing Animal Loss Rate (Barnyard/Confined Area): Fecal Coliform # NOQA + 'GRLBFC': 0, # Average Grazing Animal Loss Rate (Barnyard/Confined Area): Fecal Coliform # NOQA + 'GRSFC': 0, # Average Grazing Animal Loss Rate (Spent in Streams): Fecal Coliform # NOQA + 'GRSN': 0, # Average Grazing Animal Loss Rate (Spent in Streams): Nitrogen + 'GRSP': 0, # Average Grazing Animal Loss Rate (Spent in Streams): Phosphorus # NOQA + 'n43b': 0, # High Density Urban (Constructed Wetlands): Required Ha + 'n43c': 0, # High Density Urban (Detention Basin): % Drainage Used + 'n43d': 0, # High Density Urban: % Impervious Surface + 'n43e': 0, # High Density Urban (Constructed Wetlands): Impervious Ha Drained # NOQA + 'n43f': 0, # High Density Urban (Detention Basin): Impervious Ha Drained + 'n43g': 0, # High Density Urban (Bioretention Areas): Impervious Ha Drained + 'n43h': 0, # High Density Urban (Bioretention Areas): Required Ha + 'n43i': 0, # Low Density Urban (Bioretention Areas): Impervious Ha Drained + 'n43j': 0, # Low Density Urban (Bioretention Areas): Required Ha + 'n44': 0, # Stream Km with Vegetated Buffer Strips: Future + 'n44b': 0, # High Density Urban (Detention Basin): Required Ha + 'n45': 0, # Stream Km with Fencing: Existing + 'n45b': 0, # Low Density Urban (Constructed Wetlands): Required Ha + 'n45c': 0, # Low Density Urban (Detention Basin): % Drainage Used + 'n45d': 0, # Low Density Urban: % Impervious Surface + 'n45e': 0, # Low Density Urban (Constructed Wetlands): Impervious Ha Drained # NOQA + 'n45f': 0, # Low Density Urban (Detention Basin): Impervious Ha Drained + 'n46': 0, # Stream Km with Fencing: Future + 'n46b': 0, # Low Density Urban (Detention Basin): Required Ha + 'n46c': 0, # Stream Km with Stabilization: Existing + 'n46d': 0, # Stream Km with Stabilization: Future + 'n46g': 0, # Stream Km in High Density Urban Areas W/Buffers: Existing + 'n46h': 0, # Stream Km in High Density Urban Areas W/Buffers: Future + 'n46i': 0, # High Density Urban Streambank Stabilization (km): Existing + 'n46j': 0, # High Density Urban Streambank Stabilization (km): Future + 'n46k': 0, # Stream Km in Low Density Urban Areas W/Buffers: Existing + 'n46l': 0, # Stream Km in Low Density Urban Areas W/Buffers: Future + 'n46m': 0, # Low Density Urban Streambank Stabilization (km): Existing + 'n46n': 0, # Low Density Urban Streambank Stabilization (km): Future + 'n46o': 0, # Unpaved Road Km with E and S Controls (km): Existing + 'n46p': 0, # Unpaved Road Km with E and S Controls (km): Future + 'n47': 0, # Number of Persons on Septic Systems: Existing + 'n48': 0, # No longer used (Default = 0) + 'n49': 0, # Number of Persons on Septic Systems: Future + 'n50': 0, # No longer used (Default = 0) + 'n51': 0, # Septic Systems Converted by Secondary Treatment Type (%) + 'n52': 0, # Septic Systems Converted by Tertiary Treatment Type (%) + 'n53': 0, # No longer used (Default = 0) + 'n54': 0, # Distribution of Pollutant Discharges by Primary Treatment Type (%): Existing # NOQA + 'n55': 0, # Distribution of Pollutant Discharges by Secondary Treatment Type (%): Existing # NOQA + 'n56': 0, # Distribution of Pollutant Discharges by Tertiary Treatment Type (%): Existing # NOQA + 'n57': 0, # Distribution of Pollutant Discharges by Primary Treatment Type (%): Future # NOQA + 'n58': 0, # Distribution of Pollutant Discharges by Secondary Treatment Type (%): Future # NOQA + 'n59': 0, # Distribution of Pollutant Discharges by Tertiary Treatment Type (%): Future # NOQA + 'n60': 0, # Distribution of Treatment Upgrades (%): Primary to Secondary + 'n61': 0, # Distribution of Treatment Upgrades (%): Primary to Tertiary + 'n62': 0, # Distribution of Treatment Upgrades (%): Secondary to Tertiary + 'n63': 0.29, # BMP 1 (Nitrogen) + 'n64': 0.41, # Vegetated Buffer Strips (Nitrogen) + 'n65': 0.08, # BMP 2 (Nitrogen) + 'n66': 0.66, # BMP 3 (Nitrogen) + 'n66b': 0.05, # BMP 4 (Nitrogen) + 'n67': 0, # BMP 5 (Nitrogen) + 'n68': 0.95, # BMP 8 (Nitrogen) + 'n68b': 0.3, # BMP 7 (Nitrogen) + 'n69': 0.56, # Streambank Fencing (Nitrogen) + 'n69b': 0.2, # Constructed Wetlands (Nitrogen) + 'n69c': 0.95, # Streambank Stabilization (Nitrogen) + 'n70': 0.29, # BMP 6 (Nitrogen) + 'n70b': 0.25, # Detention Basins (Nitrogen) + 'n71': 0.5, # BMP 1 (Phosphorus) + 'n71b': 0.28, # Bioretention Areas (Nitrogen) + 'n72': 0.4, # Vegetated Buffer Strips (Phosphorus) + 'n73': 0.22, # BMP 2 (Phosphorus) + 'n74': 0.1, # BMP 3 (Phosphorus) + 'n74b': 0.1, # BMP 4 (Phosphorus) + 'n75': 0, # BMP 5 (Phosphorus) + 'n76': 0.95, # BMP 8 (Phosphorus) + 'n76b': 0.3, # BMP 7 (Phosphorus) + 'n77': 0.78, # Streambank Fencing (Phosphorus) + 'n77b': 0.45, # Constructed Wetlands (Phosphorus) + 'n77c': 0.95, # Streambank Stabilization (Phosphorus) + 'n78': 0.44, # BMP 6 (Phosphorus) + 'n78b': 0.35, # Detention Basins (Phosphorus) + 'n79': 0.35, # BMP 1 (Sediment) + 'n79b': 0.44, # Bioretention Areas (Phosphorus) + 'n79c': 0.63, # Bioretention Areas (Sediment) + 'n80': 0.53, # Vegetated Buffer Strips (Sediment) + 'n81': 0.3, # BMP 2 (Sediment) + 'n82': 0.17, # BMP 3 (Sediment) + 'n82b': 0.16, # BMP 4 (Sediment) + 'n83': 0, # BMP 5 (Sediment) + 'n84': 0.95, # BMP 8 (Sediment) + 'n84b': 0.38, # BMP 7 (Sediment) + 'n85': 0.76, # Streambank Fencing (Sediment) + 'n85b': 0.6, # Constructed Wetlands (Sediment) + 'n85c': 0.55, # Detention Basins (Sediment) + 'n85d': 0.95, # Streambank Stabilization (Sediment) + 'n85e': 0.02, # Unpaved Road (kg/meter) (Nitrogen) + 'n85f': 0.0035, # Unpaved Road (kg/meter) (Phosphorus) + 'n85g': 2.55, # Unpaved Road (kg/meter) (Sediment) + 'n85h': 0.75, # AWMS (Livestock) (Nitrogen) + 'n85i': 0.75, # AWMS (Livestock) (Phosphorus) + 'n85j': 0.14, # AWMS (Poultry) (Nitrogen) + 'n85k': 0.14, # AWMS (Poultry) (Phosphorus) + 'n85l': 0.15, # Runoff Control (Nitrogen) + 'n85m': 0.15, # Runoff Control (Phosphorus) + 'n85n': 0.21, # Phytase in Feed (Phosphorus) + 'n85o': 0.7, # Vegetated Buffer Strips (Pathogens) + 'n85p': 1, # Streambank Fencing (Pathogens) + 'n85q': 0.85, # AWMS (Livestock) (Pathogens) + 'n85r': 0.14, # AWMS (Poultry) (Pathogens) + 'n85s': 0.15, # Runoff Control (Pathogens) + 'n85t': 0.71, # Constructed Wetlands (Pathogens) + 'n85u': 0.82, # Bioretention Areas (Pathogens) + 'n85v': 0.71, # Detention Basins (Pathogens) + 'Qretention': 0, # Detention Basin: Amount of runoff retention (cm) + 'FilterWidth': 0, # Stream Protection: Vegetative buffer strip width (meters) # NOQA + 'Capacity': 0, # Detention Basin: Detention basin volume (cubic meters) + 'BasinDeadStorage': 0, # Detention Basin: Basin dead storage (cubic meters) + 'BasinArea': 0, # Detention Basin: Basin surface area (square meters) + 'DaysToDrain': 0, # Detention Basin: Basin days to drain + 'CleanMon': 0, # Detention Basin: Basin cleaning month + 'PctAreaInfil': 0, # Infiltration/Bioretention: Fraction of area treated (0-1) # NOQA + 'PctStrmBuf': 0, # Stream Protection: Fraction of streams treated (0-1) + 'UrbBankStab': 0, # Stream Protection: Streams w/bank stabilization (km) + 'ISRR': np.zeros(6), # Impervious Surface Reduction (% Reduction) of Urban Land Uses # NOQA + 'ISRA': np.zeros(6), # Impervious Surface Reduction (Area) of Urban Land Uses # NOQA + 'SweepType': 1, # Street Sweeping: Sweep Type (1-2) + 'UrbSweepFrac': 1, # Street Sweeping: Fraction of area treated (0-1) + 'StreetSweepNo': np.zeros(12), # Street sweeping times per month + 'n108': 0, # Row Crops: Sediment (kg x 1000) + 'n109': 0, # Row Crops: Nitrogen (kg) + 'n110': 0, # Row Crops: Phosphorus (kg) + 'n111': 0, # Hay/Pasture: Sediment (kg x 1000) + 'n111b': 0, # High Density Urban: Sediment (kg x 1000) + 'n111c': 0, # Low Density Urban: Sediment (kg x 1000) + 'n111d': 0, # Unpaved Roads: Sediment (kg x 1000) + 'n112': 0, # Hay/Pasture: Nitrogen (kg) + 'n112b': 0, # High Density Urban: Nitrogen (kg) + 'n112c': 0, # Low Density Urban: Nitrogen (kg) + 'n112d': 0, # Unpaved Roads: Nitrogen (kg) + 'n113': 0, # Hay/Pasture: Phosphorus (kg) + 'n113b': 0, # High Density Urban: Phosphorus (kg) + 'n113c': 0, # Low Density Urban: Phosphorus (kg) + 'n113d': 0, # Unpaved Roads: Phosphorus (kg) + 'n114': 0, # Other: Sediment (kg x 1000) + 'n115': 0, # Other: Nitrogen (kg) + 'n115b': 0, # Farm Animals: Nitrogen (kg) + 'n116': 0, # Other: Phosphorus (kg) + 'n116b': 0, # Farm Animals: Phosphorus (kg) + 'n117': 0, # Streambank Erosion: Sediment (kg x 1000) + 'n118': 0, # Streambank Erosion: Nitrogen (kg) + 'n119': 0, # Streambank Erosion: Phosphorus (kg) + 'n120': 0, # Groundwater/Subsurface: Nitrogen (kg) + 'n121': 0, # Groundwater/Subsurface: Phosphorus (kg) + 'n122': 0, # Point Source Discharges: Nitrogen (kg) + 'n123': 0, # Point Source Discharges: Phosphorus (kg) + 'n124': 0, # Septic Systems: Nitrogen (kg) + 'n125': 0, # Septic Systems: Phosphorus (kg) + 'n126': 0, # Total: Sediment (kg x 1000) + 'n127': 0, # Total: Nitrogen (kg) + 'n128': 0, # Total: Phosphorus (kg) + 'n129': 0, # Percent Reduction: Sediment (%) + 'n130': 0, # Percent Reduction: Nitrogen (%) + 'n131': 0, # Percent Reduction: Phosphorus (%) + 'n132': 0, # Estimated Scenario Cost $: Total + 'n133': 0, # Estimated Scenario Cost $: Agricultural BMPs + 'n134': 0, # Estimated Scenario Cost $: Waste Water Upgrades + 'n135': 0, # Estimated Scenario Cost $: Urban BMPs + 'n136': 0, # Estimated Scenario Cost $: Stream Protection + 'n137': 0, # Estimated Scenario Cost $: Unpaved Road Protection + 'n138': 0, # Estimated Scenario Cost $: Animal BMPs + 'n139': 0, # Pathogen Loads (Farm Animals): Existing (orgs/month) + 'n140': 0, # Pathogen Loads (Wastewater Treatment Plants): Existing (orgs/month) # NOQA + 'n141': 0, # Pathogen Loads (Septic Systems): Existing (orgs/month) + 'n142': 0, # Pathogen Loads (Urban Areas): Existing (orgs/month) + 'n143': 0, # Pathogen Loads (Wildlife): Existing (orgs/month) + 'n144': 0, # Pathogen Loads (Total): Existing (orgs/month) + 'n145': 0, # Pathogen Loads (Farm Animals): Future (orgs/month) + 'n146': 0, # Pathogen Loads (Wastewater Treatment Plants): Future (orgs/month) # NOQA + 'n147': 0, # Pathogen Loads (Septic Systems): Future (orgs/month) + 'n148': 0, # Pathogen Loads (Urban Areas): Future (orgs/month) + 'n149': 0, # Pathogen Loads (Wildlife): Future (orgs/month) + 'n150': 0, # Pathogen Loads (Total): Future (orgs/month) + 'n151': 0, # Pathogen Loads: Percent Reduction (%) + 'InitNgN': 2373, # Initial Non-Grazing Animal Totals: Nitrogen (kg/yr) + 'InitNgP': 785, # Initial Non-Grazing Animal Totals: Phosphorus (kg/yr) + 'InitNgFC': 3.38e+9, # Initial Non-Grazing Animal Totals: Fecal Coliforms (orgs/yr) # NOQA + 'NGAppSum': 0.55, # Non-Grazing Manure Data Check: Land Applied (%) + 'NGBarnSum': 0.28, # Non-Grazing Manure Data Check: In Confined Areas (%) + 'NGTotSum': 0.83, # Non-Grazing Manure Data Check: Total (<= 1) + 'InitGrN': 2373, # Initial Grazing Animal Totals: Nitrogen (kg/yr) + 'InitGrP': 785, # Initial Grazing Animal Totals: Phosphorus (kg/yr) + 'InitGrFC': 3.38e+9, # Initial Grazing Animal Totals: Fecal Coliforms (orgs/yr) # NOQA + 'GRAppSum': 0.52, # Grazing Manure Data Check: Land Applied (%) + 'GRBarnSum': 0.18, # Grazing Manure Data Check: In Confined Areas (%) + 'GRTotSum': 1, # Grazing Manure Data Check: Total (<= 1) + 'AnimalFlag': 1, # Flag: Animal Layer Detected (0 No; 1 Yes) + 'WildOrgsDay': 5.0e+8, # Wildlife Loading Rate (org/animal/per day) + 'WildDensity': 25, # Wildlife Density (animals/square mile) + 'WuDieoff': 0.9, # Wildlife/Urban Die-Off Rate + 'UrbEMC': 9600, # Urban EMC (org/100ml) + 'SepticOrgsDay': 2.0e+9, # Septic Loading Rate (org/person per day) + 'SepticFailure': 0, # Malfunctioning System Rate (0 - 1) + 'WWTPConc': 200, # Wastewater Treatment Plants Loading Rate (cfu/100ml) + 'InstreamDieoff': 0.5, # In-Stream Die-Off Rate + 'AWMSGrPct': 0, # Animal Waste Management Systems: Livestock (%) + 'AWMSNgPct': 0, # Animal Waste Management Systems: Poultry (%) + 'RunContPct': 0, # Runoff Control (%) + 'PhytasePct': 0, # Phytase in Feed (%), + 'AnimalName': ['Dairy Cows', 'Beef Cows', 'Broilers', 'Layers', + 'Hogs/Swine', 'Sheep', 'Horses', 'Turkeys', 'Other'], + 'NumAnimals': np.zeros(9), + 'GrazingAnimal': ['Y', 'Y', 'N', 'N', + 'N', 'Y', 'Y', 'N', 'N'], + 'AvgAnimalWt': np.array([640, 360, 0.9, 1.8, + 61, 50, 500, 6.8, 0]), # Average Animal Weight (kg) # NOQA + 'AnimalDailyN': np.array([0.44, 0.31, 1.07, 0.85, + 0.48, 0.37, 0.28, 0.59, 0]), # Animal Daily Loads: Nitrogen (kg/AEU) # NOQA + 'AnimalDailyP': np.array([0.07, 0.09, 0.30, 0.29, + 0.15, 0.10, 0.28, 0.59, 0]), # Animal Daily Loads: Phosphorus (kg/AEU) # NOQA + 'FCOrgsPerDay': np.array([1.0e+11, 1.0e+11, 1.4e+8, 1.4e+8, + 1.1e+10, 1.2e+10, 4.2e+8, 9.5e+7, 0]), # Fecal Coliforms (orgs/day) # NOQA + 'Month': ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', + 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'], + 'NGPctManApp': np.array([0.01, 0.01, 0.15, 0.10, 0.05, 0.03, + 0.03, 0.03, 0.11, 0.10, 0.10, 0.08]), # Manure Spreading: % Of Annual Load Applied To Crops/Pasture # NOQA + 'NGAppNRate': np.ones(12) * 0.05, # Manure Spreading: Base Nitrogen Loss Rate # NOQA + 'NGAppPRate': np.ones(12) * 0.07, # Manure Spreading: Base Phosphorus Loss Rate # NOQA + 'NGAppFCRate': np.ones(12) * 0.12, # Manure Spreading: Base Fecal Coliform Loss Rate # NOQA + 'NGPctSoilIncRate': np.zeros(12), # Manure Spreading: % Of Manure Load Incorporated Into Soil # NOQA + 'NGBarnNRate': np.ones(12) * 0.2, # Barnyard/Confined Area: Base Nitrogen Loss Rate # NOQA + 'NGBarnPRate': np.ones(12) * 0.2, # Barnyard/Confined Area: Base Phosphorus Loss Rate # NOQA + 'NGBarnFCRate': np.ones(12) * 0.12, # Barnyard/Confined Area: Base Fecal Coliform Loss Rate # NOQA + 'PctGrazing': np.array([0.02, 0.02, 0.10, 0.25, 0.50, 0.50, + 0.50, 0.50, 0.50, 0.40, 0.25, 0.10]), # Grazing Land: % Of Time Spent Grazing # NOQA + 'PctStreams': np.ones(12) * 0.05, # Grazing Land: % Of Time Spent In Streams # NOQA + 'GrazingNRate': np.ones(12) * 0.05, # Grazing Land: Base Nitrogen Loss Rate # NOQA + 'GrazingPRate': np.ones(12) * 0.07, # Grazing Land: Base Phosphorus Loss Rate # NOQA + 'GrazingFCRate': np.ones(12) * 0.12, # Grazing Land: Base Fecal Coliform Loss Rate # NOQA + 'GRPctManApp': np.array([0.01, 0.01, 0.10, 0.05, 0.05, 0.03, + 0.03, 0.03, 0.11, 0.06, 0.02, 0.02]), # Manure Spreading: % Of Annual Load Applied To Crops/Pasture # NOQA + 'GRAppNRate': np.ones(12) * 0.05, # Manure Spreading: Base Nitrogen Loss Rate # NOQA + 'GRAppPRate': np.ones(12) * 0.07, # Manure Spreading: Base Phosphorus Loss Rate # NOQA + 'GRAppFCRate': np.ones(12) * 0.12, # Manure Spreading: Base Fecal Coliform Loss Rate # NOQA + 'GRPctSoilIncRate': np.zeros(12), # Manure Spreading: % Of Manure Load Incorporated Into Soil # NOQA + 'GRBarnNRate': np.ones(12) * 0.2, # Barnyard/Confined Area: Base Nitrogen Loss Rate # NOQA + 'GRBarnPRate': np.ones(12) * 0.2, # Barnyard/Confined Area: Base Phosphorus Loss Rate # NOQA + 'GRBarnFCRate': np.ones(12) * 0.12, # Barnyard/Confined Area: Base Fecal Coliform Loss Rate # NOQA + 'ShedAreaDrainLake': 0, # Percentage of watershed area that drains into a lake or wetlands: (0 - 1) # NOQA + 'RetentNLake': 0.12, # Lake Retention Rate: Nitrogen + 'RetentPLake': 0.29, # Lake Retention Rate: Phosphorus + 'RetentSedLake': 0.84, # Lake Retention Rate: Sediment + 'AttenFlowDist': 0, # Attenuation: Flow Distance (km) + 'AttenFlowVel': 4, # Attenuation: Flow Velocity (km/hr) + 'AttenLossRateN': 0.287, # Attenuation: Loss Rate: Nitrogen + 'AttenLossRateP': 0.226, # Attenuation: Loss Rate: Phosphorus + 'AttenLossRateTSS': 0, # Attenuation: Loss Rate: Total Suspended Solids + 'AttenLossRatePath': 0, # Attenuation: Loss Rate: Pathogens + 'StreamFlowVolAdj': 1, # Streamflow Volume Adjustment Factor +} From 27b8741369ce9c42dce04ac5e038af738463e7b0 Mon Sep 17 00:00:00 2001 From: Joe Tarricone Date: Mon, 11 Apr 2016 11:58:53 -0400 Subject: [PATCH 006/136] Process overlapping BMPs/land usage polygons --- src/mmw/apps/modeling/tasks.py | 11 ++++++++--- src/mmw/js/src/modeling/models.js | 16 +++++++++++++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index 472be85d3..13a971266 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -226,9 +226,14 @@ def run_tr55(censuses, model_input, cached_aoi_census=None): # Calculate total areas for each type modification area_sums = {} for piece in modification_pieces: - kind = piece['value'] + kinds = piece['value'] area = piece['area'] + if 'bmp' in kinds: + kind = kinds['bmp'] + else: + kind = kinds['reclass'] + if kind in area_sums: area_sums[kind] += area else: @@ -322,9 +327,9 @@ def change_key(modification): value = modification['value'] if name == 'landcover': - return {'change': ':%s:' % value} + return {'change': ':%s:' % value['reclass']} elif name == 'conservation_practice': - return {'change': '::%s' % value} + return {'change': '::%s' % value['bmp']} elif name == 'both': return {'change': ':%s:%s' % (value['reclass'], value['bmp'])} diff --git a/src/mmw/js/src/modeling/models.js b/src/mmw/js/src/modeling/models.js index 125897859..be132fa5f 100644 --- a/src/mmw/js/src/modeling/models.js +++ b/src/mmw/js/src/modeling/models.js @@ -389,6 +389,7 @@ function _alterModifications(rawModifications) { overlapPiece.value.reclass = newPiece.value; } + overlapPiece.area = turfArea(overlapPiece.shape); pieces.push(overlapPiece); /* remove the overlapping portion from both parents */ @@ -413,7 +414,20 @@ function _alterModifications(rawModifications) { pieces = _.filter(pieces, validShape); } - return pieces; + return _.map(pieces, function(piece) { //ensures 'value' is uniform type for each piece + var p = piece; + if ( typeof p.value === 'string') { + var v = {}; + if (p.name === reclass) { + v.reclass = p.value; + } + else if (p.name === bmp) { + v.bmp = p.value; + } + p.value = v; + } + return p; + }); } var alterModifications = _.memoize(_alterModifications, function(_, hash) {return hash;}); From 7d1fb2537dd1e6e9be4eed1ac4e3e0112d3f3306 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 8 Apr 2016 16:57:38 -0400 Subject: [PATCH 007/136] Create GWLF-E Model and return static values Since the gwlfe.run method currently prints a number of lines to stdout instead of returning a value, we simply initialize a gwlfe DataModel and return it as JSON. --- src/mmw/apps/modeling/tasks.py | 18 +++++-- src/mmw/mmw/settings/gwlfe_settings.py | 71 ++++++++++++-------------- 2 files changed, 47 insertions(+), 42 deletions(-) diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index 8fcc00f03..a8a47d1ac 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -15,6 +15,11 @@ from tr55.model import simulate_day +# from gwlfe import gwlfe +from gwlfe.datamodel import DataModel + +from django.conf import settings + logger = logging.getLogger(__name__) KG_PER_POUND = 0.453592 @@ -24,12 +29,19 @@ @shared_task def start_gwlfe_job(model_input): """ - Runs a GWLFE job. For now, just wait 3s and return the input. + Runs a GWLFE job. For now, just wait 3s and return the default values. """ # TODO remove sleep and call actual GWLFE code time.sleep(3) - response_json = { - } + + model = DataModel(settings.GWLFE_DEFAULTS) + + # TODO Run the actual model. + # Currently it writes to stdout and some files. + # Must have it return results in a data structure. + # gwlfe.run(model) + + response_json = model.tojson() if model else {} return response_json diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index 5182c8552..f3f8459a0 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -341,45 +341,38 @@ 'AWMSNgPct': 0, # Animal Waste Management Systems: Poultry (%) 'RunContPct': 0, # Runoff Control (%) 'PhytasePct': 0, # Phytase in Feed (%), - 'AnimalName': ['Dairy Cows', 'Beef Cows', 'Broilers', 'Layers', - 'Hogs/Swine', 'Sheep', 'Horses', 'Turkeys', 'Other'], - 'NumAnimals': np.zeros(9), - 'GrazingAnimal': ['Y', 'Y', 'N', 'N', - 'N', 'Y', 'Y', 'N', 'N'], - 'AvgAnimalWt': np.array([640, 360, 0.9, 1.8, - 61, 50, 500, 6.8, 0]), # Average Animal Weight (kg) # NOQA - 'AnimalDailyN': np.array([0.44, 0.31, 1.07, 0.85, - 0.48, 0.37, 0.28, 0.59, 0]), # Animal Daily Loads: Nitrogen (kg/AEU) # NOQA - 'AnimalDailyP': np.array([0.07, 0.09, 0.30, 0.29, - 0.15, 0.10, 0.28, 0.59, 0]), # Animal Daily Loads: Phosphorus (kg/AEU) # NOQA - 'FCOrgsPerDay': np.array([1.0e+11, 1.0e+11, 1.4e+8, 1.4e+8, - 1.1e+10, 1.2e+10, 4.2e+8, 9.5e+7, 0]), # Fecal Coliforms (orgs/day) # NOQA - 'Month': ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', - 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'], - 'NGPctManApp': np.array([0.01, 0.01, 0.15, 0.10, 0.05, 0.03, - 0.03, 0.03, 0.11, 0.10, 0.10, 0.08]), # Manure Spreading: % Of Annual Load Applied To Crops/Pasture # NOQA - 'NGAppNRate': np.ones(12) * 0.05, # Manure Spreading: Base Nitrogen Loss Rate # NOQA - 'NGAppPRate': np.ones(12) * 0.07, # Manure Spreading: Base Phosphorus Loss Rate # NOQA - 'NGAppFCRate': np.ones(12) * 0.12, # Manure Spreading: Base Fecal Coliform Loss Rate # NOQA - 'NGPctSoilIncRate': np.zeros(12), # Manure Spreading: % Of Manure Load Incorporated Into Soil # NOQA - 'NGBarnNRate': np.ones(12) * 0.2, # Barnyard/Confined Area: Base Nitrogen Loss Rate # NOQA - 'NGBarnPRate': np.ones(12) * 0.2, # Barnyard/Confined Area: Base Phosphorus Loss Rate # NOQA - 'NGBarnFCRate': np.ones(12) * 0.12, # Barnyard/Confined Area: Base Fecal Coliform Loss Rate # NOQA - 'PctGrazing': np.array([0.02, 0.02, 0.10, 0.25, 0.50, 0.50, - 0.50, 0.50, 0.50, 0.40, 0.25, 0.10]), # Grazing Land: % Of Time Spent Grazing # NOQA - 'PctStreams': np.ones(12) * 0.05, # Grazing Land: % Of Time Spent In Streams # NOQA - 'GrazingNRate': np.ones(12) * 0.05, # Grazing Land: Base Nitrogen Loss Rate # NOQA - 'GrazingPRate': np.ones(12) * 0.07, # Grazing Land: Base Phosphorus Loss Rate # NOQA - 'GrazingFCRate': np.ones(12) * 0.12, # Grazing Land: Base Fecal Coliform Loss Rate # NOQA - 'GRPctManApp': np.array([0.01, 0.01, 0.10, 0.05, 0.05, 0.03, - 0.03, 0.03, 0.11, 0.06, 0.02, 0.02]), # Manure Spreading: % Of Annual Load Applied To Crops/Pasture # NOQA - 'GRAppNRate': np.ones(12) * 0.05, # Manure Spreading: Base Nitrogen Loss Rate # NOQA - 'GRAppPRate': np.ones(12) * 0.07, # Manure Spreading: Base Phosphorus Loss Rate # NOQA - 'GRAppFCRate': np.ones(12) * 0.12, # Manure Spreading: Base Fecal Coliform Loss Rate # NOQA - 'GRPctSoilIncRate': np.zeros(12), # Manure Spreading: % Of Manure Load Incorporated Into Soil # NOQA - 'GRBarnNRate': np.ones(12) * 0.2, # Barnyard/Confined Area: Base Nitrogen Loss Rate # NOQA - 'GRBarnPRate': np.ones(12) * 0.2, # Barnyard/Confined Area: Base Phosphorus Loss Rate # NOQA - 'GRBarnFCRate': np.ones(12) * 0.12, # Barnyard/Confined Area: Base Fecal Coliform Loss Rate # NOQA + + 'AnimalName': ['Dairy Cows', 'Beef Cows', 'Broilers', 'Layers', 'Hogs/Swine', 'Sheep', 'Horses', 'Turkeys', 'Other'] , # NOQA + 'NumAnimals': np.array([ 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00]), # NOQA + 'GrazingAnimal': [ 'Y', 'Y', 'N', 'N', 'N', 'Y', 'Y', 'N', 'N'] , # NOQA + 'AvgAnimalWt': np.array([ 640.00, 360.00, 0.90, 1.80, 61.00, 50.00, 500.00, 6.80, 0.00]), # Average Animal Weight (kg) # NOQA + 'AnimalDailyN': np.array([ 0.44, 0.31, 1.07, 0.85, 0.48, 0.37, 0.28, 0.59, 0.00]), # Animal Daily Loads: Nitrogen (kg/AEU) # NOQA + 'AnimalDailyP': np.array([ 0.07, 0.09, 0.30, 0.29, 0.15, 0.10, 0.06, 0.20, 0.00]), # Animal Daily Loads: Phosphorus (kg/AEU) # NOQA + 'FCOrgsPerDay': np.array([ 1.0e+11, 1.0e+11, 1.4e+08, 1.4e+08, 1.1e+10, 1.2e+10, 4.2e+08, 9.5e+07, 0.00]), # Fecal Coliforms (orgs/day) # NOQA + + 'Month': ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'], # NOQA + 'NGPctManApp': np.array([ 0.01, 0.01, 0.15, 0.10, 0.05, 0.03, 0.03, 0.03, 0.11, 0.10, 0.10, 0.08]), # Manure Spreading: % Of Annual Load Applied To Crops/Pasture # NOQA + 'NGAppNRate': np.array([ 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05]), # Manure Spreading: Base Nitrogen Loss Rate # NOQA + 'NGAppPRate': np.array([ 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07]), # Manure Spreading: Base Phosphorus Loss Rate # NOQA + 'NGAppFCRate': np.array([ 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]), # Manure Spreading: Base Fecal Coliform Loss Rate # NOQA + 'NGPctSoilIncRate': np.array([ 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00]), # Manure Spreading: % Of Manure Load Incorporated Into Soil # NOQA + 'NGBarnNRate': np.array([ 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20]), # Barnyard/Confined Area: Base Nitrogen Loss Rate # NOQA + 'NGBarnPRate': np.array([ 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20]), # Barnyard/Confined Area: Base Phosphorus Loss Rate # NOQA + 'NGBarnFCRate': np.array([ 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]), # Barnyard/Confined Area: Base Fecal Coliform Loss Rate # NOQA + 'PctGrazing': np.array([ 0.02, 0.02, 0.10, 0.25, 0.50, 0.50, 0.50, 0.50, 0.50, 0.40, 0.25, 0.10]), # Grazing Land: % Of Time Spent Grazing # NOQA + 'PctStreams': np.array([ 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05]), # Grazing Land: % Of Time Spent In Streams # NOQA + 'GrazingNRate': np.array([ 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05]), # Grazing Land: Base Nitrogen Loss Rate # NOQA + 'GrazingPRate': np.array([ 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07]), # Grazing Land: Base Phosphorus Loss Rate # NOQA + 'GrazingFCRate': np.array([ 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]), # Grazing Land: Base Fecal Coliform Loss Rate # NOQA + 'GRPctManApp': np.array([ 0.01, 0.01, 0.10, 0.05, 0.05, 0.03, 0.03, 0.03, 0.11, 0.06, 0.02, 0.02]), # Manure Spreading: % Of Annual Load Applied To Crops/Pasture # NOQA + 'GRAppNRate': np.array([ 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05]), # Manure Spreading: Base Nitrogen Loss Rate # NOQA + 'GRAppPRate': np.array([ 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07]), # Manure Spreading: Base Phosphorus Loss Rate # NOQA + 'GRAppFCRate': np.array([ 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]), # Manure Spreading: Base Fecal Coliform Loss Rate # NOQA + 'GRPctSoilIncRate': np.array([ 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00]), # Manure Spreading: % Of Manure Load Incorporated Into Soil # NOQA + 'GRBarnNRate': np.array([ 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20]), # Barnyard/Confined Area: Base Nitrogen Loss Rate # NOQA + 'GRBarnPRate': np.array([ 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20]), # Barnyard/Confined Area: Base Phosphorus Loss Rate # NOQA + 'GRBarnFCRate': np.array([ 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]), # Barnyard/Confined Area: Base Fecal Coliform Loss Rate # NOQA + 'ShedAreaDrainLake': 0, # Percentage of watershed area that drains into a lake or wetlands: (0 - 1) # NOQA 'RetentNLake': 0.12, # Lake Retention Rate: Nitrogen 'RetentPLake': 0.29, # Lake Retention Rate: Phosphorus From ba3dedddccae2f4b36f0c1c4b3c2add8ce277bf2 Mon Sep 17 00:00:00 2001 From: Hector Castro Date: Tue, 12 Apr 2016 13:52:34 -0400 Subject: [PATCH 008/136] Bump Beaver version to avoid Mosquito dependency Something happened to a transitive dependency of Beaver, which caused installs to fail. This bumps to a version of Beaver that removes the broken dependency (Mosquito). See also: https://github.com/python-beaver/python-beaver/pull/399 --- deployment/ansible/group_vars/all | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deployment/ansible/group_vars/all b/deployment/ansible/group_vars/all index 0a725dd2e..731d643ba 100644 --- a/deployment/ansible/group_vars/all +++ b/deployment/ansible/group_vars/all @@ -40,6 +40,8 @@ docker_version: "1.9.*" docker_py_version: "1.2.3" docker_options: "--storage-driver=aufs" +beaver_version: "36.2.0" + sjs_host: "localhost" sjs_port: 8090 sjs_container_image: "quay.io/azavea/spark-jobserver:0.6.1" From 48d16e5faed68af59cd3d95ab2be1da53299ae69 Mon Sep 17 00:00:00 2001 From: Hector Castro Date: Tue, 12 Apr 2016 19:37:10 -0400 Subject: [PATCH 009/136] Override Beaver version in project wrapper role The top level group variable for beaver_version was being overwritten by the wrapper role's variable. Changing it here actually allows it to take effect on an Ansible run. --- deployment/ansible/group_vars/all | 2 -- .../ansible/roles/model-my-watershed.beaver/vars/main.yml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/deployment/ansible/group_vars/all b/deployment/ansible/group_vars/all index 731d643ba..0a725dd2e 100644 --- a/deployment/ansible/group_vars/all +++ b/deployment/ansible/group_vars/all @@ -40,8 +40,6 @@ docker_version: "1.9.*" docker_py_version: "1.2.3" docker_options: "--storage-driver=aufs" -beaver_version: "36.2.0" - sjs_host: "localhost" sjs_port: 8090 sjs_container_image: "quay.io/azavea/spark-jobserver:0.6.1" diff --git a/deployment/ansible/roles/model-my-watershed.beaver/vars/main.yml b/deployment/ansible/roles/model-my-watershed.beaver/vars/main.yml index 7f49eb2dc..7647057ef 100644 --- a/deployment/ansible/roles/model-my-watershed.beaver/vars/main.yml +++ b/deployment/ansible/roles/model-my-watershed.beaver/vars/main.yml @@ -1,3 +1,3 @@ --- -beaver_version: "33.3.0" +beaver_version: "36.2.0" beaver_redis_url: "redis://{{ redis_host }}:{{ redis_port }}/0" From 1687984129e297c3119fa432aca7e8a39d8d3eb7 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 21 Apr 2016 12:34:14 -0400 Subject: [PATCH 010/136] Prepare for populating model * Parse incoming JSON to dictionary * Transform AOI geojson to a geom object, for easy WKTification * Instantiate a model as `z` using defaults. We will add calculated values to this model object as we go along * Remove dummy wait time --- src/mmw/apps/modeling/tasks.py | 17 ++++++++++------- src/mmw/apps/modeling/views.py | 2 +- src/mmw/mmw/settings/base.py | 2 +- src/mmw/mmw/settings/gwlfe_settings.py | 3 +++ 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index 61cf819d2..c097daaa4 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals from __future__ import absolute_import -import time import requests from celery import shared_task import json @@ -19,29 +18,33 @@ from gwlfe.datamodel import DataModel from django.conf import settings +from django.contrib.gis.geos import GEOSGeometry logger = logging.getLogger(__name__) KG_PER_POUND = 0.453592 CM_PER_INCH = 2.54 +ACRES_PER_SQM = 0.000247105 @shared_task def start_gwlfe_job(model_input): """ - Runs a GWLFE job. For now, just wait 3s and return the default values. + Runs a GWLFE job. """ - # TODO remove sleep and call actual GWLFE code - time.sleep(3) + aoi_geom = GEOSGeometry(json.dumps(model_input['area_of_interest']), + srid=4326) + aoi_area = aoi_geom.transform(5070, clone=True).area # Square Meters - model = DataModel(settings.GWLFE_DEFAULTS) + # Data Model is called z by convention + z = DataModel(settings.GWLFE_DEFAULTS) # TODO Run the actual model. # Currently it writes to stdout and some files. # Must have it return results in a data structure. - # gwlfe.run(model) + # gwlfe.run(z) - response_json = model.tojson() if model else {} + response_json = z.tojson() if z else {} return response_json diff --git a/src/mmw/apps/modeling/views.py b/src/mmw/apps/modeling/views.py index 0218c8f9d..1370da2a3 100644 --- a/src/mmw/apps/modeling/views.py +++ b/src/mmw/apps/modeling/views.py @@ -195,7 +195,7 @@ def start_gwlfe(request, format=None): """ user = request.user if request.user.is_authenticated() else None created = now() - model_input = request.POST['model_input'] + model_input = json.loads(request.POST['model_input']) job = Job.objects.create(created_at=created, result='', error='', traceback='', user=user, status='started') diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index b93fa7f32..5b3beb8c8 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -13,7 +13,7 @@ from sys import path from layer_settings import LAYERS, VIZER_URLS # NOQA -from gwlfe_settings import GWLFE_DEFAULTS # NOQA +from gwlfe_settings import GWLFE_DEFAULTS, GWLFE_CONFIG # NOQA # Normally you should not import ANYTHING from Django directly # into your settings, but ImproperlyConfigured is an exception. diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index f3f8459a0..be59913b4 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -385,3 +385,6 @@ 'AttenLossRatePath': 0, # Attenuation: Loss Rate: Pathogens 'StreamFlowVolAdj': 1, # Streamflow Volume Adjustment Factor } + +GWLFE_CONFIG = { +} From 72f145e81eaf53771688005d12e665891d0181e9 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 21 Apr 2016 12:38:58 -0400 Subject: [PATCH 011/136] Switch to enumerations * Prefer enumerations defined in GWLF-E to string values * Remove # NOQA indicators from the settings file, since it is ignored anyway by flake8 --- src/mmw/mmw/settings/gwlfe_settings.py | 137 +++++++++++++------------ 1 file changed, 71 insertions(+), 66 deletions(-) diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index be59913b4..6ed9d25de 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -1,5 +1,10 @@ import numpy as np +from gwlfe.enums import (LandUse as lu, + ETflag as et, + YesOrNo as b, + SweepType) + GWLFE_DEFAULTS = { 'NRur': 10, # Number of Rural Land Use Categories 'NUrb': 6, # Number of Urban Land Use Categories @@ -11,17 +16,17 @@ 'InitSnow': 0, # Initial Snow Days 'TileDrainRatio': 0, # Tile Drain Ratio 'TileDrainDensity': 0, # Tile Drain Density - 'ETFlag': 0, # ET Flag: 0 for Hammon method, 1 for Blainy-Criddle method - 'AntMoist': np.zeros(5), # Antecedent Rain + Melt Moisture Conditions for Days 1 to 5 # NOQA + 'ETFlag': et.HAMON_METHOD, # ET Flag: 0 for Hamon method, 1 for Blainy-Criddle method + 'AntMoist': np.zeros(5), # Antecedent Rain + Melt Moisture Conditions for Days 1 to 5 'StreamWithdrawal': np.zeros(12), # Surface Water Withdrawal/Extraction 'GroundWithdrawal': np.zeros(12), # Groundwater Withdrawal/Extraction 'PcntET': np.ones(12), # Percent monthly adjustment for ET calculation - 'PhysFlag': 0, # Flag: Physiographic Province Layer Detected (0 No; 1 Yes) - 'PointFlag': 1, # Flag: Point Source Layer Detected (0 No; 1 Yes) - 'SeptSysFlag': 0, # Flag: Septic System Layer Detected (0 No; 1 Yes) - 'CountyFlag': 0, # Flag: County Layer Detected (0 No; 1 Yes) - 'SoilPFlag': 1, # Flag: Soil P Layer Detected (0 No; 1 Yes) - 'GWNFlag': 1, # Flag: Groundwater N Layer Detected (0 No; 1 Yes) + 'PhysFlag': b.NO, # Flag: Physiographic Province Layer Detected (0 No; 1 Yes) + 'PointFlag': b.YES, # Flag: Point Source Layer Detected (0 No; 1 Yes) + 'SeptSysFlag': b.NO, # Flag: Septic System Layer Detected (0 No; 1 Yes) + 'CountyFlag': b.NO, # Flag: County Layer Detected (0 No; 1 Yes) + 'SoilPFlag': b.YES, # Flag: Soil P Layer Detected (0 No; 1 Yes) + 'GWNFlag': b.YES, # Flag: Groundwater N Layer Detected (0 No; 1 Yes) 'SedAAdjust': 1, # Default Percent ET 'SedNitr': 2000, # Soil Concentration: N (mg/l) 'BankNFrac': 0.25, # % Bank N Fraction (0 - 1) @@ -34,7 +39,7 @@ 'Nqual': 3, # Number of Contaminants (Default = 3) 'Contaminant': ['Nitrogen', 'Phosphorus', 'Sediment'], 'UrbBMPRed': np.zeros((16, 3)), # Urban BMP Reduction - 'SepticFlag': 1, # Flag: Septic Systems Layer Detected (0 No; 1 Yes) + 'SepticFlag': b.YES, # Flag: Septic Systems Layer Detected (0 No; 1 Yes) 'NumPondSys': np.zeros(12), # Number of People on Pond Systems 'NumShortSys': np.zeros(12), # Number of People on Short Circuit Systems 'NumDischargeSys': np.zeros(12), # Number of People on Discharge Systems @@ -130,20 +135,20 @@ 'n41h': 0, # Phytase in Feed: Existing (%) 'n41i': 0, # Phytase in Feed: Future (%) 'n43': 0, # Stream Km with Vegetated Buffer Strips: Existing - 'GRLBN': 0, # Average Grazing Animal Loss Rate (Barnyard/Confined Area): Nitrogen # NOQA - 'NGLBN': 0, # Average Non-Grazing Animal Loss Rate (Barnyard/Confined Area): Nitrogen # NOQA - 'GRLBP': 0, # Average Grazing Animal Loss Rate (Barnyard/Confined Area): Phosphorus # NOQA - 'NGLBP': 0, # Average Non-Grazing Animal Loss Rate (Barnyard/Confined Area): Phosphorus # NOQA - 'NGLManP': 0, # Average Non-Grazing Animal Loss Rate (Manure Spreading): Phosphorus # NOQA - 'NGLBFC': 0, # Average Non-Grazing Animal Loss Rate (Barnyard/Confined Area): Fecal Coliform # NOQA - 'GRLBFC': 0, # Average Grazing Animal Loss Rate (Barnyard/Confined Area): Fecal Coliform # NOQA - 'GRSFC': 0, # Average Grazing Animal Loss Rate (Spent in Streams): Fecal Coliform # NOQA + 'GRLBN': 0, # Average Grazing Animal Loss Rate (Barnyard/Confined Area): Nitrogen + 'NGLBN': 0, # Average Non-Grazing Animal Loss Rate (Barnyard/Confined Area): Nitrogen + 'GRLBP': 0, # Average Grazing Animal Loss Rate (Barnyard/Confined Area): Phosphorus + 'NGLBP': 0, # Average Non-Grazing Animal Loss Rate (Barnyard/Confined Area): Phosphorus + 'NGLManP': 0, # Average Non-Grazing Animal Loss Rate (Manure Spreading): Phosphorus + 'NGLBFC': 0, # Average Non-Grazing Animal Loss Rate (Barnyard/Confined Area): Fecal Coliform + 'GRLBFC': 0, # Average Grazing Animal Loss Rate (Barnyard/Confined Area): Fecal Coliform + 'GRSFC': 0, # Average Grazing Animal Loss Rate (Spent in Streams): Fecal Coliform 'GRSN': 0, # Average Grazing Animal Loss Rate (Spent in Streams): Nitrogen - 'GRSP': 0, # Average Grazing Animal Loss Rate (Spent in Streams): Phosphorus # NOQA + 'GRSP': 0, # Average Grazing Animal Loss Rate (Spent in Streams): Phosphorus 'n43b': 0, # High Density Urban (Constructed Wetlands): Required Ha 'n43c': 0, # High Density Urban (Detention Basin): % Drainage Used 'n43d': 0, # High Density Urban: % Impervious Surface - 'n43e': 0, # High Density Urban (Constructed Wetlands): Impervious Ha Drained # NOQA + 'n43e': 0, # High Density Urban (Constructed Wetlands): Impervious Ha Drained 'n43f': 0, # High Density Urban (Detention Basin): Impervious Ha Drained 'n43g': 0, # High Density Urban (Bioretention Areas): Impervious Ha Drained 'n43h': 0, # High Density Urban (Bioretention Areas): Required Ha @@ -155,7 +160,7 @@ 'n45b': 0, # Low Density Urban (Constructed Wetlands): Required Ha 'n45c': 0, # Low Density Urban (Detention Basin): % Drainage Used 'n45d': 0, # Low Density Urban: % Impervious Surface - 'n45e': 0, # Low Density Urban (Constructed Wetlands): Impervious Ha Drained # NOQA + 'n45e': 0, # Low Density Urban (Constructed Wetlands): Impervious Ha Drained 'n45f': 0, # Low Density Urban (Detention Basin): Impervious Ha Drained 'n46': 0, # Stream Km with Fencing: Future 'n46b': 0, # Low Density Urban (Detention Basin): Required Ha @@ -178,12 +183,12 @@ 'n51': 0, # Septic Systems Converted by Secondary Treatment Type (%) 'n52': 0, # Septic Systems Converted by Tertiary Treatment Type (%) 'n53': 0, # No longer used (Default = 0) - 'n54': 0, # Distribution of Pollutant Discharges by Primary Treatment Type (%): Existing # NOQA - 'n55': 0, # Distribution of Pollutant Discharges by Secondary Treatment Type (%): Existing # NOQA - 'n56': 0, # Distribution of Pollutant Discharges by Tertiary Treatment Type (%): Existing # NOQA - 'n57': 0, # Distribution of Pollutant Discharges by Primary Treatment Type (%): Future # NOQA - 'n58': 0, # Distribution of Pollutant Discharges by Secondary Treatment Type (%): Future # NOQA - 'n59': 0, # Distribution of Pollutant Discharges by Tertiary Treatment Type (%): Future # NOQA + 'n54': 0, # Distribution of Pollutant Discharges by Primary Treatment Type (%): Existing + 'n55': 0, # Distribution of Pollutant Discharges by Secondary Treatment Type (%): Existing + 'n56': 0, # Distribution of Pollutant Discharges by Tertiary Treatment Type (%): Existing + 'n57': 0, # Distribution of Pollutant Discharges by Primary Treatment Type (%): Future + 'n58': 0, # Distribution of Pollutant Discharges by Secondary Treatment Type (%): Future + 'n59': 0, # Distribution of Pollutant Discharges by Tertiary Treatment Type (%): Future 'n60': 0, # Distribution of Treatment Upgrades (%): Primary to Secondary 'n61': 0, # Distribution of Treatment Upgrades (%): Primary to Tertiary 'n62': 0, # Distribution of Treatment Upgrades (%): Secondary to Tertiary @@ -247,18 +252,18 @@ 'n85u': 0.82, # Bioretention Areas (Pathogens) 'n85v': 0.71, # Detention Basins (Pathogens) 'Qretention': 0, # Detention Basin: Amount of runoff retention (cm) - 'FilterWidth': 0, # Stream Protection: Vegetative buffer strip width (meters) # NOQA + 'FilterWidth': 0, # Stream Protection: Vegetative buffer strip width (meters) 'Capacity': 0, # Detention Basin: Detention basin volume (cubic meters) 'BasinDeadStorage': 0, # Detention Basin: Basin dead storage (cubic meters) 'BasinArea': 0, # Detention Basin: Basin surface area (square meters) 'DaysToDrain': 0, # Detention Basin: Basin days to drain 'CleanMon': 0, # Detention Basin: Basin cleaning month - 'PctAreaInfil': 0, # Infiltration/Bioretention: Fraction of area treated (0-1) # NOQA + 'PctAreaInfil': 0, # Infiltration/Bioretention: Fraction of area treated (0-1) 'PctStrmBuf': 0, # Stream Protection: Fraction of streams treated (0-1) 'UrbBankStab': 0, # Stream Protection: Streams w/bank stabilization (km) - 'ISRR': np.zeros(6), # Impervious Surface Reduction (% Reduction) of Urban Land Uses # NOQA - 'ISRA': np.zeros(6), # Impervious Surface Reduction (Area) of Urban Land Uses # NOQA - 'SweepType': 1, # Street Sweeping: Sweep Type (1-2) + 'ISRR': np.zeros(6), # Impervious Surface Reduction (% Reduction) of Urban Land Uses + 'ISRA': np.zeros(6), # Impervious Surface Reduction (Area) of Urban Land Uses + 'SweepType': SweepType.MECHANICAL, # Street Sweeping: Sweep Type (1-2) 'UrbSweepFrac': 1, # Street Sweeping: Fraction of area treated (0-1) 'StreetSweepNo': np.zeros(12), # Street sweeping times per month 'n108': 0, # Row Crops: Sediment (kg x 1000) @@ -304,13 +309,13 @@ 'n137': 0, # Estimated Scenario Cost $: Unpaved Road Protection 'n138': 0, # Estimated Scenario Cost $: Animal BMPs 'n139': 0, # Pathogen Loads (Farm Animals): Existing (orgs/month) - 'n140': 0, # Pathogen Loads (Wastewater Treatment Plants): Existing (orgs/month) # NOQA + 'n140': 0, # Pathogen Loads (Wastewater Treatment Plants): Existing (orgs/month) 'n141': 0, # Pathogen Loads (Septic Systems): Existing (orgs/month) 'n142': 0, # Pathogen Loads (Urban Areas): Existing (orgs/month) 'n143': 0, # Pathogen Loads (Wildlife): Existing (orgs/month) 'n144': 0, # Pathogen Loads (Total): Existing (orgs/month) 'n145': 0, # Pathogen Loads (Farm Animals): Future (orgs/month) - 'n146': 0, # Pathogen Loads (Wastewater Treatment Plants): Future (orgs/month) # NOQA + 'n146': 0, # Pathogen Loads (Wastewater Treatment Plants): Future (orgs/month) 'n147': 0, # Pathogen Loads (Septic Systems): Future (orgs/month) 'n148': 0, # Pathogen Loads (Urban Areas): Future (orgs/month) 'n149': 0, # Pathogen Loads (Wildlife): Future (orgs/month) @@ -318,17 +323,17 @@ 'n151': 0, # Pathogen Loads: Percent Reduction (%) 'InitNgN': 2373, # Initial Non-Grazing Animal Totals: Nitrogen (kg/yr) 'InitNgP': 785, # Initial Non-Grazing Animal Totals: Phosphorus (kg/yr) - 'InitNgFC': 3.38e+9, # Initial Non-Grazing Animal Totals: Fecal Coliforms (orgs/yr) # NOQA + 'InitNgFC': 3.38e+9, # Initial Non-Grazing Animal Totals: Fecal Coliforms (orgs/yr) 'NGAppSum': 0.55, # Non-Grazing Manure Data Check: Land Applied (%) 'NGBarnSum': 0.28, # Non-Grazing Manure Data Check: In Confined Areas (%) 'NGTotSum': 0.83, # Non-Grazing Manure Data Check: Total (<= 1) 'InitGrN': 2373, # Initial Grazing Animal Totals: Nitrogen (kg/yr) 'InitGrP': 785, # Initial Grazing Animal Totals: Phosphorus (kg/yr) - 'InitGrFC': 3.38e+9, # Initial Grazing Animal Totals: Fecal Coliforms (orgs/yr) # NOQA + 'InitGrFC': 3.38e+9, # Initial Grazing Animal Totals: Fecal Coliforms (orgs/yr) 'GRAppSum': 0.52, # Grazing Manure Data Check: Land Applied (%) 'GRBarnSum': 0.18, # Grazing Manure Data Check: In Confined Areas (%) 'GRTotSum': 1, # Grazing Manure Data Check: Total (<= 1) - 'AnimalFlag': 1, # Flag: Animal Layer Detected (0 No; 1 Yes) + 'AnimalFlag': b.YES, # Flag: Animal Layer Detected (0 No; 1 Yes) 'WildOrgsDay': 5.0e+8, # Wildlife Loading Rate (org/animal/per day) 'WildDensity': 25, # Wildlife Density (animals/square mile) 'WuDieoff': 0.9, # Wildlife/Urban Die-Off Rate @@ -342,38 +347,38 @@ 'RunContPct': 0, # Runoff Control (%) 'PhytasePct': 0, # Phytase in Feed (%), - 'AnimalName': ['Dairy Cows', 'Beef Cows', 'Broilers', 'Layers', 'Hogs/Swine', 'Sheep', 'Horses', 'Turkeys', 'Other'] , # NOQA - 'NumAnimals': np.array([ 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00]), # NOQA - 'GrazingAnimal': [ 'Y', 'Y', 'N', 'N', 'N', 'Y', 'Y', 'N', 'N'] , # NOQA - 'AvgAnimalWt': np.array([ 640.00, 360.00, 0.90, 1.80, 61.00, 50.00, 500.00, 6.80, 0.00]), # Average Animal Weight (kg) # NOQA - 'AnimalDailyN': np.array([ 0.44, 0.31, 1.07, 0.85, 0.48, 0.37, 0.28, 0.59, 0.00]), # Animal Daily Loads: Nitrogen (kg/AEU) # NOQA - 'AnimalDailyP': np.array([ 0.07, 0.09, 0.30, 0.29, 0.15, 0.10, 0.06, 0.20, 0.00]), # Animal Daily Loads: Phosphorus (kg/AEU) # NOQA - 'FCOrgsPerDay': np.array([ 1.0e+11, 1.0e+11, 1.4e+08, 1.4e+08, 1.1e+10, 1.2e+10, 4.2e+08, 9.5e+07, 0.00]), # Fecal Coliforms (orgs/day) # NOQA + 'AnimalName': ['Dairy Cows', 'Beef Cows', 'Broilers', 'Layers', 'Hogs/Swine', 'Sheep', 'Horses', 'Turkeys', 'Other'] , + 'NumAnimals': np.array([ 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00]), + 'GrazingAnimal': [ b.YES, b.YES, b.NO, b.NO, b.NO, b.YES, b.YES, b.NO, b.NO] , + 'AvgAnimalWt': np.array([ 640.00, 360.00, 0.90, 1.80, 61.00, 50.00, 500.00, 6.80, 0.00]), # Average Animal Weight (kg) + 'AnimalDailyN': np.array([ 0.44, 0.31, 1.07, 0.85, 0.48, 0.37, 0.28, 0.59, 0.00]), # Animal Daily Loads: Nitrogen (kg/AEU) + 'AnimalDailyP': np.array([ 0.07, 0.09, 0.30, 0.29, 0.15, 0.10, 0.06, 0.20, 0.00]), # Animal Daily Loads: Phosphorus (kg/AEU) + 'FCOrgsPerDay': np.array([ 1.0e+11, 1.0e+11, 1.4e+08, 1.4e+08, 1.1e+10, 1.2e+10, 4.2e+08, 9.5e+07, 0.00]), # Fecal Coliforms (orgs/day) - 'Month': ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'], # NOQA - 'NGPctManApp': np.array([ 0.01, 0.01, 0.15, 0.10, 0.05, 0.03, 0.03, 0.03, 0.11, 0.10, 0.10, 0.08]), # Manure Spreading: % Of Annual Load Applied To Crops/Pasture # NOQA - 'NGAppNRate': np.array([ 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05]), # Manure Spreading: Base Nitrogen Loss Rate # NOQA - 'NGAppPRate': np.array([ 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07]), # Manure Spreading: Base Phosphorus Loss Rate # NOQA - 'NGAppFCRate': np.array([ 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]), # Manure Spreading: Base Fecal Coliform Loss Rate # NOQA - 'NGPctSoilIncRate': np.array([ 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00]), # Manure Spreading: % Of Manure Load Incorporated Into Soil # NOQA - 'NGBarnNRate': np.array([ 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20]), # Barnyard/Confined Area: Base Nitrogen Loss Rate # NOQA - 'NGBarnPRate': np.array([ 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20]), # Barnyard/Confined Area: Base Phosphorus Loss Rate # NOQA - 'NGBarnFCRate': np.array([ 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]), # Barnyard/Confined Area: Base Fecal Coliform Loss Rate # NOQA - 'PctGrazing': np.array([ 0.02, 0.02, 0.10, 0.25, 0.50, 0.50, 0.50, 0.50, 0.50, 0.40, 0.25, 0.10]), # Grazing Land: % Of Time Spent Grazing # NOQA - 'PctStreams': np.array([ 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05]), # Grazing Land: % Of Time Spent In Streams # NOQA - 'GrazingNRate': np.array([ 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05]), # Grazing Land: Base Nitrogen Loss Rate # NOQA - 'GrazingPRate': np.array([ 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07]), # Grazing Land: Base Phosphorus Loss Rate # NOQA - 'GrazingFCRate': np.array([ 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]), # Grazing Land: Base Fecal Coliform Loss Rate # NOQA - 'GRPctManApp': np.array([ 0.01, 0.01, 0.10, 0.05, 0.05, 0.03, 0.03, 0.03, 0.11, 0.06, 0.02, 0.02]), # Manure Spreading: % Of Annual Load Applied To Crops/Pasture # NOQA - 'GRAppNRate': np.array([ 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05]), # Manure Spreading: Base Nitrogen Loss Rate # NOQA - 'GRAppPRate': np.array([ 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07]), # Manure Spreading: Base Phosphorus Loss Rate # NOQA - 'GRAppFCRate': np.array([ 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]), # Manure Spreading: Base Fecal Coliform Loss Rate # NOQA - 'GRPctSoilIncRate': np.array([ 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00]), # Manure Spreading: % Of Manure Load Incorporated Into Soil # NOQA - 'GRBarnNRate': np.array([ 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20]), # Barnyard/Confined Area: Base Nitrogen Loss Rate # NOQA - 'GRBarnPRate': np.array([ 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20]), # Barnyard/Confined Area: Base Phosphorus Loss Rate # NOQA - 'GRBarnFCRate': np.array([ 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]), # Barnyard/Confined Area: Base Fecal Coliform Loss Rate # NOQA + 'Month': ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'], + 'NGPctManApp': np.array([ 0.01, 0.01, 0.15, 0.10, 0.05, 0.03, 0.03, 0.03, 0.11, 0.10, 0.10, 0.08]), # Manure Spreading: % Of Annual Load Applied To Crops/Pasture + 'NGAppNRate': np.array([ 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05]), # Manure Spreading: Base Nitrogen Loss Rate + 'NGAppPRate': np.array([ 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07]), # Manure Spreading: Base Phosphorus Loss Rate + 'NGAppFCRate': np.array([ 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]), # Manure Spreading: Base Fecal Coliform Loss Rate + 'NGPctSoilIncRate': np.array([ 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00]), # Manure Spreading: % Of Manure Load Incorporated Into Soil + 'NGBarnNRate': np.array([ 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20]), # Barnyard/Confined Area: Base Nitrogen Loss Rate + 'NGBarnPRate': np.array([ 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20]), # Barnyard/Confined Area: Base Phosphorus Loss Rate + 'NGBarnFCRate': np.array([ 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]), # Barnyard/Confined Area: Base Fecal Coliform Loss Rate + 'PctGrazing': np.array([ 0.02, 0.02, 0.10, 0.25, 0.50, 0.50, 0.50, 0.50, 0.50, 0.40, 0.25, 0.10]), # Grazing Land: % Of Time Spent Grazing + 'PctStreams': np.array([ 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05]), # Grazing Land: % Of Time Spent In Streams + 'GrazingNRate': np.array([ 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05]), # Grazing Land: Base Nitrogen Loss Rate + 'GrazingPRate': np.array([ 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07]), # Grazing Land: Base Phosphorus Loss Rate + 'GrazingFCRate': np.array([ 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]), # Grazing Land: Base Fecal Coliform Loss Rate + 'GRPctManApp': np.array([ 0.01, 0.01, 0.10, 0.05, 0.05, 0.03, 0.03, 0.03, 0.11, 0.06, 0.02, 0.02]), # Manure Spreading: % Of Annual Load Applied To Crops/Pasture + 'GRAppNRate': np.array([ 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05]), # Manure Spreading: Base Nitrogen Loss Rate + 'GRAppPRate': np.array([ 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07, 0.07]), # Manure Spreading: Base Phosphorus Loss Rate + 'GRAppFCRate': np.array([ 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]), # Manure Spreading: Base Fecal Coliform Loss Rate + 'GRPctSoilIncRate': np.array([ 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00]), # Manure Spreading: % Of Manure Load Incorporated Into Soil + 'GRBarnNRate': np.array([ 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20]), # Barnyard/Confined Area: Base Nitrogen Loss Rate + 'GRBarnPRate': np.array([ 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20, 0.20]), # Barnyard/Confined Area: Base Phosphorus Loss Rate + 'GRBarnFCRate': np.array([ 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]), # Barnyard/Confined Area: Base Fecal Coliform Loss Rate - 'ShedAreaDrainLake': 0, # Percentage of watershed area that drains into a lake or wetlands: (0 - 1) # NOQA + 'ShedAreaDrainLake': 0, # Percentage of watershed area that drains into a lake or wetlands: (0 - 1) 'RetentNLake': 0.12, # Lake Retention Rate: Nitrogen 'RetentPLake': 0.29, # Lake Retention Rate: Phosphorus 'RetentSedLake': 0.84, # Lake Retention Rate: Sediment From 73e8d3daac95a318c617ca1d44b56676376188a6 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 21 Apr 2016 12:47:48 -0400 Subject: [PATCH 012/136] Add static values * Add more static values to the defaults * Add day lengths, which are calculated from a given formula --- src/mmw/apps/modeling/mapshed.py | 26 +++++++++++++ src/mmw/apps/modeling/tasks.py | 6 +++ src/mmw/mmw/settings/gwlfe_settings.py | 52 ++++++++++++++++++++++++++ 3 files changed, 84 insertions(+) create mode 100644 src/mmw/apps/modeling/mapshed.py diff --git a/src/mmw/apps/modeling/mapshed.py b/src/mmw/apps/modeling/mapshed.py new file mode 100644 index 000000000..c7aa8f7ed --- /dev/null +++ b/src/mmw/apps/modeling/mapshed.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import absolute_import + +import math +import numpy as np + + +def day_lengths(geom): + """ + Given a geometry in EPSG:4326, returns an array of 12 floats, each + representing the average number of daylight hours at that geometry's + centroid for each month. + """ + latitude = geom.centroid[1] + lengths = np.zeros(12) + + for month in range(12): + # Magic formula taken from original MapShed source + lengths[month] = 7.63942 * math.acos(0.43481 * + math.tan(latitude * 0.017453) * + math.cos(0.0172 * + (month * 30.4375 - 5))) + + return lengths diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index c097daaa4..f22cc6838 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -17,6 +17,9 @@ # from gwlfe import gwlfe from gwlfe.datamodel import DataModel +from apps.modeling.mapshed import (day_lengths, + ) + from django.conf import settings from django.contrib.gis.geos import GEOSGeometry @@ -39,6 +42,9 @@ def start_gwlfe_job(model_input): # Data Model is called z by convention z = DataModel(settings.GWLFE_DEFAULTS) + # Statically calculated lookup values + z.DayHrs = day_lengths(aoi_geom) + # TODO Run the actual model. # Currently it writes to stdout and some files. # Must have it return results in a data structure. diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index 6ed9d25de..7d6b605b2 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -21,6 +21,31 @@ 'StreamWithdrawal': np.zeros(12), # Surface Water Withdrawal/Extraction 'GroundWithdrawal': np.zeros(12), # Groundwater Withdrawal/Extraction 'PcntET': np.ones(12), # Percent monthly adjustment for ET calculation + 'Landuse': [lu.HAY_PAST, # Maps NLCD 81 + lu.CROPLAND, # Maps NLCD 82 + lu.FOREST, # Maps NLCD 41, 42, 43, 52 + lu.WETLAND, # Maps NLCD 90, 95 + lu.DISTURBED, # Does not map to NLCD + lu.TURFGRASS, # Does not map to NLCD + lu.OPEN_LAND, # Maps NLCD 21, 71 + lu.BARE_ROCK, # Maps NLCD 12, 31 + lu.SANDY_AREAS, # Does not map to NLCD + lu.UNPAVED_ROAD, # Does not map to NLCD + lu.LD_MIXED, # Maps NLCD 22 + lu.MD_MIXED, # Maps NLCD 23 + lu.HD_MIXED, # Maps NLCD 24 + lu.LD_RESIDENTIAL, # Does not map to NLCD + lu.MD_RESIDENTIAL, # Does not map to NLCD + lu.HD_RESIDENTIAL, # Does not map to NLCD + ], + 'Imper': np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # Impervious surface area percentage + 0.15, 0.52, 0.87, 0.15, 0.52, 0.87]), # only defined for urban land use types + 'CNI': np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # Curve Number for Impervious Surfaces + 92, 98, 98, 92, 92, 92]), # only defined for urban land use types + 'CNP': np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # Curve Number for Pervious Surfaces + 74, 79, 79, 74, 74, 74]), # only defined for urban land use types + 'TotSusSolids': np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # Total Suspended Solids factor + 60, 70, 80, 90, 100, 110]), # only defined for urban land use types 'PhysFlag': b.NO, # Flag: Physiographic Province Layer Detected (0 No; 1 Yes) 'PointFlag': b.YES, # Flag: Point Source Layer Detected (0 No; 1 Yes) 'SeptSysFlag': b.NO, # Flag: Septic System Layer Detected (0 No; 1 Yes) @@ -36,8 +61,35 @@ 'LastManureMonth': 0, # MS Period 1: Last Month 'FirstManureMonth2': 0, # MS Period 2: First Month 'LastManureMonth2': 0, # MS Period 2: Last Month + 'NitrConc': np.array([0.75, 2.90, 0.19, 0.19, 0.02, # Dissolved Runoff Coefficient: Nitrogen mg/l + 2.50, 0.50, 0.30, 0.10, 0.19, # only defined for rural land use types + 0, 0, 0, 0, 0, 0]), + 'PhosConc': np.array([0.01, 0.01, 0.01, 0.01, 0.01, # Dissolved Runoff Coefficient: Phosphorus mg/l + 0.01, 0.01, 0.01, 0.01, 0.01, # only defined for rural land use types + 0, 0, 0, 0, 0, 0]), 'Nqual': 3, # Number of Contaminants (Default = 3) 'Contaminant': ['Nitrogen', 'Phosphorus', 'Sediment'], + 'LoadRateImp': np.array([[], [], [], [], [], [], [], [], [], [], # Load Rate on Impervious Surfaces per urban land use per contaminant + [0.095, 0.0095, 2.80], # Ld_Mixed + [0.105, 0.0105, 6.20], # Md_Mixed + [0.110, 0.0115, 2.80], # Hd_Mixed + [0.095, 0.0095, 2.50], # Ld_Residential + [0.100, 0.0115, 6.20], # Md_Residential + [0.105, 0.0120, 5.00]]), # Hd_Residential + 'LoadRatePerv': np.array([[], [], [], [], [], [], [], [], [], [], # Load Rate on Pervious Surfaces per urban land use per contaminant + [0.015, 0.0021, 0.80], # Ld_Mixed + [0.015, 0.0021, 0.80], # Md_Mixed + [0.015, 0.0021, 0.80], # Hd_Mixed + [0.015, 0.0019, 1.30], # Ld_Residential + [0.015, 0.0039, 1.10], # Md_Residential + [0.015, 0.0078, 1.50]]), # Hd_Residential + 'DisFract': np.array([[], [], [], [], [], [], [], [], [], [], # Dissolved Fraction per urban land use per contaminant + [0.33, 0.40, 0], # Ld_Mixed + [0.33, 0.40, 0], # Md_Mixed + [0.33, 0.40, 0], # Hd_Mixed + [0.28, 0.37, 0], # Ld_Residential + [0.28, 0.37, 0], # Md_Residential + [0.28, 0.37, 0]]), # Hd_Residential 'UrbBMPRed': np.zeros((16, 3)), # Urban BMP Reduction 'SepticFlag': b.YES, # Flag: Septic Systems Layer Detected (0 No; 1 Yes) 'NumPondSys': np.zeros(12), # Number of People on Pond Systems From 9b94b93055e2f7a70b8df786752047edf2ee5902 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 21 Apr 2016 12:52:32 -0400 Subject: [PATCH 013/136] Gather data from Weather Stations dataset --- src/mmw/apps/modeling/mapshed.py | 77 ++++++++++++++++++++++++++ src/mmw/apps/modeling/tasks.py | 13 +++++ src/mmw/mmw/settings/gwlfe_settings.py | 1 + 3 files changed, 91 insertions(+) diff --git a/src/mmw/apps/modeling/mapshed.py b/src/mmw/apps/modeling/mapshed.py index c7aa8f7ed..c212fa3c6 100644 --- a/src/mmw/apps/modeling/mapshed.py +++ b/src/mmw/apps/modeling/mapshed.py @@ -6,6 +6,16 @@ import math import numpy as np +from collections import namedtuple + +from gwlfe.enums import GrowFlag + +from django.conf import settings +from django.db import connection + +NUM_WEATHER_STATIONS = settings.GWLFE_CONFIG['NumWeatherStations'] +MONTHS = settings.GWLFE_DEFAULTS['Month'] + def day_lengths(geom): """ @@ -24,3 +34,70 @@ def day_lengths(geom): (month * 30.4375 - 5))) return lengths + + +def nearest_weather_stations(geom, n=NUM_WEATHER_STATIONS): + """ + Given a geometry, returns a list of the n closest weather stations to it + """ + sql = ''' + SELECT station, location, meanrh, meanwind, meanprecip, + begyear, endyear, eroscoeff, rain_cool, rain_warm, + etadj, grw_start, grw_end + FROM ms_weather_station + ORDER BY geom <-> ST_SetSRID(ST_GeomFromText(%s), 4326) + LIMIT %s; + ''' + + with connection.cursor() as cursor: + cursor.execute(sql, [geom.wkt, n]) + + # Return all rows from cursor as namedtuple + weather_station = namedtuple('WeatherStation', + [col[0] for col in cursor.description]) + return [weather_station(*row) for row in cursor.fetchall()] + + +def growing_season(ws): + """ + Given an array of weather stations, returns an array of 12 integers, each 1 + or 0, indicating whether it is a growing season or not respectively. + We adopt a liberal approach, unioning the ranges to get a superset which is + a growing season for any weather station. + """ + + start = min([MONTHS.index(w.grw_start) for w in ws]) + end = max([MONTHS.index(w.grw_end) for w in ws]) + + season = [GrowFlag.NON_GROWING_SEASON] * 12 + season[start:end] = [GrowFlag.GROWING_SEASON] * (end - start) + + return season + + +def erosion_coeff(ws, season): + """ + Given an array of weather stations and a growing season array, returns an + array of 12 decimals, one for the erosion coefficient of each month. For + months that are in the growing season, we average the `rain_warm` of both + the weather stations, and for months outside the growing season, we average + `rain_cool` instead. + """ + + avg_warm = np.mean([w.rain_warm for w in ws]) + avg_cool = np.mean([w.rain_cool for w in ws]) + + return np.array([avg_warm if month == GrowFlag.GROWING_SEASON else avg_cool + for month in season]) + + +def et_adjustment(ws): + """ + Given an array of weather stations, returns an array of 12 decimals, one + for the ET Adjustment of each month. We average the `etadj` of all weather + stations, and use that value for all months. + """ + + avg_etadj = np.mean([w.etadj for w in ws]) + + return np.array([avg_etadj] * 12) diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index f22cc6838..7b4c9561a 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -18,6 +18,10 @@ from gwlfe.datamodel import DataModel from apps.modeling.mapshed import (day_lengths, + nearest_weather_stations, + growing_season, + erosion_coeff, + et_adjustment, ) from django.conf import settings @@ -45,6 +49,15 @@ def start_gwlfe_job(model_input): # Statically calculated lookup values z.DayHrs = day_lengths(aoi_geom) + # Data from the Weather Stations dataset + ws = nearest_weather_stations(aoi_geom) + z.Grow = growing_season(ws) + z.Acoef = erosion_coeff(ws, z.Grow) + z.PcntET = et_adjustment(ws) + z.WxYrBeg = max([w.begyear for w in ws]) + z.WxYrEnd = min([w.endyear for w in ws]) + z.WxYrs = z.WxYrEnd - z.WxYrBeg + 1 + # TODO Run the actual model. # Currently it writes to stdout and some files. # Must have it return results in a data structure. diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index 7d6b605b2..65c69a49a 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -444,4 +444,5 @@ } GWLFE_CONFIG = { + 'NumWeatherStations': 2, # Number of weather stations to consider for each polygon } From 9179ba024efe83350fab66e79acd8ab632808ca0 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 21 Apr 2016 12:54:12 -0400 Subject: [PATCH 014/136] Gather data from Animals dataset --- src/mmw/apps/modeling/mapshed.py | 71 ++++++++++++++++++++++++++ src/mmw/apps/modeling/tasks.py | 11 ++++ src/mmw/mmw/settings/gwlfe_settings.py | 3 ++ 3 files changed, 85 insertions(+) diff --git a/src/mmw/apps/modeling/mapshed.py b/src/mmw/apps/modeling/mapshed.py index c212fa3c6..dd0ee1f97 100644 --- a/src/mmw/apps/modeling/mapshed.py +++ b/src/mmw/apps/modeling/mapshed.py @@ -15,6 +15,8 @@ NUM_WEATHER_STATIONS = settings.GWLFE_CONFIG['NumWeatherStations'] MONTHS = settings.GWLFE_DEFAULTS['Month'] +LIVESTOCK = settings.GWLFE_CONFIG['Livestock'] +POULTRY = settings.GWLFE_CONFIG['Poultry'] def day_lengths(geom): @@ -101,3 +103,72 @@ def et_adjustment(ws): avg_etadj = np.mean([w.etadj for w in ws]) return np.array([avg_etadj] * 12) + + +def animal_energy_units(geom): + """ + Given a geometry, returns the total livestock and poultry AEUs within it + """ + sql = ''' + WITH clipped_counties AS ( + SELECT ST_Intersection(geom, + ST_SetSRID(ST_GeomFromText(%s), + 4326)) AS geom_clipped, + ms_county_animals.* + FROM ms_county_animals + WHERE ST_Intersects(geom, + ST_SetSRID(ST_GeomFromText(%s), + 4326)) + ), clipped_counties_with_area AS ( + SELECT ST_Area(geom_clipped) / ST_Area(geom) AS clip_percent, + clipped_counties.* + FROM clipped_counties + ) + SELECT SUM(beef_ha * totalha * clip_percent) AS beef_cows, + SUM(broiler_ha * totalha * clip_percent) AS broilers, + SUM(dairy_ha * totalha * clip_percent) AS dairy_cows, + SUM(sheep_ha * totalha * clip_percent) AS sheep, + SUM(hog_ha * totalha * clip_percent) AS hogs, + SUM(horse_ha * totalha * clip_percent) AS horses, + SUM(layer_ha * totalha * clip_percent) AS layers, + SUM(turkey_ha * totalha * clip_percent) AS turkeys + FROM clipped_counties_with_area; + ''' + + with connection.cursor() as cursor: + cursor.execute(sql, [geom.wkt, geom.wkt]) + + # Convert result to dictionary + columns = [col[0] for col in cursor.description] + values = cursor.fetchone() # Only one row since aggregate query + aeu = dict(zip(columns, values)) + + livestock_aeu = round(sum(aeu[animal] for animal in LIVESTOCK)) + poultry_aeu = round(sum(aeu[animal] for animal in POULTRY)) + + return livestock_aeu, poultry_aeu + + +def manure_spread(aeu): + """ + Given Animal Energy Units, returns two 16-item lists, containing nitrogen + and phosphorus manure spreading values for each of the 16 land use types. + If a given land use index is marked as having manure spreading applied in + the configuration, it will have a value calculated below, otherwise it + will be set to 0. + """ + n_list = np.zeros(16) + p_list = np.zeros(16) + + if 1.0 <= aeu: + n_spread, p_spread = 4.88, 0.86 + elif 0.5 < aeu < 1.0: + n_spread, p_spread = 3.66, 0.57 + else: + n_spread, p_spread = 2.44, 0.38 + + for lu in settings.GWLFE_CONFIG['ManureSpreadingLandUseIndices']: + n_list[lu] = n_spread + p_list[lu] = p_spread + + return n_list, p_list diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index 7b4c9561a..292fc13c6 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -22,6 +22,8 @@ growing_season, erosion_coeff, et_adjustment, + animal_energy_units, + manure_spread, ) from django.conf import settings @@ -58,6 +60,15 @@ def start_gwlfe_job(model_input): z.WxYrEnd = min([w.endyear for w in ws]) z.WxYrs = z.WxYrEnd - z.WxYrBeg + 1 + # Data from the County Animals dataset + livestock_aeu, poultry_aeu = animal_energy_units(aoi_geom) + z.AEU = livestock_aeu / (aoi_area * ACRES_PER_SQM) + z.n41j = livestock_aeu + z.n41k = poultry_aeu + z.n41l = livestock_aeu + poultry_aeu + + z.ManNitr, z.ManPhos = manure_spread(z.AEU) + # TODO Run the actual model. # Currently it writes to stdout and some files. # Must have it return results in a data structure. diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index 65c69a49a..a2549dcfa 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -445,4 +445,7 @@ GWLFE_CONFIG = { 'NumWeatherStations': 2, # Number of weather stations to consider for each polygon + 'Livestock': ['dairy_cows', 'beef_cows', 'hogs', 'sheep', 'horses'], + 'Poultry': ['broilers', 'layers', 'turkeys'], + 'ManureSpreadingLandUseIndices': [0, 1], # Land Use Indices where manure spreading applies. Currently Hay/Past and Cropland. } From 598981511bca896a982c423176a04ccecf1d4fec Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 21 Apr 2016 12:54:55 -0400 Subject: [PATCH 015/136] Gather data from Streams dataset --- src/mmw/apps/modeling/mapshed.py | 23 +++++++++++++++++++++++ src/mmw/apps/modeling/tasks.py | 5 +++++ 2 files changed, 28 insertions(+) diff --git a/src/mmw/apps/modeling/mapshed.py b/src/mmw/apps/modeling/mapshed.py index dd0ee1f97..b4f3e8464 100644 --- a/src/mmw/apps/modeling/mapshed.py +++ b/src/mmw/apps/modeling/mapshed.py @@ -172,3 +172,26 @@ def manure_spread(aeu): p_list[lu] = p_spread return n_list, p_list + + +def stream_length(geom, drb=False): + """ + Given a geometry, finds the total length of streams in meters within it. + If the drb flag is set, we use the Delaware River Basin dataset instead + of NHD Flowline. + """ + sql = ''' + SELECT ROUND(SUM(ST_Length( + ST_Transform( + ST_Intersection(geom, + ST_SetSRID(ST_GeomFromText(%s), 4326)), + 5070)))) + FROM {datasource} + WHERE ST_Intersects(geom, + ST_SetSRID(ST_GeomFromText(%s), 4326)); + '''.format(datasource='drb_streams_50' if drb else 'nhdflowline') + + with connection.cursor() as cursor: + cursor.execute(sql, [geom.wkt, geom.wkt]) + + return cursor.fetchone()[0] # Aggregate query returns single value diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index 292fc13c6..a34ef4c3b 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -24,6 +24,7 @@ et_adjustment, animal_energy_units, manure_spread, + stream_length, ) from django.conf import settings @@ -69,6 +70,10 @@ def start_gwlfe_job(model_input): z.ManNitr, z.ManPhos = manure_spread(z.AEU) + # Data from Streams dataset + z.StreamLength = stream_length(aoi_geom) # Meters + z.n42b = round(z.StreamLength / 1000) # Kilometers + # TODO Run the actual model. # Currently it writes to stdout and some files. # Must have it return results in a data structure. From 719cc43a613314b85064e36d943130d9665168ec Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 21 Apr 2016 12:59:07 -0400 Subject: [PATCH 016/136] Gather data from Point Source Discharge dataset --- src/mmw/apps/modeling/mapshed.py | 29 ++++++++++++++++++++++++++ src/mmw/apps/modeling/tasks.py | 7 +++++++ src/mmw/mmw/settings/gwlfe_settings.py | 1 + 3 files changed, 37 insertions(+) diff --git a/src/mmw/apps/modeling/mapshed.py b/src/mmw/apps/modeling/mapshed.py index b4f3e8464..1809668b2 100644 --- a/src/mmw/apps/modeling/mapshed.py +++ b/src/mmw/apps/modeling/mapshed.py @@ -15,8 +15,10 @@ NUM_WEATHER_STATIONS = settings.GWLFE_CONFIG['NumWeatherStations'] MONTHS = settings.GWLFE_DEFAULTS['Month'] +MONTHDAYS = settings.GWLFE_CONFIG['MonthDays'] LIVESTOCK = settings.GWLFE_CONFIG['Livestock'] POULTRY = settings.GWLFE_CONFIG['Poultry'] +LITERS_PER_MGAL = 3785412 def day_lengths(geom): @@ -195,3 +197,30 @@ def stream_length(geom, drb=False): cursor.execute(sql, [geom.wkt, geom.wkt]) return cursor.fetchone()[0] # Aggregate query returns single value + + +def point_source_discharge(geom, area): + """ + Given a geometry and its area in square meters, returns three lists, + each with 12 values, one for each month, containing the Nitrogen Load (in + kg), Phosphorus Load (in kg), and Discharge (in liters per square meter) + """ + sql = ''' + SELECT SUM(mgd) AS mg_d, + SUM(kgn_yr) / 12 AS kgn_month, + SUM(kgp_yr) / 12 AS kgp_month + FROM ms_pointsource + WHERE ST_Intersects(geom, + ST_SetSRID(ST_GeomFromText(%s), 4326)); + ''' + + with connection.cursor() as cursor: + cursor.execute(sql, [geom.wkt]) + mg_d, kgn_month, kgp_month = cursor.fetchone() + + n_load = np.array([kgn_month] * 12) + p_load = np.array([kgp_month] * 12) + discharge = np.array([float(mg_d * days * LITERS_PER_MGAL) / area + for days in MONTHDAYS]) + + return n_load, p_load, discharge diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index a34ef4c3b..96b74f9eb 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -25,6 +25,7 @@ animal_energy_units, manure_spread, stream_length, + point_source_discharge, ) from django.conf import settings @@ -74,6 +75,12 @@ def start_gwlfe_job(model_input): z.StreamLength = stream_length(aoi_geom) # Meters z.n42b = round(z.StreamLength / 1000) # Kilometers + # Data from Point Source Discharge dataset + n_load, p_load, discharge = point_source_discharge(aoi_geom, aoi_area) + z.PointNitr = n_load + z.PointPhos = p_load + z.PointFlow = discharge + # TODO Run the actual model. # Currently it writes to stdout and some files. # Must have it return results in a data structure. diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index a2549dcfa..b6b15370f 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -448,4 +448,5 @@ 'Livestock': ['dairy_cows', 'beef_cows', 'hogs', 'sheep', 'horses'], 'Poultry': ['broilers', 'layers', 'turkeys'], 'ManureSpreadingLandUseIndices': [0, 1], # Land Use Indices where manure spreading applies. Currently Hay/Past and Cropland. + 'MonthDays': [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31], } From fc1a41c3171072110bd0fbdada13a1ab1cb277c3 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 21 Apr 2016 12:59:23 -0400 Subject: [PATCH 017/136] Gather data from Weather dataset --- src/mmw/apps/modeling/mapshed.py | 72 ++++++++++++++++++++++++++ src/mmw/apps/modeling/tasks.py | 6 +++ src/mmw/mmw/settings/gwlfe_settings.py | 1 + 3 files changed, 79 insertions(+) diff --git a/src/mmw/apps/modeling/mapshed.py b/src/mmw/apps/modeling/mapshed.py index 1809668b2..b387ae21a 100644 --- a/src/mmw/apps/modeling/mapshed.py +++ b/src/mmw/apps/modeling/mapshed.py @@ -19,6 +19,7 @@ LIVESTOCK = settings.GWLFE_CONFIG['Livestock'] POULTRY = settings.GWLFE_CONFIG['Poultry'] LITERS_PER_MGAL = 3785412 +WEATHER_NULL = settings.GWLFE_CONFIG['WeatherNull'] def day_lengths(geom): @@ -224,3 +225,74 @@ def point_source_discharge(geom, area): for days in MONTHDAYS]) return n_load, p_load, discharge + + +def weather_data(ws, begyear, endyear): + """ + Given a list of Weather Stations and beginning and end years, returns two + 3D arrays, one for average temperature and the other for precipitation, + for each day in each month in each year in the range, averaged over all + stations in the list, in the format: + array[year][month][day] = value + where `year` 0 corresponds to the first year in the range, 1 to the second, + and so on; `month` 0 corresponds to January, 1 to February, and so on; + `day` 0 corresponds to the 1st of the month, 1 to the 2nd, and so on. Cells + with no corresponding values are marked as None. + """ + temp_sql = ''' + SELECT year, EXTRACT(MONTH FROM TO_DATE(month, 'MON')) AS month, + AVG("1") AS "1", AVG("2") AS "2", AVG("3") AS "3", + AVG("4") AS "4", AVG("5") AS "5", AVG("6") AS "6", + AVG("7") AS "7", AVG("8") AS "8", AVG("9") AS "9", + AVG("10") AS "10", AVG("11") AS "11", AVG("12") AS "12", + AVG("13") AS "13", AVG("14") AS "14", AVG("15") AS "15", + AVG("16") AS "16", AVG("17") AS "17", AVG("18") AS "18", + AVG("19") AS "19", AVG("20") AS "20", AVG("21") AS "21", + AVG("22") AS "22", AVG("23") AS "23", AVG("24") AS "24", + AVG("25") AS "25", AVG("26") AS "26", AVG("27") AS "27", + AVG("28") AS "28", AVG("29") AS "29", AVG("30") AS "30", + AVG("31") AS "31" + FROM ms_weather + WHERE station IN %s + AND measure IN ('TMax', 'TMin') + AND year BETWEEN %s AND %s + GROUP BY year, month + ORDER BY year, month; + ''' + prcp_sql = ''' + SELECT year, EXTRACT(MONTH FROM TO_DATE(month, 'MON')) AS month, + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", + "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", + "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", + "31" + FROM ms_weather + WHERE station IN %s + AND measure = 'Prcp' + AND year BETWEEN %s AND %s + ORDER BY year, month; + ''' + + year_range = endyear - begyear + 1 + stations = tuple([w.station for w in ws]) + temps = np.zeros((year_range, 12, 31)) + prcps = np.zeros((year_range, 12, 31)) + + with connection.cursor() as cursor: + cursor.execute(temp_sql, [stations, begyear, endyear]) + for row in cursor.fetchall(): + year = row[0] - begyear + month = row[1] - 1 + for day in range(31): + t = row[day + 2] + temps[year, month, day] = t if t != WEATHER_NULL else None + + with connection.cursor() as cursor: + cursor.execute(prcp_sql, [stations, begyear, endyear]) + for row in cursor.fetchall(): + year = row[0] - begyear + month = row[1] - 1 + for day in range(31): + p = row[day + 2] + prcps[year, month, day] = p if p != WEATHER_NULL else None + + return temps, prcps diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index 96b74f9eb..d0420ca90 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -26,6 +26,7 @@ manure_spread, stream_length, point_source_discharge, + weather_data, ) from django.conf import settings @@ -81,6 +82,11 @@ def start_gwlfe_job(model_input): z.PointPhos = p_load z.PointFlow = discharge + # Data from National Weather dataset + temps, prcps = weather_data(ws, z.WxYrBeg, z.WxYrEnd) + z.Temp = temps + z.Prec = prcps + # TODO Run the actual model. # Currently it writes to stdout and some files. # Must have it return results in a data structure. diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index b6b15370f..4575189ff 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -449,4 +449,5 @@ 'Poultry': ['broilers', 'layers', 'turkeys'], 'ManureSpreadingLandUseIndices': [0, 1], # Land Use Indices where manure spreading applies. Currently Hay/Past and Cropland. 'MonthDays': [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31], + 'WeatherNull': -99999, # This value is used to indicate NULL in ms_weather dataset } From f49eb1d1e2e5d72fd7b0e8769ba09fb7de47702d Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Mon, 18 Apr 2016 11:09:22 -0400 Subject: [PATCH 018/136] Fix select model package in ITSI mode This changes the behavior of selecting a model package when the itsi_embed flag is true, so that the user will be prompted if they select a different model package after returning from the model view, and will simply return to the same model if selecting the same model package. All other behavior should be the same as before. --- .../src/analyze/templates/resultsWindow.html | 6 +- src/mmw/js/src/analyze/views.js | 47 ++++++++ src/mmw/js/src/modeling/controllers.js | 111 ++++++++++-------- 3 files changed, 113 insertions(+), 51 deletions(-) diff --git a/src/mmw/js/src/analyze/templates/resultsWindow.html b/src/mmw/js/src/analyze/templates/resultsWindow.html index 9dc0b3643..62da0184c 100644 --- a/src/mmw/js/src/analyze/templates/resultsWindow.html +++ b/src/mmw/js/src/analyze/templates/resultsWindow.html @@ -9,7 +9,7 @@
  • @@ -20,11 +20,11 @@ {% for modelPackage in modelPackages %}
  • {% if modelPackage.disabled %} - + {{ modelPackage.display_name }} (Coming Soon) {% else %} - + {{ modelPackage.display_name }} {% endif %} diff --git a/src/mmw/js/src/analyze/views.js b/src/mmw/js/src/analyze/views.js index f53b59fcf..46f82d548 100644 --- a/src/mmw/js/src/analyze/views.js +++ b/src/mmw/js/src/analyze/views.js @@ -4,8 +4,11 @@ var $ = require('jquery'), _ = require('lodash'), Marionette = require('../../shim/backbone.marionette'), App = require('../app'), + router = require('../router').router, models = require('./models'), settings = require('../core/settings'), + modalModels = require('../core/modals/models'), + modalViews = require('../core/modals/views'), coreModels = require('../core/models'), chart = require('../core/chart'), utils = require('../core/utils'), @@ -30,6 +33,50 @@ var ResultsView = Marionette.LayoutView.extend({ analyzeRegion: '#analyze-tab-contents' }, + ui: { + 'modelPackageLinks': 'a.model-package', + }, + + events: { + 'click @ui.modelPackageLinks': 'selectModelPackage', + }, + + selectModelPackage: function (e) { + e.preventDefault(); + + var modelPackages = settings.get('model_packages'), + modelPackageName = $(e.target).data('id'), + modelPackage = _.find(modelPackages, {name: modelPackageName}), + newProjectUrl = '/project/new/' + modelPackageName, + projectUrl = '/project'; + + if (!modelPackage.disabled) { + if (settings.get('itsi_embed') && App.currentProject && !App.currentProject.get('needs_reset')) { + var currModelPackageName = App.currentProject.get('model_package'); + if (modelPackageName === currModelPackageName) { + // Go to existing project + router.navigate(projectUrl, {trigger: true}); + } else { + var confirmNewProject = new modalViews.ConfirmView({ + model: new modalModels.ConfirmModel({ + question: 'If you change the model you will lose your current work.', + confirmLabel: 'Switch Model', + cancelLabel: 'Cancel', + feedbackRequired: true + }), + }); + + confirmNewProject.on('confirmation', function() { + router.navigate(newProjectUrl, {trigger: true}); + }); + confirmNewProject.render(); + } + } else { + router.navigate(newProjectUrl, {trigger: true}); + } + } + }, + onShow: function() { this.showDetailsRegion(); }, diff --git a/src/mmw/js/src/modeling/controllers.js b/src/mmw/js/src/modeling/controllers.js index b6bdd09e8..dd1df2e0e 100644 --- a/src/mmw/js/src/modeling/controllers.js +++ b/src/mmw/js/src/modeling/controllers.js @@ -72,49 +72,13 @@ var ModelingController = { project = App.currentProject; // Reset flag is set so clear off old project data. if (project.get('needs_reset')) { - project.set({ - 'user_id': App.user.get('id'), - 'area_of_interest': App.map.get('areaOfInterest'), - 'area_of_interest_name': App.map.get('areaOfInterestName'), - 'needs_reset': false - }); - - // Clear current scenarios and start over. - // Must convert to an array first to avoid conflicts with - // collection events that are disassociating the model during - // the loop. - var locks = []; - _.each(project.get('scenarios').toArray(), function(model) { - var $lock = $.Deferred(); - locks.push($lock); - model.destroy({ - success: function() { - $lock.resolve(); - } - }); - }); - - // When all models have been deleted... - $.when.apply($, locks).then(function() { - setupNewProjectScenarios(project); - - // Don't reinitialize scenario events. - if (!project.get('scenarios_events_initialized')) { - initScenarioEvents(project); - project.set('scenarios_events_initialized', true); - } - // Make sure to save the new project id onto scenarios. - project.addIdsToScenarios(); - // Save to ensure we capture AOI. - project.save(); - - // Now render. - initViews(project); - - updateUrl(); - }); + itsiResetProject(project); } else { initViews(project); + if (!project.get('scenarios_events_initialized')) { + initScenarioEvents(project); + project.set('scenarios_events_initialized', true); + } updateUrl(); } } else { @@ -142,14 +106,19 @@ var ModelingController = { makeNewProject: function(modelPackage) { var project; - var lock = $.Deferred(); - project = makeProject(modelPackage, lock); - - App.currentProject = project; + if (settings.get('itsi_embed')) { + project = App.currentProject; + project.set('model_package', modelPackage); + itsiResetProject(project); + } else { + var lock = $.Deferred(); + project = makeProject(modelPackage, lock); + App.currentProject = project; - setupNewProjectScenarios(project); - finishProjectSetup(project, lock); - updateUrl(); + setupNewProjectScenarios(project); + finishProjectSetup(project, lock); + updateUrl(); + } }, projectCleanUp: function() { @@ -202,6 +171,50 @@ var ModelingController = { } }; +function itsiResetProject(project) { + project.set({ + 'user_id': App.user.get('id'), + 'area_of_interest': App.map.get('areaOfInterest'), + 'area_of_interest_name': App.map.get('areaOfInterestName'), + 'needs_reset': false + }); + + // Clear current scenarios and start over. + // Must convert to an array first to avoid conflicts with + // collection events that are disassociating the model during + // the loop. + var locks = []; + _.each(project.get('scenarios').toArray(), function(model) { + var $lock = $.Deferred(); + locks.push($lock); + model.destroy({ + success: function() { + $lock.resolve(); + } + }); + }); + + // When all models have been deleted... + $.when.apply($, locks).then(function() { + setupNewProjectScenarios(project); + + // Don't reinitialize scenario events. + if (!project.get('scenarios_events_initialized')) { + initScenarioEvents(project); + project.set('scenarios_events_initialized', true); + } + + // Make sure to save the new project id onto scenarios. + project.addIdsToScenarios(); + // Save to ensure we capture AOI. + project.save(); + + // Now render. + initViews(project); + updateUrl(); + }); +} + function finishProjectSetup(project, lock) { lock.done(function() { project.on('change:id', updateUrl); @@ -220,6 +233,8 @@ function projectCleanUp() { App.currentProject.off('change:id', updateUrl); scenarios.off('change:activeScenario change:id', updateScenario); + App.currentProject.set('scenarios_events_initialized', false); + // App.projectNumber holds the number of the project that was // in use when the user left the `/project` page. The intent // is to allow the same project to be returned-to via the UI From 75cf0494b630c568bd1ffb47cf54844391ada746 Mon Sep 17 00:00:00 2001 From: Hector Castro Date: Fri, 22 Apr 2016 11:46:56 -0400 Subject: [PATCH 019/136] Remove remaining Ansible 2.0 deprecation warnings Stops use of "bare" variables along with `with_dict` and updates Beaver role that was doing something similar. --- deployment/ansible/roles.yml | 2 +- .../roles/model-my-watershed.app/tasks/configuration.yml | 2 +- .../roles/model-my-watershed.base/tasks/configuration.yml | 2 +- .../model-my-watershed.celery-worker/tasks/configuration.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deployment/ansible/roles.yml b/deployment/ansible/roles.yml index 263c19a6a..a38c6a62b 100644 --- a/deployment/ansible/roles.yml +++ b/deployment/ansible/roles.yml @@ -43,7 +43,7 @@ - src: azavea.mapnik version: 0.1.0 - src: azavea.beaver - version: 0.1.3 + version: 1.0.1 - src: azavea.java version: 0.2.1 - src: azavea.docker diff --git a/deployment/ansible/roles/model-my-watershed.app/tasks/configuration.yml b/deployment/ansible/roles/model-my-watershed.app/tasks/configuration.yml index 12cb4037e..ae4442265 100644 --- a/deployment/ansible/roles/model-my-watershed.app/tasks/configuration.yml +++ b/deployment/ansible/roles/model-my-watershed.app/tasks/configuration.yml @@ -5,7 +5,7 @@ owner=root group=mmw mode=0750 - with_dict: app_config + with_dict: "{{ app_config }}" notify: - Restart mmw-app diff --git a/deployment/ansible/roles/model-my-watershed.base/tasks/configuration.yml b/deployment/ansible/roles/model-my-watershed.base/tasks/configuration.yml index 5b3d42ef5..4237acac5 100644 --- a/deployment/ansible/roles/model-my-watershed.base/tasks/configuration.yml +++ b/deployment/ansible/roles/model-my-watershed.base/tasks/configuration.yml @@ -25,7 +25,7 @@ owner=root group=mmw mode=0750 - with_dict: envdir_config + with_dict: "{{ envdir_config }}" notify: - Restart mmw-app - Restart Celery diff --git a/deployment/ansible/roles/model-my-watershed.celery-worker/tasks/configuration.yml b/deployment/ansible/roles/model-my-watershed.celery-worker/tasks/configuration.yml index 95e2065a3..c8126fe8e 100644 --- a/deployment/ansible/roles/model-my-watershed.celery-worker/tasks/configuration.yml +++ b/deployment/ansible/roles/model-my-watershed.celery-worker/tasks/configuration.yml @@ -5,6 +5,6 @@ owner=root group=mmw mode=0750 - with_dict: app_config + with_dict: "{{ app_config }}" notify: - Restart Celery From 2082b9424f81ac8d969ec3a3b9a3359fe66a11ea Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 22 Apr 2016 14:29:56 -0400 Subject: [PATCH 020/136] Handle empty database results * If no weather stations are found, then raise an exception, since they are necessary for multiple calculations * For singleton values, return 0 if nothing is found * For arrays, return arrays of 0 if nothing is found --- src/mmw/apps/modeling/mapshed.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed.py b/src/mmw/apps/modeling/mapshed.py index b387ae21a..4db9caaa5 100644 --- a/src/mmw/apps/modeling/mapshed.py +++ b/src/mmw/apps/modeling/mapshed.py @@ -57,6 +57,9 @@ def nearest_weather_stations(geom, n=NUM_WEATHER_STATIONS): with connection.cursor() as cursor: cursor.execute(sql, [geom.wkt, n]) + if cursor.rowcount == 0: + raise Exception("No weather stations found.") + # Return all rows from cursor as namedtuple weather_station = namedtuple('WeatherStation', [col[0] for col in cursor.description]) @@ -146,8 +149,8 @@ def animal_energy_units(geom): values = cursor.fetchone() # Only one row since aggregate query aeu = dict(zip(columns, values)) - livestock_aeu = round(sum(aeu[animal] for animal in LIVESTOCK)) - poultry_aeu = round(sum(aeu[animal] for animal in POULTRY)) + livestock_aeu = round(sum(aeu[animal] or 0 for animal in LIVESTOCK)) + poultry_aeu = round(sum(aeu[animal] or 0 for animal in POULTRY)) return livestock_aeu, poultry_aeu @@ -197,7 +200,7 @@ def stream_length(geom, drb=False): with connection.cursor() as cursor: cursor.execute(sql, [geom.wkt, geom.wkt]) - return cursor.fetchone()[0] # Aggregate query returns single value + return cursor.fetchone()[0] or 0 # Aggregate query returns singleton def point_source_discharge(geom, area): @@ -219,10 +222,10 @@ def point_source_discharge(geom, area): cursor.execute(sql, [geom.wkt]) mg_d, kgn_month, kgp_month = cursor.fetchone() - n_load = np.array([kgn_month] * 12) - p_load = np.array([kgp_month] * 12) + n_load = np.array([kgn_month] * 12) if kgn_month else np.zeros(12) + p_load = np.array([kgp_month] * 12) if kgp_month else np.zeros(12) discharge = np.array([float(mg_d * days * LITERS_PER_MGAL) / area - for days in MONTHDAYS]) + for days in MONTHDAYS]) if mg_d else np.zeros(12) return n_load, p_load, discharge From 0d9e1415780aa27fa5a231f29199c37f802a7638 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 21 Apr 2016 17:25:38 -0400 Subject: [PATCH 021/136] Upgrade geoprocessing service to 1.0.0 --- deployment/ansible/group_vars/all | 2 +- .../templates/spark-jobserver.conf.j2 | 2 +- src/mmw/mmw/settings/base.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deployment/ansible/group_vars/all b/deployment/ansible/group_vars/all index 0a725dd2e..d7abf19cc 100644 --- a/deployment/ansible/group_vars/all +++ b/deployment/ansible/group_vars/all @@ -44,7 +44,7 @@ sjs_host: "localhost" sjs_port: 8090 sjs_container_image: "quay.io/azavea/spark-jobserver:0.6.1" -geop_version: "0.4.0" +geop_version: "1.0.0" nginx_cache_dir: "/var/cache/nginx" observation_api_url: "http://www.wikiwatershed-vs.org/" diff --git a/deployment/ansible/roles/model-my-watershed.spark-jobserver/templates/spark-jobserver.conf.j2 b/deployment/ansible/roles/model-my-watershed.spark-jobserver/templates/spark-jobserver.conf.j2 index 6ae84711b..da674f966 100644 --- a/deployment/ansible/roles/model-my-watershed.spark-jobserver/templates/spark-jobserver.conf.j2 +++ b/deployment/ansible/roles/model-my-watershed.spark-jobserver/templates/spark-jobserver.conf.j2 @@ -31,7 +31,7 @@ spark { master = "local[*]" context-settings.passthrough.spark.serializer = "org.apache.spark.serializer.KryoSerializer" - context-settings.passthrough.spark.kryo.registrator = "geotrellis.spark.io.hadoop.KryoRegistrator" + context-settings.passthrough.spark.kryo.registrator = "geotrellis.spark.io.kryo.KryoRegistrator" } ######### diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index 5b3beb8c8..10c949ada 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -368,8 +368,8 @@ def get_env_setting(setting): 'geometry': None, 'tileCRS': 'ConusAlbers', 'polyCRS': 'LatLng', - 'nlcdLayer': 'nlcd-2011-30m-epsg5070', - 'soilLayer': 'ssurgo-hydro-groups-30m-epsg5070', + 'nlcdLayer': 'nlcd-2011-30m-epsg5070-0.10.0', + 'soilLayer': 'ssurgo-hydro-groups-30m-epsg5070-0.10.0', 'zoom': 0 } } From b2c967f026fd206c2577d40042a6e14f403bd46e Mon Sep 17 00:00:00 2001 From: Hector Castro Date: Tue, 26 Apr 2016 11:18:03 -0400 Subject: [PATCH 022/136] Ensure that Windshaft is restarted after log rotation If Windshaft isn't restarted after a log rotation event, it maintains a handle to the previous log file. This prevent new log messages from being written to `/var/log/mmw-tiler.log`, which also prevents them from being slurped up by Beaver. --- .../model-my-watershed.tiler/templates/logrotate-mmw-tiler.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deployment/ansible/roles/model-my-watershed.tiler/templates/logrotate-mmw-tiler.j2 b/deployment/ansible/roles/model-my-watershed.tiler/templates/logrotate-mmw-tiler.j2 index 4e2bcd50a..acc55a97d 100644 --- a/deployment/ansible/roles/model-my-watershed.tiler/templates/logrotate-mmw-tiler.j2 +++ b/deployment/ansible/roles/model-my-watershed.tiler/templates/logrotate-mmw-tiler.j2 @@ -3,5 +3,8 @@ {{ tiler_log_rotate_interval }} compress missingok + postrotate + restart mmw-tiler + endscript notifempty } From 31650a43f9e31a32def09952184b086912593a9e Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 26 Apr 2016 17:36:36 -0400 Subject: [PATCH 023/136] Make SJS calls more generic Now that we need to support multiple Spark JobServer endpoints for different purposes, we decouple the data formatting routines from the SJS communication routines. We also include more detail in the exception messaging, which should make debugging easier. --- src/mmw/apps/modeling/geoprocessing.py | 120 +++++++++++++++---------- src/mmw/mmw/settings/base.py | 2 +- 2 files changed, 72 insertions(+), 50 deletions(-) diff --git a/src/mmw/apps/modeling/geoprocessing.py b/src/mmw/apps/modeling/geoprocessing.py index ee8c19e35..c1cca93cb 100644 --- a/src/mmw/apps/modeling/geoprocessing.py +++ b/src/mmw/apps/modeling/geoprocessing.py @@ -52,6 +52,70 @@ } +@statsd.timer(__name__ + '.sjs_submit') +def sjs_submit(host, port, args, data): + """ + Submits a job to Spark Job Server. Returns its Job ID, which + can be used with sjs_retrieve to get the final result. + """ + url = 'http://{}:{}/jobs?{}'.format(host, port, args) + response = requests.post(url, data=json.dumps(data)) + + if response.ok: + job = response.json() + else: + raise Exception('Unable to submit job to Spark JobServer.\n' + 'Details = {}'.format(response.text)) + + if job['status'] == 'STARTED': + return job['result']['jobId'] + else: + raise Exception('Submitted job did not start in Spark JobServer.\n' + 'Details = {}'.format(response.text)) + + +@statsd.timer(__name__ + '.sjs_retrieve') +def sjs_retrieve(host, port, job_id, retry=None): + """ + Given a job ID, will try to retrieve its value. If the job is + still running, will call the optional retry function before + proceeding. + """ + url = 'http://{}:{}/jobs/{}'.format(host, port, job_id) + response = requests.get(url) + + if response.ok: + job = response.json() + else: + raise Exception('Unable to retrieve job {} from Spark JobServer.\n' + 'Details = {}'.format(job_id, response.text)) + + if job['status'] == 'FINISHED': + return job['result'] + elif job['status'] == 'RUNNING': + if retry: + try: + retry() + except MaxRetriesExceededError: + delete = requests.delete(url) # Job took too long, terminate + if delete.ok: + raise Exception('Job {} timed out, ' + 'deleted.'.format(job_id)) + else: + raise Exception('Job {} timed out, unable to delete.\n' + 'Details: {}'.format(job_id, delete.text)) + else: + delete = requests.delete(url) # Job in unusual state, terminate + if delete.ok: + raise Exception('Job {} was {}, deleted'.format(job_id, + job['status'])) + else: + raise Exception('Job {} was {}, could not delete.\n' + 'Details = {}'.format(job_id, + job['status'], + delete.text)) + + @statsd.timer(__name__ + '.histogram_start') def histogram_start(polygons): """ @@ -62,27 +126,13 @@ def histogram_start(polygons): This is the top-half of the function. """ - @statsd.timer(__name__ + '.histogram_start.sjs_post') - def post(url, data): - return requests.post(url, data) - host = settings.GEOP['host'] port = settings.GEOP['port'] - path = settings.GEOP['path'] - request = settings.GEOP['request'].copy() - request['input']['geometry'] = polygons - url = "http://%s:%s%s" % (host, port, path) - - response = post(url, data=json.dumps(request)) - if response.ok: - data = response.json() - else: - raise Exception('Unable to communicate with SJS (top-half).') + args = settings.GEOP['args'] + data = settings.GEOP['request'].copy() + data['input']['geometry'] = polygons - if data['status'] == 'STARTED': - return data['result']['jobId'] - else: - raise Exception('Job submission failed.') + return sjs_submit(host, port, args, data) @statsd.timer(__name__ + '.histogram_finish') @@ -97,40 +147,12 @@ def dict_to_array(d): result.append(((k1, k2), v)) return result - @statsd.timer(__name__ + '.histogram_finish.sjs_get') - def get(url): - return requests.get(url) - - @statsd.timer(__name__ + '.histogram_finish.sjs_delete') - def delete(url): - response = requests.delete(url) - return response.ok - host = settings.GEOP['host'] port = settings.GEOP['port'] - url = "http://%s:%s/jobs/%s" % (host, port, job_id) - response = get(url) - if response.ok: - data = response.json() - else: - raise Exception('Unable to communicate with SJS (bottom-half).') - - if data['status'] == 'FINISHED': - return [dict_to_array(d) for d in data['result']] - elif data['status'] == 'RUNNING': - try: - retry() - except MaxRetriesExceededError, X: # job took too long, terminate - if delete(url): - raise X - else: - raise Exception('Job timed out, unable to delete.') - else: - if delete(url): # job failed, terminate - raise Exception('Job failed, deleted.') - else: - raise Exception('Job failed, unable to delete.') + data = sjs_retrieve(host, port, job_id, retry) + + return [dict_to_array(d) for d in data] def histogram_to_x(data, nucleus, update_rule, after_rule): diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index 10c949ada..1905107c1 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -362,7 +362,7 @@ def get_env_setting(setting): GEOP = { 'host': environ.get('MMW_GEOPROCESSING_HOST', 'localhost'), 'port': environ.get('MMW_GEOPROCESSING_PORT', '8090'), - 'path': '/jobs?context=geoprocessing&appName=geoprocessing-%s&classPath=org.wikiwatershed.mmw.geoprocessing.SummaryJob' % environ.get('MMW_GEOPROCESSING_VERSION', '0.1.0'), # NOQA + 'args': 'context=geoprocessing&appName=geoprocessing-%s&classPath=org.wikiwatershed.mmw.geoprocessing.SummaryJob' % environ.get('MMW_GEOPROCESSING_VERSION', '0.1.0'), # NOQA 'request': { 'input': { 'geometry': None, From 6b4cc8092d6d6fabd20424785a4b00736531659a Mon Sep 17 00:00:00 2001 From: Kevin DeLoach Date: Wed, 27 Apr 2016 18:08:03 -0400 Subject: [PATCH 024/136] Re-enable vagrant synced folder The ability to run scripts and transfer files between the host and guest via the `/vagrant` synced folder is occasionally very convenient. --- Vagrantfile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 20b70313a..4923dd2fe 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -47,8 +47,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| services.vm.hostname = "services" services.vm.network "private_network", ip: ENV.fetch("MMW_SERVICES_IP", "33.33.34.30") - services.vm.synced_folder ".", "/vagrant", disabled: true - # Graphite Web services.vm.network "forwarded_port", { guest: 8080, @@ -90,7 +88,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| worker.vm.hostname = "worker" worker.vm.network "private_network", ip: ENV.fetch("MMW_WORKER_IP", "33.33.34.20") - worker.vm.synced_folder ".", "/vagrant", disabled: true worker.vm.synced_folder "src/mmw", "/opt/app/" if ENV["VAGRANT_ENV"].nil? || ENV["VAGRANT_ENV"] != "TEST" @@ -125,8 +122,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| app.vm.hostname = "app" app.vm.network "private_network", ip: ENV.fetch("MMW_APP_IP", "33.33.34.10") - app.vm.synced_folder ".", "/vagrant", disabled: true - if Vagrant::Util::Platform.windows? || Vagrant::Util::Platform.cygwin? app.vm.synced_folder "src/mmw", "/opt/app/", type: "rsync", rsync__exclude: ["node_modules/", "apps/"] app.vm.synced_folder "src/mmw/apps", "/opt/app/apps" @@ -167,8 +162,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| tiler.vm.hostname = "tiler" tiler.vm.network "private_network", ip: ENV.fetch("MMW_TILER_IP", "33.33.34.35") - tiler.vm.synced_folder ".", "/vagrant", disabled: true - if Vagrant::Util::Platform.windows? || Vagrant::Util::Platform.cygwin? tiler.vm.synced_folder "src/tiler", "/opt/tiler/", type: "rsync", rsync__exclude: ["node_modules/"] else From d1c8f9b117a5f4da641cb32233e3f44a81ed4350 Mon Sep 17 00:00:00 2001 From: Kevin DeLoach Date: Wed, 27 Apr 2016 18:26:22 -0400 Subject: [PATCH 025/136] Restart gunicorn after exiting the debugserver This fixes the problem where you kill your debug server and then you have to manually restart the gunicorn process on the app VM. Now, the service should restart automatically, after exiting the debugserver. --- scripts/debugserver.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/debugserver.sh b/scripts/debugserver.sh index 634adce5f..f36b25106 100755 --- a/scripts/debugserver.sh +++ b/scripts/debugserver.sh @@ -7,3 +7,4 @@ set -x vagrant ssh app -c "sudo service mmw-app stop || /bin/true" vagrant ssh app -c "cd /opt/app/ && envdir /etc/mmw.d/env gunicorn --config /etc/mmw.d/gunicorn.py mmw.wsgi" +vagrant ssh app -c "sudo start mmw-app" From 53918b00803dd9393ecd2274412fc28de367bcb9 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 27 Apr 2016 19:59:38 -0400 Subject: [PATCH 026/136] Add EPSG:5070 spatial reference to database For a number of MapShed operations, we need to convert data stored as EPSG:4326 (LatLng) to EPSG:5070 (ConusAlbers). To support operations such as ST_Transform(), we need to first add the spatial reference to the system. --- .../migrations/0018_postgis_add_EPSG5070.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 src/mmw/apps/modeling/migrations/0018_postgis_add_EPSG5070.py diff --git a/src/mmw/apps/modeling/migrations/0018_postgis_add_EPSG5070.py b/src/mmw/apps/modeling/migrations/0018_postgis_add_EPSG5070.py new file mode 100644 index 000000000..0aad1ff1b --- /dev/null +++ b/src/mmw/apps/modeling/migrations/0018_postgis_add_EPSG5070.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('modeling', '0017_add_gwlfe'), + ] + + operations = [ + migrations.RunSQL( + '''INSERT INTO spatial_ref_sys (srid, auth_name, auth_srid, proj4text, srtext) + SELECT 5070, 'EPSG', 5070, '+proj=aea +lat_1=29.5 +lat_2=45.5 +lat_0=23 +lon_0=-96 +x_0=0 +y_0=0 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs ', 'PROJCS["NAD83 / Conus Albers",GEOGCS["NAD83",DATUM["North_American_Datum_1983",SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4269"]],PROJECTION["Albers_Conic_Equal_Area"],PARAMETER["standard_parallel_1",29.5],PARAMETER["standard_parallel_2",45.5],PARAMETER["latitude_of_center",23],PARAMETER["longitude_of_center",-96],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["X",EAST],AXIS["Y",NORTH],AUTHORITY["EPSG","5070"]]' + WHERE NOT EXISTS ( + SELECT * + FROM spatial_ref_sys + WHERE srid = 5070 + )''' + ) + ] From 0ae3840ce197dc05e703973115e87f6479621371 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Tue, 26 Apr 2016 12:13:18 -0400 Subject: [PATCH 027/136] Run GWLF-E using sample GMS file on backend --- src/mmw/apps/modeling/data/sample_input.gms | 5841 +++++++++++++++++++ src/mmw/apps/modeling/tasks.py | 14 +- 2 files changed, 5849 insertions(+), 6 deletions(-) create mode 100755 src/mmw/apps/modeling/data/sample_input.gms diff --git a/src/mmw/apps/modeling/data/sample_input.gms b/src/mmw/apps/modeling/data/sample_input.gms new file mode 100755 index 000000000..498f55818 --- /dev/null +++ b/src/mmw/apps/modeling/data/sample_input.gms @@ -0,0 +1,5841 @@ +10,6,"4" +"1.4.0",.06,0,10,0,0,.149,13.154,45178,22800,22378,1843,1404,2.67,8.74802590162132E-02,15,2000,2014,.00064029392381193195,4129,0,0,0,.291239452651974 +0 +0 +0 +0 +0 +"Jan",.63,9.4,0,.2,0,.007,.9 +"Feb",.68,10.4,0,.2,0,.007,.9 +"Mar",.7,11.8,0,.2,0,.007,.9 +"Apr",.72,13.2,0,.3,0,.007,.9 +"May",.91,14.3,1,.3,0,.007,.9 +"Jun",1.02,14.9,1,.3,0,.007,.9 +"Jul",1.08,14.6,1,.3,0,.007,.9 +"Aug",1.12,13.6,1,.3,0,.007,.9 +"Sep",1.14,12.2,1,.12,0,.007,.9 +"Oct",.97,10.8,0,.12,0,.007,.9 +"Nov",.88,9.7,0,.12,0,.007,.9 +"Dec",.82,9.1,0,.12,0,.007,.9 +"Hay/Past",957,63,.29,.525,.03,.45 +"Cropland",1505,75,.29,.545,.42,.45 +"Forest",1033,60,.294,.8,.002,.45 +"Wetland",46,80,.286,.299,.01,.1 +"Disturbed",7,85,.297,.529,.08,.1 +"Turfgrass",37,58,.3,.697,.03,.2 +"Open_Land",186,82,.293,.604,.04,.45 +"Bare_Rock",0,0,0,0,0,0 +"Sandy_Areas",0,0,0,0,0,0 +"Unpaved_Road",0,0,0,0,0,0 +"Ld_Mixed",50,.15,92,74,60 +"Md_Mixed",47,.52,98,79,70 +"Hd_Mixed",84,.87,98,79,80 +"Ld_Residential",131,.15,92,74,90 +"Md_Residential",46,.52,92,74,100 +"Hd_Residential",0,0,0,0,0 +1,0,0,1,1,1,1 +2000,528,7.83,.03,.5,.5 +2,0,0,0,0 +.75,.178574398 +2.9,.178574398 +.19,.01 +.19,.01 +.02,.01 +2.5,.174451303827751 +.5,.01 +0,0 +0,0 +0,0 +3 +"Nitrogen" +"Phosphorus" +"Sediment" +.095,.015,.33,0 +.0095,.0021,.4,0 +2.8,.8,0,0 +.105,.015,.33,0 +.0105,.0021,.4,0 +6.2,.8,0,0 +.11,.015,.33,0 +.0115,.0021,.4,0 +2.8,.8,0,0 +.095,.015,.28,0 +.0095,.0019,.37,0 +2.5,1.3,0,0 +.1,.015,.28,0 +.0115,.0039,.37,0 +6.2,1.1,0,0 +0,0,0,0 +0,0,0,0 +0,0,0,0 +2.44,.38 +2.44,.38 +.68,.05,2.8420384063938E-04 +.68,.05,2.5670024315815E-04 +.68,.05,2.8420384063938E-04 +.68,.05,2.75035974812303E-04 +.68,.05,2.8420384063938E-04 +.68,.05,2.75035974812303E-04 +.68,.05,2.8420384063938E-04 +.68,.05,2.8420384063938E-04 +.68,.05,2.75035974812303E-04 +.68,.05,2.8420384063938E-04 +.68,.05,2.75035974812303E-04 +.68,.05,2.8420384063938E-04 +1 +686,0,0,0,0 +686,0,0,0,0 +686,0,0,0,0 +686,0,0,0,0 +686,0,0,0,0 +686,0,0,0,0 +686,0,0,0,0 +686,0,0,0,0 +686,0,0,0,0 +686,0,0,0,0 +686,0,0,0,0 +686,0,0,0,0 +12,2.5,1.6,.4 +15,.1,50 +"scenario4",1,"","1.4.0" +"scenario4" +0,0,0,0,0,0,0 +0,0,0,0,0,0,0,0,0,0,0 +0,0,0,0,0,0,0,0,0,0,0 +0,0,0,0 +1505,84,5,957,50,3,6,6 +20,0,0,0,0,55,0,0,0,4.6,0,34,0 +20,0,0,0,0,55,0,0,0,4.6,0,0,0,34,0,0 +0,0,0,0,0,0,0,0,0 +.748578391551584,62,62,100,100,0,0,0,0,0,0,0,22.8,45.2,0,1,0,0,0,0,0,0,0,0,0,0 +0,3,50,0,0,0,0,0,0,1,0,.6,0,2,25,0,0 +.6,0,0,0,.96,.63,0,0,0,0,0,0,0,0,0,0 +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +.29,.41,.08,.07,.05,0,.95,.3,.56,.2,.95,.29,.25 +.5,.28,.4,.22,.1,.1,0,.95,.3,.78,.45,.95,.44,.35 +.35,.44,.63,.53,.3,.17,.16,0,.95,.38,.76,.6,.55,.95,.02,.0035,2.55 +.75,.75,.14,.14,.15,.15,.21,.7,1,.75,.14,.15,.71,.82,.71 +.14,.56,.14,.56,.42,.1,.6,.1,.6,.5,.99,.99,200,200 +74.13,61.78,889.58,9320.57,24.71,33112.12,82.02,19768.43,932.06,12355.27,1250,520,300,2.5,271.82,0,26440.28,24.71,18.31 +15000,250,300,0,0,0,150,0,0,0,0,0,0,"None" +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +"Output" +0,0,0 +0,0,0,0,0,0,0,0,0,0,0,0 +0,0,0,0,0 +0,0,0 +0,0 +0,0 +0,0 +0,0,0 +0,0,0 +0,0,0,0,0,0,0,0,0 +0,0,0,0,0,0,0,0,0,0,0 +144034,45743,90000000000000,.8,.2,1,135961,22147,2.9E+16,.52,.18,1,0 +500000000,25,.9,9600,2000000000,0,200,.5,0,0,0,0 +"Dairy Cows",1239,"Y",640,.44,.07,100000000000 +"Beef Cows",0,"Y",360,.31,.09,100000000000 +"Broilers",155000,"N",.9,1.07,.3,140000000 +"Layers",155000,"N",1.8,.85,.29,140000000 +"Hogs/Swine",280,"N",61,.48,.15,11000000000 +"Sheep",110,"Y",50,.37,.1,12000000000 +"Horses",154,"Y",500,.28,.06,420000000 +"Turkeys",0,"N",6.8,.59,.2,95000000 +"Other",0,"N",0,0,0,0 +"Jan",.01,.15,.1,.12,0,.3,.2,.12 +"Feb",.01,.15,.1,.12,0,.3,.2,.12 +"Mar",.15,.15,.1,.12,0,.3,.2,.12 +"Apr",.1,.15,.1,.12,0,.3,.2,.12 +"May",.05,.15,.1,.12,0,.3,.2,.12 +"Jun",.03,.15,.1,.12,0,.3,.2,.12 +"Jul",.03,.15,.1,.12,0,.3,.2,.12 +"Aug",.03,.15,.1,.12,0,.3,.2,.12 +"Sep",.11,.15,.1,.12,0,.3,.2,.12 +"Oct",.1,.15,.1,.12,0,.3,.2,.12 +"Nov",.1,.15,.1,.12,0,.3,.2,.12 +"Dec",.08,.15,.1,.12,0,.3,.2,.12 +"Jan",.02,.05,.15,.1,.12,.01,.15,.1,.12,0,.3,.2,.12 +"Feb",.02,.05,.15,.1,.12,.01,.15,.1,.12,0,.3,.2,.12 +"Mar",.1,.05,.15,.1,.12,.1,.15,.1,.12,0,.3,.2,.12 +"Apr",.25,.05,.15,.1,.12,.05,.15,.1,.12,0,.3,.2,.12 +"May",.5,.05,.15,.1,.12,.05,.15,.1,.12,0,.3,.2,.12 +"Jun",.5,.05,.15,.1,.12,.03,.15,.1,.12,0,.3,.2,.12 +"Jul",.5,.05,.15,.1,.12,.03,.15,.1,.12,0,.3,.2,.12 +"Aug",.5,.05,.15,.1,.12,.03,.15,.1,.12,0,.3,.2,.12 +"Sep",.5,.05,.15,.1,.12,.11,.15,.1,.12,0,.3,.2,.12 +"Oct",.4,.05,.15,.1,.12,.06,.15,.1,.12,0,.3,.2,.12 +"Nov",.25,.05,.15,.1,.12,.02,.15,.1,.12,0,.3,.2,.12 +"Dec",.1,.05,.15,.1,.12,.02,.15,.1,.12,0,.3,.2,.12 +0,.12,.29,.84,26.052,4,.287,.226,0,0,1 +31,"Jan",2000 +2,0 +8,0 +11,.13 +13,.38 +3,.01 +0,.01 +3,0 +0,0 +3,.01 +7,1.82 +6,.09 +4,0 +2,.03 +-5,0 +-5,0 +3,0 +-6,0 +-12,0 +-5,0 +-3,.76 +-8,0 +-12,0 +-7,0 +-1,0 +-3,1.33 +-4,.01 +-10,0 +-8,0 +-8,0 +-9,1.46 +-3,.71 +29,"Feb",2000 +-7,0 +-5,0 +-8,.03 +-3,.24 +-2,.04 +-2,0 +-3,0 +-7,0 +-4,0 +0,0 +4,.11 +-3,0 +-6,.03 +4,.91 +1,0 +2,0 +1,0 +-1,.86 +1,1.02 +1,0 +1,0 +2,0 +3,0 +5,0 +11,0 +9,.36 +9,.62 +8,.93 +7,0 +31,"Mar",2000 +6,0 +7,0 +5,0 +5,0 +7,0 +6,0 +9,0 +14,0 +16,.04 +15,0 +9,1.73 +5,.91 +3,0 +6,0 +10,0 +12,.57 +7,1.28 +-1,.1 +3,0 +6,0 +5,6.34 +6,2.87 +10,0 +11,0 +13,.09 +12,.15 +10,.76 +11,1.31 +7,.03 +6,0 +6,0 +30,"Apr",2000 +9,0 +13,.03 +17,0 +15,.98 +7,0 +12,0 +13,0 +15,1.1 +5,1.41 +8,0 +8,.03 +6,.01 +4,0 +8,0 +11,.09 +20,.71 +13,1.24 +6,.27 +12,0 +14,0 +12,.93 +12,.29 +9,0 +12,0 +12,0 +9,0 +8,.08 +11,0 +11,0 +12,0 +31,"May",2000 +12,0 +15,.58 +13,0 +16,0 +21,.23 +22,0 +24,0 +24,0 +24,0 +23,.8 +18,.27 +21,0 +22,.95 +19,.3 +13,0 +12,.01 +17,.06 +22,.09 +19,2.39 +12,1.61 +13,.42 +14,.88 +16,.24 +19,1.83 +18,.04 +17,0 +16,.41 +13,.08 +15,.13 +13,0 +14,0 +30,"Jun",2000 +21,0 +24,0 +22,.1 +17,0 +17,0 +14,2.08 +18,.06 +19,0 +22,0 +25,0 +26,0 +25,.23 +19,.23 +16,.37 +20,.27 +24,.32 +25,.03 +24,.66 +20,1.23 +20,0 +22,1.04 +24,1.26 +22,0 +22,0 +26,1.46 +25,3.24 +25,.04 +22,.06 +22,.66 +19,.42 +31,"Jul",2000 +20,0 +21,0 +22,.32 +25,.14 +23,0 +20,0 +20,.08 +17,0 +21,0 +26,.6 +24,0 +20,0 +20,0 +22,.08 +21,2.31 +21,.01 +22,.2 +23,0 +20,.48 +20,.09 +20,0 +21,0 +20,0 +20,.08 +20,.04 +19,.8 +21,0 +22,0 +23,0 +25,0 +26,0 +31,"Aug",2000 +26,0 +25,0 +25,.18 +23,.55 +20,0 +18,.51 +27,.08 +26,0 +26,.03 +25,.09 +23,0 +22,.08 +19,.04 +19,.14 +21,.01 +23,.57 +20,.03 +18,.18 +18,0 +17,0 +15,0 +17,0 +20,.61 +21,.46 +20,0 +20,0 +21,.53 +23,.41 +22,.01 +23,.01 +25,.23 +30,"Sep",2000 +26,7.37 +26,.01 +24,.67 +24,0 +17,0 +13,0 +14,0 +17,0 +22,0 +22,0 +23,0 +24,.01 +21,2.76 +17,1.88 +19,1.59 +14,0 +14,0 +15,0 +17,1.47 +20,.32 +21,0 +14,0 +15,.11 +20,0 +13,1.87 +10,3.09 +12,0 +12,.03 +8,0 +10,0 +31,"Oct",2000 +12,0 +16,0 +18,0 +19,.57 +18,.04 +18,0 +12,0 +7,0 +4,0 +7,.09 +10,0 +12,0 +12,0 +14,0 +16,0 +14,.1 +14,.62 +15,.66 +12,0 +12,0 +14,0 +12,0 +8,0 +11,0 +14,0 +14,0 +13,0 +12,0 +7,0 +7,0 +8,0 +30,"Nov",2000 +8,0 +9,0 +10,0 +12,0 +7,0 +6,0 +6,0 +12,0 +13,.25 +14,1.35 +10,0 +7,0 +5,0 +8,.53 +4,0 +2,0 +4,0 +2,0 +-1,0 +-1,.03 +-1,0 +-2,0 +-4,0 +-3,0 +-2,.22 +7,1.6 +6,.06 +5,.01 +5,.39 +4,.44 +31,"Dec",2000 +1,0 +-2,0 +-4,0 +-2,0 +1,0 +-3,0 +-3,0 +0,0 +-1,0 +-2,.03 +0,0 +3,.01 +-5,0 +-2,3.1 +0,.03 +2,.74 +8,4.64 +-3,0 +-4,.09 +-5,.23 +-8,0 +-6,.01 +-10,0 +-7,0 +-7,0 +-10,0 +-9,0 +-8,0 +-6,0 +-6,0 +-3,0 +31,"Jan",2001 +-3,0 +-5,0 +-7,0 +-3,0 +-6,.34 +-4,0 +-1,0 +-1,.23 +-3,0 +-2,0 +0,0 +0,0 +-2,0 +-1,0 +2,.25 +3,0 +1,0 +-1,.23 +1,2.08 +1,.95 +-4,.52 +-8,0 +-8,0 +-5,0 +-3,0 +-6,0 +0,0 +-2,0 +-4,0 +4,1.52 +2,.01 +28,"Feb",2001 +3,0 +2,0 +-2,0 +-1,0 +0,2.15 +3,0 +2,0 +-1,0 +6,0 +8,0 +-2,0 +-3,.04 +4,0 +4,.36 +6,.1 +3,.6 +0,.18 +-3,0 +-2,0 +6,0 +4,.04 +-6,.64 +-2,0 +-2,0 +4,.37 +4,0 +3,0 +3,0 +31,"Mar",2001 +0,0 +3,0 +6,0 +3,.62 +-2,1.22 +-2,0 +4,0 +1,0 +3,0 +2,0 +3,0 +2,.44 +6,1.75 +5,.04 +4,.05 +6,.52 +5,.23 +4,.06 +4,0 +3,0 +6,2.06 +7,.08 +8,0 +4,0 +1,0 +0,0 +-2,0 +1,0 +3,1.21 +5,3.05 +5,.05 +30,"Apr",2001 +6,0 +5,.04 +6,.03 +8,.06 +7,0 +10,.66 +10,.03 +7,.09 +17,.42 +16,.29 +11,.9 +13,.04 +16,0 +13,0 +12,.39 +10,1.13 +5,.86 +4,.67 +6,0 +8,0 +15,.04 +19,0 +21,0 +20,.01 +10,0 +8,0 +12,0 +13,0 +8,0 +12,0 +31,"May",2001 +17,0 +19,0 +21,0 +22,0 +20,0 +13,0 +11,0 +12,0 +18,.14 +18,0 +19,0 +20,.11 +15,0 +11,0 +13,0 +13,0 +13,0 +14,0 +20,0 +16,.06 +13,1.83 +18,.98 +18,.32 +18,.01 +19,.19 +16,2.22 +17,1.13 +17,.2 +16,.24 +15,0 +13,0 +30,"Jun",2001 +13,.7 +18,.5 +18,0 +17,0 +20,0 +20,0 +20,0 +18,0 +18,0 +19,0 +22,.03 +23,0 +24,0 +25,0 +24,.13 +23,2.51 +24,.37 +22,0 +23,0 +25,.64 +24,.11 +24,1.07 +21,.84 +19,0 +21,0 +22,0 +24,0 +26,0 +26,0 +27,0 +31,"Jul",2001 +25,.34 +19,.65 +17,0 +23,.88 +22,.2 +19,0 +19,0 +23,.19 +24,1.19 +24,.64 +22,0 +20,0 +18,0 +20,0 +21,0 +22,0 +25,0 +24,.01 +23,0 +21,0 +20,0 +21,0 +24,0 +28,0 +29,0 +25,.19 +19,.04 +20,0 +21,.08 +22,.01 +23,0 +31,"Aug",2001 +23,0 +23,0 +25,0 +26,.57 +26,.81 +28,0 +30,0 +30,0 +29,0 +28,.94 +23,2.83 +25,1.41 +25,.33 +25,.01 +23,0 +24,0 +26,0 +23,0 +23,.38 +25,.89 +22,0 +22,0 +23,0 +23,.03 +22,0 +22,0 +23,.27 +25,0 +23,.1 +23,.05 +25,.06 +30,"Sep",2001 +21,.09 +17,0 +19,0 +24,.04 +20,.04 +18,0 +20,0 +22,0 +23,0 +22,.27 +19,0 +18,0 +20,0 +18,.39 +13,0 +14,0 +16,0 +17,0 +18,0 +21,1.18 +21,.27 +22,0 +20,0 +21,1.83 +17,4 +11,0 +13,0 +11,0 +13,0 +11,0 +31,"Oct",2001 +14,0 +16,0 +19,0 +18,0 +18,.09 +17,.03 +9,0 +5,0 +7,0 +11,0 +15,0 +16,.04 +19,0 +17,.03 +13,.41 +12,.37 +11,.39 +8,0 +9,0 +14,0 +15,0 +16,0 +17,0 +20,0 +20,0 +11,0 +7,0 +5,0 +5,0 +9,0 +8,0 +30,"Nov",2001 +12,0 +16,0 +17,.01 +11,0 +10,0 +6,0 +10,0 +9,0 +10,0 +8,0 +6,0 +2,0 +4,0 +6,0 +11,0 +11,0 +8,0 +6,0 +7,0 +8,.25 +1,0 +3,0 +6,0 +9,.09 +15,1.66 +10,.05 +7,0 +13,0 +13,0 +16,.2 +31,"Dec",2001 +13,0 +7,0 +6,0 +9,.03 +15,0 +13,0 +10,.04 +4,.84 +4,.33 +1,.04 +7,.23 +4,.04 +9,.04 +11,.62 +7,.51 +0,0 +5,.39 +10,.71 +6,0 +3,0 +2,0 +1,0 +2,.18 +2,.17 +-2,0 +-2,0 +-5,0 +-1,0 +0,0 +-4,0 +-7,0 +31,"Jan",2002 +-5,0 +-4,0 +-3,0 +-2,0 +-1,0 +-2,.88 +-2,.94 +-2,0 +-2,0 +3,0 +4,.95 +3,0 +3,0 +1,0 +3,0 +3,0 +4,0 +-1,0 +-4,.89 +-4,.28 +-2,0 +3,0 +3,0 +5,2.24 +4,0 +4,0 +7,0 +8,0 +10,0 +12,.43 +7,.48 +28,"Feb",2002 +10,.24 +3,0 +0,0 +-1,.01 +-4,0 +0,.01 +1,.01 +4,0 +4,0 +6,.03 +2,.05 +0,0 +1,0 +-2,0 +2,0 +5,0 +5,0 +1,0 +2,0 +10,0 +11,.01 +7,0 +2,0 +2,0 +5,0 +8,.18 +4,.14 +0,0 +31,"Mar",2002 +0,0 +2,.72 +9,1.63 +0,0 +-4,0 +6,0 +7,0 +10,0 +12,.01 +9,.08 +0,0 +4,0 +5,.93 +12,0 +15,0 +13,0 +3,.25 +2,.91 +5,.03 +5,2.68 +7,0 +0,0 +3,0 +4,0 +6,0 +5,.79 +5,.91 +5,0 +9,0 +14,0 +12,.36 +30,"Apr",2002 +8,.1 +10,0 +12,.51 +5,0 +2,0 +2,.01 +2,0 +8,.05 +18,.17 +12,.33 +10,0 +9,.03 +17,.25 +18,.42 +20,1.41 +23,0 +23,.11 +23,0 +23,.06 +20,.24 +11,.28 +9,.8 +7,0 +7,0 +10,.33 +9,0 +9,.15 +15,2.64 +14,.29 +11,.22 +31,"May",2002 +13,.1 +20,1.93 +15,.7 +9,0 +12,0 +14,0 +20,0 +18,.03 +15,1.49 +17,.04 +13,0 +18,.99 +20,3.05 +14,.8 +13,.1 +16,0 +21,.03 +13,2.26 +9,.01 +8,0 +8,0 +10,0 +14,0 +18,0 +19,0 +18,0 +20,.1 +21,.01 +22,0 +21,0 +24,.06 +30,"Jun",2002 +23,.06 +21,0 +18,0 +19,0 +25,.15 +23,2.04 +19,1.41 +19,0 +20,0 +22,0 +25,0 +26,0 +22,.17 +16,1.64 +18,.8 +19,0 +19,0 +18,.03 +20,.75 +21,.01 +22,0 +22,0 +23,0 +25,.06 +26,.08 +27,.01 +26,.89 +24,.61 +22,0 +23,0 +31,"Jul",2002 +24,0 +27,0 +28,0 +28,0 +26,0 +21,0 +20,0 +23,0 +26,.19 +23,.09 +19,0 +19,0 +20,0 +22,.22 +24,0 +25,0 +25,0 +28,0 +27,.03 +25,.04 +25,0 +27,0 +29,.08 +25,.66 +23,0 +20,.09 +23,0 +27,1.18 +28,0 +28,0 +27,0 +31,"Aug",2002 +28,0 +28,0 +28,1.56 +28,0 +27,.04 +24,.01 +19,0 +19,0 +20,0 +22,0 +24,0 +27,0 +28,0 +29,0 +28,0 +28,0 +27,0 +28,0 +28,.7 +26,.3 +23,0 +26,.04 +26,.67 +25,1.8 +24,.13 +22,0 +22,0 +21,1.13 +17,2.13 +18,.01 +20,0 +30,"Sep",2002 +16,.72 +20,.04 +21,.01 +25,0 +21,0 +18,0 +19,0 +20,0 +21,0 +24,0 +23,0 +17,0 +18,0 +21,.03 +23,.53 +23,.22 +20,0 +19,0 +20,0 +23,0 +23,.41 +23,.01 +19,.51 +16,0 +16,.01 +17,2.2 +19,3.14 +18,.93 +15,0 +17,0 +31,"Oct",2002 +19,0 +22,0 +22,0 +22,1.02 +22,.08 +15,0 +18,0 +11,0 +12,0 +16,1.56 +14,5.73 +16,.24 +15,.08 +10,.04 +8,.01 +8,5.11 +11,.06 +8,.09 +9,.14 +11,0 +8,0 +8,0 +8,0 +7,.22 +6,.91 +12,1.1 +11,0 +8,0 +4,1.23 +2,1.93 +5,.48 +30,"Nov",2002 +4,.01 +4,0 +4,0 +5,0 +4,.51 +8,.66 +5,0 +8,0 +9,0 +14,0 +15,.03 +12,1.8 +6,.04 +8,0 +8,0 +8,2.3 +4,2.45 +4,.84 +3,.01 +6,0 +5,.91 +7,.56 +4,.04 +5,0 +6,0 +5,.01 +1,.52 +-1,0 +1,0 +6,0 +31,"Dec",2002 +-1,0 +0,0 +-3,0 +-5,0 +-4,1.55 +-7,0 +-9,0 +-4,0 +-9,0 +-8,0 +-4,4 +2,.28 +0,.84 +4,.55 +5,0 +2,0 +-3,0 +-3,0 +3,.01 +8,.93 +3,0 +4,.01 +4,0 +1,.13 +0,3.56 +1,0 +-3,0 +-3,0 +0,0 +-2,.03 +2,0 +31,"Jan",2003 +3,1.68 +2,.41 +0,1.31 +1,.11 +-2,.44 +-1,.18 +-4,0 +2,.01 +5,0 +4,0 +-2,0 +-3,0 +-3,0 +-5,0 +-5,0 +-7,.11 +-7,.08 +-10,0 +-8,0 +-4,0 +-6,0 +-8,0 +-10,0 +-7,0 +-5,0 +-3,.05 +-9,.04 +-11,0 +-3,.19 +-7,0 +0,0 +28,"Feb",2003 +2,.11 +3,0 +3,0 +4,.7 +-1,0 +-5,.11 +-1,1.05 +-8,0 +-7,0 +-1,.06 +-6,0 +-7,.01 +-9,0 +-6,0 +-5,.18 +-9,1.85 +-8,1.97 +-4,.22 +-5,0 +1,0 +-3,.19 +3,3.09 +1,.11 +-2,0 +-3,0 +-8,.08 +-4,.04 +-1,.17 +31,"Mar",2003 +0,0 +4,.65 +-4,0 +-5,0 +4,.1 +1,.76 +-6,0 +0,0 +3,0 +-4,0 +-5,0 +2,0 +4,.13 +-1,.04 +5,0 +9,0 +14,.08 +13,0 +8,0 +5,3.07 +9,.81 +10,0 +9,0 +9,0 +11,0 +12,.7 +10,.39 +10,.04 +15,.41 +7,1.35 +3,.01 +30,"Apr",2003 +5,0 +14,0 +14,0 +10,.06 +6,.27 +5,0 +3,1 +0,.1 +3,.77 +7,0 +6,2.3 +13,.11 +11,0 +10,0 +17,0 +18,0 +14,.01 +5,.38 +12,0 +10,0 +11,.01 +13,.36 +10,0 +9,0 +12,.19 +12,2.55 +14,0 +15,0 +15,.14 +14,0 +31,"May",2003 +18,0 +20,0 +16,0 +14,0 +9,.1 +12,.09 +19,.25 +19,.85 +16,.55 +17,.55 +20,.08 +18,.01 +12,0 +14,0 +13,0 +13,1.35 +10,.19 +10,.13 +14,0 +15,0 +16,.74 +14,.01 +12,.38 +14,1.78 +15,.47 +14,2.59 +14,.04 +15,.34 +17,.01 +18,0 +18,.36 +30,"Jun",2003 +14,.97 +14,.03 +14,.98 +12,2.54 +17,.11 +16,0 +16,2.83 +16,0 +20,.2 +20,0 +23,.01 +24,.1 +24,.19 +24,.17 +22,.2 +18,0 +15,.2 +18,.62 +22,1.3 +19,4.14 +16,.9 +18,.64 +22,0 +23,0 +24,0 +26,0 +25,0 +22,0 +23,0 +24,0 +31,"Jul",2003 +22,0 +22,.01 +23,0 +25,0 +26,.76 +26,.36 +26,.22 +26,0 +25,.04 +21,.3 +24,.06 +22,0 +22,0 +22,0 +22,0 +26,0 +22,0 +22,.03 +22,.04 +21,0 +26,.2 +25,2.58 +24,1.38 +22,1.18 +22,0 +24,0 +26,.09 +24,.03 +21,0 +21,0 +22,.06 +31,"Aug",2003 +24,.19 +26,.05 +26,.04 +25,.15 +23,1.49 +23,1.56 +22,.04 +24,0 +25,.1 +25,.03 +25,1.16 +26,0 +26,0 +26,0 +25,0 +24,5.08 +23,.24 +22,0 +22,0 +23,0 +24,0 +26,.22 +22,.61 +18,0 +22,0 +24,.22 +23,.13 +22,0 +24,.36 +24,.25 +20,0 +30,"Sep",2003 +22,1.5 +21,3.81 +19,.56 +22,1.31 +18,0 +17,0 +18,0 +20,0 +20,0 +17,0 +18,0 +19,.15 +20,1.28 +24,.36 +21,1.61 +18,0 +17,0 +17,1.17 +21,2.22 +20,0 +18,0 +21,.38 +19,6.87 +16,0 +18,.3 +19,.69 +21,0 +18,.53 +14,.19 +11,0 +31,"Oct",2003 +11,.15 +8,0 +7,0 +9,.85 +8,0 +9,0 +10,0 +15,0 +15,0 +15,0 +16,0 +16,0 +15,0 +14,1.37 +13,2.44 +11,.01 +11,.47 +10,.33 +10,0 +9,0 +16,0 +12,.03 +5,0 +5,0 +7,0 +15,.3 +13,3.26 +7,.17 +10,2.64 +10,2.59 +10,0 +30,"Nov",2003 +15,0 +18,0 +19,0 +18,0 +15,1.47 +14,.52 +12,.19 +6,0 +0,0 +1,0 +4,.11 +10,.67 +9,.08 +5,.01 +6,0 +7,.04 +9,.28 +7,.01 +15,2.83 +8,.7 +9,0 +10,0 +8,0 +10,.38 +4,.01 +2,0 +5,.05 +8,2.6 +6,.88 +6,0 +31,"Dec",2003 +4,.04 +2,0 +-3,0 +-2,0 +-1,1.16 +-3,.37 +-4,0 +-4,0 +-3,0 +3,.41 +8,2.71 +2,0 +-2,0 +-1,1.51 +1,.2 +1,0 +3,1.68 +-1,0 +-2,0 +-1,0 +1,0 +4,0 +8,0 +8,1.44 +1,0 +1,0 +3,0 +1,0 +3,0 +5,.08 +3,0 +31,"Jan",2004 +3,0 +3,.28 +10,.03 +8,.34 +4,.97 +0,0 +-4,0 +-4,0 +-6,0 +-13,0 +-8,0 +0,0 +2,0 +-6,.01 +-10,.1 +-10,0 +-6,.14 +-2,2.04 +-5,0 +-5,0 +-8,0 +-4,0 +-8,0 +-9,.17 +-13,.14 +-9,.65 +-8,.32 +-6,.17 +-8,0 +-8,0 +-9,0 +29,"Feb",2004 +-7,0 +-8,0 +-3,1.87 +1,0 +-4,.08 +-1,4.64 +1,.36 +-4,0 +-2,0 +2,0 +1,0 +0,0 +0,0 +0,0 +-3,0 +-7,0 +-4,0 +-1,0 +3,0 +3,0 +4,0 +3,0 +1,0 +0,.32 +-1,0 +-1,0 +2,0 +5,0 +6,0 +31,"Mar",2004 +7,0 +12,0 +8,0 +9,.36 +10,.03 +12,1.31 +10,.18 +6,.27 +2,0 +2,.05 +4,0 +4,0 +3,0 +1,0 +8,0 +4,1.38 +0,.11 +2,.22 +2,.81 +3,.05 +6,0 +0,0 +1,0 +5,.03 +10,.01 +15,0 +14,.17 +12,0 +8,0 +5,0 +6,.05 +30,"Apr",2004 +10,2.12 +8,.7 +8,.08 +5,.7 +2,0 +6,0 +13,0 +8,.32 +11,.14 +8,0 +9,.05 +8,1.16 +6,1.8 +9,.69 +10,0 +8,0 +15,0 +19,0 +21,0 +18,0 +16,.15 +18,.04 +17,.83 +16,.67 +13,.5 +12,3.62 +12,.11 +9,.01 +15,0 +18,0 +31,"May",2004 +21,0 +21,.55 +14,1.74 +10,.04 +12,.04 +13,.08 +18,.29 +15,0 +18,.29 +22,.24 +22,0 +23,0 +23,0 +24,.18 +24,.89 +22,.48 +22,0 +22,.93 +20,1.26 +18,.04 +23,.17 +23,.06 +25,0 +26,.19 +23,.18 +22,.57 +22,.01 +22,0 +17,0 +16,0 +16,.44 +30,"Jun",2004 +19,.44 +18,.41 +19,.11 +17,.01 +16,3.24 +15,2.15 +21,0 +22,0 +25,0 +25,.34 +18,2.31 +17,.08 +17,0 +23,.89 +24,1.22 +25,.41 +25,3.9 +25,1.97 +23,0 +17,0 +18,0 +23,.28 +23,.03 +22,0 +23,.67 +22,.04 +18,0 +19,0 +19,.24 +20,0 +31,"Jul",2004 +22,0 +23,.03 +22,0 +22,.01 +27,0 +25,0 +23,1.19 +24,.14 +22,0 +21,0 +23,0 +22,6.45 +22,.11 +22,1.84 +21,.98 +21,.03 +22,.01 +22,1.51 +23,.04 +23,.33 +24,0 +24,1.14 +25,2.84 +22,.52 +20,0 +22,.01 +24,3.15 +24,1.28 +23,0 +25,0 +27,1.02 +31,"Aug",2004 +25,3.45 +25,0 +25,.9 +25,.7 +23,.98 +18,0 +16,0 +18,0 +20,0 +23,0 +23,.84 +24,1.21 +22,3.01 +20,.01 +22,.04 +22,0 +21,0 +22,0 +24,.3 +25,.56 +23,3.25 +18,0 +20,1.8 +22,0 +23,0 +22,0 +24,0 +25,0 +25,0 +25,.05 +23,0 +30,"Sep",2004 +20,0 +20,0 +20,0 +22,0 +22,0 +19,0 +22,0 +21,.44 +24,.53 +21,.03 +19,0 +20,0 +21,0 +20,0 +18,.11 +22,.05 +22,.08 +19,7.47 +13,0 +13,0 +17,0 +19,0 +21,0 +21,0 +19,0 +19,0 +18,0 +20,5.21 +19,1.44 +17,.33 +31,"Oct",2004 +16,0 +16,.09 +14,.17 +15,0 +12,0 +11,0 +14,0 +16,0 +16,0 +15,0 +12,0 +10,0 +9,.03 +12,1.6 +13,.23 +11,.27 +8,0 +7,.11 +11,1.3 +10,.13 +10,.38 +10,.19 +7,0 +7,0 +11,0 +11,0 +10,0 +10,0 +10,0 +16,.98 +17,0 +30,"Nov",2004 +12,0 +13,0 +12,0 +5,2.03 +9,.11 +9,0 +11,0 +9,0 +2,0 +2,0 +7,0 +7,1.82 +4,.25 +2,0 +5,0 +7,0 +6,0 +11,0 +10,0 +11,.55 +11,.24 +8,.03 +8,.05 +13,.27 +11,.67 +3,0 +6,.11 +10,3.82 +4,.01 +5,.05 +31,"Dec",2004 +8,1.69 +2,.03 +2,.01 +0,0 +4,0 +3,.06 +6,1.1 +8,.03 +4,.85 +7,1.8 +7,.1 +4,.01 +4,.09 +0,0 +-3,0 +0,0 +2,0 +-1,0 +-1,.01 +-9,.03 +-5,0 +2,0 +8,2.25 +0,.04 +-6,0 +-6,0 +-6,0 +-6,0 +3,0 +3,0 +8,0 +31,"Jan",2005 +8,0 +3,.04 +6,.11 +9,.11 +5,1.74 +3,.64 +3,.05 +3,1.36 +3,0 +4,0 +3,.62 +4,.44 +11,.15 +9,3.63 +-1,0 +-2,0 +-6,0 +-9,0 +-10,.23 +-5,0 +-10,0 +-12,1.28 +-10,.01 +-11,.08 +-3,0 +-1,0 +-8,0 +-11,0 +-9,.03 +-2,.14 +-6,0 +28,"Feb",2005 +-6,0 +-6,0 +-3,.05 +2,.08 +4,0 +2,0 +2,0 +6,0 +7,.05 +4,.05 +0,0 +2,0 +1,0 +3,1.02 +8,.37 +7,.39 +1,0 +-4,0 +-5,0 +-2,.14 +0,.47 +3,.08 +2,0 +-2,.38 +-6,.38 +-4,0 +-4,0 +-1,.76 +31,"Mar",2005 +-1,.38 +-1,0 +-4,0 +-6,0 +-3,0 +2,0 +9,0 +5,.43 +-5,0 +-4,0 +2,0 +1,0 +1,0 +0,0 +1,0 +1,0 +3,0 +3,0 +4,0 +7,.39 +6,.19 +5,0 +5,3.94 +3,.15 +5,0 +3,0 +6,.32 +5,3.67 +10,.3 +8,0 +8,0 +30,"Apr",2005 +12,.11 +11,4.38 +6,.9 +8,.01 +10,0 +16,0 +18,.3 +15,.13 +11,0 +12,0 +14,0 +9,0 +8,0 +11,0 +10,0 +7,0 +11,0 +14,0 +17,0 +19,0 +17,0 +10,.32 +14,1.59 +9,.56 +8,0 +12,0 +16,.17 +11,0 +10,0 +14,1.03 +31,"May",2005 +12,.18 +8,.1 +7,.06 +10,0 +9,0 +10,0 +12,0 +15,0 +16,0 +16,0 +20,0 +18,.01 +11,0 +20,.06 +21,.1 +16,.69 +12,0 +13,0 +14,0 +13,.97 +13,.05 +15,.08 +14,.08 +14,.41 +11,.08 +18,0 +17,0 +18,.77 +16,0 +15,.03 +16,0 +30,"Jun",2005 +18,0 +18,0 +17,.76 +19,.22 +19,.01 +25,1.75 +24,.98 +26,.13 +25,.08 +26,.03 +26,0 +26,0 +27,.04 +27,.06 +27,0 +23,.03 +19,0 +18,0 +18,0 +18,0 +20,0 +22,0 +19,0 +22,0 +25,0 +25,0 +25,.13 +28,.19 +26,.7 +26,2.5 +31,"Jul",2005 +26,.25 +23,0 +21,0 +23,0 +24,1.18 +26,.7 +24,.77 +21,5.78 +22,.01 +23,0 +23,0 +25,0 +26,0 +26,.09 +25,.77 +25,.65 +26,1.77 +27,0 +27,.01 +25,.04 +25,.04 +26,.1 +24,0 +22,0 +26,1.19 +27,0 +28,.2 +24,.03 +22,0 +24,0 +24,0 +31,"Aug",2005 +25,0 +26,0 +26,0 +28,0 +27,0 +24,0 +25,.15 +24,.25 +24,.34 +24,0 +26,0 +27,.24 +29,0 +29,0 +27,0 +22,1.35 +23,.1 +23,0 +23,.32 +25,0 +27,0 +22,0 +20,0 +20,0 +19,0 +21,0 +22,.03 +24,.69 +24,.11 +25,.08 +27,.05 +30,"Sep",2005 +22,0 +22,0 +21,0 +20,0 +20,0 +19,0 +19,0 +19,0 +21,0 +21,0 +18,0 +21,0 +23,0 +22,.25 +25,.01 +26,0 +24,1.74 +21,0 +22,0 +24,0 +21,0 +20,0 +23,0 +21,0 +20,0 +22,.28 +18,.13 +15,0 +18,.11 +12,0 +31,"Oct",2005 +14,0 +17,0 +17,0 +18,0 +21,0 +20,.03 +22,6.52 +17,10.33 +12,.5 +15,.03 +16,.01 +14,.13 +13,1.38 +17,.22 +18,.01 +15,0 +13,0 +15,0 +14,0 +13,0 +9,.81 +10,2.34 +10,.44 +8,.33 +8,2.2 +8,.22 +7,.01 +6,0 +5,0 +9,0 +10,0 +30,"Nov",2005 +11,0 +10,.04 +10,0 +12,0 +13,0 +14,.25 +11,.34 +9,0 +13,.05 +11,.01 +4,0 +7,0 +10,0 +11,0 +12,.08 +14,2.36 +3,.55 +-1,0 +2,0 +5,0 +5,.89 +5,1.36 +1,.03 +5,.03 +2,0 +3,0 +6,.04 +13,.01 +17,1.12 +9,.98 +31,"Dec",2005 +1,.01 +1,0 +-2,0 +-1,.42 +-1,.01 +-1,0 +-4,0 +-5,0 +-3,1.26 +-7,0 +-5,0 +-2,0 +-9,0 +-13,0 +-8,.8 +1,2.53 +0,0 +-1,0 +-3,0 +-5,0 +-5,0 +0,0 +5,0 +4,0 +0,1.26 +4,.13 +4,0 +2,0 +6,.65 +4,0 +0,.15 +31,"Jan",2006 +2,.01 +1,2.77 +4,.38 +3,0 +5,0 +2,0 +-2,0 +3,0 +8,0 +5,0 +4,.84 +8,0 +6,0 +9,.84 +-1,0 +-4,0 +0,.01 +10,2.74 +3,0 +7,0 +7,0 +2,.1 +2,1.91 +2,.06 +3,.06 +-1,0 +-1,0 +4,0 +4,.19 +8,.03 +8,.66 +28,"Feb",2006 +2,0 +5,0 +10,1.45 +9,1.07 +6,.46 +2,0 +0,0 +-2,0 +-2,0 +-2,0 +0,.32 +-2,.74 +-6,0 +-3,0 +0,0 +4,0 +7,0 +-4,0 +-7,0 +-2,0 +-2,0 +-1,0 +4,.03 +1,0 +3,0 +0,0 +-4,.03 +-3,0 +31,"Mar",2006 +2,0 +-1,.72 +-1,0 +1,0 +3,0 +2,0 +1,0 +2,0 +10,.01 +17,0 +11,0 +12,1.27 +17,0 +14,0 +4,0 +6,0 +5,0 +2,0 +2,0 +1,0 +-1,0 +0,0 +4,0 +3,0 +5,0 +6,0 +6,0 +7,0 +10,0 +10,0 +14,0 +30,"Apr",2006 +16,.06 +13,0 +10,.55 +9,.75 +6,.13 +8,0 +11,.32 +10,1.75 +5,.01 +7,0 +12,0 +14,0 +17,.1 +14,.11 +17,.01 +13,0 +11,.01 +13,0 +14,0 +17,0 +15,.32 +10,4.33 +12,1.85 +13,.14 +13,.13 +10,.14 +11,0 +12,0 +10,0 +11,0 +31,"May",2006 +11,0 +15,0 +15,0 +16,0 +19,0 +16,0 +11,0 +13,.04 +15,0 +15,0 +17,1.68 +16,2.01 +17,.15 +15,.15 +13,1.12 +14,.28 +15,.03 +15,.17 +13,.15 +14,0 +13,0 +13,0 +12,0 +13,0 +16,0 +19,.14 +21,.1 +20,0 +23,0 +25,0 +26,0 +30,"Jun",2006 +25,.66 +25,1.41 +20,1.94 +16,0 +17,0 +18,0 +18,.53 +19,.1 +19,.65 +17,.14 +16,0 +16,0 +19,0 +20,.03 +20,1.78 +19,0 +21,0 +24,0 +24,.11 +24,.08 +22,0 +25,.09 +25,.43 +24,.32 +22,1.07 +24,4.72 +24,5.26 +26,2.06 +24,0 +21,0 +31,"Jul",2006 +22,3.56 +26,.55 +25,.75 +26,.55 +24,.42 +21,.88 +19,0 +20,0 +22,0 +23,1.02 +26,0 +27,.08 +26,.94 +24,0 +25,.29 +25,.55 +27,.18 +27,1.4 +26,.25 +25,0 +27,0 +26,.1 +23,.9 +22,0 +24,0 +25,0 +27,.36 +26,1.5 +26,0 +27,0 +27,0 +31,"Aug",2006 +28,.01 +29,0 +29,0 +27,0 +23,0 +24,0 +27,0 +25,0 +20,0 +22,0 +21,0 +18,0 +18,0 +22,0 +25,0 +21,0 +21,0 +22,0 +24,0 +23,.27 +23,0 +22,0 +23,.95 +23,.11 +24,.38 +23,.03 +24,1.46 +24,.48 +25,.39 +22,.25 +19,0 +30,"Sep",2006 +16,.62 +17,3.73 +18,.37 +18,.29 +19,.15 +19,.01 +19,.06 +20,.13 +21,.03 +18,0 +18,0 +16,.09 +17,1.73 +18,2.49 +19,1.83 +20,.23 +20,.01 +19,0 +21,0 +16,0 +13,0 +13,0 +21,.04 +21,.05 +15,0 +15,0 +15,0 +18,3.47 +15,1.02 +10,0 +31,"Oct",2006 +15,.19 +14,0 +19,0 +19,0 +18,.05 +11,1.04 +11,.04 +14,0 +16,0 +18,0 +17,.01 +15,0 +7,0 +6,0 +6,0 +8,0 +13,1.84 +16,.14 +16,.11 +14,.74 +9,.01 +8,0 +9,.04 +7,.01 +8,0 +6,0 +4,.55 +10,4.04 +7,0 +8,.01 +13,0 +30,"Nov",2006 +14,.81 +9,.67 +4,0 +1,0 +4,0 +5,0 +6,.08 +12,4.5 +14,.01 +12,0 +14,0 +13,.67 +11,.18 +12,.11 +13,.03 +16,5.55 +12,1.22 +6,0 +4,0 +4,0 +2,0 +3,.55 +5,1.55 +8,0 +4,0 +6,0 +7,.03 +8,0 +9,0 +15,.01 +31,"Dec",2006 +16,.25 +8,.03 +1,0 +1,0 +-2,0 +1,0 +3,0 +-2,0 +-2,0 +3,0 +6,0 +5,0 +9,.37 +8,0 +7,0 +5,0 +6,0 +11,0 +5,0 +1,0 +3,.06 +6,.98 +9,1.97 +5,0 +2,.97 +8,.37 +4,0 +3,0 +3,0 +5,0 +2,.09 +31,"Jan",2007 +7,2.41 +5,0 +4,0 +6,0 +10,.25 +16,.25 +8,.74 +6,2.49 +2,0 +-1,0 +-2,0 +3,.04 +8,.29 +10,.18 +10,.04 +6,.01 +-4,0 +-4,0 +1,0 +-2,0 +-5,.01 +-3,.03 +1,0 +-1,0 +-2,.03 +-8,0 +-4,0 +1,.01 +-3,0 +-3,0 +-4,0 +28,"Feb",2007 +-3,0 +0,.28 +-5,0 +-7,0 +-11,0 +-11,0 +-9,.04 +-9,0 +-6,0 +-6,0 +-7,0 +0,0 +-2,1.71 +-4,5.66 +-9,0 +-8,0 +-5,0 +-5,0 +-7,0 +1,.03 +3,0 +-250,.19 +0,0 +-3,0 +-1,.66 +0,.36 +2,0 +1,0 +31,"Mar",2007 +2,.61 +7,2.5 +4,.04 +2,0 +0,0 +-5,0 +-8,.17 +-9,0 +-3,0 +5,.25 +4,0 +5,0 +10,0 +15,0 +4,.7 +1,3 +-3,.53 +-1,0 +-3,.03 +5,.01 +1,0 +12,.17 +14,.99 +10,.33 +10,0 +7,0 +16,0 +15,0 +8,0 +8,0 +9,0 +30,"Apr",2007 +9,.06 +14,.18 +13,0 +8,.81 +3,.04 +2,0 +1,0 +1,0 +2,0 +3,0 +3,.08 +7,2.13 +7,.01 +5,.25 +7,5.18 +4,.86 +8,.01 +8,0 +10,.03 +11,0 +13,0 +15,0 +18,0 +20,0 +14,.05 +12,.69 +11,1.88 +14,.15 +14,0 +17,0 +31,"May",2007 +14,0 +16,.01 +13,0 +12,0 +13,0 +13,0 +10,0 +13,0 +18,0 +20,.03 +22,0 +20,.34 +16,.52 +12,0 +19,0 +22,1.14 +15,.1 +14,.05 +14,.01 +16,0 +16,.01 +15,0 +17,0 +19,0 +21,0 +24,0 +23,.58 +22,.03 +20,0 +20,0 +24,0 +30,"Jun",2007 +25,.08 +24,.32 +23,.76 +22,.66 +21,0 +18,0 +19,0 +25,.06 +25,0 +22,0 +22,.15 +23,.44 +22,.18 +16,0 +16,0 +22,0 +22,0 +25,0 +26,1.97 +23,1.8 +21,0 +21,0 +17,0 +19,0 +22,.04 +26,0 +27,.75 +26,.56 +23,.65 +22,0 +31,"Jul",2007 +18,0 +16,0 +18,0 +21,.56 +23,3.28 +23,.65 +21,0 +24,0 +26,0 +27,.38 +24,1.08 +21,.13 +21,0 +20,0 +25,0 +25,0 +24,0 +25,.05 +25,.14 +23,0 +20,0 +22,0 +20,0 +20,0 +23,0 +25,0 +26,.64 +25,.42 +24,4.2 +24,.01 +24,0 +31,"Aug",2007 +24,0 +26,0 +26,0 +27,0 +25,.04 +26,.24 +27,0 +29,.74 +27,2.82 +22,1.97 +21,0 +23,0 +25,0 +22,0 +22,0 +25,.32 +25,.83 +21,0 +18,.27 +16,2.27 +16,2.96 +16,.38 +20,0 +25,0 +28,.36 +24,3.29 +21,0 +22,0 +22,0 +23,0 +24,0 +30,"Sep",2007 +19,0 +18,0 +21,0 +22,0 +21,0 +24,0 +25,0 +26,0 +24,0 +25,.13 +23,1.57 +17,0 +18,0 +19,.01 +16,.15 +12,0 +12,0 +14,0 +16,0 +19,0 +19,0 +23,.04 +21,0 +18,0 +21,.03 +24,.18 +24,.01 +22,.08 +17,0 +15,0 +31,"Oct",2007 +17,0 +18,0 +21,0 +22,0 +22,0 +22,.03 +22,0 +23,0 +25,1.02 +22,.55 +15,1.05 +12,.05 +9,0 +11,0 +13,0 +15,0 +19,.06 +19,.08 +21,.43 +16,.3 +15,0 +16,0 +21,.04 +17,1.03 +12,.84 +12,2.24 +16,4.37 +11,.03 +6,0 +8,0 +10,0 +30,"Nov",2007 +12,0 +6,0 +9,0 +7,0 +7,.14 +9,.23 +5,0 +1,0 +1,.13 +4,.28 +4,.01 +8,.04 +11,.53 +8,.04 +10,2.36 +4,0 +3,0 +3,.56 +1,.51 +6,.69 +11,0 +14,.1 +3,0 +-1,0 +3,0 +8,1.37 +11,.67 +2,0 +6,.04 +3,0 +31,"Dec",2007 +-1,0 +-1,.89 +2,.83 +0,0 +-3,.36 +-8,.03 +-7,.08 +2,.17 +0,.1 +3,.37 +4,.1 +7,.01 +1,1.79 +2,.09 +-1,.19 +1,3.12 +0,0 +-2,0 +0,0 +2,0 +1,.01 +3,.05 +9,.57 +6,.19 +1,0 +-1,.06 +3,.11 +3,1.05 +5,.38 +1,.52 +2,.69 +31,"Jan",2008 +3,.01 +-1,0 +-6,0 +-3,0 +-1,.11 +5,.09 +11,0 +10,0 +11,.15 +4,.04 +7,.48 +3,0 +2,.11 +3,.23 +-1,0 +0,0 +-2,.72 +3,.33 +0,0 +-4,0 +-8,0 +-4,0 +-2,0 +-5,0 +-4,0 +-5,0 +0,0 +-2,0 +-2,.38 +4,.33 +-3,0 +29,"Feb",2008 +1,4.39 +1,.38 +2,0 +2,0 +5,.08 +12,.19 +10,0 +3,.09 +3,0 +-1,0 +-9,0 +-7,.36 +-3,4.91 +-2,0 +2,0 +-2,0 +2,0 +10,.46 +2,0 +-2,.25 +-7,.05 +-3,.72 +-1,.08 +-2,0 +1,0 +2,1.35 +1,.22 +-5,0 +-4,0 +31,"Mar",2008 +2,.19 +1,0 +7,0 +12,.55 +10,2.65 +3,.03 +3,.6 +9,1.08 +3,0 +3,0 +5,.04 +5,0 +5,0 +8,.03 +10,.01 +7,.33 +3,0 +5,.03 +7,1.46 +10,.84 +5,0 +4,0 +2,0 +2,0 +2,0 +9,0 +6,.04 +7,0 +3,0 +2,0 +7,.29 +30,"Apr",2008 +15,.14 +9,0 +3,.28 +6,.9 +10,.03 +8,0 +7,0 +10,0 +10,0 +15,0 +18,.53 +18,.56 +11,0 +5,0 +7,0 +9,0 +12,0 +16,0 +17,0 +19,.44 +16,.13 +15,0 +16,0 +17,0 +15,0 +17,.7 +12,1.13 +10,2.86 +11,.15 +8,0 +31,"May",2008 +9,.06 +15,0 +16,0 +14,.14 +13,0 +15,0 +16,0 +20,.14 +15,1.84 +12,.01 +11,.13 +10,1.31 +13,.1 +14,0 +18,.01 +16,3.06 +13,.06 +14,1.16 +12,0 +8,1.42 +13,.01 +12,.03 +14,0 +14,0 +15,0 +19,.15 +23,.93 +17,.51 +15,0 +17,0 +21,1.09 +30,"Jun",2008 +21,0 +19,0 +20,.44 +22,1.54 +23,.04 +23,0 +27,0 +28,0 +28,0 +28,2.13 +25,.03 +23,0 +24,0 +25,.22 +24,.01 +22,0 +19,.13 +16,.2 +16,0 +19,0 +22,0 +24,0 +23,0 +20,.36 +21,0 +23,.04 +25,.53 +25,.22 +25,0 +23,.03 +31,"Jul",2008 +22,0 +22,0 +24,0 +23,.75 +23,.58 +22,0 +25,.47 +25,1.55 +25,.43 +24,0 +22,0 +24,0 +25,.15 +23,4.17 +22,0 +23,0 +25,0 +26,0 +26,0 +27,0 +26,.1 +26,0 +24,2.4 +22,.9 +21,0 +23,0 +24,0 +22,0 +24,0 +25,0 +24,.61 +31,"Aug",2008 +25,0 +24,0 +21,0 +22,0 +23,.03 +26,0 +23,0 +21,.03 +20,0 +21,.51 +18,.47 +20,0 +20,0 +22,.14 +21,.95 +21,0 +21,0 +22,0 +22,0 +17,0 +19,0 +21,0 +20,0 +24,0 +24,.01 +20,0 +18,0 +20,.01 +20,2.03 +23,.13 +21,0 +30,"Sep",2008 +20,0 +22,0 +22,0 +24,0 +25,0 +24,6.85 +21,.19 +20,0 +22,1.27 +18,.14 +18,0 +19,1.3 +24,.18 +26,0 +24,0 +18,0 +16,0 +17,0 +15,0 +14,0 +17,0 +18,0 +17,0 +15,0 +15,.06 +16,.27 +19,5.85 +21,1.02 +18,0 +15,.08 +31,"Oct",2008 +17,.44 +12,0 +12,0 +12,0 +13,0 +12,0 +9,0 +11,0 +19,0 +16,0 +15,0 +15,0 +16,0 +18,0 +20,0 +20,0 +12,0 +9,0 +7,0 +7,0 +8,0 +8,0 +5,0 +5,0 +13,3.12 +10,.43 +7,.43 +6,2.2 +6,.01 +6,0 +8,0 +30,"Nov",2008 +11,0 +8,0 +9,.01 +10,.04 +14,.05 +16,.01 +16,.01 +13,.05 +8,0 +5,0 +3,0 +5,0 +7,1.28 +11,.05 +16,1.23 +9,.01 +4,0 +0,0 +-2,0 +2,0 +-1,.28 +-3,.06 +-2,.03 +0,.13 +4,.25 +4,0 +3,0 +3,0 +2,0 +3,1.97 +31,"Dec",2008 +4,.6 +1,0 +0,0 +3,.18 +0,0 +-5,0 +-3,0 +-6,0 +3,0 +9,1.22 +5,3.49 +2,2.22 +-2,0 +1,0 +10,.76 +7,.52 +1,.79 +0,.03 +1,2.1 +-1,.29 +-1,0 +-8,0 +-7,.05 +3,1.96 +6,0 +0,.25 +3,.13 +13,0 +5,0 +2,0 +0,0 +31,"Jan",2009 +-5,0 +-2,0 +0,0 +-2,0 +4,0 +3,.38 +0,3.15 +1,.03 +-2,0 +-4,.06 +-2,.08 +-3,0 +-2,0 +-4,0 +-7,0 +-11,0 +-12,0 +-4,.01 +-6,.13 +-6,0 +-8,0 +-4,0 +1,0 +1,0 +-7,0 +-6,0 +-6,.28 +-2,1.4 +-3,.01 +-4,0 +-5,0 +28,"Feb",2009 +2,0 +3,0 +0,.48 +-4,.28 +-8,.01 +-5,0 +2,0 +9,0 +2,0 +5,0 +10,.05 +11,.04 +5,0 +1,.04 +0,.05 +-1,0 +-1,0 +0,.24 +3,.19 +-2,0 +-1,0 +2,0 +-2,0 +-3,0 +-2,0 +2,.01 +9,.1 +5,0 +31,"Mar",2009 +1,.03 +-4,.47 +-8,0 +-6,0 +-1,0 +9,0 +11,0 +14,0 +10,0 +7,0 +12,0 +6,0 +2,0 +4,0 +6,0 +6,.14 +8,.19 +9,0 +10,.18 +3,0 +0,0 +4,0 +3,0 +1,0 +3,0 +6,.46 +8,.33 +10,.76 +14,1.14 +9,.38 +7,0 +30,"Apr",2009 +8,.29 +13,.06 +16,2.78 +11,.01 +12,0 +12,.03 +6,0 +6,0 +6,0 +10,0 +11,1 +6,0 +4,0 +7,1.32 +7,1.73 +8,.03 +10,0 +13,0 +14,0 +11,1.56 +13,.65 +10,.52 +9,.11 +11,0 +20,0 +22,0 +23,0 +23,0 +18,0 +13,0 +31,"May",2009 +17,1.55 +17,.58 +14,1.02 +12,1.54 +12,.56 +14,.51 +17,.81 +17,0 +21,.1 +17,0 +12,0 +12,0 +12,0 +16,1.28 +20,.43 +20,.25 +16,.24 +11,0 +11,0 +16,0 +18,0 +19,0 +22,0 +23,.1 +22,.08 +16,.36 +15,.23 +19,.03 +20,1.87 +18,.05 +18,0 +30,"Jun",2009 +15,0 +20,1.05 +20,1.93 +15,2.11 +14,2.07 +19,.05 +20,0 +23,0 +24,.03 +21,0 +22,.47 +24,.01 +23,.01 +22,.83 +22,0 +19,0 +16,.71 +20,1.41 +19,0 +21,2.13 +22,0 +23,0 +22,0 +22,0 +23,0 +25,0 +23,0 +21,.06 +22,.06 +22,.38 +31,"Jul",2009 +22,.76 +22,1.27 +22,0 +22,0 +19,0 +20,0 +21,0 +19,0 +19,0 +21,0 +21,0 +22,0 +20,0 +19,0 +20,0 +26,0 +23,.38 +22,.08 +20,0 +21,0 +21,0 +22,.25 +22,.85 +22,.44 +22,.1 +24,.88 +23,0 +24,.22 +25,0 +24,.52 +24,.19 +31,"Aug",2009 +22,.01 +24,1.98 +23,0 +23,0 +24,0 +24,.06 +21,0 +22,.01 +25,1.12 +27,1.14 +26,.42 +25,2.17 +24,.99 +23,0 +23,0 +24,0 +25,0 +25,.38 +25,.32 +25,2.81 +26,1.41 +24,.38 +24,.11 +22,0 +22,0 +23,0 +22,0 +22,2.76 +23,2.02 +22,0 +18,0 +30,"Sep",2009 +16,0 +17,0 +18,0 +19,0 +20,0 +20,0 +20,0 +19,0 +20,.55 +17,.28 +15,6.12 +17,.7 +20,.13 +19,0 +19,0 +19,.23 +16,1.04 +19,.03 +16,0 +14,0 +16,0 +19,0 +23,.53 +23,.13 +19,0 +14,.6 +18,1.52 +16,.01 +15,0 +13,0 +31,"Oct",2009 +10,0 +12,.04 +18,.05 +15,0 +12,0 +10,0 +17,0 +13,0 +18,.61 +16,.06 +10,0 +8,0 +12,0 +7,0 +6,1.69 +4,1.7 +5,2.63 +8,1.23 +6,0 +9,0 +13,0 +15,0 +13,.55 +18,1.97 +12,.29 +9,0 +11,1.49 +14,2.12 +13,0 +12,.01 +16,.18 +30,"Nov",2009 +11,.37 +9,0 +9,0 +6,0 +5,.04 +5,.03 +4,0 +11,0 +11,0 +12,0 +11,.24 +9,.05 +10,.24 +12,.14 +14,0 +12,0 +8,0 +8,0 +11,.37 +10,.72 +7,0 +6,.48 +6,.17 +9,.86 +10,.51 +8,.06 +6,0 +5,0 +10,0 +9,.5 +31,"Dec",2009 +4,0 +4,.9 +11,1.74 +6,0 +2,.71 +-1,.11 +-1,0 +2,.03 +6,2.88 +2,0 +-4,0 +-2,0 +-1,1.61 +5,0 +5,0 +1,0 +-2,0 +-4,0 +-5,1.7 +-3,.25 +-2,0 +-2,0 +-6,0 +-4,0 +-4,.19 +6,.89 +4,.23 +2,0 +-4,0 +-5,0 +-2,.62 +31,"Jan",2010 +0,.19 +-3,0 +-7,0 +-4,0 +-4,0 +-1,0 +0,0 +-4,.14 +-4,0 +-7,0 +-5,0 +-2,0 +-2,0 +-1,0 +2,0 +5,0 +2,1.65 +5,.17 +2,0 +0,0 +-1,0 +1,0 +-1,0 +3,.06 +11,1.98 +4,.01 +0,0 +0,0 +-5,0 +-8,0 +-6,0 +28,"Feb",2010 +-5,0 +-4,0 +0,.18 +-1,0 +-1,.74 +-3,.7 +-8,0 +-6,0 +-6,.53 +-4,2.62 +-2,0 +-4,0 +-6,0 +-3,0 +-6,0 +-2,.04 +-4,0 +2,0 +3,0 +1,0 +0,0 +-1,.28 +1,.46 +3,.03 +0,.03 +-1,.04 +0,0 +2,0 +31,"Mar",2010 +3,0 +2,.06 +3,.01 +2,0 +3,0 +3,0 +7,0 +7,0 +7,0 +8,0 +9,0 +8,.3 +8,1.28 +8,.75 +7,.25 +11,.05 +8,0 +9,0 +11,0 +12,0 +14,0 +16,1.04 +11,.5 +11,0 +11,0 +9,.55 +1,0 +5,.62 +11,.8 +8,1.63 +12,0 +30,"Apr",2010 +13,0 +15,0 +14,0 +15,0 +17,0 +21,.08 +21,0 +19,.25 +13,.67 +10,0 +12,0 +12,0 +8,.37 +10,0 +12,0 +18,.23 +14,.24 +7,0 +10,0 +10,0 +10,.1 +14,0 +12,.03 +11,0 +12,.88 +10,1.26 +10,.14 +8,0 +11,0 +16,0 +31,"May",2010 +18,0 +24,.08 +21,1.7 +18,0 +18,0 +17,0 +17,0 +12,0 +8,0 +8,0 +8,.28 +9,1.08 +12,0 +21,.29 +19,.19 +17,0 +16,.06 +12,.5 +14,0 +19,0 +21,0 +21,0 +20,.05 +21,.08 +23,.19 +24,0 +24,1.02 +21,2.02 +21,.25 +23,.25 +24,2.54 +30,"Jun",2010 +23,1.47 +23,0 +24,.7 +24,0 +25,0 +24,0 +18,0 +17,.9 +17,.55 +21,.37 +20,0 +22,.14 +22,.58 +22,0 +21,0 +20,.05 +21,.27 +20,.01 +21,0 +23,0 +22,0 +22,.25 +22,0 +24,.08 +22,.23 +23,0 +23,.03 +23,0 +23,0 +20,0 +31,"Jul",2010 +18,0 +18,0 +19,0 +22,0 +25,0 +27,0 +29,0 +29,0 +28,0 +27,0 +24,3.04 +24,0 +24,2.86 +24,2.78 +26,0 +27,0 +26,0 +27,0 +27,0 +27,3.01 +25,.13 +26,.1 +27,0 +29,0 +28,1.46 +22,1.12 +23,0 +26,0 +26,0 +22,0 +21,0 +31,"Aug",2010 +23,.13 +21,.03 +25,0 +27,0 +26,0 +25,0 +22,0 +23,0 +26,0 +28,0 +28,0 +26,1.47 +21,.46 +22,0 +21,.28 +26,.37 +25,.38 +24,.15 +24,.03 +24,0 +23,0 +26,.42 +23,.09 +20,.14 +21,0 +21,0 +19,0 +20,0 +23,0 +24,0 +25,0 +30,"Sep",2010 +26,0 +26,0 +27,0 +24,0 +17,0 +17,0 +21,0 +26,0 +22,0 +19,0 +18,0 +20,.46 +18,1.22 +20,0 +18,0 +18,.03 +19,1.13 +17,0 +18,0 +21,0 +14,0 +19,0 +25,.25 +25,.69 +27,0 +23,0 +19,.88 +22,1.18 +19,.05 +19,1.74 +31,"Oct",2010 +18,6.73 +13,0 +12,0 +11,1.32 +11,1.03 +12,.13 +16,.14 +14,0 +15,0 +15,0 +18,0 +19,.51 +12,0 +9,1.02 +11,.91 +12,0 +12,0 +11,0 +11,.81 +10,.01 +10,.01 +9,0 +10,0 +13,0 +15,0 +16,.03 +19,.88 +15,.13 +11,0 +8,0 +8,0 +30,"Nov",2010 +6,0 +4,0 +5,0 +9,2.73 +8,.13 +5,0 +4,0 +7,0 +9,0 +10,0 +7,0 +6,0 +7,0 +8,0 +9,0 +10,.84 +13,.88 +8,0 +5,0 +5,0 +5,0 +10,0 +9,0 +7,0 +4,.38 +4,.03 +2,0 +1,0 +4,0 +10,.38 +31,"Dec",2010 +8,3.59 +1,0 +2,0 +0,0 +1,0 +0,0 +-3,0 +-2,0 +-5,0 +-3,0 +2,.11 +4,2.45 +-1,0 +-6,0 +-4,0 +-6,0 +-4,0 +-4,0 +-4,0 +-3,0 +0,0 +0,0 +1,0 +-1,0 +-2,0 +-3,0 +-2,0 +-1,0 +-2,0 +0,0 +2,0 +31,"Jan",2011 +6,0 +3,.08 +-2,0 +0,0 +-1,0 +-3,0 +-3,.14 +-6,.06 +-4,0 +-5,0 +-6,.43 +-4,.43 +-5,0 +-5,0 +-3,0 +-3,0 +-4,0 +-2,.9 +3,.18 +1,0 +-4,0 +-10,0 +-10,0 +-11,0 +-2,0 +-1,2.53 +0,.27 +-1,.22 +-3,0 +-3,0 +-6,0 +28,"Feb",2011 +-2,.93 +2,1.6 +-3,0 +-6,0 +-2,.76 +2,0 +1,.04 +-1,0 +-7,0 +-7,0 +-6,0 +-3,0 +-1,0 +7,0 +1,0 +3,0 +9,0 +13,0 +8,0 +2,.1 +0,.64 +-5,.36 +-5,0 +-2,.04 +4,1.82 +2,0 +7,0 +9,.85 +31,"Mar",2011 +4,0 +4,0 +-1,0 +2,0 +8,0 +11,4.05 +7,1.77 +2,0 +3,0 +8,3.67 +8,.94 +8,0 +8,0 +3,0 +3,.04 +8,1 +10,.04 +15,0 +13,0 +6,0 +9,.8 +7,0 +4,1.28 +2,.32 +0,0 +-1,0 +2,0 +1,0 +2,0 +2,.04 +4,.29 +30,"Apr",2011 +4,.29 +6,0 +8,.1 +16,.08 +14,1.21 +6,0 +8,0 +7,1.63 +8,.46 +12,0 +19,.11 +18,1.65 +9,1.12 +13,0 +12,0 +11,1.93 +12,1.41 +11,0 +11,.32 +17,.42 +17,0 +7,.06 +12,1.08 +19,.79 +21,.33 +23,0 +23,0 +21,3.34 +15,0 +13,0 +31,"May",2011 +13,0 +16,.04 +21,.01 +14,.99 +13,.18 +12,.03 +14,0 +14,0 +14,0 +15,0 +16,0 +17,0 +17,0 +16,.03 +20,1.31 +20,2.96 +18,2.02 +18,.74 +17,1.55 +17,1.02 +19,.03 +20,0 +21,.76 +22,.46 +22,0 +24,.11 +23,.17 +23,0 +24,0 +26,0 +27,0 +30,"Jun",2011 +26,0 +22,0 +18,0 +17,.17 +21,.46 +20,0 +22,0 +26,0 +28,.11 +26,.13 +25,2.22 +24,2.06 +18,0 +19,0 +20,0 +20,.29 +22,.58 +22,0 +23,0 +23,0 +24,.93 +26,.24 +26,.03 +24,.56 +22,0 +21,0 +22,0 +23,.15 +23,0 +21,0 +31,"Jul",2011 +20,0 +22,0 +26,.52 +26,0 +24,0 +26,.09 +26,0 +25,3.64 +24,.19 +24,0 +25,0 +27,.15 +25,0 +22,0 +22,0 +23,0 +24,0 +27,0 +28,.03 +27,0 +29,0 +32,0 +30,.01 +29,0 +27,.43 +28,.18 +24,0 +24,.74 +28,.93 +28,0 +25,.05 +31,"Aug",2011 +26,.05 +25,.08 +25,1.21 +23,.19 +23,0 +24,.34 +27,.84 +26,0 +24,1.99 +23,.36 +22,0 +20,0 +21,.41 +22,1.21 +23,.06 +23,0 +23,0 +25,.57 +23,1.17 +23,0 +24,.79 +21,.18 +19,0 +21,0 +25,.38 +24,.76 +25,2.22 +22,5.59 +18,0 +19,0 +20,0 +30,"Sep",2011 +20,0 +22,.06 +23,0 +24,0 +24,4.57 +18,4.69 +19,5.44 +21,8.23 +24,0 +24,0 +22,.01 +21,.04 +22,0 +23,.48 +18,1.96 +12,0 +13,.17 +13,.04 +13,0 +19,.09 +20,.29 +23,.05 +21,2.37 +21,.03 +21,0 +22,0 +22,.08 +23,.05 +21,1.18 +17,0 +31,"Oct",2011 +15,.18 +8,.93 +9,.09 +12,.04 +15,0 +12,0 +12,0 +14,0 +17,0 +17,0 +17,0 +17,.64 +16,1.55 +17,.93 +15,.15 +14,0 +15,0 +13,.01 +16,1.13 +16,.42 +10,0 +7,0 +9,0 +11,.03 +10,.03 +12,0 +13,.27 +8,.17 +4,3.18 +3,.14 +3,0 +30,"Nov",2011 +8,0 +7,0 +8,0 +7,0 +4,0 +8,0 +8,0 +11,0 +9,0 +8,.13 +6,0 +6,0 +7,0 +13,0 +15,.05 +13,1.78 +8,.19 +2,0 +4,0 +8,.09 +10,.32 +8,3.76 +9,2.63 +7,0 +8,0 +8,0 +11,0 +17,0 +12,1.87 +7,.01 +31,"Dec",2011 +4,0 +3,0 +2,0 +5,0 +9,0 +13,.19 +10,2.91 +5,.99 +4,0 +1,0 +0,0 +-1,0 +2,0 +3,0 +9,.06 +7,0 +7,0 +-1,0 +2,0 +6,.03 +9,.41 +10,.81 +5,1.55 +0,0 +2,0 +2,0 +5,1.28 +3,.24 +-3,0 +2,0 +7,.03 +31,"Jan",2012 +5,.97 +3,0 +-4,0 +-7,0 +0,0 +5,0 +7,0 +4,0 +1,0 +2,0 +2,.53 +7,.8 +4,1.28 +0,0 +-6,0 +-5,.1 +6,.47 +3,0 +-4,0 +-3,0 +-3,1.18 +-4,0 +1,.15 +6,0 +4,0 +3,.76 +9,.55 +6,0 +3,.01 +1,0 +7,0 +29,"Feb",2012 +9,0 +5,0 +2,0 +2,0 +1,0 +2,0 +2,0 +0,.15 +-1,.1 +1,.03 +-3,.38 +-3,0 +2,0 +5,.03 +3,0 +4,.23 +3,.24 +3,0 +3,0 +0,0 +3,0 +7,0 +9,0 +7,.57 +3,0 +3,0 +7,0 +5,0 +5,2.55 +31,"Mar",2012 +5,.01 +6,.2 +8,.32 +3,0 +0,0 +4,0 +9,0 +16,0 +11,.17 +3,0 +6,0 +10,0 +17,.03 +16,0 +13,0 +13,0 +14,.03 +14,0 +15,0 +18,.01 +18,0 +16,0 +18,0 +14,2.07 +12,.47 +10,0 +4,0 +10,.11 +13,0 +7,0 +8,.71 +30,"Apr",2012 +9,.41 +10,0 +8,0 +12,0 +9,0 +7,0 +8,0 +10,0 +12,0 +11,0 +6,.03 +8,0 +8,0 +10,0 +19,.94 +21,0 +20,0 +15,.19 +15,.04 +14,0 +16,.64 +12,1.78 +6,1.98 +9,.39 +9,0 +10,.04 +10,0 +5,0 +9,0 +9,0 +31,"May",2012 +17,.58 +16,.38 +18,.75 +21,.11 +20,.15 +18,0 +14,.15 +17,.65 +19,.11 +15,.04 +13,0 +16,0 +19,0 +19,.7 +20,3.06 +22,.03 +20,0 +15,0 +18,0 +21,0 +21,.3 +20,.01 +21,.64 +22,.06 +23,0 +24,0 +25,.06 +24,.39 +26,.89 +24,1.28 +22,0 +30,"Jun",2012 +21,7.05 +17,.11 +17,.08 +17,.52 +15,.3 +16,0 +18,0 +19,0 +22,0 +23,0 +24,.65 +22,.97 +21,0 +19,0 +20,0 +20,0 +19,0 +17,0 +22,0 +26,0 +27,0 +28,.04 +24,.22 +22,0 +22,0 +18,0 +20,0 +23,0 +26,.03 +27,.01 +31,"Jul",2012 +26,0 +26,0 +24,0 +28,0 +28,0 +27,0 +30,.19 +28,0 +25,0 +26,0 +24,0 +24,0 +25,0 +23,1.21 +26,.86 +26,.19 +28,0 +29,1.02 +26,1.28 +21,.81 +20,.44 +23,0 +26,.27 +27,.17 +23,.43 +25,.56 +26,.14 +25,.13 +24,1.32 +24,0 +24,.65 +31,"Aug",2012 +24,0 +24,0 +26,.03 +26,0 +28,.44 +27,.6 +23,0 +24,0 +26,.01 +25,1.17 +24,0 +23,0 +22,0 +23,1.51 +23,.27 +23,0 +24,.01 +22,.01 +19,0 +21,.33 +21,0 +21,0 +21,0 +22,0 +23,.1 +24,0 +24,0 +25,.36 +22,0 +21,0 +23,0 +30,"Sep",2012 +24,0 +23,.32 +23,.28 +26,2.54 +25,.34 +23,0 +25,0 +25,1.88 +18,.05 +17,0 +15,0 +17,0 +18,0 +19,0 +18,0 +16,0 +16,0 +21,4.46 +15,.27 +14,0 +17,0 +22,.05 +15,0 +12,0 +13,0 +20,.01 +20,1.03 +20,.64 +16,0 +14,.25 +31,"Oct",2012 +14,0 +17,2.67 +20,3.43 +19,.01 +19,0 +15,0 +9,.05 +8,.18 +11,0 +14,.18 +11,0 +8,0 +5,0 +12,0 +16,.74 +12,0 +9,0 +13,.08 +16,3.16 +14,.62 +11,0 +11,0 +13,.15 +17,0 +18,0 +17,0 +16,0 +15,.46 +13,3.67 +8,4.15 +6,.08 +30,"Nov",2012 +7,0 +7,0 +6,0 +5,0 +3,0 +2,0 +3,0 +7,0 +6,0 +6,0 +10,0 +12,0 +10,2.03 +3,0 +2,0 +3,0 +4,0 +4,0 +4,0 +5,0 +5,0 +5,0 +6,0 +6,0 +1,0 +1,0 +1,1.19 +1,0 +1,0 +1,0 +31,"Dec",2012 +5,0 +8,.03 +12,0 +14,0 +9,0 +1,0 +1,.25 +1,.05 +6,.46 +9,.05 +4,.1 +2,0 +2,0 +2,0 +2,0 +2,.28 +7,.25 +8,.03 +5,0 +3,1.35 +7,2.54 +2,0 +2,0 +-1,.38 +0,0 +1,1.35 +2,.28 +0,0 +-1,.43 +0,0 +1,0 +31,"Jan",2013 +0,0 +-1,0 +-3,0 +0,0 +0,0 +1,.03 +3,0 +-1,0 +1,.03 +3,0 +3,.69 +7,.37 +6,0 +8,.2 +4,.69 +1,1.77 +2,0 +0,0 +3,0 +5,0 +-1,0 +-6,0 +-10,0 +-8,0 +-9,.14 +-9,.14 +-7,0 +-3,.47 +7,0 +10,4.62 +6,5.13 +28,"Feb",2013 +-2,0 +-6,.13 +-5,.09 +-3,0 +-4,0 +0,0 +-2,0 +1,.34 +-1,.18 +-3,0 +4,.65 +3,0 +0,.06 +2,.19 +5,.23 +3,.43 +-3,0 +-3,0 +0,.15 +-1,0 +-2,0 +-2,0 +3,.15 +3,0 +2,0 +2,.38 +6,.72 +5,0 +31,"Mar",2013 +3,0 +2,0 +0,0 +0,0 +2,0 +2,.51 +4,.03 +3,0 +6,0 +6,0 +8,0 +10,3.44 +4,.01 +2,0 +3,0 +5,.37 +1,.05 +0,1.14 +6,.09 +4,0 +2,0 +1,0 +2,0 +1,0 +2,.98 +5,.05 +5,0 +5,0 +7,0 +6,0 +6,.01 +30,"Apr",2013 +8,.11 +3,0 +2,0 +3,0 +8,0 +8,0 +10,0 +15,0 +19,0 +21,0 +20,.13 +12,1.28 +9,0 +10,0 +14,.05 +15,0 +17,.08 +15,.11 +17,1.65 +12,1.61 +5,0 +7,0 +10,0 +12,.11 +12,.2 +9,0 +10,0 +12,.04 +14,1.05 +15,.13 +31,"May",2013 +12,0 +12,0 +12,0 +12,0 +11,0 +11,0 +16,.13 +18,.55 +16,.11 +19,1.16 +21,.23 +14,0 +8,0 +8,.05 +15,.08 +20,.19 +18,0 +18,.05 +17,.03 +22,0 +22,0 +25,.04 +23,.89 +17,.34 +14,0 +15,0 +13,0 +15,.69 +23,0 +24,0 +25,0 +30,"Jun",2013 +25,0 +26,.13 +25,.83 +18,0 +18,0 +19,.06 +18,3.14 +20,.33 +20,0 +22,3.49 +22,.25 +22,0 +21,2.12 +20,.14 +19,0 +20,.06 +23,.85 +22,.58 +20,.05 +19,0 +20,0 +22,0 +22,0 +24,0 +25,0 +25,1.7 +25,1.47 +24,.06 +24,0 +25,.24 +31,"Jul",2013 +26,1.88 +26,.39 +26,.06 +27,.11 +27,0 +27,0 +27,.15 +25,.18 +25,.09 +26,.09 +25,0 +23,.93 +25,.93 +27,0 +28,0 +28,0 +28,0 +29,0 +29,0 +28,0 +27,0 +27,2.59 +25,1.61 +21,0 +19,0 +22,0 +23,0 +23,.51 +21,0 +21,0 +22,0 +31,"Aug",2013 +22,1.02 +23,0 +21,.04 +19,.03 +19,0 +20,.04 +22,.34 +25,.75 +26,.76 +22,0 +23,0 +24,0 +22,5.17 +18,0 +17,0 +18,0 +19,0 +20,.08 +19,0 +22,0 +22,0 +25,0 +21,0 +19,0 +18,0 +22,0 +25,0 +24,.09 +24,.66 +25,0 +26,0 +30,"Sep",2013 +26,0 +26,.18 +24,.67 +23,0 +20,0 +16,0 +16,0 +21,0 +17,0 +27,0 +27,0 +26,1.12 +20,.75 +16,0 +14,0 +18,1.19 +13,0 +12,0 +14,0 +17,0 +19,.93 +16,1.04 +14,0 +12,0 +13,0 +15,0 +18,0 +15,0 +14,0 +15,0 +31,"Oct",2013 +17,0 +19,0 +20,0 +21,0 +24,.08 +23,.2 +21,3.64 +13,0 +12,0 +12,2.57 +13,10.59 +15,2.18 +16,0 +14,0 +13,0 +14,0 +17,.25 +14,.19 +10,.24 +9,.28 +10,0 +13,0 +8,0 +5,0 +5,0 +5,0 +7,0 +6,0 +8,0 +9,0 +13,.04 +30,"Nov",2013 +17,.67 +13,0 +9,0 +2,0 +6,0 +12,0 +12,.34 +6,0 +3,0 +6,0 +6,0 +5,0 +0,0 +3,0 +4,0 +8,.03 +11,0 +12,.51 +7,0 +1,0 +1,0 +7,.04 +7,0 +0,0 +-4,0 +2,1.65 +4,2.97 +0,0 +-1,0 +-3,0 +31,"Dec",2013 +-1,0 +1,0 +3,0 +5,0 +10,.06 +9,1.99 +1,.04 +-2,2.03 +-1,.46 +-1,.66 +-7,0 +-6,0 +-6,0 +-3,.83 +-2,0 +-2,0 +-6,.15 +-4,0 +-3,0 +3,0 +5,0 +11,.09 +11,1.41 +2,.5 +-4,0 +-2,.09 +-2,.08 +3,.03 +5,1.21 +4,1.03 +-2,0 +31,"Jan",2014 +-3,0 +-3,.25 +-9,.67 +-12,0 +-6,.53 +0,.48 +-12,0 +-11,0 +-5,0 +-1,.66 +7,1.65 +6,.04 +2,0 +6,.69 +2,0 +0,0 +-1,0 +-1,.11 +-4,0 +1,0 +-3,1 +-14,.29 +-13,0 +-11,0 +-8,.01 +-8,0 +-3,0 +-9,0 +-12,0 +-12,0 +-5,0 +28,"Feb",2014 +1,.01 +2,0 +1,2.48 +-7,.06 +-2,3.45 +-4,0 +-6,0 +-7,0 +-8,.13 +-8,.08 +-11,0 +-8,1.09 +-4,1.65 +1,0 +-2,.06 +-5,0 +-8,0 +-2,.6 +0,.55 +1,0 +4,.08 +3,0 +3,.08 +1,0 +-5,0 +-6,.04 +-8,0 +-10,0 +31,"Mar",2014 +-7,0 +0,.09 +-5,.19 +-9,0 +-4,0 +-5,0 +-2,0 +2,0 +5,0 +5,0 +7,0 +8,.37 +-1,.23 +-1,0 +9,0 +4,0 +-2,.06 +2,0 +3,.61 +6,.23 +5,0 +8,0 +6,0 +-2,0 +-3,.09 +-2,0 +-1,0 +8,.36 +8,2.02 +5,3.43 +8,.05 +30,"Apr",2014 +7,.89 +9,.14 +12,.25 +8,.2 +7,0 +4,0 +6,1.19 +10,.22 +8,0 +9,0 +16,.1 +14,.1 +17,0 +20,0 +16,2.4 +6,.57 +4,0 +5,0 +10,0 +10,0 +9,0 +13,.33 +11,.04 +9,0 +9,.55 +14,.42 +12,0 +8,.06 +10,1.69 +9,5.87 +31,"May",2014 +16,.93 +14,0 +12,.27 +12,0 +12,0 +13,0 +11,0 +17,0 +18,0 +19,.47 +18,0 +19,0 +19,.11 +16,0 +20,1.82 +18,5.5 +12,0 +12,0 +12,0 +15,.48 +14,.55 +19,.15 +17,0 +15,0 +19,0 +21,0 +23,.14 +18,.25 +12,.39 +17,1.74 +16,.01 +30,"Jun",2014 +17,.48 +20,0 +22,.2 +21,.08 +19,.25 +17,0 +18,0 +20,0 +22,.32 +21,.88 +20,1.35 +18,.23 +24,3.33 +19,0 +18,0 +21,0 +26,0 +28,0 +24,.24 +18,0 +20,.83 +19,0 +21,0 +23,0 +26,.53 +25,1.75 +23,0 +22,0 +23,0 +24,0 +31,"Jul",2014 +26,0 +28,.51 +27,.25 +23,.2 +19,0 +21,0 +27,0 +26,.18 +25,1.03 +24,1.84 +23,1.69 +24,0 +26,.17 +24,1.77 +25,2.53 +22,.38 +20,0 +19,0 +21,0 +21,.06 +22,0 +24,0 +26,2.51 +24,1.24 +19,0 +21,0 +25,4.32 +24,3.59 +19,.25 +18,0 +21,0 +31,"Aug",2014 +23,.13 +23,.28 +22,0 +22,.01 +22,0 +22,0 +21,0 +19,0 +20,0 +21,0 +21,0 +20,2.97 +23,1.18 +17,0 +17,0 +17,0 +19,.06 +19,0 +21,0 +22,.11 +22,.01 +21,0 +20,0 +19,.03 +20,0 +21,0 +22,0 +22,.36 +17,0 +20,0 +25,.71 +30,"Sep",2014 +25,.39 +25,.55 +24,.14 +22,0 +25,0 +26,.33 +23,1.03 +18,0 +19,0 +21,0 +22,0 +17,0 +14,.18 +12,.23 +14,0 +15,.06 +16,0 +16,0 +15,0 +19,0 +20,.06 +18,.09 +13,0 +14,0 +16,1 +17,.19 +18,0 +19,0 +20,0 +20,0 +31,"Oct",2014 +18,.41 +17,0 +16,.17 +15,.65 +9,.01 +12,0 +17,.05 +17,.23 +14,0 +11,.08 +10,.83 +12,.1 +17,.08 +20,.06 +21,3.43 +17,2.51 +16,.36 +15,0 +10,0 +8,.03 +13,.13 +13,.41 +12,.17 +16,0 +12,0 +14,0 +10,0 +15,0 +15,.13 +9,.06 +9,0 +30,"Nov",2014 +9,.05 +8,.01 +9,0 +13,0 +12,0 +10,1.96 +7,0 +3,0 +6,0 +8,0 +9,0 +12,0 +4,.38 +1,.06 +0,0 +0,.17 +5,1.63 +1,0 +-5,0 +1,0 +0,0 +-2,0 +3,0 +14,.93 +11,0 +4,2.54 +1,.03 +-1,0 +-2,0 +4,0 +31,"Dec",2014 +7,.44 +3,.67 +3,.56 +2,0 +1,.56 +5,1.51 +2,.15 +-2,0 +2,1.21 +4,.14 +1,.25 +1,0 +0,0 +4,0 +4,0 +2,.84 +7,0 +2,0 +2,0 +2,0 +-2,0 +-2,.13 +6,.38 +9,2.04 +10,.08 +3,0 +5,0 +6,.13 +3,0 +-1,0 +-4,0 +0,0 diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index d0420ca90..ffa3b1fbe 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -8,13 +8,14 @@ import json import logging from math import sqrt +from os.path import join, dirname, abspath from apps.modeling.geoprocessing import histogram_start, histogram_finish, \ data_to_survey, data_to_censuses from tr55.model import simulate_day -# from gwlfe import gwlfe +from gwlfe import gwlfe, parser from gwlfe.datamodel import DataModel from apps.modeling.mapshed import (day_lengths, @@ -87,12 +88,13 @@ def start_gwlfe_job(model_input): z.Temp = temps z.Prec = prcps - # TODO Run the actual model. - # Currently it writes to stdout and some files. - # Must have it return results in a data structure. - # gwlfe.run(z) + # TODO pass real input to model instead of reading it from gms file + gms_filename = join(dirname(abspath(__file__)), 'data/sample_input.gms') + gms_file = open(gms_filename, 'r') + z = parser.GmsReader(gms_file).read() - response_json = z.tojson() if z else {} + # The frontend expects an object with runoff and quality as keys. + response_json = {'runoff': gwlfe.run(z)} return response_json From 97b96008205076269d1ab8bc5cf595324b695ff1 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Tue, 26 Apr 2016 12:31:50 -0400 Subject: [PATCH 028/136] Show table and chart for GWLFE-E runoff * Render chart and table, using tr55/runoff as an example. * Add dropdown selector to runoff view so user can select variable to view in chart * Add utility for drawing line charts using NVD3 --- src/mmw/js/src/core/chart.js | 41 +++- .../gwlfe/runoff/templates/result.html | 3 + .../gwlfe/runoff/templates/selector.html | 7 + .../gwlfe/runoff/templates/table.html | 28 +++ src/mmw/js/src/modeling/gwlfe/runoff/views.js | 184 ++++++++++++++++++ src/mmw/js/src/modeling/models.js | 11 +- src/mmw/js/src/modeling/views.js | 4 +- src/mmw/sass/pages/_model.scss | 8 +- 8 files changed, 277 insertions(+), 9 deletions(-) create mode 100644 src/mmw/js/src/modeling/gwlfe/runoff/templates/result.html create mode 100644 src/mmw/js/src/modeling/gwlfe/runoff/templates/selector.html create mode 100644 src/mmw/js/src/modeling/gwlfe/runoff/templates/table.html create mode 100644 src/mmw/js/src/modeling/gwlfe/runoff/views.js diff --git a/src/mmw/js/src/core/chart.js b/src/mmw/js/src/core/chart.js index da46c73b1..97551c0e8 100644 --- a/src/mmw/js/src/core/chart.js +++ b/src/mmw/js/src/core/chart.js @@ -246,7 +246,46 @@ function renderVerticalBarChart(chartEl, data, options) { }); } +// data is same format as for renderVerticalBarChart +function renderLineChart(chartEl, data, options) { + var chart = nv.models.lineChart(), + svg = makeSvg(chartEl); + + options = options || {}; + _.defaults(options, { + margin: {top: 20, right: 30, bottom: 40, left: 60} + }); + + nv.addGraph(function() { + chart.showLegend(false) + .margin(options.margin); + + chart.xAxis + .tickValues(options.xTickValues) + .tickFormat(function(month) { + return options.xAxisLabel(month); + }); + + chart.yAxis + .axisLabel(options.yAxisLabel) + .tickFormat(d3.format('.02f')); + + chart.tooltip.valueFormatter(function(d) { + return chart.yAxis.tickFormat()(d) + ' ' + options.yAxisUnit; + }); + + handleCommonOptions(chart, options); + + d3.select(svg) + .datum(data) + .call(chart); + + return chart; + }); +} + module.exports = { renderHorizontalBarChart: renderHorizontalBarChart, - renderVerticalBarChart: renderVerticalBarChart + renderVerticalBarChart: renderVerticalBarChart, + renderLineChart: renderLineChart }; diff --git a/src/mmw/js/src/modeling/gwlfe/runoff/templates/result.html b/src/mmw/js/src/modeling/gwlfe/runoff/templates/result.html new file mode 100644 index 000000000..233917277 --- /dev/null +++ b/src/mmw/js/src/modeling/gwlfe/runoff/templates/result.html @@ -0,0 +1,3 @@ +
    +
    +
    diff --git a/src/mmw/js/src/modeling/gwlfe/runoff/templates/selector.html b/src/mmw/js/src/modeling/gwlfe/runoff/templates/selector.html new file mode 100644 index 000000000..4db32f808 --- /dev/null +++ b/src/mmw/js/src/modeling/gwlfe/runoff/templates/selector.html @@ -0,0 +1,7 @@ + diff --git a/src/mmw/js/src/modeling/gwlfe/runoff/templates/table.html b/src/mmw/js/src/modeling/gwlfe/runoff/templates/table.html new file mode 100644 index 000000000..a7766a0f3 --- /dev/null +++ b/src/mmw/js/src/modeling/gwlfe/runoff/templates/table.html @@ -0,0 +1,28 @@ + + + + {% for columnName in columnNames %} + {% if loop.index0 == 0 %} + + {% else %} + + {% endif %} + {% endfor %} + + + + {% for row in rows %} + + {% for value in row %} + {% if loop.index0 == 0 %} + + {% else %} + + {% endif %} + {% endfor %} + + {% endfor %} + +
    + {{ columnName }} + {{ columnName }}
    {{ monthNames[value] }}{{ value|round(2)|toLocaleString }}
    diff --git a/src/mmw/js/src/modeling/gwlfe/runoff/views.js b/src/mmw/js/src/modeling/gwlfe/runoff/views.js new file mode 100644 index 000000000..63308de8d --- /dev/null +++ b/src/mmw/js/src/modeling/gwlfe/runoff/views.js @@ -0,0 +1,184 @@ +"use strict"; + +var $ = require('jquery'), + _ = require('underscore'), + Marionette = require('../../../../shim/backbone.marionette'), + chart = require('../../../core/chart.js'), + barChartTmpl = require('../../../core/templates/barChart.html'), + selectorTmpl = require('./templates/selector.html'), + resultTmpl = require('./templates/result.html'), + tableTmpl = require('./templates/table.html'); + +var runoffVars = [ + { name: 'AvPrecipitation', display: 'Precip' }, + { name: 'AvEvapoTrans', display: 'ET' }, + { name: 'AvRunoff', display: 'Surface Runoff' }, + { name: 'AvTileDrain', display: 'Subsurface Flow' }, + { name: 'AvPtSrcFlow', display: 'Point Src Flow' }, + { name: 'AvStreamFlow', display: 'Stream Flow' }, + ], + monthNames = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; + +var ResultView = Marionette.LayoutView.extend({ + className: 'tab-pane', + + template: resultTmpl, + + regions: { + selectorRegion: '.runoff-selector-region', + chartRegion: '.runoff-chart-region', + tableRegion: '.runoff-table-region' + }, + + modelEvents: { + 'change': 'onShow' + }, + + initialize: function(options) { + this.compareMode = options.compareMode; + this.scenario = options.scenario; + + this.model.set('activeVar', runoffVars[0].name); + }, + + onShow: function() { + this.selectorRegion.reset(); + this.tableRegion.reset(); + this.chartRegion.reset(); + + if (this.model.get('result')) { + if (!this.compareMode) { + this.tableRegion.show(new TableView({ + model: this.model, + })); + } + + this.selectorRegion.show(new SelectorView({ + model: this.model + })); + + this.chartRegion.show(new ChartView({ + model: this.model, + scenario: this.scenario, + compareMode: this.compareMode + })); + } + } +}); + +var SelectorView = Marionette.ItemView.extend({ + template: selectorTmpl, + + ui: { + runoffVarSelector: 'select' + }, + + events: { + 'change @ui.runoffVarSelector': 'updateRunoffVar' + }, + + modelEvents: { + 'change:activeVar': 'render' + }, + + updateRunoffVar: function() { + var activeRunoffVar = this.ui.runoffVarSelector.val(); + this.model.set('activeVar', activeRunoffVar); + }, + + templateHelpers: function() { + return { + runoffVars: runoffVars + }; + } +}); + +var ChartView = Marionette.ItemView.extend({ + template: barChartTmpl, + className: 'chart-container runoff-chart-container', + + initialize: function(options) { + this.scenario = options.scenario; + this.compareMode = options.compareMode; + }, + + onAttach: function() { + this.addChart(); + }, + + addChart: function() { + var chartEl = this.$el.find('.bar-chart').get(0), + result = this.model.get('result'), + activeVar = this.model.get('activeVar'); + + $(chartEl).empty(); + + if (result) { + var data = [ + { + key: _.findWhere(runoffVars, {name: activeVar}).display, + values: _.map(result.monthly, function(monthResult, monthInd) { + return { + x: monthInd, + y: Number(monthResult[activeVar]) + }; + }) + } + ], + chartOptions = { + yAxisLabel: 'Water Depth (cm)', + yAxisUnit: 'cm', + xAxisLabel: function(xValue) { + return monthNames[xValue]; + }, + xTickValues: _.range(12), + margin: this.compareMode ? + {top: 20, right: 120, bottom: 40, left: 60} : + {top: 20, right: 20, bottom: 40, left: 60} + }; + + chart.renderLineChart(chartEl, data, chartOptions); + } + } + +}); + +function monthSorter(a, b) { + if (a.month < b.month) { + return -1; + } else if (a.month > b.month) { + return 1; + } + return 0; +} + +window.monthSorter = monthSorter; + +var TableView = Marionette.CompositeView.extend({ + template: tableTmpl, + + onAttach: function() { + $('[data-toggle="table"]').bootstrapTable(); + }, + + templateHelpers: function() { + var columnNames = ['Month'].concat(_.map(runoffVars, function(runoffVar) { + return runoffVar.display; + })), + rows = _.map(this.model.get('result').monthly, function(month, i) { + return [i].concat(_.map(runoffVars, function(runoffVar) { + return Number(month[runoffVar.name]); + })); + }); + + return { + columnNames: columnNames, + rows: rows, + monthNames: monthNames, + }; + } +}); + +module.exports = { + ResultView: ResultView +}; diff --git a/src/mmw/js/src/modeling/models.js b/src/mmw/js/src/modeling/models.js index 39b3b835e..7a6e25fb5 100644 --- a/src/mmw/js/src/modeling/models.js +++ b/src/mmw/js/src/modeling/models.js @@ -61,7 +61,8 @@ var ResultModel = Backbone.Model.extend({ inputmod_hash: null, // MD5 string generated from result result: null, // The actual result object polling: false, // True if currently polling - active: false // True if currently selected in Compare UI + active: false, // True if currently selected in Compare UI + activeVar: null // For GWLFE, the currently selected variable in the UI } }); @@ -411,11 +412,11 @@ function _alterModifications(rawModifications) { var p = piece; if ( typeof p.value === 'string') { var v = {}; - if (p.name === reclass) { - v.reclass = p.value; + if (p.name === reclass) { + v.reclass = p.value; } - else if (p.name === bmp) { - v.bmp = p.value; + else if (p.name === bmp) { + v.bmp = p.value; } p.value = v; } diff --git a/src/mmw/js/src/modeling/views.js b/src/mmw/js/src/modeling/views.js index 1dfade654..d9570da18 100644 --- a/src/mmw/js/src/modeling/views.js +++ b/src/mmw/js/src/modeling/views.js @@ -25,8 +25,8 @@ scenarioToolbarTabContentTmpl = require('./templates/scenarioToolbarTabContent.html'), tr55RunoffViews = require('./tr55/runoff/views.js'), tr55QualityViews = require('./tr55/quality/views.js'), - gwlfeRunoffViews = tr55RunoffViews, // TODO actually make views for gwlfe - gwlfeQualityViews = tr55QualityViews; + gwlfeRunoffViews = require('./gwlfe/runoff/views.js'), + gwlfeQualityViews = tr55QualityViews; // TODO make gwlfe quality views var ENTER_KEYCODE = 13, ESCAPE_KEYCODE = 27; diff --git a/src/mmw/sass/pages/_model.scss b/src/mmw/sass/pages/_model.scss index 38dc9bd76..7db53d570 100644 --- a/src/mmw/sass/pages/_model.scss +++ b/src/mmw/sass/pages/_model.scss @@ -56,7 +56,13 @@ .runoff-chart-container.current-conditions { height: 300px; - width: 100% + width: 100%; + } + + .runoff-selector-region { + select { + width: 50%; + } } } From 17ddca15d6c39a4e14eeaf555082b22e8da8ea0e Mon Sep 17 00:00:00 2001 From: Hector Castro Date: Fri, 29 Apr 2016 14:06:59 -0400 Subject: [PATCH 029/136] Meet Paramiko dependency before installing Ansible The loose dependency on Paramiko is pulling in a version with a backwards incompatible change, which is causing the Ansible install to fail. Until Ansible addresses this issue, I'd rather explicitly install an older version of Paramiko that satisfies Ansible dependency requirements. See also: ansible/ansible#15665 --- deployment/packer/template.js | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment/packer/template.js b/deployment/packer/template.js index 2b6915b94..a18ed7230 100644 --- a/deployment/packer/template.js +++ b/deployment/packer/template.js @@ -111,6 +111,7 @@ "sleep 5", "sudo apt-get update -qq", "sudo apt-get install python-pip python-dev -y", + "sudo pip install paramiko==1.16.0", "sudo pip install ansible==2.0.1.0", "sudo /bin/sh -c 'echo {{user `version`}} > /srv/version.txt'" ] From d627e830d0e79a82f670d2d076e86294dd3851e1 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Mon, 2 May 2016 15:00:05 -0400 Subject: [PATCH 030/136] Move MapShed tasks, calcs into a module Since MapShed will require a number of Celery tasks to interface with the geoprocessing service, it is prudent to collect them into their own module. --- src/mmw/apps/modeling/mapshed/__init__.py | 0 .../modeling/{mapshed.py => mapshed/calcs.py} | 0 .../{ => mapshed}/data/sample_input.gms | 0 src/mmw/apps/modeling/mapshed/tasks.py | 93 +++++++++++++++++++ src/mmw/apps/modeling/tasks.py | 78 ---------------- src/mmw/apps/modeling/views.py | 9 +- 6 files changed, 98 insertions(+), 82 deletions(-) create mode 100644 src/mmw/apps/modeling/mapshed/__init__.py rename src/mmw/apps/modeling/{mapshed.py => mapshed/calcs.py} (100%) rename src/mmw/apps/modeling/{ => mapshed}/data/sample_input.gms (100%) create mode 100644 src/mmw/apps/modeling/mapshed/tasks.py diff --git a/src/mmw/apps/modeling/mapshed/__init__.py b/src/mmw/apps/modeling/mapshed/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/mmw/apps/modeling/mapshed.py b/src/mmw/apps/modeling/mapshed/calcs.py similarity index 100% rename from src/mmw/apps/modeling/mapshed.py rename to src/mmw/apps/modeling/mapshed/calcs.py diff --git a/src/mmw/apps/modeling/data/sample_input.gms b/src/mmw/apps/modeling/mapshed/data/sample_input.gms similarity index 100% rename from src/mmw/apps/modeling/data/sample_input.gms rename to src/mmw/apps/modeling/mapshed/data/sample_input.gms diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py new file mode 100644 index 000000000..f2ef663e0 --- /dev/null +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import absolute_import + +from os.path import join, dirname, abspath +from ast import literal_eval as make_tuple + +from celery import shared_task + +from gwlfe import gwlfe, parser +from gwlfe.datamodel import DataModel + +from django.conf import settings +from django.contrib.gis.geos import GEOSGeometry + +from apps.modeling.geoprocessing import sjs_submit, sjs_retrieve +from apps.modeling.mapshed.calcs import (day_lengths, + nearest_weather_stations, + growing_season, + erosion_coeff, + et_adjustment, + animal_energy_units, + manure_spread, + stream_length, + point_source_discharge, + weather_data, + ) + +ACRES_PER_SQM = 0.000247105 + + +@shared_task +def collect_data(geojson): + geom = GEOSGeometry(geojson, srid=4326) + area = geom.transform(5070, clone=True).area # Square Meters + + # Data Model is called z by convention + z = DataModel(settings.GWLFE_DEFAULTS) + + # Statically calculated lookup values + z.DayHrs = day_lengths(geom) + + # Data from the Weather Stations dataset + ws = nearest_weather_stations(geom) + z.Grow = growing_season(ws) + z.Acoef = erosion_coeff(ws, z.Grow) + z.PcntET = et_adjustment(ws) + z.WxYrBeg = max([w.begyear for w in ws]) + z.WxYrEnd = min([w.endyear for w in ws]) + z.WxYrs = z.WxYrEnd - z.WxYrBeg + 1 + + # Data from the County Animals dataset + livestock_aeu, poultry_aeu = animal_energy_units(geom) + z.AEU = livestock_aeu / (area * ACRES_PER_SQM) + z.n41j = livestock_aeu + z.n41k = poultry_aeu + z.n41l = livestock_aeu + poultry_aeu + + z.ManNitr, z.ManPhos = manure_spread(z.AEU) + + # Data from Streams dataset + z.StreamLength = stream_length(geom) # Meters + z.n42b = round(z.StreamLength / 1000) # Kilometers + + # Data from Point Source Discharge dataset + n_load, p_load, discharge = point_source_discharge(geom, area) + z.PointNitr = n_load + z.PointPhos = p_load + z.PointFlow = discharge + + # Data from National Weather dataset + temps, prcps = weather_data(ws, z.WxYrBeg, z.WxYrEnd) + z.Temp = temps + z.Prec = prcps + + # TODO pass real input to model instead of reading it from gms file + gms_filename = join(dirname(abspath(__file__)), 'data/sample_input.gms') + gms_file = open(gms_filename, 'r') + z = parser.GmsReader(gms_file).read() + + # The frontend expects an object with runoff and quality as keys. + response_json = {'runoff': gwlfe.run(z)} + + return response_json + + +@shared_task +def start_gwlfe_job(model_input): + geom = GEOSGeometry(json.dumps(model_input['area_of_interest']), + srid=4326) + + return collect_data.s(geom.geojson) diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index ffa3b1fbe..df477189f 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -8,31 +8,12 @@ import json import logging from math import sqrt -from os.path import join, dirname, abspath from apps.modeling.geoprocessing import histogram_start, histogram_finish, \ data_to_survey, data_to_censuses from tr55.model import simulate_day -from gwlfe import gwlfe, parser -from gwlfe.datamodel import DataModel - -from apps.modeling.mapshed import (day_lengths, - nearest_weather_stations, - growing_season, - erosion_coeff, - et_adjustment, - animal_energy_units, - manure_spread, - stream_length, - point_source_discharge, - weather_data, - ) - -from django.conf import settings -from django.contrib.gis.geos import GEOSGeometry - logger = logging.getLogger(__name__) KG_PER_POUND = 0.453592 @@ -40,65 +21,6 @@ ACRES_PER_SQM = 0.000247105 -@shared_task -def start_gwlfe_job(model_input): - """ - Runs a GWLFE job. - """ - aoi_geom = GEOSGeometry(json.dumps(model_input['area_of_interest']), - srid=4326) - aoi_area = aoi_geom.transform(5070, clone=True).area # Square Meters - - # Data Model is called z by convention - z = DataModel(settings.GWLFE_DEFAULTS) - - # Statically calculated lookup values - z.DayHrs = day_lengths(aoi_geom) - - # Data from the Weather Stations dataset - ws = nearest_weather_stations(aoi_geom) - z.Grow = growing_season(ws) - z.Acoef = erosion_coeff(ws, z.Grow) - z.PcntET = et_adjustment(ws) - z.WxYrBeg = max([w.begyear for w in ws]) - z.WxYrEnd = min([w.endyear for w in ws]) - z.WxYrs = z.WxYrEnd - z.WxYrBeg + 1 - - # Data from the County Animals dataset - livestock_aeu, poultry_aeu = animal_energy_units(aoi_geom) - z.AEU = livestock_aeu / (aoi_area * ACRES_PER_SQM) - z.n41j = livestock_aeu - z.n41k = poultry_aeu - z.n41l = livestock_aeu + poultry_aeu - - z.ManNitr, z.ManPhos = manure_spread(z.AEU) - - # Data from Streams dataset - z.StreamLength = stream_length(aoi_geom) # Meters - z.n42b = round(z.StreamLength / 1000) # Kilometers - - # Data from Point Source Discharge dataset - n_load, p_load, discharge = point_source_discharge(aoi_geom, aoi_area) - z.PointNitr = n_load - z.PointPhos = p_load - z.PointFlow = discharge - - # Data from National Weather dataset - temps, prcps = weather_data(ws, z.WxYrBeg, z.WxYrEnd) - z.Temp = temps - z.Prec = prcps - - # TODO pass real input to model instead of reading it from gms file - gms_filename = join(dirname(abspath(__file__)), 'data/sample_input.gms') - gms_file = open(gms_filename, 'r') - z = parser.GmsReader(gms_file).read() - - # The frontend expects an object with runoff and quality as keys. - response_json = {'runoff': gwlfe.run(z)} - - return response_json - - @shared_task def start_rwd_job(location, snapping): """ diff --git a/src/mmw/apps/modeling/views.py b/src/mmw/apps/modeling/views.py index 1370da2a3..d58285019 100644 --- a/src/mmw/apps/modeling/views.py +++ b/src/mmw/apps/modeling/views.py @@ -25,6 +25,7 @@ from apps.core.models import Job from apps.core.tasks import save_job_error, save_job_result from apps.modeling import tasks +from apps.modeling.mapshed.tasks import start_gwlfe_job from apps.modeling.models import Project, Scenario from apps.modeling.serializers import (ProjectSerializer, ProjectListingSerializer, @@ -216,10 +217,10 @@ def _initiate_gwlfe_job_chain(model_input, job_id): exchange = MAGIC_EXCHANGE routing_key = choose_worker() - return chain(tasks.start_gwlfe_job.s(model_input) - .set(exchange=exchange, routing_key=routing_key), - save_job_result.s(job_id, model_input)) \ - .apply_async(link_error=save_job_error.s(job_id)) + return (start_gwlfe_job(model_input).set(exchange=exchange, + routing_key=routing_key) | + save_job_result.s(job_id, model_input) + ).apply_async(link_error=save_job_error.s(job_id)) @decorators.api_view(['POST']) From 41cfbd7989df1b1fd3b7be2551e704ee1301be24 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Mon, 2 May 2016 15:05:01 -0400 Subject: [PATCH 031/136] Refactor settings to support multiple calls As we add multiple calls to the geoprocessing service, we refactor their save structure to support the them. --- src/mmw/apps/modeling/geoprocessing.py | 4 ++-- src/mmw/mmw/settings/base.py | 22 +++++++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/mmw/apps/modeling/geoprocessing.py b/src/mmw/apps/modeling/geoprocessing.py index c1cca93cb..a913418a1 100644 --- a/src/mmw/apps/modeling/geoprocessing.py +++ b/src/mmw/apps/modeling/geoprocessing.py @@ -128,8 +128,8 @@ def histogram_start(polygons): """ host = settings.GEOP['host'] port = settings.GEOP['port'] - args = settings.GEOP['args'] - data = settings.GEOP['request'].copy() + args = settings.GEOP['args']['SummaryJob'] + data = settings.GEOP['json']['nlcdSoilCensus'].copy() data['input']['geometry'] = polygons return sjs_submit(host, port, args, data) diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index 1905107c1..eef6b0ebb 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -362,15 +362,19 @@ def get_env_setting(setting): GEOP = { 'host': environ.get('MMW_GEOPROCESSING_HOST', 'localhost'), 'port': environ.get('MMW_GEOPROCESSING_PORT', '8090'), - 'args': 'context=geoprocessing&appName=geoprocessing-%s&classPath=org.wikiwatershed.mmw.geoprocessing.SummaryJob' % environ.get('MMW_GEOPROCESSING_VERSION', '0.1.0'), # NOQA - 'request': { - 'input': { - 'geometry': None, - 'tileCRS': 'ConusAlbers', - 'polyCRS': 'LatLng', - 'nlcdLayer': 'nlcd-2011-30m-epsg5070-0.10.0', - 'soilLayer': 'ssurgo-hydro-groups-30m-epsg5070-0.10.0', - 'zoom': 0 + 'args': { + 'SummaryJob': 'context=geoprocessing&appName=geoprocessing-%s&classPath=org.wikiwatershed.mmw.geoprocessing.SummaryJob' % environ.get('MMW_GEOPROCESSING_VERSION', '0.1.0'), + }, + 'json': { + 'nlcdSoilCensus': { + 'input': { + 'geometry': None, + 'tileCRS': 'ConusAlbers', + 'polyCRS': 'LatLng', + 'nlcdLayer': 'nlcd-2011-30m-epsg5070-0.10.0', + 'soilLayer': 'ssurgo-hydro-groups-30m-epsg5070-0.10.0', + 'zoom': 0 + } } } } From b6b9119be672c686d524667fac1e477d35dbc779 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Mon, 2 May 2016 15:11:02 -0400 Subject: [PATCH 032/136] Add MapShed SJS tasks Given an operation name and input data, mapshed_start method will look up geoprocessing service settings for that operation, hydrate the template with the input data, and send that request. Then, mapshed_finish method will look up the results, and will transform the output of the service into a format that can easily be used. --- src/mmw/apps/modeling/mapshed/tasks.py | 26 ++++++++++++++++++++++++++ src/mmw/mmw/settings/base.py | 1 + 2 files changed, 27 insertions(+) diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index f2ef663e0..729a51437 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -30,6 +30,32 @@ ACRES_PER_SQM = 0.000247105 +@shared_task +def mapshed_start(opname, input_data): + host = settings.GEOP['host'] + port = settings.GEOP['port'] + args = settings.GEOP['args']['MapshedJob'] + + data = settings.GEOP['json'][opname].copy() + data['input'].update(input_data) + + job_id = sjs_submit(host, port, args, data) + + return { + 'host': host, + 'port': port, + 'job_id': job_id + } + + +@shared_task(bind=True, default_retry_delay=1, max_retries=42) +def mapshed_finish(self, incoming): + sjs_result = sjs_retrieve(retry=self.retry, **incoming) + + # Convert string "List(1,2,3)" into tuple (1,2,3) for each key + return {make_tuple(key[4:]): val for key, val in sjs_result.items()} + + @shared_task def collect_data(geojson): geom = GEOSGeometry(geojson, srid=4326) diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index eef6b0ebb..04f80c2a0 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -364,6 +364,7 @@ def get_env_setting(setting): 'port': environ.get('MMW_GEOPROCESSING_PORT', '8090'), 'args': { 'SummaryJob': 'context=geoprocessing&appName=geoprocessing-%s&classPath=org.wikiwatershed.mmw.geoprocessing.SummaryJob' % environ.get('MMW_GEOPROCESSING_VERSION', '0.1.0'), + 'MapshedJob': 'context=geoprocessing&appName=geoprocessing-%s&classPath=org.wikiwatershed.mmw.geoprocessing.MapshedJob' % environ.get('MMW_GEOPROCESSING_VERSION', '0.1.0') }, 'json': { 'nlcdSoilCensus': { From d1d7b8721a08be76f9d4b470d26cfeb98a469f77 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Mon, 2 May 2016 15:15:06 -0400 Subject: [PATCH 033/136] Get streams from database In order to overlay streams with a raster dataset such as NLCD, we need to send the vector streams to the geoprocessing service. This method finds all the streams contained within the polygon. The result can contain LineStrings and MultiLineStrings. The query also returns some empty GeometryCollections, which we exclude with `NOT ST_IsEmpty`. --- src/mmw/apps/modeling/mapshed/calcs.py | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index 4db9caaa5..44cc8e912 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -203,6 +203,33 @@ def stream_length(geom, drb=False): return cursor.fetchone()[0] or 0 # Aggregate query returns singleton +def streams(geom, drb=False): + """ + Given a geometry, returns a list of GeoJSON objects, either LineStrings or + MultiLineStrings, representing the set of streams contained inside the + geometry, in LatLng. If the drb flag is set, we use the Delaware River + Basin dataset instead of NHD Flowline. + """ + sql = ''' + WITH clipped_streams AS ( + SELECT ST_Intersection(geom, + ST_SetSRID(ST_GeomFromText(%s), 4326)) + AS stream + FROM {datasource} + WHERE ST_Intersects(geom, + ST_SetSRID(ST_GeomFromText(%s), 4326)) + ) + SELECT ST_AsGeoJSON(ST_Force2D(stream)) + FROM clipped_streams + WHERE NOT ST_IsEmpty(stream) + '''.format(datasource='drb_streams_50' if drb else 'nhdflowline') + + with connection.cursor() as cursor: + cursor.execute(sql, [geom.wkt, geom.wkt]) + + return [row[0] for row in cursor.fetchall()] # List of GeoJSON strings + + def point_source_discharge(geom, area): """ Given a geometry and its area in square meters, returns three lists, From e3fd6e35281c11be6eaa362d4d74b7a3cd212843 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Mon, 2 May 2016 15:27:04 -0400 Subject: [PATCH 034/136] Add geop tasks to chain, derive AgLength We add geop_tasks that given a geometry returns a group of Celery chains, each corresponding to one combination of rasters and vectors. Each chain is a sequence of mapshed_start, mapshed_finish and some post-processing of the results. The first task we add is nlcd_streams, which combines NLCD raster with Streams vector data, and calculates the length of streams in agricultural NLCD codes (81 and 82). We also add a dummy identity task which alleviates an issue with Celery which causes problems when a chord is the penultimate task in a chain. --- src/mmw/apps/modeling/mapshed/tasks.py | 56 ++++++++++++++++++++++++-- src/mmw/mmw/settings/base.py | 14 +++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index 729a51437..4b589f155 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -6,7 +6,7 @@ from os.path import join, dirname, abspath from ast import literal_eval as make_tuple -from celery import shared_task +from celery import shared_task, group from gwlfe import gwlfe, parser from gwlfe.datamodel import DataModel @@ -22,6 +22,7 @@ et_adjustment, animal_energy_units, manure_spread, + streams, stream_length, point_source_discharge, weather_data, @@ -57,7 +58,7 @@ def mapshed_finish(self, incoming): @shared_task -def collect_data(geojson): +def collect_data(geop_results, geojson): geom = GEOSGeometry(geojson, srid=4326) area = geom.transform(5070, clone=True).area # Square Meters @@ -100,6 +101,12 @@ def collect_data(geojson): z.Temp = temps z.Prec = prcps + # Flatten geoprocessing results into one dictionary + geop_result = {k: v for r in geop_results for k, v in r.items()} + + z.AgLength = geop_result['AgStreamPct'] * z.StreamLength + z.UrbLength = z.StreamLength - z.AgLength + # TODO pass real input to model instead of reading it from gms file gms_filename = join(dirname(abspath(__file__)), 'data/sample_input.gms') gms_file = open(gms_filename, 'r') @@ -116,4 +123,47 @@ def start_gwlfe_job(model_input): geom = GEOSGeometry(json.dumps(model_input['area_of_interest']), srid=4326) - return collect_data.s(geom.geojson) + return (group(geop_tasks(geom)) | + identity.s() | + collect_data.s(geom.geojson)) + + +@shared_task +def nlcd_streams(result): + ag_streams = sum(result.get(nlcd, 0) for nlcd in [81, 82]) + total = sum(result.values()) + + ag_stream_percent = ag_streams / total + + return { + 'AgStreamPct': ag_stream_percent + } + + +def geop_tasks(geom): + # List of tuples of (opname, data, callback) for each geop task + definitions = [ + ('nlcd_streams', + {'polygon': [geom.geojson], 'vector': streams(geom)}, + nlcd_streams), + # TODO Remove this second dummy call with actual geop task + # We need at least two for the flatten to work correctly. + ('nlcd_streams', + {'polygon': [geom.geojson], 'vector': streams(geom)}, + nlcd_streams) + ] + + return [(mapshed_start.s(opname, data) | + mapshed_finish.s() | + callback.s()) + for (opname, data, callback) in definitions] + + +@shared_task +def identity(x): + """ + Simple identity function that returns its argument. + Used as a buffer in a chord as a workaround to + https://github.com/celery/celery/issues/3191 + """ + return x diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index 04f80c2a0..8c2c73eb2 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -375,6 +375,20 @@ def get_env_setting(setting): 'nlcdLayer': 'nlcd-2011-30m-epsg5070-0.10.0', 'soilLayer': 'ssurgo-hydro-groups-30m-epsg5070-0.10.0', 'zoom': 0 + }, + }, + 'nlcd_streams': { + 'input': { + 'polygon': [], + 'polygonCRS': 'LatLng', + 'vector': [], + 'vectorCRS': 'LatLng', + 'rasters': [ + 'nlcd-2011-30m-epsg5070-0.10.0' + ], + 'rasterCRS': 'ConusAlbers', + 'operationType': 'RasterLinesJoin', + 'zoom': 0 } } } From 0223088abd522bdb31e2302804a6754aef114b01 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 4 May 2016 17:48:29 -0400 Subject: [PATCH 035/136] Celery Errors: More descriptive geop errors If the geoprocessing service returns an error, we now include that error message in the exception as well. This makes debugging them considerably easier, since one no longer need to inspect Spark JobServer with a particular Job ID to get the details. That can still be done to see a full stack trace. --- src/mmw/apps/modeling/geoprocessing.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/mmw/apps/modeling/geoprocessing.py b/src/mmw/apps/modeling/geoprocessing.py index a913418a1..9426524d7 100644 --- a/src/mmw/apps/modeling/geoprocessing.py +++ b/src/mmw/apps/modeling/geoprocessing.py @@ -105,15 +105,18 @@ def sjs_retrieve(host, port, job_id, retry=None): raise Exception('Job {} timed out, unable to delete.\n' 'Details: {}'.format(job_id, delete.text)) else: + if job['status'] == 'ERROR': + status = 'ERROR ({}: {})'.format(job['result']['errorClass'], + job['result']['message']) + else: + status = job['status'] + delete = requests.delete(url) # Job in unusual state, terminate if delete.ok: - raise Exception('Job {} was {}, deleted'.format(job_id, - job['status'])) + raise Exception('Job {} was {}, deleted'.format(job_id, status)) else: raise Exception('Job {} was {}, could not delete.\n' - 'Details = {}'.format(job_id, - job['status'], - delete.text)) + 'Details = {}'.format(job_id, status, delete.text)) @statsd.timer(__name__ + '.histogram_start') From 14af08a244d89ea9c8b9e5504bd2bf48721636bf Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 4 May 2016 17:52:41 -0400 Subject: [PATCH 036/136] Celery Errors: Limited Chord Unlocks In cases where one has a group of chains as a chord header, and a task in the middle of a chain raises an Exception, the final task in the chain would not execute and thus not be marked as "ready", and thus the group would not be marked as "ready". Thus, the chord would attempt to retry indefinitely. Unfortunately, this indefinite retrial is hard-coded into Celery. Thus, we shim the chord_unlock function with a variant that takes the configurable CELERY_CHORD_UNLOCK_MAX_RETRIES variable. This will ensure that chord unlocking would not proceed infinitely when a chain in a group in the chord header fails. This overridden method must be defined before a Celery instance is created, thus the placing of the method at the beginning of the file. --- src/mmw/mmw/celery.py | 76 ++++++++++++++++++++++++++++++++++++ src/mmw/mmw/settings/base.py | 2 + 2 files changed, 78 insertions(+) diff --git a/src/mmw/mmw/celery.py b/src/mmw/mmw/celery.py index 2e1db1b6a..fbe620421 100644 --- a/src/mmw/mmw/celery.py +++ b/src/mmw/mmw/celery.py @@ -2,12 +2,88 @@ import os import rollbar +import logging from celery import Celery +from celery._state import connect_on_app_finalize from celery.signals import task_failure from django.conf import settings + +@connect_on_app_finalize +def add_unlock_chord_task_shim(app): + """ + Override native unlock_chord to support configurable max_retries. + Original code taken from https://goo.gl/3mX0ie + + This task is used by result backends without native chord support. + It joins chords by creating a task chain polling the header for completion. + """ + from celery.canvas import maybe_signature + from celery.exceptions import ChordError + from celery.result import allow_join_result, result_from_tuple + + logger = logging.getLogger(__name__) + + MAX_RETRIES = settings.CELERY_CHORD_UNLOCK_MAX_RETRIES + + @app.task(name='celery.chord_unlock', shared=False, default_retry_delay=1, + ignore_result=True, lazy=False, bind=True, + max_retries=MAX_RETRIES) + def unlock_chord(self, group_id, callback, interval=None, + max_retries=MAX_RETRIES, result=None, + Result=app.AsyncResult, GroupResult=app.GroupResult, + result_from_tuple=result_from_tuple, **kwargs): + if interval is None: + interval = self.default_retry_delay + + # check if the task group is ready, and if so apply the callback. + callback = maybe_signature(callback, app) + deps = GroupResult( + group_id, + [result_from_tuple(r, app=app) for r in result], + app=app, + ) + j = deps.join_native if deps.supports_native_join else deps.join + + try: + ready = deps.ready() + except Exception as exc: + raise self.retry( + exc=exc, countdown=interval, max_retries=max_retries, + ) + else: + if not ready: + raise self.retry(countdown=interval, max_retries=max_retries) + + callback = maybe_signature(callback, app=app) + try: + with allow_join_result(): + ret = j(timeout=3.0, propagate=True) + except Exception as exc: + try: + culprit = next(deps._failed_join_report()) + reason = 'Dependency {0.id} raised {1!r}'.format( + culprit, exc, + ) + except StopIteration: + reason = repr(exc) + logger.error('Chord %r raised: %r', group_id, exc, exc_info=1) + app.backend.chord_error_from_stack(callback, + ChordError(reason)) + else: + try: + callback.delay(ret) + except Exception as exc: + logger.error('Chord %r raised: %r', group_id, exc, exc_info=1) + app.backend.chord_error_from_stack( + callback, + exc=ChordError('Callback error: {0!r}'.format(exc)), + ) + return unlock_chord + + # set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mmw.settings.production') diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index 8c2c73eb2..afb46e14a 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -124,6 +124,8 @@ def get_env_setting(setting): STATSD_CELERY_SIGNALS = True CELERY_WORKER_DIRECT = True CELERY_CREATE_MISSING_QUEUES = True +CELERY_CHORD_PROPAGATES = True +CELERY_CHORD_UNLOCK_MAX_RETRIES = 3 # END CELERY CONFIGURATION From 84e24e02e6290c1c92093a16d9440464207923b3 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 4 May 2016 18:00:20 -0400 Subject: [PATCH 037/136] Celery Errors: Promptly mark jobs as failed While the previous commits take care of failings of Celery, they do not trigger the save_job_error handler. Thus, the job is not marked as failed and the front-end keeps polling until it reaches its own timeout. Apparently, specifying a link_error to the final chain is not good enough to catch errors in nested chains / chords / groups. Thus, we edit our chain definition as follows: * start_gwlfe_job has been replaced by its contents, so there is one less level if indirection. * The error handler is also passed to geop_tasks, which in turn applies it to the last task in each chain. This ensures that as soon as one of the chains fail, the job is marked as such and the front-end can stop polling. * Earlier tasks in the geop chains handle exceptions, and forward them to the next task. It is the responsibility of the final task to inspect the result for the 'error' key, and raise an exception if found. By convention, the exception should be pre- fixed with the task's name for easy identification in the logs. --- src/mmw/apps/modeling/mapshed/tasks.py | 54 +++++++++++++++----------- src/mmw/apps/modeling/views.py | 18 ++++++--- 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index 4b589f155..515dd1763 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -6,7 +6,7 @@ from os.path import join, dirname, abspath from ast import literal_eval as make_tuple -from celery import shared_task, group +from celery import shared_task from gwlfe import gwlfe, parser from gwlfe.datamodel import DataModel @@ -40,21 +40,36 @@ def mapshed_start(opname, input_data): data = settings.GEOP['json'][opname].copy() data['input'].update(input_data) - job_id = sjs_submit(host, port, args, data) + try: + job_id = sjs_submit(host, port, args, data) - return { - 'host': host, - 'port': port, - 'job_id': job_id - } + return { + 'host': host, + 'port': port, + 'job_id': job_id + } + + except Exception as x: + return { + 'error': x.message + } @shared_task(bind=True, default_retry_delay=1, max_retries=42) def mapshed_finish(self, incoming): - sjs_result = sjs_retrieve(retry=self.retry, **incoming) + if 'error' in incoming: + return incoming - # Convert string "List(1,2,3)" into tuple (1,2,3) for each key - return {make_tuple(key[4:]): val for key, val in sjs_result.items()} + try: + sjs_result = sjs_retrieve(retry=self.retry, **incoming) + + # Convert string "List(1,2,3)" into tuple (1,2,3) for each key + return {make_tuple(key[4:]): val for key, val in sjs_result.items()} + + except Exception as x: + return { + 'error': x.message + } @shared_task @@ -118,18 +133,11 @@ def collect_data(geop_results, geojson): return response_json -@shared_task -def start_gwlfe_job(model_input): - geom = GEOSGeometry(json.dumps(model_input['area_of_interest']), - srid=4326) - - return (group(geop_tasks(geom)) | - identity.s() | - collect_data.s(geom.geojson)) - - -@shared_task +@shared_task(throws=Exception) def nlcd_streams(result): + if 'error' in result: + raise Exception('[nlcd_streams] {}'.format(result['error'])) + ag_streams = sum(result.get(nlcd, 0) for nlcd in [81, 82]) total = sum(result.values()) @@ -140,7 +148,7 @@ def nlcd_streams(result): } -def geop_tasks(geom): +def geop_tasks(geom, errback): # List of tuples of (opname, data, callback) for each geop task definitions = [ ('nlcd_streams', @@ -155,7 +163,7 @@ def geop_tasks(geom): return [(mapshed_start.s(opname, data) | mapshed_finish.s() | - callback.s()) + callback.s().set(link_error=errback)) for (opname, data, callback) in definitions] diff --git a/src/mmw/apps/modeling/views.py b/src/mmw/apps/modeling/views.py index d58285019..7a83a4032 100644 --- a/src/mmw/apps/modeling/views.py +++ b/src/mmw/apps/modeling/views.py @@ -18,14 +18,14 @@ from django.contrib.gis.geos import GEOSGeometry import celery -from celery import chain +from celery import chain, group from retry import retry from apps.core.models import Job from apps.core.tasks import save_job_error, save_job_result from apps.modeling import tasks -from apps.modeling.mapshed.tasks import start_gwlfe_job +from apps.modeling.mapshed.tasks import geop_tasks, collect_data from apps.modeling.models import Project, Scenario from apps.modeling.serializers import (ProjectSerializer, ProjectListingSerializer, @@ -216,11 +216,17 @@ def start_gwlfe(request, format=None): def _initiate_gwlfe_job_chain(model_input, job_id): exchange = MAGIC_EXCHANGE routing_key = choose_worker() + errback = save_job_error.s(job_id) - return (start_gwlfe_job(model_input).set(exchange=exchange, - routing_key=routing_key) | - save_job_result.s(job_id, model_input) - ).apply_async(link_error=save_job_error.s(job_id)) + geom = GEOSGeometry(json.dumps(model_input['area_of_interest']), + srid=4326) + + chain = (group(geop_tasks(geom, errback)).set(exchange=exchange, + routing_key=routing_key) | + collect_data.s(geom.geojson).set(link_error=errback) | + save_job_result.s(job_id, model_input)) + + return chain.apply_async(link_error=errback) @decorators.api_view(['POST']) From aa8c192e4b838badb7c47f32281a6c18d0d0ec22 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 4 May 2016 18:54:00 -0400 Subject: [PATCH 038/136] Celery Errors: Buffer handling chord errors Attaching an error handler to a chord's body causes weird errors, so we place a simple buffer between geop_tasks and collect_data, making it not the chord body and thus handle errors correctly. At first I used the simple identity function as the buffer, but since we need to flatten the result of geop_tasks anyway, the simple method was updated to do something useful instead. --- src/mmw/apps/modeling/mapshed/tasks.py | 16 ++++++++-------- src/mmw/apps/modeling/views.py | 3 ++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index 515dd1763..14e5609c9 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -73,7 +73,7 @@ def mapshed_finish(self, incoming): @shared_task -def collect_data(geop_results, geojson): +def collect_data(geop_result, geojson): geom = GEOSGeometry(geojson, srid=4326) area = geom.transform(5070, clone=True).area # Square Meters @@ -116,9 +116,6 @@ def collect_data(geop_results, geojson): z.Temp = temps z.Prec = prcps - # Flatten geoprocessing results into one dictionary - geop_result = {k: v for r in geop_results for k, v in r.items()} - z.AgLength = geop_result['AgStreamPct'] * z.StreamLength z.UrbLength = z.StreamLength - z.AgLength @@ -168,10 +165,13 @@ def geop_tasks(geom, errback): @shared_task -def identity(x): +def combine(geop_results): """ - Simple identity function that returns its argument. - Used as a buffer in a chord as a workaround to + Flattens the incoming results dictionaries into one + which has all the keys of the components. + + This could be a part of collect_data, but we need + a buffer in a chord as a workaround to https://github.com/celery/celery/issues/3191 """ - return x + return {k: v for r in geop_results for k, v in r.items()} diff --git a/src/mmw/apps/modeling/views.py b/src/mmw/apps/modeling/views.py index 7a83a4032..0a22e5f22 100644 --- a/src/mmw/apps/modeling/views.py +++ b/src/mmw/apps/modeling/views.py @@ -25,7 +25,7 @@ from apps.core.models import Job from apps.core.tasks import save_job_error, save_job_result from apps.modeling import tasks -from apps.modeling.mapshed.tasks import geop_tasks, collect_data +from apps.modeling.mapshed.tasks import geop_tasks, collect_data, combine from apps.modeling.models import Project, Scenario from apps.modeling.serializers import (ProjectSerializer, ProjectListingSerializer, @@ -223,6 +223,7 @@ def _initiate_gwlfe_job_chain(model_input, job_id): chain = (group(geop_tasks(geom, errback)).set(exchange=exchange, routing_key=routing_key) | + combine.s() | collect_data.s(geom.geojson).set(link_error=errback) | save_job_result.s(job_id, model_input)) From f7ee11ab6c830afbb026b160b54f2a6c4405c87b Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 4 May 2016 20:18:37 -0400 Subject: [PATCH 039/136] Celery Errors: Don't trap retries Retrying fires an exception so that no further code gets executed after the retry. Unfortunately, the Retry was being captured in the except block, causing false errors. Instead, it is now re- raised, absorbed by the Celery Worker as the proper signal. See http://docs.celeryq.org/en/latest/userguide/tasks.html#retrying for details. --- src/mmw/apps/modeling/mapshed/tasks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index 14e5609c9..64ee63eb2 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -7,6 +7,7 @@ from ast import literal_eval as make_tuple from celery import shared_task +from celery.exceptions import Retry from gwlfe import gwlfe, parser from gwlfe.datamodel import DataModel @@ -66,6 +67,8 @@ def mapshed_finish(self, incoming): # Convert string "List(1,2,3)" into tuple (1,2,3) for each key return {make_tuple(key[4:]): val for key, val in sjs_result.items()} + except Retry as r: + raise r except Exception as x: return { 'error': x.message From ba5e569ab872ed5dade01108aa04252dd0937f3a Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 4 May 2016 23:49:20 -0400 Subject: [PATCH 040/136] Celery Errors: Explain, record rationale --- src/mmw/apps/modeling/mapshed/tasks.py | 35 ++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index 64ee63eb2..446d4a0f9 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -68,6 +68,11 @@ def mapshed_finish(self, incoming): return {make_tuple(key[4:]): val for key, val in sjs_result.items()} except Retry as r: + # Celery throws a Retry exception when self.retry is called to stop + # the execution of any further code, and to indicate to the worker + # that the same task is going to be retried. + # We capture and re-raise Retry to continue this behavior, and ensure + # that it doesn't get passed to the next task like every other error. raise r except Exception as x: return { @@ -135,6 +140,36 @@ def collect_data(geop_result, geojson): @shared_task(throws=Exception) def nlcd_streams(result): + """ + From a dictionary mapping NLCD codes to the count of stream pixels on + each, return a dictionary with a key 'AgStreamPct' which indicates the + percent of streams in agricultural areas, namely NLCD 81 Pasture/Hay + and 82 Cultivated Crops. + + In addition, we inspect the result to see if it includes an 'error' key. + If so, it would indicate that a preceeding task has thrown an exception, + and thus we throw an exception with that message. + + We throw the actual exception in this final task in the chain, rather than + one of the preceeding ones, because when creating a chain the resulting + AsyncResult points to the last task in the chain. If task in the middle of + the chain fails, and the last task doesn't run, the AsyncResult is never + marked as Ready. In the case of pure chains this can be addressed with a + simple link_error, but in the case of chords the entire set of tasks in + the chord's header needs to be Ready before the body can execute. Thus, if + the final task never runs, Celery repeatedly calls chord_unlock infinitely + causing overflows. By throwing the exception in the final task instead of + an intermediate one, we ensure that the group is marked as Ready, and the + chord is notified of the failure and suspends execution gracefully. + + Furthermore, this task is decorated with 'throws=Exception' so that the + exception is logged as INFO, rather than ERROR. This is because we have + already logged it as an ERROR the first time it was thrown up the chain, + and this will reduce noise in the logs. + + This task should be used as a template for making other geoprocessing + post-processing tasks, to be used in geop_tasks. + """ if 'error' in result: raise Exception('[nlcd_streams] {}'.format(result['error'])) From 25d9c673c7e784a623da68411db3c0f8f8343ee1 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 5 May 2016 12:07:45 -0400 Subject: [PATCH 041/136] Upgrade Geoprocessing to 1.1.0 This adds support for MapshedJob which will be used for calculating various values needed for running GWLF-E. --- deployment/ansible/group_vars/all | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/ansible/group_vars/all b/deployment/ansible/group_vars/all index d7abf19cc..8956e9b32 100644 --- a/deployment/ansible/group_vars/all +++ b/deployment/ansible/group_vars/all @@ -44,7 +44,7 @@ sjs_host: "localhost" sjs_port: 8090 sjs_container_image: "quay.io/azavea/spark-jobserver:0.6.1" -geop_version: "1.0.0" +geop_version: "1.1.0" nginx_cache_dir: "/var/cache/nginx" observation_api_url: "http://www.wikiwatershed-vs.org/" From 8bd52a983a714cb261f6a5894aee1f25a267138c Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 6 May 2016 18:40:48 -0400 Subject: [PATCH 042/136] Parse SJS Results in post-processing step Since the SJS results may contain tuples as keys which cannot be transferred between Celery tasks as they are not JSON serializable we switch to parsing them in the post-processing step instead. --- src/mmw/apps/modeling/mapshed/tasks.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index 446d4a0f9..ff24e3575 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -62,10 +62,7 @@ def mapshed_finish(self, incoming): return incoming try: - sjs_result = sjs_retrieve(retry=self.retry, **incoming) - - # Convert string "List(1,2,3)" into tuple (1,2,3) for each key - return {make_tuple(key[4:]): val for key, val in sjs_result.items()} + return sjs_retrieve(retry=self.retry, **incoming) except Retry as r: # Celery throws a Retry exception when self.retry is called to stop @@ -139,7 +136,7 @@ def collect_data(geop_result, geojson): @shared_task(throws=Exception) -def nlcd_streams(result): +def nlcd_streams(sjs_result): """ From a dictionary mapping NLCD codes to the count of stream pixels on each, return a dictionary with a key 'AgStreamPct' which indicates the @@ -170,8 +167,13 @@ def nlcd_streams(result): This task should be used as a template for making other geoprocessing post-processing tasks, to be used in geop_tasks. """ - if 'error' in result: - raise Exception('[nlcd_streams] {}'.format(result['error'])) + if 'error' in sjs_result: + raise Exception('[nlcd_streams] {}'.format(sjs_result['error'])) + + # Parse SJS results + # This can't be done in mapshed_finish because the keys may be tuples, + # which are not JSON serializable and thus can't be shared between tasks + result = parse_sjs_result(sjs_result) ag_streams = sum(result.get(nlcd, 0) for nlcd in [81, 82]) total = sum(result.values()) @@ -213,3 +215,8 @@ def combine(geop_results): https://github.com/celery/celery/issues/3191 """ return {k: v for r in geop_results for k, v in r.items()} + + +def parse_sjs_result(sjs_result): + # Convert string "List(1,2,3)" into tuple (1,2,3) for each key + return {make_tuple(key[4:]): val for key, val in sjs_result.items()} From ab570c3d5b697ea0769bcd5f83c8f0e6b1a59a98 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 6 May 2016 19:39:52 -0400 Subject: [PATCH 043/136] Calculate Curve Number and Sediment Phosphorus --- src/mmw/apps/modeling/mapshed/calcs.py | 75 ++++++++++++++++++++++ src/mmw/apps/modeling/mapshed/tasks.py | 52 +++++++++++++-- src/mmw/mmw/settings/base.py | 16 ++++- src/mmw/mmw/settings/gwlfe_settings.py | 88 ++++++++++++++++++++++++++ 4 files changed, 224 insertions(+), 7 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index 44cc8e912..5ece353e5 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -20,6 +20,7 @@ POULTRY = settings.GWLFE_CONFIG['Poultry'] LITERS_PER_MGAL = 3785412 WEATHER_NULL = settings.GWLFE_CONFIG['WeatherNull'] +AG_NLCD_CODES = settings.GWLFE_CONFIG['AgriculturalNLCDCodes'] def day_lengths(geom): @@ -326,3 +327,77 @@ def weather_data(ws, begyear, endyear): prcps[year, month, day] = p if p != WEATHER_NULL else None return temps, prcps + + +def curve_number(n_count, ng_count): + """ + Given a dictionary mapping NLCD codes to counts of cells, and another + mapping pairs of NLCD codes and Hydrological Soil Groups, returns the + curve number for each non-urban MapShed land use type by calculating the + average hydrological soil group for each land use type, rounded to the + nearest integer, and looking up the value in the CURVE_NUMBER table. + + Original at Class1.vb@1.3.0:7257-7267 + """ + + # Calculate average hydrological soil group for each NLCD type by + # reducing [(n, g): c] to [n: avg(g * c)] + n_gavg = {} + for (n, g), count in ng_count.iteritems(): + n_gavg[n] = float(g) * count / n_count[n] + n_gavg.get(n, 0) + + def cni(nlcd): + # Helper method to lookup values from CURVE_NUMBER table + return settings.CURVE_NUMBER[nlcd][int(round(n_gavg.get(nlcd, 0)))] + + def cni_avg(nlcds): + # Helper method to average non-zero values only + vals = [cni(nlcd) for nlcd in nlcds] + sum_vals = sum(vals) + nonzero_vals = len([v for v in vals if v > 0]) + + return float(sum_vals) / nonzero_vals if nonzero_vals > 0 else 0 + + return [ + cni(81), # Hay/Pasture + cni(82), # Cropland + cni_avg([41, 42, 43, 52]), # Forest + cni_avg([90, 95]), # Wetland + 0, # Disturbed + 0, # Turf Grass + cni_avg([21, 71]), # Open Land + cni_avg([12, 31]), # Bare Rock + 0, # Sandy Areas + 0, # Unpaved Road + 0, 0, 0, 0, 0, 0 # Urban Land Use Types + ] + + +def sediment_phosphorus(nt_count): + """ + Given a dictionary mapping pairs of NLCD Codes and Soil Textures to counts + of cells, returns the average concentration of phosphorus in the soil by + looking up the value in the SOILP table. NLCD Codes 81 and 82 correspond to + agricultural values, and the rest to non-agricultural ones. + + Original at Class1.vb@1.3.0:8975-8988 + """ + + ag_textures = {} + nag_textures = {} + total = sum(nt_count.values()) + + for (n, t), count in nt_count.iteritems(): + if n in AG_NLCD_CODES: + ag_textures[t] = count + ag_textures.get(t, 0) + else: + nag_textures[t] = count + nag_textures.get(t, 0) + + ag_sedp = sum(count * settings.SOILP[t][0] + for t, count in ag_textures.iteritems()) + nag_sedp = sum(count * settings.SOILP[t][1] + for t, count in nag_textures.iteritems()) + + sedp = float(ag_sedp + nag_sedp) / total + + return sedp * 1.6 diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index ff24e3575..dae17d62c 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals from __future__ import absolute_import +import numpy as np + from os.path import join, dirname, abspath from ast import literal_eval as make_tuple @@ -27,8 +29,11 @@ stream_length, point_source_discharge, weather_data, + curve_number, + sediment_phosphorus, ) +AG_NLCD_CODES = settings.GWLFE_CONFIG['AgriculturalNLCDCodes'] ACRES_PER_SQM = 0.000247105 @@ -124,6 +129,9 @@ def collect_data(geop_result, geojson): z.AgLength = geop_result['AgStreamPct'] * z.StreamLength z.UrbLength = z.StreamLength - z.AgLength + z.CN = np.array(geop_result['cn']) + z.SedPhos = geop_result['sed_phos'] + # TODO pass real input to model instead of reading it from gms file gms_filename = join(dirname(abspath(__file__)), 'data/sample_input.gms') gms_file = open(gms_filename, 'r') @@ -175,7 +183,7 @@ def nlcd_streams(sjs_result): # which are not JSON serializable and thus can't be shared between tasks result = parse_sjs_result(sjs_result) - ag_streams = sum(result.get(nlcd, 0) for nlcd in [81, 82]) + ag_streams = sum(result.get(nlcd, 0) for nlcd in AG_NLCD_CODES) total = sum(result.values()) ag_stream_percent = ag_streams / total @@ -185,17 +193,49 @@ def nlcd_streams(sjs_result): } +@shared_task(throws=Exception) +def nlcd_soils(sjs_result): + """ + Results are expected to be in the format: + { + (NLCD ID, Soil Group ID, Soil Texture ID): Count, + } + + We calculate a number of values relying on various combinations + of these raster datasets. + """ + if 'error' in sjs_result: + raise Exception('[nlcd_soils] {}'.format(sjs_result['error'])) + + ngt_count = parse_sjs_result(sjs_result) + + # Split combined counts into separate ones for processing + # Reduce [(n, g, t): c] to + n_count = {} # [n: sum(c)] + ng_count = {} # [(n, g): sum(c)] + nt_count = {} # [(n, t): sum(c)] + for (n, g, t), count in ngt_count.iteritems(): + n_count[n] = count + n_count.get(n, 0) + nt_count[(n, t)] = count + nt_count.get((n, t), 0) + + # Map soil group values to usable subset + g2 = settings.SOIL_GROUP[g] + ng_count[(n, g2)] = count + ng_count.get((n, g2), 0) + + return { + 'cn': curve_number(n_count, ng_count), + 'sed_phos': sediment_phosphorus(nt_count), + } + + def geop_tasks(geom, errback): # List of tuples of (opname, data, callback) for each geop task definitions = [ ('nlcd_streams', {'polygon': [geom.geojson], 'vector': streams(geom)}, nlcd_streams), - # TODO Remove this second dummy call with actual geop task - # We need at least two for the flatten to work correctly. - ('nlcd_streams', - {'polygon': [geom.geojson], 'vector': streams(geom)}, - nlcd_streams) + + ('nlcd_soils', {'polygon': [geom.geojson]}, nlcd_soils), ] return [(mapshed_start.s(opname, data) | diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index afb46e14a..7f9977aac 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -13,7 +13,7 @@ from sys import path from layer_settings import LAYERS, VIZER_URLS # NOQA -from gwlfe_settings import GWLFE_DEFAULTS, GWLFE_CONFIG # NOQA +from gwlfe_settings import GWLFE_DEFAULTS, GWLFE_CONFIG, SOIL_GROUP, SOILP, CURVE_NUMBER # NOQA # Normally you should not import ANYTHING from Django directly # into your settings, but ImproperlyConfigured is an exception. @@ -392,6 +392,20 @@ def get_env_setting(setting): 'operationType': 'RasterLinesJoin', 'zoom': 0 } + }, + 'nlcd_soils': { + 'input': { + 'polygon': [], + 'polygonCRS': 'LatLng', + 'rasters': [ + 'nlcd-2011-30m-epsg5070-0.10.0', + 'ssurgo-hydro-groups-30m-epsg5070-0.10.0', + 'us-ssugro-texture-id-30m-epsg5070' + ], + 'rasterCRS': 'ConusAlbers', + 'operationType': 'RasterJoin', + 'zoom': 0 + } } } } diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index 4575189ff..4fe74c210 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -5,6 +5,8 @@ YesOrNo as b, SweepType) +NODATA = -2147483648 # Scala's Int.MinValue is NODATA in GeoTrellis + GWLFE_DEFAULTS = { 'NRur': 10, # Number of Rural Land Use Categories 'NUrb': 6, # Number of Urban Land Use Categories @@ -448,6 +450,92 @@ 'Livestock': ['dairy_cows', 'beef_cows', 'hogs', 'sheep', 'horses'], 'Poultry': ['broilers', 'layers', 'turkeys'], 'ManureSpreadingLandUseIndices': [0, 1], # Land Use Indices where manure spreading applies. Currently Hay/Past and Cropland. + 'AgriculturalNLCDCodes': [81, 82], # NLCD codes considered agricultural. Correspond to Hay/Past and Cropland 'MonthDays': [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31], 'WeatherNull': -99999, # This value is used to indicate NULL in ms_weather dataset } + +# Maps NLCD + Hygrological Soil Group to a Curve Number. Keys are NLCD Codes. +# Array Index 0 has no value, rest map to SOIL_GROUP value. +# Original at Class1.vb@1.3.0:4174-4315 +CURVE_NUMBER = { + 11: [0, 100, 100, 100, 100], # Water + 12: [0, 72, 82, 87, 89], # Perennial Ice/Snow + 21: [0, 72, 82, 87, 89], # Developed Open Space + 31: [0, 72, 82, 87, 89], # Barren Land + 41: [0, 37, 60, 73, 80], # Deciduous Forest + 42: [0, 37, 60, 73, 80], # Evergreen Forest + 43: [0, 37, 60, 73, 80], # Mixed Forest + 52: [0, 37, 60, 73, 80], # Shrub/Scrub + 71: [0, 72, 82, 87, 89], # Grassland/Herbaceous + 81: [0, 43, 63, 75, 81], # Pasture/Hay + 82: [0, 64, 75, 82, 85], # Cultivated Crops + 90: [0, 69, 80, 87, 90], # Woody Wetlands + 95: [0, 69, 80, 87, 90], # Emergent Herbaceous Wetlands +} + +# Maps Hydrological Soil Groups to subset we can use +SOIL_GROUP = { + 1: 1, # A + 2: 2, # B + 3: 3, # C + 4: 4, # D + 5: 3, # A/D -> C + 6: 3, # B/D -> C + 7: 4, # C/D -> D + NODATA: 3, # NODATA -> C +} + +# Maps Soil Texture values to Agricultural and Non-Agricultural Phosphorus levels +SOILP = { + 1: [900, 420], # Clay + 2: [690, 266], # Fine sandy loam + 3: [0, 0], # Boulders + 4: [0, 0], # Gravel + 5: [650, 230], # Very fine sandy loam + 6: [300, 100], # Gypsiferous material + 7: [600, 200], # Loamy coarse sand + 8: [1000, 600], # Muck + 9: [0, 0], # Marl + 10: [870, 400], # Clay loam + 11: [600, 200], # Very fine sand + 12: [0, 0], # Artifacts + 13: [780, 332], # Silt + 14: [100, 100], # Material + 15: [100, 100], # Weathered bedrock + 16: [1000, 600], # Mucky peat + 17: [0, 0], # Consolidated permafrost (ice rich) + 18: [580, 180], # Coarse sand + 19: [630, 220], # Loamy fine sand + 20: [0, 0], # Fragmental material + 21: [0, 0], # Bedrock + 22: [600, 200], # Fine sand + 23: [0, 0], # + 24: [0, 0], # Channers + 25: [600, 200], # Pumiceous + 26: [600, 200], # Loamy sand + 27: [0, 0], # Water + 28: [660, 244], # Sandy loam + 29: [680, 266], # Sandy clay loam + 30: [1000, 600], # Moderately decomposed plant material + 31: [300, 100], # Cobbles + 32: [580, 180], # Sand + 33: [0, 0], # Cinders + 34: [840, 376], # Silty clay loam + 35: [650, 230], # Loamy very fine sand + 36: [600, 200], # Coarse sandy loam + 37: [0, 0], # Paragravel + 38: [600, 200], # Variable + 39: [0, 0], # Unweathered bedrock + 40: [720, 288], # Loam + 41: [780, 332], # Silt loam + 42: [800, 320], # Sandy clay + 43: [300, 100], # Stones + 44: [1000, 600], # Highly decomposed plant material + 45: [1000, 600], # Slightly decomposed plant material + 46: [600, 200], # LoamGravel + 47: [840, 376], # Silty clay + 48: [1000, 600], # Peat + 49: [300, 100], # SandGravel + NODATA: [100, 200], # NODATA +} From 414614c4fc971b8413fc84e6295b7d212998473d Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 6 May 2016 12:51:29 -0400 Subject: [PATCH 044/136] Add NLCD + Stream values We ignore Unpaved Roads, thus n42c is initialized as 0 with all other static values. The rest are calculated from the NLCD + Streams geoprocessing call. Also fix the rounding of n42b, which needs exactly one decimal place. --- src/mmw/apps/modeling/mapshed/tasks.py | 32 ++++++++++++++++++-------- src/mmw/mmw/settings/gwlfe_settings.py | 1 + 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index dae17d62c..0ec0c8e73 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -112,8 +112,8 @@ def collect_data(geop_result, geojson): z.ManNitr, z.ManPhos = manure_spread(z.AEU) # Data from Streams dataset - z.StreamLength = stream_length(geom) # Meters - z.n42b = round(z.StreamLength / 1000) # Kilometers + z.StreamLength = stream_length(geom) # Meters + z.n42b = round(z.StreamLength / 1000, 1) # Kilometers # Data from Point Source Discharge dataset n_load, p_load, discharge = point_source_discharge(geom, area) @@ -126,8 +126,12 @@ def collect_data(geop_result, geojson): z.Temp = temps z.Prec = prcps - z.AgLength = geop_result['AgStreamPct'] * z.StreamLength + # Begin processing geop_result + z.AgLength = geop_result['ag_stream_pct'] * z.StreamLength z.UrbLength = z.StreamLength - z.AgLength + z.n42 = round(z.AgLength / 1000, 1) + z.n46e = geop_result['med_high_urban_stream_pct'] * z.StreamLength / 1000 + z.n46f = geop_result['low_urban_stream_pct'] * z.StreamLength / 1000 z.CN = np.array(geop_result['cn']) z.SedPhos = geop_result['sed_phos'] @@ -147,9 +151,11 @@ def collect_data(geop_result, geojson): def nlcd_streams(sjs_result): """ From a dictionary mapping NLCD codes to the count of stream pixels on - each, return a dictionary with a key 'AgStreamPct' which indicates the - percent of streams in agricultural areas, namely NLCD 81 Pasture/Hay - and 82 Cultivated Crops. + each, return a dictionary with keys 'ag_stream_pct', 'low_urban_stream_pct' + and 'med_high_urban_stream_pct' which indicate the percent of streams in + agricultural areas (namely NLCD 81 Pasture/Hay and 82 Cultivated Crops), + low density urban areas (NLCD 22), and medium and high density urban areas + (NLCD 23 and 24) respectively. In addition, we inspect the result to see if it includes an 'error' key. If so, it would indicate that a preceeding task has thrown an exception, @@ -183,13 +189,21 @@ def nlcd_streams(sjs_result): # which are not JSON serializable and thus can't be shared between tasks result = parse_sjs_result(sjs_result) - ag_streams = sum(result.get(nlcd, 0) for nlcd in AG_NLCD_CODES) + ag_count = sum(result.get(nlcd, 0) for nlcd in AG_NLCD_CODES) + low_urban_count = result.get(22, 0) + med_high_urban_count = sum(result.get(nlcd, 0) for nlcd in [23, 24]) total = sum(result.values()) - ag_stream_percent = ag_streams / total + ag, low, med_high = (float(count) / total + if total > 0 else 0 + for count in (ag_count, + low_urban_count, + med_high_urban_count)) return { - 'AgStreamPct': ag_stream_percent + 'ag_stream_pct': ag, + 'low_urban_stream_pct': low, + 'med_high_urban_stream_pct': med_high } diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index 4fe74c210..b09e65dd8 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -188,6 +188,7 @@ 'n41g': 0, # Runoff Control: Future (%) 'n41h': 0, # Phytase in Feed: Existing (%) 'n41i': 0, # Phytase in Feed: Future (%) + 'n42c': 0, # Unpaved Road Length (km) 'n43': 0, # Stream Km with Vegetated Buffer Strips: Existing 'GRLBN': 0, # Average Grazing Animal Loss Rate (Barnyard/Confined Area): Nitrogen 'NGLBN': 0, # Average Non-Grazing Animal Loss Rate (Barnyard/Confined Area): Nitrogen From 15d5b1510a6928c43518353bd2555a31244a8414 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 10 May 2016 19:13:15 -0400 Subject: [PATCH 045/136] Retry submitting jobs to Spark JobServer Because of the multiplicity of jobs required for MapShed, we face overcrowding of Spark JobServer from time to time, when there are no more slots for new jobs. In such a case, we should retry the submission of the job a few times before giving up. --- src/mmw/apps/modeling/geoprocessing.py | 14 +++++++++++--- src/mmw/apps/modeling/mapshed/tasks.py | 8 +++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/mmw/apps/modeling/geoprocessing.py b/src/mmw/apps/modeling/geoprocessing.py index 9426524d7..48a0278ca 100644 --- a/src/mmw/apps/modeling/geoprocessing.py +++ b/src/mmw/apps/modeling/geoprocessing.py @@ -53,7 +53,7 @@ @statsd.timer(__name__ + '.sjs_submit') -def sjs_submit(host, port, args, data): +def sjs_submit(host, port, args, data, retry=None): """ Submits a job to Spark Job Server. Returns its Job ID, which can be used with sjs_retrieve to get the final result. @@ -64,8 +64,16 @@ def sjs_submit(host, port, args, data): if response.ok: job = response.json() else: - raise Exception('Unable to submit job to Spark JobServer.\n' - 'Details = {}'.format(response.text)) + error = response.json() + if error['status'] == 'NO SLOTS AVAILABLE' and retry: + try: + retry() + except MaxRetriesExceededError: + raise Exception('No slots available in Spark JobServer.\n' + 'Details = {}'.format(response.text)) + else: + raise Exception('Unable to submit job to Spark JobServer.\n' + 'Details = {}'.format(response.text)) if job['status'] == 'STARTED': return job['result']['jobId'] diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index 0ec0c8e73..64bd80859 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -37,8 +37,8 @@ ACRES_PER_SQM = 0.000247105 -@shared_task -def mapshed_start(opname, input_data): +@shared_task(bind=True, default_retry_delay=1, max_retries=42) +def mapshed_start(self, opname, input_data): host = settings.GEOP['host'] port = settings.GEOP['port'] args = settings.GEOP['args']['MapshedJob'] @@ -47,7 +47,7 @@ def mapshed_start(opname, input_data): data['input'].update(input_data) try: - job_id = sjs_submit(host, port, args, data) + job_id = sjs_submit(host, port, args, data, retry=self.retry) return { 'host': host, @@ -55,6 +55,8 @@ def mapshed_start(opname, input_data): 'job_id': job_id } + except Retry as r: + raise r except Exception as x: return { 'error': x.message From 809930d7db79536d760e79e13d89103ff7a8b956 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 10 May 2016 19:37:19 -0400 Subject: [PATCH 046/136] Retry SummaryJob as well Since SummaryJob is often triggered after MapshedJob when coming back to an existing project, we add the retrial to that as well to ensure that it doesn't give up after the first failure. --- src/mmw/apps/modeling/geoprocessing.py | 4 ++-- src/mmw/apps/modeling/tasks.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/mmw/apps/modeling/geoprocessing.py b/src/mmw/apps/modeling/geoprocessing.py index 48a0278ca..7d4c896b1 100644 --- a/src/mmw/apps/modeling/geoprocessing.py +++ b/src/mmw/apps/modeling/geoprocessing.py @@ -128,7 +128,7 @@ def sjs_retrieve(host, port, job_id, retry=None): @statsd.timer(__name__ + '.histogram_start') -def histogram_start(polygons): +def histogram_start(polygons, retry=None): """ Together, histogram_start and histogram_finish implement a function which takes a list of polygons or multipolygons as input, @@ -143,7 +143,7 @@ def histogram_start(polygons): data = settings.GEOP['json']['nlcdSoilCensus'].copy() data['input']['geometry'] = polygons - return sjs_submit(host, port, args, data) + return sjs_submit(host, port, args, data, retry) @statsd.timer(__name__ + '.histogram_finish') diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index df477189f..673e587a5 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -46,8 +46,8 @@ def start_rwd_job(location, snapping): return response_json -@shared_task -def start_histogram_job(json_polygon): +@shared_task(bind=True, default_retry_delay=1, max_retries=42) +def start_histogram_job(self, json_polygon): """ Calls the histogram_start function to kick off the SJS job to generate a histogram of the provided polygon (i.e. AoI). @@ -60,12 +60,12 @@ def start_histogram_job(json_polygon): return { 'pixel_width': aoi_resolution(polygon), - 'sjs_job_id': histogram_start([json_polygon]) + 'sjs_job_id': histogram_start([json_polygon], self.retry) } -@shared_task -def start_histograms_job(polygons): +@shared_task(bind=True, default_retry_delay=1, max_retries=42) +def start_histograms_job(self, polygons): """ Calls the histogram_start function to kick off the SJS job to generate a histogram of the provided polygons (i.e. AoI + modifications, @@ -79,7 +79,7 @@ def start_histograms_job(polygons): return { 'pixel_width': None, - 'sjs_job_id': histogram_start(json_polygons) + 'sjs_job_id': histogram_start(json_polygons, self.retry) } From f816a20921da95366ca044beba2c78ec42e45c4d Mon Sep 17 00:00:00 2001 From: Kelly Innes Date: Mon, 9 May 2016 09:04:02 -0400 Subject: [PATCH 047/136] Calculate additional MapShed values - calculate `SedDelivRatio` from input polygon - calculate `BasinAreaHa` from input polygon - calculate `AvgAwc` from `us-ssugro-aws100-30m-epsg5070` - calculate `GrNitrConc` from `us-groundwater-nitrogen-30m-epsg5070` - calculate `GrPhosConc` from `us-groundwater-nitrogen-30m-epsg5070` - add `gwn` and `avg_awc` fns to `tasks.py` - add calculations to `calcs.py` - register layers in `base.py` --- src/mmw/apps/modeling/mapshed/calcs.py | 42 +++++++++++++++++++++ src/mmw/apps/modeling/mapshed/tasks.py | 52 +++++++++++++++++++++++++- src/mmw/mmw/settings/base.py | 22 +++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index 5ece353e5..6d17c12a1 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -401,3 +401,45 @@ def sediment_phosphorus(nt_count): sedp = float(ag_sedp + nag_sedp) / total return sedp * 1.6 + + +def groundwater_nitrogen_conc(gwn_dict): + """ + Calculate GwN and GwP from a dictionary of NLCD land use keys + paired with the number of cells as values + + Original at Class1.vb@1.3.0:9007-9022 + """ + + # Discard any key-value pairs for keys < 1 + valid_res = {k: gwn_dict[k] for k in gwn_dict.keys() if k > 0} + # Combine values for all keys >= 20 onto one key + valid_res[20] = sum([gwn_dict[k] for k in valid_res.keys() if k >= 20]) + valid_total_cells = sum([v for v in valid_res.values()]) + + weighted_conc = 0 + if valid_total_cells > 0: + weighted_conc = sum([float(gwn * count)/valid_total_cells + for gwn, count in valid_res.iteritems()]) + + groundwater_nitrogen_conc = (0.7973 * weighted_conc) - 0.692 + groundwater_phosphorus_conc = (0.0049 * weighted_conc) + 0.0089 + + if groundwater_nitrogen_conc < 0.34: + groundwater_nitrogen_conc = 0.34 + + return groundwater_nitrogen_conc, groundwater_phosphorus_conc + + +def sediment_delivery_ratio(area_sq_km): + """ + Calculate Sediment Delivery Ratio from the basin area in square km + + Original at Class1.vb@1.3.0:9334-9340 + """ + + if area_sq_km < 50: + return (0.000005 * (area_sq_km ** 2) - + (0.0014 * area_sq_km) + 0.198) + else: + return 0.451 * (area_sq_km ** (0 - 0.298)) diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index 64bd80859..b36a5f842 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -31,10 +31,14 @@ weather_data, curve_number, sediment_phosphorus, + groundwater_nitrogen_conc, + sediment_delivery_ratio, ) AG_NLCD_CODES = settings.GWLFE_CONFIG['AgriculturalNLCDCodes'] ACRES_PER_SQM = 0.000247105 +HECTARES_PER_SQM = 0.0001 +SQKM_PER_SQM = 0.000001 @shared_task(bind=True, default_retry_delay=1, max_retries=42) @@ -138,6 +142,13 @@ def collect_data(geop_result, geojson): z.CN = np.array(geop_result['cn']) z.SedPhos = geop_result['sed_phos'] + # Additional calculated values + z.SedDelivRatio = sediment_delivery_ratio(area * SQKM_PER_SQM) + z.TotArea = area * HECTARES_PER_SQM + z.GrNitrConc = geop_result['gr_nitr_conc'] + z.GrPhosConc = geop_result['gr_phos_conc'] + z.MaxWaterCap = geop_result['avg_awc'] + # TODO pass real input to model instead of reading it from gms file gms_filename = join(dirname(abspath(__file__)), 'data/sample_input.gms') gms_file = open(gms_filename, 'r') @@ -244,14 +255,53 @@ def nlcd_soils(sjs_result): } +@shared_task(throws=Exception) +def gwn(sjs_result): + """ + Derive Groundwater Nitrogen and Phosphorus + """ + if 'error' in sjs_result: + raise Exception('[gwn] {}' + .format(sjs_result['error'])) + + result = parse_sjs_result(sjs_result) + gr_nitr_conc, gr_phos_conc = groundwater_nitrogen_conc(result) + + return { + 'gr_nitr_conc': gr_nitr_conc, + 'gr_phos_conc': gr_phos_conc + } + + +@shared_task(throws=Exception) +def avg_awc(sjs_result): + """ + Get `AvgAwc` from MMW-Geoprocessing endpoint + + Original at Class1.vb@1.3.0:4150 + """ + if 'error' in sjs_result: + raise Exception('[awc] {}' + .format(sjs_result['error'])) + + result = parse_sjs_result(sjs_result) + + return { + 'avg_awc': result.values()[0] + } + + def geop_tasks(geom, errback): # List of tuples of (opname, data, callback) for each geop task definitions = [ ('nlcd_streams', {'polygon': [geom.geojson], 'vector': streams(geom)}, nlcd_streams), - ('nlcd_soils', {'polygon': [geom.geojson]}, nlcd_soils), + ('gwn', + {'polygon': [geom.geojson]}, gwn), + ('avg_awc', + {'polygon': [geom.geojson]}, avg_awc), ] return [(mapshed_start.s(opname, data) | diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index 7f9977aac..1800d848b 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -406,6 +406,28 @@ def get_env_setting(setting): 'operationType': 'RasterJoin', 'zoom': 0 } + }, + 'gwn': { + 'input': { + 'polygon': [], + 'polygonCRS': 'LatLng', + 'rasters': [ + 'us-groundwater-nitrogen-30m-epsg5070' + ], + 'rasterCRS': 'ConusAlbers', + 'operationType': 'RasterJoin', + 'zoom': 0 + } + }, + 'avg_awc': { + 'input': { + 'polygon': [], + 'targetRaster': 'us-ssugro-aws100-30m-epsg5070', + 'rasterCRS': 'ConusAlbers', + 'polygonCRS': 'LatLng', + 'operationType': 'RasterAverage', + 'zoom': 0 + } } } } From c09f723a2897b71a07ee23f3bc3263366a26d8c6 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 10 May 2016 23:42:54 -0400 Subject: [PATCH 048/136] Calculate KV --- src/mmw/apps/modeling/mapshed/calcs.py | 20 ++++++++++++++++++++ src/mmw/apps/modeling/mapshed/tasks.py | 2 ++ src/mmw/mmw/settings/gwlfe_settings.py | 1 + 3 files changed, 23 insertions(+) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index 6d17c12a1..a7898b358 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -14,6 +14,7 @@ from django.db import connection NUM_WEATHER_STATIONS = settings.GWLFE_CONFIG['NumWeatherStations'] +KV_FACTOR = settings.GWLFE_CONFIG['KvFactor'] MONTHS = settings.GWLFE_DEFAULTS['Month'] MONTHDAYS = settings.GWLFE_CONFIG['MonthDays'] LIVESTOCK = settings.GWLFE_CONFIG['Livestock'] @@ -112,6 +113,25 @@ def et_adjustment(ws): return np.array([avg_etadj] * 12) +def kv_coefficient(ecs): + """ + Given an array of erosion coefficients, returns an array of 12 decimals, + one for the KV coefficient of each month. The KV of a month is initialized + to the corresponding erosion coefficient value times the KV Factor, and + then averaged over the preceeding month. January being the first month is + not averaged. + + Original at Class1.vb@1.3.0:4989-4995 + """ + + kv = [ec * KV_FACTOR for ec in ecs] + + for m in range(1, 12): + kv[m] = (kv[m] + kv[m-1]) / 2 + + return np.array(kv) + + def animal_energy_units(geom): """ Given a geometry, returns the total livestock and poultry AEUs within it diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index b36a5f842..f55a349c2 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -23,6 +23,7 @@ growing_season, erosion_coeff, et_adjustment, + kv_coefficient, animal_energy_units, manure_spread, streams, @@ -104,6 +105,7 @@ def collect_data(geop_result, geojson): z.Grow = growing_season(ws) z.Acoef = erosion_coeff(ws, z.Grow) z.PcntET = et_adjustment(ws) + z.KV = kv_coefficient(z.Acoef) z.WxYrBeg = max([w.begyear for w in ws]) z.WxYrEnd = min([w.endyear for w in ws]) z.WxYrs = z.WxYrEnd - z.WxYrBeg + 1 diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index b09e65dd8..40726ee19 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -448,6 +448,7 @@ GWLFE_CONFIG = { 'NumWeatherStations': 2, # Number of weather stations to consider for each polygon + 'KvFactor': 1.16, # Original at Class1.vb@1.3.0:4987 'Livestock': ['dairy_cows', 'beef_cows', 'hogs', 'sheep', 'horses'], 'Poultry': ['broilers', 'layers', 'turkeys'], 'ManureSpreadingLandUseIndices': [0, 1], # Land Use Indices where manure spreading applies. Currently Hay/Past and Cropland. From 510ed51fbbffc08e3b45bf4a472244318a9b8025 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 11 May 2016 00:06:33 -0400 Subject: [PATCH 049/136] Calculate Area for each land use --- src/mmw/apps/modeling/mapshed/calcs.py | 34 ++++++++++++++++++++++++++ src/mmw/apps/modeling/mapshed/tasks.py | 3 +++ 2 files changed, 37 insertions(+) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index a7898b358..ca1490528 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -463,3 +463,37 @@ def sediment_delivery_ratio(area_sq_km): (0.0014 * area_sq_km) + 0.198) else: return 0.451 * (area_sq_km ** (0 - 0.298)) + + +def landuse_pcts(n_count): + """ + Given a dictionary mapping NLCD Codes to counts of cells, returns a + dictionary mapping MapShed Land Use Types to percent area covered. + """ + + total = sum(n_count.values()) + if total > 0: + n_pct = {nlcd: float(count) / total + for nlcd, count in n_count.iteritems()} + else: + n_pct = {nlcd: 0 for nlcd in n_count.keys()} + + return [ + n_pct.get(81, 0), # Hay/Pasture + n_pct.get(82, 0), # Cropland + n_pct.get(41, 0) + n_pct.get(42, 0) + + n_pct.get(43, 0) + n_pct.get(52, 0), # Forest + n_pct.get(90, 0) + n_pct.get(95, 0), # Wetland + 0, # Disturbed + 0, # Turf Grass + n_pct.get(21, 0) + n_pct.get(71, 0), # Open Land + n_pct.get(12, 0) + n_pct.get(31, 0), # Bare Rock + 0, # Sandy Areas + 0, # Unpaved Road + n_pct.get(22, 0), # Low Density Mixed + n_pct.get(23, 0), # Medium Density Mixed + n_pct.get(24, 0), # High Density Mixed + 0, # Low Density Residential + 0, # Medium Density Residential + 0, # High Density Residential + ] diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index f55a349c2..c0f393f91 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -34,6 +34,7 @@ sediment_phosphorus, groundwater_nitrogen_conc, sediment_delivery_ratio, + landuse_pcts, ) AG_NLCD_CODES = settings.GWLFE_CONFIG['AgriculturalNLCDCodes'] @@ -143,6 +144,7 @@ def collect_data(geop_result, geojson): z.CN = np.array(geop_result['cn']) z.SedPhos = geop_result['sed_phos'] + z.Area = np.array(geop_result['landuse_pcts'] * area * HECTARES_PER_SQM) # Additional calculated values z.SedDelivRatio = sediment_delivery_ratio(area * SQKM_PER_SQM) @@ -254,6 +256,7 @@ def nlcd_soils(sjs_result): return { 'cn': curve_number(n_count, ng_count), 'sed_phos': sediment_phosphorus(nt_count), + 'landuse_pcts': landuse_pcts(n_count), } From 72bf818ec0413da6d59f6d449f68d829f964d35c Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 11 May 2016 00:16:54 -0400 Subject: [PATCH 050/136] Calculate NormalSys from land use area --- src/mmw/apps/modeling/mapshed/calcs.py | 18 ++++++++++++++++++ src/mmw/apps/modeling/mapshed/tasks.py | 3 +++ src/mmw/mmw/settings/gwlfe_settings.py | 2 ++ 3 files changed, 23 insertions(+) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index ca1490528..28ce15405 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -497,3 +497,21 @@ def landuse_pcts(n_count): 0, # Medium Density Residential 0, # High Density Residential ] + + +def normal_sys(lu_area): + """ + Given the land use area in hectares, estimates the number of normal septic + systems based on the constants SSLDR and SSLDM for Residential and Mixed + land uses respectively. + Since in this version Residential land uses are always 0, the value is + effectively dependent only on the area of medium density mixed land use. + However, we replicate the original formula for consistency. + + Original at Class1.vb@1.3.0:9577-9579 + """ + + SSLDR = settings.GWLFE_CONFIG['SSLDR'] + SSLDM = settings.GWLFE_CONFIG['SSLDM'] + + return SSLDR * lu_area[14] + SSLDM * lu_area[11] diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index c0f393f91..928a5269f 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -35,6 +35,7 @@ groundwater_nitrogen_conc, sediment_delivery_ratio, landuse_pcts, + normal_sys, ) AG_NLCD_CODES = settings.GWLFE_CONFIG['AgriculturalNLCDCodes'] @@ -146,6 +147,8 @@ def collect_data(geop_result, geojson): z.SedPhos = geop_result['sed_phos'] z.Area = np.array(geop_result['landuse_pcts'] * area * HECTARES_PER_SQM) + z.NormalSys = normal_sys(z.Area) + # Additional calculated values z.SedDelivRatio = sediment_delivery_ratio(area * SQKM_PER_SQM) z.TotArea = area * HECTARES_PER_SQM diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index 40726ee19..df6a51a2e 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -454,6 +454,8 @@ 'ManureSpreadingLandUseIndices': [0, 1], # Land Use Indices where manure spreading applies. Currently Hay/Past and Cropland. 'AgriculturalNLCDCodes': [81, 82], # NLCD codes considered agricultural. Correspond to Hay/Past and Cropland 'MonthDays': [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31], + 'SSLDR': 4.4, + 'SSLDM': 2.2, 'WeatherNull': -99999, # This value is used to indicate NULL in ms_weather dataset } From 071996ecab6c7f34008e7c372f26238367c04a8f Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 11 May 2016 18:10:21 -0400 Subject: [PATCH 051/136] Calculate n23, n23b, n24, n24b --- src/mmw/apps/modeling/mapshed/tasks.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index 928a5269f..d926085e0 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -149,7 +149,12 @@ def collect_data(geop_result, geojson): z.NormalSys = normal_sys(z.Area) - # Additional calculated values + # Original at Class1.vb@1.3.0:9803-9807 + z.n23 = z.Area[1] # Row Crops Area + z.n23b = z.Area[13] # High Density Mixed Urban Area + z.n24 = z.Area[0] # Hay/Pasture Area + z.n24b = z.Area[11] # Low Density Mixed Urban Area + z.SedDelivRatio = sediment_delivery_ratio(area * SQKM_PER_SQM) z.TotArea = area * HECTARES_PER_SQM z.GrNitrConc = geop_result['gr_nitr_conc'] From c867a77a04fefecb7bf863cc330bea50b1692f27 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 11 May 2016 17:58:07 -0400 Subject: [PATCH 052/136] Upgrade mmw-geoprocessing to 1.2.0 --- deployment/ansible/group_vars/all | 2 +- src/mmw/mmw/settings/base.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/deployment/ansible/group_vars/all b/deployment/ansible/group_vars/all index 8956e9b32..298f5824e 100644 --- a/deployment/ansible/group_vars/all +++ b/deployment/ansible/group_vars/all @@ -44,7 +44,7 @@ sjs_host: "localhost" sjs_port: 8090 sjs_container_image: "quay.io/azavea/spark-jobserver:0.6.1" -geop_version: "1.1.0" +geop_version: "1.2.0" nginx_cache_dir: "/var/cache/nginx" observation_api_url: "http://www.wikiwatershed-vs.org/" diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index 1800d848b..3a419cfb3 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -403,7 +403,7 @@ def get_env_setting(setting): 'us-ssugro-texture-id-30m-epsg5070' ], 'rasterCRS': 'ConusAlbers', - 'operationType': 'RasterJoin', + 'operationType': 'RasterGroupedCount', 'zoom': 0 } }, @@ -415,7 +415,7 @@ def get_env_setting(setting): 'us-groundwater-nitrogen-30m-epsg5070' ], 'rasterCRS': 'ConusAlbers', - 'operationType': 'RasterJoin', + 'operationType': 'RasterGroupedCount', 'zoom': 0 } }, @@ -423,9 +423,10 @@ def get_env_setting(setting): 'input': { 'polygon': [], 'targetRaster': 'us-ssugro-aws100-30m-epsg5070', + 'rasters': [], 'rasterCRS': 'ConusAlbers', 'polygonCRS': 'LatLng', - 'operationType': 'RasterAverage', + 'operationType': 'RasterGroupedAverage', 'zoom': 0 } } From 1773459347967df95ab39b39e462f7aa0efca059 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 12 May 2016 11:33:41 -0400 Subject: [PATCH 053/136] Lint --- src/mmw/mmw/settings/base.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index 3a419cfb3..557a86cbd 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -365,18 +365,18 @@ def get_env_setting(setting): 'host': environ.get('MMW_GEOPROCESSING_HOST', 'localhost'), 'port': environ.get('MMW_GEOPROCESSING_PORT', '8090'), 'args': { - 'SummaryJob': 'context=geoprocessing&appName=geoprocessing-%s&classPath=org.wikiwatershed.mmw.geoprocessing.SummaryJob' % environ.get('MMW_GEOPROCESSING_VERSION', '0.1.0'), - 'MapshedJob': 'context=geoprocessing&appName=geoprocessing-%s&classPath=org.wikiwatershed.mmw.geoprocessing.MapshedJob' % environ.get('MMW_GEOPROCESSING_VERSION', '0.1.0') + 'SummaryJob': 'context=geoprocessing&appName=geoprocessing-%s&classPath=org.wikiwatershed.mmw.geoprocessing.SummaryJob' % environ.get('MMW_GEOPROCESSING_VERSION', '0.1.0'), # NOQA + 'MapshedJob': 'context=geoprocessing&appName=geoprocessing-%s&classPath=org.wikiwatershed.mmw.geoprocessing.MapshedJob' % environ.get('MMW_GEOPROCESSING_VERSION', '0.1.0'), # NOQA }, 'json': { 'nlcdSoilCensus': { 'input': { - 'geometry': None, - 'tileCRS': 'ConusAlbers', - 'polyCRS': 'LatLng', - 'nlcdLayer': 'nlcd-2011-30m-epsg5070-0.10.0', - 'soilLayer': 'ssurgo-hydro-groups-30m-epsg5070-0.10.0', - 'zoom': 0 + 'geometry': None, + 'tileCRS': 'ConusAlbers', + 'polyCRS': 'LatLng', + 'nlcdLayer': 'nlcd-2011-30m-epsg5070-0.10.0', + 'soilLayer': 'ssurgo-hydro-groups-30m-epsg5070-0.10.0', + 'zoom': 0 }, }, 'nlcd_streams': { @@ -428,7 +428,7 @@ def get_env_setting(setting): 'polygonCRS': 'LatLng', 'operationType': 'RasterGroupedAverage', 'zoom': 0 - } + } } } } From dedb95fef798a92d1d4a2260a78e864fdcb9d91a Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 12 May 2016 11:36:22 -0400 Subject: [PATCH 054/136] Wait longer for geoprocessing to finish --- src/mmw/mmw/settings/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index 557a86cbd..69bd82009 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -125,7 +125,7 @@ def get_env_setting(setting): CELERY_WORKER_DIRECT = True CELERY_CREATE_MISSING_QUEUES = True CELERY_CHORD_PROPAGATES = True -CELERY_CHORD_UNLOCK_MAX_RETRIES = 3 +CELERY_CHORD_UNLOCK_MAX_RETRIES = 10 # END CELERY CONFIGURATION From 2d6db0ba8ccc9a09c1b11aae010b023e95554875 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 12 May 2016 12:41:00 -0400 Subject: [PATCH 055/136] Include goats in sheep count --- src/mmw/apps/modeling/mapshed/calcs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index 6d17c12a1..d176dfff2 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -134,6 +134,7 @@ def animal_energy_units(geom): SELECT SUM(beef_ha * totalha * clip_percent) AS beef_cows, SUM(broiler_ha * totalha * clip_percent) AS broilers, SUM(dairy_ha * totalha * clip_percent) AS dairy_cows, + SUM(goat_ha * totalha * clip_percent) + SUM(sheep_ha * totalha * clip_percent) AS sheep, SUM(hog_ha * totalha * clip_percent) AS hogs, SUM(horse_ha * totalha * clip_percent) AS horses, From b4a657f6bd30b85846c0570050f80a965203c30f Mon Sep 17 00:00:00 2001 From: Hector Castro Date: Thu, 12 May 2016 14:56:54 -0400 Subject: [PATCH 056/136] Bump libpq version to 9.6.x The main PostgreSQL APT repository bumped their version of libpq. --- deployment/ansible/group_vars/all | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/ansible/group_vars/all b/deployment/ansible/group_vars/all index 298f5824e..b3c988c0e 100644 --- a/deployment/ansible/group_vars/all +++ b/deployment/ansible/group_vars/all @@ -21,7 +21,7 @@ postgresql_database: mmw postgresql_version: "9.4" postgresql_package_version: "9.4.*.pgdg14.04+1" postgresql_support_repository_channel: "main" -postgresql_support_libpq_version: "9.5.*.pgdg14.04+1" +postgresql_support_libpq_version: "9.6*.pgdg14.04+1" postgresql_support_psycopg2_version: "2.6" postgis_version: "2.1" postgis_package_version: "2.1.*.pgdg14.04+1" From b198a44bdf1639a457f825ddb50866a2aaece449 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Thu, 12 May 2016 16:20:22 -0400 Subject: [PATCH 057/136] Compute values based on nlcd, slope, and kfactor rasters Compute AgSlope3, AgSlope3To8, n41, AvSlope, AvKF, and KF --- src/mmw/apps/modeling/mapshed/tasks.py | 150 ++++++++++++++++++++++++- src/mmw/mmw/settings/base.py | 40 ++++++- src/mmw/mmw/settings/gwlfe_settings.py | 1 + 3 files changed, 188 insertions(+), 3 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index d926085e0..96e4b26e0 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -38,6 +38,7 @@ normal_sys, ) +NLU = settings.GWLFE_DEFAULTS['NLU'] AG_NLCD_CODES = settings.GWLFE_CONFIG['AgriculturalNLCDCodes'] ACRES_PER_SQM = 0.000247105 HECTARES_PER_SQM = 0.0001 @@ -61,7 +62,6 @@ def mapshed_start(self, opname, input_data): 'port': port, 'job_id': job_id } - except Retry as r: raise r except Exception as x: @@ -77,7 +77,6 @@ def mapshed_finish(self, incoming): try: return sjs_retrieve(retry=self.retry, **incoming) - except Retry as r: # Celery throws a Retry exception when self.retry is called to stop # the execution of any further code, and to indicate to the worker @@ -149,12 +148,22 @@ def collect_data(geop_result, geojson): z.NormalSys = normal_sys(z.Area) + z.AgSlope3 = (geop_result['ag_slope_3_pct'] * area) * HECTARES_PER_SQM + z.AgSlope3To8 = (geop_result['ag_slope_3_8_pct'] * area) * HECTARES_PER_SQM + z.n41 = geop_result['n41'] + + z.AvSlope = geop_result['avg_slope'] + + z.AvKF = geop_result['avg_kf'] + z.KF = geop_result['kf'] + # Original at Class1.vb@1.3.0:9803-9807 z.n23 = z.Area[1] # Row Crops Area z.n23b = z.Area[13] # High Density Mixed Urban Area z.n24 = z.Area[0] # Hay/Pasture Area z.n24b = z.Area[11] # Low Density Mixed Urban Area + # Additional calculated values z.SedDelivRatio = sediment_delivery_ratio(area * SQKM_PER_SQM) z.TotArea = area * HECTARES_PER_SQM z.GrNitrConc = geop_result['gr_nitr_conc'] @@ -304,6 +313,134 @@ def avg_awc(sjs_result): } +@shared_task(throws=Exception) +def nlcd_slope(result): + if 'error' in result: + raise Exception('[nlcd_slope] {}'.format(result['error'])) + + result = parse_sjs_result(result) + + ag_slope_3_count = 0 + ag_slope_3_8_count = 0 + ag_count = 0 + total_count = 0 + + for (nlcd_code, slope), count in result.iteritems(): + if nlcd_code in AG_NLCD_CODES: + if slope > 3: + ag_slope_3_count += count + if 3 < slope < 8: + ag_slope_3_8_count += count + ag_count += count + + total_count += count + + # percent of AOI that is agricultural with slope > 3% + # see Class1.vb#7223 + ag_slope_3_pct = (float(ag_slope_3_count) / total_count + if total_count > 0 else 0.0) + + # percent of AOI that is agricultural with 3% < slope < 8% + ag_slope_3_8_pct = (float(ag_slope_3_8_count) / total_count + if total_count > 0 else 0.0) + + # percent of agricultural parts of AOI with slope > 3% + # see Class1.vb#9864 + n41 = float(ag_slope_3_count) / ag_count if ag_count > 0 else 0.0 + + output = { + 'ag_slope_3_pct': ag_slope_3_pct, + 'ag_slope_3_8_pct': ag_slope_3_8_pct, + 'n41': n41, + } + + # TODO remove before merging + print(output) + + return output + + +@shared_task(throws=Exception) +def slope(result): + if 'error' in result: + raise Exception('[slope] {}'.format(result['error'])) + + result = parse_sjs_result(result) + + # average slope over the AOI + # see Class1.vb#6252 + avg_slope = result + + output = { + 'avg_slope': avg_slope + } + + # TODO remove before merging + print(output) + + return output + + +def get_lu_index(nlcd): + if nlcd == 81: + lu_index = 1 + elif nlcd == 82: + lu_index = 2 + elif nlcd in [41, 42, 43]: + lu_index = 3 + elif nlcd in [90, 95]: + lu_index = 4 + elif nlcd in [21, 71]: + lu_index = 7 + elif nlcd in [12, 31]: + lu_index = 8 + elif nlcd == 22: + lu_index = 10 + elif nlcd == 23: + lu_index = 11 + elif nlcd == 24: + lu_index = 12 + elif nlcd == 11: + lu_index = 16 + else: + return None + + return lu_index - 1 + + +@shared_task(throws=Exception) +def nlcd_kfactor(result): + if 'error' in result: + raise Exception('[nlcd_kfactor] {}'.format(result['error'])) + + result = parse_sjs_result(result) + + # average kfactor for each land use + # see Class1.vb#6431 + kf = [0.0] * NLU + for nlcd_code, kfactor in result.iteritems(): + lu_ind = get_lu_index(nlcd_code) + if lu_ind: + kf[lu_ind] = kfactor + + # average kfactor across all land uses, ignoring zero values + # see Class1.vb#4151 + num_nonzero_kf = len([k for k in kf if k != 0.0]) + avg_kf = 0.0 + if num_nonzero_kf != 0: + avg_kf = sum(kf) / num_nonzero_kf + + output = { + 'kf': kf, + 'avg_kf': avg_kf + } + + # TODO remove before merging + print(output) + + return output + + def geop_tasks(geom, errback): # List of tuples of (opname, data, callback) for each geop task definitions = [ @@ -315,6 +452,15 @@ def geop_tasks(geom, errback): {'polygon': [geom.geojson]}, gwn), ('avg_awc', {'polygon': [geom.geojson]}, avg_awc), + ('nlcd_slope', + {'polygon': [geom.geojson]}, + nlcd_slope), + ('slope', + {'polygon': [geom.geojson]}, + slope), + ('nlcd_kfactor', + {'polygon': [geom.geojson]}, + nlcd_kfactor) ] return [(mapshed_start.s(opname, data) | diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index 69bd82009..85fb0c32f 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -13,7 +13,8 @@ from sys import path from layer_settings import LAYERS, VIZER_URLS # NOQA -from gwlfe_settings import GWLFE_DEFAULTS, GWLFE_CONFIG, SOIL_GROUP, SOILP, CURVE_NUMBER # NOQA +from gwlfe_settings import (GWLFE_DEFAULTS, GWLFE_CONFIG, SOIL_GROUP, # NOQA + SOILP, CURVE_NUMBER) # NOQA # Normally you should not import ANYTHING from Django directly # into your settings, but ImproperlyConfigured is an exception. @@ -429,6 +430,43 @@ def get_env_setting(setting): 'operationType': 'RasterGroupedAverage', 'zoom': 0 } + }, + 'nlcd_slope': { + 'input': { + 'polygon': [], + 'polygonCRS': 'LatLng', + 'rasters': [ + 'nlcd-2011-30m-epsg5070-0.10.0', + 'us-percent-slope-30m-epsg5070', + ], + 'rasterCRS': 'ConusAlbers', + 'operationType': 'RasterGroupedCount', + 'zoom': 0 + } + }, + 'slope': { + 'input': { + 'polygon': [], + 'polygonCRS': 'LatLng', + 'rasters': [], + 'targetRaster': 'us-percent-slope-30m-epsg5070', + 'rasterCRS': 'ConusAlbers', + 'operationType': 'RasterGroupedAverage', + 'zoom': 0 + } + }, + 'nlcd_kfactor': { + 'input': { + 'polygon': [], + 'polygonCRS': 'LatLng', + 'rasters': [ + 'nlcd-2011-30m-epsg5070-0.10.0' + ], + 'targetRaster': 'us-ssugro-kfactor-30m-epsg5070', + 'rasterCRS': 'ConusAlbers', + 'operationType': 'RasterGroupedAverage', + 'zoom': 0 + } } } } diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index df6a51a2e..503c48f1f 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -10,6 +10,7 @@ GWLFE_DEFAULTS = { 'NRur': 10, # Number of Rural Land Use Categories 'NUrb': 6, # Number of Urban Land Use Categories + 'NLU': 16, # Total Number of Land Use Categories 'TranVersionNo': '1.4.0', # GWLF-E Version 'RecessionCoef': 0.06, # Recession Coefficient 'SeepCoef': 0, # Seepage Coefficient From 81d0c1da77b53876d18cd06b6641961159abda2e Mon Sep 17 00:00:00 2001 From: Hector Castro Date: Thu, 12 May 2016 21:56:03 -0400 Subject: [PATCH 058/136] Revert "Bump libpq version to 9.6.x" This reverts commit b4a657f6bd30b85846c0570050f80a965203c30f. --- deployment/ansible/group_vars/all | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/ansible/group_vars/all b/deployment/ansible/group_vars/all index b3c988c0e..298f5824e 100644 --- a/deployment/ansible/group_vars/all +++ b/deployment/ansible/group_vars/all @@ -21,7 +21,7 @@ postgresql_database: mmw postgresql_version: "9.4" postgresql_package_version: "9.4.*.pgdg14.04+1" postgresql_support_repository_channel: "main" -postgresql_support_libpq_version: "9.6*.pgdg14.04+1" +postgresql_support_libpq_version: "9.5.*.pgdg14.04+1" postgresql_support_psycopg2_version: "2.6" postgis_version: "2.1" postgis_package_version: "2.1.*.pgdg14.04+1" From 274c3ceafeb7b73e19f414b9bbde807ab72df3e0 Mon Sep 17 00:00:00 2001 From: Hector Castro Date: Thu, 12 May 2016 22:14:47 -0400 Subject: [PATCH 059/136] Bump OpenJDK version to 7u101 Includes a number of CVE fixes. --- deployment/ansible/group_vars/all | 2 +- deployment/ansible/roles.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/ansible/group_vars/all b/deployment/ansible/group_vars/all index 298f5824e..403d0e624 100644 --- a/deployment/ansible/group_vars/all +++ b/deployment/ansible/group_vars/all @@ -30,7 +30,7 @@ elasticsearch_cluster_name: "logstash" nodejs_npm_version: 2.1.17 -java_version: "7u95-*" +java_version: "7u101-*" graphite_carbon_version: "0.9.13-pre1" graphite_whisper_version: "0.9.13-pre1" diff --git a/deployment/ansible/roles.yml b/deployment/ansible/roles.yml index a38c6a62b..d077eb38c 100644 --- a/deployment/ansible/roles.yml +++ b/deployment/ansible/roles.yml @@ -45,6 +45,6 @@ - src: azavea.beaver version: 1.0.1 - src: azavea.java - version: 0.2.1 + version: 0.2.5 - src: azavea.docker version: 1.0.2 From 1e98343033351224717555be4aa7fb22558299c2 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Fri, 13 May 2016 11:16:39 -0400 Subject: [PATCH 060/136] Various fixes and compute SedAFactor --- src/mmw/apps/modeling/mapshed/calcs.py | 30 +++++++++++++++++++++++--- src/mmw/apps/modeling/mapshed/tasks.py | 25 +++++++++++---------- src/mmw/mmw/settings/base.py | 2 +- src/mmw/mmw/settings/gwlfe_settings.py | 3 ++- 4 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index f40dc0b52..01ce940e3 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -7,12 +7,14 @@ import numpy as np from collections import namedtuple +import decimal from gwlfe.enums import GrowFlag from django.conf import settings from django.db import connection +NRur = settings.GWLFE_DEFAULTS['NRur'] NUM_WEATHER_STATIONS = settings.GWLFE_CONFIG['NumWeatherStations'] KV_FACTOR = settings.GWLFE_CONFIG['KvFactor'] MONTHS = settings.GWLFE_DEFAULTS['Month'] @@ -124,7 +126,7 @@ def kv_coefficient(ecs): Original at Class1.vb@1.3.0:4989-4995 """ - kv = [ec * KV_FACTOR for ec in ecs] + kv = [ec * decimal.Decimal(KV_FACTOR) for ec in ecs] for m in range(1, 12): kv[m] = (kv[m] + kv[m-1]) / 2 @@ -468,8 +470,9 @@ def sediment_delivery_ratio(area_sq_km): def landuse_pcts(n_count): """ - Given a dictionary mapping NLCD Codes to counts of cells, returns a - dictionary mapping MapShed Land Use Types to percent area covered. + Given a dictionary mapping NLCD Codes to counts of cells, returns an + array mapping MapShed Land Use Types (as the index) to percent + area covered. """ total = sum(n_count.values()) @@ -516,3 +519,24 @@ def normal_sys(lu_area): SSLDM = settings.GWLFE_CONFIG['SSLDM'] return SSLDR * lu_area[14] + SSLDM * lu_area[11] + + +def sed_a_factor(landuse_pct_vals, cn, AEU, AvKF, AvSlope): + # see Class1.vb#10518 + urban_pct = sum(landuse_pct_vals[NRur:]) + + # see Class1.vb#10512 + avg_cn = 0 + nonzero_pct_sum = 0 + for (cn_val, pct) in zip(cn, landuse_pct_vals): + avg_cn += cn_val * pct + if cn_val != 0: + nonzero_pct_sum += pct + # We need to normalize since the land uses with cn_val ==0 + # shouldn't be counted as part of the average. + avg_cn = avg_cn / nonzero_pct_sum if nonzero_pct_sum > 0 else 0.0 + + # see Class1.vb#10521 + return ((0.00467 * urban_pct) + (0.000863 * AEU) + + (0.000001 * avg_cn) + (0.000425 * AvKF) + + (0.000001 * AvSlope) - 0.000036) diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index 96e4b26e0..9b9598fac 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -36,9 +36,11 @@ sediment_delivery_ratio, landuse_pcts, normal_sys, + sed_a_factor ) -NLU = settings.GWLFE_DEFAULTS['NLU'] +NLU = settings.GWLFE_CONFIG['NLU'] +NRur = settings.GWLFE_DEFAULTS['NRur'] AG_NLCD_CODES = settings.GWLFE_CONFIG['AgriculturalNLCDCodes'] ACRES_PER_SQM = 0.000247105 HECTARES_PER_SQM = 0.0001 @@ -144,7 +146,7 @@ def collect_data(geop_result, geojson): z.CN = np.array(geop_result['cn']) z.SedPhos = geop_result['sed_phos'] - z.Area = np.array(geop_result['landuse_pcts'] * area * HECTARES_PER_SQM) + z.Area = np.array(geop_result['landuse_pcts']) * area * HECTARES_PER_SQM z.NormalSys = normal_sys(z.Area) @@ -169,7 +171,10 @@ def collect_data(geop_result, geojson): z.GrNitrConc = geop_result['gr_nitr_conc'] z.GrPhosConc = geop_result['gr_phos_conc'] z.MaxWaterCap = geop_result['avg_awc'] - + z.SedAFactor = sed_a_factor(geop_result['landuse_pcts'], + z.CN, z.AEU, z.AvKF, z.AvSlope) + # TODO remove before merging + print('SedAFactor: ' + z.SedAFactor) # TODO pass real input to model instead of reading it from gms file gms_filename = join(dirname(abspath(__file__)), 'data/sample_input.gms') gms_file = open(gms_filename, 'r') @@ -351,7 +356,7 @@ def nlcd_slope(result): output = { 'ag_slope_3_pct': ag_slope_3_pct, 'ag_slope_3_8_pct': ag_slope_3_8_pct, - 'n41': n41, + 'n41': n41 } # TODO remove before merging @@ -369,7 +374,7 @@ def slope(result): # average slope over the AOI # see Class1.vb#6252 - avg_slope = result + avg_slope = result[0] output = { 'avg_slope': avg_slope @@ -386,7 +391,7 @@ def get_lu_index(nlcd): lu_index = 1 elif nlcd == 82: lu_index = 2 - elif nlcd in [41, 42, 43]: + elif nlcd in [41, 42, 43, 52]: lu_index = 3 elif nlcd in [90, 95]: lu_index = 4 @@ -395,13 +400,11 @@ def get_lu_index(nlcd): elif nlcd in [12, 31]: lu_index = 8 elif nlcd == 22: - lu_index = 10 - elif nlcd == 23: lu_index = 11 - elif nlcd == 24: + elif nlcd == 23: lu_index = 12 - elif nlcd == 11: - lu_index = 16 + elif nlcd == 24: + lu_index = 13 else: return None diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index 85fb0c32f..f5ee5ac1f 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -126,7 +126,7 @@ def get_env_setting(setting): CELERY_WORKER_DIRECT = True CELERY_CREATE_MISSING_QUEUES = True CELERY_CHORD_PROPAGATES = True -CELERY_CHORD_UNLOCK_MAX_RETRIES = 10 +CELERY_CHORD_UNLOCK_MAX_RETRIES = 20 # END CELERY CONFIGURATION diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index 503c48f1f..606988ab4 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -10,7 +10,6 @@ GWLFE_DEFAULTS = { 'NRur': 10, # Number of Rural Land Use Categories 'NUrb': 6, # Number of Urban Land Use Categories - 'NLU': 16, # Total Number of Land Use Categories 'TranVersionNo': '1.4.0', # GWLF-E Version 'RecessionCoef': 0.06, # Recession Coefficient 'SeepCoef': 0, # Seepage Coefficient @@ -448,6 +447,8 @@ } GWLFE_CONFIG = { + 'NLU': (GWLFE_DEFAULTS['NUrb'] + + GWLFE_DEFAULTS['NRur']), # Total Number of Land Use Categories 'NumWeatherStations': 2, # Number of weather stations to consider for each polygon 'KvFactor': 1.16, # Original at Class1.vb@1.3.0:4987 'Livestock': ['dairy_cows', 'beef_cows', 'hogs', 'sheep', 'horses'], From c03ed9109727c4ca34bb28afa2ba285794b24801 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Fri, 13 May 2016 12:52:11 -0400 Subject: [PATCH 061/136] Add BMP configs for GWLFE * Add configurations for GWLFE BMPs * Allow style for GWLFE BMPs to be copied from other modification configs --- src/mmw/js/src/core/modificationConfig.json | 47 ++++++++++++++++ .../src/modeling/modificationConfigUtils.js | 54 +++++++++++-------- 2 files changed, 79 insertions(+), 22 deletions(-) diff --git a/src/mmw/js/src/core/modificationConfig.json b/src/mmw/js/src/core/modificationConfig.json index 0460a9347..3374fae4d 100644 --- a/src/mmw/js/src/core/modificationConfig.json +++ b/src/mmw/js/src/core/modificationConfig.json @@ -133,5 +133,52 @@ "name": "C/D - Medium/Very Slow Infiltration", "shortName": "C/D - Medium/Very Slow", "summary": "Group C/D soils have slow infiltration rates where artificially drained but behave as Group D soils where undrained." + }, + "cover_crops": { + "name": "Cover Crops", + "summary": "Use of annual or perennial plant cover to protect the soil from erosion during the time period between the harvesting and planting of the primary crop. In addition to reducing soil erosion, cover crops can also limit nitrogen loss from cropped areas. Use of annual or perennial plant cover to protect the soil from erosion during the time period between the harvesting and planting of the primary crop. In addition to reducing soil erosion, cover crops can also limit nitrogen loss from cropped areas.", + "copyStyle": "deciduous_forest" + }, + "conservation_tillage": { + "name": "Conservation Tillage", + "summary": "The purpose of this BMP is to leave enough residue from harvested crops on the soil surface (at least 30%) to significantly reduce soil erosion. Variations include seasonal residue management, mulch tillage and no-till planting.", + "copyStyle": "barren_land" + }, + "nutrient_management": { + "name": "Nutrient Management", + "summary": "Refers to the planned use of organic and/or inorganic nutrients to sustain optimum crop production while at the same time protecting the quality of nearby water resources. This practice usually includes a farm-wide nutrient management plan that minimizes the use of animal wastes or commercial fertilizers to the greatest degree practicable.", + "copyStyle": "pasture" + }, + "waste_management": { + "name": "Animal Waste Management Systems (Livestock & Poultry)", + "shortName": "Waste Management", + "summary": "These are systems that are designed to collect runoff and/or wastes from confined animal operations for the purpose of breaking down organic wastes via aerobic or anaerobic processes. Typical examples include waste lagoons or holding tanks that collect the wastes and prevent their discharge to nearby streams.", + "copyStyle": "grassland" + }, + "buffer_strips": { + "name": "Vegetated Buffer Strips", + "shortName": "Veg Buffer Strips", + "summary": "Areas of trees and/or grasses planted along streams or lakes that are designed to capture and renovate surface runoff and shallow subsurface flow from agricultural and urban areas via the processes of filtration, infiltration, absorption, adsorption, uptake, denitrification, volatilization, and deposition. A buffer width of 30 m (roughly 100 ft) is assumed.", + "copyStyle": "no_till" + }, + "streambank_fencing": { + "name": "Streambank Fencing", + "summary": "This practice involves the construction of fencing that prohibits cattle from trampling stream banks, destroying protective vegetation, stirring up sediment in the streambed, and depositing organic waste directly into the stream.", + "copyStyle": "cultivated_crops" + }, + "streambank_stabilization": { + "name": "Streambank Stabilization", + "summary": "The use of rip-rap, gabion walls, or a “bio-engineering” solution of some type along the edges of a stream to protect the banks during periods of heavy stream flow, thereby reducing direct stream bank erosion. The banks may also be covered with rocks, grass, trees, shrubs, and other protective surfaces to reduce erosion as well.", + "copyStyle": "woody_wetlands" + }, + "water_retention": { + "name": "Surface Water Retention", + "summary": "With this practice, surface runoff from impervious areas is detained via this use of designed structures (e.g., retention/detention basins, constructed wetlands) for the purpose of reducing peak flows and promoting the reduction of nutrient and sediment loads primarily via sedimentation and/or plant uptake.", + "copyStyle": "rain_garden" + }, + "infiltration": { + "name": "Infiltration / Bioretention", + "summary": "Various approaches that achieve runoff reduction and the treatment of urban pollutants (primarily nutrients) through the promotion of infiltration, evapotranspiration and renovation via subsurface flow. Examples include the use of pervious materials/porous pavement, infiltration basins/trenches, and vegetated roofs. It is assumed that runoff from a 2.54 cm (I.0 in) rainfall event is captured.", + "copyStyle": "infiltration_trench" } } diff --git a/src/mmw/js/src/modeling/modificationConfigUtils.js b/src/mmw/js/src/modeling/modificationConfigUtils.js index d4b7aa2de..97accfbbe 100644 --- a/src/mmw/js/src/modeling/modificationConfigUtils.js +++ b/src/mmw/js/src/modeling/modificationConfigUtils.js @@ -11,13 +11,17 @@ function resetConfig() { modificationConfig = require('../core/modificationConfig.json'); } +function unknownModKey(modKey) { + console.warn('Unknown Land Cover or Conservation Practice: ' + modKey); + return ''; +} + // modKey should be a key in modificationsConfig (eg. 'open_water'). function getHumanReadableName(modKey) { if (modificationConfig[modKey]) { return modificationConfig[modKey].name; } - console.warn('Unknown Land Cover or Conservation Practice: ' + modKey); - return ''; + return unknownModKey(modKey); } // If no shortName, just use name. @@ -29,36 +33,42 @@ function getHumanReadableShortName(modKey) { return modificationConfig[modKey].name; } } - console.warn('Unknown Land Cover or Conservation Practice: ' + modKey); - return ''; + return unknownModKey(modKey); } function getHumanReadableSummary(modKey) { if (modificationConfig[modKey]) { return modificationConfig[modKey].summary || ''; } - console.warn('Unknown Land Cover or Conservation Practice: ' + modKey); - return ''; + return unknownModKey(modKey); } var getDrawOpts = function(modKey) { - if (modKey && modificationConfig[modKey] && modificationConfig[modKey].strokeColor) { - return { - color: modificationConfig[modKey].strokeColor, - opacity: 1, - weight: 3, - fillColor: 'url(#fill-' + modKey + ')', - fillOpacity: 0.74 - }; + var defaultStyle = { + color: '#888', + opacity: 1, + weight: 3, + fillColor: '#888', + fillOpacity: 0.74 + }; + + if (modKey && modificationConfig[modKey]) { + var config = modificationConfig[modKey]; + if (config.copyStyle) { + return getDrawOpts(config.copyStyle); + } else if (config.strokeColor){ + return { + color: config.strokeColor, + opacity: 1, + weight: 3, + fillColor: 'url(#fill-' + modKey + ')', + fillOpacity: 0.74 + }; + } else { + return defaultStyle; + } } else { - // Unknown modKey, return generic grey - return { - color: '#888', - opacity: 1, - weight: 3, - fillColor: '#888', - fillOpacity: 0.74 - }; + return defaultStyle; } }; From 3e9a0662e1a540758eadaf5e0f4fd79cc794c44e Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 13 May 2016 16:26:10 -0400 Subject: [PATCH 062/136] Remove accidentally committed changes Visual Studio Code merged #1289 in unexpected fashion. This cleares up the accidental changes that were committed. --- src/mmw/apps/modeling/mapshed/tasks.py | 65 +++++++++++--------------- 1 file changed, 28 insertions(+), 37 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index 9b9598fac..ec211bc67 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -173,8 +173,7 @@ def collect_data(geop_result, geojson): z.MaxWaterCap = geop_result['avg_awc'] z.SedAFactor = sed_a_factor(geop_result['landuse_pcts'], z.CN, z.AEU, z.AvKF, z.AvSlope) - # TODO remove before merging - print('SedAFactor: ' + z.SedAFactor) + # TODO pass real input to model instead of reading it from gms file gms_filename = join(dirname(abspath(__file__)), 'data/sample_input.gms') gms_file = open(gms_filename, 'r') @@ -359,9 +358,6 @@ def nlcd_slope(result): 'n41': n41 } - # TODO remove before merging - print(output) - return output @@ -380,37 +376,9 @@ def slope(result): 'avg_slope': avg_slope } - # TODO remove before merging - print(output) - return output -def get_lu_index(nlcd): - if nlcd == 81: - lu_index = 1 - elif nlcd == 82: - lu_index = 2 - elif nlcd in [41, 42, 43, 52]: - lu_index = 3 - elif nlcd in [90, 95]: - lu_index = 4 - elif nlcd in [21, 71]: - lu_index = 7 - elif nlcd in [12, 31]: - lu_index = 8 - elif nlcd == 22: - lu_index = 11 - elif nlcd == 23: - lu_index = 12 - elif nlcd == 24: - lu_index = 13 - else: - return None - - return lu_index - 1 - - @shared_task(throws=Exception) def nlcd_kfactor(result): if 'error' in result: @@ -423,7 +391,7 @@ def nlcd_kfactor(result): kf = [0.0] * NLU for nlcd_code, kfactor in result.iteritems(): lu_ind = get_lu_index(nlcd_code) - if lu_ind: + if lu_ind is not None: kf[lu_ind] = kfactor # average kfactor across all land uses, ignoring zero values @@ -438,9 +406,6 @@ def nlcd_kfactor(result): 'avg_kf': avg_kf } - # TODO remove before merging - print(output) - return output @@ -488,3 +453,29 @@ def combine(geop_results): def parse_sjs_result(sjs_result): # Convert string "List(1,2,3)" into tuple (1,2,3) for each key return {make_tuple(key[4:]): val for key, val in sjs_result.items()} + + +def get_lu_index(nlcd): + # Convert NLCD code into MapShed Land Use Index + if nlcd == 81: + lu_index = 1 + elif nlcd == 82: + lu_index = 2 + elif nlcd in [41, 42, 43, 52]: + lu_index = 3 + elif nlcd in [90, 95]: + lu_index = 4 + elif nlcd in [21, 71]: + lu_index = 7 + elif nlcd in [12, 31]: + lu_index = 8 + elif nlcd == 22: + lu_index = 11 + elif nlcd == 23: + lu_index = 12 + elif nlcd == 24: + lu_index = 13 + else: + return None + + return lu_index - 1 From 8ef780dc5e27182e4c1aa4d29da55f5f0832708a Mon Sep 17 00:00:00 2001 From: Hector Castro Date: Mon, 16 May 2016 13:34:08 -0400 Subject: [PATCH 063/136] Use more precise Apache version Previously, the default glob for Apache was `2.4.*`, which captures a more recent 2.4.10 package. Pinning to `2.4.7-*` ensures that everyone is on the same version in the Ubuntu repositories. --- deployment/ansible/group_vars/all | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deployment/ansible/group_vars/all b/deployment/ansible/group_vars/all index 403d0e624..b5bdb13ca 100644 --- a/deployment/ansible/group_vars/all +++ b/deployment/ansible/group_vars/all @@ -30,6 +30,8 @@ elasticsearch_cluster_name: "logstash" nodejs_npm_version: 2.1.17 +apache_version: "2.4.7-*" + java_version: "7u101-*" graphite_carbon_version: "0.9.13-pre1" From c42dd4a966e268da31d5b2f7b4baf9eab01cb2dc Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Fri, 13 May 2016 13:01:14 -0400 Subject: [PATCH 064/136] Refactor modifications control and add GWLFE control * Refactor modifications controls to get row configuration from model * Consolidate DrawControlView and ModificationControlView into single class for simplicity * Rename stuff to make things easier to understand * Add manualMode to dropdown. When true, modifications are manually entered (GWLFE), and drawn (Tr55) otherwise. * Manually control the opening/closing of the dropdown to allow for different behavior depending on drawMode * When manualMode=true, clicking on thumbnail leads to newly added ManualEntryView * Use GwlfeConservationPracticeView when model package is GWLFE --- .../src/core/templates/modificationPopup.html | 2 +- src/mmw/js/src/modeling/controls.js | 188 +++++++++++++++--- src/mmw/js/src/modeling/filters.js | 10 +- src/mmw/js/src/modeling/models.js | 19 +- .../controls/conservationPractice.html | 24 --- .../templates/controls/landCover.html | 33 --- .../templates/controls/manualEntry.html | 18 ++ .../templates/controls/modDropdown.html | 8 + .../modeling/templates/controls/summary.html | 2 - .../templates/controls/thumbSelect.html | 25 +++ .../modeling/templates/controls/utils.html | 10 - .../templates/scenarioToolbarTabContent.html | 2 +- src/mmw/sass/components/_dropdowns.scss | 28 ++- 13 files changed, 260 insertions(+), 109 deletions(-) delete mode 100644 src/mmw/js/src/modeling/templates/controls/conservationPractice.html delete mode 100644 src/mmw/js/src/modeling/templates/controls/landCover.html create mode 100644 src/mmw/js/src/modeling/templates/controls/manualEntry.html create mode 100644 src/mmw/js/src/modeling/templates/controls/modDropdown.html delete mode 100644 src/mmw/js/src/modeling/templates/controls/summary.html create mode 100644 src/mmw/js/src/modeling/templates/controls/thumbSelect.html delete mode 100644 src/mmw/js/src/modeling/templates/controls/utils.html diff --git a/src/mmw/js/src/core/templates/modificationPopup.html b/src/mmw/js/src/core/templates/modificationPopup.html index c605446f3..d4690d4c2 100644 --- a/src/mmw/js/src/core/templates/modificationPopup.html +++ b/src/mmw/js/src/core/templates/modificationPopup.html @@ -1,4 +1,4 @@ -

    {{ value|tr55Name }}

    +

    {{ value|modName }}

    diff --git a/src/mmw/js/src/modeling/controls.js b/src/mmw/js/src/modeling/controls.js index 42291edb4..ee987ac6a 100644 --- a/src/mmw/js/src/modeling/controls.js +++ b/src/mmw/js/src/modeling/controls.js @@ -2,17 +2,16 @@ var $ = require('jquery'), _ = require('underscore'), - Backbone = require('../../shim/backbone'), Marionette = require('../../shim/backbone.marionette'), App = require('../app'), drawUtils = require('../draw/utils'), coreUtils = require('../core/utils'), models = require('./models'), modificationConfigUtils = require('./modificationConfigUtils'), - landCoverTmpl = require('./templates/controls/landCover.html'), - conservationPracticeTmpl = require('./templates/controls/conservationPractice.html'), precipitationTmpl = require('./templates/controls/precipitation.html'), - summaryTmpl = require('./templates/controls/summary.html'); + manualEntryTmpl = require('./templates/controls/manualEntry.html'), + thumbSelectTmpl = require('./templates/controls/thumbSelect.html'), + modDropdownTmpl = require('./templates/controls/modDropdown.html'); // Simulation input controls base class. var ControlView = Marionette.LayoutView.extend({ @@ -35,24 +34,51 @@ var ControlView = Marionette.LayoutView.extend({ } }); -// Drawing control base class. -var DrawControlView = ControlView.extend({ +var ThumbSelectView = Marionette.ItemView.extend({ + template: thumbSelectTmpl, + + initialize: function(options) { + this.model.set('activeMod', null); + this.addModification = options.addModification; + }, + ui: { + thumb: '.thumb', drawControl: '[data-value]' }, events: { - 'click @ui.drawControl': 'startDrawing' + 'click @ui.drawControl': 'onThumbClick', + 'mouseenter @ui.thumb': 'onThumbHover' + }, + + modelEvents: { + 'change:activeMod': 'render' + }, + + onThumbHover: function(e) { + var value = $(e.currentTarget).data('value'); + this.model.set('activeMod', value); + }, + + onThumbClick: function(e) { + var $el = $(e.currentTarget), + controlName = this.model.get('controlName'), + controlValue = $el.data('value'); + + if (this.model.get('manualMode')) { + this.startManual(controlName, controlValue); + } else { + this.startDrawing(controlName, controlValue); + } }, - startDrawing: function(e) { + startDrawing: function(controlName, controlValue) { var self = this, - $el = $(e.currentTarget), - controlName = this.getControlName(), - controlValue = $el.data('value'), map = App.getLeafletMap(), drawOpts = modificationConfigUtils.getDrawOpts(controlValue); + this.model.set('dropdownOpen', false); drawUtils.drawPolygon(map, drawOpts).then(function(geojson) { self.addModification(new models.ModificationModel({ name: controlName, @@ -60,40 +86,106 @@ var DrawControlView = ControlView.extend({ shape: geojson })); }); + }, + + startManual: function(controlName, controlValue) { + this.model.set('manualMod', controlValue); } }); -var SummaryView = Marionette.ItemView.extend({ - template: summaryTmpl +var ManualEntryView = Marionette.ItemView.extend({ + template: manualEntryTmpl, + + ui: { + 'backButton': '.back-button', + 'convertButton': '.convert-button' + }, + + events: { + 'click @ui.backButton': 'clearManualMod', + 'click @ui.convertButton': 'convert' + }, + + clearManualMod: function() { + this.model.set('manualMod', null); + }, + + convert: function() { + // TODO add modification + this.model.set('dropdownOpen', false); + this.clearManualMod(); + } }); -var ModificationsView = DrawControlView.extend({ - ui: _.defaults({ - thumb: '.thumb', - button: 'button' - }, DrawControlView.prototype.ui), +var ModificationsView = ControlView.extend({ + template: modDropdownTmpl, + + initialize: function(options) { + ControlView.prototype.initialize.apply(this, [options]); + var self = this; + + // If clicked outside this view and dropdown is open, then close it. + $(document).mouseup(function(e) { + var isTargetOutsideDropdown = $(e.target).parents('.dropdown-menu').length === 0; + + if (isTargetOutsideDropdown && self.model.get('dropdownOpen')) { + self.model.set('dropdownOpen', false); + } + }); + }, + + ui: { + dropdownButton: '.dropdown-button' + }, + + events: { + 'click @ui.dropdownButton': 'onClickDropdownButton' + }, + + modelEvents: { + 'change:dropdownOpen': 'render', + 'change:manualMod': 'updateContent' + }, regions: { - summaryRegion: '.summary-region' + modContentRegion: '.mod-content-region' }, - events: _.defaults({ - 'mouseenter @ui.thumb': 'onMouseHover' - }, DrawControlView.prototype.events), + onClickDropdownButton: function() { + if (!this.model.get('dropdownOpen')) { + this.model.set('dropdownOpen', true); + } + }, - onMouseHover: function(e) { - var value = $(e.currentTarget).data('value'); + onRender: function() { + this.updateContent(); + }, - this.summaryRegion.show(new SummaryView({ - model: new Backbone.Model({ - value: value - }) - })); + updateContent: function() { + if (this.model.get('manualMod')) { + this.modContentRegion.show(new ManualEntryView({model: this.model})); + } else { + this.modContentRegion.show(new ThumbSelectView({ + addModification: this.addModification, + model: this.model + })); + } } }); var LandCoverView = ModificationsView.extend({ - template: landCoverTmpl, + initialize: function(options) { + ModificationsView.prototype.initialize.apply(this, [options]); + this.model.set({ + controlName: this.getControlName(), + controlDisplayName: 'Land Cover', + modRows: [ + ['open_water', 'developed_open', 'developed_low', 'developed_med'], + ['developed_high', 'barren_land', 'deciduous_forest', 'shrub'], + ['grassland', 'pasture', 'cultivated_crops', 'woody_wetlands'] + ] + }); + }, getControlName: function() { return 'landcover'; @@ -101,13 +193,44 @@ var LandCoverView = ModificationsView.extend({ }); var ConservationPracticeView = ModificationsView.extend({ - template: conservationPracticeTmpl, + initialize: function(options) { + ModificationsView.prototype.initialize.apply(this, [options]); + this.model.set({ + controlName: this.getControlName(), + controlDisplayName: 'Conservation Practice', + modRows: [ + ['rain_garden', 'infiltration_trench', 'porous_paving'], + ['green_roof', 'no_till', 'cluster_housing'] + ] + }); + }, getControlName: function() { return 'conservation_practice'; } }); +var GwlfeConservationPracticeView = ModificationsView.extend({ + initialize: function(options) { + ModificationsView.prototype.initialize.apply(this, [options]); + this.model.set({ + controlName: this.getControlName(), + controlDisplayName: 'Conservation Practice', + manualMode: true, + manualMod: null, + modRows: [ + ['cover_crops', 'conservation_tillage', 'nutrient_management'], + ['waste_management', 'buffer_strips', 'streambank_fencing'], + ['streambank_stabilization', 'water_retention', 'infiltration'] + ] + }); + }, + + getControlName: function() { + return 'gwlfe_conservation_practice'; + } +}); + var PrecipitationSynchronizer = (function() { var isEnabled = false, precipViews = []; @@ -250,6 +373,8 @@ function getControlView(controlName) { return LandCoverView; case 'conservation_practice': return ConservationPracticeView; + case 'gwlfe_conservation_practice': + return GwlfeConservationPracticeView; case 'precipitation': return PrecipitationView; } @@ -259,6 +384,7 @@ function getControlView(controlName) { module.exports = { LandCoverView: LandCoverView, ConservationPracticeView: ConservationPracticeView, + GwlfeConservationPracticeView: GwlfeConservationPracticeView, PrecipitationView: PrecipitationView, getControlView: getControlView, PrecipitationSynchronizer: PrecipitationSynchronizer diff --git a/src/mmw/js/src/modeling/filters.js b/src/mmw/js/src/modeling/filters.js index ed59bfafc..e90b0d50f 100644 --- a/src/mmw/js/src/modeling/filters.js +++ b/src/mmw/js/src/modeling/filters.js @@ -3,14 +3,18 @@ var nunjucks = require('nunjucks'), modUtils = require('./modificationConfigUtils'); -nunjucks.env.addFilter('tr55Name', function(val) { +nunjucks.env.addFilter('modName', function(val) { return modUtils.getHumanReadableName(val); }); -nunjucks.env.addFilter('tr55ShortName', function(val) { +nunjucks.env.addFilter('modShortName', function(val) { return modUtils.getHumanReadableShortName(val); }); -nunjucks.env.addFilter('tr55Summary', function(val) { +nunjucks.env.addFilter('modSummary', function(val) { return modUtils.getHumanReadableSummary(val); }); + +nunjucks.env.addFilter('modFill', function(val) { + return modUtils.getDrawOpts(val).fillColor; +}); diff --git a/src/mmw/js/src/modeling/models.js b/src/mmw/js/src/modeling/models.js index 7a6e25fb5..b886d09a0 100644 --- a/src/mmw/js/src/modeling/models.js +++ b/src/mmw/js/src/modeling/models.js @@ -16,7 +16,14 @@ var TR55_PACKAGE = 'tr-55'; var ModelPackageControlModel = Backbone.Model.extend({ defaults: { - name: '' + name: '', + controlName: '', + controlDisplayName: '', + manualMode: false, + manualMod: '', + activeMod: '', + modRows: null, + dropdownOpen: false }, // Return true if this is an input control and false if it is a @@ -846,8 +853,14 @@ function getControlsForModelPackage(modelPackageName, options) { ]); } } else if (modelPackageName === GWLFE) { - // TODO add controls for GWLFE - return new ModelPackageControlsCollection([]); + if (options && (options.compareMode || + options.is_current_conditions)) { + return new ModelPackageControlsCollection(); + } else { + return new ModelPackageControlsCollection([ + new ModelPackageControlModel({ name: 'gwlfe_conservation_practice' }) + ]); + } } throw 'Model package not supported ' + modelPackageName; diff --git a/src/mmw/js/src/modeling/templates/controls/conservationPractice.html b/src/mmw/js/src/modeling/templates/controls/conservationPractice.html deleted file mode 100644 index 3458591c2..000000000 --- a/src/mmw/js/src/modeling/templates/controls/conservationPractice.html +++ /dev/null @@ -1,24 +0,0 @@ -{% import './utils.html' as utils %} - - diff --git a/src/mmw/js/src/modeling/templates/controls/landCover.html b/src/mmw/js/src/modeling/templates/controls/landCover.html deleted file mode 100644 index fc7bec29c..000000000 --- a/src/mmw/js/src/modeling/templates/controls/landCover.html +++ /dev/null @@ -1,33 +0,0 @@ -{% import './utils.html' as utils %} - - diff --git a/src/mmw/js/src/modeling/templates/controls/manualEntry.html b/src/mmw/js/src/modeling/templates/controls/manualEntry.html new file mode 100644 index 000000000..1a44a5ae4 --- /dev/null +++ b/src/mmw/js/src/modeling/templates/controls/manualEntry.html @@ -0,0 +1,18 @@ +
    +
    +
    + +
    {{ manualMod|modShortName}}
    +
    + + + +
    +
    + + +
    + +
    diff --git a/src/mmw/js/src/modeling/templates/controls/modDropdown.html b/src/mmw/js/src/modeling/templates/controls/modDropdown.html new file mode 100644 index 000000000..669670a76 --- /dev/null +++ b/src/mmw/js/src/modeling/templates/controls/modDropdown.html @@ -0,0 +1,8 @@ +
    + + +
    diff --git a/src/mmw/js/src/modeling/templates/controls/summary.html b/src/mmw/js/src/modeling/templates/controls/summary.html deleted file mode 100644 index 14e4e5c3a..000000000 --- a/src/mmw/js/src/modeling/templates/controls/summary.html +++ /dev/null @@ -1,2 +0,0 @@ -

    {{ value|tr55Name }}

    -

    {{ value|tr55Summary }}

    diff --git a/src/mmw/js/src/modeling/templates/controls/thumbSelect.html b/src/mmw/js/src/modeling/templates/controls/thumbSelect.html new file mode 100644 index 000000000..b8f5491f7 --- /dev/null +++ b/src/mmw/js/src/modeling/templates/controls/thumbSelect.html @@ -0,0 +1,25 @@ +
    +
    + {% for modRow in modRows %} +
      + {% for modKey in modRow %} +
    • +
      + + + +
      + +
      • + {% endfor %} +
    + {% endfor %} +
    + + {% if activeMod %} +
    +

    {{ activeMod|modName }}

    +

    {{ activeMod|modSummary }}

    +
    + {% endif %} +
    diff --git a/src/mmw/js/src/modeling/templates/controls/utils.html b/src/mmw/js/src/modeling/templates/controls/utils.html deleted file mode 100644 index 5660244fc..000000000 --- a/src/mmw/js/src/modeling/templates/controls/utils.html +++ /dev/null @@ -1,10 +0,0 @@ -{% macro thumb(value, size=56) %} -
  • -
    - - - -
    - -
    • -{% endmacro %} diff --git a/src/mmw/js/src/modeling/templates/scenarioToolbarTabContent.html b/src/mmw/js/src/modeling/templates/scenarioToolbarTabContent.html index d4a014fef..17e1ca279 100644 --- a/src/mmw/js/src/modeling/templates/scenarioToolbarTabContent.html +++ b/src/mmw/js/src/modeling/templates/scenarioToolbarTabContent.html @@ -31,7 +31,7 @@ {% for model in models %} - + {% if editable %} + {% endif %} {% endfor %} @@ -17,9 +17,9 @@ {% for value in row %} {% if loop.index0 == 0 %} - + {% else %} - + {% endif %} {% endfor %} diff --git a/src/mmw/js/src/modeling/models.js b/src/mmw/js/src/modeling/models.js index b886d09a0..ed1d22598 100644 --- a/src/mmw/js/src/modeling/models.js +++ b/src/mmw/js/src/modeling/models.js @@ -625,9 +625,9 @@ var ScenarioModel = Backbone.Model.extend({ this.get('results').forEach(function(resultModel) { var resultName = resultModel.get('name'); - if (serverResults[resultName]) { + if (serverResults) { resultModel.set({ - 'result': serverResults[resultName], + 'result': serverResults, 'inputmod_hash': serverResults.inputmod_hash }); } else { diff --git a/src/mmw/js/src/modeling/tr55/quality/templates/tableRow.html b/src/mmw/js/src/modeling/tr55/quality/templates/tableRow.html index 2a5c64829..5862ef3b2 100644 --- a/src/mmw/js/src/modeling/tr55/quality/templates/tableRow.html +++ b/src/mmw/js/src/modeling/tr55/quality/templates/tableRow.html @@ -1,3 +1,3 @@ - - + + diff --git a/src/mmw/js/src/modeling/tr55/quality/views.js b/src/mmw/js/src/modeling/tr55/quality/views.js index 522518d23..3a1da4cfb 100644 --- a/src/mmw/js/src/modeling/tr55/quality/views.js +++ b/src/mmw/js/src/modeling/tr55/quality/views.js @@ -50,7 +50,7 @@ var ResultView = Marionette.LayoutView.extend({ })); } else { var dataCollection = new Backbone.Collection( - this.model.get('result') + this.model.get('result').quality ); this.tableRegion.show(new TableView({ @@ -161,7 +161,7 @@ var CompareChartView = Marionette.ItemView.extend({ } var chartEl = this.$el.find('.bar-chart').get(0), - result = this.model.get('result'), + result = this.model.get('result').quality, seriesDisplayNames = ['Oxygen Demand', 'Suspended Solids', 'Nitrogen', diff --git a/src/mmw/js/src/modeling/tr55/runoff/templates/tableRow.html b/src/mmw/js/src/modeling/tr55/runoff/templates/tableRow.html index 0db0a1c01..f66cd9225 100644 --- a/src/mmw/js/src/modeling/tr55/runoff/templates/tableRow.html +++ b/src/mmw/js/src/modeling/tr55/runoff/templates/tableRow.html @@ -1,3 +1,3 @@ - - + + diff --git a/src/mmw/js/src/modeling/tr55/runoff/views.js b/src/mmw/js/src/modeling/tr55/runoff/views.js index 50ea4fe78..3f4bd8665 100644 --- a/src/mmw/js/src/modeling/tr55/runoff/views.js +++ b/src/mmw/js/src/modeling/tr55/runoff/views.js @@ -88,7 +88,7 @@ var ChartView = Marionette.ItemView.extend({ } var chartEl = this.$el.find('.bar-chart').get(0), - result = this.model.get('result'), + result = this.model.get('result').runoff, seriesNames = ['inf', 'runoff', 'et'], seriesDisplayNames = ['Infiltration', 'Runoff', 'Evapotranspiration'], labelNames, @@ -155,7 +155,7 @@ var TableView = Marionette.CompositeView.extend({ initialize: function() { this.aoiVolumeModel = this.options.aoiVolumeModel; - this.tr55Results = this.model.get('result'); + this.tr55Results = this.model.get('result').runoff; this.collection = this.formatData(); }, diff --git a/src/mmw/js/src/modeling/views.js b/src/mmw/js/src/modeling/views.js index d9570da18..e8562cae1 100644 --- a/src/mmw/js/src/modeling/views.js +++ b/src/mmw/js/src/modeling/views.js @@ -26,7 +26,7 @@ tr55RunoffViews = require('./tr55/runoff/views.js'), tr55QualityViews = require('./tr55/quality/views.js'), gwlfeRunoffViews = require('./gwlfe/runoff/views.js'), - gwlfeQualityViews = tr55QualityViews; // TODO make gwlfe quality views + gwlfeQualityViews = require('./gwlfe/quality/views.js'); var ENTER_KEYCODE = 13, ESCAPE_KEYCODE = 27; diff --git a/src/mmw/sass/pages/_model.scss b/src/mmw/sass/pages/_model.scss index 7db53d570..118e6e639 100644 --- a/src/mmw/sass/pages/_model.scss +++ b/src/mmw/sass/pages/_model.scss @@ -49,6 +49,10 @@ height: 100%; width: 100%; + .mean-flow { + font-size: 16px; + } + .runoff-chart-container { height: 300px; width: 100%; From 37fa37b46b20b871f6ca2b9ff7e23d656ad874d5 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 18 May 2016 18:01:57 -0400 Subject: [PATCH 076/136] Poll for MapShed before running GWLF-E * Add a MapshedTaskRunner that can poll for results at the MapShed endpoint. Unlike the GWLFE and TR55 task runners which are attached to the ScenarioModel, this is instantiated locally only for the purpose of fetching GIS Data. * Add functions fetchGisData and fetchGisDataIfNeeded to the ProjectModel which run MapShed in the case of GWLFE projects, and nothing in the case of TR55 projects. These return promises so the callers can be notified when the run is complete. If a promise already exists, we simply return that instead of firing a second run, which prevents duplication. In case of TR55 where no GIS Data needs to be gathered, we return an already resolved promise which doesn't block further execution. --- src/mmw/js/src/modeling/models.js | 148 ++++++++++++++++++++++++++---- 1 file changed, 132 insertions(+), 16 deletions(-) diff --git a/src/mmw/js/src/modeling/models.js b/src/mmw/js/src/modeling/models.js index baa0c6670..c3cc64c03 100644 --- a/src/mmw/js/src/modeling/models.js +++ b/src/mmw/js/src/modeling/models.js @@ -1,6 +1,7 @@ "use strict"; -var Backbone = require('../../shim/backbone'), +var $ = require('jquery'), + Backbone = require('../../shim/backbone'), _ = require('lodash'), utils = require('../core/utils'), settings = require('../core/settings'), @@ -10,6 +11,7 @@ var Backbone = require('../../shim/backbone'), turfErase = require('turf-erase'), turfIntersect = require('turf-intersect'); +var MAPSHED = 'mapshed'; var GWLFE = 'gwlfe'; var TR55_TASK = 'tr55'; var TR55_PACKAGE = 'tr-55'; @@ -51,6 +53,16 @@ var Tr55TaskModel = coreModels.TaskModel.extend({ ) }); +var MapshedTaskModel = coreModels.TaskModel.extend({ + defaults: _.extend( + { + taskName: MAPSHED, + taskType: 'modeling' + }, + coreModels.TaskModel.prototype.defaults + ) +}); + var GwlfeTaskModel = coreModels.TaskModel.extend({ defaults: _.extend( { @@ -291,6 +303,88 @@ var ProjectModel = Backbone.Model.extend({ } return url; + }, + + /** + * If a project is of the GWLFE package, we trigger the mapshed GIS + * data gathering chain, and poll for it to finish. Once it finishes, + * we resolve the lock to indicate the project is ready for further + * processing. + * If the project is not of the GWLFE package, we simply resolve the + * lock immediately and return. + */ + fetchGisData: function() { + if (this.get('model_package') === GWLFE) { + var aoi = this.get('area_of_interest'), + promise = $.Deferred(), + taskModel = createTaskModel(MAPSHED), + taskHelper = { + postData: { + mapshed_input: JSON.stringify({ + area_of_interest: aoi + }) + }, + + onStart: function() { + console.log('Starting polling for MAPSHED'); + }, + + startFailure: function(response) { + console.log('Failed to start gathering data for MAPSHED'); + + if (response.responseJSON && response.responseJSON.error) { + console.log(response.responseJSON.error); + } + + promise.reject(); + }, + + pollSuccess: function() { + promise.resolve(taskModel.get('result')); + }, + + pollFailure: function() { + console.log('Failed to gather data required for MAPSHED'); + promise.reject(); + } + }; + + taskModel.start(taskHelper); + return promise; + } else { + // Currently there are no methods for fetching GIS Data if the + // model package is not GWLFE. Thus we return a resolved promise + // so that any execution waiting on this can continue immediately. + return $.when(); + } + }, + + /** + * Returns a promise that completes when GIS Data has been fetched. If fetching + * is not required, returns an immediatley resolved promise. + */ + fetchGisDataIfNeeded: function() { + var self = this, + saveProjectAndScenarios = _.bind(self.saveProjectAndScenarios, self); + + if (self.get('gis_data') === null && self.fetchGisDataPromise === undefined) { + self.fetchGisDataPromise = self.fetchGisData(); + self.fetchGisDataPromise + .done(function(result) { + if (result) { + self.set('gis_data', result); + saveProjectAndScenarios(); + } + }) + .always(function() { + // Clear promise once it completes, so we start a new one + // next time. + delete self.fetchGisDataPromise; + }); + } + + // Return fetchGisDataPromise if it exists, else an immediately resolved one. + return self.fetchGisDataPromise || $.when(); } }); @@ -612,7 +706,10 @@ var ScenarioModel = Backbone.Model.extend({ }); if (needsResults) { - this.fetchResults(); + var fetchResults = _.bind(this.fetchResults, this); + App.currentProject + .fetchGisDataIfNeeded() + .then(fetchResults); } }, @@ -653,21 +750,9 @@ var ScenarioModel = Backbone.Model.extend({ var self = this, results = this.get('results'), taskModel = this.get('taskModel'), - nonZeroModifications = this.get('modifications').filter(function(mod) { - return mod.get('effectiveArea') > 0; - }), + gisData = this.getGisData(), taskHelper = { - postData: { - model_input: JSON.stringify({ - inputs: self.get('inputs').toJSON(), - modification_pieces: alterModifications(nonZeroModifications, self.get('modification_hash')), - area_of_interest: App.currentProject.get('area_of_interest'), - aoi_census: self.get('aoi_census'), - modification_censuses: self.get('modification_censuses'), - inputmod_hash: self.get('inputmod_hash'), - modification_hash: self.get('modification_hash') - }) - }, + postData: gisData, onStart: function() { results.setPolling(true); @@ -716,6 +801,35 @@ var ScenarioModel = Backbone.Model.extend({ var hash = utils.getCollectionHash(this.get('modifications')); this.set('modification_hash', hash); + }, + + getGisData: function() { + var self = this, + project = App.currentProject; + + switch(App.currentProject.get('model_package')) { + case TR55_PACKAGE: + var nonZeroModifications = self.get('modifications').filter(function(mod) { + return mod.get('effectiveArea') > 0; + }); + + return { + model_input: JSON.stringify({ + inputs: self.get('inputs').toJSON(), + modification_pieces: alterModifications(nonZeroModifications, self.get('modification_hash')), + area_of_interest: project.get('area_of_interest'), + aoi_census: self.get('aoi_census'), + modification_censuses: self.get('modification_censuses'), + inputmod_hash: self.get('inputmod_hash'), + modification_hash: self.get('modification_hash') + }) + }; + + case GWLFE: + return { + model_input: JSON.stringify(project.get('gis_data')) + }; + } } }); @@ -873,6 +987,8 @@ function createTaskModel(modelPackage) { return new Tr55TaskModel(); case GWLFE: return new GwlfeTaskModel(); + case MAPSHED: + return new MapshedTaskModel(); } throw 'Model package not supported: ' + modelPackage; } From 063e917fa80cca815009a4a9406219a5f5a776ab Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Mon, 23 May 2016 14:47:13 -0400 Subject: [PATCH 077/136] Various data format fixes for GWLF-E These errors were preventing MapShed data from executing in GWLF-E * Shape of CNI and CNP needs to be (3, 16), with only index 1 populated with values, the rest 0. * Shape of UrbBMPRed needs to be (16, 3) instead of (12, 3) * ManNitr and ManPhos need to be of size 2 instead of 16 * NumAnimals need to be ints instead of floats * Temp and Precip cannot contain NULLs, so we put in the value from the database itself (which in this case is -99999) --- src/mmw/apps/modeling/mapshed/calcs.py | 27 ++++++++------------------ src/mmw/mmw/settings/gwlfe_settings.py | 16 +++++++++------ 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index a58a23980..42f1930e9 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -20,7 +20,6 @@ LIVESTOCK = settings.GWLFE_CONFIG['Livestock'] POULTRY = settings.GWLFE_CONFIG['Poultry'] LITERS_PER_MGAL = 3785412 -WEATHER_NULL = settings.GWLFE_CONFIG['WeatherNull'] AG_NLCD_CODES = settings.GWLFE_CONFIG['AgriculturalNLCDCodes'] KM_PER_M = 0.001 @@ -180,14 +179,11 @@ def animal_energy_units(geom): def manure_spread(aeu): """ - Given Animal Energy Units, returns two 16-item lists, containing nitrogen - and phosphorus manure spreading values for each of the 16 land use types. - If a given land use index is marked as having manure spreading applied in - the configuration, it will have a value calculated below, otherwise it - will be set to 0. + Given Animal Energy Units, returns two lists, containing nitrogen and + phosphorus manure spreading values for each of the manure spreading land + use types. """ - n_list = [0.0] * 16 - p_list = [0.0] * 16 + num_land_uses = len(settings.GWLFE_CONFIG['ManureSpreadingLandUseIndices']) if 1.0 <= aeu: n_spread, p_spread = 4.88, 0.86 @@ -196,11 +192,7 @@ def manure_spread(aeu): else: n_spread, p_spread = 2.44, 0.38 - for lu in settings.GWLFE_CONFIG['ManureSpreadingLandUseIndices']: - n_list[lu] = n_spread - p_list[lu] = p_spread - - return n_list, p_list + return [n_spread] * num_land_uses, [p_spread] * num_land_uses def ls_factors(lu_strms, total_strm_len, areas, avg_slope): @@ -330,8 +322,7 @@ def weather_data(ws, begyear, endyear): array[year][month][day] = value where `year` 0 corresponds to the first year in the range, 1 to the second, and so on; `month` 0 corresponds to January, 1 to February, and so on; - `day` 0 corresponds to the 1st of the month, 1 to the 2nd, and so on. Cells - with no corresponding values are marked as None. + `day` 0 corresponds to the 1st of the month, 1 to the 2nd, and so on. """ temp_sql = ''' SELECT year, EXTRACT(MONTH FROM TO_DATE(month, 'MON')) AS month, @@ -377,8 +368,7 @@ def weather_data(ws, begyear, endyear): year = int(row[0]) - begyear month = int(row[1]) - 1 for day in range(31): - t = float(row[day + 2]) - temps[year][month][day] = t if t != WEATHER_NULL else None + temps[year][month][day] = float(row[day + 2]) with connection.cursor() as cursor: cursor.execute(prcp_sql, [stations, begyear, endyear]) @@ -386,8 +376,7 @@ def weather_data(ws, begyear, endyear): year = int(row[0]) - begyear month = int(row[1]) - 1 for day in range(31): - p = float(row[day + 2]) - prcps[year][month][day] = p if p != WEATHER_NULL else None + prcps[year][month][day] = float(row[day + 2]) return temps, prcps diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index 7fd94e243..37c7bde99 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -40,10 +40,14 @@ ], 'Imper': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # Impervious surface area percentage 0.15, 0.52, 0.87, 0.15, 0.52, 0.87], # only defined for urban land use types - 'CNI': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # Curve Number for Impervious Surfaces - 92, 98, 98, 92, 92, 92], # only defined for urban land use types - 'CNP': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # Curve Number for Pervious Surfaces - 74, 79, 79, 74, 74, 74], # only defined for urban land use types + 'CNI': [[0] * 16, + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # Curve Number for Impervious Surfaces + 92, 98, 98, 92, 92, 92], # only defined for urban land use types + [0] * 16], + 'CNP': [[0] * 16, + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # Curve Number for Pervious Surfaces + 74, 79, 79, 74, 74, 74], # only defined for urban land use types + [0] * 16], 'TotSusSolids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # Total Suspended Solids factor 60, 70, 80, 90, 100, 110], # only defined for urban land use types 'PhysFlag': b.NO, # Flag: Physiographic Province Layer Detected (0 No; 1 Yes) @@ -90,7 +94,7 @@ [0.28, 0.37, 0], # Ld_Residential [0.28, 0.37, 0], # Md_Residential [0.28, 0.37, 0]], # Hd_Residential - 'UrbBMPRed': [[0.0] * 3 for m in range(12)], # Urban BMP Reduction + 'UrbBMPRed': [[0.0] * 3 for l in range(16)], # Urban BMP Reduction 'SepticFlag': b.YES, # Flag: Septic Systems Layer Detected (0 No; 1 Yes) 'NumPondSys': [0] * 12, # Number of People on Pond Systems 'NumShortSys': [0] * 12, # Number of People on Short Circuit Systems @@ -401,7 +405,7 @@ 'PhytasePct': 0, # Phytase in Feed (%), 'AnimalName': ['Dairy Cows', 'Beef Cows', 'Broilers', 'Layers', 'Hogs/Swine', 'Sheep', 'Horses', 'Turkeys', 'Other'], - 'NumAnimals': [ 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00], + 'NumAnimals': [ 0, 0, 0, 0, 0, 0, 0, 0, 0], 'GrazingAnimal': [ b.YES, b.YES, b.NO, b.NO, b.NO, b.YES, b.YES, b.NO, b.NO], 'AvgAnimalWt': [ 640.00, 360.00, 0.90, 1.80, 61.00, 50.00, 500.00, 6.80, 0.00], # Average Animal Weight (kg) 'AnimalDailyN': [ 0.44, 0.31, 1.07, 0.85, 0.48, 0.37, 0.28, 0.59, 0.00], # Animal Daily Loads: Nitrogen (kg/AEU) From c191864de9ec38f490028a19568ef8d41418cb98 Mon Sep 17 00:00:00 2001 From: Joe Tarricone Date: Mon, 16 May 2016 10:33:19 -0400 Subject: [PATCH 078/136] Add endpoint to run GWLF-E if passed a full input dataset. --- .../modeling/mapshed/data/sample_input.gms | 5841 ----------------- src/mmw/apps/modeling/tasks.py | 15 +- 2 files changed, 6 insertions(+), 5850 deletions(-) delete mode 100755 src/mmw/apps/modeling/mapshed/data/sample_input.gms diff --git a/src/mmw/apps/modeling/mapshed/data/sample_input.gms b/src/mmw/apps/modeling/mapshed/data/sample_input.gms deleted file mode 100755 index 498f55818..000000000 --- a/src/mmw/apps/modeling/mapshed/data/sample_input.gms +++ /dev/null @@ -1,5841 +0,0 @@ -10,6,"4" -"1.4.0",.06,0,10,0,0,.149,13.154,45178,22800,22378,1843,1404,2.67,8.74802590162132E-02,15,2000,2014,.00064029392381193195,4129,0,0,0,.291239452651974 -0 -0 -0 -0 -0 -"Jan",.63,9.4,0,.2,0,.007,.9 -"Feb",.68,10.4,0,.2,0,.007,.9 -"Mar",.7,11.8,0,.2,0,.007,.9 -"Apr",.72,13.2,0,.3,0,.007,.9 -"May",.91,14.3,1,.3,0,.007,.9 -"Jun",1.02,14.9,1,.3,0,.007,.9 -"Jul",1.08,14.6,1,.3,0,.007,.9 -"Aug",1.12,13.6,1,.3,0,.007,.9 -"Sep",1.14,12.2,1,.12,0,.007,.9 -"Oct",.97,10.8,0,.12,0,.007,.9 -"Nov",.88,9.7,0,.12,0,.007,.9 -"Dec",.82,9.1,0,.12,0,.007,.9 -"Hay/Past",957,63,.29,.525,.03,.45 -"Cropland",1505,75,.29,.545,.42,.45 -"Forest",1033,60,.294,.8,.002,.45 -"Wetland",46,80,.286,.299,.01,.1 -"Disturbed",7,85,.297,.529,.08,.1 -"Turfgrass",37,58,.3,.697,.03,.2 -"Open_Land",186,82,.293,.604,.04,.45 -"Bare_Rock",0,0,0,0,0,0 -"Sandy_Areas",0,0,0,0,0,0 -"Unpaved_Road",0,0,0,0,0,0 -"Ld_Mixed",50,.15,92,74,60 -"Md_Mixed",47,.52,98,79,70 -"Hd_Mixed",84,.87,98,79,80 -"Ld_Residential",131,.15,92,74,90 -"Md_Residential",46,.52,92,74,100 -"Hd_Residential",0,0,0,0,0 -1,0,0,1,1,1,1 -2000,528,7.83,.03,.5,.5 -2,0,0,0,0 -.75,.178574398 -2.9,.178574398 -.19,.01 -.19,.01 -.02,.01 -2.5,.174451303827751 -.5,.01 -0,0 -0,0 -0,0 -3 -"Nitrogen" -"Phosphorus" -"Sediment" -.095,.015,.33,0 -.0095,.0021,.4,0 -2.8,.8,0,0 -.105,.015,.33,0 -.0105,.0021,.4,0 -6.2,.8,0,0 -.11,.015,.33,0 -.0115,.0021,.4,0 -2.8,.8,0,0 -.095,.015,.28,0 -.0095,.0019,.37,0 -2.5,1.3,0,0 -.1,.015,.28,0 -.0115,.0039,.37,0 -6.2,1.1,0,0 -0,0,0,0 -0,0,0,0 -0,0,0,0 -2.44,.38 -2.44,.38 -.68,.05,2.8420384063938E-04 -.68,.05,2.5670024315815E-04 -.68,.05,2.8420384063938E-04 -.68,.05,2.75035974812303E-04 -.68,.05,2.8420384063938E-04 -.68,.05,2.75035974812303E-04 -.68,.05,2.8420384063938E-04 -.68,.05,2.8420384063938E-04 -.68,.05,2.75035974812303E-04 -.68,.05,2.8420384063938E-04 -.68,.05,2.75035974812303E-04 -.68,.05,2.8420384063938E-04 -1 -686,0,0,0,0 -686,0,0,0,0 -686,0,0,0,0 -686,0,0,0,0 -686,0,0,0,0 -686,0,0,0,0 -686,0,0,0,0 -686,0,0,0,0 -686,0,0,0,0 -686,0,0,0,0 -686,0,0,0,0 -686,0,0,0,0 -12,2.5,1.6,.4 -15,.1,50 -"scenario4",1,"","1.4.0" -"scenario4" -0,0,0,0,0,0,0 -0,0,0,0,0,0,0,0,0,0,0 -0,0,0,0,0,0,0,0,0,0,0 -0,0,0,0 -1505,84,5,957,50,3,6,6 -20,0,0,0,0,55,0,0,0,4.6,0,34,0 -20,0,0,0,0,55,0,0,0,4.6,0,0,0,34,0,0 -0,0,0,0,0,0,0,0,0 -.748578391551584,62,62,100,100,0,0,0,0,0,0,0,22.8,45.2,0,1,0,0,0,0,0,0,0,0,0,0 -0,3,50,0,0,0,0,0,0,1,0,.6,0,2,25,0,0 -.6,0,0,0,.96,.63,0,0,0,0,0,0,0,0,0,0 -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -.29,.41,.08,.07,.05,0,.95,.3,.56,.2,.95,.29,.25 -.5,.28,.4,.22,.1,.1,0,.95,.3,.78,.45,.95,.44,.35 -.35,.44,.63,.53,.3,.17,.16,0,.95,.38,.76,.6,.55,.95,.02,.0035,2.55 -.75,.75,.14,.14,.15,.15,.21,.7,1,.75,.14,.15,.71,.82,.71 -.14,.56,.14,.56,.42,.1,.6,.1,.6,.5,.99,.99,200,200 -74.13,61.78,889.58,9320.57,24.71,33112.12,82.02,19768.43,932.06,12355.27,1250,520,300,2.5,271.82,0,26440.28,24.71,18.31 -15000,250,300,0,0,0,150,0,0,0,0,0,0,"None" -0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1 -0 -0 -0 -0 -0 -0 -0 -0 -0 -0 -0 -0 -"Output" -0,0,0 -0,0,0,0,0,0,0,0,0,0,0,0 -0,0,0,0,0 -0,0,0 -0,0 -0,0 -0,0 -0,0,0 -0,0,0 -0,0,0,0,0,0,0,0,0 -0,0,0,0,0,0,0,0,0,0,0 -144034,45743,90000000000000,.8,.2,1,135961,22147,2.9E+16,.52,.18,1,0 -500000000,25,.9,9600,2000000000,0,200,.5,0,0,0,0 -"Dairy Cows",1239,"Y",640,.44,.07,100000000000 -"Beef Cows",0,"Y",360,.31,.09,100000000000 -"Broilers",155000,"N",.9,1.07,.3,140000000 -"Layers",155000,"N",1.8,.85,.29,140000000 -"Hogs/Swine",280,"N",61,.48,.15,11000000000 -"Sheep",110,"Y",50,.37,.1,12000000000 -"Horses",154,"Y",500,.28,.06,420000000 -"Turkeys",0,"N",6.8,.59,.2,95000000 -"Other",0,"N",0,0,0,0 -"Jan",.01,.15,.1,.12,0,.3,.2,.12 -"Feb",.01,.15,.1,.12,0,.3,.2,.12 -"Mar",.15,.15,.1,.12,0,.3,.2,.12 -"Apr",.1,.15,.1,.12,0,.3,.2,.12 -"May",.05,.15,.1,.12,0,.3,.2,.12 -"Jun",.03,.15,.1,.12,0,.3,.2,.12 -"Jul",.03,.15,.1,.12,0,.3,.2,.12 -"Aug",.03,.15,.1,.12,0,.3,.2,.12 -"Sep",.11,.15,.1,.12,0,.3,.2,.12 -"Oct",.1,.15,.1,.12,0,.3,.2,.12 -"Nov",.1,.15,.1,.12,0,.3,.2,.12 -"Dec",.08,.15,.1,.12,0,.3,.2,.12 -"Jan",.02,.05,.15,.1,.12,.01,.15,.1,.12,0,.3,.2,.12 -"Feb",.02,.05,.15,.1,.12,.01,.15,.1,.12,0,.3,.2,.12 -"Mar",.1,.05,.15,.1,.12,.1,.15,.1,.12,0,.3,.2,.12 -"Apr",.25,.05,.15,.1,.12,.05,.15,.1,.12,0,.3,.2,.12 -"May",.5,.05,.15,.1,.12,.05,.15,.1,.12,0,.3,.2,.12 -"Jun",.5,.05,.15,.1,.12,.03,.15,.1,.12,0,.3,.2,.12 -"Jul",.5,.05,.15,.1,.12,.03,.15,.1,.12,0,.3,.2,.12 -"Aug",.5,.05,.15,.1,.12,.03,.15,.1,.12,0,.3,.2,.12 -"Sep",.5,.05,.15,.1,.12,.11,.15,.1,.12,0,.3,.2,.12 -"Oct",.4,.05,.15,.1,.12,.06,.15,.1,.12,0,.3,.2,.12 -"Nov",.25,.05,.15,.1,.12,.02,.15,.1,.12,0,.3,.2,.12 -"Dec",.1,.05,.15,.1,.12,.02,.15,.1,.12,0,.3,.2,.12 -0,.12,.29,.84,26.052,4,.287,.226,0,0,1 -31,"Jan",2000 -2,0 -8,0 -11,.13 -13,.38 -3,.01 -0,.01 -3,0 -0,0 -3,.01 -7,1.82 -6,.09 -4,0 -2,.03 --5,0 --5,0 -3,0 --6,0 --12,0 --5,0 --3,.76 --8,0 --12,0 --7,0 --1,0 --3,1.33 --4,.01 --10,0 --8,0 --8,0 --9,1.46 --3,.71 -29,"Feb",2000 --7,0 --5,0 --8,.03 --3,.24 --2,.04 --2,0 --3,0 --7,0 --4,0 -0,0 -4,.11 --3,0 --6,.03 -4,.91 -1,0 -2,0 -1,0 --1,.86 -1,1.02 -1,0 -1,0 -2,0 -3,0 -5,0 -11,0 -9,.36 -9,.62 -8,.93 -7,0 -31,"Mar",2000 -6,0 -7,0 -5,0 -5,0 -7,0 -6,0 -9,0 -14,0 -16,.04 -15,0 -9,1.73 -5,.91 -3,0 -6,0 -10,0 -12,.57 -7,1.28 --1,.1 -3,0 -6,0 -5,6.34 -6,2.87 -10,0 -11,0 -13,.09 -12,.15 -10,.76 -11,1.31 -7,.03 -6,0 -6,0 -30,"Apr",2000 -9,0 -13,.03 -17,0 -15,.98 -7,0 -12,0 -13,0 -15,1.1 -5,1.41 -8,0 -8,.03 -6,.01 -4,0 -8,0 -11,.09 -20,.71 -13,1.24 -6,.27 -12,0 -14,0 -12,.93 -12,.29 -9,0 -12,0 -12,0 -9,0 -8,.08 -11,0 -11,0 -12,0 -31,"May",2000 -12,0 -15,.58 -13,0 -16,0 -21,.23 -22,0 -24,0 -24,0 -24,0 -23,.8 -18,.27 -21,0 -22,.95 -19,.3 -13,0 -12,.01 -17,.06 -22,.09 -19,2.39 -12,1.61 -13,.42 -14,.88 -16,.24 -19,1.83 -18,.04 -17,0 -16,.41 -13,.08 -15,.13 -13,0 -14,0 -30,"Jun",2000 -21,0 -24,0 -22,.1 -17,0 -17,0 -14,2.08 -18,.06 -19,0 -22,0 -25,0 -26,0 -25,.23 -19,.23 -16,.37 -20,.27 -24,.32 -25,.03 -24,.66 -20,1.23 -20,0 -22,1.04 -24,1.26 -22,0 -22,0 -26,1.46 -25,3.24 -25,.04 -22,.06 -22,.66 -19,.42 -31,"Jul",2000 -20,0 -21,0 -22,.32 -25,.14 -23,0 -20,0 -20,.08 -17,0 -21,0 -26,.6 -24,0 -20,0 -20,0 -22,.08 -21,2.31 -21,.01 -22,.2 -23,0 -20,.48 -20,.09 -20,0 -21,0 -20,0 -20,.08 -20,.04 -19,.8 -21,0 -22,0 -23,0 -25,0 -26,0 -31,"Aug",2000 -26,0 -25,0 -25,.18 -23,.55 -20,0 -18,.51 -27,.08 -26,0 -26,.03 -25,.09 -23,0 -22,.08 -19,.04 -19,.14 -21,.01 -23,.57 -20,.03 -18,.18 -18,0 -17,0 -15,0 -17,0 -20,.61 -21,.46 -20,0 -20,0 -21,.53 -23,.41 -22,.01 -23,.01 -25,.23 -30,"Sep",2000 -26,7.37 -26,.01 -24,.67 -24,0 -17,0 -13,0 -14,0 -17,0 -22,0 -22,0 -23,0 -24,.01 -21,2.76 -17,1.88 -19,1.59 -14,0 -14,0 -15,0 -17,1.47 -20,.32 -21,0 -14,0 -15,.11 -20,0 -13,1.87 -10,3.09 -12,0 -12,.03 -8,0 -10,0 -31,"Oct",2000 -12,0 -16,0 -18,0 -19,.57 -18,.04 -18,0 -12,0 -7,0 -4,0 -7,.09 -10,0 -12,0 -12,0 -14,0 -16,0 -14,.1 -14,.62 -15,.66 -12,0 -12,0 -14,0 -12,0 -8,0 -11,0 -14,0 -14,0 -13,0 -12,0 -7,0 -7,0 -8,0 -30,"Nov",2000 -8,0 -9,0 -10,0 -12,0 -7,0 -6,0 -6,0 -12,0 -13,.25 -14,1.35 -10,0 -7,0 -5,0 -8,.53 -4,0 -2,0 -4,0 -2,0 --1,0 --1,.03 --1,0 --2,0 --4,0 --3,0 --2,.22 -7,1.6 -6,.06 -5,.01 -5,.39 -4,.44 -31,"Dec",2000 -1,0 --2,0 --4,0 --2,0 -1,0 --3,0 --3,0 -0,0 --1,0 --2,.03 -0,0 -3,.01 --5,0 --2,3.1 -0,.03 -2,.74 -8,4.64 --3,0 --4,.09 --5,.23 --8,0 --6,.01 --10,0 --7,0 --7,0 --10,0 --9,0 --8,0 --6,0 --6,0 --3,0 -31,"Jan",2001 --3,0 --5,0 --7,0 --3,0 --6,.34 --4,0 --1,0 --1,.23 --3,0 --2,0 -0,0 -0,0 --2,0 --1,0 -2,.25 -3,0 -1,0 --1,.23 -1,2.08 -1,.95 --4,.52 --8,0 --8,0 --5,0 --3,0 --6,0 -0,0 --2,0 --4,0 -4,1.52 -2,.01 -28,"Feb",2001 -3,0 -2,0 --2,0 --1,0 -0,2.15 -3,0 -2,0 --1,0 -6,0 -8,0 --2,0 --3,.04 -4,0 -4,.36 -6,.1 -3,.6 -0,.18 --3,0 --2,0 -6,0 -4,.04 --6,.64 --2,0 --2,0 -4,.37 -4,0 -3,0 -3,0 -31,"Mar",2001 -0,0 -3,0 -6,0 -3,.62 --2,1.22 --2,0 -4,0 -1,0 -3,0 -2,0 -3,0 -2,.44 -6,1.75 -5,.04 -4,.05 -6,.52 -5,.23 -4,.06 -4,0 -3,0 -6,2.06 -7,.08 -8,0 -4,0 -1,0 -0,0 --2,0 -1,0 -3,1.21 -5,3.05 -5,.05 -30,"Apr",2001 -6,0 -5,.04 -6,.03 -8,.06 -7,0 -10,.66 -10,.03 -7,.09 -17,.42 -16,.29 -11,.9 -13,.04 -16,0 -13,0 -12,.39 -10,1.13 -5,.86 -4,.67 -6,0 -8,0 -15,.04 -19,0 -21,0 -20,.01 -10,0 -8,0 -12,0 -13,0 -8,0 -12,0 -31,"May",2001 -17,0 -19,0 -21,0 -22,0 -20,0 -13,0 -11,0 -12,0 -18,.14 -18,0 -19,0 -20,.11 -15,0 -11,0 -13,0 -13,0 -13,0 -14,0 -20,0 -16,.06 -13,1.83 -18,.98 -18,.32 -18,.01 -19,.19 -16,2.22 -17,1.13 -17,.2 -16,.24 -15,0 -13,0 -30,"Jun",2001 -13,.7 -18,.5 -18,0 -17,0 -20,0 -20,0 -20,0 -18,0 -18,0 -19,0 -22,.03 -23,0 -24,0 -25,0 -24,.13 -23,2.51 -24,.37 -22,0 -23,0 -25,.64 -24,.11 -24,1.07 -21,.84 -19,0 -21,0 -22,0 -24,0 -26,0 -26,0 -27,0 -31,"Jul",2001 -25,.34 -19,.65 -17,0 -23,.88 -22,.2 -19,0 -19,0 -23,.19 -24,1.19 -24,.64 -22,0 -20,0 -18,0 -20,0 -21,0 -22,0 -25,0 -24,.01 -23,0 -21,0 -20,0 -21,0 -24,0 -28,0 -29,0 -25,.19 -19,.04 -20,0 -21,.08 -22,.01 -23,0 -31,"Aug",2001 -23,0 -23,0 -25,0 -26,.57 -26,.81 -28,0 -30,0 -30,0 -29,0 -28,.94 -23,2.83 -25,1.41 -25,.33 -25,.01 -23,0 -24,0 -26,0 -23,0 -23,.38 -25,.89 -22,0 -22,0 -23,0 -23,.03 -22,0 -22,0 -23,.27 -25,0 -23,.1 -23,.05 -25,.06 -30,"Sep",2001 -21,.09 -17,0 -19,0 -24,.04 -20,.04 -18,0 -20,0 -22,0 -23,0 -22,.27 -19,0 -18,0 -20,0 -18,.39 -13,0 -14,0 -16,0 -17,0 -18,0 -21,1.18 -21,.27 -22,0 -20,0 -21,1.83 -17,4 -11,0 -13,0 -11,0 -13,0 -11,0 -31,"Oct",2001 -14,0 -16,0 -19,0 -18,0 -18,.09 -17,.03 -9,0 -5,0 -7,0 -11,0 -15,0 -16,.04 -19,0 -17,.03 -13,.41 -12,.37 -11,.39 -8,0 -9,0 -14,0 -15,0 -16,0 -17,0 -20,0 -20,0 -11,0 -7,0 -5,0 -5,0 -9,0 -8,0 -30,"Nov",2001 -12,0 -16,0 -17,.01 -11,0 -10,0 -6,0 -10,0 -9,0 -10,0 -8,0 -6,0 -2,0 -4,0 -6,0 -11,0 -11,0 -8,0 -6,0 -7,0 -8,.25 -1,0 -3,0 -6,0 -9,.09 -15,1.66 -10,.05 -7,0 -13,0 -13,0 -16,.2 -31,"Dec",2001 -13,0 -7,0 -6,0 -9,.03 -15,0 -13,0 -10,.04 -4,.84 -4,.33 -1,.04 -7,.23 -4,.04 -9,.04 -11,.62 -7,.51 -0,0 -5,.39 -10,.71 -6,0 -3,0 -2,0 -1,0 -2,.18 -2,.17 --2,0 --2,0 --5,0 --1,0 -0,0 --4,0 --7,0 -31,"Jan",2002 --5,0 --4,0 --3,0 --2,0 --1,0 --2,.88 --2,.94 --2,0 --2,0 -3,0 -4,.95 -3,0 -3,0 -1,0 -3,0 -3,0 -4,0 --1,0 --4,.89 --4,.28 --2,0 -3,0 -3,0 -5,2.24 -4,0 -4,0 -7,0 -8,0 -10,0 -12,.43 -7,.48 -28,"Feb",2002 -10,.24 -3,0 -0,0 --1,.01 --4,0 -0,.01 -1,.01 -4,0 -4,0 -6,.03 -2,.05 -0,0 -1,0 --2,0 -2,0 -5,0 -5,0 -1,0 -2,0 -10,0 -11,.01 -7,0 -2,0 -2,0 -5,0 -8,.18 -4,.14 -0,0 -31,"Mar",2002 -0,0 -2,.72 -9,1.63 -0,0 --4,0 -6,0 -7,0 -10,0 -12,.01 -9,.08 -0,0 -4,0 -5,.93 -12,0 -15,0 -13,0 -3,.25 -2,.91 -5,.03 -5,2.68 -7,0 -0,0 -3,0 -4,0 -6,0 -5,.79 -5,.91 -5,0 -9,0 -14,0 -12,.36 -30,"Apr",2002 -8,.1 -10,0 -12,.51 -5,0 -2,0 -2,.01 -2,0 -8,.05 -18,.17 -12,.33 -10,0 -9,.03 -17,.25 -18,.42 -20,1.41 -23,0 -23,.11 -23,0 -23,.06 -20,.24 -11,.28 -9,.8 -7,0 -7,0 -10,.33 -9,0 -9,.15 -15,2.64 -14,.29 -11,.22 -31,"May",2002 -13,.1 -20,1.93 -15,.7 -9,0 -12,0 -14,0 -20,0 -18,.03 -15,1.49 -17,.04 -13,0 -18,.99 -20,3.05 -14,.8 -13,.1 -16,0 -21,.03 -13,2.26 -9,.01 -8,0 -8,0 -10,0 -14,0 -18,0 -19,0 -18,0 -20,.1 -21,.01 -22,0 -21,0 -24,.06 -30,"Jun",2002 -23,.06 -21,0 -18,0 -19,0 -25,.15 -23,2.04 -19,1.41 -19,0 -20,0 -22,0 -25,0 -26,0 -22,.17 -16,1.64 -18,.8 -19,0 -19,0 -18,.03 -20,.75 -21,.01 -22,0 -22,0 -23,0 -25,.06 -26,.08 -27,.01 -26,.89 -24,.61 -22,0 -23,0 -31,"Jul",2002 -24,0 -27,0 -28,0 -28,0 -26,0 -21,0 -20,0 -23,0 -26,.19 -23,.09 -19,0 -19,0 -20,0 -22,.22 -24,0 -25,0 -25,0 -28,0 -27,.03 -25,.04 -25,0 -27,0 -29,.08 -25,.66 -23,0 -20,.09 -23,0 -27,1.18 -28,0 -28,0 -27,0 -31,"Aug",2002 -28,0 -28,0 -28,1.56 -28,0 -27,.04 -24,.01 -19,0 -19,0 -20,0 -22,0 -24,0 -27,0 -28,0 -29,0 -28,0 -28,0 -27,0 -28,0 -28,.7 -26,.3 -23,0 -26,.04 -26,.67 -25,1.8 -24,.13 -22,0 -22,0 -21,1.13 -17,2.13 -18,.01 -20,0 -30,"Sep",2002 -16,.72 -20,.04 -21,.01 -25,0 -21,0 -18,0 -19,0 -20,0 -21,0 -24,0 -23,0 -17,0 -18,0 -21,.03 -23,.53 -23,.22 -20,0 -19,0 -20,0 -23,0 -23,.41 -23,.01 -19,.51 -16,0 -16,.01 -17,2.2 -19,3.14 -18,.93 -15,0 -17,0 -31,"Oct",2002 -19,0 -22,0 -22,0 -22,1.02 -22,.08 -15,0 -18,0 -11,0 -12,0 -16,1.56 -14,5.73 -16,.24 -15,.08 -10,.04 -8,.01 -8,5.11 -11,.06 -8,.09 -9,.14 -11,0 -8,0 -8,0 -8,0 -7,.22 -6,.91 -12,1.1 -11,0 -8,0 -4,1.23 -2,1.93 -5,.48 -30,"Nov",2002 -4,.01 -4,0 -4,0 -5,0 -4,.51 -8,.66 -5,0 -8,0 -9,0 -14,0 -15,.03 -12,1.8 -6,.04 -8,0 -8,0 -8,2.3 -4,2.45 -4,.84 -3,.01 -6,0 -5,.91 -7,.56 -4,.04 -5,0 -6,0 -5,.01 -1,.52 --1,0 -1,0 -6,0 -31,"Dec",2002 --1,0 -0,0 --3,0 --5,0 --4,1.55 --7,0 --9,0 --4,0 --9,0 --8,0 --4,4 -2,.28 -0,.84 -4,.55 -5,0 -2,0 --3,0 --3,0 -3,.01 -8,.93 -3,0 -4,.01 -4,0 -1,.13 -0,3.56 -1,0 --3,0 --3,0 -0,0 --2,.03 -2,0 -31,"Jan",2003 -3,1.68 -2,.41 -0,1.31 -1,.11 --2,.44 --1,.18 --4,0 -2,.01 -5,0 -4,0 --2,0 --3,0 --3,0 --5,0 --5,0 --7,.11 --7,.08 --10,0 --8,0 --4,0 --6,0 --8,0 --10,0 --7,0 --5,0 --3,.05 --9,.04 --11,0 --3,.19 --7,0 -0,0 -28,"Feb",2003 -2,.11 -3,0 -3,0 -4,.7 --1,0 --5,.11 --1,1.05 --8,0 --7,0 --1,.06 --6,0 --7,.01 --9,0 --6,0 --5,.18 --9,1.85 --8,1.97 --4,.22 --5,0 -1,0 --3,.19 -3,3.09 -1,.11 --2,0 --3,0 --8,.08 --4,.04 --1,.17 -31,"Mar",2003 -0,0 -4,.65 --4,0 --5,0 -4,.1 -1,.76 --6,0 -0,0 -3,0 --4,0 --5,0 -2,0 -4,.13 --1,.04 -5,0 -9,0 -14,.08 -13,0 -8,0 -5,3.07 -9,.81 -10,0 -9,0 -9,0 -11,0 -12,.7 -10,.39 -10,.04 -15,.41 -7,1.35 -3,.01 -30,"Apr",2003 -5,0 -14,0 -14,0 -10,.06 -6,.27 -5,0 -3,1 -0,.1 -3,.77 -7,0 -6,2.3 -13,.11 -11,0 -10,0 -17,0 -18,0 -14,.01 -5,.38 -12,0 -10,0 -11,.01 -13,.36 -10,0 -9,0 -12,.19 -12,2.55 -14,0 -15,0 -15,.14 -14,0 -31,"May",2003 -18,0 -20,0 -16,0 -14,0 -9,.1 -12,.09 -19,.25 -19,.85 -16,.55 -17,.55 -20,.08 -18,.01 -12,0 -14,0 -13,0 -13,1.35 -10,.19 -10,.13 -14,0 -15,0 -16,.74 -14,.01 -12,.38 -14,1.78 -15,.47 -14,2.59 -14,.04 -15,.34 -17,.01 -18,0 -18,.36 -30,"Jun",2003 -14,.97 -14,.03 -14,.98 -12,2.54 -17,.11 -16,0 -16,2.83 -16,0 -20,.2 -20,0 -23,.01 -24,.1 -24,.19 -24,.17 -22,.2 -18,0 -15,.2 -18,.62 -22,1.3 -19,4.14 -16,.9 -18,.64 -22,0 -23,0 -24,0 -26,0 -25,0 -22,0 -23,0 -24,0 -31,"Jul",2003 -22,0 -22,.01 -23,0 -25,0 -26,.76 -26,.36 -26,.22 -26,0 -25,.04 -21,.3 -24,.06 -22,0 -22,0 -22,0 -22,0 -26,0 -22,0 -22,.03 -22,.04 -21,0 -26,.2 -25,2.58 -24,1.38 -22,1.18 -22,0 -24,0 -26,.09 -24,.03 -21,0 -21,0 -22,.06 -31,"Aug",2003 -24,.19 -26,.05 -26,.04 -25,.15 -23,1.49 -23,1.56 -22,.04 -24,0 -25,.1 -25,.03 -25,1.16 -26,0 -26,0 -26,0 -25,0 -24,5.08 -23,.24 -22,0 -22,0 -23,0 -24,0 -26,.22 -22,.61 -18,0 -22,0 -24,.22 -23,.13 -22,0 -24,.36 -24,.25 -20,0 -30,"Sep",2003 -22,1.5 -21,3.81 -19,.56 -22,1.31 -18,0 -17,0 -18,0 -20,0 -20,0 -17,0 -18,0 -19,.15 -20,1.28 -24,.36 -21,1.61 -18,0 -17,0 -17,1.17 -21,2.22 -20,0 -18,0 -21,.38 -19,6.87 -16,0 -18,.3 -19,.69 -21,0 -18,.53 -14,.19 -11,0 -31,"Oct",2003 -11,.15 -8,0 -7,0 -9,.85 -8,0 -9,0 -10,0 -15,0 -15,0 -15,0 -16,0 -16,0 -15,0 -14,1.37 -13,2.44 -11,.01 -11,.47 -10,.33 -10,0 -9,0 -16,0 -12,.03 -5,0 -5,0 -7,0 -15,.3 -13,3.26 -7,.17 -10,2.64 -10,2.59 -10,0 -30,"Nov",2003 -15,0 -18,0 -19,0 -18,0 -15,1.47 -14,.52 -12,.19 -6,0 -0,0 -1,0 -4,.11 -10,.67 -9,.08 -5,.01 -6,0 -7,.04 -9,.28 -7,.01 -15,2.83 -8,.7 -9,0 -10,0 -8,0 -10,.38 -4,.01 -2,0 -5,.05 -8,2.6 -6,.88 -6,0 -31,"Dec",2003 -4,.04 -2,0 --3,0 --2,0 --1,1.16 --3,.37 --4,0 --4,0 --3,0 -3,.41 -8,2.71 -2,0 --2,0 --1,1.51 -1,.2 -1,0 -3,1.68 --1,0 --2,0 --1,0 -1,0 -4,0 -8,0 -8,1.44 -1,0 -1,0 -3,0 -1,0 -3,0 -5,.08 -3,0 -31,"Jan",2004 -3,0 -3,.28 -10,.03 -8,.34 -4,.97 -0,0 --4,0 --4,0 --6,0 --13,0 --8,0 -0,0 -2,0 --6,.01 --10,.1 --10,0 --6,.14 --2,2.04 --5,0 --5,0 --8,0 --4,0 --8,0 --9,.17 --13,.14 --9,.65 --8,.32 --6,.17 --8,0 --8,0 --9,0 -29,"Feb",2004 --7,0 --8,0 --3,1.87 -1,0 --4,.08 --1,4.64 -1,.36 --4,0 --2,0 -2,0 -1,0 -0,0 -0,0 -0,0 --3,0 --7,0 --4,0 --1,0 -3,0 -3,0 -4,0 -3,0 -1,0 -0,.32 --1,0 --1,0 -2,0 -5,0 -6,0 -31,"Mar",2004 -7,0 -12,0 -8,0 -9,.36 -10,.03 -12,1.31 -10,.18 -6,.27 -2,0 -2,.05 -4,0 -4,0 -3,0 -1,0 -8,0 -4,1.38 -0,.11 -2,.22 -2,.81 -3,.05 -6,0 -0,0 -1,0 -5,.03 -10,.01 -15,0 -14,.17 -12,0 -8,0 -5,0 -6,.05 -30,"Apr",2004 -10,2.12 -8,.7 -8,.08 -5,.7 -2,0 -6,0 -13,0 -8,.32 -11,.14 -8,0 -9,.05 -8,1.16 -6,1.8 -9,.69 -10,0 -8,0 -15,0 -19,0 -21,0 -18,0 -16,.15 -18,.04 -17,.83 -16,.67 -13,.5 -12,3.62 -12,.11 -9,.01 -15,0 -18,0 -31,"May",2004 -21,0 -21,.55 -14,1.74 -10,.04 -12,.04 -13,.08 -18,.29 -15,0 -18,.29 -22,.24 -22,0 -23,0 -23,0 -24,.18 -24,.89 -22,.48 -22,0 -22,.93 -20,1.26 -18,.04 -23,.17 -23,.06 -25,0 -26,.19 -23,.18 -22,.57 -22,.01 -22,0 -17,0 -16,0 -16,.44 -30,"Jun",2004 -19,.44 -18,.41 -19,.11 -17,.01 -16,3.24 -15,2.15 -21,0 -22,0 -25,0 -25,.34 -18,2.31 -17,.08 -17,0 -23,.89 -24,1.22 -25,.41 -25,3.9 -25,1.97 -23,0 -17,0 -18,0 -23,.28 -23,.03 -22,0 -23,.67 -22,.04 -18,0 -19,0 -19,.24 -20,0 -31,"Jul",2004 -22,0 -23,.03 -22,0 -22,.01 -27,0 -25,0 -23,1.19 -24,.14 -22,0 -21,0 -23,0 -22,6.45 -22,.11 -22,1.84 -21,.98 -21,.03 -22,.01 -22,1.51 -23,.04 -23,.33 -24,0 -24,1.14 -25,2.84 -22,.52 -20,0 -22,.01 -24,3.15 -24,1.28 -23,0 -25,0 -27,1.02 -31,"Aug",2004 -25,3.45 -25,0 -25,.9 -25,.7 -23,.98 -18,0 -16,0 -18,0 -20,0 -23,0 -23,.84 -24,1.21 -22,3.01 -20,.01 -22,.04 -22,0 -21,0 -22,0 -24,.3 -25,.56 -23,3.25 -18,0 -20,1.8 -22,0 -23,0 -22,0 -24,0 -25,0 -25,0 -25,.05 -23,0 -30,"Sep",2004 -20,0 -20,0 -20,0 -22,0 -22,0 -19,0 -22,0 -21,.44 -24,.53 -21,.03 -19,0 -20,0 -21,0 -20,0 -18,.11 -22,.05 -22,.08 -19,7.47 -13,0 -13,0 -17,0 -19,0 -21,0 -21,0 -19,0 -19,0 -18,0 -20,5.21 -19,1.44 -17,.33 -31,"Oct",2004 -16,0 -16,.09 -14,.17 -15,0 -12,0 -11,0 -14,0 -16,0 -16,0 -15,0 -12,0 -10,0 -9,.03 -12,1.6 -13,.23 -11,.27 -8,0 -7,.11 -11,1.3 -10,.13 -10,.38 -10,.19 -7,0 -7,0 -11,0 -11,0 -10,0 -10,0 -10,0 -16,.98 -17,0 -30,"Nov",2004 -12,0 -13,0 -12,0 -5,2.03 -9,.11 -9,0 -11,0 -9,0 -2,0 -2,0 -7,0 -7,1.82 -4,.25 -2,0 -5,0 -7,0 -6,0 -11,0 -10,0 -11,.55 -11,.24 -8,.03 -8,.05 -13,.27 -11,.67 -3,0 -6,.11 -10,3.82 -4,.01 -5,.05 -31,"Dec",2004 -8,1.69 -2,.03 -2,.01 -0,0 -4,0 -3,.06 -6,1.1 -8,.03 -4,.85 -7,1.8 -7,.1 -4,.01 -4,.09 -0,0 --3,0 -0,0 -2,0 --1,0 --1,.01 --9,.03 --5,0 -2,0 -8,2.25 -0,.04 --6,0 --6,0 --6,0 --6,0 -3,0 -3,0 -8,0 -31,"Jan",2005 -8,0 -3,.04 -6,.11 -9,.11 -5,1.74 -3,.64 -3,.05 -3,1.36 -3,0 -4,0 -3,.62 -4,.44 -11,.15 -9,3.63 --1,0 --2,0 --6,0 --9,0 --10,.23 --5,0 --10,0 --12,1.28 --10,.01 --11,.08 --3,0 --1,0 --8,0 --11,0 --9,.03 --2,.14 --6,0 -28,"Feb",2005 --6,0 --6,0 --3,.05 -2,.08 -4,0 -2,0 -2,0 -6,0 -7,.05 -4,.05 -0,0 -2,0 -1,0 -3,1.02 -8,.37 -7,.39 -1,0 --4,0 --5,0 --2,.14 -0,.47 -3,.08 -2,0 --2,.38 --6,.38 --4,0 --4,0 --1,.76 -31,"Mar",2005 --1,.38 --1,0 --4,0 --6,0 --3,0 -2,0 -9,0 -5,.43 --5,0 --4,0 -2,0 -1,0 -1,0 -0,0 -1,0 -1,0 -3,0 -3,0 -4,0 -7,.39 -6,.19 -5,0 -5,3.94 -3,.15 -5,0 -3,0 -6,.32 -5,3.67 -10,.3 -8,0 -8,0 -30,"Apr",2005 -12,.11 -11,4.38 -6,.9 -8,.01 -10,0 -16,0 -18,.3 -15,.13 -11,0 -12,0 -14,0 -9,0 -8,0 -11,0 -10,0 -7,0 -11,0 -14,0 -17,0 -19,0 -17,0 -10,.32 -14,1.59 -9,.56 -8,0 -12,0 -16,.17 -11,0 -10,0 -14,1.03 -31,"May",2005 -12,.18 -8,.1 -7,.06 -10,0 -9,0 -10,0 -12,0 -15,0 -16,0 -16,0 -20,0 -18,.01 -11,0 -20,.06 -21,.1 -16,.69 -12,0 -13,0 -14,0 -13,.97 -13,.05 -15,.08 -14,.08 -14,.41 -11,.08 -18,0 -17,0 -18,.77 -16,0 -15,.03 -16,0 -30,"Jun",2005 -18,0 -18,0 -17,.76 -19,.22 -19,.01 -25,1.75 -24,.98 -26,.13 -25,.08 -26,.03 -26,0 -26,0 -27,.04 -27,.06 -27,0 -23,.03 -19,0 -18,0 -18,0 -18,0 -20,0 -22,0 -19,0 -22,0 -25,0 -25,0 -25,.13 -28,.19 -26,.7 -26,2.5 -31,"Jul",2005 -26,.25 -23,0 -21,0 -23,0 -24,1.18 -26,.7 -24,.77 -21,5.78 -22,.01 -23,0 -23,0 -25,0 -26,0 -26,.09 -25,.77 -25,.65 -26,1.77 -27,0 -27,.01 -25,.04 -25,.04 -26,.1 -24,0 -22,0 -26,1.19 -27,0 -28,.2 -24,.03 -22,0 -24,0 -24,0 -31,"Aug",2005 -25,0 -26,0 -26,0 -28,0 -27,0 -24,0 -25,.15 -24,.25 -24,.34 -24,0 -26,0 -27,.24 -29,0 -29,0 -27,0 -22,1.35 -23,.1 -23,0 -23,.32 -25,0 -27,0 -22,0 -20,0 -20,0 -19,0 -21,0 -22,.03 -24,.69 -24,.11 -25,.08 -27,.05 -30,"Sep",2005 -22,0 -22,0 -21,0 -20,0 -20,0 -19,0 -19,0 -19,0 -21,0 -21,0 -18,0 -21,0 -23,0 -22,.25 -25,.01 -26,0 -24,1.74 -21,0 -22,0 -24,0 -21,0 -20,0 -23,0 -21,0 -20,0 -22,.28 -18,.13 -15,0 -18,.11 -12,0 -31,"Oct",2005 -14,0 -17,0 -17,0 -18,0 -21,0 -20,.03 -22,6.52 -17,10.33 -12,.5 -15,.03 -16,.01 -14,.13 -13,1.38 -17,.22 -18,.01 -15,0 -13,0 -15,0 -14,0 -13,0 -9,.81 -10,2.34 -10,.44 -8,.33 -8,2.2 -8,.22 -7,.01 -6,0 -5,0 -9,0 -10,0 -30,"Nov",2005 -11,0 -10,.04 -10,0 -12,0 -13,0 -14,.25 -11,.34 -9,0 -13,.05 -11,.01 -4,0 -7,0 -10,0 -11,0 -12,.08 -14,2.36 -3,.55 --1,0 -2,0 -5,0 -5,.89 -5,1.36 -1,.03 -5,.03 -2,0 -3,0 -6,.04 -13,.01 -17,1.12 -9,.98 -31,"Dec",2005 -1,.01 -1,0 --2,0 --1,.42 --1,.01 --1,0 --4,0 --5,0 --3,1.26 --7,0 --5,0 --2,0 --9,0 --13,0 --8,.8 -1,2.53 -0,0 --1,0 --3,0 --5,0 --5,0 -0,0 -5,0 -4,0 -0,1.26 -4,.13 -4,0 -2,0 -6,.65 -4,0 -0,.15 -31,"Jan",2006 -2,.01 -1,2.77 -4,.38 -3,0 -5,0 -2,0 --2,0 -3,0 -8,0 -5,0 -4,.84 -8,0 -6,0 -9,.84 --1,0 --4,0 -0,.01 -10,2.74 -3,0 -7,0 -7,0 -2,.1 -2,1.91 -2,.06 -3,.06 --1,0 --1,0 -4,0 -4,.19 -8,.03 -8,.66 -28,"Feb",2006 -2,0 -5,0 -10,1.45 -9,1.07 -6,.46 -2,0 -0,0 --2,0 --2,0 --2,0 -0,.32 --2,.74 --6,0 --3,0 -0,0 -4,0 -7,0 --4,0 --7,0 --2,0 --2,0 --1,0 -4,.03 -1,0 -3,0 -0,0 --4,.03 --3,0 -31,"Mar",2006 -2,0 --1,.72 --1,0 -1,0 -3,0 -2,0 -1,0 -2,0 -10,.01 -17,0 -11,0 -12,1.27 -17,0 -14,0 -4,0 -6,0 -5,0 -2,0 -2,0 -1,0 --1,0 -0,0 -4,0 -3,0 -5,0 -6,0 -6,0 -7,0 -10,0 -10,0 -14,0 -30,"Apr",2006 -16,.06 -13,0 -10,.55 -9,.75 -6,.13 -8,0 -11,.32 -10,1.75 -5,.01 -7,0 -12,0 -14,0 -17,.1 -14,.11 -17,.01 -13,0 -11,.01 -13,0 -14,0 -17,0 -15,.32 -10,4.33 -12,1.85 -13,.14 -13,.13 -10,.14 -11,0 -12,0 -10,0 -11,0 -31,"May",2006 -11,0 -15,0 -15,0 -16,0 -19,0 -16,0 -11,0 -13,.04 -15,0 -15,0 -17,1.68 -16,2.01 -17,.15 -15,.15 -13,1.12 -14,.28 -15,.03 -15,.17 -13,.15 -14,0 -13,0 -13,0 -12,0 -13,0 -16,0 -19,.14 -21,.1 -20,0 -23,0 -25,0 -26,0 -30,"Jun",2006 -25,.66 -25,1.41 -20,1.94 -16,0 -17,0 -18,0 -18,.53 -19,.1 -19,.65 -17,.14 -16,0 -16,0 -19,0 -20,.03 -20,1.78 -19,0 -21,0 -24,0 -24,.11 -24,.08 -22,0 -25,.09 -25,.43 -24,.32 -22,1.07 -24,4.72 -24,5.26 -26,2.06 -24,0 -21,0 -31,"Jul",2006 -22,3.56 -26,.55 -25,.75 -26,.55 -24,.42 -21,.88 -19,0 -20,0 -22,0 -23,1.02 -26,0 -27,.08 -26,.94 -24,0 -25,.29 -25,.55 -27,.18 -27,1.4 -26,.25 -25,0 -27,0 -26,.1 -23,.9 -22,0 -24,0 -25,0 -27,.36 -26,1.5 -26,0 -27,0 -27,0 -31,"Aug",2006 -28,.01 -29,0 -29,0 -27,0 -23,0 -24,0 -27,0 -25,0 -20,0 -22,0 -21,0 -18,0 -18,0 -22,0 -25,0 -21,0 -21,0 -22,0 -24,0 -23,.27 -23,0 -22,0 -23,.95 -23,.11 -24,.38 -23,.03 -24,1.46 -24,.48 -25,.39 -22,.25 -19,0 -30,"Sep",2006 -16,.62 -17,3.73 -18,.37 -18,.29 -19,.15 -19,.01 -19,.06 -20,.13 -21,.03 -18,0 -18,0 -16,.09 -17,1.73 -18,2.49 -19,1.83 -20,.23 -20,.01 -19,0 -21,0 -16,0 -13,0 -13,0 -21,.04 -21,.05 -15,0 -15,0 -15,0 -18,3.47 -15,1.02 -10,0 -31,"Oct",2006 -15,.19 -14,0 -19,0 -19,0 -18,.05 -11,1.04 -11,.04 -14,0 -16,0 -18,0 -17,.01 -15,0 -7,0 -6,0 -6,0 -8,0 -13,1.84 -16,.14 -16,.11 -14,.74 -9,.01 -8,0 -9,.04 -7,.01 -8,0 -6,0 -4,.55 -10,4.04 -7,0 -8,.01 -13,0 -30,"Nov",2006 -14,.81 -9,.67 -4,0 -1,0 -4,0 -5,0 -6,.08 -12,4.5 -14,.01 -12,0 -14,0 -13,.67 -11,.18 -12,.11 -13,.03 -16,5.55 -12,1.22 -6,0 -4,0 -4,0 -2,0 -3,.55 -5,1.55 -8,0 -4,0 -6,0 -7,.03 -8,0 -9,0 -15,.01 -31,"Dec",2006 -16,.25 -8,.03 -1,0 -1,0 --2,0 -1,0 -3,0 --2,0 --2,0 -3,0 -6,0 -5,0 -9,.37 -8,0 -7,0 -5,0 -6,0 -11,0 -5,0 -1,0 -3,.06 -6,.98 -9,1.97 -5,0 -2,.97 -8,.37 -4,0 -3,0 -3,0 -5,0 -2,.09 -31,"Jan",2007 -7,2.41 -5,0 -4,0 -6,0 -10,.25 -16,.25 -8,.74 -6,2.49 -2,0 --1,0 --2,0 -3,.04 -8,.29 -10,.18 -10,.04 -6,.01 --4,0 --4,0 -1,0 --2,0 --5,.01 --3,.03 -1,0 --1,0 --2,.03 --8,0 --4,0 -1,.01 --3,0 --3,0 --4,0 -28,"Feb",2007 --3,0 -0,.28 --5,0 --7,0 --11,0 --11,0 --9,.04 --9,0 --6,0 --6,0 --7,0 -0,0 --2,1.71 --4,5.66 --9,0 --8,0 --5,0 --5,0 --7,0 -1,.03 -3,0 --250,.19 -0,0 --3,0 --1,.66 -0,.36 -2,0 -1,0 -31,"Mar",2007 -2,.61 -7,2.5 -4,.04 -2,0 -0,0 --5,0 --8,.17 --9,0 --3,0 -5,.25 -4,0 -5,0 -10,0 -15,0 -4,.7 -1,3 --3,.53 --1,0 --3,.03 -5,.01 -1,0 -12,.17 -14,.99 -10,.33 -10,0 -7,0 -16,0 -15,0 -8,0 -8,0 -9,0 -30,"Apr",2007 -9,.06 -14,.18 -13,0 -8,.81 -3,.04 -2,0 -1,0 -1,0 -2,0 -3,0 -3,.08 -7,2.13 -7,.01 -5,.25 -7,5.18 -4,.86 -8,.01 -8,0 -10,.03 -11,0 -13,0 -15,0 -18,0 -20,0 -14,.05 -12,.69 -11,1.88 -14,.15 -14,0 -17,0 -31,"May",2007 -14,0 -16,.01 -13,0 -12,0 -13,0 -13,0 -10,0 -13,0 -18,0 -20,.03 -22,0 -20,.34 -16,.52 -12,0 -19,0 -22,1.14 -15,.1 -14,.05 -14,.01 -16,0 -16,.01 -15,0 -17,0 -19,0 -21,0 -24,0 -23,.58 -22,.03 -20,0 -20,0 -24,0 -30,"Jun",2007 -25,.08 -24,.32 -23,.76 -22,.66 -21,0 -18,0 -19,0 -25,.06 -25,0 -22,0 -22,.15 -23,.44 -22,.18 -16,0 -16,0 -22,0 -22,0 -25,0 -26,1.97 -23,1.8 -21,0 -21,0 -17,0 -19,0 -22,.04 -26,0 -27,.75 -26,.56 -23,.65 -22,0 -31,"Jul",2007 -18,0 -16,0 -18,0 -21,.56 -23,3.28 -23,.65 -21,0 -24,0 -26,0 -27,.38 -24,1.08 -21,.13 -21,0 -20,0 -25,0 -25,0 -24,0 -25,.05 -25,.14 -23,0 -20,0 -22,0 -20,0 -20,0 -23,0 -25,0 -26,.64 -25,.42 -24,4.2 -24,.01 -24,0 -31,"Aug",2007 -24,0 -26,0 -26,0 -27,0 -25,.04 -26,.24 -27,0 -29,.74 -27,2.82 -22,1.97 -21,0 -23,0 -25,0 -22,0 -22,0 -25,.32 -25,.83 -21,0 -18,.27 -16,2.27 -16,2.96 -16,.38 -20,0 -25,0 -28,.36 -24,3.29 -21,0 -22,0 -22,0 -23,0 -24,0 -30,"Sep",2007 -19,0 -18,0 -21,0 -22,0 -21,0 -24,0 -25,0 -26,0 -24,0 -25,.13 -23,1.57 -17,0 -18,0 -19,.01 -16,.15 -12,0 -12,0 -14,0 -16,0 -19,0 -19,0 -23,.04 -21,0 -18,0 -21,.03 -24,.18 -24,.01 -22,.08 -17,0 -15,0 -31,"Oct",2007 -17,0 -18,0 -21,0 -22,0 -22,0 -22,.03 -22,0 -23,0 -25,1.02 -22,.55 -15,1.05 -12,.05 -9,0 -11,0 -13,0 -15,0 -19,.06 -19,.08 -21,.43 -16,.3 -15,0 -16,0 -21,.04 -17,1.03 -12,.84 -12,2.24 -16,4.37 -11,.03 -6,0 -8,0 -10,0 -30,"Nov",2007 -12,0 -6,0 -9,0 -7,0 -7,.14 -9,.23 -5,0 -1,0 -1,.13 -4,.28 -4,.01 -8,.04 -11,.53 -8,.04 -10,2.36 -4,0 -3,0 -3,.56 -1,.51 -6,.69 -11,0 -14,.1 -3,0 --1,0 -3,0 -8,1.37 -11,.67 -2,0 -6,.04 -3,0 -31,"Dec",2007 --1,0 --1,.89 -2,.83 -0,0 --3,.36 --8,.03 --7,.08 -2,.17 -0,.1 -3,.37 -4,.1 -7,.01 -1,1.79 -2,.09 --1,.19 -1,3.12 -0,0 --2,0 -0,0 -2,0 -1,.01 -3,.05 -9,.57 -6,.19 -1,0 --1,.06 -3,.11 -3,1.05 -5,.38 -1,.52 -2,.69 -31,"Jan",2008 -3,.01 --1,0 --6,0 --3,0 --1,.11 -5,.09 -11,0 -10,0 -11,.15 -4,.04 -7,.48 -3,0 -2,.11 -3,.23 --1,0 -0,0 --2,.72 -3,.33 -0,0 --4,0 --8,0 --4,0 --2,0 --5,0 --4,0 --5,0 -0,0 --2,0 --2,.38 -4,.33 --3,0 -29,"Feb",2008 -1,4.39 -1,.38 -2,0 -2,0 -5,.08 -12,.19 -10,0 -3,.09 -3,0 --1,0 --9,0 --7,.36 --3,4.91 --2,0 -2,0 --2,0 -2,0 -10,.46 -2,0 --2,.25 --7,.05 --3,.72 --1,.08 --2,0 -1,0 -2,1.35 -1,.22 --5,0 --4,0 -31,"Mar",2008 -2,.19 -1,0 -7,0 -12,.55 -10,2.65 -3,.03 -3,.6 -9,1.08 -3,0 -3,0 -5,.04 -5,0 -5,0 -8,.03 -10,.01 -7,.33 -3,0 -5,.03 -7,1.46 -10,.84 -5,0 -4,0 -2,0 -2,0 -2,0 -9,0 -6,.04 -7,0 -3,0 -2,0 -7,.29 -30,"Apr",2008 -15,.14 -9,0 -3,.28 -6,.9 -10,.03 -8,0 -7,0 -10,0 -10,0 -15,0 -18,.53 -18,.56 -11,0 -5,0 -7,0 -9,0 -12,0 -16,0 -17,0 -19,.44 -16,.13 -15,0 -16,0 -17,0 -15,0 -17,.7 -12,1.13 -10,2.86 -11,.15 -8,0 -31,"May",2008 -9,.06 -15,0 -16,0 -14,.14 -13,0 -15,0 -16,0 -20,.14 -15,1.84 -12,.01 -11,.13 -10,1.31 -13,.1 -14,0 -18,.01 -16,3.06 -13,.06 -14,1.16 -12,0 -8,1.42 -13,.01 -12,.03 -14,0 -14,0 -15,0 -19,.15 -23,.93 -17,.51 -15,0 -17,0 -21,1.09 -30,"Jun",2008 -21,0 -19,0 -20,.44 -22,1.54 -23,.04 -23,0 -27,0 -28,0 -28,0 -28,2.13 -25,.03 -23,0 -24,0 -25,.22 -24,.01 -22,0 -19,.13 -16,.2 -16,0 -19,0 -22,0 -24,0 -23,0 -20,.36 -21,0 -23,.04 -25,.53 -25,.22 -25,0 -23,.03 -31,"Jul",2008 -22,0 -22,0 -24,0 -23,.75 -23,.58 -22,0 -25,.47 -25,1.55 -25,.43 -24,0 -22,0 -24,0 -25,.15 -23,4.17 -22,0 -23,0 -25,0 -26,0 -26,0 -27,0 -26,.1 -26,0 -24,2.4 -22,.9 -21,0 -23,0 -24,0 -22,0 -24,0 -25,0 -24,.61 -31,"Aug",2008 -25,0 -24,0 -21,0 -22,0 -23,.03 -26,0 -23,0 -21,.03 -20,0 -21,.51 -18,.47 -20,0 -20,0 -22,.14 -21,.95 -21,0 -21,0 -22,0 -22,0 -17,0 -19,0 -21,0 -20,0 -24,0 -24,.01 -20,0 -18,0 -20,.01 -20,2.03 -23,.13 -21,0 -30,"Sep",2008 -20,0 -22,0 -22,0 -24,0 -25,0 -24,6.85 -21,.19 -20,0 -22,1.27 -18,.14 -18,0 -19,1.3 -24,.18 -26,0 -24,0 -18,0 -16,0 -17,0 -15,0 -14,0 -17,0 -18,0 -17,0 -15,0 -15,.06 -16,.27 -19,5.85 -21,1.02 -18,0 -15,.08 -31,"Oct",2008 -17,.44 -12,0 -12,0 -12,0 -13,0 -12,0 -9,0 -11,0 -19,0 -16,0 -15,0 -15,0 -16,0 -18,0 -20,0 -20,0 -12,0 -9,0 -7,0 -7,0 -8,0 -8,0 -5,0 -5,0 -13,3.12 -10,.43 -7,.43 -6,2.2 -6,.01 -6,0 -8,0 -30,"Nov",2008 -11,0 -8,0 -9,.01 -10,.04 -14,.05 -16,.01 -16,.01 -13,.05 -8,0 -5,0 -3,0 -5,0 -7,1.28 -11,.05 -16,1.23 -9,.01 -4,0 -0,0 --2,0 -2,0 --1,.28 --3,.06 --2,.03 -0,.13 -4,.25 -4,0 -3,0 -3,0 -2,0 -3,1.97 -31,"Dec",2008 -4,.6 -1,0 -0,0 -3,.18 -0,0 --5,0 --3,0 --6,0 -3,0 -9,1.22 -5,3.49 -2,2.22 --2,0 -1,0 -10,.76 -7,.52 -1,.79 -0,.03 -1,2.1 --1,.29 --1,0 --8,0 --7,.05 -3,1.96 -6,0 -0,.25 -3,.13 -13,0 -5,0 -2,0 -0,0 -31,"Jan",2009 --5,0 --2,0 -0,0 --2,0 -4,0 -3,.38 -0,3.15 -1,.03 --2,0 --4,.06 --2,.08 --3,0 --2,0 --4,0 --7,0 --11,0 --12,0 --4,.01 --6,.13 --6,0 --8,0 --4,0 -1,0 -1,0 --7,0 --6,0 --6,.28 --2,1.4 --3,.01 --4,0 --5,0 -28,"Feb",2009 -2,0 -3,0 -0,.48 --4,.28 --8,.01 --5,0 -2,0 -9,0 -2,0 -5,0 -10,.05 -11,.04 -5,0 -1,.04 -0,.05 --1,0 --1,0 -0,.24 -3,.19 --2,0 --1,0 -2,0 --2,0 --3,0 --2,0 -2,.01 -9,.1 -5,0 -31,"Mar",2009 -1,.03 --4,.47 --8,0 --6,0 --1,0 -9,0 -11,0 -14,0 -10,0 -7,0 -12,0 -6,0 -2,0 -4,0 -6,0 -6,.14 -8,.19 -9,0 -10,.18 -3,0 -0,0 -4,0 -3,0 -1,0 -3,0 -6,.46 -8,.33 -10,.76 -14,1.14 -9,.38 -7,0 -30,"Apr",2009 -8,.29 -13,.06 -16,2.78 -11,.01 -12,0 -12,.03 -6,0 -6,0 -6,0 -10,0 -11,1 -6,0 -4,0 -7,1.32 -7,1.73 -8,.03 -10,0 -13,0 -14,0 -11,1.56 -13,.65 -10,.52 -9,.11 -11,0 -20,0 -22,0 -23,0 -23,0 -18,0 -13,0 -31,"May",2009 -17,1.55 -17,.58 -14,1.02 -12,1.54 -12,.56 -14,.51 -17,.81 -17,0 -21,.1 -17,0 -12,0 -12,0 -12,0 -16,1.28 -20,.43 -20,.25 -16,.24 -11,0 -11,0 -16,0 -18,0 -19,0 -22,0 -23,.1 -22,.08 -16,.36 -15,.23 -19,.03 -20,1.87 -18,.05 -18,0 -30,"Jun",2009 -15,0 -20,1.05 -20,1.93 -15,2.11 -14,2.07 -19,.05 -20,0 -23,0 -24,.03 -21,0 -22,.47 -24,.01 -23,.01 -22,.83 -22,0 -19,0 -16,.71 -20,1.41 -19,0 -21,2.13 -22,0 -23,0 -22,0 -22,0 -23,0 -25,0 -23,0 -21,.06 -22,.06 -22,.38 -31,"Jul",2009 -22,.76 -22,1.27 -22,0 -22,0 -19,0 -20,0 -21,0 -19,0 -19,0 -21,0 -21,0 -22,0 -20,0 -19,0 -20,0 -26,0 -23,.38 -22,.08 -20,0 -21,0 -21,0 -22,.25 -22,.85 -22,.44 -22,.1 -24,.88 -23,0 -24,.22 -25,0 -24,.52 -24,.19 -31,"Aug",2009 -22,.01 -24,1.98 -23,0 -23,0 -24,0 -24,.06 -21,0 -22,.01 -25,1.12 -27,1.14 -26,.42 -25,2.17 -24,.99 -23,0 -23,0 -24,0 -25,0 -25,.38 -25,.32 -25,2.81 -26,1.41 -24,.38 -24,.11 -22,0 -22,0 -23,0 -22,0 -22,2.76 -23,2.02 -22,0 -18,0 -30,"Sep",2009 -16,0 -17,0 -18,0 -19,0 -20,0 -20,0 -20,0 -19,0 -20,.55 -17,.28 -15,6.12 -17,.7 -20,.13 -19,0 -19,0 -19,.23 -16,1.04 -19,.03 -16,0 -14,0 -16,0 -19,0 -23,.53 -23,.13 -19,0 -14,.6 -18,1.52 -16,.01 -15,0 -13,0 -31,"Oct",2009 -10,0 -12,.04 -18,.05 -15,0 -12,0 -10,0 -17,0 -13,0 -18,.61 -16,.06 -10,0 -8,0 -12,0 -7,0 -6,1.69 -4,1.7 -5,2.63 -8,1.23 -6,0 -9,0 -13,0 -15,0 -13,.55 -18,1.97 -12,.29 -9,0 -11,1.49 -14,2.12 -13,0 -12,.01 -16,.18 -30,"Nov",2009 -11,.37 -9,0 -9,0 -6,0 -5,.04 -5,.03 -4,0 -11,0 -11,0 -12,0 -11,.24 -9,.05 -10,.24 -12,.14 -14,0 -12,0 -8,0 -8,0 -11,.37 -10,.72 -7,0 -6,.48 -6,.17 -9,.86 -10,.51 -8,.06 -6,0 -5,0 -10,0 -9,.5 -31,"Dec",2009 -4,0 -4,.9 -11,1.74 -6,0 -2,.71 --1,.11 --1,0 -2,.03 -6,2.88 -2,0 --4,0 --2,0 --1,1.61 -5,0 -5,0 -1,0 --2,0 --4,0 --5,1.7 --3,.25 --2,0 --2,0 --6,0 --4,0 --4,.19 -6,.89 -4,.23 -2,0 --4,0 --5,0 --2,.62 -31,"Jan",2010 -0,.19 --3,0 --7,0 --4,0 --4,0 --1,0 -0,0 --4,.14 --4,0 --7,0 --5,0 --2,0 --2,0 --1,0 -2,0 -5,0 -2,1.65 -5,.17 -2,0 -0,0 --1,0 -1,0 --1,0 -3,.06 -11,1.98 -4,.01 -0,0 -0,0 --5,0 --8,0 --6,0 -28,"Feb",2010 --5,0 --4,0 -0,.18 --1,0 --1,.74 --3,.7 --8,0 --6,0 --6,.53 --4,2.62 --2,0 --4,0 --6,0 --3,0 --6,0 --2,.04 --4,0 -2,0 -3,0 -1,0 -0,0 --1,.28 -1,.46 -3,.03 -0,.03 --1,.04 -0,0 -2,0 -31,"Mar",2010 -3,0 -2,.06 -3,.01 -2,0 -3,0 -3,0 -7,0 -7,0 -7,0 -8,0 -9,0 -8,.3 -8,1.28 -8,.75 -7,.25 -11,.05 -8,0 -9,0 -11,0 -12,0 -14,0 -16,1.04 -11,.5 -11,0 -11,0 -9,.55 -1,0 -5,.62 -11,.8 -8,1.63 -12,0 -30,"Apr",2010 -13,0 -15,0 -14,0 -15,0 -17,0 -21,.08 -21,0 -19,.25 -13,.67 -10,0 -12,0 -12,0 -8,.37 -10,0 -12,0 -18,.23 -14,.24 -7,0 -10,0 -10,0 -10,.1 -14,0 -12,.03 -11,0 -12,.88 -10,1.26 -10,.14 -8,0 -11,0 -16,0 -31,"May",2010 -18,0 -24,.08 -21,1.7 -18,0 -18,0 -17,0 -17,0 -12,0 -8,0 -8,0 -8,.28 -9,1.08 -12,0 -21,.29 -19,.19 -17,0 -16,.06 -12,.5 -14,0 -19,0 -21,0 -21,0 -20,.05 -21,.08 -23,.19 -24,0 -24,1.02 -21,2.02 -21,.25 -23,.25 -24,2.54 -30,"Jun",2010 -23,1.47 -23,0 -24,.7 -24,0 -25,0 -24,0 -18,0 -17,.9 -17,.55 -21,.37 -20,0 -22,.14 -22,.58 -22,0 -21,0 -20,.05 -21,.27 -20,.01 -21,0 -23,0 -22,0 -22,.25 -22,0 -24,.08 -22,.23 -23,0 -23,.03 -23,0 -23,0 -20,0 -31,"Jul",2010 -18,0 -18,0 -19,0 -22,0 -25,0 -27,0 -29,0 -29,0 -28,0 -27,0 -24,3.04 -24,0 -24,2.86 -24,2.78 -26,0 -27,0 -26,0 -27,0 -27,0 -27,3.01 -25,.13 -26,.1 -27,0 -29,0 -28,1.46 -22,1.12 -23,0 -26,0 -26,0 -22,0 -21,0 -31,"Aug",2010 -23,.13 -21,.03 -25,0 -27,0 -26,0 -25,0 -22,0 -23,0 -26,0 -28,0 -28,0 -26,1.47 -21,.46 -22,0 -21,.28 -26,.37 -25,.38 -24,.15 -24,.03 -24,0 -23,0 -26,.42 -23,.09 -20,.14 -21,0 -21,0 -19,0 -20,0 -23,0 -24,0 -25,0 -30,"Sep",2010 -26,0 -26,0 -27,0 -24,0 -17,0 -17,0 -21,0 -26,0 -22,0 -19,0 -18,0 -20,.46 -18,1.22 -20,0 -18,0 -18,.03 -19,1.13 -17,0 -18,0 -21,0 -14,0 -19,0 -25,.25 -25,.69 -27,0 -23,0 -19,.88 -22,1.18 -19,.05 -19,1.74 -31,"Oct",2010 -18,6.73 -13,0 -12,0 -11,1.32 -11,1.03 -12,.13 -16,.14 -14,0 -15,0 -15,0 -18,0 -19,.51 -12,0 -9,1.02 -11,.91 -12,0 -12,0 -11,0 -11,.81 -10,.01 -10,.01 -9,0 -10,0 -13,0 -15,0 -16,.03 -19,.88 -15,.13 -11,0 -8,0 -8,0 -30,"Nov",2010 -6,0 -4,0 -5,0 -9,2.73 -8,.13 -5,0 -4,0 -7,0 -9,0 -10,0 -7,0 -6,0 -7,0 -8,0 -9,0 -10,.84 -13,.88 -8,0 -5,0 -5,0 -5,0 -10,0 -9,0 -7,0 -4,.38 -4,.03 -2,0 -1,0 -4,0 -10,.38 -31,"Dec",2010 -8,3.59 -1,0 -2,0 -0,0 -1,0 -0,0 --3,0 --2,0 --5,0 --3,0 -2,.11 -4,2.45 --1,0 --6,0 --4,0 --6,0 --4,0 --4,0 --4,0 --3,0 -0,0 -0,0 -1,0 --1,0 --2,0 --3,0 --2,0 --1,0 --2,0 -0,0 -2,0 -31,"Jan",2011 -6,0 -3,.08 --2,0 -0,0 --1,0 --3,0 --3,.14 --6,.06 --4,0 --5,0 --6,.43 --4,.43 --5,0 --5,0 --3,0 --3,0 --4,0 --2,.9 -3,.18 -1,0 --4,0 --10,0 --10,0 --11,0 --2,0 --1,2.53 -0,.27 --1,.22 --3,0 --3,0 --6,0 -28,"Feb",2011 --2,.93 -2,1.6 --3,0 --6,0 --2,.76 -2,0 -1,.04 --1,0 --7,0 --7,0 --6,0 --3,0 --1,0 -7,0 -1,0 -3,0 -9,0 -13,0 -8,0 -2,.1 -0,.64 --5,.36 --5,0 --2,.04 -4,1.82 -2,0 -7,0 -9,.85 -31,"Mar",2011 -4,0 -4,0 --1,0 -2,0 -8,0 -11,4.05 -7,1.77 -2,0 -3,0 -8,3.67 -8,.94 -8,0 -8,0 -3,0 -3,.04 -8,1 -10,.04 -15,0 -13,0 -6,0 -9,.8 -7,0 -4,1.28 -2,.32 -0,0 --1,0 -2,0 -1,0 -2,0 -2,.04 -4,.29 -30,"Apr",2011 -4,.29 -6,0 -8,.1 -16,.08 -14,1.21 -6,0 -8,0 -7,1.63 -8,.46 -12,0 -19,.11 -18,1.65 -9,1.12 -13,0 -12,0 -11,1.93 -12,1.41 -11,0 -11,.32 -17,.42 -17,0 -7,.06 -12,1.08 -19,.79 -21,.33 -23,0 -23,0 -21,3.34 -15,0 -13,0 -31,"May",2011 -13,0 -16,.04 -21,.01 -14,.99 -13,.18 -12,.03 -14,0 -14,0 -14,0 -15,0 -16,0 -17,0 -17,0 -16,.03 -20,1.31 -20,2.96 -18,2.02 -18,.74 -17,1.55 -17,1.02 -19,.03 -20,0 -21,.76 -22,.46 -22,0 -24,.11 -23,.17 -23,0 -24,0 -26,0 -27,0 -30,"Jun",2011 -26,0 -22,0 -18,0 -17,.17 -21,.46 -20,0 -22,0 -26,0 -28,.11 -26,.13 -25,2.22 -24,2.06 -18,0 -19,0 -20,0 -20,.29 -22,.58 -22,0 -23,0 -23,0 -24,.93 -26,.24 -26,.03 -24,.56 -22,0 -21,0 -22,0 -23,.15 -23,0 -21,0 -31,"Jul",2011 -20,0 -22,0 -26,.52 -26,0 -24,0 -26,.09 -26,0 -25,3.64 -24,.19 -24,0 -25,0 -27,.15 -25,0 -22,0 -22,0 -23,0 -24,0 -27,0 -28,.03 -27,0 -29,0 -32,0 -30,.01 -29,0 -27,.43 -28,.18 -24,0 -24,.74 -28,.93 -28,0 -25,.05 -31,"Aug",2011 -26,.05 -25,.08 -25,1.21 -23,.19 -23,0 -24,.34 -27,.84 -26,0 -24,1.99 -23,.36 -22,0 -20,0 -21,.41 -22,1.21 -23,.06 -23,0 -23,0 -25,.57 -23,1.17 -23,0 -24,.79 -21,.18 -19,0 -21,0 -25,.38 -24,.76 -25,2.22 -22,5.59 -18,0 -19,0 -20,0 -30,"Sep",2011 -20,0 -22,.06 -23,0 -24,0 -24,4.57 -18,4.69 -19,5.44 -21,8.23 -24,0 -24,0 -22,.01 -21,.04 -22,0 -23,.48 -18,1.96 -12,0 -13,.17 -13,.04 -13,0 -19,.09 -20,.29 -23,.05 -21,2.37 -21,.03 -21,0 -22,0 -22,.08 -23,.05 -21,1.18 -17,0 -31,"Oct",2011 -15,.18 -8,.93 -9,.09 -12,.04 -15,0 -12,0 -12,0 -14,0 -17,0 -17,0 -17,0 -17,.64 -16,1.55 -17,.93 -15,.15 -14,0 -15,0 -13,.01 -16,1.13 -16,.42 -10,0 -7,0 -9,0 -11,.03 -10,.03 -12,0 -13,.27 -8,.17 -4,3.18 -3,.14 -3,0 -30,"Nov",2011 -8,0 -7,0 -8,0 -7,0 -4,0 -8,0 -8,0 -11,0 -9,0 -8,.13 -6,0 -6,0 -7,0 -13,0 -15,.05 -13,1.78 -8,.19 -2,0 -4,0 -8,.09 -10,.32 -8,3.76 -9,2.63 -7,0 -8,0 -8,0 -11,0 -17,0 -12,1.87 -7,.01 -31,"Dec",2011 -4,0 -3,0 -2,0 -5,0 -9,0 -13,.19 -10,2.91 -5,.99 -4,0 -1,0 -0,0 --1,0 -2,0 -3,0 -9,.06 -7,0 -7,0 --1,0 -2,0 -6,.03 -9,.41 -10,.81 -5,1.55 -0,0 -2,0 -2,0 -5,1.28 -3,.24 --3,0 -2,0 -7,.03 -31,"Jan",2012 -5,.97 -3,0 --4,0 --7,0 -0,0 -5,0 -7,0 -4,0 -1,0 -2,0 -2,.53 -7,.8 -4,1.28 -0,0 --6,0 --5,.1 -6,.47 -3,0 --4,0 --3,0 --3,1.18 --4,0 -1,.15 -6,0 -4,0 -3,.76 -9,.55 -6,0 -3,.01 -1,0 -7,0 -29,"Feb",2012 -9,0 -5,0 -2,0 -2,0 -1,0 -2,0 -2,0 -0,.15 --1,.1 -1,.03 --3,.38 --3,0 -2,0 -5,.03 -3,0 -4,.23 -3,.24 -3,0 -3,0 -0,0 -3,0 -7,0 -9,0 -7,.57 -3,0 -3,0 -7,0 -5,0 -5,2.55 -31,"Mar",2012 -5,.01 -6,.2 -8,.32 -3,0 -0,0 -4,0 -9,0 -16,0 -11,.17 -3,0 -6,0 -10,0 -17,.03 -16,0 -13,0 -13,0 -14,.03 -14,0 -15,0 -18,.01 -18,0 -16,0 -18,0 -14,2.07 -12,.47 -10,0 -4,0 -10,.11 -13,0 -7,0 -8,.71 -30,"Apr",2012 -9,.41 -10,0 -8,0 -12,0 -9,0 -7,0 -8,0 -10,0 -12,0 -11,0 -6,.03 -8,0 -8,0 -10,0 -19,.94 -21,0 -20,0 -15,.19 -15,.04 -14,0 -16,.64 -12,1.78 -6,1.98 -9,.39 -9,0 -10,.04 -10,0 -5,0 -9,0 -9,0 -31,"May",2012 -17,.58 -16,.38 -18,.75 -21,.11 -20,.15 -18,0 -14,.15 -17,.65 -19,.11 -15,.04 -13,0 -16,0 -19,0 -19,.7 -20,3.06 -22,.03 -20,0 -15,0 -18,0 -21,0 -21,.3 -20,.01 -21,.64 -22,.06 -23,0 -24,0 -25,.06 -24,.39 -26,.89 -24,1.28 -22,0 -30,"Jun",2012 -21,7.05 -17,.11 -17,.08 -17,.52 -15,.3 -16,0 -18,0 -19,0 -22,0 -23,0 -24,.65 -22,.97 -21,0 -19,0 -20,0 -20,0 -19,0 -17,0 -22,0 -26,0 -27,0 -28,.04 -24,.22 -22,0 -22,0 -18,0 -20,0 -23,0 -26,.03 -27,.01 -31,"Jul",2012 -26,0 -26,0 -24,0 -28,0 -28,0 -27,0 -30,.19 -28,0 -25,0 -26,0 -24,0 -24,0 -25,0 -23,1.21 -26,.86 -26,.19 -28,0 -29,1.02 -26,1.28 -21,.81 -20,.44 -23,0 -26,.27 -27,.17 -23,.43 -25,.56 -26,.14 -25,.13 -24,1.32 -24,0 -24,.65 -31,"Aug",2012 -24,0 -24,0 -26,.03 -26,0 -28,.44 -27,.6 -23,0 -24,0 -26,.01 -25,1.17 -24,0 -23,0 -22,0 -23,1.51 -23,.27 -23,0 -24,.01 -22,.01 -19,0 -21,.33 -21,0 -21,0 -21,0 -22,0 -23,.1 -24,0 -24,0 -25,.36 -22,0 -21,0 -23,0 -30,"Sep",2012 -24,0 -23,.32 -23,.28 -26,2.54 -25,.34 -23,0 -25,0 -25,1.88 -18,.05 -17,0 -15,0 -17,0 -18,0 -19,0 -18,0 -16,0 -16,0 -21,4.46 -15,.27 -14,0 -17,0 -22,.05 -15,0 -12,0 -13,0 -20,.01 -20,1.03 -20,.64 -16,0 -14,.25 -31,"Oct",2012 -14,0 -17,2.67 -20,3.43 -19,.01 -19,0 -15,0 -9,.05 -8,.18 -11,0 -14,.18 -11,0 -8,0 -5,0 -12,0 -16,.74 -12,0 -9,0 -13,.08 -16,3.16 -14,.62 -11,0 -11,0 -13,.15 -17,0 -18,0 -17,0 -16,0 -15,.46 -13,3.67 -8,4.15 -6,.08 -30,"Nov",2012 -7,0 -7,0 -6,0 -5,0 -3,0 -2,0 -3,0 -7,0 -6,0 -6,0 -10,0 -12,0 -10,2.03 -3,0 -2,0 -3,0 -4,0 -4,0 -4,0 -5,0 -5,0 -5,0 -6,0 -6,0 -1,0 -1,0 -1,1.19 -1,0 -1,0 -1,0 -31,"Dec",2012 -5,0 -8,.03 -12,0 -14,0 -9,0 -1,0 -1,.25 -1,.05 -6,.46 -9,.05 -4,.1 -2,0 -2,0 -2,0 -2,0 -2,.28 -7,.25 -8,.03 -5,0 -3,1.35 -7,2.54 -2,0 -2,0 --1,.38 -0,0 -1,1.35 -2,.28 -0,0 --1,.43 -0,0 -1,0 -31,"Jan",2013 -0,0 --1,0 --3,0 -0,0 -0,0 -1,.03 -3,0 --1,0 -1,.03 -3,0 -3,.69 -7,.37 -6,0 -8,.2 -4,.69 -1,1.77 -2,0 -0,0 -3,0 -5,0 --1,0 --6,0 --10,0 --8,0 --9,.14 --9,.14 --7,0 --3,.47 -7,0 -10,4.62 -6,5.13 -28,"Feb",2013 --2,0 --6,.13 --5,.09 --3,0 --4,0 -0,0 --2,0 -1,.34 --1,.18 --3,0 -4,.65 -3,0 -0,.06 -2,.19 -5,.23 -3,.43 --3,0 --3,0 -0,.15 --1,0 --2,0 --2,0 -3,.15 -3,0 -2,0 -2,.38 -6,.72 -5,0 -31,"Mar",2013 -3,0 -2,0 -0,0 -0,0 -2,0 -2,.51 -4,.03 -3,0 -6,0 -6,0 -8,0 -10,3.44 -4,.01 -2,0 -3,0 -5,.37 -1,.05 -0,1.14 -6,.09 -4,0 -2,0 -1,0 -2,0 -1,0 -2,.98 -5,.05 -5,0 -5,0 -7,0 -6,0 -6,.01 -30,"Apr",2013 -8,.11 -3,0 -2,0 -3,0 -8,0 -8,0 -10,0 -15,0 -19,0 -21,0 -20,.13 -12,1.28 -9,0 -10,0 -14,.05 -15,0 -17,.08 -15,.11 -17,1.65 -12,1.61 -5,0 -7,0 -10,0 -12,.11 -12,.2 -9,0 -10,0 -12,.04 -14,1.05 -15,.13 -31,"May",2013 -12,0 -12,0 -12,0 -12,0 -11,0 -11,0 -16,.13 -18,.55 -16,.11 -19,1.16 -21,.23 -14,0 -8,0 -8,.05 -15,.08 -20,.19 -18,0 -18,.05 -17,.03 -22,0 -22,0 -25,.04 -23,.89 -17,.34 -14,0 -15,0 -13,0 -15,.69 -23,0 -24,0 -25,0 -30,"Jun",2013 -25,0 -26,.13 -25,.83 -18,0 -18,0 -19,.06 -18,3.14 -20,.33 -20,0 -22,3.49 -22,.25 -22,0 -21,2.12 -20,.14 -19,0 -20,.06 -23,.85 -22,.58 -20,.05 -19,0 -20,0 -22,0 -22,0 -24,0 -25,0 -25,1.7 -25,1.47 -24,.06 -24,0 -25,.24 -31,"Jul",2013 -26,1.88 -26,.39 -26,.06 -27,.11 -27,0 -27,0 -27,.15 -25,.18 -25,.09 -26,.09 -25,0 -23,.93 -25,.93 -27,0 -28,0 -28,0 -28,0 -29,0 -29,0 -28,0 -27,0 -27,2.59 -25,1.61 -21,0 -19,0 -22,0 -23,0 -23,.51 -21,0 -21,0 -22,0 -31,"Aug",2013 -22,1.02 -23,0 -21,.04 -19,.03 -19,0 -20,.04 -22,.34 -25,.75 -26,.76 -22,0 -23,0 -24,0 -22,5.17 -18,0 -17,0 -18,0 -19,0 -20,.08 -19,0 -22,0 -22,0 -25,0 -21,0 -19,0 -18,0 -22,0 -25,0 -24,.09 -24,.66 -25,0 -26,0 -30,"Sep",2013 -26,0 -26,.18 -24,.67 -23,0 -20,0 -16,0 -16,0 -21,0 -17,0 -27,0 -27,0 -26,1.12 -20,.75 -16,0 -14,0 -18,1.19 -13,0 -12,0 -14,0 -17,0 -19,.93 -16,1.04 -14,0 -12,0 -13,0 -15,0 -18,0 -15,0 -14,0 -15,0 -31,"Oct",2013 -17,0 -19,0 -20,0 -21,0 -24,.08 -23,.2 -21,3.64 -13,0 -12,0 -12,2.57 -13,10.59 -15,2.18 -16,0 -14,0 -13,0 -14,0 -17,.25 -14,.19 -10,.24 -9,.28 -10,0 -13,0 -8,0 -5,0 -5,0 -5,0 -7,0 -6,0 -8,0 -9,0 -13,.04 -30,"Nov",2013 -17,.67 -13,0 -9,0 -2,0 -6,0 -12,0 -12,.34 -6,0 -3,0 -6,0 -6,0 -5,0 -0,0 -3,0 -4,0 -8,.03 -11,0 -12,.51 -7,0 -1,0 -1,0 -7,.04 -7,0 -0,0 --4,0 -2,1.65 -4,2.97 -0,0 --1,0 --3,0 -31,"Dec",2013 --1,0 -1,0 -3,0 -5,0 -10,.06 -9,1.99 -1,.04 --2,2.03 --1,.46 --1,.66 --7,0 --6,0 --6,0 --3,.83 --2,0 --2,0 --6,.15 --4,0 --3,0 -3,0 -5,0 -11,.09 -11,1.41 -2,.5 --4,0 --2,.09 --2,.08 -3,.03 -5,1.21 -4,1.03 --2,0 -31,"Jan",2014 --3,0 --3,.25 --9,.67 --12,0 --6,.53 -0,.48 --12,0 --11,0 --5,0 --1,.66 -7,1.65 -6,.04 -2,0 -6,.69 -2,0 -0,0 --1,0 --1,.11 --4,0 -1,0 --3,1 --14,.29 --13,0 --11,0 --8,.01 --8,0 --3,0 --9,0 --12,0 --12,0 --5,0 -28,"Feb",2014 -1,.01 -2,0 -1,2.48 --7,.06 --2,3.45 --4,0 --6,0 --7,0 --8,.13 --8,.08 --11,0 --8,1.09 --4,1.65 -1,0 --2,.06 --5,0 --8,0 --2,.6 -0,.55 -1,0 -4,.08 -3,0 -3,.08 -1,0 --5,0 --6,.04 --8,0 --10,0 -31,"Mar",2014 --7,0 -0,.09 --5,.19 --9,0 --4,0 --5,0 --2,0 -2,0 -5,0 -5,0 -7,0 -8,.37 --1,.23 --1,0 -9,0 -4,0 --2,.06 -2,0 -3,.61 -6,.23 -5,0 -8,0 -6,0 --2,0 --3,.09 --2,0 --1,0 -8,.36 -8,2.02 -5,3.43 -8,.05 -30,"Apr",2014 -7,.89 -9,.14 -12,.25 -8,.2 -7,0 -4,0 -6,1.19 -10,.22 -8,0 -9,0 -16,.1 -14,.1 -17,0 -20,0 -16,2.4 -6,.57 -4,0 -5,0 -10,0 -10,0 -9,0 -13,.33 -11,.04 -9,0 -9,.55 -14,.42 -12,0 -8,.06 -10,1.69 -9,5.87 -31,"May",2014 -16,.93 -14,0 -12,.27 -12,0 -12,0 -13,0 -11,0 -17,0 -18,0 -19,.47 -18,0 -19,0 -19,.11 -16,0 -20,1.82 -18,5.5 -12,0 -12,0 -12,0 -15,.48 -14,.55 -19,.15 -17,0 -15,0 -19,0 -21,0 -23,.14 -18,.25 -12,.39 -17,1.74 -16,.01 -30,"Jun",2014 -17,.48 -20,0 -22,.2 -21,.08 -19,.25 -17,0 -18,0 -20,0 -22,.32 -21,.88 -20,1.35 -18,.23 -24,3.33 -19,0 -18,0 -21,0 -26,0 -28,0 -24,.24 -18,0 -20,.83 -19,0 -21,0 -23,0 -26,.53 -25,1.75 -23,0 -22,0 -23,0 -24,0 -31,"Jul",2014 -26,0 -28,.51 -27,.25 -23,.2 -19,0 -21,0 -27,0 -26,.18 -25,1.03 -24,1.84 -23,1.69 -24,0 -26,.17 -24,1.77 -25,2.53 -22,.38 -20,0 -19,0 -21,0 -21,.06 -22,0 -24,0 -26,2.51 -24,1.24 -19,0 -21,0 -25,4.32 -24,3.59 -19,.25 -18,0 -21,0 -31,"Aug",2014 -23,.13 -23,.28 -22,0 -22,.01 -22,0 -22,0 -21,0 -19,0 -20,0 -21,0 -21,0 -20,2.97 -23,1.18 -17,0 -17,0 -17,0 -19,.06 -19,0 -21,0 -22,.11 -22,.01 -21,0 -20,0 -19,.03 -20,0 -21,0 -22,0 -22,.36 -17,0 -20,0 -25,.71 -30,"Sep",2014 -25,.39 -25,.55 -24,.14 -22,0 -25,0 -26,.33 -23,1.03 -18,0 -19,0 -21,0 -22,0 -17,0 -14,.18 -12,.23 -14,0 -15,.06 -16,0 -16,0 -15,0 -19,0 -20,.06 -18,.09 -13,0 -14,0 -16,1 -17,.19 -18,0 -19,0 -20,0 -20,0 -31,"Oct",2014 -18,.41 -17,0 -16,.17 -15,.65 -9,.01 -12,0 -17,.05 -17,.23 -14,0 -11,.08 -10,.83 -12,.1 -17,.08 -20,.06 -21,3.43 -17,2.51 -16,.36 -15,0 -10,0 -8,.03 -13,.13 -13,.41 -12,.17 -16,0 -12,0 -14,0 -10,0 -15,0 -15,.13 -9,.06 -9,0 -30,"Nov",2014 -9,.05 -8,.01 -9,0 -13,0 -12,0 -10,1.96 -7,0 -3,0 -6,0 -8,0 -9,0 -12,0 -4,.38 -1,.06 -0,0 -0,.17 -5,1.63 -1,0 --5,0 -1,0 -0,0 --2,0 -3,0 -14,.93 -11,0 -4,2.54 -1,.03 --1,0 --2,0 -4,0 -31,"Dec",2014 -7,.44 -3,.67 -3,.56 -2,0 -1,.56 -5,1.51 -2,.15 --2,0 -2,1.21 -4,.14 -1,.25 -1,0 -0,0 -4,0 -4,0 -2,.84 -7,0 -2,0 -2,0 -2,0 --2,0 --2,.13 -6,.38 -9,2.04 -10,.08 -3,0 -5,0 -6,.13 -3,0 --1,0 --4,0 -0,0 diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index 3bdca505b..c99d3c832 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -8,7 +8,6 @@ import requests from math import sqrt -from os.path import join, dirname, abspath from celery import shared_task @@ -16,7 +15,8 @@ data_to_survey, data_to_censuses from tr55.model import simulate_day -from gwlfe import gwlfe, parser +from gwlfe import gwlfe +from gwlfe.datamodel import DataModel logger = logging.getLogger(__name__) @@ -332,10 +332,7 @@ def change_key(modification): @shared_task def run_gwlfe(model_input): - # TODO pass real input to model instead of reading it from gms file - gms_filename = join(dirname(abspath(__file__)), - 'mapshed/data/sample_input.gms') - gms_file = open(gms_filename, 'r') - z = parser.GmsReader(gms_file).read() - - return gwlfe.run(z) + # The frontend expects an object with runoff and quality as keys. + z = DataModel(model_input) + response_json = {'runoff': gwlfe.run(z)} + return response_json From 561bea14110e0b285092d8717d1117a12a2a3630 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Mon, 23 May 2016 15:31:03 -0400 Subject: [PATCH 079/136] Improve style of GWLF-E BMP UI * Make style of Animal Waste Management (Poultry) slightly different from (Livestock) * Add Rural and Urban thumbnail groups * Add more vertical space in thumbnail view so labels are fully visible * For BMPs with multiple data model variables, change the label of the input to clarify which data model variable is being converted * In manual entry view: change size and color of info and error messages, improve spacing, and put fill pattern in line under header to improve readability * Use table to display data model values --- src/mmw/js/src/core/modificationConfig.json | 2 +- src/mmw/js/src/modeling/controls.js | 56 ++++++++++++++----- .../gwlfe/quality/templates/table.html | 42 +++++++------- .../src/modeling/gwlfeModificationConfig.js | 18 +++--- .../templates/controls/inputInfo.html | 7 +-- .../templates/controls/manualEntry.html | 25 ++++++--- .../templates/controls/thumbSelect.html | 31 +++++----- src/mmw/sass/components/_dropdowns.scss | 37 +++++++++--- 8 files changed, 140 insertions(+), 78 deletions(-) diff --git a/src/mmw/js/src/core/modificationConfig.json b/src/mmw/js/src/core/modificationConfig.json index 9e3246b89..8c9856931 100644 --- a/src/mmw/js/src/core/modificationConfig.json +++ b/src/mmw/js/src/core/modificationConfig.json @@ -160,7 +160,7 @@ "name": "Animal Waste Management Systems (Poultry)", "shortName": "Poultry Waste Man.", "summary": "These are systems that are designed to collect runoff and/or wastes from confined animal operations for the purpose of breaking down organic wastes via aerobic or anaerobic processes. Typical examples include waste lagoons or holding tanks that collect the wastes and prevent their discharge to nearby streams.", - "copyStyle": "grassland" + "copyStyle": "shrub" }, "buffer_strips": { "name": "Vegetated Buffer Strips (Rural)", diff --git a/src/mmw/js/src/modeling/controls.js b/src/mmw/js/src/modeling/controls.js index eab3cd3b6..91db9b61e 100644 --- a/src/mmw/js/src/modeling/controls.js +++ b/src/mmw/js/src/modeling/controls.js @@ -111,8 +111,22 @@ var InputInfoView = Marionette.ItemView.extend({ }, templateHelpers: function() { + var output = this.model.get('output'), + errorMessage = output && output.errorMessages[this.userInputName], + infoMessage = output && output.infoMessages[this.userInputName], + isError = false, + message; + + if (errorMessage) { + isError = true; + message = errorMessage; + } else if (infoMessage) { + message = infoMessage; + } + return { - userInputName: this.userInputName + message: message, + isError: isError }; } }); @@ -335,11 +349,14 @@ var LandCoverView = ModificationsView.extend({ this.model.set({ controlName: this.getControlName(), controlDisplayName: 'Land Cover', - modRows: [ - ['open_water', 'developed_open', 'developed_low', 'developed_med'], - ['developed_high', 'barren_land', 'deciduous_forest', 'shrub'], - ['grassland', 'pasture', 'cultivated_crops', 'woody_wetlands'] - ] + modRowGroups: [{ + name: '', + rows: [ + ['open_water', 'developed_open', 'developed_low', 'developed_med'], + ['developed_high', 'barren_land', 'deciduous_forest', 'shrub'], + ['grassland', 'pasture', 'cultivated_crops', 'woody_wetlands'] + ] + }] }); }, @@ -354,10 +371,13 @@ var ConservationPracticeView = ModificationsView.extend({ this.model.set({ controlName: this.getControlName(), controlDisplayName: 'Conservation Practice', - modRows: [ - ['rain_garden', 'infiltration_trench', 'porous_paving'], - ['green_roof', 'no_till', 'cluster_housing'] - ] + modRowGroups: [{ + name: '', + rows: [ + ['rain_garden', 'infiltration_trench', 'porous_paving'], + ['green_roof', 'no_till', 'cluster_housing'] + ] + }] }); }, @@ -383,10 +403,18 @@ var GwlfeConservationPracticeView = ModificationsView.extend({ controlDisplayName: 'Conservation Practice', manualMode: true, manualMod: null, - modRows: [ - ['cover_crops', 'conservation_tillage', 'nutrient_management', 'waste_management_livestock'], - ['waste_management_poultry', 'buffer_strips', 'streambank_fencing', 'streambank_stabilization'], - ['urban_buffer_strips', 'urban_streambank_stabilization', 'water_retention', 'infiltration'] + modRowGroups: [ + { + name: 'Rural', + rows: [ + ['cover_crops', 'conservation_tillage', 'nutrient_management', 'waste_management_livestock'], + ['waste_management_poultry', 'buffer_strips', 'streambank_fencing', 'streambank_stabilization'] + ] + }, + { + name: 'Urban', + rows: [['urban_buffer_strips', 'urban_streambank_stabilization', 'water_retention', 'infiltration']] + } ], dataModel: mockDataModel, errorMessages: null, diff --git a/src/mmw/js/src/modeling/gwlfe/quality/templates/table.html b/src/mmw/js/src/modeling/gwlfe/quality/templates/table.html index a2b05edc8..1978e3ee7 100644 --- a/src/mmw/js/src/modeling/gwlfe/quality/templates/table.html +++ b/src/mmw/js/src/modeling/gwlfe/quality/templates/table.html @@ -1,30 +1,30 @@ {% macro table(columnNames, rows, isSortable) %} -
    Total Area
    {{ model.get('value')|tr55Name }}{{ model.get('value')|modName }} {{ model.get('effectiveArea')|round(2)|toLocaleString(2) }} {{ model.get('effectiveUnits') }} diff --git a/src/mmw/js/src/modeling/templates/controls/userInput.html b/src/mmw/js/src/modeling/templates/controls/userInput.html new file mode 100644 index 000000000..60fc2145c --- /dev/null +++ b/src/mmw/js/src/modeling/templates/controls/userInput.html @@ -0,0 +1,5 @@ +
    + + +
    +
    From c50df79a95d671adf0606b1668cbfb925e1a6c3a Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Thu, 19 May 2016 09:37:03 -0400 Subject: [PATCH 075/136] Add table for GWLF-E quality results * Make it so that each result view gets the whole result object from the server, to better accomodate GWLF-E whose output doesn't have a clean break into runoff and quality. * Make table number formatting consistent across app. --- src/mmw/apps/modeling/tasks.py | 5 +- .../gwlfe/quality/templates/result.html | 1 + .../gwlfe/quality/templates/table.html | 35 ++++++++++ .../js/src/modeling/gwlfe/quality/views.js | 70 +++++++++++++++++++ .../gwlfe/runoff/templates/table.html | 6 +- src/mmw/js/src/modeling/models.js | 4 +- .../tr55/quality/templates/tableRow.html | 4 +- src/mmw/js/src/modeling/tr55/quality/views.js | 4 +- .../tr55/runoff/templates/tableRow.html | 4 +- src/mmw/js/src/modeling/tr55/runoff/views.js | 4 +- src/mmw/js/src/modeling/views.js | 2 +- src/mmw/sass/pages/_model.scss | 4 ++ 12 files changed, 125 insertions(+), 18 deletions(-) create mode 100644 src/mmw/js/src/modeling/gwlfe/quality/templates/result.html create mode 100644 src/mmw/js/src/modeling/gwlfe/quality/templates/table.html create mode 100644 src/mmw/js/src/modeling/gwlfe/quality/views.js diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index ae6098f93..3bdca505b 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -338,7 +338,4 @@ def run_gwlfe(model_input): gms_file = open(gms_filename, 'r') z = parser.GmsReader(gms_file).read() - # The frontend expects an object with runoff and quality as keys. - response_json = {'runoff': gwlfe.run(z)} - - return response_json + return gwlfe.run(z) diff --git a/src/mmw/js/src/modeling/gwlfe/quality/templates/result.html b/src/mmw/js/src/modeling/gwlfe/quality/templates/result.html new file mode 100644 index 000000000..06d8a9895 --- /dev/null +++ b/src/mmw/js/src/modeling/gwlfe/quality/templates/result.html @@ -0,0 +1 @@ +
    diff --git a/src/mmw/js/src/modeling/gwlfe/quality/templates/table.html b/src/mmw/js/src/modeling/gwlfe/quality/templates/table.html new file mode 100644 index 000000000..a2b05edc8 --- /dev/null +++ b/src/mmw/js/src/modeling/gwlfe/quality/templates/table.html @@ -0,0 +1,35 @@ +{% macro table(columnNames, rows, isSortable) %} + + + + {% for columnName in columnNames %} + {% if isSortable %} + + {% else %} + + {% endif %} + {% endfor %} + + + + {% for row in rows %} + + {% for value in row %} + {% if loop.index0 == 0 %} + + {% else %} + + {% endif %} + {% endfor %} + + {% endfor %} + +
    {{ columnName }}{{ columnName }}
    {{ value }}{{ value|round(2)|toLocaleString(2) }}
    +{% endmacro %} + + +
    + Mean Flow: {{ MeanFlow|round(0)|toLocaleString }} (m3/year) and {{ MeanFlowPerSecond|round(0)|toLocaleString }} (m3/s) +
    +{{ table(columnNames, landUseRows, true) }} +{{ table(columnNames, summaryRows, false) }} diff --git a/src/mmw/js/src/modeling/gwlfe/quality/views.js b/src/mmw/js/src/modeling/gwlfe/quality/views.js new file mode 100644 index 000000000..a2cca409b --- /dev/null +++ b/src/mmw/js/src/modeling/gwlfe/quality/views.js @@ -0,0 +1,70 @@ +"use strict"; + +var $ = require('jquery'), + _ = require('underscore'), + Marionette = require('../../../../shim/backbone.marionette'), + resultTmpl = require('./templates/result.html'), + tableTmpl = require('./templates/table.html'); + +var ResultView = Marionette.LayoutView.extend({ + className: 'tab-pane', + + template: resultTmpl, + + regions: { + tableRegion: '.quality-table-region', + }, + + modelEvents: { + 'change': 'onShow' + }, + + initialize: function(options) { + this.compareMode = options.compareMode; + this.scenario = options.scenario; + }, + + onShow: function() { + var result = this.model.get('result'); + this.tableRegion.reset(); + + if (result && result.Loads) { + if (!this.compareMode) { + this.tableRegion.show(new TableView({ + model: this.model, + })); + } + } + } +}); + +var TableView = Marionette.CompositeView.extend({ + template: tableTmpl, + + onAttach: function() { + $('[data-toggle="table"]').bootstrapTable(); + }, + + templateHelpers: function() { + function makeRow(load) { + return [load.Source, load.Sediment, load.TotalN, load.TotalP]; + } + + var result = this.model.get('result'), + columnNames = ['Sources', 'Sediment (kg)', 'Total Nitrogen (kg)', 'Total Phosphorus (kg)'], + landUseRows = _.map(result.Loads.slice(0,15), makeRow), + summaryRows = _.map(result.Loads.slice(15), makeRow); + + return { + MeanFlow: result.MeanFlow, + MeanFlowPerSecond: result.MeanFlowPerSecond, + columnNames: columnNames, + landUseRows: landUseRows, + summaryRows: summaryRows + }; + } +}); + +module.exports = { + ResultView: ResultView +}; diff --git a/src/mmw/js/src/modeling/gwlfe/runoff/templates/table.html b/src/mmw/js/src/modeling/gwlfe/runoff/templates/table.html index a7766a0f3..93cdb0dd6 100644 --- a/src/mmw/js/src/modeling/gwlfe/runoff/templates/table.html +++ b/src/mmw/js/src/modeling/gwlfe/runoff/templates/table.html @@ -7,7 +7,7 @@ {{ columnName }} {% else %} -     		{{ columnName }}{{ columnName }}
    {{ monthNames[value] }}{{ monthNames[value] }}{{ value|round(2)|toLocaleString }}{{ value|round(2)|toLocaleString(2) }}
    {{ measure }}{{ load|round(3)|toLocaleString }}{{ concentration|round(1)|toLocaleString }}{{ load|round(3)|toLocaleString(3) }}{{ concentration|round(1)|toLocaleString(1) }}{{ runoffType }}{{ depth|round(3)|toLocaleString }}{{ adjustedVolume|round(2)|toLocaleString }}{{ depth|round(3)|toLocaleString(3) }}{{ adjustedVolume|round(2)|toLocaleString(2) }}
    - +
    + + + {% for columnName in columnNames %} + {% if isSortable %} + + {% else %} + + {% endif %} + {% endfor %} + + + + {% for row in rows %} - {% for columnName in columnNames %} - {% if isSortable %} - + {% for value in row %} + {% if loop.index0 == 0 %} + {% else %} - + {% endif %} {% endfor %} - - - {% for row in rows %} - - {% for value in row %} - {% if loop.index0 == 0 %} - - {% else %} - - {% endif %} - {% endfor %} - - {% endfor %} - -
    {{ columnName }}{{ columnName }}
    {{ columnName }}{{ value }}{{ columnName }}{{ value|round(2)|toLocaleString(2) }}
    {{ value }}{{ value|round(2)|toLocaleString(2) }}
    + {% endfor %} + + {% endmacro %} diff --git a/src/mmw/js/src/modeling/gwlfeModificationConfig.js b/src/mmw/js/src/modeling/gwlfeModificationConfig.js index 314c6d6c1..082be0091 100644 --- a/src/mmw/js/src/modeling/gwlfeModificationConfig.js +++ b/src/mmw/js/src/modeling/gwlfeModificationConfig.js @@ -25,6 +25,7 @@ var n23Name = 'n23', PctAreaInfilName = 'PctAreaInfil', UrbAreaTotalName = 'UrbAreaTotal', lengthToConvertName = 'lengthToConvert', + lengthToConvertInAgName = 'lengthToConvertInAg', areaToConvertName = 'areaToConvert', percentAeuToConvertName = 'percentAeuToConvert', percentAreaToConvertName = 'percentAreaToConvert', @@ -51,15 +52,16 @@ function fromPairs(pairs) { } var displayNames = fromPairs([ - [n23Name, 'Area of row crops (ha)'], - [areaToConvertName, 'Area to convert (ha)'], [percentAeuToConvertName, '% of AEUs to convert'], [percentAreaToConvertName, '% of area to convert'], - [n42Name, 'Total length of streams (km) in ag areas'], - [n42bName, 'Total length of streams (km) in entire watershed'], + [areaToConvertName, 'Area to convert (ha)'], [lengthToConvertName, 'Length to convert (km)'], + [lengthToConvertInAgName, 'Length to convert in ag areas (km)'], + [n23Name, 'Area of row crops (ha)'], + [n42Name, 'Length of streams in ag areas (km)'], + [n42bName, 'Length of streams in watershed (km)'], [UrbAreaTotalName, 'Total urban area (ha)'], - [UrbLengthName, 'Total length (km) of streams in non-ag areas'] + [UrbLengthName, 'Length of streams in non-ag areas (km)'] ]); function getPercentStr(fraction) { @@ -208,9 +210,9 @@ function makeRuralStreamsBmpConfig(outputName) { return { dataModelNames: [n42Name, n42bName], - userInputNames: [lengthToConvertName], - validate: makeThresholdValidateFn(n42Name, LENGTH, lengthToConvertName), - computeOutput: makeComputeOutputFn(n42Name, lengthToConvertName, getOutput) + userInputNames: [lengthToConvertInAgName], + validate: makeThresholdValidateFn(n42Name, LENGTH, lengthToConvertInAgName), + computeOutput: makeComputeOutputFn(n42Name, lengthToConvertInAgName, getOutput) }; } diff --git a/src/mmw/js/src/modeling/templates/controls/inputInfo.html b/src/mmw/js/src/modeling/templates/controls/inputInfo.html index a50f91c6b..883eb0ff1 100644 --- a/src/mmw/js/src/modeling/templates/controls/inputInfo.html +++ b/src/mmw/js/src/modeling/templates/controls/inputInfo.html @@ -1,4 +1,3 @@ -
    - {{ output.errorMessages[userInputName] }} - {{ output.infoMessages[userInputName] }} -
    + diff --git a/src/mmw/js/src/modeling/templates/controls/manualEntry.html b/src/mmw/js/src/modeling/templates/controls/manualEntry.html index a75751922..49dc0232c 100644 --- a/src/mmw/js/src/modeling/templates/controls/manualEntry.html +++ b/src/mmw/js/src/modeling/templates/controls/manualEntry.html @@ -6,18 +6,25 @@
    {{ manualMod|modShortName}}
    - - + + -
      - {% for dataModelName in modConfig.dataModelNames %} -
    • {{ displayNames[dataModelName] }}: {{ dataModel[dataModelName] }}
      • - {% endfor %} -
    +
    + + + {% for dataModelName in modConfig.dataModelNames %} + + + + + {% endfor %} + +
    {{ displayNames[dataModelName] }}{{ dataModel[dataModelName]|round(2)|toLocaleString(2) }}
    -
    +
    - + +
    diff --git a/src/mmw/js/src/modeling/templates/controls/thumbSelect.html b/src/mmw/js/src/modeling/templates/controls/thumbSelect.html index b8f5491f7..776517cf7 100644 --- a/src/mmw/js/src/modeling/templates/controls/thumbSelect.html +++ b/src/mmw/js/src/modeling/templates/controls/thumbSelect.html @@ -1,18 +1,23 @@
    - {% for modRow in modRows %} -
      - {% for modKey in modRow %} -
    • -
      - - - -
      - -
      • - {% endfor %} -
    + {% for modRowGroup in modRowGroups %} + {% if modRowGroup.name %} +
    {{ modRowGroup.name }}
    + {% endif %} + {% for row in modRowGroup.rows %} +
      + {% for modKey in row %} +
    • +
      + + + +
      + +
      • + {% endfor %} +
    + {% endfor %} {% endfor %}
    diff --git a/src/mmw/sass/components/_dropdowns.scss b/src/mmw/sass/components/_dropdowns.scss index 233bfb6bc..b63db8431 100644 --- a/src/mmw/sass/components/_dropdowns.scss +++ b/src/mmw/sass/components/_dropdowns.scss @@ -64,22 +64,44 @@ } .manual-entry { - .header { - position: relative; + min-width: 250px; + .header { .header-text { - position: absolute; - top: 50%; - transform: translateY(-50%); + background-color: $ui-light; .name, .back-button { display: inline; } } + + svg { + position: absolute; + } } - .amount-form { - margin: 10px; + .content { + padding: 1rem; + + .form-group { + margin-bottom: 0; + } + + .info-region { + min-height: 25px; + + .info-message { + color: $ui-grey; + } + + .error-message { + color: $ui-danger; + } + } + + .amount-form { + margin: 10px; + } } } @@ -108,7 +130,6 @@ float: left; display: block; width: 60px; - height: 90px; margin-bottom: 0.2rem; transition: all 0.4s ease-out; margin-right: 1rem; From 6755a39c5f416e78f187b93f4dae351200c54725 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Mon, 23 May 2016 16:37:03 -0400 Subject: [PATCH 080/136] Run GWLF-E via parser It is easiest for us to create a temporary GMS file and run that through the GWLF-E parser than it is to run a direct dictionary. We also remove JSON.stringify where it would doubly JSON-encode a value. --- src/mmw/apps/modeling/tasks.py | 30 ++++++++++++++++++++++++------ src/mmw/js/src/modeling/models.js | 2 +- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index c99d3c832..611ea3d78 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -8,6 +8,7 @@ import requests from math import sqrt +from StringIO import StringIO from celery import shared_task @@ -15,8 +16,7 @@ data_to_survey, data_to_censuses from tr55.model import simulate_day -from gwlfe import gwlfe -from gwlfe.datamodel import DataModel +from gwlfe import gwlfe, parser logger = logging.getLogger(__name__) @@ -332,7 +332,25 @@ def change_key(modification): @shared_task def run_gwlfe(model_input): - # The frontend expects an object with runoff and quality as keys. - z = DataModel(model_input) - response_json = {'runoff': gwlfe.run(z)} - return response_json + """ + Given a model_input resulting from a MapShed run, converts that dictionary + to an intermediate GMS file representation, which is then parsed by GWLF-E + to create the final data model z. We run GWLF-E on this final data model + and return the results. + + This intermediate GMS file representation needs to be created because most + of GWLF-E logic is written to handle GMS files, and to support dictionaries + directly we would have to replicate all that logic. Thus, it is easier to + simply create a GMS file and have it read that. + """ + pre_z = parser.DataModel(model_input) + output = StringIO() + writer = parser.GmsWriter(output) + writer.write(pre_z) + + output.seek(0) + + reader = parser.GmsReader(output) + z = reader.read() + + return gwlfe.run(z) diff --git a/src/mmw/js/src/modeling/models.js b/src/mmw/js/src/modeling/models.js index b2f460c3f..df678b15c 100644 --- a/src/mmw/js/src/modeling/models.js +++ b/src/mmw/js/src/modeling/models.js @@ -829,7 +829,7 @@ var ScenarioModel = Backbone.Model.extend({ case GWLFE: return { - model_input: JSON.stringify(project.get('gis_data')) + model_input: project.get('gis_data') }; } } From 377295a9e917f28e91a330e5d077368febd6a148 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 24 May 2016 10:47:00 -0400 Subject: [PATCH 081/136] Pull GWLF-E from PyPI --- src/mmw/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mmw/requirements/base.txt b/src/mmw/requirements/base.txt index 2a521df70..9347b7c22 100644 --- a/src/mmw/requirements/base.txt +++ b/src/mmw/requirements/base.txt @@ -10,7 +10,7 @@ python-omgeo==1.7.2 rauth==0.7.1 djangorestframework-gis==0.8.2 tr55==1.1.3 +gwlf-e==0.3.0 requests==2.9.1 rollbar>=0.11.0,<=0.12.0 retry==0.9.1 --e git+https://github.com/WikiWatershed/gwlf-e.git@develop#egg=gwlf-e From 7ab7e5e9d8e33d07b6bcf12137547ca7539d940b Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 24 May 2016 11:01:18 -0400 Subject: [PATCH 082/136] Enable GWLF-E in Production --- src/mmw/mmw/settings/production.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mmw/mmw/settings/production.py b/src/mmw/mmw/settings/production.py index 7308ee2f2..c44b822a2 100644 --- a/src/mmw/mmw/settings/production.py +++ b/src/mmw/mmw/settings/production.py @@ -108,7 +108,7 @@ 'ZoomControl', ] -DISABLED_MODEL_PACKAGES = [GWLFE] +DISABLED_MODEL_PACKAGES = [] # END UI CONFIGURATION From ec0e6842d4c13ff4303c33ddd4fdf97ddfd5ff76 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Tue, 24 May 2016 17:26:14 -0400 Subject: [PATCH 083/136] Change 'Convert' to 'Apply' and 'Modify' The term 'convert' made more sense when we thought that BMPs were simply converting one land use to another. But we're actually leaving the land use intact, and just modifying it by applying BMPs. --- src/mmw/js/src/modeling/controls.js | 8 +-- .../src/modeling/gwlfeModificationConfig.js | 54 +++++++++---------- .../templates/controls/manualEntry.html | 2 +- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/mmw/js/src/modeling/controls.js b/src/mmw/js/src/modeling/controls.js index 91db9b61e..dee384a3e 100644 --- a/src/mmw/js/src/modeling/controls.js +++ b/src/mmw/js/src/modeling/controls.js @@ -161,12 +161,12 @@ var ManualEntryView = Marionette.CompositeView.extend({ ui: { 'backButton': '.back-button', - 'convertButton': '.convert-button' + 'applyButton': '.apply-button' }, events: { 'click @ui.backButton': 'clearManualMod', - 'click @ui.convertButton': 'convert', + 'click @ui.applyButton': 'applyModification', 'keyup': 'onKeyUp' }, @@ -191,7 +191,7 @@ var ManualEntryView = Marionette.CompositeView.extend({ onKeyUp: function(e) { if (e.keyCode === ENTER_KEYCODE) { - this.convert(); + this.applyModification(); } }, @@ -233,7 +233,7 @@ var ManualEntryView = Marionette.CompositeView.extend({ }); }, - convert: function() { + applyModification: function() { this.computeOutput(); var output = this.model.get('output'); if (output && gwlfeConfig.isValid(output.errorMessages)) { diff --git a/src/mmw/js/src/modeling/gwlfeModificationConfig.js b/src/mmw/js/src/modeling/gwlfeModificationConfig.js index 082be0091..05b004cff 100644 --- a/src/mmw/js/src/modeling/gwlfeModificationConfig.js +++ b/src/mmw/js/src/modeling/gwlfeModificationConfig.js @@ -24,11 +24,11 @@ var n23Name = 'n23', QretentionName = 'Qretention', PctAreaInfilName = 'PctAreaInfil', UrbAreaTotalName = 'UrbAreaTotal', - lengthToConvertName = 'lengthToConvert', - lengthToConvertInAgName = 'lengthToConvertInAg', - areaToConvertName = 'areaToConvert', - percentAeuToConvertName = 'percentAeuToConvert', - percentAreaToConvertName = 'percentAreaToConvert', + lengthToModifyName = 'lengthToModify', + lengthToModifyInAgName = 'lengthToModifyInAg', + areaToModifyName = 'areaToModify', + percentAeuToModifyName = 'percentAeuToModify', + percentAreaToModifyName = 'percentAreaToModify', AREA = 'area', LENGTH = 'length', FilterWidthDefault = 30, @@ -52,11 +52,11 @@ function fromPairs(pairs) { } var displayNames = fromPairs([ - [percentAeuToConvertName, '% of AEUs to convert'], - [percentAreaToConvertName, '% of area to convert'], - [areaToConvertName, 'Area to convert (ha)'], - [lengthToConvertName, 'Length to convert (km)'], - [lengthToConvertInAgName, 'Length to convert in ag areas (km)'], + [percentAeuToModifyName, '% of AEUs to modify'], + [percentAreaToModifyName, '% of area to modify'], + [areaToModifyName, 'Area to modify (ha)'], + [lengthToModifyName, 'Length to modify (km)'], + [lengthToModifyInAgName, 'Length to modify in ag areas (km)'], [n23Name, 'Area of row crops (ha)'], [n42Name, 'Length of streams in ag areas (km)'], [n42bName, 'Length of streams in watershed (km)'], @@ -175,24 +175,24 @@ function makeAgBmpConfig(outputName) { return { dataModelNames: [n23Name], - userInputNames: [areaToConvertName], - validate: makeThresholdValidateFn(n23Name, AREA, areaToConvertName), - computeOutput: makeComputeOutputFn(n23Name, areaToConvertName, getOutput) + userInputNames: [areaToModifyName], + validate: makeThresholdValidateFn(n23Name, AREA, areaToModifyName), + computeOutput: makeComputeOutputFn(n23Name, areaToModifyName, getOutput) }; } function makeAeuBmpConfig(outputName) { return { dataModelNames: [], - userInputNames: [percentAeuToConvertName], - validate: makePercentValidateFn(percentAeuToConvertName), + userInputNames: [percentAeuToModifyName], + validate: makePercentValidateFn(percentAeuToModifyName), computeOutput: function(dataModel, cleanUserInput) { - var percentAeuToConvert = cleanUserInput[percentAeuToConvertName], + var percentAeuToModify = cleanUserInput[percentAeuToModifyName], output = {}, infoMessages = {}; - if (percentAeuToConvert) { - output[outputName] = percentAeuToConvert; + if (percentAeuToModify) { + output[outputName] = percentAeuToModify; } return formatOutput(infoMessages, output); @@ -210,27 +210,27 @@ function makeRuralStreamsBmpConfig(outputName) { return { dataModelNames: [n42Name, n42bName], - userInputNames: [lengthToConvertInAgName], - validate: makeThresholdValidateFn(n42Name, LENGTH, lengthToConvertInAgName), - computeOutput: makeComputeOutputFn(n42Name, lengthToConvertInAgName, getOutput) + userInputNames: [lengthToModifyInAgName], + validate: makeThresholdValidateFn(n42Name, LENGTH, lengthToModifyInAgName), + computeOutput: makeComputeOutputFn(n42Name, lengthToModifyInAgName, getOutput) }; } function makeUrbanStreamsBmpConfig(getOutput) { return { dataModelNames: [UrbLengthName], - userInputNames: [lengthToConvertName], - validate: makeThresholdValidateFn(UrbLengthName, LENGTH, lengthToConvertName), - computeOutput: makeComputeOutputFn(UrbLengthName, lengthToConvertName, getOutput) + userInputNames: [lengthToModifyName], + validate: makeThresholdValidateFn(UrbLengthName, LENGTH, lengthToModifyName), + computeOutput: makeComputeOutputFn(UrbLengthName, lengthToModifyName, getOutput) }; } function makeUrbanAreaBmpConfig(getOutput) { return { dataModelNames: [UrbAreaTotalName], - userInputNames: [areaToConvertName], - validate: makeThresholdValidateFn(UrbAreaTotalName, AREA, areaToConvertName), - computeOutput: makeComputeOutputFn(UrbAreaTotalName, areaToConvertName, getOutput) + userInputNames: [areaToModifyName], + validate: makeThresholdValidateFn(UrbAreaTotalName, AREA, areaToModifyName), + computeOutput: makeComputeOutputFn(UrbAreaTotalName, areaToModifyName, getOutput) }; } diff --git a/src/mmw/js/src/modeling/templates/controls/manualEntry.html b/src/mmw/js/src/modeling/templates/controls/manualEntry.html index 49dc0232c..2209cb914 100644 --- a/src/mmw/js/src/modeling/templates/controls/manualEntry.html +++ b/src/mmw/js/src/modeling/templates/controls/manualEntry.html @@ -25,6 +25,6 @@
    - +
    From 3bb27cbd5aa3fdc16dcd128942cc969d0fbde56e Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 24 May 2016 13:21:48 -0400 Subject: [PATCH 084/136] Add GMS Export Endpoint We go with POST /api/modeling/export/gms/ rather than the more compact GET /project/{id}/export/gms/ because we need to support unsaved projects. --- src/mmw/apps/modeling/tasks.py | 19 ++++++++++++++----- src/mmw/apps/modeling/urls.py | 1 + src/mmw/apps/modeling/views.py | 20 ++++++++++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index 611ea3d78..1544eda61 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -343,14 +343,23 @@ def run_gwlfe(model_input): directly we would have to replicate all that logic. Thus, it is easier to simply create a GMS file and have it read that. """ - pre_z = parser.DataModel(model_input) + output = to_gms_file(model_input) + + reader = parser.GmsReader(output) + z = reader.read() + + return gwlfe.run(z) + + +def to_gms_file(mapshed_data): + """ + Given a dictionary of MapShed data, uses GWLF-E to convert it to a GMS file + """ + pre_z = parser.DataModel(mapshed_data) output = StringIO() writer = parser.GmsWriter(output) writer.write(pre_z) output.seek(0) - reader = parser.GmsReader(output) - z = reader.read() - - return gwlfe.run(z) + return output diff --git a/src/mmw/apps/modeling/urls.py b/src/mmw/apps/modeling/urls.py index 592864d6f..485f68c4b 100644 --- a/src/mmw/apps/modeling/urls.py +++ b/src/mmw/apps/modeling/urls.py @@ -29,4 +29,5 @@ url(r'start/gwlfe/$', views.start_gwlfe, name='start_gwlfe'), url(r'boundary-layers/(?P\w+)/(?P[0-9]+)/$', views.boundary_layer_detail, name='boundary_layer_detail'), + url(r'export/gms/?$', views.export_gms, name='export_gms'), ) diff --git a/src/mmw/apps/modeling/views.py b/src/mmw/apps/modeling/views.py index f2a3360de..11a4a18b6 100644 --- a/src/mmw/apps/modeling/views.py +++ b/src/mmw/apps/modeling/views.py @@ -16,6 +16,8 @@ from django.conf import settings from django.db import connection from django.contrib.gis.geos import GEOSGeometry +from django.http import HttpResponse +from django.core.servers.basehttp import FileWrapper import celery from celery import chain, group @@ -263,6 +265,24 @@ def _initiate_mapshed_job_chain(mapshed_input, job_id): return chain.apply_async(link_error=errback) +@decorators.api_view(['POST']) +@decorators.permission_classes((AllowAny, )) +def export_gms(request, format=None): + mapshed_data = json.loads(request.POST.get('mapshed_data', '{}')) + filename = request.POST.get('filename', None) + + if not mapshed_data or not filename: + return Response('Must specify mapshed_data and filename', + status.HTTP_400_BAD_REQUEST) + + gms_file = tasks.to_gms_file(mapshed_data) + + response = HttpResponse(FileWrapper(gms_file), content_type='text/plain') + response['Content-Disposition'] = 'attachment; '\ + 'filename={}.gms'.format(filename) + return response + + @decorators.api_view(['POST']) @decorators.permission_classes((AllowAny, )) def start_analyze(request, format=None): From 5465892fd03a8a9cb1973348ec08d0f1ffc96b7f Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 24 May 2016 14:49:19 -0400 Subject: [PATCH 085/136] Add GMS Export item in project menu This will only be available to GWLF-E projects that have a real value for gis_data. Clicking the button will download a .gms file with the same name as the project. --- src/mmw/js/src/core/csrf.js | 4 ++++ .../js/src/modeling/templates/projectMenu.html | 8 ++++++++ src/mmw/js/src/modeling/views.js | 16 ++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/src/mmw/js/src/core/csrf.js b/src/mmw/js/src/core/csrf.js index 3f697c7d4..0c603e6e6 100644 --- a/src/mmw/js/src/core/csrf.js +++ b/src/mmw/js/src/core/csrf.js @@ -41,3 +41,7 @@ exports.jqueryAjaxSetupOptions = { } } }; + +exports.getToken = function() { + return getCookie('csrftoken'); +} diff --git a/src/mmw/js/src/modeling/templates/projectMenu.html b/src/mmw/js/src/modeling/templates/projectMenu.html index 48ba4d5f3..1ba1a69ac 100644 --- a/src/mmw/js/src/modeling/templates/projectMenu.html +++ b/src/mmw/js/src/modeling/templates/projectMenu.html @@ -23,6 +23,14 @@

    {{ name }}

    {% if itsi %}
  • Embed in ITSI
    • {% endif %} + {% if gwlfe %} +
    + + + +
    +
  • Export GMS
    • + {% endif %} {% if user and not itsi_embed %}
  • My Projects
    • diff --git a/src/mmw/js/src/modeling/views.js b/src/mmw/js/src/modeling/views.js index e8562cae1..432af02ef 100644 --- a/src/mmw/js/src/modeling/views.js +++ b/src/mmw/js/src/modeling/views.js @@ -5,6 +5,7 @@ Marionette = require('../../shim/backbone.marionette'), App = require('../app'), settings = require('../core/settings'), + csrf = require('../core/csrf'), models = require('./models'), controls = require('./controls'), analyzeViews = require('../analyze/views.js'), @@ -95,6 +96,8 @@ var ProjectMenuView = Marionette.ItemView.extend({ print: '#print-project', save: '#save-project', privacy: '#project-privacy', + exportGms: '#export-gms', + exportGmsForm: '#export-gms-form', itsiClone: '#itsi-clone' }, @@ -105,6 +108,7 @@ var ProjectMenuView = Marionette.ItemView.extend({ 'click @ui.print': 'printProject', 'click @ui.save': 'saveProjectOrLoginUser', 'click @ui.privacy': 'setProjectPrivacy', + 'click @ui.exportGms': 'downloadGmsFile', 'click @ui.itsiClone': 'getItsiEmbedLink' }, @@ -115,6 +119,11 @@ var ProjectMenuView = Marionette.ItemView.extend({ itsi: App.user.get('itsi'), itsi_embed: settings.get('itsi_embed'), editable: isEditable(this.model), + csrftoken: csrf.getToken(), + gwlfe: this.model.get('model_package') === models.GWLFE && + this.model.get('gis_data') !== null && + this.model.get('gis_data') !== '' && + this.model.get('gis_data') !== '{}', is_new: this.model.isNew() }; }, @@ -231,6 +240,13 @@ var ProjectMenuView = Marionette.ItemView.extend({ }); }, + downloadGmsFile: function() { + // We can't download a file from an AJAX call. One either has to + // load the data in an iframe, or submit a form that responds with + // Content-Disposition: attachment. We prefer submitting a form. + this.ui.exportGmsForm.submit(); + }, + getItsiEmbedLink: function() { var self = this, embedLink = window.location.origin + From 88b30e66723c15730d63efbee61700b66ad2b27d Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 25 May 2016 16:26:24 -0400 Subject: [PATCH 086/136] Force registration of MapShed tasks Since MapShed tasks are defined in a submodule to a Django app, and never explicitly specified in INSTALLED_APPS, the autoloading functionality did not work reliably in all environments. Particularly, it would work in local vagrant setups, but not on the staging AWS server. By including it in CELERY_IMPORTS we ensure that the tasks are always loaded. --- src/mmw/mmw/settings/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index f5ee5ac1f..58871f097 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -117,7 +117,10 @@ def get_env_setting(setting): environ.get('MMW_CACHE_HOST', 'localhost'), environ.get('MMW_CACHE_PORT', 6379)) -CELERY_IMPORTS = ('celery.task.http',) +CELERY_IMPORTS = ('celery.task.http', + # Submodule task is not always autodiscovered + 'apps.modeling.mapshed.tasks', + ) CELERY_ACCEPT_CONTENT = ['json'] CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' From 58737712c0cbfb3380177644e1c082b33c21d40a Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 26 May 2016 12:14:20 -0400 Subject: [PATCH 087/136] Convert decimals to floats The database returns Decimal values, however they are not JSON serializable, thus we convert them to floats which are. This is required not just for communicating to the front-end but also between Celery tasks, which use JSON serialization as the intermediate format. --- src/mmw/apps/modeling/mapshed/calcs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index 42f1930e9..48497d60e 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -305,8 +305,8 @@ def point_source_discharge(geom, area): cursor.execute(sql, [geom.wkt]) mg_d, kgn_month, kgp_month = cursor.fetchone() - n_load = [kgn_month] * 12 if kgn_month else [0.0] * 12 - p_load = [kgp_month] * 12 if kgp_month else [0.0] * 12 + n_load = [float(kgn_month)] * 12 if kgn_month else [0.0] * 12 + p_load = [float(kgp_month)] * 12 if kgp_month else [0.0] * 12 discharge = [float(mg_d * days * LITERS_PER_MGAL) / area for days in MONTHDAYS] if mg_d else [0.0] * 12 From 886dcf5ea688b67ed7c851ef53c30d3e30a0d7ad Mon Sep 17 00:00:00 2001 From: Hector Castro Date: Fri, 27 May 2016 09:53:21 -0400 Subject: [PATCH 088/136] Bump pyrollbar to 0.12.1 See also: https://github.com/rollbar/pyrollbar/blob/master/CHANGELOG.md --- src/mmw/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mmw/requirements/base.txt b/src/mmw/requirements/base.txt index 9347b7c22..c39f9d79e 100644 --- a/src/mmw/requirements/base.txt +++ b/src/mmw/requirements/base.txt @@ -12,5 +12,5 @@ djangorestframework-gis==0.8.2 tr55==1.1.3 gwlf-e==0.3.0 requests==2.9.1 -rollbar>=0.11.0,<=0.12.0 +rollbar==0.12.1 retry==0.9.1 From 60b983d7c1fe718a1d6205cf59bf03745b7ab4eb Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 27 May 2016 10:49:17 -0400 Subject: [PATCH 089/136] Lint --- src/mmw/js/src/core/csrf.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mmw/js/src/core/csrf.js b/src/mmw/js/src/core/csrf.js index 0c603e6e6..89bd49025 100644 --- a/src/mmw/js/src/core/csrf.js +++ b/src/mmw/js/src/core/csrf.js @@ -44,4 +44,4 @@ exports.jqueryAjaxSetupOptions = { exports.getToken = function() { return getCookie('csrftoken'); -} +}; From 6365f4a4154144142cc18908c97edb7c279c44ff Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 27 May 2016 10:52:38 -0400 Subject: [PATCH 090/136] refactor: Reduce GWLF-E runs by hashing input, mods For TR-55, we use the inputmod_hash field to judge whether the results should be fetched again or not. If they don't match, then we surmise that the results are stale and must be run again. Previously, we didn't have an inputmod_hash field in the GWLF-E endpoint. Thus, when the front-end would compare its value to the locally computed one, they would always mismatch and the results would always be reran. Now that we do incorporate the field, the results should only rerun when the inputs / modifications are actually different. --- src/mmw/apps/modeling/tasks.py | 7 +++++-- src/mmw/apps/modeling/views.py | 7 ++++--- src/mmw/js/src/modeling/models.js | 1 + 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/mmw/apps/modeling/tasks.py b/src/mmw/apps/modeling/tasks.py index 1544eda61..92d62029a 100644 --- a/src/mmw/apps/modeling/tasks.py +++ b/src/mmw/apps/modeling/tasks.py @@ -331,7 +331,7 @@ def change_key(modification): @shared_task -def run_gwlfe(model_input): +def run_gwlfe(model_input, inputmod_hash): """ Given a model_input resulting from a MapShed run, converts that dictionary to an intermediate GMS file representation, which is then parsed by GWLF-E @@ -348,7 +348,10 @@ def run_gwlfe(model_input): reader = parser.GmsReader(output) z = reader.read() - return gwlfe.run(z) + result = gwlfe.run(z) + result['inputmod_hash'] = inputmod_hash + + return result def to_gms_file(mapshed_data): diff --git a/src/mmw/apps/modeling/views.py b/src/mmw/apps/modeling/views.py index 11a4a18b6..2db982e55 100644 --- a/src/mmw/apps/modeling/views.py +++ b/src/mmw/apps/modeling/views.py @@ -199,10 +199,11 @@ def start_gwlfe(request, format=None): user = request.user if request.user.is_authenticated() else None created = now() model_input = json.loads(request.POST['model_input']) + inputmod_hash = request.POST.get('inputmod_hash', '') job = Job.objects.create(created_at=created, result='', error='', traceback='', user=user, status='started') - task_list = _initiate_gwlfe_job_chain(model_input, job.id) + task_list = _initiate_gwlfe_job_chain(model_input, inputmod_hash, job.id) job.uuid = task_list.id job.save() @@ -215,8 +216,8 @@ def start_gwlfe(request, format=None): ) -def _initiate_gwlfe_job_chain(model_input, job_id): - chain = (tasks.run_gwlfe.s(model_input) | +def _initiate_gwlfe_job_chain(model_input, inputmod_hash, job_id): + chain = (tasks.run_gwlfe.s(model_input, inputmod_hash) | save_job_result.s(job_id, model_input)) return chain.apply_async(link_error=save_job_error.s(job_id)) diff --git a/src/mmw/js/src/modeling/models.js b/src/mmw/js/src/modeling/models.js index df678b15c..27bfaa8f3 100644 --- a/src/mmw/js/src/modeling/models.js +++ b/src/mmw/js/src/modeling/models.js @@ -829,6 +829,7 @@ var ScenarioModel = Backbone.Model.extend({ case GWLFE: return { + inputmod_hash: self.get('inputmod_hash'), model_input: project.get('gis_data') }; } From 74c1d4ae1385cd2ae4f0e5d5416755b5fa463a60 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 27 May 2016 11:01:50 -0400 Subject: [PATCH 091/136] refactor: fetchResultsIfNeeded now returns promise In order to not send multiple fetchResultsIfNeeded requests when fetchGisDataPromise succeeds, when need to refactor that to return promises as well. We do this as follows: 1. We update ScenarioModel.fetchResultsIfNeeded to return a combination of all fetchResults promises for the scenario. These are the startPromise, which tracks the initial call, and the pollingPromise, which tracks subsequent job checks. 2. We update ProjectModel.fetchResultsIfNeeded to return a combination of all its scenario's fetchResultsIfNeeded promises. --- src/mmw/js/src/modeling/models.js | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/mmw/js/src/modeling/models.js b/src/mmw/js/src/modeling/models.js index 27bfaa8f3..9ae05c46c 100644 --- a/src/mmw/js/src/modeling/models.js +++ b/src/mmw/js/src/modeling/models.js @@ -166,13 +166,17 @@ var ProjectModel = Backbone.Model.extend({ }, fetchResultsIfNeeded: function() { + var promises = []; + this.get('scenarios').forEach(function(scenario) { - scenario.fetchResultsIfNeeded(); + promises.push(scenario.fetchResultsIfNeeded()); }); this.get('scenarios').on('add', function(scenario) { scenario.fetchResultsIfNeeded(); }); + + return $.when.apply($, promises); }, updateName: function(newName) { @@ -699,7 +703,8 @@ var ScenarioModel = Backbone.Model.extend({ }, fetchResultsIfNeeded: function() { - var inputmod_hash = this.get('inputmod_hash'), + var self = this, + inputmod_hash = this.get('inputmod_hash'), needsResults = this.get('results').some(function(resultModel) { var emptyResults = !resultModel.get('result'), staleResults = inputmod_hash !== resultModel.get('inputmod_hash'); @@ -707,12 +712,24 @@ var ScenarioModel = Backbone.Model.extend({ return emptyResults || staleResults; }); - if (needsResults) { - var fetchResults = _.bind(this.fetchResults, this); - App.currentProject - .fetchGisDataIfNeeded() - .then(fetchResults); + if (needsResults && self.fetchResultsPromise === undefined) { + var fetchResults = _.bind(self.fetchResults, self), + fetchGisDataPromise = App.currentProject.fetchGisDataIfNeeded(); + + self.fetchResultsPromise = fetchGisDataPromise.then(function() { + var promises = fetchResults(); + return $.when(promises.startPromise, promises.pollingPromise); + }); + + self.fetchResultsPromise + .always(function() { + // Clear promise so we start a new one next time + delete self.fetchResultsPromise; + }); } + + // Return fetchResultsPromise if it exists, else an immediately resovled one. + return self.fetchResultsPromise || $.when(); }, setResults: function() { From 9a53203f0673b943fdc84a91986d3bb5f39eeeee Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 27 May 2016 11:05:25 -0400 Subject: [PATCH 092/136] refactor: Extract Task View Model and Message View This model was devised to display the loading and error values in the Analyze phase. Since we need to do the same thing for when gathering MapShed data, we extract this functionality into core models and views so it can be reused. We also add the ability to add a custom message to the loading stage. --- src/mmw/js/src/analyze/models.js | 16 ---------------- src/mmw/js/src/analyze/views.js | 17 ++++++----------- src/mmw/js/src/core/models.js | 16 ++++++++++++++++ .../{analyze => core}/templates/message.html | 0 src/mmw/js/src/core/views.js | 7 +++++++ 5 files changed, 29 insertions(+), 27 deletions(-) rename src/mmw/js/src/{analyze => core}/templates/message.html (100%) diff --git a/src/mmw/js/src/analyze/models.js b/src/mmw/js/src/analyze/models.js index bad85eca4..55b26d166 100644 --- a/src/mmw/js/src/analyze/models.js +++ b/src/mmw/js/src/analyze/models.js @@ -26,25 +26,9 @@ var AnalyzeTaskModel = coreModels.TaskModel.extend({ ) }); -var AnalyzeMessageModel = Backbone.Model.extend({ - message: null, - iconClass: null, - - setError: function() { - this.set('message', 'Error'); - this.set('iconClasses', 'fa fa-exclamation-triangle'); - }, - - setAnalyzing: function() { - this.set('message', 'Analyzing'); - this.set('iconClasses', 'fa fa-circle-o-notch fa-spin'); - } -}); - module.exports = { AnalyzeTaskModel: AnalyzeTaskModel, LayerModel: LayerModel, LayerCollection: LayerCollection, LayerCategoryCollection: LayerCategoryCollection, - AnalyzeMessageModel: AnalyzeMessageModel }; diff --git a/src/mmw/js/src/analyze/views.js b/src/mmw/js/src/analyze/views.js index 46f82d548..34d7acf31 100644 --- a/src/mmw/js/src/analyze/views.js +++ b/src/mmw/js/src/analyze/views.js @@ -10,10 +10,10 @@ var $ = require('jquery'), modalModels = require('../core/modals/models'), modalViews = require('../core/modals/views'), coreModels = require('../core/models'), + coreViews = require('../core/views'), chart = require('../core/chart'), utils = require('../core/utils'), windowTmpl = require('./templates/window.html'), - messageTmpl = require('./templates/message.html'), detailsTmpl = require('../modeling/templates/resultsDetails.html'), aoiHeaderTmpl = require('./templates/aoiHeader.html'), tableTmpl = require('./templates/table.html'), @@ -164,21 +164,21 @@ var AnalyzeWindow = Marionette.LayoutView.extend({ }, showAnalyzingMessage: function() { - var messageModel = new models.AnalyzeMessageModel(); + var messageModel = new coreModels.TaskMessageViewModel(); - messageModel.setAnalyzing(); + messageModel.setWorking('Analyzing'); - this.detailsRegion.show(new MessageView({ + this.detailsRegion.show(new coreViews.TaskMessageView({ model: messageModel })); }, showErrorMessage: function() { - var messageModel = new models.AnalyzeMessageModel(); + var messageModel = new coreModels.TaskMessageViewModel(); messageModel.setError(); - this.detailsRegion.show(new MessageView({ + this.detailsRegion.show(new coreViews.TaskMessageView({ model: messageModel })); }, @@ -194,11 +194,6 @@ var AnalyzeWindow = Marionette.LayoutView.extend({ } }); -var MessageView = Marionette.ItemView.extend({ - template: messageTmpl, - className: 'analyze-message-region' -}); - var DetailsView = Marionette.LayoutView.extend({ template: detailsTmpl, regions: { diff --git a/src/mmw/js/src/core/models.js b/src/mmw/js/src/core/models.js index 1c12a1014..cc7f233b7 100644 --- a/src/mmw/js/src/core/models.js +++ b/src/mmw/js/src/core/models.js @@ -213,6 +213,21 @@ var TaskModel = Backbone.Model.extend({ } }); +var TaskMessageViewModel = Backbone.Model.extend({ + message: null, + iconClass: null, + + setError: function() { + this.set('message', 'Error'); + this.set('iconClasses', 'fa fa-exclamation-triangle'); + }, + + setWorking: function(message) { + this.set('message', message); + this.set('iconClasses', 'fa fa-circle-o-notch fa-spin'); + } +}); + // A collection of data points, useful for tables. var LandUseCensusCollection = Backbone.Collection.extend({ comparator: 'nlcd' @@ -272,6 +287,7 @@ var AppStateModel = Backbone.Model.extend({ module.exports = { MapModel: MapModel, TaskModel: TaskModel, + TaskMessageViewModel: TaskMessageViewModel, LandUseCensusCollection: LandUseCensusCollection, SoilCensusCollection: SoilCensusCollection, GeoModel: GeoModel, diff --git a/src/mmw/js/src/analyze/templates/message.html b/src/mmw/js/src/core/templates/message.html similarity index 100% rename from src/mmw/js/src/analyze/templates/message.html rename to src/mmw/js/src/core/templates/message.html diff --git a/src/mmw/js/src/core/views.js b/src/mmw/js/src/core/views.js index c66ff0344..a0bbd9f90 100644 --- a/src/mmw/js/src/core/views.js +++ b/src/mmw/js/src/core/views.js @@ -10,6 +10,7 @@ var L = require('leaflet'), coreUtils = require('./utils'), modificationConfigUtils = require('../modeling/modificationConfigUtils'), headerTmpl = require('./templates/header.html'), + messageTmpl = require('./templates/message.html'), modificationPopupTmpl = require('./templates/modificationPopup.html'), areaOfInterestTmpl = require('./templates/areaOfInterestHeader.html'), modalModels = require('./modals/models'), @@ -841,10 +842,16 @@ var AreaOfInterestView = Marionette.ItemView.extend({ } }); +var TaskMessageView = Marionette.ItemView.extend({ + template: messageTmpl, + className: 'analyze-message-region' +}); + module.exports = { HeaderView: HeaderView, MapView: MapView, RootView: RootView, AreaOfInterestView: AreaOfInterestView, + TaskMessageView: TaskMessageView, ModificationPopupView: ModificationPopupView }; From 93020a4ad6f133c770aaeae9f45e66170d8a4200 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 27 May 2016 11:07:47 -0400 Subject: [PATCH 093/136] Show data loading and error states Using the just extracted TaskMessageViewModel and TaskMessageView, show a spinner or an error depending on the state of the project's fetchGisDataIfNeeded promise. Once that resolves, show a spinner or an error depending on the state of the project's fetchResultsIfNeeded promise. Once that resolves, show the final results. --- src/mmw/js/src/modeling/views.js | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/mmw/js/src/modeling/views.js b/src/mmw/js/src/modeling/views.js index 432af02ef..720863cd1 100644 --- a/src/mmw/js/src/modeling/views.js +++ b/src/mmw/js/src/modeling/views.js @@ -8,6 +8,8 @@ csrf = require('../core/csrf'), models = require('./models'), controls = require('./controls'), + coreModels = require('../core/models'), + coreViews = require('../core/views'), analyzeViews = require('../analyze/views.js'), analyzeModels = require('../analyze/models.js'), modalModels = require('../core/modals/models'), @@ -654,17 +656,35 @@ var ResultsView = Marionette.LayoutView.extend({ initialize: function(options) { var scenarios = this.model.get('scenarios'); - this.listenTo(scenarios, 'change:active', this.showDetailsRegion); + this.listenTo(scenarios, 'change:active', this.onShow); if (options.lock) { this.lock = options.lock; } - this.model.fetchResultsIfNeeded(); + this.fetchGisDataPromise = this.model.fetchGisDataIfNeeded(); + this.fetchResultsPromise = this.model.fetchResultsIfNeeded(); }, onShow: function() { - this.showDetailsRegion(); + var self = this, + tmvModel = new coreModels.TaskMessageViewModel(), + errorHandler = function() { + tmvModel.setError(); + self.modelingRegion.show(new coreViews.TaskMessageView({ model: tmvModel })); + }; + + tmvModel.setWorking('Gathering Data'); + self.modelingRegion.show(new coreViews.TaskMessageView({ model: tmvModel })); + + self.fetchGisDataPromise.done(function() { + tmvModel.setWorking('Calculating Results'); + self.modelingRegion.show(new coreViews.TaskMessageView({ model: tmvModel })); + }).fail(errorHandler); + + self.fetchResultsPromise.done(function() { + self.showDetailsRegion(); + }).fail(errorHandler); }, onRender: function() { From b325cf8546cec25171de54ed749eb39ad0430c9c Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Thu, 26 May 2016 11:52:35 -0400 Subject: [PATCH 094/136] Add UrbTotalArea to Mapshed output This variable is used in the last two BMPs --- src/mmw/apps/modeling/mapshed/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index 3fb90a3b2..ca3d552c1 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -144,6 +144,7 @@ def collect_data(geop_result, geojson): z['SedPhos'] = geop_result['sed_phos'] z['Area'] = [percent * area * HECTARES_PER_SQM for percent in geop_result['landuse_pcts']] + z['UrbAreaTotal'] = sum(z['Area'][NRur:]) z['NormalSys'] = normal_sys(z['Area']) From 955d5fae90b20194b184750e3e2af1e57d66a14e Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Thu, 26 May 2016 11:53:38 -0400 Subject: [PATCH 095/136] Move Leaflet controls to right side This is to make room for the BMP bar --- src/mmw/js/src/core/views.js | 8 ++++---- src/mmw/sass/base/_map.scss | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/mmw/js/src/core/views.js b/src/mmw/js/src/core/views.js index a0bbd9f90..3a7575011 100644 --- a/src/mmw/js/src/core/views.js +++ b/src/mmw/js/src/core/views.js @@ -154,7 +154,7 @@ var HeaderView = Marionette.ItemView.extend({ // Init the locate plugin button and add it to the map. function addLocateMeButton(map, maxZoom, maxAge) { var locateOptions = { - position: 'topleft', + position: 'topright', metric: false, drawCircle: false, showPopup: false, @@ -248,7 +248,7 @@ var MapView = Marionette.ItemView.extend({ } if (options.addZoomControl) { - map.addControl(new L.Control.Zoom({position: 'topleft'})); + map.addControl(new L.Control.Zoom({position: 'topright'})); } var maxGeolocationAge = 60000; @@ -259,7 +259,7 @@ var MapView = Marionette.ItemView.extend({ if (options.addLayerSelector) { var layerOptions = { autoZIndex: false, - position: 'topleft', + position: 'topright', collapsed: false }; @@ -413,7 +413,7 @@ var MapView = Marionette.ItemView.extend({ minZoom: 0}); leafletLayer = new L.TileLayer(tileUrl, layer); if (layer.has_opacity_slider) { - var slider = new OpacityControl({position: 'topleft'}); + var slider = new OpacityControl({position: 'topright'}); slider.setOpacityLayer(leafletLayer); leafletLayer.slider = slider; diff --git a/src/mmw/sass/base/_map.scss b/src/mmw/sass/base/_map.scss index e4430b752..26ed385b5 100644 --- a/src/mmw/sass/base/_map.scss +++ b/src/mmw/sass/base/_map.scss @@ -122,7 +122,7 @@ display: none; position: absolute; top: 120px; - left: 60px; + right: 60px; // Be as tall as the map container, // minus some space for the attribution height: 83%; From 4b8d0eb9df11937c8b2e560c92e3159a28bd0ea6 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Thu, 26 May 2016 11:55:23 -0400 Subject: [PATCH 096/136] Round numbers in manual entry UI --- src/mmw/js/src/modeling/gwlfeModificationConfig.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/mmw/js/src/modeling/gwlfeModificationConfig.js b/src/mmw/js/src/modeling/gwlfeModificationConfig.js index 05b004cff..4896ab2e3 100644 --- a/src/mmw/js/src/modeling/gwlfeModificationConfig.js +++ b/src/mmw/js/src/modeling/gwlfeModificationConfig.js @@ -110,8 +110,11 @@ function makeThresholdValidateFn(thresholdName, thresholdType, userInputName) { cleanUserInput = {}, threshold = dataModel[thresholdName], userInputVal = convertToNumber(userInput[userInputName]), + thresholdNum = Number(threshold).toFixed(2).toLocaleString('en', { + minimumFractionDigits: 2 + }), errorMessage = 'Enter ' + thresholdType + - ' > 0 and <= ' + threshold; + ' > 0 and <= ' + thresholdNum; if (userInputVal) { if (userInputVal > threshold || userInputVal <= 0) { From 631b46ea25f8c35806cda1e8d1b0912f3c667992 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Thu, 26 May 2016 11:56:16 -0400 Subject: [PATCH 097/136] Hide compare button for GWLF-E models --- src/mmw/js/src/modeling/templates/scenariosBar.html | 8 +++++--- src/mmw/js/src/modeling/views.js | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/mmw/js/src/modeling/templates/scenariosBar.html b/src/mmw/js/src/modeling/templates/scenariosBar.html index 9a0e33d9a..1251cafa2 100644 --- a/src/mmw/js/src/modeling/templates/scenariosBar.html +++ b/src/mmw/js/src/modeling/templates/scenariosBar.html @@ -9,6 +9,8 @@
    -
    - Compare -
    +{% if showCompare %} +
    + Compare +
    +{% endif %} diff --git a/src/mmw/js/src/modeling/views.js b/src/mmw/js/src/modeling/views.js index 720863cd1..77fa7f199 100644 --- a/src/mmw/js/src/modeling/views.js +++ b/src/mmw/js/src/modeling/views.js @@ -282,10 +282,12 @@ var ScenariosView = Marionette.LayoutView.extend({ // Check the first scenario in the collection as a proxy for the // entire collection. var scenario = this.collection.first(), + showCompare = App.currentProject.get('model_package') === models.TR55_PACKAGE, compareUrl = this.projectModel.getCompareUrl(); return { editable: isEditable(scenario), + showCompare: showCompare, compareUrl: compareUrl }; }, From 165a690ed8302992efb5f4055c4b93955c75aed1 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Thu, 26 May 2016 11:57:27 -0400 Subject: [PATCH 098/136] Hide 'Conservation Practices' until Mapshed finishes This is because the BMP UI needs access to variables returned by Mashed --- src/mmw/js/src/modeling/views.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/mmw/js/src/modeling/views.js b/src/mmw/js/src/modeling/views.js index 77fa7f199..89d6f8e45 100644 --- a/src/mmw/js/src/modeling/views.js +++ b/src/mmw/js/src/modeling/views.js @@ -54,6 +54,8 @@ var ModelingHeaderView = Marionette.LayoutView.extend({ }, reRender: function() { + var self = this; + this.projectMenuRegion.empty(); this.projectMenuRegion.show(new ProjectMenuView({ model: this.model @@ -66,10 +68,12 @@ var ModelingHeaderView = Marionette.LayoutView.extend({ })); this.toolbarRegion.empty(); - this.toolbarRegion.show(new ToolbarTabContentsView({ - collection: this.model.get('scenarios'), - model_package: this.model.get('model_package') - })); + App.currentProject.fetchGisDataIfNeeded().done(function() { + self.toolbarRegion.show(new ToolbarTabContentsView({ + collection: self.model.get('scenarios'), + model_package: self.model.get('model_package') + })); + }); }, onShow: function() { From 4803cbbbef0dea4a81a1c83fe81e5337fc5e82c1 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Thu, 26 May 2016 12:02:32 -0400 Subject: [PATCH 099/136] Connect BMP UI to the ScenarioModel * Use data from Mapshed instead of mock data * Add GwlfeModificationModel * Add modifications to ScenarioModel instead of printing to console * Merge modifications into postData when doing GWLFE run --- src/mmw/js/src/modeling/controls.js | 32 ++++++++------- src/mmw/js/src/modeling/models.js | 61 +++++++++++++++++++++++------ 2 files changed, 66 insertions(+), 27 deletions(-) diff --git a/src/mmw/js/src/modeling/controls.js b/src/mmw/js/src/modeling/controls.js index dee384a3e..cc0dcc696 100644 --- a/src/mmw/js/src/modeling/controls.js +++ b/src/mmw/js/src/modeling/controls.js @@ -174,6 +174,10 @@ var ManualEntryView = Marionette.CompositeView.extend({ childViewContainer: '#user-input-region', + initialize: function(options) { + this.addModification = options.addModification; + }, + childViewOptions: function() { return { parentModel: this.model @@ -229,19 +233,26 @@ var ManualEntryView = Marionette.CompositeView.extend({ userInput, modConfig.validate, modConfig.computeOutput); this.model.set({ - output: output + output: output, + userInput: userInput }); }, applyModification: function() { this.computeOutput(); - var output = this.model.get('output'); + var modKey = this.model.get('manualMod'), + userInput = this.model.get('userInput'), + output = this.model.get('output'); + if (output && gwlfeConfig.isValid(output.errorMessages)) { this.clearManualMod(); this.closeDropdown(); - // TODO call this.addModification after infrastructure is in place - // to handle Gwlfe scenarios and modifications - console.log(output.output); + + this.addModification(new models.GwlfeModificationModel({ + modKey: modKey, + userInput: userInput, + output: output.output + })); } } }); @@ -386,15 +397,6 @@ var ConservationPracticeView = ModificationsView.extend({ } }); -// TODO use the real Mapshed data model from the server -var mockDataModel = { - n23: 500, - n42: 300, - n42b: 1000, - UrbAreaTotal: 900, - UrbLength: 200 -}; - var GwlfeConservationPracticeView = ModificationsView.extend({ initialize: function(options) { ModificationsView.prototype.initialize.apply(this, [options]); @@ -416,7 +418,7 @@ var GwlfeConservationPracticeView = ModificationsView.extend({ rows: [['urban_buffer_strips', 'urban_streambank_stabilization', 'water_retention', 'infiltration']] } ], - dataModel: mockDataModel, + dataModel: JSON.parse(App.currentProject.get('gis_data')), errorMessages: null, infoMessages: null }); diff --git a/src/mmw/js/src/modeling/models.js b/src/mmw/js/src/modeling/models.js index 9ae05c46c..b2b2cbff6 100644 --- a/src/mmw/js/src/modeling/models.js +++ b/src/mmw/js/src/modeling/models.js @@ -582,6 +582,14 @@ var ModificationsCollection = Backbone.Collection.extend({ model: ModificationModel }); +var GwlfeModificationModel = Backbone.Model.extend({ + defaults: { + modKey: null, + output: null, + userInput: null + } +}); + var ScenarioModel = Backbone.Model.extend({ urlRoot: '/api/modeling/scenarios/', @@ -605,16 +613,18 @@ var ScenarioModel = Backbone.Model.extend({ Backbone.Model.prototype.initialize.apply(this, arguments); this.set('user_id', App.user.get('id')); - // TODO The default modifications might be a function - // of the model_package in the future. - _.defaults(attrs, { - inputs: [ - { - name: 'precipitation', - value: 0.984252 // equal to 2.5 cm. - } - ] - }); + var defaultMods = {}; + if (App.currentProject.get('model_package') === TR55_PACKAGE) { + defaultMods = { + inputs: [ + { + name: 'precipitation', + value: 0.984252 // equal to 2.5 cm. + } + ] + }; + } + _.defaults(attrs, defaultMods); this.set('inputs', new ModificationsCollection(attrs.inputs)); this.set('modifications', new ModificationsCollection(attrs.modifications)); @@ -660,7 +670,20 @@ var ScenarioModel = Backbone.Model.extend({ }, addModification: function(modification) { - this.get('modifications').add(modification); + var modifications = this.get('modifications'); + + // For GWLFE, first remove existing mod with the same key since it + // doesn't make sense to have multiples of the same type of BMP. + if (App.currentProject.get('model_package') === GWLFE) { + var modKey = modification.get('modKey'), + matches = modifications.where({'modKey': modKey}); + + if (matches) { + modifications.remove(matches[0], {silent: true}); + } + } + + modifications.add(modification); }, addOrReplaceInput: function(input) { @@ -803,6 +826,10 @@ var ScenarioModel = Backbone.Model.extend({ } }; + // TODO remove before merging + console.log('gwlfe postData'); + console.log(JSON.parse(gisData.model_input)); + return taskModel.start(taskHelper); }, @@ -845,9 +872,18 @@ var ScenarioModel = Backbone.Model.extend({ }; case GWLFE: + // Merge the values that came back from Mapshed with the values + // in the modifications from the user. + var modifications = self.get('modifications'), + mergedGisData = JSON.parse(project.get('gis_data')); + + modifications.forEach(function(mod) { + Object.assign(mergedGisData, mod.get('output')); + }); + return { inputmod_hash: self.get('inputmod_hash'), - model_input: project.get('gis_data') + model_input: JSON.stringify(mergedGisData) }; } } @@ -1061,6 +1097,7 @@ module.exports = { ProjectModel: ProjectModel, ProjectCollection: ProjectCollection, ModificationModel: ModificationModel, + GwlfeModificationModel: GwlfeModificationModel, ModificationsCollection: ModificationsCollection, ScenarioModel: ScenarioModel, ScenariosCollection: ScenariosCollection From 6491a72341243468683f756cf7974a0550f5babf Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 27 May 2016 17:47:43 -0400 Subject: [PATCH 100/136] Analyze only when needed, return promise Similar to the recent refactoring done for fetchGisDataIfNeeded and fetchResultsIfNeeded. --- src/mmw/js/src/analyze/models.js | 32 ++++++++++++++++++++++++++++++-- src/mmw/js/src/analyze/views.js | 29 +++++++---------------------- 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/mmw/js/src/analyze/models.js b/src/mmw/js/src/analyze/models.js index 55b26d166..cd7c539e4 100644 --- a/src/mmw/js/src/analyze/models.js +++ b/src/mmw/js/src/analyze/models.js @@ -1,7 +1,8 @@ "use strict"; -var Backbone = require('../../shim/backbone'), +var $ = require('jquery'), _ = require('lodash'), + Backbone = require('../../shim/backbone'), coreModels = require('../core/models'); var LayerModel = Backbone.Model.extend({}); @@ -20,10 +21,37 @@ var LayerCategoryCollection = Backbone.Collection.extend({ var AnalyzeTaskModel = coreModels.TaskModel.extend({ defaults: _.extend( { + area_of_interest: null, taskName: 'analyze', taskType: 'modeling' }, coreModels.TaskModel.prototype.defaults - ) + ), + + /** + * Returns a promise that completes when Analysis has been fetched. If + * fetching is not required, returns an immediatley resolved promise. + */ + fetchAnalysisIfNeeded: function() { + var self = this, + aoi = self.get('area_of_interest'), + result = self.get('result'); + + if (aoi && !result && self.fetchAnalysisPromise === undefined) { + var promises = self.start({ + postData: { + 'area_of_interest': JSON.stringify(aoi) + } + }); + self.fetchAnalysisPromise = $.when(promises.startPromise, + promises.pollingPromise); + self.fetchAnalysisPromise + .always(function() { + delete self.fetchAnalysisPromise; + }); + } + + return self.fetchAnalysisPromise || $.when(); + } }); module.exports = { diff --git a/src/mmw/js/src/analyze/views.js b/src/mmw/js/src/analyze/views.js index 34d7acf31..a3398347c 100644 --- a/src/mmw/js/src/analyze/views.js +++ b/src/mmw/js/src/analyze/views.js @@ -139,28 +139,13 @@ var AnalyzeWindow = Marionette.LayoutView.extend({ var self = this; - if (!this.model.get('result')) { - var aoi = JSON.stringify(this.model.get('area_of_interest')), - taskHelper = { - pollSuccess: function() { - self.showDetailsRegion(); - }, - - pollFailure: function() { - self.showErrorMessage(); - }, - - startFailure: function() { - self.showErrorMessage(); - }, - - postData: {'area_of_interest': aoi} - }; - - this.model.start(taskHelper); - } else { - self.showDetailsRegion(); - } + this.model.fetchAnalysisIfNeeded() + .done(function() { + self.showDetailsRegion(); + }) + .fail(function() { + self.showErrorMessage(); + }); }, showAnalyzingMessage: function() { From df879612e292b7c94e35d05a86af4af1b722287e Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 27 May 2016 17:56:21 -0400 Subject: [PATCH 101/136] Don't wait for Results to Analyze Previously, we would wait for the data gathering and result calculating steps to finish before showing the Analyze tab. This was an extraneous wait, since the analysis is already available. Now, we show the analysis results either immediately (when coming from the Analyze tab) or in parallel (when reloading a scenario) to the results. --- src/mmw/js/src/modeling/views.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/mmw/js/src/modeling/views.js b/src/mmw/js/src/modeling/views.js index 720863cd1..09b16c7e1 100644 --- a/src/mmw/js/src/modeling/views.js +++ b/src/mmw/js/src/modeling/views.js @@ -668,12 +668,18 @@ var ResultsView = Marionette.LayoutView.extend({ onShow: function() { var self = this, + aoi = JSON.stringify(App.map.get('areaOfInterest')), + analyzeModel = App.analyzeModel || createTaskModel(aoi), tmvModel = new coreModels.TaskMessageViewModel(), errorHandler = function() { tmvModel.setError(); self.modelingRegion.show(new coreViews.TaskMessageView({ model: tmvModel })); }; + this.analyzeRegion.show(new analyzeViews.AnalyzeWindow({ + model: analyzeModel + })); + tmvModel.setWorking('Gathering Data'); self.modelingRegion.show(new coreViews.TaskMessageView({ model: tmvModel })); @@ -697,14 +703,7 @@ var ResultsView = Marionette.LayoutView.extend({ showDetailsRegion: function() { var scenarios = this.model.get('scenarios'), - scenario = scenarios.getActiveScenario(), - aoi = JSON.stringify(App.map.get('areaOfInterest')), - analyzeModel = App.analyzeModel !== undefined ? App.analyzeModel : - createTaskModel(aoi); - - this.analyzeRegion.show(new analyzeViews.AnalyzeWindow({ - model: analyzeModel - })); + scenario = scenarios.getActiveScenario(); if (scenario) { this.modelingRegion.show(new ResultsDetailsView({ From 07432234a9869a00a03c634c45279944d54886e3 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 27 May 2016 18:09:12 -0400 Subject: [PATCH 102/136] Remember Analyze results between views Now we will remember Analyze results not only when going from Analyze to Model, but also from Model to Analyze. We clear any saved results when leaving Draw, so that if users go all the ways back to Draw and choose a different Area of Interest, that actually triggers a legitimate re-analysis. --- src/mmw/js/src/analyze/controllers.js | 7 +++++-- src/mmw/js/src/draw/controllers.js | 1 + src/mmw/js/src/modeling/views.js | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/mmw/js/src/analyze/controllers.js b/src/mmw/js/src/analyze/controllers.js index 27c016d60..56d6eeb69 100644 --- a/src/mmw/js/src/analyze/controllers.js +++ b/src/mmw/js/src/analyze/controllers.js @@ -49,12 +49,15 @@ var AnalyzeController = { analyze: function() { var aoi = JSON.stringify(App.map.get('areaOfInterest')), - analyzeModel = createTaskModel(aoi), + analyzeModel = App.analyzeModel || createTaskModel(aoi), analyzeResults = new views.ResultsView({ model: analyzeModel }); - App.analyzeModel = analyzeModel; + if (!App.analyzeModel) { + App.analyzeModel = analyzeModel; + } + App.state.set('current_page_title', 'Geospatial Analysis'); App.rootView.sidebarRegion.show(analyzeResults); diff --git a/src/mmw/js/src/draw/controllers.js b/src/mmw/js/src/draw/controllers.js index 6ed395cbb..7558fe219 100644 --- a/src/mmw/js/src/draw/controllers.js +++ b/src/mmw/js/src/draw/controllers.js @@ -53,6 +53,7 @@ var DrawController = { }, drawCleanUp: function() { + delete App.analyzeModel; App.rootView.geocodeSearchRegion.empty(); App.rootView.drawToolsRegion.empty(); App.rootView.footerRegion.empty(); diff --git a/src/mmw/js/src/modeling/views.js b/src/mmw/js/src/modeling/views.js index 09b16c7e1..eec992e61 100644 --- a/src/mmw/js/src/modeling/views.js +++ b/src/mmw/js/src/modeling/views.js @@ -676,6 +676,10 @@ var ResultsView = Marionette.LayoutView.extend({ self.modelingRegion.show(new coreViews.TaskMessageView({ model: tmvModel })); }; + if (!App.analyzeModel) { + App.analyzeModel = analyzeModel; + } + this.analyzeRegion.show(new analyzeViews.AnalyzeWindow({ model: analyzeModel })); From f75f4c360e040a1f91205555826999c1cda0f5c7 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Thu, 26 May 2016 12:08:59 -0400 Subject: [PATCH 103/136] Add BMP sidebar for GWLF-E models * Split the ScenarioToolbarTabContent view into Tr55ScenarioToolbarTabContentView (with modifications dropdown) and GwlfeScenarioToolbarTabContentView (with BMP sidebar) * Add sidebar and popup that shows when hovering over thumbnails in sidebar --- src/mmw/js/src/compare/views.js | 4 +- .../templates/controls/modDropdown.html | 2 +- .../gwlfeScenarioToolbarTabContent.html | 48 +++++ ...tml => tr55ScenarioToolbarTabContent.html} | 0 src/mmw/js/src/modeling/tests.js | 4 +- src/mmw/js/src/modeling/views.js | 182 ++++++++++++++---- src/mmw/sass/base/_header.scss | 48 +++++ src/mmw/sass/base/_map.scss | 7 +- src/mmw/sass/components/_dropdowns.scss | 6 +- src/mmw/sass/components/_tabs.scss | 2 - 10 files changed, 259 insertions(+), 44 deletions(-) create mode 100644 src/mmw/js/src/modeling/templates/gwlfeScenarioToolbarTabContent.html rename src/mmw/js/src/modeling/templates/{scenarioToolbarTabContent.html => tr55ScenarioToolbarTabContent.html} (100%) diff --git a/src/mmw/js/src/compare/views.js b/src/mmw/js/src/compare/views.js index b2279eb05..576462b4d 100644 --- a/src/mmw/js/src/compare/views.js +++ b/src/mmw/js/src/compare/views.js @@ -246,7 +246,9 @@ var CompareModelingView = Marionette.LayoutView.extend({ {compareMode: true} ); - this.controlsRegion.show(new modelingViews.ToolbarTabContentView({ + // TODO this needs to be generalized if we want the compare view + // to work with GWLF-E + this.controlsRegion.show(new modelingViews.Tr55ToolbarTabContentView({ model: this.model, collection: controls, compareMode: true diff --git a/src/mmw/js/src/modeling/templates/controls/modDropdown.html b/src/mmw/js/src/modeling/templates/controls/modDropdown.html index 669670a76..9256bbc24 100644 --- a/src/mmw/js/src/modeling/templates/controls/modDropdown.html +++ b/src/mmw/js/src/modeling/templates/controls/modDropdown.html @@ -1,4 +1,4 @@ -
    +
    diff --git a/src/mmw/js/src/modeling/templates/gwlfeScenarioToolbarTabContent.html b/src/mmw/js/src/modeling/templates/gwlfeScenarioToolbarTabContent.html new file mode 100644 index 000000000..95a477b3e --- /dev/null +++ b/src/mmw/js/src/modeling/templates/gwlfeScenarioToolbarTabContent.html @@ -0,0 +1,48 @@ +
    + +{% if not compareMode and modifications.length > 0 %} +
    +
    + +
    + {% for mod in modifications %} +
    + + + +
    + {% endfor %} +
    + + {% if activeMod %} +
    +
    +
    +
    + {{ activeMod.modKey|modShortName }} +
    + + + + + + {% for inputName, inputVal in activeMod.userInput %} + + + + + {% endfor %} + +
    {{ displayNames[inputName] }}{{ inputVal|round(0)|toLocaleString(0) }}
    + +
    +
    + {% endif %} +{% endif %} diff --git a/src/mmw/js/src/modeling/templates/scenarioToolbarTabContent.html b/src/mmw/js/src/modeling/templates/tr55ScenarioToolbarTabContent.html similarity index 100% rename from src/mmw/js/src/modeling/templates/scenarioToolbarTabContent.html rename to src/mmw/js/src/modeling/templates/tr55ScenarioToolbarTabContent.html diff --git a/src/mmw/js/src/modeling/tests.js b/src/mmw/js/src/modeling/tests.js index ca63ac6f1..48ec5c542 100644 --- a/src/mmw/js/src/modeling/tests.js +++ b/src/mmw/js/src/modeling/tests.js @@ -156,7 +156,7 @@ describe('Modeling', function() { }); }); - describe('ToolbarTabContentView', function() { + describe('Tr55ToolbarTabContentView', function() { beforeEach(function() { this.userId = 1; App.user.id = this.userId; @@ -164,7 +164,7 @@ describe('Modeling', function() { name: 'New Scenario', modifications: [] }); - this.view = new views.ToolbarTabContentView({ + this.view = new views.Tr55ToolbarTabContentView({ model: this.model, collection: models.getControlsForModelPackage('tr-55') }); diff --git a/src/mmw/js/src/modeling/views.js b/src/mmw/js/src/modeling/views.js index 89d6f8e45..b44c00efc 100644 --- a/src/mmw/js/src/modeling/views.js +++ b/src/mmw/js/src/modeling/views.js @@ -1,6 +1,6 @@ "use strict"; - var _ = require('lodash'), +var _ = require('lodash'), $ = require('jquery'), Marionette = require('../../shim/backbone.marionette'), App = require('../app'), @@ -10,6 +10,7 @@ controls = require('./controls'), coreModels = require('../core/models'), coreViews = require('../core/views'), + gwlfeConfig = require('./gwlfeModificationConfig'), analyzeViews = require('../analyze/views.js'), analyzeModels = require('../analyze/models.js'), modalModels = require('../core/modals/models'), @@ -25,7 +26,8 @@ scenarioMenuTmpl = require('./templates/scenarioMenu.html'), scenarioMenuItemTmpl = require('./templates/scenarioMenuItem.html'), projectMenuTmpl = require('./templates/projectMenu.html'), - scenarioToolbarTabContentTmpl = require('./templates/scenarioToolbarTabContent.html'), + tr55ScenarioToolbarTabContentTmpl = require('./templates/tr55ScenarioToolbarTabContent.html'), + gwlfeScenarioToolbarTabContentTmpl = require('./templates/gwlfeScenarioToolbarTabContent.html'), tr55RunoffViews = require('./tr55/runoff/views.js'), tr55QualityViews = require('./tr55/quality/views.js'), gwlfeRunoffViews = require('./gwlfe/runoff/views.js'), @@ -527,8 +529,8 @@ var ScenarioDropDownMenuView = Marionette.CompositeView.extend({ // The toolbar that contains the modification and input tools // for a scenario. var ToolbarTabContentView = Marionette.CompositeView.extend({ + template: tr55ScenarioToolbarTabContentTmpl, model: models.ScenarioModel, - template: scenarioToolbarTabContentTmpl, collection: models.ModelPackageControlsCollection, childViewContainer: '.controls', @@ -558,32 +560,12 @@ var ToolbarTabContentView = Marionette.CompositeView.extend({ 'change:active': 'render' }, - ui: { - deleteModification: '[data-delete]' - }, - - events: { - 'click @ui.deleteModification': 'deleteModification' - }, - initialize: function(options) { this.compareMode = options.compareMode; var modificationsColl = this.model.get('modifications'); this.listenTo(modificationsColl, 'add remove reset', this.render); }, - templateHelpers: function() { - var shapes = this.model.get('modifications'), - groupedShapes = shapes.groupBy('name'); - - return { - compareMode: this.compareMode, - shapes: shapes, - groupedShapes: groupedShapes, - editable: isEditable(this.model) - }; - }, - // Only display modification controls if scenario is editable. // Input controls should always be shown. filter: function(modelPackageControl) { @@ -602,6 +584,47 @@ var ToolbarTabContentView = Marionette.CompositeView.extend({ } }, + getChildView: function(modelPackageControl) { + var controlName = modelPackageControl.get('name'); + + return controls.getControlView(controlName); + }, + + // For a given ModelPackageControlModel (ex. Precipitation), return + // the instance of that control model from this scenario inputs. + getInputControlModel: function(modelPackageControl) { + return this.model.get('inputs').findWhere({ + name: modelPackageControl.get('name') + }); + } +}); + +// The toolbar that contains the modification and input tools +// for a scenario. +var Tr55ToolbarTabContentView = ToolbarTabContentView.extend({ + template: tr55ScenarioToolbarTabContentTmpl, + model: models.ScenarioModel, + + ui: { + deleteModification: '[data-delete]' + }, + + events: { + 'click @ui.deleteModification': 'deleteModification' + }, + + templateHelpers: function() { + var shapes = this.model.get('modifications'), + groupedShapes = shapes.groupBy('name'); + + return { + compareMode: this.compareMode, + shapes: shapes, + groupedShapes: groupedShapes, + editable: isEditable(this.model) + }; + }, + deleteModification: function(e) { var $el = $(e.currentTarget), cid = $el.data('delete'), @@ -609,20 +632,106 @@ var ToolbarTabContentView = Marionette.CompositeView.extend({ modification = modificationsColl.get(cid); modificationsColl.remove(modification); + } +}); + +// The toolbar that contains the modification and input tools +// for a scenario. +var GwlfeToolbarTabContentView = ToolbarTabContentView.extend({ + template: gwlfeScenarioToolbarTabContentTmpl, + + model: models.ScenarioModel, + + ui: { + thumb: '#gwlfe-modifications-bar .thumb', + deleteButton: '.delete-button', + closeButton: 'button.close' }, - getChildView: function(modelPackageControl) { - var controlName = modelPackageControl.get('name'); + events: { + 'click @ui.thumb': 'onThumbClick', + 'click @ui.deleteButton': 'deleteModification', + 'click @ui.closeButton': 'closePopup' + }, - return controls.getControlView(controlName); + modelEvents: _.defaults({ + 'change:activeModKey': 'render', + 'change:modifications': 'render' + }, ToolbarTabContentView.prototype.modelEvents), + + initialize: function(options) { + ToolbarTabContentView.prototype.initialize.apply(this, [options]); + + var self = this; + function closePopupOnOutsideClick(e) { + var isTargetOutside = $(e.target).parents('#gwlfe-modifications-popup').length === 0; + if (self.model.get('activeModKey') && isTargetOutside) { + self.closePopup(); + } + } + + $(document).on('mouseup', function(e) { + closePopupOnOutsideClick(e); + }); }, - // For a given ModelPackageControlModel (ex. Precipitation), return - // the instance of that control model from this scenario inputs. - getInputControlModel: function(modelPackageControl) { - return this.model.get('inputs').findWhere({ - name: modelPackageControl.get('name') + setupTooltips: function() { + var options = this.model.get('activeModKey') ? 'destroy' : null; + $('#gwlfe-modifications-bar .thumb').tooltip(options); + $('#gwlfe-modifications-bar i').tooltip(options); + }, + + onRender: function() { + ToolbarTabContentView.prototype.onRender.apply(this); + this.setupTooltips(); + }, + + onShow: function() { + this.setupTooltips(); + }, + + closePopup: function() { + this.model.set('activeModKey', null); + }, + + getActiveMod: function() { + var activeModKey = this.model.get('activeModKey'), + modifications = this.model.get('modifications'); + return activeModKey && modifications.where({modKey: activeModKey})[0]; + }, + + deleteModification: function() { + var activeMod = this.getActiveMod(), + modifications = this.model.get('modifications'); + + if (activeMod) { + modifications.remove(activeMod); + this.closePopup(); + } + }, + + onThumbClick: function(e) { + var modKey = $(e.currentTarget).data('value'), + thumbOffset = $(e.target).offset(); + + this.model.set('activeModKey', modKey); + + $('#gwlfe-modifications-popup').offset({ + top: thumbOffset.top - 50 }); + }, + + templateHelpers: function() { + var activeMod = this.getActiveMod(), + modifications = this.model.get('modifications').toJSON(); + + activeMod = activeMod ? activeMod.toJSON() : null; + + return { + modifications: modifications, + activeMod: activeMod, + displayNames: gwlfeConfig.displayNames + }; } }); @@ -631,7 +740,14 @@ var ToolbarTabContentView = Marionette.CompositeView.extend({ var ToolbarTabContentsView = Marionette.CollectionView.extend({ collection: models.ScenariosCollection, className: 'tab-content', - childView: ToolbarTabContentView, + getChildView: function() { + var isGwlfe = App.currentProject.get('model_package') === 'gwlfe'; + if (isGwlfe) { + return GwlfeToolbarTabContentView; + } else { + return Tr55ToolbarTabContentView; + } + }, childViewOptions: function(model) { var controls = models.getControlsForModelPackage( this.options.model_package, @@ -923,7 +1039,7 @@ module.exports = { ScenariosView: ScenariosView, ScenarioTabPanelsView: ScenarioTabPanelsView, ScenarioDropDownMenuView: ScenarioDropDownMenuView, - ToolbarTabContentView: ToolbarTabContentView, + Tr55ToolbarTabContentView: Tr55ToolbarTabContentView, ProjectMenuView: ProjectMenuView, getResultView: getResultView }; diff --git a/src/mmw/sass/base/_header.scss b/src/mmw/sass/base/_header.scss index c13e91bec..a40ea1361 100644 --- a/src/mmw/sass/base/_header.scss +++ b/src/mmw/sass/base/_header.scss @@ -189,6 +189,54 @@ header { background-color: #D0D0D0; height: $toolbar-height; + #gwlfe-modifications-bar, #gwlfe-modifications-popup { + padding: 5px; + position: absolute; + display: block; + + .thumb { + cursor: pointer; + border: solid 2px transparent; + height: 40px; + width: 40px; + } + + .thumb:hover { + border: solid 2px $black; + } + } + + #gwlfe-modifications-bar { + top: 60px; + left: 0; + background-color: #D0D0D0; + z-index: 1001; + + .title-icon { + text-align: center; + font-size: 15px; + } + } + + #gwlfe-modifications-popup { + left: 46px; + z-index: 1003; + + .popover-content { + padding: 12px; + + .table { + margin-bottom: 10px; + } + } + + .close { + position: absolute; + top: 5px; + right: 5px; + } + } + #modification-btn-wrapper { float: right; } diff --git a/src/mmw/sass/base/_map.scss b/src/mmw/sass/base/_map.scss index 26ed385b5..aa124bff2 100644 --- a/src/mmw/sass/base/_map.scss +++ b/src/mmw/sass/base/_map.scss @@ -116,13 +116,18 @@ display: block; } + &.opacity_slider_control { + position: absolute; + top: 134px; + } + &.leaflet-control-layers { min-width: 246px; margin-right: 50px; display: none; position: absolute; top: 120px; - right: 60px; + right: 15px; // Be as tall as the map container, // minus some space for the attribution height: 83%; diff --git a/src/mmw/sass/components/_dropdowns.scss b/src/mmw/sass/components/_dropdowns.scss index b63db8431..8d422c37e 100644 --- a/src/mmw/sass/components/_dropdowns.scss +++ b/src/mmw/sass/components/_dropdowns.scss @@ -162,8 +162,6 @@ } } -//Modification Dropdown -#modifications { - max-height: 400px; - overflow-y: auto; +.modification-dropdown { + z-index: 1002; } diff --git a/src/mmw/sass/components/_tabs.scss b/src/mmw/sass/components/_tabs.scss index 726f2e0cd..3dd5d20fc 100644 --- a/src/mmw/sass/components/_tabs.scss +++ b/src/mmw/sass/components/_tabs.scss @@ -156,6 +156,4 @@ } } } - - } From 9d2151b08460eddc9810399e85f2f7ed222d6060 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Thu, 26 May 2016 16:53:59 -0400 Subject: [PATCH 104/136] Disable thumbnails for invalid BMPs BMPs can be invalid for an AOI when areas and lengths to modify are zero. * Add validateDataModel field to GWLF-E modification configs * For each modKey, compute whether it is valid * Disable thumbnails for invalid BMPs (gray out and disable click) --- src/mmw/js/src/modeling/controls.js | 28 ++++++++++++++----- .../src/modeling/gwlfeModificationConfig.js | 17 +++++++++++ .../templates/controls/thumbSelect.html | 6 ++-- src/mmw/sass/components/_dropdowns.scss | 6 +++- 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/src/mmw/js/src/modeling/controls.js b/src/mmw/js/src/modeling/controls.js index cc0dcc696..8387887fb 100644 --- a/src/mmw/js/src/modeling/controls.js +++ b/src/mmw/js/src/modeling/controls.js @@ -44,7 +44,19 @@ var ThumbSelectView = Marionette.ItemView.extend({ template: thumbSelectTmpl, initialize: function(options) { - this.model.set('activeMod', null); + var modKeys = _.flatten(_.pluck(this.model.get('modRowGroups'), 'rows'), true), + dataModel = this.model.get('dataModel'), + manualMode = this.model.get('manualMode'), + modEnabled = {}; + + _.forEach(modKeys, function(modKey) { + modEnabled[modKey] = manualMode ? gwlfeConfig.configs[modKey].validateDataModel(dataModel) : true; + }); + + this.model.set({ + activeMod: null, + modEnabled: modEnabled + }); this.addModification = options.addModification; }, @@ -63,8 +75,8 @@ var ThumbSelectView = Marionette.ItemView.extend({ }, onThumbHover: function(e) { - var value = $(e.currentTarget).data('value'); - this.model.set('activeMod', value); + var modKey = $(e.currentTarget).data('value'); + this.model.set('activeMod', modKey); }, onThumbClick: function(e) { @@ -72,10 +84,12 @@ var ThumbSelectView = Marionette.ItemView.extend({ controlName = this.model.get('controlName'), controlValue = $el.data('value'); - if (this.model.get('manualMode')) { - this.startManual(controlName, controlValue); - } else { - this.startDrawing(controlName, controlValue); + if (this.model.get('modEnabled')[controlValue]) { + if (this.model.get('manualMode')) { + this.startManual(controlName, controlValue); + } else { + this.startDrawing(controlName, controlValue); + } } }, diff --git a/src/mmw/js/src/modeling/gwlfeModificationConfig.js b/src/mmw/js/src/modeling/gwlfeModificationConfig.js index 4896ab2e3..ca76d8764 100644 --- a/src/mmw/js/src/modeling/gwlfeModificationConfig.js +++ b/src/mmw/js/src/modeling/gwlfeModificationConfig.js @@ -169,6 +169,16 @@ function makeComputeOutputFn(dataModelName, inputName, getOutput) { }; } +// Returns a function that checks that each variable in dataModelNames +// is positive. Useful for checking if a BMP is applicable to an AOI. +function makeValidateDataModelFn(dataModelNames) { + return function(dataModel) { + return _.every(dataModelNames, function(dataModelName) { + return dataModel[dataModelName] > 0; + }); + }; +} + function makeAgBmpConfig(outputName) { function getOutput(inputVal, fractionVal) { return fromPairs([ @@ -178,6 +188,7 @@ function makeAgBmpConfig(outputName) { return { dataModelNames: [n23Name], + validateDataModel: makeValidateDataModelFn([n23Name]), userInputNames: [areaToModifyName], validate: makeThresholdValidateFn(n23Name, AREA, areaToModifyName), computeOutput: makeComputeOutputFn(n23Name, areaToModifyName, getOutput) @@ -187,6 +198,7 @@ function makeAgBmpConfig(outputName) { function makeAeuBmpConfig(outputName) { return { dataModelNames: [], + validateDataModel: makeValidateDataModelFn([]), userInputNames: [percentAeuToModifyName], validate: makePercentValidateFn(percentAeuToModifyName), computeOutput: function(dataModel, cleanUserInput) { @@ -213,6 +225,7 @@ function makeRuralStreamsBmpConfig(outputName) { return { dataModelNames: [n42Name, n42bName], + validateDataModel: makeValidateDataModelFn([n42Name]), userInputNames: [lengthToModifyInAgName], validate: makeThresholdValidateFn(n42Name, LENGTH, lengthToModifyInAgName), computeOutput: makeComputeOutputFn(n42Name, lengthToModifyInAgName, getOutput) @@ -222,6 +235,7 @@ function makeRuralStreamsBmpConfig(outputName) { function makeUrbanStreamsBmpConfig(getOutput) { return { dataModelNames: [UrbLengthName], + validateDataModel: makeValidateDataModelFn([UrbLengthName]), userInputNames: [lengthToModifyName], validate: makeThresholdValidateFn(UrbLengthName, LENGTH, lengthToModifyName), computeOutput: makeComputeOutputFn(UrbLengthName, lengthToModifyName, getOutput) @@ -231,6 +245,7 @@ function makeUrbanStreamsBmpConfig(getOutput) { function makeUrbanAreaBmpConfig(getOutput) { return { dataModelNames: [UrbAreaTotalName], + validateDataModel: makeValidateDataModelFn([UrbAreaTotalName]), userInputNames: [areaToModifyName], validate: makeThresholdValidateFn(UrbAreaTotalName, AREA, areaToModifyName), computeOutput: makeComputeOutputFn(UrbAreaTotalName, areaToModifyName, getOutput) @@ -242,6 +257,8 @@ function makeUrbanAreaBmpConfig(getOutput) { to generate the UI for that modification. - dataModelNames is a list of variable names in the Mapshed data model (received from the backend) which should be displayed to the user. + - validateDataModel is a function which validates the values of the + dataModel which can be used to tell if the BMP is valid for an AOI - userInputNames is a list of non-Mapshed variables used to reference values the user enters into the UI. currently all the BMPs only reference a single user input variable, so using a list is overkill, but it could be useful in diff --git a/src/mmw/js/src/modeling/templates/controls/thumbSelect.html b/src/mmw/js/src/modeling/templates/controls/thumbSelect.html index 776517cf7..341b97c6c 100644 --- a/src/mmw/js/src/modeling/templates/controls/thumbSelect.html +++ b/src/mmw/js/src/modeling/templates/controls/thumbSelect.html @@ -3,17 +3,17 @@ {% for modRowGroup in modRowGroups %} {% if modRowGroup.name %}
    {{ modRowGroup.name }}
    - {% endif %} + {% endif %} {% for row in modRowGroup.rows %}
      {% for modKey in row %}
    • -
      +
      - +
      • {% endfor %}
    diff --git a/src/mmw/sass/components/_dropdowns.scss b/src/mmw/sass/components/_dropdowns.scss index 8d422c37e..613227b13 100644 --- a/src/mmw/sass/components/_dropdowns.scss +++ b/src/mmw/sass/components/_dropdowns.scss @@ -147,12 +147,16 @@ margin-bottom: 0.2rem; display: block; - &:hover { + &:not(.disabled):hover { border-color: $ui-primary; cursor: pointer; } } } + + .disabled { + opacity: 0.4; + } } } From 1dc28b9c6760082f4908281bdf15f4e2f9abb2b8 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Fri, 27 May 2016 15:12:14 -0400 Subject: [PATCH 105/136] Improve info message in manual entry UI Instead of just showing '3%' as the info message, we now show '3% of total urban area' or the like. --- .../js/src/modeling/gwlfeModificationConfig.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/mmw/js/src/modeling/gwlfeModificationConfig.js b/src/mmw/js/src/modeling/gwlfeModificationConfig.js index ca76d8764..f99fb4ec5 100644 --- a/src/mmw/js/src/modeling/gwlfeModificationConfig.js +++ b/src/mmw/js/src/modeling/gwlfeModificationConfig.js @@ -64,8 +64,17 @@ var displayNames = fromPairs([ [UrbLengthName, 'Length of streams in non-ag areas (km)'] ]); -function getPercentStr(fraction) { - return Math.round(fraction * 100) + '%'; +var shortDisplayNames = fromPairs([ + [n23Name, 'area of row crops'], + [n42Name, 'length of streams in ag areas'], + [n42bName, 'length of streams in watershed'], + [UrbAreaTotalName, 'total urban area'], + [UrbLengthName, 'length of streams in non-ag areas'] +]); + +function getPercentStr(fraction, dataModelName) { + var dataModelDisplayName = shortDisplayNames[dataModelName] || displayNames[dataModelName]; + return Math.round(fraction * 100) + '% of ' + dataModelDisplayName; } function convertToNumber(x) { @@ -161,7 +170,7 @@ function makeComputeOutputFn(dataModelName, inputName, getOutput) { if (inputVal) { fractionVal = inputVal / dataModelVal; - infoMessages[inputName] = getPercentStr(fractionVal); + infoMessages[inputName] = getPercentStr(fractionVal, dataModelName); output = getOutput(inputVal, fractionVal); } From 4952bc766edb7f926a64f386b53fd4ed5d7eadcc Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 31 May 2016 11:26:37 -0400 Subject: [PATCH 106/136] refactor: Singleton AnalyzeTaskModel per App Previously there was code duplication in an effort to maintain a single copy of AnalyzeTaskModel for the entire App. Now we collect all that logic into a 'static' helper method in AnalyzeTaskModel and reuse it everywhere. --- src/mmw/js/src/analyze/controllers.js | 21 +++------------------ src/mmw/js/src/analyze/models.js | 18 ++++++++++++++++++ src/mmw/js/src/modeling/views.js | 18 ++---------------- 3 files changed, 23 insertions(+), 34 deletions(-) diff --git a/src/mmw/js/src/analyze/controllers.js b/src/mmw/js/src/analyze/controllers.js index 56d6eeb69..6a6a30893 100644 --- a/src/mmw/js/src/analyze/controllers.js +++ b/src/mmw/js/src/analyze/controllers.js @@ -1,7 +1,6 @@ "use strict"; -var _ = require('underscore'), - App = require('../app'), +var App = require('../app'), router = require('../router').router, views = require('./views'), models = require('./models'), @@ -48,16 +47,12 @@ var AnalyzeController = { }, analyze: function() { - var aoi = JSON.stringify(App.map.get('areaOfInterest')), - analyzeModel = App.analyzeModel || createTaskModel(aoi), + var aoi = App.map.get('areaOfInterest'), + analyzeModel = models.AnalyzeTaskModel.getSingleton(App, aoi), analyzeResults = new views.ResultsView({ model: analyzeModel }); - if (!App.analyzeModel) { - App.analyzeModel = analyzeModel; - } - App.state.set('current_page_title', 'Geospatial Analysis'); App.rootView.sidebarRegion.show(analyzeResults); @@ -68,16 +63,6 @@ var AnalyzeController = { } }; -// Pass in the serialized Area of Interest for -// caching purposes (_.memoize returns the same -// results for any object), and deserialize -// the AoI for use on the model. -var createTaskModel = _.memoize(function(aoi) { - return new models.AnalyzeTaskModel({ - area_of_interest: JSON.parse(aoi) - }); -}); - module.exports = { AnalyzeController: AnalyzeController }; diff --git a/src/mmw/js/src/analyze/models.js b/src/mmw/js/src/analyze/models.js index cd7c539e4..05bc936ce 100644 --- a/src/mmw/js/src/analyze/models.js +++ b/src/mmw/js/src/analyze/models.js @@ -54,6 +54,24 @@ var AnalyzeTaskModel = coreModels.TaskModel.extend({ } }); +AnalyzeTaskModel.getSingleton = function(App, aoi) { + if (!App.analyzeModel) { + App.analyzeModel = createTaskModel(JSON.stringify(aoi)); + } + + return App.analyzeModel; +}; + +// Pass in the serialized Area of Interest for +// caching purposes (_.memoize returns the same +// results for any object), and deserialize +// the AoI for use on the model. +var createTaskModel = _.memoize(function(aoi) { + return new AnalyzeTaskModel({ + area_of_interest: JSON.parse(aoi) + }); +}); + module.exports = { AnalyzeTaskModel: AnalyzeTaskModel, LayerModel: LayerModel, diff --git a/src/mmw/js/src/modeling/views.js b/src/mmw/js/src/modeling/views.js index eec992e61..a15d86e9e 100644 --- a/src/mmw/js/src/modeling/views.js +++ b/src/mmw/js/src/modeling/views.js @@ -668,18 +668,14 @@ var ResultsView = Marionette.LayoutView.extend({ onShow: function() { var self = this, - aoi = JSON.stringify(App.map.get('areaOfInterest')), - analyzeModel = App.analyzeModel || createTaskModel(aoi), + aoi = App.map.get('areaOfInterest'), + analyzeModel = analyzeModels.AnalyzeTaskModel.getSingleton(App, aoi), tmvModel = new coreModels.TaskMessageViewModel(), errorHandler = function() { tmvModel.setError(); self.modelingRegion.show(new coreViews.TaskMessageView({ model: tmvModel })); }; - if (!App.analyzeModel) { - App.analyzeModel = analyzeModel; - } - this.analyzeRegion.show(new analyzeViews.AnalyzeWindow({ model: analyzeModel })); @@ -904,16 +900,6 @@ function getResultView(modelPackage, resultName) { } } -// Pass in the serialized Area of Interest for -// caching purposes (_.memoize returns the same -// results for any object), and deserialize -// the AoI for use on the model. -var createTaskModel = _.memoize(function(aoi) { - return new analyzeModels.AnalyzeTaskModel({ - area_of_interest: JSON.parse(aoi) - }); -}); - module.exports = { ResultsView: ResultsView, ModelingHeaderView: ModelingHeaderView, From 0b8e9b2f3befe72f1d9a6e9eda7110fc3326fb59 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Tue, 31 May 2016 12:03:54 -0400 Subject: [PATCH 107/136] Convert UrbLength to km Previously, this variable was in units of meters, but the label said km. --- src/mmw/js/src/modeling/controls.js | 2 +- src/mmw/js/src/modeling/gwlfeModificationConfig.js | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/mmw/js/src/modeling/controls.js b/src/mmw/js/src/modeling/controls.js index 8387887fb..d13c2664a 100644 --- a/src/mmw/js/src/modeling/controls.js +++ b/src/mmw/js/src/modeling/controls.js @@ -432,7 +432,7 @@ var GwlfeConservationPracticeView = ModificationsView.extend({ rows: [['urban_buffer_strips', 'urban_streambank_stabilization', 'water_retention', 'infiltration']] } ], - dataModel: JSON.parse(App.currentProject.get('gis_data')), + dataModel: gwlfeConfig.cleanDataModel(JSON.parse(App.currentProject.get('gis_data'))), errorMessages: null, infoMessages: null }); diff --git a/src/mmw/js/src/modeling/gwlfeModificationConfig.js b/src/mmw/js/src/modeling/gwlfeModificationConfig.js index f99fb4ec5..faba06758 100644 --- a/src/mmw/js/src/modeling/gwlfeModificationConfig.js +++ b/src/mmw/js/src/modeling/gwlfeModificationConfig.js @@ -72,6 +72,12 @@ var shortDisplayNames = fromPairs([ [UrbLengthName, 'length of streams in non-ag areas'] ]); +function cleanDataModel(dataModel) { + return _.mapValues(dataModel, function(val, varName) { + return varName === UrbLengthName ? val / 1000 : val; + }); +} + function getPercentStr(fraction, dataModelName) { var dataModelDisplayName = shortDisplayNames[dataModelName] || displayNames[dataModelName]; return Math.round(fraction * 100) + '% of ' + dataModelDisplayName; @@ -310,6 +316,7 @@ var configs = { }; module.exports = { + cleanDataModel: cleanDataModel, displayNames: displayNames, isValid: isValid, aggregateOutput: aggregateOutput, From 4132d48d10e418b77f1ce9919c9a2329e36849f4 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 31 May 2016 12:51:21 -0400 Subject: [PATCH 108/136] Use DRB Stream Length for LS We prefer DRB Stream Lengths for LS calculations if possible for the selected Area of Interest. If the AoI is outside the DRB, then we fall back to using standard NHD+ stream lengths. --- src/mmw/apps/modeling/mapshed/tasks.py | 6 +++++- src/mmw/mmw/settings/base.py | 2 +- src/mmw/mmw/settings/layer_settings.py | 5 +++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index ca3d552c1..a6068c331 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -37,6 +37,7 @@ NLU = settings.GWLFE_CONFIG['NLU'] NRur = settings.GWLFE_DEFAULTS['NRur'] AG_NLCD_CODES = settings.GWLFE_CONFIG['AgriculturalNLCDCodes'] +DRB = settings.DRB_PERIMETER ACRES_PER_SQM = 0.000247105 HECTARES_PER_SQM = 0.0001 SQKM_PER_SQM = 0.000001 @@ -172,8 +173,11 @@ def collect_data(geop_result, geojson): z['SedAFactor'] = sed_a_factor(geop_result['landuse_pcts'], z['CN'], z['AEU'], z['AvKF'], z['AvSlope']) + ls_stream_length = (stream_length(geom, drb=True) if geom.within(DRB) + else z['StreamLength']) + z['LS'] = ls_factors(geop_result['lu_stream_pct'], - z['StreamLength'], z['Area'], z['AvSlope']) + ls_stream_length, z['Area'], z['AvSlope']) return z diff --git a/src/mmw/mmw/settings/base.py b/src/mmw/mmw/settings/base.py index 58871f097..b966acf3d 100644 --- a/src/mmw/mmw/settings/base.py +++ b/src/mmw/mmw/settings/base.py @@ -12,7 +12,7 @@ from os.path import abspath, basename, dirname, join, normpath from sys import path -from layer_settings import LAYERS, VIZER_URLS # NOQA +from layer_settings import LAYERS, VIZER_URLS, DRB_PERIMETER # NOQA from gwlfe_settings import (GWLFE_DEFAULTS, GWLFE_CONFIG, SOIL_GROUP, # NOQA SOILP, CURVE_NUMBER) # NOQA diff --git a/src/mmw/mmw/settings/layer_settings.py b/src/mmw/mmw/settings/layer_settings.py index fa366406b..1410b4d4c 100644 --- a/src/mmw/mmw/settings/layer_settings.py +++ b/src/mmw/mmw/settings/layer_settings.py @@ -13,6 +13,8 @@ from os.path import join, dirname, abspath import json +from django.contrib.gis.geos import GEOSGeometry + # Simplified perimeter of the Delaware River Basin (DRB). drb_perimeter_path = join(dirname(abspath(__file__)), 'data/drb_perimeter.json') @@ -207,6 +209,9 @@ 'basemap': True, } ] + +DRB_PERIMETER = GEOSGeometry(json.dumps(drb_perimeter['geometry']), srid=4326) + # Vizer observation meta data URL. Happens to be proxied through a local app # server to avoid Cross Domain request errors VIZER_ROOT = '/observation/services/get_asset_info.php?' From 2834f5ba0093ca968d1fe9fe744d2e0a9687c567 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 31 May 2016 15:06:11 -0400 Subject: [PATCH 109/136] Export GMS from Scenario instead of Project Since each scenario can contribute different values to the GMS file given its modifications, we change to exporting from the scenario instead of the project. Furthermore, we now allow Current Conditions to have a dropdown menu, since we need it to allow GMS Export. While this does allow duplication of the "Share" functionality, since sharing the Current Conditions scenario is the same as sharing the project, consistency in menus is more important, and this duplication is preferable to Current Conditions not having a Share option at all when all other scenarios have it. --- .../src/modeling/templates/projectMenu.html | 8 ---- .../modeling/templates/scenarioTabPanel.html | 10 ++++- src/mmw/js/src/modeling/views.js | 39 +++++++++++-------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/mmw/js/src/modeling/templates/projectMenu.html b/src/mmw/js/src/modeling/templates/projectMenu.html index 1ba1a69ac..48ba4d5f3 100644 --- a/src/mmw/js/src/modeling/templates/projectMenu.html +++ b/src/mmw/js/src/modeling/templates/projectMenu.html @@ -23,14 +23,6 @@

    {{ name }}

    {% if itsi %}
  • Embed in ITSI
    • {% endif %} - {% if gwlfe %} -
    - - - -
    -
  • Export GMS
    • - {% endif %} {% if user and not itsi_embed %}
  • My Projects
    • diff --git a/src/mmw/js/src/modeling/templates/scenarioTabPanel.html b/src/mmw/js/src/modeling/templates/scenarioTabPanel.html index 05c9b88cd..3f77f8888 100644 --- a/src/mmw/js/src/modeling/templates/scenarioTabPanel.html +++ b/src/mmw/js/src/modeling/templates/scenarioTabPanel.html @@ -1,5 +1,5 @@ {{ name }} -{% if active and editable and not is_current_conditions %} +{% if active and editable and (gwlfe or not is_current_conditions or not is_new) %}
    diff --git a/src/mmw/js/src/modeling/views.js b/src/mmw/js/src/modeling/views.js index 4139a18ea..f7900efdd 100644 --- a/src/mmw/js/src/modeling/views.js +++ b/src/mmw/js/src/modeling/views.js @@ -104,8 +104,6 @@ var ProjectMenuView = Marionette.ItemView.extend({ print: '#print-project', save: '#save-project', privacy: '#project-privacy', - exportGms: '#export-gms', - exportGmsForm: '#export-gms-form', itsiClone: '#itsi-clone' }, @@ -116,7 +114,6 @@ var ProjectMenuView = Marionette.ItemView.extend({ 'click @ui.print': 'printProject', 'click @ui.save': 'saveProjectOrLoginUser', 'click @ui.privacy': 'setProjectPrivacy', - 'click @ui.exportGms': 'downloadGmsFile', 'click @ui.itsiClone': 'getItsiEmbedLink' }, @@ -127,11 +124,6 @@ var ProjectMenuView = Marionette.ItemView.extend({ itsi: App.user.get('itsi'), itsi_embed: settings.get('itsi_embed'), editable: isEditable(this.model), - csrftoken: csrf.getToken(), - gwlfe: this.model.get('model_package') === models.GWLFE && - this.model.get('gis_data') !== null && - this.model.get('gis_data') !== '' && - this.model.get('gis_data') !== '{}', is_new: this.model.isNew() }; }, @@ -248,13 +240,6 @@ var ProjectMenuView = Marionette.ItemView.extend({ }); }, - downloadGmsFile: function() { - // We can't download a file from an AJAX call. One either has to - // load the data in an iframe, or submit a form that responds with - // Content-Disposition: attachment. We prefer submitting a form. - this.ui.exportGmsForm.submit(); - }, - getItsiEmbedLink: function() { var self = this, embedLink = window.location.origin + @@ -356,7 +341,19 @@ var ScenarioTabPanelView = Marionette.ItemView.extend({ }, templateHelpers: function() { + var gis_data = this.model.getGisData().model_input, + gwlfe = App.currentProject.get('model_package') === models.GWLFE && + gis_data !== null && + gis_data !== '{}' && + gis_data !== '', + filename = App.currentProject.get('name').replace(' ', '_') + + '__' + this.model.get('name').replace(' ', '_'); + return { + gwlfe: gwlfe, + csrftoken: csrf.getToken(), + filename: filename, + gis_data: gis_data, cid: this.model.cid, editable: isEditable(this.model), is_new: this.model.isNew() @@ -373,6 +370,8 @@ var ScenarioTabPanelView = Marionette.ItemView.extend({ rename: '[data-action="rename"]', print: '[data-action="print"]', duplicate: '[data-action="duplicate"]', + exportGms: '[data-action="export-gms"]', + exportGmsForm: '#export-gms-form', nameField: '.tab-name' }, @@ -383,7 +382,8 @@ var ScenarioTabPanelView = Marionette.ItemView.extend({ 'click @ui.print': function() { window.print(); }, - 'click @ui.duplicate': 'duplicateScenario' + 'click @ui.duplicate': 'duplicateScenario', + 'click @ui.exportGms': 'downloadGmsFile', }, renameScenario: function() { @@ -452,6 +452,13 @@ var ScenarioTabPanelView = Marionette.ItemView.extend({ duplicateScenario: function() { this.model.collection.duplicateScenario(this.model.cid); + }, + + downloadGmsFile: function() { + // We can't download a file from an AJAX call. One either has to + // load the data in an iframe, or submit a form that responds with + // Content-Disposition: attachment. We prefer submitting a form. + this.ui.exportGmsForm.submit(); } }); From e8e7b1b17c524405e63bf45c16e88c5e7a7d348b Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 31 May 2016 15:07:00 -0400 Subject: [PATCH 110/136] cleanup: Remove extraneous logging --- src/mmw/js/src/modeling/models.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/mmw/js/src/modeling/models.js b/src/mmw/js/src/modeling/models.js index b2b2cbff6..3f7d4d170 100644 --- a/src/mmw/js/src/modeling/models.js +++ b/src/mmw/js/src/modeling/models.js @@ -826,10 +826,6 @@ var ScenarioModel = Backbone.Model.extend({ } }; - // TODO remove before merging - console.log('gwlfe postData'); - console.log(JSON.parse(gisData.model_input)); - return taskModel.start(taskHelper); }, From eae344667327ab5a4ae3a565fb5d96d721e461eb Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 31 May 2016 21:15:49 -0400 Subject: [PATCH 111/136] Restore sidebar collapse functionality The sidebar collapse functionality would previously be enabled by the pre-handler of the /project route. Since we switched to using the /project/new route to initialie each project for its chosen model package, the original handler was no longer executing, and the button wasn't enabled. By including that instruction in the new handler, we ensure that the button is made visible. In addition, we push down the map controls a bit so they don't overlap the button. --- src/mmw/js/src/modeling/controllers.js | 1 + src/mmw/sass/base/_map.scss | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mmw/js/src/modeling/controllers.js b/src/mmw/js/src/modeling/controllers.js index dd1df2e0e..9ec16648e 100644 --- a/src/mmw/js/src/modeling/controllers.js +++ b/src/mmw/js/src/modeling/controllers.js @@ -119,6 +119,7 @@ var ModelingController = { finishProjectSetup(project, lock); updateUrl(); } + App.rootView.showCollapsable(); }, projectCleanUp: function() { diff --git a/src/mmw/sass/base/_map.scss b/src/mmw/sass/base/_map.scss index aa124bff2..7a78ec8cc 100644 --- a/src/mmw/sass/base/_map.scss +++ b/src/mmw/sass/base/_map.scss @@ -103,7 +103,7 @@ } .leaflet-top { - top: 10px; + top: 45px; } .leaflet-control-layers-expanded { From d7bc34db3dd6fcc8c3fb791f647c58ef8ce32855 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 2 Jun 2016 00:25:51 -0400 Subject: [PATCH 112/136] Convert temps to Celsius, precips to cm The source data is in Fahrenheit and inches, however GWLF-E needs these values in Celsius and cm. --- src/mmw/apps/modeling/mapshed/calcs.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index 48497d60e..eb93b9a2b 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -22,6 +22,7 @@ LITERS_PER_MGAL = 3785412 AG_NLCD_CODES = settings.GWLFE_CONFIG['AgriculturalNLCDCodes'] KM_PER_M = 0.001 +CM_PER_INCH = 2.54 def day_lengths(geom): @@ -324,6 +325,10 @@ def weather_data(ws, begyear, endyear): and so on; `month` 0 corresponds to January, 1 to February, and so on; `day` 0 corresponds to the 1st of the month, 1 to the 2nd, and so on. """ + # Utility function to convert Fahrenheit to Celsius + def f_to_c(f): + return (f - 32) * 5.0 / 9.0 + temp_sql = ''' SELECT year, EXTRACT(MONTH FROM TO_DATE(month, 'MON')) AS month, AVG("1") AS "1", AVG("2") AS "2", AVG("3") AS "3", @@ -368,7 +373,7 @@ def weather_data(ws, begyear, endyear): year = int(row[0]) - begyear month = int(row[1]) - 1 for day in range(31): - temps[year][month][day] = float(row[day + 2]) + temps[year][month][day] = f_to_c(float(row[day + 2])) with connection.cursor() as cursor: cursor.execute(prcp_sql, [stations, begyear, endyear]) @@ -376,7 +381,7 @@ def weather_data(ws, begyear, endyear): year = int(row[0]) - begyear month = int(row[1]) - 1 for day in range(31): - prcps[year][month][day] = float(row[day + 2]) + prcps[year][month][day] = float(row[day + 2]) * CM_PER_INCH return temps, prcps From 4ec59002325dc912a212ee573ea16bfea86c1059 Mon Sep 17 00:00:00 2001 From: Matthew McFarland Date: Thu, 2 Jun 2016 10:17:05 -0400 Subject: [PATCH 113/136] Use AvGroundWater for Subsurface flow display Replaces an incorrect key for display. --- src/mmw/js/src/modeling/gwlfe/runoff/views.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mmw/js/src/modeling/gwlfe/runoff/views.js b/src/mmw/js/src/modeling/gwlfe/runoff/views.js index 63308de8d..d5fad3827 100644 --- a/src/mmw/js/src/modeling/gwlfe/runoff/views.js +++ b/src/mmw/js/src/modeling/gwlfe/runoff/views.js @@ -13,7 +13,7 @@ var runoffVars = [ { name: 'AvPrecipitation', display: 'Precip' }, { name: 'AvEvapoTrans', display: 'ET' }, { name: 'AvRunoff', display: 'Surface Runoff' }, - { name: 'AvTileDrain', display: 'Subsurface Flow' }, + { name: 'AvGroundWater', display: 'Subsurface Flow' }, { name: 'AvPtSrcFlow', display: 'Point Src Flow' }, { name: 'AvStreamFlow', display: 'Stream Flow' }, ], From 53b8af7dfedf322575a4322fa0d76b2ffa73c4ee Mon Sep 17 00:00:00 2001 From: Matthew McFarland Date: Thu, 2 Jun 2016 10:17:34 -0400 Subject: [PATCH 114/136] Rename Runoff -> Hydrology It was brought to our attention several times that the tab "Runoff" contains more than just runoff value and is better described as "Hydrology". Leaving TR55 for now, since there are tutorials referencing the tab as such. --- src/mmw/js/src/modeling/models.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mmw/js/src/modeling/models.js b/src/mmw/js/src/modeling/models.js index 3f7d4d170..3a54d2545 100644 --- a/src/mmw/js/src/modeling/models.js +++ b/src/mmw/js/src/modeling/models.js @@ -1064,7 +1064,7 @@ function createTaskResultCollection(modelPackage) { return new ResultCollection([ { name: 'runoff', - displayName: 'Runoff', + displayName: 'Hydrology', result: null }, { From 289a21ba00c52d62607f83546e6739c15a9ecb4b Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Mon, 6 Jun 2016 14:24:19 -0400 Subject: [PATCH 115/136] Discharge in cm/month, instead of L/m2/month --- src/mmw/apps/modeling/mapshed/calcs.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index eb93b9a2b..ff4e10b09 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -19,10 +19,11 @@ MONTHDAYS = settings.GWLFE_CONFIG['MonthDays'] LIVESTOCK = settings.GWLFE_CONFIG['Livestock'] POULTRY = settings.GWLFE_CONFIG['Poultry'] -LITERS_PER_MGAL = 3785412 +M3_PER_MGAL = 3785.41178 AG_NLCD_CODES = settings.GWLFE_CONFIG['AgriculturalNLCDCodes'] KM_PER_M = 0.001 CM_PER_INCH = 2.54 +CM_PER_M = 100.0 def day_lengths(geom): @@ -291,7 +292,7 @@ def point_source_discharge(geom, area): """ Given a geometry and its area in square meters, returns three lists, each with 12 values, one for each month, containing the Nitrogen Load (in - kg), Phosphorus Load (in kg), and Discharge (in liters per square meter) + kg), Phosphorus Load (in kg), and Discharge (in centimeters per month). """ sql = ''' SELECT SUM(mgd) AS mg_d, @@ -308,7 +309,7 @@ def point_source_discharge(geom, area): n_load = [float(kgn_month)] * 12 if kgn_month else [0.0] * 12 p_load = [float(kgp_month)] * 12 if kgp_month else [0.0] * 12 - discharge = [float(mg_d * days * LITERS_PER_MGAL) / area + discharge = [float(mg_d) * days * M3_PER_MGAL * CM_PER_M / area for days in MONTHDAYS] if mg_d else [0.0] * 12 return n_load, p_load, discharge From 31fca1a35e9a3ed5b15b1017b776aeb5b87de12c Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 8 Jun 2016 14:45:10 -0400 Subject: [PATCH 116/136] Don't fail on empty streams Though the results may be inaccurate, we can add a disclaimer at a later time to notify the user. Crashing is not acceptable. --- src/mmw/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mmw/requirements/base.txt b/src/mmw/requirements/base.txt index c39f9d79e..5c86aeb57 100644 --- a/src/mmw/requirements/base.txt +++ b/src/mmw/requirements/base.txt @@ -10,7 +10,7 @@ python-omgeo==1.7.2 rauth==0.7.1 djangorestframework-gis==0.8.2 tr55==1.1.3 -gwlf-e==0.3.0 +gwlf-e==0.4.0 requests==2.9.1 rollbar==0.12.1 retry==0.9.1 From 4dd050057a6ecb3601f7928f1395106c8c635591 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 9 Jun 2016 12:17:28 -0400 Subject: [PATCH 117/136] Map NLCD 21 to Low Density Mixed This earlier mapped to Open Land. Switch on instruction from BME. --- src/mmw/apps/modeling/mapshed/calcs.py | 6 +++--- src/mmw/apps/modeling/mapshed/tasks.py | 10 +++++----- src/mmw/mmw/settings/gwlfe_settings.py | 5 ++--- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index ff4e10b09..4dcfbf8e2 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -423,7 +423,7 @@ def cni_avg(nlcds): cni_avg([90, 95]), # Wetland 0, # Disturbed 0, # Turf Grass - cni_avg([21, 71]), # Open Land + cni(71), # Open Land cni_avg([12, 31]), # Bare Rock 0, # Sandy Areas 0, # Unpaved Road @@ -525,11 +525,11 @@ def landuse_pcts(n_count): n_pct.get(90, 0) + n_pct.get(95, 0), # Wetland 0, # Disturbed 0, # Turf Grass - n_pct.get(21, 0) + n_pct.get(71, 0), # Open Land + n_pct.get(71, 0), # Open Land n_pct.get(12, 0) + n_pct.get(31, 0), # Bare Rock 0, # Sandy Areas 0, # Unpaved Road - n_pct.get(22, 0), # Low Density Mixed + n_pct.get(21, 0) + n_pct.get(22, 0), # Low Density Mixed n_pct.get(23, 0), # Medium Density Mixed n_pct.get(24, 0), # High Density Mixed 0, # Low Density Residential diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index a6068c331..2c983ed3a 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -189,8 +189,8 @@ def nlcd_streams(sjs_result): each, return a dictionary with keys 'ag_stream_pct', 'low_urban_stream_pct' and 'med_high_urban_stream_pct' which indicate the percent of streams in agricultural areas (namely NLCD 81 Pasture/Hay and 82 Cultivated Crops), - low density urban areas (NLCD 22), and medium and high density urban areas - (NLCD 23 and 24) respectively. + low density urban areas (NLCD 21 and 22), and medium and high density urban + areas (NLCD 23 and 24) respectively. In addition, we inspect the result to see if it includes an 'error' key. If so, it would indicate that a preceeding task has thrown an exception, @@ -225,7 +225,7 @@ def nlcd_streams(sjs_result): result = parse_sjs_result(sjs_result) ag_count = sum(result.get(nlcd, 0) for nlcd in AG_NLCD_CODES) - low_urban_count = result.get(22, 0) + low_urban_count = sum(result.get(nlcd, 0) for nlcd in [21, 22]) med_high_urban_count = sum(result.get(nlcd, 0) for nlcd in [23, 24]) total = sum(result.values()) @@ -468,11 +468,11 @@ def get_lu_index(nlcd): lu_index = 3 elif nlcd in [90, 95]: lu_index = 4 - elif nlcd in [21, 71]: + elif nlcd == 71: lu_index = 7 elif nlcd in [12, 31]: lu_index = 8 - elif nlcd == 22: + elif nlcd in [21, 22]: lu_index = 11 elif nlcd == 23: lu_index = 12 diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index 37c7bde99..8a2a4446b 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -27,11 +27,11 @@ lu.WETLAND, # Maps NLCD 90, 95 lu.DISTURBED, # Does not map to NLCD lu.TURFGRASS, # Does not map to NLCD - lu.OPEN_LAND, # Maps NLCD 21, 71 + lu.OPEN_LAND, # Maps NLCD 71 lu.BARE_ROCK, # Maps NLCD 12, 31 lu.SANDY_AREAS, # Does not map to NLCD lu.UNPAVED_ROAD, # Does not map to NLCD - lu.LD_MIXED, # Maps NLCD 22 + lu.LD_MIXED, # Maps NLCD 21, 22 lu.MD_MIXED, # Maps NLCD 23 lu.HD_MIXED, # Maps NLCD 24 lu.LD_RESIDENTIAL, # Does not map to NLCD @@ -537,7 +537,6 @@ CURVE_NUMBER = { 11: [0, 100, 100, 100, 100], # Water 12: [0, 72, 82, 87, 89], # Perennial Ice/Snow - 21: [0, 72, 82, 87, 89], # Developed Open Space 31: [0, 72, 82, 87, 89], # Barren Land 41: [0, 37, 60, 73, 80], # Deciduous Forest 42: [0, 37, 60, 73, 80], # Evergreen Forest From d59c7eeee43ff6ae74f0151593ef6d7edea9096b Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 9 Jun 2016 13:26:54 -0400 Subject: [PATCH 118/136] Look up C and P values Not sure how these were missed earlier. --- src/mmw/apps/modeling/mapshed/calcs.py | 32 ++++++++++++++++++++++++++ src/mmw/apps/modeling/mapshed/tasks.py | 3 +++ src/mmw/mmw/settings/gwlfe_settings.py | 4 ++++ 3 files changed, 39 insertions(+) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index 4dcfbf8e2..654a14994 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -575,3 +575,35 @@ def sed_a_factor(landuse_pct_vals, cn, AEU, AvKF, AvSlope): return ((0.00467 * urban_pct) + (0.000863 * AEU) + (0.000001 * avg_cn) + (0.000425 * AvKF) + (0.000001 * AvSlope) - 0.000036) + + +def p_factors(avg_slope): + """ + Given the average slope, calculates the P Factor for rural land use types. + + Original at Class1.vb@1.3.0:4393-4470 + """ + if 0 <= avg_slope < 2.1: + ag_p = 0.52 + elif 2.1 <= avg_slope < 7.1: + ag_p = 0.45 + elif 7.1 <= avg_slope < 12.1: + ag_p = 0.52 + elif 12.1 <= avg_slope < 18.1: + ag_p = 0.66 + else: + ag_p = 0.74 + + return [ + ag_p, # Hay/Pasture + ag_p, # Cropland + ag_p, # Forest + 0.1, # Wetland + 0.1, # Disturbed + 0.2, # Turf Grass + ag_p, # Open Land + ag_p, # Bare Rock + ag_p, # Sandy Areas + 1.0, # Unpaved + 0, 0, 0, 0, 0, 0 # Urban Land Use Types + ] diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index 2c983ed3a..93ecedbcb 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -20,6 +20,7 @@ kv_coefficient, animal_energy_units, ls_factors, + p_factors, manure_spread, streams, stream_length, @@ -179,6 +180,8 @@ def collect_data(geop_result, geojson): z['LS'] = ls_factors(geop_result['lu_stream_pct'], ls_stream_length, z['Area'], z['AvSlope']) + z['P'] = p_factors(z['AvSlope']) + return z diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index 8a2a4446b..54e5fb8ec 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -40,6 +40,8 @@ ], 'Imper': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # Impervious surface area percentage 0.15, 0.52, 0.87, 0.15, 0.52, 0.87], # only defined for urban land use types + 'C': [0.03, 0.42, 0.002, 0.01, 0.08, 0.03, 0.04, 0.001, 0.01, 0.8, # C Factor Defaults + 0, 0, 0, 0, 0, 0], # only defined for rural land use types 'CNI': [[0] * 16, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # Curve Number for Impervious Surfaces 92, 98, 98, 92, 92, 92], # only defined for urban land use types @@ -469,6 +471,7 @@ 'AntMoist', 'Area', 'AvgAnimalWt', + 'C', 'CN', 'CNI', 'CNP', @@ -515,6 +518,7 @@ 'NumPondSys', 'NumSewerSys', 'NumShortSys', + 'P', 'PcntET', 'PctGrazing', 'PctStreams', From 0f357039a566225312059d7f5968c84fbd4b3cd5 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 9 Jun 2016 14:54:24 -0400 Subject: [PATCH 119/136] Correct day length calculation The original VB code used 1-based indexing, which we had missed here. Also add a reference to original code. --- src/mmw/apps/modeling/mapshed/calcs.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index 654a14994..dfb29ddb1 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -31,18 +31,20 @@ def day_lengths(geom): Given a geometry in EPSG:4326, returns an array of 12 floats, each representing the average number of daylight hours at that geometry's centroid for each month. + + Original at Class1.vb@1.3.0:8878-8889 """ latitude = geom.centroid[1] lengths = [0.0] * 12 - for month in range(12): + for m in range(12): # Magic formula taken from original MapShed source - lengths[month] = 7.63942 * math.acos(0.43481 * - math.tan(latitude * 0.017453) * - math.cos(0.0172 * - (month * 30.4375 - 5))) + lengths[m] = 7.63942 * math.acos(0.43481 * + math.tan(0.017453 * latitude) * + math.cos(0.0172 * + ((m + 1) * 30.4375 - 5))) - return lengths + return [round(l, 1) for l in lengths] def nearest_weather_stations(geom, n=NUM_WEATHER_STATIONS): From cbe32dab62ab67a03afaf5807b09587e89dd1008 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 9 Jun 2016 16:10:29 -0400 Subject: [PATCH 120/136] Correct LS calculation Previously we were multiplying the NHD Land Use Stream Percent value with DRB Stream Length inside ls_factors, which was wrong. Now we multiply DRB Land Use Stream Percent values with DRB Stream length when in the DRB, and NHD Land Use Stream Percent values with NHD Stream Length when outside it. --- src/mmw/apps/modeling/mapshed/tasks.py | 43 +++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index 93ecedbcb..d76ef856b 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -174,11 +174,15 @@ def collect_data(geop_result, geojson): z['SedAFactor'] = sed_a_factor(geop_result['landuse_pcts'], z['CN'], z['AEU'], z['AvKF'], z['AvSlope']) - ls_stream_length = (stream_length(geom, drb=True) if geom.within(DRB) - else z['StreamLength']) + if geom.within(DRB): + ls_stream_length = stream_length(geom, drb=True) + ls_stream_pcts = geop_result['lu_stream_pct_drb'] + else: + ls_stream_length = z['StreamLength'] + ls_stream_pcts = geop_result['lu_stream_pct'] - z['LS'] = ls_factors(geop_result['lu_stream_pct'], - ls_stream_length, z['Area'], z['AvSlope']) + z['LS'] = ls_factors(ls_stream_pcts, ls_stream_length, + z['Area'], z['AvSlope']) z['P'] = p_factors(z['AvSlope']) @@ -237,7 +241,7 @@ def nlcd_streams(sjs_result): for count in (ag_count, low_urban_count, med_high_urban_count)) - lu_stream_pct = [0.0] * 16 + lu_stream_pct = [0.0] * NLU for nlcd, stream_count in result.iteritems(): lu = get_lu_index(nlcd) if lu is not None: @@ -251,6 +255,29 @@ def nlcd_streams(sjs_result): } +@shared_task(throws=Exception) +def nlcd_streams_drb(sjs_result): + """ + This callback is run when the geometry falls within the DRB. We calculate + the percentage of DRB streams in each land use type. + """ + if 'error' in sjs_result: + raise Exception('[nlcd_streams_drb] {}'.format(sjs_result['error'])) + + result = parse_sjs_result(sjs_result) + total = sum(result.values()) + + lu_stream_pct_drb = [0.0] * NLU + for nlcd, stream_count in result.iteritems(): + lu = get_lu_index(nlcd) + if lu is not None: + lu_stream_pct_drb[lu] += float(stream_count) / total + + return { + 'lu_stream_pct_drb': lu_stream_pct_drb + } + + @shared_task(throws=Exception) def nlcd_soils(sjs_result): """ @@ -437,6 +464,12 @@ def geop_tasks(geom, errback): nlcd_kfactor) ] + if geom.within(DRB): + definitions.append(('nlcd_streams', + {'polygon': [geom.geojson], + 'vector': streams(geom, drb=True)}, + nlcd_streams_drb)) + return [(mapshed_start.s(opname, data) | mapshed_finish.s() | callback.s().set(link_error=errback)) From 34362e341a93c99252881f504ba40401029ad4bb Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 9 Jun 2016 18:04:23 -0400 Subject: [PATCH 121/136] Correct KV calculation Previously we were erroneously calculating KV from the erosion coefficient. It must instead be calculated from the ET coefficient. --- src/mmw/apps/modeling/mapshed/calcs.py | 23 ++++++++++------ src/mmw/apps/modeling/mapshed/tasks.py | 3 ++- src/mmw/mmw/settings/gwlfe_settings.py | 36 ++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 9 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index dfb29ddb1..382898b3a 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -117,21 +117,28 @@ def et_adjustment(ws): return [avg_etadj] * 12 -def kv_coefficient(ecs): +def kv_coefficient(area_pcts, season): """ - Given an array of erosion coefficients, returns an array of 12 decimals, - one for the KV coefficient of each month. The KV of a month is initialized - to the corresponding erosion coefficient value times the KV Factor, and - then averaged over the preceeding month. January being the first month is - not averaged. + Given arrays of land use area percentages and growing season, returns an + array of 12 floats, one for the KV coefficient of each month. The KV of a + month is initialized to the sum of ETs of each land use type weighted by + its area percent, using growth or dormant coefficients depending on whether + or not the month is in the growing season. The value is then averaged over + the preceeding month and multiplied by the KV Factor. January being the + first month is not averaged. Original at Class1.vb@1.3.0:4989-4995 """ - kv = [ec * KV_FACTOR for ec in ecs] + et_grow = sum([et * area_pct for et, area_pct in + zip(settings.GWLFE_CONFIG['ETGrowCoeff'], area_pcts)]) + et_dorm = sum([et * area_pct for et, area_pct in + zip(settings.GWLFE_CONFIG['ETDormCoeff'], area_pcts)]) + kv = [et_grow if m == GrowFlag.GROWING_SEASON else et_dorm for m in season] + kv[0] *= KV_FACTOR for m in range(1, 12): - kv[m] = (kv[m] + kv[m-1]) / 2 + kv[m] = KV_FACTOR * (kv[m] + kv[m-1]) / 2 return kv diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index d76ef856b..0ac400454 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -105,7 +105,6 @@ def collect_data(geop_result, geojson): z['Grow'] = growing_season(ws) z['Acoef'] = erosion_coeff(ws, z['Grow']) z['PcntET'] = et_adjustment(ws) - z['KV'] = kv_coefficient(z['Acoef']) z['WxYrBeg'] = int(max([w.begyear for w in ws])) z['WxYrEnd'] = int(min([w.endyear for w in ws])) z['WxYrs'] = z['WxYrEnd'] - z['WxYrBeg'] + 1 @@ -160,6 +159,8 @@ def collect_data(geop_result, geojson): z['AvKF'] = geop_result['avg_kf'] z['KF'] = geop_result['kf'] + z['KV'] = kv_coefficient(geop_result['landuse_pcts'], z['Grow']) + # Original at Class1.vb@1.3.0:9803-9807 z['n23'] = z['Area'][1] # Row Crops Area z['n23b'] = z['Area'][13] # High Density Mixed Urban Area diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index 54e5fb8ec..80f1dac2e 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -463,6 +463,42 @@ 'SSLDR': 4.4, 'SSLDM': 2.2, 'WeatherNull': -99999, # This value is used to indicate NULL in ms_weather dataset + 'ETGrowCoeff': [ + 1.00, # Hay/Pasture + 0.80, # Cropland + 0.77, # Forest + 1.00, # Wetlands + 0.30, # Disturbed + 1.00, # Turf Grass + 1.00, # Open Land + 0.10, # Bare Rock + 0.30, # Sandy Areas + 1.00, # Unpaved Roads + 0.85, # Low Density Mixed + 0.48, # Medium Density Mixed + 0.70, # High Density Mixed + 0.85, # Low Density Residential + 0.48, # Medium Density Residential + 0.13, # High Density Residential + ], + 'ETDormCoeff': [ + 0.80, # Hay/Pasture + 0.30, # Cropland + 0.51, # Forest + 1.00, # Wetlands + 0.30, # Disturbed + 0.80, # Turf Grass + 0.80, # Open Land + 0.10, # Bare Rock + 0.30, # Sandy Areas + 1.00, # Unpaved Roads + 0.80, # Low Density Mixed + 0.48, # Medium Density Mixed + 0.70, # High Density Mixed + 0.85, # Low Density Residential + 0.48, # Medium Density Residential + 0.13, # High Density Residential + ], 'ArrayFields': [ # These may be optionally converted to numpy arrays before running the model # NOQA 'Acoef', 'AnimalDailyN', From 940d4eb241e01bd44102ac5a623b1ba36f1992d1 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 10 Jun 2016 16:05:43 -0400 Subject: [PATCH 122/136] Correctly calculate PhosConc Previously we were using the default value of 0.01 for all land use types, whereas we should use the calculated values for the agricultural land use types and turf grass. --- src/mmw/apps/modeling/mapshed/calcs.py | 28 ++++++++++++++++++++++++++ src/mmw/apps/modeling/mapshed/tasks.py | 2 ++ src/mmw/mmw/settings/gwlfe_settings.py | 3 --- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index 382898b3a..0d5e80afb 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -616,3 +616,31 @@ def p_factors(avg_slope): 1.0, # Unpaved 0, 0, 0, 0, 0, 0 # Urban Land Use Types ] + + +def phosphorus_conc(sed_phos): + """ + Given the average concentration of phosphorus dissolved in sediments for + the entire polygon, returns average concentration per land use type based + on pre-baked estimates. + + Original at Class1.vb@1.3.0:8975-9001,9350-9359 + """ + psed = sed_phos / 1.6 + stp = 190 * psed / 836 + prunoff = (1.98 * stp + 79) / 1000 + prunoff_turf = 2.9 * psed / 836 + + return [ + prunoff, # Hay/Pasture + prunoff, # Cropland + 0.01, # Forest + 0.01, # Wetland + 0.01, # Disturbed + prunoff_turf, # Turf Grass + 0.01, # Open Land + 0.01, # Bare Rock + 0.01, # Sandy Areas + 0.01, # Unpaved + 0, 0, 0, 0, 0, 0 # Urban Land Use Types + ] diff --git a/src/mmw/apps/modeling/mapshed/tasks.py b/src/mmw/apps/modeling/mapshed/tasks.py index 0ac400454..ee5b2cc76 100644 --- a/src/mmw/apps/modeling/mapshed/tasks.py +++ b/src/mmw/apps/modeling/mapshed/tasks.py @@ -28,6 +28,7 @@ weather_data, curve_number, sediment_phosphorus, + phosphorus_conc, groundwater_nitrogen_conc, sediment_delivery_ratio, landuse_pcts, @@ -146,6 +147,7 @@ def collect_data(geop_result, geojson): z['Area'] = [percent * area * HECTARES_PER_SQM for percent in geop_result['landuse_pcts']] z['UrbAreaTotal'] = sum(z['Area'][NRur:]) + z['PhosConc'] = phosphorus_conc(z['SedPhos']) z['NormalSys'] = normal_sys(z['Area']) diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index 80f1dac2e..93297fb14 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -70,9 +70,6 @@ 'NitrConc': [0.75, 2.90, 0.19, 0.19, 0.02, # Dissolved Runoff Coefficient: Nitrogen mg/l 2.50, 0.50, 0.30, 0.10, 0.19, # only defined for rural land use types 0, 0, 0, 0, 0, 0], - 'PhosConc': [0.01, 0.01, 0.01, 0.01, 0.01, # Dissolved Runoff Coefficient: Phosphorus mg/l - 0.01, 0.01, 0.01, 0.01, 0.01, # only defined for rural land use types - 0, 0, 0, 0, 0, 0], 'Nqual': 3, # Number of Contaminants (Default = 3) 'Contaminant': ['Nitrogen', 'Phosphorus', 'Sediment'], 'LoadRateImp': [[], [], [], [], [], [], [], [], [], [], # Load Rate on Impervious Surfaces per urban land use per contaminant From 3385cb91b82a8afbd335151d50dfe17f3a27eaea Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 10 Jun 2016 18:58:26 -0400 Subject: [PATCH 123/136] Add units to Hydrology table All columns are in centimeters. --- src/mmw/js/src/modeling/gwlfe/runoff/templates/table.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mmw/js/src/modeling/gwlfe/runoff/templates/table.html b/src/mmw/js/src/modeling/gwlfe/runoff/templates/table.html index 93cdb0dd6..b4d5908e3 100644 --- a/src/mmw/js/src/modeling/gwlfe/runoff/templates/table.html +++ b/src/mmw/js/src/modeling/gwlfe/runoff/templates/table.html @@ -7,7 +7,7 @@ {{ columnName }} {% else %} - {{ columnName }} + {{ columnName }} (cm) {% endif %} {% endfor %} From 57a5bb52f62cf0611e090e80d5881783ad9f1863 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 10 Jun 2016 19:34:33 -0400 Subject: [PATCH 124/136] Project menu should be on top --- src/mmw/sass/base/_header.scss | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/mmw/sass/base/_header.scss b/src/mmw/sass/base/_header.scss index a40ea1361..4810ef799 100644 --- a/src/mmw/sass/base/_header.scss +++ b/src/mmw/sass/base/_header.scss @@ -120,7 +120,7 @@ header { position: absolute; top: 10px; right: 0; - z-index: 200; + z-index: ($zindex-dropdown + 10); text-transform: uppercase; } } @@ -141,6 +141,12 @@ header { padding: 7px; } + #scenarios-drop-down-region { + .dropdown-menu { + z-index: ($zindex-dropdown + 10); + } + } + .scenario-tabs-wrapper { position: absolute; top: 0; From ff34cd513c97bab049960981efe39499cc47e791 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 10 Jun 2016 19:49:52 -0400 Subject: [PATCH 125/136] Download correctly named GMS files Previously we would only update the filename when switching tabs of scenarios. Now the correct filename is given as soon as the user renames the project. --- src/mmw/js/src/modeling/templates/scenarioTabPanel.html | 2 +- src/mmw/js/src/modeling/views.js | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/mmw/js/src/modeling/templates/scenarioTabPanel.html b/src/mmw/js/src/modeling/templates/scenarioTabPanel.html index 3f77f8888..183a48894 100644 --- a/src/mmw/js/src/modeling/templates/scenarioTabPanel.html +++ b/src/mmw/js/src/modeling/templates/scenarioTabPanel.html @@ -17,7 +17,7 @@ {% if gwlfe %}
    - +
  • Export GMS
    • diff --git a/src/mmw/js/src/modeling/views.js b/src/mmw/js/src/modeling/views.js index f7900efdd..a4e39080b 100644 --- a/src/mmw/js/src/modeling/views.js +++ b/src/mmw/js/src/modeling/views.js @@ -345,14 +345,11 @@ var ScenarioTabPanelView = Marionette.ItemView.extend({ gwlfe = App.currentProject.get('model_package') === models.GWLFE && gis_data !== null && gis_data !== '{}' && - gis_data !== '', - filename = App.currentProject.get('name').replace(' ', '_') + - '__' + this.model.get('name').replace(' ', '_'); + gis_data !== ''; return { gwlfe: gwlfe, csrftoken: csrf.getToken(), - filename: filename, gis_data: gis_data, cid: this.model.cid, editable: isEditable(this.model), @@ -458,6 +455,10 @@ var ScenarioTabPanelView = Marionette.ItemView.extend({ // We can't download a file from an AJAX call. One either has to // load the data in an iframe, or submit a form that responds with // Content-Disposition: attachment. We prefer submitting a form. + var filename = App.currentProject.get('name').replace(' ', '_') + + '__' + this.model.get('name').replace(' ', '_'); + + this.ui.exportGmsForm.find('.gms-filename').val(filename); this.ui.exportGmsForm.submit(); } }); From a429d0b536c56ac3f4e4cdfbd7496fa916d9eac4 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Mon, 13 Jun 2016 15:48:37 -0400 Subject: [PATCH 126/136] Show error message when drawn AoI too big Since the amount of area we can analyze is limited by our computational capacity, we disallow areas of interest above a certain size. Previously we would simply cancel the drawing of a very large polygon with no user feedback. Now they user is given an error message when they draw one that is too big. --- src/mmw/js/src/draw/views.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mmw/js/src/draw/views.js b/src/mmw/js/src/draw/views.js index df90b60fb..450843033 100644 --- a/src/mmw/js/src/draw/views.js +++ b/src/mmw/js/src/draw/views.js @@ -67,6 +67,7 @@ function validateShape(polygon) { message += Math.floor(area) + ' square km were selected, '; message += 'but the maximum supported size is currently '; message += MAX_AREA + ' square km.'; + window.alert(message); d.reject(message); } else { d.resolve(polygon); From 6415d33d8136d9347e156384bb1fbf60c78c4225 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 10 Jun 2016 18:24:27 -0400 Subject: [PATCH 127/136] Correct AEU calculation Previously we were not multiplying the number by weight, which is needed for conversion of plain population to AEU. --- src/mmw/apps/modeling/mapshed/calcs.py | 15 +++++++++++---- src/mmw/mmw/settings/gwlfe_settings.py | 10 ++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/mmw/apps/modeling/mapshed/calcs.py b/src/mmw/apps/modeling/mapshed/calcs.py index 0d5e80afb..be09c1f4c 100644 --- a/src/mmw/apps/modeling/mapshed/calcs.py +++ b/src/mmw/apps/modeling/mapshed/calcs.py @@ -17,6 +17,7 @@ KV_FACTOR = settings.GWLFE_CONFIG['KvFactor'] MONTHS = settings.GWLFE_DEFAULTS['Month'] MONTHDAYS = settings.GWLFE_CONFIG['MonthDays'] +WEIGHTOF = settings.GWLFE_CONFIG['AvgAnimalWt'] LIVESTOCK = settings.GWLFE_CONFIG['Livestock'] POULTRY = settings.GWLFE_CONFIG['Poultry'] M3_PER_MGAL = 3785.41178 @@ -146,6 +147,8 @@ def kv_coefficient(area_pcts, season): def animal_energy_units(geom): """ Given a geometry, returns the total livestock and poultry AEUs within it + + Original at Class1.vb@1.3.0:9230-9247 """ sql = ''' WITH clipped_counties AS ( @@ -180,10 +183,14 @@ def animal_energy_units(geom): # Convert result to dictionary columns = [col[0] for col in cursor.description] values = cursor.fetchone() # Only one row since aggregate query - aeu = dict(zip(columns, values)) - - livestock_aeu = round(sum(aeu[animal] or 0 for animal in LIVESTOCK)) - poultry_aeu = round(sum(aeu[animal] or 0 for animal in POULTRY)) + population = dict(zip(columns, values)) + + livestock_aeu = round(sum(population.get(animal, 0) * + WEIGHTOF[animal] / 1000 + for animal in LIVESTOCK)) + poultry_aeu = round(sum(population.get(animal, 0) * + WEIGHTOF[animal] / 1000 + for animal in POULTRY)) return livestock_aeu, poultry_aeu diff --git a/src/mmw/mmw/settings/gwlfe_settings.py b/src/mmw/mmw/settings/gwlfe_settings.py index 93297fb14..a7ec7778e 100644 --- a/src/mmw/mmw/settings/gwlfe_settings.py +++ b/src/mmw/mmw/settings/gwlfe_settings.py @@ -454,6 +454,16 @@ 'KvFactor': 1.16, # Original at Class1.vb@1.3.0:4987 'Livestock': ['dairy_cows', 'beef_cows', 'hogs', 'sheep', 'horses'], 'Poultry': ['broilers', 'layers', 'turkeys'], + 'AvgAnimalWt': { # Original at Class1.vb@1.3.0:9048-9056 + 'dairy_cows': 640.0, + 'beef_cows': 360.0, + 'broilers': 0.9, + 'layers': 1.8, + 'hogs': 61.0, + 'sheep': 50.0, + 'horses': 500.0, + 'turkeys': 6.8, + }, 'ManureSpreadingLandUseIndices': [0, 1], # Land Use Indices where manure spreading applies. Currently Hay/Past and Cropland. 'AgriculturalNLCDCodes': [81, 82], # NLCD codes considered agricultural. Correspond to Hay/Past and Cropland 'MonthDays': [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31], From f89730a5dd9c98e5fbcfbed4645a298189eec9f7 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 14 Jun 2016 10:56:42 -0400 Subject: [PATCH 128/136] Make message friendlier --- src/mmw/js/src/draw/views.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/mmw/js/src/draw/views.js b/src/mmw/js/src/draw/views.js index 450843033..94fffee08 100644 --- a/src/mmw/js/src/draw/views.js +++ b/src/mmw/js/src/draw/views.js @@ -62,11 +62,10 @@ function validateShape(polygon) { d = new $.Deferred(); if (area > MAX_AREA) { - var message = ''; - message += 'Sorry, your Area of Interest is too large.\n\n'; - message += Math.floor(area) + ' square km were selected, '; - message += 'but the maximum supported size is currently '; - message += MAX_AREA + ' square km.'; + var message = 'Sorry, your Area of Interest is too large.\n\n' + + Math.floor(area).toLocaleString() + ' km² were selected, ' + + 'but the maximum supported size is currently ' + + MAX_AREA.toLocaleString() + ' km².'; window.alert(message); d.reject(message); } else { From 1e7e130d316615971050fdeebc4f575ce741b499 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Mon, 13 Jun 2016 16:48:27 -0400 Subject: [PATCH 129/136] Show explicit timeout message Previously both error and timeout conditions had the same messages for the user. This was confusing and unhelpful, especially as the most common case for error was a very large area of interest. By including this in our response to the user, we make the app easier to deal with and understand. --- src/mmw/js/src/analyze/views.js | 18 ++++++++---------- src/mmw/js/src/core/models.js | 10 +++++++++- src/mmw/js/src/modeling/models.js | 4 ++-- src/mmw/js/src/modeling/views.js | 8 ++++++-- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/mmw/js/src/analyze/views.js b/src/mmw/js/src/analyze/views.js index a3398347c..23310b897 100644 --- a/src/mmw/js/src/analyze/views.js +++ b/src/mmw/js/src/analyze/views.js @@ -137,15 +137,9 @@ var AnalyzeWindow = Marionette.LayoutView.extend({ onShow: function() { this.showAnalyzingMessage(); - var self = this; - this.model.fetchAnalysisIfNeeded() - .done(function() { - self.showDetailsRegion(); - }) - .fail(function() { - self.showErrorMessage(); - }); + .done(_.bind(this.showDetailsRegion, this)) + .fail(_.bind(this.showErrorMessage, this)); }, showAnalyzingMessage: function() { @@ -158,10 +152,14 @@ var AnalyzeWindow = Marionette.LayoutView.extend({ })); }, - showErrorMessage: function() { + showErrorMessage: function(err) { var messageModel = new coreModels.TaskMessageViewModel(); - messageModel.setError(); + if (err && err.timeout) { + messageModel.setTimeoutError(); + } else { + messageModel.setError(); + } this.detailsRegion.show(new coreViews.TaskMessageView({ model: messageModel diff --git a/src/mmw/js/src/core/models.js b/src/mmw/js/src/core/models.js index cc7f233b7..2c1608bfb 100644 --- a/src/mmw/js/src/core/models.js +++ b/src/mmw/js/src/core/models.js @@ -183,7 +183,7 @@ var TaskModel = Backbone.Model.extend({ // pollInterval has elapsed. var getResults = function() { if (elapsed >= self.get('timeout')) { - defer.reject(); + defer.reject({timeout: true}); return; } @@ -222,6 +222,14 @@ var TaskMessageViewModel = Backbone.Model.extend({ this.set('iconClasses', 'fa fa-exclamation-triangle'); }, + setTimeoutError: function() { + var message = 'Operation took too long
    ' + + 'Consider trying a smaller area of interest.'; + + this.set('message', message); + this.set('iconClasses', 'fa fa-exclamation-triangle'); + }, + setWorking: function(message) { this.set('message', message); this.set('iconClasses', 'fa fa-circle-o-notch fa-spin'); diff --git a/src/mmw/js/src/modeling/models.js b/src/mmw/js/src/modeling/models.js index 3a54d2545..8eb1f6965 100644 --- a/src/mmw/js/src/modeling/models.js +++ b/src/mmw/js/src/modeling/models.js @@ -349,9 +349,9 @@ var ProjectModel = Backbone.Model.extend({ promise.resolve(taskModel.get('result')); }, - pollFailure: function() { + pollFailure: function(err) { console.log('Failed to gather data required for MAPSHED'); - promise.reject(); + promise.reject(err); } }; diff --git a/src/mmw/js/src/modeling/views.js b/src/mmw/js/src/modeling/views.js index a4e39080b..d21e4ee77 100644 --- a/src/mmw/js/src/modeling/views.js +++ b/src/mmw/js/src/modeling/views.js @@ -801,8 +801,12 @@ var ResultsView = Marionette.LayoutView.extend({ aoi = App.map.get('areaOfInterest'), analyzeModel = analyzeModels.AnalyzeTaskModel.getSingleton(App, aoi), tmvModel = new coreModels.TaskMessageViewModel(), - errorHandler = function() { - tmvModel.setError(); + errorHandler = function(err) { + if (err && err.timeout) { + tmvModel.setTimeoutError(); + } else { + tmvModel.setError(); + } self.modelingRegion.show(new coreViews.TaskMessageView({ model: tmvModel })); }; From 6c52fbe8afc751b3ce3dd9acfaa9e01ff592bdad Mon Sep 17 00:00:00 2001 From: Hector Castro Date: Wed, 15 Jun 2016 12:55:10 -0400 Subject: [PATCH 130/136] Use datahub CDN endpoint for NLCD and SSURGO tiles Instead of serving tiles directly from Amazon S3, use the new CDN enabled datahub endpoint. See also: https://github.com/azavea/azavea-data-hub/pull/37 --- src/mmw/mmw/settings/layer_settings.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/mmw/mmw/settings/layer_settings.py b/src/mmw/mmw/settings/layer_settings.py index 1410b4d4c..b625f3fca 100644 --- a/src/mmw/mmw/settings/layer_settings.py +++ b/src/mmw/mmw/settings/layer_settings.py @@ -143,8 +143,7 @@ 'short_display': 'NLCD', 'helptext': 'National Land Cover Database defines' 'land use across the U.S.', - 'url': 'https://s3.amazonaws.com/com.azavea.datahub.tms/' - 'nlcd/{z}/{x}/{y}.png', + 'url': 'https://{s}.tiles.azavea.com/nlcd/{z}/{x}/{y}.png', 'raster': True, 'overlay': True, 'maxNativeZoom': 13, @@ -160,8 +159,7 @@ 'soil\'s runoff potential. The four Hydrologic Soils Groups' 'are A, B, C and D. Where A\'s generally have the smallest ' 'runoff potential and D\'s the greatest.', - 'url': 'https://s3.amazonaws.com/com.azavea.datahub.tms/' - 'ssurgo-hydro-group-30m/{z}/{x}/{y}.png', + 'url': 'https://{s}.tiles.azavea.com/ssurgo-hydro-group-30m/{z}/{x}/{y}.png', 'raster': True, 'overlay': True, 'maxNativeZoom': 13, From 466be7aa3d171716939465d052e42c3334d6aa16 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Wed, 15 Jun 2016 11:18:52 -0400 Subject: [PATCH 131/136] Show units in summary loads in GWLFE quality tables * Upgrade to gwlfe 0.5.0 which adds SummaryLoads which have Unit field --- .../modeling/gwlfe/quality/templates/table.html | 4 ++-- src/mmw/js/src/modeling/gwlfe/quality/views.js | 16 ++++++++++------ src/mmw/requirements/base.txt | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/mmw/js/src/modeling/gwlfe/quality/templates/table.html b/src/mmw/js/src/modeling/gwlfe/quality/templates/table.html index 1978e3ee7..bd8a92765 100644 --- a/src/mmw/js/src/modeling/gwlfe/quality/templates/table.html +++ b/src/mmw/js/src/modeling/gwlfe/quality/templates/table.html @@ -31,5 +31,5 @@
    Mean Flow: {{ MeanFlow|round(0)|toLocaleString }} (m3/year) and {{ MeanFlowPerSecond|round(0)|toLocaleString }} (m3/s)
    -{{ table(columnNames, landUseRows, true) }} -{{ table(columnNames, summaryRows, false) }} +{{ table(landUseColumns, landUseRows, true) }} +{{ table(summaryColumns, summaryRows, false) }} diff --git a/src/mmw/js/src/modeling/gwlfe/quality/views.js b/src/mmw/js/src/modeling/gwlfe/quality/views.js index a2cca409b..d239a7da5 100644 --- a/src/mmw/js/src/modeling/gwlfe/quality/views.js +++ b/src/mmw/js/src/modeling/gwlfe/quality/views.js @@ -46,20 +46,24 @@ var TableView = Marionette.CompositeView.extend({ }, templateHelpers: function() { - function makeRow(load) { - return [load.Source, load.Sediment, load.TotalN, load.TotalP]; + function makeRow(addUnit, load) { + var source = addUnit ? load.Source + ' (' + load.Unit + ')' + : load.Source; + return [source, load.Sediment, load.TotalN, load.TotalP]; } var result = this.model.get('result'), - columnNames = ['Sources', 'Sediment (kg)', 'Total Nitrogen (kg)', 'Total Phosphorus (kg)'], - landUseRows = _.map(result.Loads.slice(0,15), makeRow), - summaryRows = _.map(result.Loads.slice(15), makeRow); + landUseColumns = ['Sources', 'Sediment (kg)', 'Total Nitrogen (kg)', 'Total Phosphorus (kg)'], + landUseRows = _.map(result.Loads, _.partial(makeRow, false)), + summaryColumns = ['Sources', 'Sediment', 'Total Nitrogen', 'Total Phosphorus'], + summaryRows = _.map(result.SummaryLoads, _.partial(makeRow, true)); return { MeanFlow: result.MeanFlow, MeanFlowPerSecond: result.MeanFlowPerSecond, - columnNames: columnNames, + landUseColumns: landUseColumns, landUseRows: landUseRows, + summaryColumns: summaryColumns, summaryRows: summaryRows }; } diff --git a/src/mmw/requirements/base.txt b/src/mmw/requirements/base.txt index 5c86aeb57..0b6b13b42 100644 --- a/src/mmw/requirements/base.txt +++ b/src/mmw/requirements/base.txt @@ -10,7 +10,7 @@ python-omgeo==1.7.2 rauth==0.7.1 djangorestframework-gis==0.8.2 tr55==1.1.3 -gwlf-e==0.4.0 +gwlf-e==0.5.0 requests==2.9.1 rollbar==0.12.1 retry==0.9.1 From 2e1b3cc63dbd335517a1ddeda77d2ccacaf7d76a Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 24 Jun 2016 17:15:18 -0400 Subject: [PATCH 132/136] Fix Google Maps Key Previously it was embedded inside a json.dumps call. Since it is never used by any JavaScript code, we extract it from client_settings and include it directly. --- src/mmw/apps/core/templates/base.html | 2 +- src/mmw/apps/home/views.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mmw/apps/core/templates/base.html b/src/mmw/apps/core/templates/base.html index ec1809586..ab9490e2e 100644 --- a/src/mmw/apps/core/templates/base.html +++ b/src/mmw/apps/core/templates/base.html @@ -54,7 +54,7 @@ {% block javascript %}