From 4586d82120d5115dd31ccedfdec545f3172c2dc5 Mon Sep 17 00:00:00 2001 From: Anthony Aufdenkampe Date: Mon, 7 Feb 2022 11:29:16 -0600 Subject: [PATCH 01/17] Update Docs ReadMe to include link to Swagger and mention that we've maintained these to work since they were first developed. --- doc/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/README.md b/doc/README.md index 85b4c95db..fc45faafe 100644 --- a/doc/README.md +++ b/doc/README.md @@ -2,7 +2,11 @@ ## Python Jupyter notebooks demonstrating the use of the Model My Watershed geoprocessing API -2018-8-19. Created by [Emilio Mayorga](https://github.com/emiliom/), University of Washington. +The following Jupyter Notebooks provide example workflows for the Model My Watershed (ModelMW) public web services Application Programming Interface (API) for automating some of the workflows that are provided by the web application. + +Detailed ModelMW web service API documentation is provided at: https://modelmywatershed.org/api/docs/ + +Example notebooks were first created by [Emilio Mayorga](https://github.com/emiliom/) (University of Washington) on 2018-8-19, and have been maintained to work with subsequent changes to the API. 1. [MMW_API_watershed_demo.ipynb](https://github.com/WikiWatershed/model-my-watershed/blob/develop/doc/MMW_API_landproperties_demo.ipynb). Go [here](http://nbviewer.jupyter.org/github/WikiWatershed/model-my-watershed/blob/develop/doc/MMW_API_watershed_demo.ipynb) to view the functioning, interactive Folium map at the end of the notebook. From a50f08bfbfcb17e0009c9fb5621b4e9beed3548a Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 8 Feb 2022 22:19:34 +0000 Subject: [PATCH 02/17] Add public endpoint for MapShed This very basic endpoint can take a GeoJSON shape or a WKAoI id and starts a MapShed job, which can be queried using the returned job id. This endpoint currently does not support layer overrides. It is also missing API documentation. --- src/mmw/apps/geoprocessing_api/urls.py | 2 ++ src/mmw/apps/geoprocessing_api/views.py | 29 ++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/mmw/apps/geoprocessing_api/urls.py b/src/mmw/apps/geoprocessing_api/urls.py index 1fc7598f2..3e6f2c148 100644 --- a/src/mmw/apps/geoprocessing_api/urls.py +++ b/src/mmw/apps/geoprocessing_api/urls.py @@ -36,5 +36,7 @@ re_path(r'jobs/' + uuid_regex, get_job, name='get_job'), re_path(r'modeling/worksheet/$', views.start_modeling_worksheet, name='start_modeling_worksheet'), + re_path(r'modeling/mapshed/$', views.start_modeling_mapshed, + name='start_modeling_mapshed'), re_path(r'watershed/$', views.start_rwd, name='start_rwd'), ] diff --git a/src/mmw/apps/geoprocessing_api/views.py b/src/mmw/apps/geoprocessing_api/views.py index c1c5d6ff0..32b669708 100644 --- a/src/mmw/apps/geoprocessing_api/views.py +++ b/src/mmw/apps/geoprocessing_api/views.py @@ -21,7 +21,10 @@ from apps.core.decorators import log_request from apps.modeling import geoprocessing from apps.modeling.mapshed.calcs import streams -from apps.modeling.mapshed.tasks import nlcd_streams +from apps.modeling.mapshed.tasks import (collect_data, + convert_data, + multi_mapshed, + nlcd_streams) from apps.modeling.serializers import AoiSerializer from apps.geoprocessing_api import schemas, tasks @@ -1365,6 +1368,30 @@ def start_modeling_worksheet(request, format=None): ], area_of_interest, user) +@swagger_auto_schema(method='post', + manual_parameters=[schemas.WKAOI], + request_body=schemas.MULTIPOLYGON, + responses={200: schemas.JOB_STARTED_RESPONSE}) +@decorators.api_view(['POST']) +@decorators.authentication_classes((SessionAuthentication, + TokenAuthentication, )) +@decorators.permission_classes((IsAuthenticated, )) +@decorators.throttle_classes([BurstRateThrottle, SustainedRateThrottle]) +@log_request +def start_modeling_mapshed(request, format=None): + user = request.user if request.user.is_authenticated else None + area_of_interest, wkaoi = _parse_input(request) + + # TODO Add support for overriding layers + layer_overrides = {} + + return start_celery_job([ + multi_mapshed(area_of_interest, wkaoi, layer_overrides), + convert_data.s(wkaoi), + collect_data.s(area_of_interest, layer_overrides=layer_overrides), + ], area_of_interest, user) + + def _initiate_rwd_job_chain(location, snapping, simplify, data_source, job_id, testing=False): errback = save_job_error.s(job_id) From b52f7fe96d8bee7d0c34da6c57eba729afefe823 Mon Sep 17 00:00:00 2001 From: simonkassel Date: Wed, 16 Feb 2022 14:34:28 -0500 Subject: [PATCH 03/17] Add to setup instructions Add instructions for how to forward the appropriate AWS credentials to the worker vm --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index cea90b067..d112ddbd2 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,8 @@ First, ensure that you have a set of Amazon Web Services (AWS) credentials with $ aws configure --profile mmw-stg ``` +You will also need to set the MMW Datahub AWS credential as your default. These are stored in lastpass under the name `MMW Azavea DataHub AWS`. Ensure that the AWS credentials file has universal read permissions. + Ensure you have the [vagrant-disksize](https://github.com/sprotheroe/vagrant-disksize) plugin installed: ```bash From 194661420bb25a707d9ad84e031e587a3a9490f4 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Thu, 10 Feb 2022 23:59:41 +0000 Subject: [PATCH 04/17] Add MODELING_REQUEST schema, use in MapShed API Previously, all endpoints took a GeoJSON as the POST body, and any additional parameters, e.g. layer name or WKAoI, were path or query parameters. However, for modeling, the requests are much more complex. They may specify multiple layer overrides, HUC IDs, and other modifications. To allow for this, we create a new MODELING_REQUEST schema, which mimics the format used in the internal API, to allow for such complex requests. The Analyze endpoints will continue to use their simple forms. This creates a disparity in the payloads / expected formats between Analyze and Modeling. However, since there is only one existing modeling endpoint (/modeling/worksheet/) and this is used internally, making this switch will be minimally disruptive. --- src/mmw/apps/geoprocessing_api/schemas.py | 17 +++++++++++++++++ src/mmw/apps/geoprocessing_api/views.py | 6 +++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/mmw/apps/geoprocessing_api/schemas.py b/src/mmw/apps/geoprocessing_api/schemas.py index 993e9eef8..c99891171 100644 --- a/src/mmw/apps/geoprocessing_api/schemas.py +++ b/src/mmw/apps/geoprocessing_api/schemas.py @@ -178,3 +178,20 @@ }, required=['location'], ) + +MODELING_REQUEST = Schema( + title='Modeling Request', + type=TYPE_OBJECT, + properties={ + 'area_of_interest': MULTIPOLYGON, + 'wkaoi': Schema( + title='Well-Known Area of Interest', + type=TYPE_STRING, + example='huc12__55174', + description='The table and ID for a well-known area of interest, ' + 'such as a HUC. ' + 'Format "table__id", eg. "huc12__55174" will analyze ' + 'the HUC-12 City of Philadelphia-Schuylkill River.', + ), + }, +) diff --git a/src/mmw/apps/geoprocessing_api/views.py b/src/mmw/apps/geoprocessing_api/views.py index 32b669708..25e51ccd0 100644 --- a/src/mmw/apps/geoprocessing_api/views.py +++ b/src/mmw/apps/geoprocessing_api/views.py @@ -26,6 +26,7 @@ multi_mapshed, nlcd_streams) from apps.modeling.serializers import AoiSerializer +from apps.modeling.views import _parse_input as _parse_modeling_input from apps.geoprocessing_api import schemas, tasks from apps.geoprocessing_api.permissions import AuthTokenSerializerAuthentication # noqa @@ -1369,8 +1370,7 @@ def start_modeling_worksheet(request, format=None): @swagger_auto_schema(method='post', - manual_parameters=[schemas.WKAOI], - request_body=schemas.MULTIPOLYGON, + request_body=schemas.MODELING_REQUEST, responses={200: schemas.JOB_STARTED_RESPONSE}) @decorators.api_view(['POST']) @decorators.authentication_classes((SessionAuthentication, @@ -1380,7 +1380,7 @@ def start_modeling_worksheet(request, format=None): @log_request def start_modeling_mapshed(request, format=None): user = request.user if request.user.is_authenticated else None - area_of_interest, wkaoi = _parse_input(request) + area_of_interest, wkaoi = _parse_modeling_input(request.data) # TODO Add support for overriding layers layer_overrides = {} From d743a91f2bb7415b85bf562c40fb8a01dd5c5ef6 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 11 Feb 2022 01:17:16 +0000 Subject: [PATCH 05/17] Add documentation, support for layer_overrides Although all the layers used in MMW can be overridden, only the __LAND__ and __STREAMS__ layers have viable alternatives. Thus, we only document support for those two here. When alternatives are added for other layer types in the future, this documentation should be expanded to include them. --- src/mmw/apps/geoprocessing_api/schemas.py | 42 +++++++++++++++++++++++ src/mmw/apps/geoprocessing_api/views.py | 3 +- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/mmw/apps/geoprocessing_api/schemas.py b/src/mmw/apps/geoprocessing_api/schemas.py index c99891171..e5dec7fd8 100644 --- a/src/mmw/apps/geoprocessing_api/schemas.py +++ b/src/mmw/apps/geoprocessing_api/schemas.py @@ -179,6 +179,47 @@ required=['location'], ) +nlcd_override_allowed_values = '", "'.join([ + 'nlcd-2019-30m-epsg5070-512-byte', + 'nlcd-2016-30m-epsg5070-512-byte', + 'nlcd-2011-30m-epsg5070-512-byte', + 'nlcd-2006-30m-epsg5070-512-byte', + 'nlcd-2001-30m-epsg5070-512-byte', + 'nlcd-2011-30m-epsg5070-512-int8', +]) +LAYER_OVERRIDES = Schema( + title='Layer Overrides', + type=TYPE_OBJECT, + description='MMW combines different datasets in model runs. These have ' + 'default values, but can be overridden by specifying them ' + 'here. Only specify a value for the layers you want to ' + 'override.', + properties={ + '__LAND__': Schema( + type=TYPE_STRING, + example='nlcd-2019-30m-epsg5070-512-byte', + description='The NLCD layer to use. Valid options are: ' + f'"{nlcd_override_allowed_values}". All "-byte" ' + 'layers are from the NLCD19 product. The "-int8" ' + 'layer is from the NLCD11 product. The default value ' + 'is NLCD19 2019 "nlcd-2019-30m-epsg5070-512-byte".', + ), + '__STREAMS__': Schema( + type=TYPE_STRING, + example='nhdhr', + description='The streams layer to use. Valid options are: ' + '"nhdhr" for NHD High Resolution Streams, "nhd" for ' + 'NHD Medium Resolution Streams, and "drb" for ' + 'Delaware High Resolution. The area of interest must ' + 'be completely within the Delaware River Basin for ' + '"drb". "nhdhr" and "nhd" can be used within the ' + 'Continental United States. In some cases, "nhdhr" ' + 'may timeout. In such cases, "nhd" can be used as a ' + 'fallback. "nhdhr" is the default.' + ) + }, +) + MODELING_REQUEST = Schema( title='Modeling Request', type=TYPE_OBJECT, @@ -193,5 +234,6 @@ 'Format "table__id", eg. "huc12__55174" will analyze ' 'the HUC-12 City of Philadelphia-Schuylkill River.', ), + 'layer_overrides': LAYER_OVERRIDES, }, ) diff --git a/src/mmw/apps/geoprocessing_api/views.py b/src/mmw/apps/geoprocessing_api/views.py index 25e51ccd0..04ad16e64 100644 --- a/src/mmw/apps/geoprocessing_api/views.py +++ b/src/mmw/apps/geoprocessing_api/views.py @@ -1382,8 +1382,7 @@ def start_modeling_mapshed(request, format=None): user = request.user if request.user.is_authenticated else None area_of_interest, wkaoi = _parse_modeling_input(request.data) - # TODO Add support for overriding layers - layer_overrides = {} + layer_overrides = request.data.get('layer_overrides', {}) return start_celery_job([ multi_mapshed(area_of_interest, wkaoi, layer_overrides), From aae12fd84dabbb71cee1a175c0ffdff9cfb9f806 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 11 Feb 2022 20:15:42 +0000 Subject: [PATCH 06/17] Describe the MapShed endpoint --- src/mmw/apps/geoprocessing_api/views.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/mmw/apps/geoprocessing_api/views.py b/src/mmw/apps/geoprocessing_api/views.py index 04ad16e64..76d20c706 100644 --- a/src/mmw/apps/geoprocessing_api/views.py +++ b/src/mmw/apps/geoprocessing_api/views.py @@ -1379,6 +1379,21 @@ def start_modeling_worksheet(request, format=None): @decorators.throttle_classes([BurstRateThrottle, SustainedRateThrottle]) @log_request def start_modeling_mapshed(request, format=None): + """ + Starts a job to prepare an input payload for GWLF-E for a given area. + + Given an area of interest or a WKAoI id, gathers data from land, soil type, + groundwater nitrogen, available water capacity, K-factor, slope, soil + nitrogen, soil phosphorus, base-flow index, and stream datasets. + + By default, NLCD 2019 and NHD High Resolution Streams are used to prepare + the input payload. This can be changed using the `layer_overrides` option. + + Only one of `area_of_interest` or `wkaoi` should be provided. If both are + given, the `area_of_interest` will be used. + + The `result` should be used with the gwlf-e endpoint. + """ user = request.user if request.user.is_authenticated else None area_of_interest, wkaoi = _parse_modeling_input(request.data) From ee5990d2d5ab37d2a9ab07e90e0e5af729242017 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Fri, 11 Feb 2022 20:37:13 +0000 Subject: [PATCH 07/17] Rename endpoint to gwlf-e/prepare GWLF-E is a more recognizable name for the general public than MapShed. Also, grouping the two endpoints under one path will improve the semantic linking of the two. The other endpoint will be called gwlf-e/run. --- src/mmw/apps/geoprocessing_api/urls.py | 4 ++-- src/mmw/apps/geoprocessing_api/views.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mmw/apps/geoprocessing_api/urls.py b/src/mmw/apps/geoprocessing_api/urls.py index 3e6f2c148..076ccb266 100644 --- a/src/mmw/apps/geoprocessing_api/urls.py +++ b/src/mmw/apps/geoprocessing_api/urls.py @@ -36,7 +36,7 @@ re_path(r'jobs/' + uuid_regex, get_job, name='get_job'), re_path(r'modeling/worksheet/$', views.start_modeling_worksheet, name='start_modeling_worksheet'), - re_path(r'modeling/mapshed/$', views.start_modeling_mapshed, - name='start_modeling_mapshed'), + re_path(r'modeling/gwlf-e/prepare/$', views.start_modeling_gwlfe_prepare, + name='start_modeling_gwlfe_prepare'), re_path(r'watershed/$', views.start_rwd, name='start_rwd'), ] diff --git a/src/mmw/apps/geoprocessing_api/views.py b/src/mmw/apps/geoprocessing_api/views.py index 76d20c706..633a70987 100644 --- a/src/mmw/apps/geoprocessing_api/views.py +++ b/src/mmw/apps/geoprocessing_api/views.py @@ -1378,7 +1378,7 @@ def start_modeling_worksheet(request, format=None): @decorators.permission_classes((IsAuthenticated, )) @decorators.throttle_classes([BurstRateThrottle, SustainedRateThrottle]) @log_request -def start_modeling_mapshed(request, format=None): +def start_modeling_gwlfe_prepare(request, format=None): """ Starts a job to prepare an input payload for GWLF-E for a given area. @@ -1392,7 +1392,7 @@ def start_modeling_mapshed(request, format=None): Only one of `area_of_interest` or `wkaoi` should be provided. If both are given, the `area_of_interest` will be used. - The `result` should be used with the gwlf-e endpoint. + The `result` should be used with the gwlf-e/run endpoint. """ user = request.user if request.user.is_authenticated else None area_of_interest, wkaoi = _parse_modeling_input(request.data) From 9368acd5f554f9446408d662bd8b9d3b700fbff0 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 16 Feb 2022 20:10:19 +0000 Subject: [PATCH 08/17] Standardize job statuses We've been using string values for a long time, which have thankfully not diverged. By wrapping them in constants we ensure their standard use throughout. --- src/mmw/apps/core/models.py | 6 ++++++ src/mmw/apps/core/tasks.py | 6 +++--- src/mmw/apps/geoprocessing_api/schemas.py | 6 ++++-- src/mmw/apps/geoprocessing_api/views.py | 12 ++++++------ src/mmw/apps/modeling/tests.py | 4 ++-- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/mmw/apps/core/models.py b/src/mmw/apps/core/models.py index 6da97b5e5..b9d817335 100644 --- a/src/mmw/apps/core/models.py +++ b/src/mmw/apps/core/models.py @@ -5,6 +5,12 @@ AUTH_USER_MODEL = getattr(settings, 'AUTH_USER_MODEL', 'auth.User') +class JobStatus: + STARTED = 'started' + COMPLETE = 'complete' + FAILED = 'failed' + + class Job(models.Model): user = models.ForeignKey(AUTH_USER_MODEL, on_delete=models.SET_NULL, diff --git a/src/mmw/apps/core/tasks.py b/src/mmw/apps/core/tasks.py index 7532a1492..a6c17e872 100644 --- a/src/mmw/apps/core/tasks.py +++ b/src/mmw/apps/core/tasks.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from django.utils.timezone import now from celery import shared_task -from apps.core.models import Job +from apps.core.models import Job, JobStatus import json import logging @@ -29,7 +29,7 @@ def save_job_error(request, exc, traceback, job_id): job.error = exc job.traceback = traceback or 'No traceback' job.delivered_at = now() - job.status = 'failed' + job.status = JobStatus.FAILED job.save() except Exception as e: logger.error('Failed to save job error status. Job will appear hung.' @@ -47,5 +47,5 @@ def save_job_result(self, result, id, model_input): job.delivered_at = now() job.uuid = self.request.id job.model_input = model_input - job.status = 'complete' + job.status = JobStatus.COMPLETE job.save() diff --git a/src/mmw/apps/geoprocessing_api/schemas.py b/src/mmw/apps/geoprocessing_api/schemas.py index e5dec7fd8..a6d443c8f 100644 --- a/src/mmw/apps/geoprocessing_api/schemas.py +++ b/src/mmw/apps/geoprocessing_api/schemas.py @@ -8,6 +8,8 @@ from django.conf import settings +from apps.core.models import JobStatus + STREAM_DATASOURCE = Parameter( 'datasource', IN_PATH, @@ -125,7 +127,7 @@ properties={ 'job': Schema(type=TYPE_STRING, format=FORMAT_UUID, example='6e514e69-f46b-47e7-9476-c1f5be0bac01'), - 'status': Schema(type=TYPE_STRING, example='started'), + 'status': Schema(type=TYPE_STRING, example=JobStatus.STARTED), } ) @@ -135,7 +137,7 @@ properties={ 'job_uuid': Schema(type=TYPE_STRING, format=FORMAT_UUID, example='6e514e69-f46b-47e7-9476-c1f5be0bac01'), - 'status': Schema(type=TYPE_STRING, example='started'), + 'status': Schema(type=TYPE_STRING, example=JobStatus.STARTED), 'result': Schema(type=TYPE_OBJECT), 'error': Schema(type=TYPE_STRING), 'started': Schema(type=TYPE_STRING, format=FORMAT_DATETIME, diff --git a/src/mmw/apps/geoprocessing_api/views.py b/src/mmw/apps/geoprocessing_api/views.py index 633a70987..bbe3e4aac 100644 --- a/src/mmw/apps/geoprocessing_api/views.py +++ b/src/mmw/apps/geoprocessing_api/views.py @@ -15,7 +15,7 @@ from django.urls import reverse from django.contrib.gis.geos import GEOSGeometry -from apps.core.models import Job +from apps.core.models import Job, JobStatus from apps.core.tasks import (save_job_error, save_job_result) from apps.core.decorators import log_request @@ -221,7 +221,7 @@ def start_rwd(request, format=None): validate_rwd(location, data_source, snapping, simplify) job = Job.objects.create(created_at=created, result='', error='', - traceback='', user=user, status='started') + traceback='', user=user, status=JobStatus.STARTED) task_list = _initiate_rwd_job_chain(location, snapping, simplify, data_source, job.id) @@ -232,7 +232,7 @@ def start_rwd(request, format=None): return Response( { 'job': task_list.id, - 'status': 'started', + 'status': JobStatus.STARTED, }, headers={'Location': reverse('geoprocessing_api:get_job', args=[task_list.id])} @@ -1425,11 +1425,11 @@ def start_celery_job(task_list, job_input, user=None, link_error=True): :param job_input: Input to the first task, used in recording started jobs :param user: The user requesting the job. Optional. :param link_error: Whether or not to apply error handler to entire chain - :return: A Response contianing the job id, marked as 'started' + :return: A Response contianing the job id, marked as JobStatus.STARTED """ created = now() job = Job.objects.create(created_at=created, result='', error='', - traceback='', user=user, status='started', + traceback='', user=user, status=JobStatus.STARTED, model_input=job_input) success = save_job_result.s(job.id, job_input) @@ -1447,7 +1447,7 @@ def start_celery_job(task_list, job_input, user=None, link_error=True): return Response( { 'job': task_chain.id, - 'status': 'started', + 'status': JobStatus.STARTED, }, headers={'Location': reverse('geoprocessing_api:get_job', args=[task_chain.id])} diff --git a/src/mmw/apps/modeling/tests.py b/src/mmw/apps/modeling/tests.py index 3c4e99c3b..d755d225b 100644 --- a/src/mmw/apps/modeling/tests.py +++ b/src/mmw/apps/modeling/tests.py @@ -11,7 +11,7 @@ from django.test.utils import override_settings from django.utils.timezone import now -from apps.core.models import Job +from apps.core.models import Job, JobStatus from apps.modeling import tasks, views from apps.modeling.models import Scenario, WeatherType @@ -232,7 +232,7 @@ def test_tr55_job_runs_in_chain(self): 'Job not found') self.assertEqual(str(found_job.status), - 'complete', + JobStatus.COMPLETE, 'Job found but incomplete.') @override_settings(**CELERY_TEST_OVERRIDES) From 1760f10c956699f75cb2eabcdef967942c2eb245 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 16 Feb 2022 20:11:35 +0000 Subject: [PATCH 09/17] Add custom exceptions for two-step endpoints For modeling, sometimes an endpoint depends on the output of a preceding preparatory action. To handle cases when a gwlf-e/run is given a job that is not ready yet, or has failed, we add errors. They use the precondition required and precondition failed HTTP status codes. --- src/mmw/apps/geoprocessing_api/exceptions.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/mmw/apps/geoprocessing_api/exceptions.py diff --git a/src/mmw/apps/geoprocessing_api/exceptions.py b/src/mmw/apps/geoprocessing_api/exceptions.py new file mode 100644 index 000000000..f68b05faa --- /dev/null +++ b/src/mmw/apps/geoprocessing_api/exceptions.py @@ -0,0 +1,14 @@ +from rest_framework import status +from rest_framework.exceptions import APIException + + +class JobNotReadyError(APIException): + status_code = status.HTTP_428_PRECONDITION_REQUIRED + default_code = 'precondition_required' + default_detail = 'The prepare job has not finished yet.' + + +class JobFailedError(APIException): + status_code = status.HTTP_412_PRECONDITION_FAILED + default_code = 'precondition_failed' + default_detail = 'The prepare job has failed.' From 85168da95667ae2bfb3734fe271400a73be95933 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 16 Feb 2022 20:24:01 +0000 Subject: [PATCH 10/17] Add public GWLF-E Endpoint at gwlf-e/run This takes the output of gwlf-e/prepare and runs it. If the prepare job is not ready or has failed, returns an error. This endpoint is based on the internal start_gwlfe endpoint. Currently, it does not handle modifications. These will be added at a later date. --- src/mmw/apps/geoprocessing_api/schemas.py | 17 ++++++ src/mmw/apps/geoprocessing_api/urls.py | 2 + src/mmw/apps/geoprocessing_api/views.py | 65 ++++++++++++++++++++++- 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/src/mmw/apps/geoprocessing_api/schemas.py b/src/mmw/apps/geoprocessing_api/schemas.py index a6d443c8f..9396520e2 100644 --- a/src/mmw/apps/geoprocessing_api/schemas.py +++ b/src/mmw/apps/geoprocessing_api/schemas.py @@ -239,3 +239,20 @@ 'layer_overrides': LAYER_OVERRIDES, }, ) + +GWLFE_REQUEST = Schema( + title='GWLF-E Request', + type=TYPE_OBJECT, + properties={ + 'input': Schema( + type=TYPE_OBJECT, + description='The result of modeling/gwlf-e/prepare/', + ), + 'job_uuid': Schema( + type=TYPE_STRING, + format=FORMAT_UUID, + example='6e514e69-f46b-47e7-9476-c1f5be0bac01', + description='The job uuid of modeling/gwlf-e/prepare/', + ), + }, +) diff --git a/src/mmw/apps/geoprocessing_api/urls.py b/src/mmw/apps/geoprocessing_api/urls.py index 076ccb266..0a41c30be 100644 --- a/src/mmw/apps/geoprocessing_api/urls.py +++ b/src/mmw/apps/geoprocessing_api/urls.py @@ -38,5 +38,7 @@ name='start_modeling_worksheet'), re_path(r'modeling/gwlf-e/prepare/$', views.start_modeling_gwlfe_prepare, name='start_modeling_gwlfe_prepare'), + re_path(r'modeling/gwlf-e/run/$', views.start_modeling_gwlfe_run, + name='start_modeling_gwlfe_run'), re_path(r'watershed/$', views.start_rwd, name='start_rwd'), ] diff --git a/src/mmw/apps/geoprocessing_api/views.py b/src/mmw/apps/geoprocessing_api/views.py index bbe3e4aac..fa660bfde 100644 --- a/src/mmw/apps/geoprocessing_api/views.py +++ b/src/mmw/apps/geoprocessing_api/views.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +import json + from celery import chain from rest_framework.response import Response @@ -14,12 +16,15 @@ from django.utils.timezone import now from django.urls import reverse from django.contrib.gis.geos import GEOSGeometry +from django.shortcuts import get_object_or_404 from apps.core.models import Job, JobStatus from apps.core.tasks import (save_job_error, save_job_result) from apps.core.decorators import log_request from apps.modeling import geoprocessing +from apps.modeling.calcs import apply_gwlfe_modifications +from apps.modeling.tasks import run_gwlfe from apps.modeling.mapshed.calcs import streams from apps.modeling.mapshed.tasks import (collect_data, convert_data, @@ -28,7 +33,7 @@ from apps.modeling.serializers import AoiSerializer from apps.modeling.views import _parse_input as _parse_modeling_input -from apps.geoprocessing_api import schemas, tasks +from apps.geoprocessing_api import exceptions, schemas, tasks from apps.geoprocessing_api.permissions import AuthTokenSerializerAuthentication # noqa from apps.geoprocessing_api.throttling import (BurstRateThrottle, SustainedRateThrottle) @@ -1406,6 +1411,64 @@ def start_modeling_gwlfe_prepare(request, format=None): ], area_of_interest, user) +@swagger_auto_schema(method='post', + request_body=schemas.GWLFE_REQUEST, + responses={200: schemas.JOB_STARTED_RESPONSE}) +@decorators.api_view(['POST']) +@decorators.authentication_classes((SessionAuthentication, + TokenAuthentication, )) +@decorators.permission_classes((IsAuthenticated, )) +@decorators.throttle_classes([BurstRateThrottle, SustainedRateThrottle]) +@log_request +def start_modeling_gwlfe_run(request, format=None): + """ + Starts a job to GWLF-E for a given prepared input. + + Given an `input` JSON of the gwlf-e/prepare endpoint's `result`, or a + `job_uuid` of a gwlf-e/prepare job, runs GWLF-E and returns a JSON + dictionary containing mean flow: total and per-second; sediment, nitrogen, + and phosphorous loads for various sources; summarized loads; and monthly + values for average precipitation, evapotranspiration, groundwater, runoff, + stream flow, point source flow, and tile drain. + + If the specified `job_uuid` is not ready or has failed, returns an error. + + For more details on the GWLF-E package, please see: [https://github.com/WikiWatershed/gwlf-e](https://github.com/WikiWatershed/gwlf-e) # NOQA + """ + user = request.user if request.user.is_authenticated else None + model_input = request.data.get('input') + + if not model_input: + job_uuid = request.data.get('job_uuid') + + if not job_uuid: + raise ValidationError( + 'At least one of `input` or `job_uuid` must be specified') + + input_job = get_object_or_404(Job, uuid=job_uuid) + if input_job.status == JobStatus.STARTED: + raise exceptions.JobNotReadyError( + f'The prepare job {job_uuid} has not finished yet.') + + if input_job.status == JobStatus.FAILED: + raise exceptions.JobFailedError( + f'The prepare job {job_uuid} has failed.') + + model_input = json.loads(input_job.result) + + # TODO #3484 Validate model_input + + # TODO #3485 Implement modifications, hash + mods = [] + hash = '' + + modified_model_input = apply_gwlfe_modifications(model_input, mods) + + return start_celery_job([ + run_gwlfe.s(modified_model_input, hash) + ], model_input, user) + + def _initiate_rwd_job_chain(location, snapping, simplify, data_source, job_id, testing=False): errback = save_job_error.s(job_id) From 991e6dfb6b69f41353cbbc234c622e228235fa6b Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 22 Feb 2022 21:32:47 +0000 Subject: [PATCH 11/17] Amend prepare documentation to include job_uuid --- src/mmw/apps/geoprocessing_api/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mmw/apps/geoprocessing_api/views.py b/src/mmw/apps/geoprocessing_api/views.py index fa660bfde..aaeadf532 100644 --- a/src/mmw/apps/geoprocessing_api/views.py +++ b/src/mmw/apps/geoprocessing_api/views.py @@ -1397,7 +1397,9 @@ def start_modeling_gwlfe_prepare(request, format=None): Only one of `area_of_interest` or `wkaoi` should be provided. If both are given, the `area_of_interest` will be used. - The `result` should be used with the gwlf-e/run endpoint. + The `result` should be used with the gwlf-e/run endpoint, by sending at as + the `input`. Alternatively, the `job` UUID can be used as well by sending + it as the `job_uuid`. """ user = request.user if request.user.is_authenticated else None area_of_interest, wkaoi = _parse_modeling_input(request.data) From b93cfbc288c7202d52439f0c0a28d5fcd8e8436a Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 23 Feb 2022 16:48:07 +0000 Subject: [PATCH 12/17] Re-enable tests These are passing now. --- src/mmw/js/src/modeling/tests.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/mmw/js/src/modeling/tests.js b/src/mmw/js/src/modeling/tests.js index 3b2a35632..486801f14 100644 --- a/src/mmw/js/src/modeling/tests.js +++ b/src/mmw/js/src/modeling/tests.js @@ -866,8 +866,7 @@ describe('Modeling', function() { self.scenarioModel.fetchResults().pollingPromise.always(function() { assert(self.setNullResultsSpy.calledOnce, 'setNullResults should have been called once'); assert.isFalse(self.setResultsSpy.called, 'setResults should not have been called'); - // TODO: Re-enable tests https://github.com/WikiWatershed/model-my-watershed/issues/3442 - // assert(saveSpy.calledTwice, 'attemptSave should have been called twice'); + assert(saveSpy.calledTwice, 'attemptSave should have been called twice'); fetchResultsAssertions(self); done(); }); @@ -878,8 +877,7 @@ describe('Modeling', function() { self.scenarioModel.fetchResults().pollingPromise.always(function() { assert(self.setResultsSpy.calledOnce, 'setResults should have been called'); assert.isFalse(self.setNullResultsSpy.called, 'setNullResults should not have been called'); - // TODO: Re-enable tests https://github.com/WikiWatershed/model-my-watershed/issues/3442 - // assert(saveSpy.calledTwice, 'attemptSave should have been called twice'); + assert(saveSpy.calledTwice, 'attemptSave should have been called twice'); fetchResultsAssertions(self); done(); }); From 5237a76144c77781bb41d58af011d750bf55d5ed Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 1 Mar 2022 16:43:54 +0000 Subject: [PATCH 13/17] Use is_anonymous as value, not function This was changed in Django 1.11: https://github.com/django/django/commit/c1aec0feda73ede09503192a66f973598aef901d Refs #3463 --- src/mmw/apps/home/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mmw/apps/home/views.py b/src/mmw/apps/home/views.py index ac4106ae3..53cc0e6c8 100644 --- a/src/mmw/apps/home/views.py +++ b/src/mmw/apps/home/views.py @@ -185,7 +185,7 @@ def project_via_hydroshare_edit(request, resource): """ # Only logged in users are allowed to edit - if request.user.is_anonymous(): + if request.user.is_anonymous: return redirect('/error/hydroshare-not-logged-in') def callback(project_id): From 792e8a88f6c5a9a67922c9da6e76287492e89d1e Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 1 Mar 2022 17:04:37 +0000 Subject: [PATCH 14/17] Switch strings to bytes As required by recent Django upgrades --- src/mmw/apps/export/hydroshare.py | 6 +++--- src/mmw/apps/export/tasks.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mmw/apps/export/hydroshare.py b/src/mmw/apps/export/hydroshare.py index b6fa9708f..7fbac1241 100644 --- a/src/mmw/apps/export/hydroshare.py +++ b/src/mmw/apps/export/hydroshare.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import json -from io import StringIO +from io import BytesIO from zipfile import ZipFile from rauth import OAuth2Service from urllib.parse import urljoin, urlparse @@ -87,7 +87,7 @@ def add_files(self, resource_id, files): {'name': 'String', 'contents': 'String'} """ zippath = resource_id + '.zip' - stream = StringIO() + stream = BytesIO() # Zip all given files into the stream with ZipFile(stream, 'w') as zf: @@ -124,7 +124,7 @@ def get_project_snapshot(self, resource_id): snapshot_path = 'mmw_project_snapshot.json' try: stream = self.getResourceFile(resource_id, snapshot_path) - fio = StringIO() + fio = BytesIO() for chunk in stream: fio.write(chunk) diff --git a/src/mmw/apps/export/tasks.py b/src/mmw/apps/export/tasks.py index edf13224e..017adc004 100644 --- a/src/mmw/apps/export/tasks.py +++ b/src/mmw/apps/export/tasks.py @@ -89,7 +89,7 @@ def create_resource(user_id, project_id, params): for ext in SHAPEFILE_EXTENSIONS: filename = f'/tmp/{resource}.{ext}' - with open(filename) as shapefile: + with open(filename, 'rb') as shapefile: hs.addResourceFile(resource, shapefile, f'area-of-interest.{ext}') os.remove(filename) From 39a275a05d13acc0c4f2de5d2268af9b97ee530f Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 1 Mar 2022 17:05:06 +0000 Subject: [PATCH 15/17] Errors no longer have message attributes Also changed in one of the Django upgrades --- src/mmw/apps/modeling/calcs.py | 6 +++--- src/mmw/apps/user/views.py | 10 +++------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/mmw/apps/modeling/calcs.py b/src/mmw/apps/modeling/calcs.py index 4960bce1b..c049b8762 100644 --- a/src/mmw/apps/modeling/calcs.py +++ b/src/mmw/apps/modeling/calcs.py @@ -62,13 +62,13 @@ def err(msg, line=None): try: begyear = datetime.strptime(rows[1][0], DATE_FORMAT).year except ValueError as ve: - err(ve.message, 2) + err(ve, 2) return None, errs try: endyear = datetime.strptime(rows[-1][0], DATE_FORMAT).year except ValueError as ve: - err(ve.message, len(rows)) + err(ve, len(rows)) return None, errs year_range = endyear - begyear + 1 @@ -126,7 +126,7 @@ def err(msg, line=None): except Exception as e: # Record error with line. idx + 2 because idx starts at 0 while # line numbers start at 1, and we need to account for the header. - err(e.message, idx + 2) + err(e, idx + 2) previous_d = d diff --git a/src/mmw/apps/user/views.py b/src/mmw/apps/user/views.py index abb9f7b14..0c4d0ac99 100644 --- a/src/mmw/apps/user/views.py +++ b/src/mmw/apps/user/views.py @@ -172,7 +172,7 @@ def itsi_auth(request): itsi_user = session.get_user() except Exception as e: # In case we are unable to reach ITSI and get an unexpected response - rollbar.report_message(f'ITSI OAuth Error: {e.message}', 'error') + rollbar.report_message(f'ITSI OAuth Error: {e}', 'error') return redirect('/error/sso') user = authenticate(sso_id=itsi_user['id']) @@ -283,11 +283,7 @@ def concord_auth(request): concord_user = session.get_user() except Exception as e: # Report OAuth error - message = 'Concord OAuth Error' - if hasattr(e, 'message'): - message += f': {e.message}' - - rollbar.report_message(message, 'error') + rollbar.report_message(f'Concord OAuth Error: {e}', 'error') return redirect('/error/sso') user = authenticate(sso_id=concord_user['id']) @@ -477,7 +473,7 @@ def hydroshare_auth(request): token = hss.set_token_from_code(code, redirect_uri, request.user) context.update({'token': token.access_token}) except (IOError, RuntimeError) as e: - context.update({'error': e.message}) + context.update({'error': e}) if context.get('error') == 'invalid_client': rollbar.report_message('HydroShare OAuth credentials ' From 2181ec1a0eb60125b50cbc11c22e2420d7140223 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 1 Mar 2022 17:07:09 +0000 Subject: [PATCH 16/17] Don't use iframes for HydroShare login Previously we used iframes for logging users into HydroShare, which was convenient and friendly, and worked in all modern browsers. For IE which did not allow third party cookies in iframes, necessary for HydroShare login, we fellback to a full-page experience. As of recent times, Chrome no longer allows third party cookies in iframes either. So, we remove the iframe flow entirely, and replace all of it with the fallback. Not as elegant as before, but functional. Refs #3440 --- src/mmw/js/src/account/views.js | 25 +------------ src/mmw/js/src/core/modals/views.js | 56 ++++++++--------------------- 2 files changed, 15 insertions(+), 66 deletions(-) diff --git a/src/mmw/js/src/account/views.js b/src/mmw/js/src/account/views.js index 10ef0c080..3571c2d5a 100644 --- a/src/mmw/js/src/account/views.js +++ b/src/mmw/js/src/account/views.js @@ -12,7 +12,6 @@ var $ = require('jquery'), modalModels = require('../core/modals/models'), models = require('./models'), settings = require('../core/settings'), - utils = require('../core/utils'), containerTmpl = require('./templates/container.html'), pageToggleTmpl = require('./templates/pageToggle.html'), linkedAccountsTmpl = require('./templates/linkedAccounts.html'), @@ -38,29 +37,7 @@ var LinkedAccountsView = Marionette.ItemView.extend({ }, linkHydroShare: function() { - var self = this, - iframe = new modalViews.IframeView({ - model: new modalModels.IframeModel({ - href: '/user/hydroshare/login/', - signalSuccess: 'mmw-hydroshare-success', - signalFailure: 'mmw-hydroshare-failure', - signalCancel: 'mmw-hydroshare-cancel', - }) - }); - - if (utils.getIEVersion()) { - // Special handling for IE which does not support 3rd party - // cookies in iframes, which are necessary for the iframe - // workflow. - - window.location.href = window.location.origin + '/user/hydroshare/login/'; - } - - iframe.render(); - iframe.on('success', function() { - // Fetch user again to save new HydroShare Access state - self.model.fetch(); - }); + window.location.href = window.location.origin + '/user/hydroshare/login/'; }, unlinkHydroShare: function() { diff --git a/src/mmw/js/src/core/modals/views.js b/src/mmw/js/src/core/modals/views.js index f22df4e89..6c4b1608c 100644 --- a/src/mmw/js/src/core/modals/views.js +++ b/src/mmw/js/src/core/modals/views.js @@ -309,52 +309,24 @@ var MultiShareView = ModalBaseView.extend({ checkbox.prop('checked', false); - if (coreUtils.getIEVersion()) { - // Special handling for IE which does not support 3rd party - // cookies in iframes, which are necessary for the iframe - // workflow. - - var confirm = new ConfirmView({ - model: new models.ConfirmModel({ - titleText: 'Link Account to HydroShare', - question: 'You must link your Model My Watershed ' + - 'account to HydroShare before you can ' + - 'export this project. This can be done ' + - 'under "Linked Accounts" in your account ' + - 'page. Would you like to do this now?', - confirmLabel: 'Link Account to HydroShare', - cancelLabel: 'Not Now' - }) - }); - - confirm.render(); - - confirm.on('confirmation', function() { - router.navigate('/account', { trigger: true }); - self.hide(); - }); - - return; - } - - var iframe = new IframeView({ - model: new models.IframeModel({ - href: '/user/hydroshare/login/', - signalSuccess: 'mmw-hydroshare-success', - signalFailure: 'mmw-hydroshare-failure', - signalCancel: 'mmw-hydroshare-cancel', + var confirm = new ConfirmView({ + model: new models.ConfirmModel({ + titleText: 'Link Account to HydroShare', + question: 'You must link your Model My Watershed ' + + 'account to HydroShare before you can ' + + 'export this project. This can be done ' + + 'under "Linked Accounts" in your account ' + + 'page. Would you like to do this now?', + confirmLabel: 'Link Account to HydroShare', + cancelLabel: 'Not Now' }) }); - iframe.render(); - - iframe.on('success', function() { - // Fetch user again to save new HydroShare Access state - self.options.app.user.fetch(); - self.ui.hydroShareNotification.addClass('hidden'); + confirm.render(); - // Export to HydroShare - self.exportToHydroShare(); + confirm.on('confirmation', function() { + router.navigate('/account', { trigger: true }); + self.hide(); }); } }, From 6757f86f295c40608efdc9b92f521f009014fded Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 1 Mar 2022 17:12:12 +0000 Subject: [PATCH 17/17] Remove dead code for iframe handling Since this is no longer used, the iframe specific code is removed. --- .../user/templates/user/hydroshare-auth.html | 10 ++--- .../core/modals/templates/iframeModal.html | 5 --- src/mmw/js/src/core/modals/views.js | 44 ------------------- 3 files changed, 4 insertions(+), 55 deletions(-) delete mode 100644 src/mmw/js/src/core/modals/templates/iframeModal.html diff --git a/src/mmw/apps/user/templates/user/hydroshare-auth.html b/src/mmw/apps/user/templates/user/hydroshare-auth.html index 85f3d9aa3..eadefbbf7 100644 --- a/src/mmw/apps/user/templates/user/hydroshare-auth.html +++ b/src/mmw/apps/user/templates/user/hydroshare-auth.html @@ -40,11 +40,10 @@

Success!

parent.postMessage(message, window.location.origin); {% if not error %} - // Redirect to home if not in iframe and already closed - // since iframes are closed after 500ms in core.modals.views.IframeView. + // Redirect to home setTimeout(function() { window.location.href = window.location.origin + '/account/'; - }, 750); + }, 500); {% endif %} }); @@ -54,11 +53,10 @@

Success!

// Disable button to prevent double-clicking document.getElementById('hydroshare-cancel').disabled = true; - // Redirect to home if not in iframe and already closed - // since iframes are closed after 500ms in core.modals.views.IframeView. + // Redirect to home setTimeout(function() { window.location.href = window.location.origin + '/account/'; - }, 750); + }, 500); }); {% endif %} diff --git a/src/mmw/js/src/core/modals/templates/iframeModal.html b/src/mmw/js/src/core/modals/templates/iframeModal.html deleted file mode 100644 index bf86447a0..000000000 --- a/src/mmw/js/src/core/modals/templates/iframeModal.html +++ /dev/null @@ -1,5 +0,0 @@ -
-
-