From 2f1d51dfea81091d092f2bb1a3f06afc5e67b799 Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Wed, 22 Jan 2025 16:12:12 +0100 Subject: [PATCH 1/8] PB-1169: Spec search forecast fields Add forecast properties to search filter for GET and POST. Refactor forecast:variable and forecast:perturbed descriptions into schema properties to be reused. --- spec/components/parameters.yaml | 44 +++++++++ spec/components/schemas.yaml | 59 +++++++++--- spec/openapi.yaml | 5 + spec/static/spec/v1/openapi.yaml | 96 +++++++++++++++++-- spec/static/spec/v1/openapitransactional.yaml | 96 +++++++++++++++++-- 5 files changed, 267 insertions(+), 33 deletions(-) diff --git a/spec/components/parameters.yaml b/spec/components/parameters.yaml index 63ca9bbd..84b8ea24 100644 --- a/spec/components/parameters.yaml +++ b/spec/components/parameters.yaml @@ -117,3 +117,47 @@ components: required: false schema: type: string + forecast_reference_datetime: + explode: false + in: query + name: forecast_reference_datetime + required: false + schema: + $ref: "./schemas.yaml#/components/schemas/datetimeQuery" + example: 2018-02-12T00%3A00%3A00Z%2F2018-03-18T12%3A31%3A12Z + style: form + forecast_horizon: + explode: false + in: query + name: forecast_horizon + required: false + schema: + $ref: "./schemas.yaml#/components/schemas/duration" + example: P3DT6H + style: form + forecast_duration: + explode: false + in: query + name: forecast_duration + required: false + schema: + $ref: "./schemas.yaml#/components/schemas/duration" + example: P3DT6H + style: form + forecast_variable: + explode: false + in: query + name: forecast_variable + required: false + schema: + $ref: "./schemas.yaml#/components/schemas/forecast_variable" + example: air_temperature + style: form + forecast_perturbed: + explode: false + in: query + name: forecast_perturbed + required: false + schema: + $ref: "./schemas.yaml#/components/schemas/forecast_perturbed" + style: form diff --git a/spec/components/schemas.yaml b/spec/components/schemas.yaml index b877cd12..5f3cee07 100644 --- a/spec/components/schemas.yaml +++ b/spec/components/schemas.yaml @@ -1096,22 +1096,9 @@ components: forecast:duration: $ref: "#/components/schemas/duration" forecast:variable: - description: >- - Name of the model variable that corresponds to the data. The variables - should correspond to the - [CF Standard Names](https://cfconventions.org/Data/cf-standard-names/current/build/cf-standard-name-table.html), - e.g. `air_temperature` for the air temperature. - example: air_temperature - type: string - nullable: true + $ref: "#/components/schemas/forecast_variable" forecast:perturbed: - description: >- - Denotes whether the data corresponds to the control run (`false`) or - perturbed runs (`true`). The property needs to be specified in both - cases as no default value is specified and as such the meaning is "unknown" - in case it's missing. - type: boolean - nullable: true + $ref: "#/components/schemas/forecast_perturbed" required: - created - updated @@ -1494,6 +1481,43 @@ components: query: $ref: "#/components/schemas/query" type: object + forecast_reference_datetimeFilter: + properties: + forecast:reference_datetime: + $ref: "#/components/schemas/datetimeQuery" + forecast_horizonFilter: + properties: + forecast:horizon: + $ref: "#/components/schemas/duration" + forecast_durationFilter: + properties: + forecast:duration: + $ref: "#/components/schemas/duration" + forecast_variableFilter: + properties: + forecast:variable: + $ref: "#/components/schemas/forecast_variable" + forecast_variable: + description: >- + Name of the model variable that corresponds to the data. The variables + should correspond to the + [CF Standard Names](https://cfconventions.org/Data/cf-standard-names/current/build/cf-standard-name-table.html), + e.g. `air_temperature` for the air temperature. + example: air_temperature + type: string + nullable: true + forecast_perturbedFilter: + properties: + forecast:perturbed: + $ref: "#/components/schemas/forecast_perturbed" + forecast_perturbed: + description: >- + Denotes whether the data corresponds to the control run (`false`) or + perturbed runs (`true`). The property needs to be specified in both + cases as no default value is specified and as such the meaning is "unknown" + in case it's missing. + type: boolean + nullable: true queryProp: anyOf: - description: >- @@ -1588,6 +1612,11 @@ components: - $ref: "#/components/schemas/collectionsFilter" - $ref: "#/components/schemas/idsFilter" - $ref: "#/components/schemas/limitFilter" + - $ref: "#/components/schemas/forecast_reference_datetimeFilter" + - $ref: "#/components/schemas/forecast_horizonFilter" + - $ref: "#/components/schemas/forecast_durationFilter" + - $ref: "#/components/schemas/forecast_variableFilter" + - $ref: "#/components/schemas/forecast_perturbedFilter" description: The search criteria type: object stac_version: diff --git a/spec/openapi.yaml b/spec/openapi.yaml index 80c8d4ad..befc9641 100644 --- a/spec/openapi.yaml +++ b/spec/openapi.yaml @@ -283,6 +283,11 @@ paths: - $ref: "./components/parameters.yaml#/components/parameters/limit" - $ref: "./components/parameters.yaml#/components/parameters/ids" - $ref: "./components/parameters.yaml#/components/parameters/collectionsArray" + - $ref: "./components/parameters.yaml#/components/parameters/forecast_reference_datetime" + - $ref: "./components/parameters.yaml#/components/parameters/forecast_horizon" + - $ref: "./components/parameters.yaml#/components/parameters/forecast_duration" + - $ref: "./components/parameters.yaml#/components/parameters/forecast_variable" + - $ref: "./components/parameters.yaml#/components/parameters/forecast_perturbed" responses: "200": content: diff --git a/spec/static/spec/v1/openapi.yaml b/spec/static/spec/v1/openapi.yaml index 552bd542..24663687 100644 --- a/spec/static/spec/v1/openapi.yaml +++ b/spec/static/spec/v1/openapi.yaml @@ -115,6 +115,50 @@ components: required: false schema: type: string + forecast_reference_datetime: + explode: false + in: query + name: forecast_reference_datetime + required: false + schema: + $ref: "#/components/schemas/datetimeQuery" + example: 2018-02-12T00%3A00%3A00Z%2F2018-03-18T12%3A31%3A12Z + style: form + forecast_horizon: + explode: false + in: query + name: forecast_horizon + required: false + schema: + $ref: "#/components/schemas/duration" + example: P3DT6H + style: form + forecast_duration: + explode: false + in: query + name: forecast_duration + required: false + schema: + $ref: "#/components/schemas/duration" + example: P3DT6H + style: form + forecast_variable: + explode: false + in: query + name: forecast_variable + required: false + schema: + $ref: "#/components/schemas/forecast_variable" + example: air_temperature + style: form + forecast_perturbed: + explode: false + in: query + name: forecast_perturbed + required: false + schema: + $ref: "#/components/schemas/forecast_perturbed" + style: form responses: Collection: headers: @@ -1331,16 +1375,9 @@ components: forecast:duration: $ref: "#/components/schemas/duration" forecast:variable: - description: >- - Name of the model variable that corresponds to the data. The variables should correspond to the [CF Standard Names](https://cfconventions.org/Data/cf-standard-names/current/build/cf-standard-name-table.html), e.g. `air_temperature` for the air temperature. - example: air_temperature - type: string - nullable: true + $ref: "#/components/schemas/forecast_variable" forecast:perturbed: - description: >- - Denotes whether the data corresponds to the control run (`false`) or perturbed runs (`true`). The property needs to be specified in both cases as no default value is specified and as such the meaning is "unknown" in case it's missing. - type: boolean - nullable: true + $ref: "#/components/schemas/forecast_perturbed" required: - created - updated @@ -1697,6 +1734,37 @@ components: query: $ref: "#/components/schemas/query" type: object + forecast_reference_datetimeFilter: + properties: + forecast:reference_datetime: + $ref: "#/components/schemas/datetimeQuery" + forecast_horizonFilter: + properties: + forecast:horizon: + $ref: "#/components/schemas/duration" + forecast_durationFilter: + properties: + forecast:duration: + $ref: "#/components/schemas/duration" + forecast_variableFilter: + properties: + forecast:variable: + $ref: "#/components/schemas/forecast_variable" + forecast_variable: + description: >- + Name of the model variable that corresponds to the data. The variables should correspond to the [CF Standard Names](https://cfconventions.org/Data/cf-standard-names/current/build/cf-standard-name-table.html), e.g. `air_temperature` for the air temperature. + example: air_temperature + type: string + nullable: true + forecast_perturbedFilter: + properties: + forecast:perturbed: + $ref: "#/components/schemas/forecast_perturbed" + forecast_perturbed: + description: >- + Denotes whether the data corresponds to the control run (`false`) or perturbed runs (`true`). The property needs to be specified in both cases as no default value is specified and as such the meaning is "unknown" in case it's missing. + type: boolean + nullable: true queryProp: anyOf: - description: >- @@ -1783,6 +1851,11 @@ components: - $ref: "#/components/schemas/collectionsFilter" - $ref: "#/components/schemas/idsFilter" - $ref: "#/components/schemas/limitFilter" + - $ref: "#/components/schemas/forecast_reference_datetimeFilter" + - $ref: "#/components/schemas/forecast_horizonFilter" + - $ref: "#/components/schemas/forecast_durationFilter" + - $ref: "#/components/schemas/forecast_variableFilter" + - $ref: "#/components/schemas/forecast_perturbedFilter" description: The search criteria type: object stac_version: @@ -2091,6 +2164,11 @@ paths: - $ref: "#/components/parameters/limit" - $ref: "#/components/parameters/ids" - $ref: "#/components/parameters/collectionsArray" + - $ref: "#/components/parameters/forecast_reference_datetime" + - $ref: "#/components/parameters/forecast_horizon" + - $ref: "#/components/parameters/forecast_duration" + - $ref: "#/components/parameters/forecast_variable" + - $ref: "#/components/parameters/forecast_perturbed" responses: "200": content: diff --git a/spec/static/spec/v1/openapitransactional.yaml b/spec/static/spec/v1/openapitransactional.yaml index 7ab6759e..e3230d2f 100644 --- a/spec/static/spec/v1/openapitransactional.yaml +++ b/spec/static/spec/v1/openapitransactional.yaml @@ -115,6 +115,50 @@ components: required: false schema: type: string + forecast_reference_datetime: + explode: false + in: query + name: forecast_reference_datetime + required: false + schema: + $ref: "#/components/schemas/datetimeQuery" + example: 2018-02-12T00%3A00%3A00Z%2F2018-03-18T12%3A31%3A12Z + style: form + forecast_horizon: + explode: false + in: query + name: forecast_horizon + required: false + schema: + $ref: "#/components/schemas/duration" + example: P3DT6H + style: form + forecast_duration: + explode: false + in: query + name: forecast_duration + required: false + schema: + $ref: "#/components/schemas/duration" + example: P3DT6H + style: form + forecast_variable: + explode: false + in: query + name: forecast_variable + required: false + schema: + $ref: "#/components/schemas/forecast_variable" + example: air_temperature + style: form + forecast_perturbed: + explode: false + in: query + name: forecast_perturbed + required: false + schema: + $ref: "#/components/schemas/forecast_perturbed" + style: form assetId: name: assetId in: path @@ -1435,16 +1479,9 @@ components: forecast:duration: $ref: "#/components/schemas/duration" forecast:variable: - description: >- - Name of the model variable that corresponds to the data. The variables should correspond to the [CF Standard Names](https://cfconventions.org/Data/cf-standard-names/current/build/cf-standard-name-table.html), e.g. `air_temperature` for the air temperature. - example: air_temperature - type: string - nullable: true + $ref: "#/components/schemas/forecast_variable" forecast:perturbed: - description: >- - Denotes whether the data corresponds to the control run (`false`) or perturbed runs (`true`). The property needs to be specified in both cases as no default value is specified and as such the meaning is "unknown" in case it's missing. - type: boolean - nullable: true + $ref: "#/components/schemas/forecast_perturbed" required: - created - updated @@ -1805,6 +1842,37 @@ components: query: $ref: "#/components/schemas/query" type: object + forecast_reference_datetimeFilter: + properties: + forecast:reference_datetime: + $ref: "#/components/schemas/datetimeQuery" + forecast_horizonFilter: + properties: + forecast:horizon: + $ref: "#/components/schemas/duration" + forecast_durationFilter: + properties: + forecast:duration: + $ref: "#/components/schemas/duration" + forecast_variableFilter: + properties: + forecast:variable: + $ref: "#/components/schemas/forecast_variable" + forecast_variable: + description: >- + Name of the model variable that corresponds to the data. The variables should correspond to the [CF Standard Names](https://cfconventions.org/Data/cf-standard-names/current/build/cf-standard-name-table.html), e.g. `air_temperature` for the air temperature. + example: air_temperature + type: string + nullable: true + forecast_perturbedFilter: + properties: + forecast:perturbed: + $ref: "#/components/schemas/forecast_perturbed" + forecast_perturbed: + description: >- + Denotes whether the data corresponds to the control run (`false`) or perturbed runs (`true`). The property needs to be specified in both cases as no default value is specified and as such the meaning is "unknown" in case it's missing. + type: boolean + nullable: true queryProp: anyOf: - description: >- @@ -1891,6 +1959,11 @@ components: - $ref: "#/components/schemas/collectionsFilter" - $ref: "#/components/schemas/idsFilter" - $ref: "#/components/schemas/limitFilter" + - $ref: "#/components/schemas/forecast_reference_datetimeFilter" + - $ref: "#/components/schemas/forecast_horizonFilter" + - $ref: "#/components/schemas/forecast_durationFilter" + - $ref: "#/components/schemas/forecast_variableFilter" + - $ref: "#/components/schemas/forecast_perturbedFilter" description: The search criteria type: object stac_version: @@ -3511,6 +3584,11 @@ paths: - $ref: "#/components/parameters/limit" - $ref: "#/components/parameters/ids" - $ref: "#/components/parameters/collectionsArray" + - $ref: "#/components/parameters/forecast_reference_datetime" + - $ref: "#/components/parameters/forecast_horizon" + - $ref: "#/components/parameters/forecast_duration" + - $ref: "#/components/parameters/forecast_variable" + - $ref: "#/components/parameters/forecast_perturbed" responses: "200": content: From ad90dab334ba4b96b1f59f23183f671bd9e98a19 Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Thu, 23 Jan 2025 10:36:14 +0100 Subject: [PATCH 2/8] PB-1169: Search forecast fields Add filters for forecast properties to search endpoints. Can filter for forecast_reference_datetime either by exact value or by a range. --- app/stac_api/managers.py | 104 +++++++++ app/stac_api/validators_serializer.py | 14 +- app/stac_api/views/general.py | 13 ++ .../tests_10/sample_data/item_samples.py | 52 ++++- app/tests/tests_10/test_search_endpoint.py | 199 ++++++++++++++++++ 5 files changed, 380 insertions(+), 2 deletions(-) diff --git a/app/stac_api/managers.py b/app/stac_api/managers.py index 7ee411e5..8b0df7f1 100644 --- a/app/stac_api/managers.py +++ b/app/stac_api/managers.py @@ -105,6 +105,47 @@ def _filter_by_datetime_range(self, start_datetime, end_datetime): ) ) + def filter_by_forecast_reference_datetime(self, date_time): + '''Filter a queryset by forecast reference datetime + + Args: + queryset: + A django queryset (https://docs.djangoproject.com/en/3.0/ref/models/querysets/) + date_time: + A string + + Returns: + The queryset filtered by date_time + ''' + start, end = self._parse_datetime_query(date_time) + if end is not None: + return self._filter_by_forecast_reference_datetime_range(start, end) + return self.filter(forecast_reference_datetime=start) + + def _filter_by_forecast_reference_datetime_range(self, start_datetime, end_datetime): + '''Filter a queryset by forecast reference datetime range + + Helper function of filter_by_forecast_reference_datetime + + Args: + queryset: + A django queryset (https://docs.djangoproject.com/en/3.0/ref/models/querysets/) + start_datetime: + A string with the start datetime + end_datetime: + A string with the end datetime + Returns: + The queryset filtered by datetime range + ''' + if start_datetime == '..': + # open start range + return self.filter(forecast_reference_datetime__lte=end_datetime) + if end_datetime == '..': + # open end range + return self.filter(forecast_reference_datetime__gte=start_datetime) + # else fixed range + return self.filter(forecast_reference_datetime__range=(start_datetime, end_datetime)) + def _parse_datetime_query(self, date_time): '''Parse the datetime query as specified in the api-spec.md. @@ -189,6 +230,54 @@ def filter_by_intersects(self, intersects): the_geom = GEOSGeometry(intersects) return self.filter(geometry__intersects=the_geom) + def filter_by_forecast_horizon(self, duration): + '''Filter by forecast horizon + + Args: + duration: string + ISO 8601 duration string + + Returns: + queryset filtered by forecast horizon + ''' + return self.filter(forecast_horizon=duration) + + def filter_by_forecast_duration(self, duration): + '''Filter by forecast duration + + Args: + duration: string + ISO 8601 duration string + + Returns: + queryset filtered by forecast duration + ''' + return self.filter(forecast_duration=duration) + + def filter_by_forecast_variable(self, val): + '''Filter by forecast variable + + Args: + val: string + foracast variable value + + Returns: + queryset filtered by forecast variable + ''' + return self.filter(forecast_variable=val) + + def filter_by_forecast_perturbed(self, pert): + '''Filter by forecast perturbed + + Args: + pert: boolean + foracast perturbed + + Returns: + queryset filtered by forecast perturbed + ''' + return self.filter(forecast_perturbed=pert) + def filter_by_query(self, query): '''Filter by the query parameter @@ -236,6 +325,21 @@ def filter_by_item_name(self, item_name_array): def filter_by_query(self, query): return self.get_queryset().filter_by_query(query) + def filter_by_forecast_reference_datetime(self, date_time): + return self.get_queryset().filter_by_forecast_reference_datetime(date_time) + + def filter_by_forecast_horizon(self, duration): + return self.get_queryset().filter_by_forecast_horizon(duration) + + def filter_by_forecast_duration(self, duration): + return self.get_queryset().filter_by_forecast_duration(duration) + + def filter_by_forecast_variable(self, val): + return self.get_queryset().filter_by_forecast_variable(val) + + def filter_by_forecast_perturbed(self, pert): + return self.get_queryset().filter_by_forecast_perturbed(pert) + class AssetUploadQuerySet(models.QuerySet): diff --git a/app/stac_api/validators_serializer.py b/app/stac_api/validators_serializer.py index d0114e6b..56a7d430 100644 --- a/app/stac_api/validators_serializer.py +++ b/app/stac_api/validators_serializer.py @@ -351,7 +351,19 @@ def validate_query_parameters_post_search(self, query_param): Copy of the harmonized QueryDict ''' accepted_query_parameters = [ - "bbox", "collections", "datetime", "ids", "intersects", "limit", "cursor", "query" + "bbox", + "collections", + "datetime", + "ids", + "intersects", + "limit", + "cursor", + "query", + "forecast_reference_datetime", + "forecast_horizon", + "forecast_duration", + "forecast_variable", + "forecast_perturbed" ] wrong_query_parameters = set(query_param.keys()).difference(set(accepted_query_parameters)) if wrong_query_parameters: diff --git a/app/stac_api/views/general.py b/app/stac_api/views/general.py index 3b4d3a71..498766ea 100644 --- a/app/stac_api/views/general.py +++ b/app/stac_api/views/general.py @@ -69,6 +69,7 @@ class SearchList(generics.GenericAPIView, mixins.ListModelMixin): # we must use the pk as ordering attribute, otherwise the cursor pagination will not work ordering = ['pk'] + # pylint: disable=too-many-branches def get_queryset(self): queryset = Item.objects.filter(collection__published=True ).prefetch_related('assets', 'links') @@ -92,6 +93,18 @@ def get_queryset(self): queryset = queryset.filter_by_query(dict_query) if 'intersects' in query_param: queryset = queryset.filter_by_intersects(json.dumps(query_param['intersects'])) + if 'forecast_reference_datetime' in query_param: + queryset = queryset.filter_by_forecast_reference_datetime( + query_param['forecast_reference_datetime'] + ) + if 'forecast_horizon' in query_param: + queryset = queryset.filter_by_forecast_horizon(query_param['forecast_horizon']) + if 'forecast_duration' in query_param: + queryset = queryset.filter_by_forecast_duration(query_param['forecast_duration']) + if 'forecast_variable' in query_param: + queryset = queryset.filter_by_forecast_variable(query_param['forecast_variable']) + if 'forecast_perturbed' in query_param: + queryset = queryset.filter_by_forecast_perturbed(query_param['forecast_perturbed']) if settings.DEBUG_ENABLE_DB_EXPLAIN_ANALYZE: logger.debug( diff --git a/app/tests/tests_10/sample_data/item_samples.py b/app/tests/tests_10/sample_data/item_samples.py index 999696a8..5b4dd12a 100644 --- a/app/tests/tests_10/sample_data/item_samples.py +++ b/app/tests/tests_10/sample_data/item_samples.py @@ -218,5 +218,55 @@ 'name': 'item-covers_switzerland', 'geometry': geometries['covers-switzerland'], 'properties_datetime': fromisoformat('2020-10-28T13:05:10Z') - } + }, + 'item-forecast-1': { + 'name': 'item-forecast-1', + 'geometry': geometries['covers-switzerland'], + 'properties_datetime': fromisoformat('2020-10-28T13:05:10Z'), + 'forecast_reference_datetime': fromisoformat('2025-01-01T13:05:10Z'), + 'forecast_horizon': 'PT6H', + 'forecast_duration': 'PT12H', + 'forecast_variable': 'T', + 'forecast_perturbed': 'False', + }, + 'item-forecast-2': { + 'name': 'item-forecast-2', + 'geometry': geometries['covers-switzerland'], + 'properties_datetime': fromisoformat('2020-10-28T13:05:10Z'), + 'forecast_reference_datetime': fromisoformat('2025-02-01T13:05:10Z'), + 'forecast_horizon': 'PT6H', + 'forecast_duration': 'PT12H', + 'forecast_variable': 'T', + 'forecast_perturbed': 'False', + }, + 'item-forecast-3': { + 'name': 'item-forecast-3', + 'geometry': geometries['covers-switzerland'], + 'properties_datetime': fromisoformat('2020-10-28T13:05:10Z'), + 'forecast_reference_datetime': fromisoformat('2025-02-01T13:05:10Z'), + 'forecast_horizon': 'PT3H', + 'forecast_duration': 'PT6H', + 'forecast_variable': 'T', + 'forecast_perturbed': 'False', + }, + 'item-forecast-4': { + 'name': 'item-forecast-4', + 'geometry': geometries['covers-switzerland'], + 'properties_datetime': fromisoformat('2020-10-28T13:05:10Z'), + 'forecast_reference_datetime': fromisoformat('2025-02-01T13:05:10Z'), + 'forecast_horizon': 'PT6H', + 'forecast_duration': 'PT12H', + 'forecast_variable': 'air_temperature', + 'forecast_perturbed': 'True', + }, + 'item-forecast-5': { + 'name': 'item-forecast-5', + 'geometry': geometries['covers-switzerland'], + 'properties_datetime': fromisoformat('2020-10-28T13:05:10Z'), + 'forecast_reference_datetime': fromisoformat('2025-04-01T13:05:10Z'), + 'forecast_horizon': 'PT6H', + 'forecast_duration': 'PT12H', + 'forecast_variable': 'air_temperature', + 'forecast_perturbed': 'False', + }, } diff --git a/app/tests/tests_10/test_search_endpoint.py b/app/tests/tests_10/test_search_endpoint.py index 63321f08..8043a263 100644 --- a/app/tests/tests_10/test_search_endpoint.py +++ b/app/tests/tests_10/test_search_endpoint.py @@ -580,3 +580,202 @@ def test_post_search_no_cache_setting(self): response.has_header('Cache-Control'), msg="Unexpected Cache-Control header in POST response" ) + + +class SearchEndpointTestForecast(StacBaseTestCase): + + @classmethod + def setUpTestData(cls): + cls.factory = Factory() + cls.collection = cls.factory.create_collection_sample().model + cls.items = cls.factory.create_item_samples( + [ + 'item-forecast-1', + 'item-forecast-2', + 'item-forecast-3', + 'item-forecast-4', + 'item-forecast-5' + ], + cls.collection, + db_create=True, + ) + cls.now = utc_aware(datetime.utcnow()) + cls.yesterday = cls.now - timedelta(days=1) + + def setUp(self): # pylint: disable=invalid-name + self.client = Client() + client_login(self.client) + self.path = f'/{STAC_BASE_V}/search' + self.maxDiff = None # pylint: disable=invalid-name + + def test_reference_datetime_exact(self): + val = '2025-01-01T13:05:10Z' + response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 1) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-1']) + + payload = {"forecast_reference_datetime": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 1) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-1']) + + val = '2025-02-01T13:05:10Z' + response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 3) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) + + payload = {"forecast_reference_datetime": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 3) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) + + def test_reference_datetime_range(self): + val = '2025-02-01T00:00:00Z/2025-02-28T00:00:00Z' + response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 3) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) + + payload = {"forecast_reference_datetime": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 3) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) + + def test_reference_datetime_open_end(self): + val = '2025-02-01T13:05:10Z/..' + response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 4) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4', 'item-5']) + + payload = {"forecast_reference_datetime": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 4) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4', 'item-5']) + + def test_reference_datetime_open_start(self): + val = '../2025-02-01T13:05:10Z' + response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 4) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-1', 'item-2', 'item-3', 'item-4']) + + payload = {"forecast_reference_datetime": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 4) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-1', 'item-2', 'item-3', 'item-4']) + + def test_horizon(self): + val = 'PT3H' + response = self.client.get(f"{self.path}?forecast_horizon={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 1) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-3']) + + payload = {"forecast_horizon": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 1) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-3']) + + def test_duration(self): + val = 'PT12H' + response = self.client.get(f"{self.path}?forecast_duration={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 4) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-1', 'item-2', 'item-4', 'item-5']) + + payload = {"forecast_duration": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 4) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-1', 'item-2', 'item-4', 'item-5']) + + def test_variable(self): + val = 'air_temperature' + response = self.client.get(f"{self.path}?forecast_variable={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 2) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-4', 'item-5']) + + payload = {"forecast_variable": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 2) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-4', 'item-5']) + + def test_perturbed(self): + val = 'True' + response = self.client.get(f"{self.path}?forecast_perturbed={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 1) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-4']) + + payload = {"forecast_perturbed": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 1) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-4']) + + def test_multiple(self): + response = self.client.get( + f"{self.path}?forecast_perturbed=False&forecast_horizon=PT6H&forecast_variable=T" + ) + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 2) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-1', 'item-2']) + + payload = { + "forecast_perturbed": "False", "forecast_horizon": "PT6H", "forecast_variable": "T" + } + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 2) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-1', 'item-2']) From b29937ce49112beb958740148a3cc07ec9c6b55b Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Thu, 23 Jan 2025 15:23:18 +0100 Subject: [PATCH 3/8] PB-1169: Spec remove forecast params from Get search Decided to remove the forecast parameters because the variable name differs to the post variable names (using underscore instead of colon) and colons need url encoding. --- spec/components/parameters.yaml | 44 ---------------- spec/openapi.yaml | 8 +-- spec/static/spec/v1/openapi.yaml | 51 +------------------ spec/static/spec/v1/openapitransactional.yaml | 51 +------------------ 4 files changed, 4 insertions(+), 150 deletions(-) diff --git a/spec/components/parameters.yaml b/spec/components/parameters.yaml index 84b8ea24..63ca9bbd 100644 --- a/spec/components/parameters.yaml +++ b/spec/components/parameters.yaml @@ -117,47 +117,3 @@ components: required: false schema: type: string - forecast_reference_datetime: - explode: false - in: query - name: forecast_reference_datetime - required: false - schema: - $ref: "./schemas.yaml#/components/schemas/datetimeQuery" - example: 2018-02-12T00%3A00%3A00Z%2F2018-03-18T12%3A31%3A12Z - style: form - forecast_horizon: - explode: false - in: query - name: forecast_horizon - required: false - schema: - $ref: "./schemas.yaml#/components/schemas/duration" - example: P3DT6H - style: form - forecast_duration: - explode: false - in: query - name: forecast_duration - required: false - schema: - $ref: "./schemas.yaml#/components/schemas/duration" - example: P3DT6H - style: form - forecast_variable: - explode: false - in: query - name: forecast_variable - required: false - schema: - $ref: "./schemas.yaml#/components/schemas/forecast_variable" - example: air_temperature - style: form - forecast_perturbed: - explode: false - in: query - name: forecast_perturbed - required: false - schema: - $ref: "./schemas.yaml#/components/schemas/forecast_perturbed" - style: form diff --git a/spec/openapi.yaml b/spec/openapi.yaml index befc9641..a49fcec1 100644 --- a/spec/openapi.yaml +++ b/spec/openapi.yaml @@ -274,7 +274,8 @@ paths: get: description: >- Retrieve Items matching filters. Intended as a shorthand API for simple - queries. + queries. To filter by forecast properties please use the + [POST /search](#tag/STAC/operation/postSearchSTAC) request. operationId: getSearchSTAC parameters: @@ -283,11 +284,6 @@ paths: - $ref: "./components/parameters.yaml#/components/parameters/limit" - $ref: "./components/parameters.yaml#/components/parameters/ids" - $ref: "./components/parameters.yaml#/components/parameters/collectionsArray" - - $ref: "./components/parameters.yaml#/components/parameters/forecast_reference_datetime" - - $ref: "./components/parameters.yaml#/components/parameters/forecast_horizon" - - $ref: "./components/parameters.yaml#/components/parameters/forecast_duration" - - $ref: "./components/parameters.yaml#/components/parameters/forecast_variable" - - $ref: "./components/parameters.yaml#/components/parameters/forecast_perturbed" responses: "200": content: diff --git a/spec/static/spec/v1/openapi.yaml b/spec/static/spec/v1/openapi.yaml index 24663687..c6e61d31 100644 --- a/spec/static/spec/v1/openapi.yaml +++ b/spec/static/spec/v1/openapi.yaml @@ -115,50 +115,6 @@ components: required: false schema: type: string - forecast_reference_datetime: - explode: false - in: query - name: forecast_reference_datetime - required: false - schema: - $ref: "#/components/schemas/datetimeQuery" - example: 2018-02-12T00%3A00%3A00Z%2F2018-03-18T12%3A31%3A12Z - style: form - forecast_horizon: - explode: false - in: query - name: forecast_horizon - required: false - schema: - $ref: "#/components/schemas/duration" - example: P3DT6H - style: form - forecast_duration: - explode: false - in: query - name: forecast_duration - required: false - schema: - $ref: "#/components/schemas/duration" - example: P3DT6H - style: form - forecast_variable: - explode: false - in: query - name: forecast_variable - required: false - schema: - $ref: "#/components/schemas/forecast_variable" - example: air_temperature - style: form - forecast_perturbed: - explode: false - in: query - name: forecast_perturbed - required: false - schema: - $ref: "#/components/schemas/forecast_perturbed" - style: form responses: Collection: headers: @@ -2156,7 +2112,7 @@ paths: /search: get: description: >- - Retrieve Items matching filters. Intended as a shorthand API for simple queries. + Retrieve Items matching filters. Intended as a shorthand API for simple queries. To filter by forecast properties please use the [POST /search](#tag/STAC/operation/postSearchSTAC) request. operationId: getSearchSTAC parameters: - $ref: "#/components/parameters/bbox" @@ -2164,11 +2120,6 @@ paths: - $ref: "#/components/parameters/limit" - $ref: "#/components/parameters/ids" - $ref: "#/components/parameters/collectionsArray" - - $ref: "#/components/parameters/forecast_reference_datetime" - - $ref: "#/components/parameters/forecast_horizon" - - $ref: "#/components/parameters/forecast_duration" - - $ref: "#/components/parameters/forecast_variable" - - $ref: "#/components/parameters/forecast_perturbed" responses: "200": content: diff --git a/spec/static/spec/v1/openapitransactional.yaml b/spec/static/spec/v1/openapitransactional.yaml index e3230d2f..4133df4d 100644 --- a/spec/static/spec/v1/openapitransactional.yaml +++ b/spec/static/spec/v1/openapitransactional.yaml @@ -115,50 +115,6 @@ components: required: false schema: type: string - forecast_reference_datetime: - explode: false - in: query - name: forecast_reference_datetime - required: false - schema: - $ref: "#/components/schemas/datetimeQuery" - example: 2018-02-12T00%3A00%3A00Z%2F2018-03-18T12%3A31%3A12Z - style: form - forecast_horizon: - explode: false - in: query - name: forecast_horizon - required: false - schema: - $ref: "#/components/schemas/duration" - example: P3DT6H - style: form - forecast_duration: - explode: false - in: query - name: forecast_duration - required: false - schema: - $ref: "#/components/schemas/duration" - example: P3DT6H - style: form - forecast_variable: - explode: false - in: query - name: forecast_variable - required: false - schema: - $ref: "#/components/schemas/forecast_variable" - example: air_temperature - style: form - forecast_perturbed: - explode: false - in: query - name: forecast_perturbed - required: false - schema: - $ref: "#/components/schemas/forecast_perturbed" - style: form assetId: name: assetId in: path @@ -3576,7 +3532,7 @@ paths: /search: get: description: >- - Retrieve Items matching filters. Intended as a shorthand API for simple queries. + Retrieve Items matching filters. Intended as a shorthand API for simple queries. To filter by forecast properties please use the [POST /search](#tag/STAC/operation/postSearchSTAC) request. operationId: getSearchSTAC parameters: - $ref: "#/components/parameters/bbox" @@ -3584,11 +3540,6 @@ paths: - $ref: "#/components/parameters/limit" - $ref: "#/components/parameters/ids" - $ref: "#/components/parameters/collectionsArray" - - $ref: "#/components/parameters/forecast_reference_datetime" - - $ref: "#/components/parameters/forecast_horizon" - - $ref: "#/components/parameters/forecast_duration" - - $ref: "#/components/parameters/forecast_variable" - - $ref: "#/components/parameters/forecast_perturbed" responses: "200": content: From 757c46dd593228d68fa948fc04ec684854dc00bc Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Thu, 23 Jan 2025 15:52:17 +0100 Subject: [PATCH 4/8] PB-1169: Remove forecast filter from GET search Decided to remove the forecast filtering options from the GET /search request due to colons in the parameter names. Also fixed the POST request body to actually use the parameters that contain a colon. --- app/stac_api/utils.py | 14 +++ app/stac_api/validators_serializer.py | 10 +- app/stac_api/views/general.py | 20 ++-- app/tests/tests_10/test_search_endpoint.py | 114 +++++---------------- 4 files changed, 52 insertions(+), 106 deletions(-) diff --git a/app/stac_api/utils.py b/app/stac_api/utils.py index af354a17..9cff55af 100644 --- a/app/stac_api/utils.py +++ b/app/stac_api/utils.py @@ -309,6 +309,20 @@ def harmonize_post_get_for_search(request): query_param['ids'] = query_param['ids'].split(',') # to array if 'collections' in query_param: query_param['collections'] = query_param['collections'].split(',') # to array + + # Forecast properties can only be filtered with method POST. + # Decision was made as `:` need to be url encoded and (at least for now) we do not need to + # support forecast filtering in the GET request. + if 'forecast:reference_datetime' in query_param: + del query_param['forecast:reference_datetime'] + if 'forecast:horizon' in query_param: + del query_param['forecast:horizon'] + if 'forecast:duration' in query_param: + del query_param['forecast:duration'] + if 'forecast:variable' in query_param: + del query_param['forecast:variable'] + if 'forecast:perturbed' in query_param: + del query_param['forecast:perturbed'] return query_param diff --git a/app/stac_api/validators_serializer.py b/app/stac_api/validators_serializer.py index 56a7d430..509d38e7 100644 --- a/app/stac_api/validators_serializer.py +++ b/app/stac_api/validators_serializer.py @@ -359,11 +359,11 @@ def validate_query_parameters_post_search(self, query_param): "limit", "cursor", "query", - "forecast_reference_datetime", - "forecast_horizon", - "forecast_duration", - "forecast_variable", - "forecast_perturbed" + "forecast:reference_datetime", + "forecast:horizon", + "forecast:duration", + "forecast:variable", + "forecast:perturbed" ] wrong_query_parameters = set(query_param.keys()).difference(set(accepted_query_parameters)) if wrong_query_parameters: diff --git a/app/stac_api/views/general.py b/app/stac_api/views/general.py index 498766ea..662ed813 100644 --- a/app/stac_api/views/general.py +++ b/app/stac_api/views/general.py @@ -93,18 +93,18 @@ def get_queryset(self): queryset = queryset.filter_by_query(dict_query) if 'intersects' in query_param: queryset = queryset.filter_by_intersects(json.dumps(query_param['intersects'])) - if 'forecast_reference_datetime' in query_param: + if 'forecast:reference_datetime' in query_param: queryset = queryset.filter_by_forecast_reference_datetime( - query_param['forecast_reference_datetime'] + query_param['forecast:reference_datetime'] ) - if 'forecast_horizon' in query_param: - queryset = queryset.filter_by_forecast_horizon(query_param['forecast_horizon']) - if 'forecast_duration' in query_param: - queryset = queryset.filter_by_forecast_duration(query_param['forecast_duration']) - if 'forecast_variable' in query_param: - queryset = queryset.filter_by_forecast_variable(query_param['forecast_variable']) - if 'forecast_perturbed' in query_param: - queryset = queryset.filter_by_forecast_perturbed(query_param['forecast_perturbed']) + if 'forecast:horizon' in query_param: + queryset = queryset.filter_by_forecast_horizon(query_param['forecast:horizon']) + if 'forecast:duration' in query_param: + queryset = queryset.filter_by_forecast_duration(query_param['forecast:duration']) + if 'forecast:variable' in query_param: + queryset = queryset.filter_by_forecast_variable(query_param['forecast:variable']) + if 'forecast:perturbed' in query_param: + queryset = queryset.filter_by_forecast_perturbed(query_param['forecast:perturbed']) if settings.DEBUG_ENABLE_DB_EXPLAIN_ANALYZE: logger.debug( diff --git a/app/tests/tests_10/test_search_endpoint.py b/app/tests/tests_10/test_search_endpoint.py index 8043a263..dadb6be3 100644 --- a/app/tests/tests_10/test_search_endpoint.py +++ b/app/tests/tests_10/test_search_endpoint.py @@ -2,6 +2,7 @@ import logging from datetime import datetime from datetime import timedelta +from urllib.parse import quote_plus from django.test import Client from django.test import override_settings @@ -609,15 +610,7 @@ def setUp(self): # pylint: disable=invalid-name self.maxDiff = None # pylint: disable=invalid-name def test_reference_datetime_exact(self): - val = '2025-01-01T13:05:10Z' - response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 1) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-1']) - - payload = {"forecast_reference_datetime": f"{val}"} + payload = {"forecast:reference_datetime": "2025-01-01T13:05:10Z"} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -625,15 +618,7 @@ def test_reference_datetime_exact(self): for feature in json_data['features']: self.assertIn(feature['id'], ['item-1']) - val = '2025-02-01T13:05:10Z' - response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 3) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) - - payload = {"forecast_reference_datetime": f"{val}"} + payload = {"forecast:reference_datetime": "2025-02-01T13:05:10Z"} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -642,15 +627,7 @@ def test_reference_datetime_exact(self): self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) def test_reference_datetime_range(self): - val = '2025-02-01T00:00:00Z/2025-02-28T00:00:00Z' - response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 3) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) - - payload = {"forecast_reference_datetime": f"{val}"} + payload = {"forecast:reference_datetime": "2025-02-01T00:00:00Z/2025-02-28T00:00:00Z"} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -659,15 +636,7 @@ def test_reference_datetime_range(self): self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) def test_reference_datetime_open_end(self): - val = '2025-02-01T13:05:10Z/..' - response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 4) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4', 'item-5']) - - payload = {"forecast_reference_datetime": f"{val}"} + payload = {"forecast:reference_datetime": "2025-02-01T13:05:10Z/.."} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -676,15 +645,7 @@ def test_reference_datetime_open_end(self): self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4', 'item-5']) def test_reference_datetime_open_start(self): - val = '../2025-02-01T13:05:10Z' - response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 4) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-1', 'item-2', 'item-3', 'item-4']) - - payload = {"forecast_reference_datetime": f"{val}"} + payload = {"forecast:reference_datetime": "../2025-02-01T13:05:10Z"} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -693,15 +654,7 @@ def test_reference_datetime_open_start(self): self.assertIn(feature['id'], ['item-1', 'item-2', 'item-3', 'item-4']) def test_horizon(self): - val = 'PT3H' - response = self.client.get(f"{self.path}?forecast_horizon={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 1) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-3']) - - payload = {"forecast_horizon": f"{val}"} + payload = {"forecast:horizon": "PT3H"} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -710,15 +663,7 @@ def test_horizon(self): self.assertIn(feature['id'], ['item-3']) def test_duration(self): - val = 'PT12H' - response = self.client.get(f"{self.path}?forecast_duration={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 4) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-1', 'item-2', 'item-4', 'item-5']) - - payload = {"forecast_duration": f"{val}"} + payload = {"forecast:duration": "PT12H"} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -727,15 +672,7 @@ def test_duration(self): self.assertIn(feature['id'], ['item-1', 'item-2', 'item-4', 'item-5']) def test_variable(self): - val = 'air_temperature' - response = self.client.get(f"{self.path}?forecast_variable={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 2) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-4', 'item-5']) - - payload = {"forecast_variable": f"{val}"} + payload = {"forecast:variable": "air_temperature"} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -744,15 +681,7 @@ def test_variable(self): self.assertIn(feature['id'], ['item-4', 'item-5']) def test_perturbed(self): - val = 'True' - response = self.client.get(f"{self.path}?forecast_perturbed={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 1) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-4']) - - payload = {"forecast_perturbed": f"{val}"} + payload = {"forecast:perturbed": "True"} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -761,17 +690,8 @@ def test_perturbed(self): self.assertIn(feature['id'], ['item-4']) def test_multiple(self): - response = self.client.get( - f"{self.path}?forecast_perturbed=False&forecast_horizon=PT6H&forecast_variable=T" - ) - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 2) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-1', 'item-2']) - payload = { - "forecast_perturbed": "False", "forecast_horizon": "PT6H", "forecast_variable": "T" + "forecast:perturbed": "False", "forecast:horizon": "PT6H", "forecast:variable": "T" } response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) @@ -779,3 +699,15 @@ def test_multiple(self): self.assertEqual(len(json_data['features']), 2) for feature in json_data['features']: self.assertIn(feature['id'], ['item-1', 'item-2']) + + def test_get_request_does_not_filter_forecast(self): + response = self.client.get( + f"{self.path}?" + quote_plus( + "forecast:reference_datetime=2025-01-01T13:05:10Z&" + "forecast:duration=PT12H&" + + "forecast:perturbed=False&" + "forecast:horizon=PT6H&" + "forecast:variable=T" + ) + ) + self.assertStatusCode(200, response) + json_data = response.json() + # As GET request should not filter for forecast expect all 5 features to be returned. + self.assertEqual(len(json_data['features']), 5) From d2492aaad2de46fc621701ccc923b42057e572d3 Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Mon, 27 Jan 2025 08:56:47 +0100 Subject: [PATCH 5/8] Fix for review comments Fix code comments. Refactor create sample data names. --- app/stac_api/managers.py | 10 ++-- app/tests/tests_10/test_search_endpoint.py | 53 ++++++++++++++-------- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/app/stac_api/managers.py b/app/stac_api/managers.py index 8b0df7f1..6075cc0e 100644 --- a/app/stac_api/managers.py +++ b/app/stac_api/managers.py @@ -109,10 +109,8 @@ def filter_by_forecast_reference_datetime(self, date_time): '''Filter a queryset by forecast reference datetime Args: - queryset: - A django queryset (https://docs.djangoproject.com/en/3.0/ref/models/querysets/) date_time: - A string + A string containing datetime like "2020-10-28T13:05:10Z" Returns: The queryset filtered by date_time @@ -128,12 +126,10 @@ def _filter_by_forecast_reference_datetime_range(self, start_datetime, end_datet Helper function of filter_by_forecast_reference_datetime Args: - queryset: - A django queryset (https://docs.djangoproject.com/en/3.0/ref/models/querysets/) start_datetime: - A string with the start datetime + A string with the start datetime or ".." to denote open start end_datetime: - A string with the end datetime + A string with the end datetime or ".." to denote open end Returns: The queryset filtered by datetime range ''' diff --git a/app/tests/tests_10/test_search_endpoint.py b/app/tests/tests_10/test_search_endpoint.py index dadb6be3..b48023a2 100644 --- a/app/tests/tests_10/test_search_endpoint.py +++ b/app/tests/tests_10/test_search_endpoint.py @@ -589,16 +589,20 @@ class SearchEndpointTestForecast(StacBaseTestCase): def setUpTestData(cls): cls.factory = Factory() cls.collection = cls.factory.create_collection_sample().model - cls.items = cls.factory.create_item_samples( - [ - 'item-forecast-1', - 'item-forecast-2', - 'item-forecast-3', - 'item-forecast-4', - 'item-forecast-5' - ], - cls.collection, - db_create=True, + cls.factory.create_item_sample( + cls.collection, 'item-forecast-1', 'item-forecast-1', db_create=True + ) + cls.factory.create_item_sample( + cls.collection, 'item-forecast-2', 'item-forecast-2', db_create=True + ) + cls.factory.create_item_sample( + cls.collection, 'item-forecast-3', 'item-forecast-3', db_create=True + ) + cls.factory.create_item_sample( + cls.collection, 'item-forecast-4', 'item-forecast-4', db_create=True + ) + cls.factory.create_item_sample( + cls.collection, 'item-forecast-5', 'item-forecast-5', db_create=True ) cls.now = utc_aware(datetime.utcnow()) cls.yesterday = cls.now - timedelta(days=1) @@ -616,7 +620,7 @@ def test_reference_datetime_exact(self): json_data = response.json() self.assertEqual(len(json_data['features']), 1) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-1']) + self.assertIn(feature['id'], ['item-forecast-1']) payload = {"forecast:reference_datetime": "2025-02-01T13:05:10Z"} response = self.client.post(self.path, data=payload, content_type="application/json") @@ -624,7 +628,7 @@ def test_reference_datetime_exact(self): json_data = response.json() self.assertEqual(len(json_data['features']), 3) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) + self.assertIn(feature['id'], ['item-forecast-2', 'item-forecast-3', 'item-forecast-4']) def test_reference_datetime_range(self): payload = {"forecast:reference_datetime": "2025-02-01T00:00:00Z/2025-02-28T00:00:00Z"} @@ -633,7 +637,7 @@ def test_reference_datetime_range(self): json_data = response.json() self.assertEqual(len(json_data['features']), 3) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) + self.assertIn(feature['id'], ['item-forecast-2', 'item-forecast-3', 'item-forecast-4']) def test_reference_datetime_open_end(self): payload = {"forecast:reference_datetime": "2025-02-01T13:05:10Z/.."} @@ -642,7 +646,10 @@ def test_reference_datetime_open_end(self): json_data = response.json() self.assertEqual(len(json_data['features']), 4) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4', 'item-5']) + self.assertIn( + feature['id'], + ['item-forecast-2', 'item-forecast-3', 'item-forecast-4', 'item-forecast-5'] + ) def test_reference_datetime_open_start(self): payload = {"forecast:reference_datetime": "../2025-02-01T13:05:10Z"} @@ -651,7 +658,10 @@ def test_reference_datetime_open_start(self): json_data = response.json() self.assertEqual(len(json_data['features']), 4) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-1', 'item-2', 'item-3', 'item-4']) + self.assertIn( + feature['id'], + ['item-forecast-1', 'item-forecast-2', 'item-forecast-3', 'item-forecast-4'] + ) def test_horizon(self): payload = {"forecast:horizon": "PT3H"} @@ -660,7 +670,7 @@ def test_horizon(self): json_data = response.json() self.assertEqual(len(json_data['features']), 1) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-3']) + self.assertIn(feature['id'], ['item-forecast-3']) def test_duration(self): payload = {"forecast:duration": "PT12H"} @@ -669,7 +679,10 @@ def test_duration(self): json_data = response.json() self.assertEqual(len(json_data['features']), 4) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-1', 'item-2', 'item-4', 'item-5']) + self.assertIn( + feature['id'], + ['item-forecast-1', 'item-forecast-2', 'item-forecast-4', 'item-forecast-5'] + ) def test_variable(self): payload = {"forecast:variable": "air_temperature"} @@ -678,7 +691,7 @@ def test_variable(self): json_data = response.json() self.assertEqual(len(json_data['features']), 2) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-4', 'item-5']) + self.assertIn(feature['id'], ['item-forecast-4', 'item-forecast-5']) def test_perturbed(self): payload = {"forecast:perturbed": "True"} @@ -687,7 +700,7 @@ def test_perturbed(self): json_data = response.json() self.assertEqual(len(json_data['features']), 1) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-4']) + self.assertIn(feature['id'], ['item-forecast-4']) def test_multiple(self): payload = { @@ -698,7 +711,7 @@ def test_multiple(self): json_data = response.json() self.assertEqual(len(json_data['features']), 2) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-1', 'item-2']) + self.assertIn(feature['id'], ['item-forecast-1', 'item-forecast-2']) def test_get_request_does_not_filter_forecast(self): response = self.client.get( From e5aa63f400b4b3ad17ef8eae2af2e04fa93e7521 Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Mon, 27 Jan 2025 10:23:49 +0100 Subject: [PATCH 6/8] PB-1169: Index item forecast fields Add index for the item forecast properties as they can be used as filters in the item search. --- ...item_fc_reference_datetime_idx_and_more.py | 36 +++++++++++++++++++ app/stac_api/models.py | 8 +++++ 2 files changed, 44 insertions(+) create mode 100644 app/stac_api/migrations/0062_item_item_fc_reference_datetime_idx_and_more.py diff --git a/app/stac_api/migrations/0062_item_item_fc_reference_datetime_idx_and_more.py b/app/stac_api/migrations/0062_item_item_fc_reference_datetime_idx_and_more.py new file mode 100644 index 00000000..5297a700 --- /dev/null +++ b/app/stac_api/migrations/0062_item_item_fc_reference_datetime_idx_and_more.py @@ -0,0 +1,36 @@ +# Generated by Django 5.0.11 on 2025-01-27 08:22 + +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ('stac_api', '0061_remove_item_forecast_mode_remove_item_forecast_param_and_more'), + ] + + operations = [ + migrations.AddIndex( + model_name='item', + index=models.Index( + fields=['forecast_reference_datetime'], name='item_fc_reference_datetime_idx' + ), + ), + migrations.AddIndex( + model_name='item', + index=models.Index(fields=['forecast_horizon'], name='item_fc_horizon_idx'), + ), + migrations.AddIndex( + model_name='item', + index=models.Index(fields=['forecast_duration'], name='item_fc_duration_idx'), + ), + migrations.AddIndex( + model_name='item', + index=models.Index(fields=['forecast_variable'], name='item_fc_variable_idx'), + ), + migrations.AddIndex( + model_name='item', + index=models.Index(fields=['forecast_perturbed'], name='item_fc_perturbed_idx'), + ), + ] diff --git a/app/stac_api/models.py b/app/stac_api/models.py index 934f613b..c6b7cd8f 100644 --- a/app/stac_api/models.py +++ b/app/stac_api/models.py @@ -370,6 +370,14 @@ class Meta: models.Index(fields=['created'], name='item_created_idx'), models.Index(fields=['updated'], name='item_updated_idx'), models.Index(fields=['properties_title'], name='item_title_idx'), + # forecast properties are "queryable" in the search endpoint + models.Index( + fields=['forecast_reference_datetime'], name='item_fc_reference_datetime_idx' + ), + models.Index(fields=['forecast_horizon'], name='item_fc_horizon_idx'), + models.Index(fields=['forecast_duration'], name='item_fc_duration_idx'), + models.Index(fields=['forecast_variable'], name='item_fc_variable_idx'), + models.Index(fields=['forecast_perturbed'], name='item_fc_perturbed_idx'), # combination of datetime and start_ and end_datetimes are used in # managers.py:110 and following models.Index( From 18d698be3a0d3d716e979e6816162484fa6b56cf Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Mon, 27 Jan 2025 10:57:40 +0100 Subject: [PATCH 7/8] Refactor models - move file Move models file into folder /models and rename file to general.py. Fix all imports that referenced the models file. --- app/stac_api/admin.py | 22 ++-- .../management/commands/calculate_extent.py | 2 +- .../management/commands/dummy_asset_upload.py | 6 +- .../management/commands/dummy_data.py | 6 +- .../management/commands/list_asset_uploads.py | 2 +- .../commands/profile_cursor_paginator.py | 2 +- .../commands/profile_item_serializer.py | 2 +- .../commands/profile_serializer_vs_no_drf.py | 2 +- .../commands/remove_expired_items.py | 6 +- .../commands/update_asset_file_size.py | 4 +- app/stac_api/migrations/0001_initial.py | 8 +- .../migrations/0005_auto_20210408_0821.py | 7 +- .../migrations/0014_auto_20210715_1358.py | 8 +- .../migrations/0032_alter_asset_file.py | 6 +- ...ncepage_landingpage_conformsto_and_more.py | 4 +- .../0036_collectionasset_and_more.py | 8 +- .../0050_collectionassetupload_and_more.py | 7 +- app/stac_api/models/__init__.py | 0 app/stac_api/{models.py => models/general.py} | 0 app/stac_api/s3_multipart_upload.py | 4 +- app/stac_api/sample_data/importer.py | 10 +- app/stac_api/serializers/collection.py | 8 +- app/stac_api/serializers/general.py | 4 +- app/stac_api/serializers/item.py | 6 +- app/stac_api/serializers/upload.py | 4 +- app/stac_api/serializers/utils.py | 6 +- app/stac_api/signals.py | 8 +- app/stac_api/validators_view.py | 8 +- app/stac_api/views/collection.py | 4 +- app/stac_api/views/general.py | 4 +- app/stac_api/views/item.py | 6 +- app/stac_api/views/test.py | 2 +- app/stac_api/views/upload.py | 10 +- app/tests/base_test_admin_page.py | 12 +- app/tests/test_admin_page.py | 12 +- app/tests/tests_09/data_factory.py | 12 +- .../tests_09/sample_data/item_samples.py | 2 +- app/tests/tests_09/test_asset_model.py | 2 +- .../tests_09/test_asset_upload_endpoint.py | 4 +- app/tests/tests_09/test_asset_upload_model.py | 4 +- app/tests/tests_09/test_assets_endpoint.py | 2 +- app/tests/tests_09/test_collection_model.py | 2 +- .../tests_09/test_collections_endpoint.py | 6 +- app/tests/tests_09/test_collections_extent.py | 2 +- app/tests/tests_09/test_generic_api.py | 2 +- app/tests/tests_09/test_item_model.py | 4 +- app/tests/tests_09/test_items_endpoint.py | 2 +- .../tests_09/test_items_endpoint_bbox.py | 2 +- app/tests/tests_09/test_serializer.py | 2 +- .../tests_09/test_serializer_asset_upload.py | 2 +- app/tests/tests_10/data_factory.py | 14 +-- .../tests_10/sample_data/item_samples.py | 2 +- app/tests/tests_10/test_asset_model.py | 2 +- .../tests_10/test_asset_upload_endpoint.py | 4 +- app/tests/tests_10/test_asset_upload_model.py | 4 +- app/tests/tests_10/test_assets_endpoint.py | 2 +- .../tests_10/test_collection_asset_model.py | 2 +- .../test_collection_asset_upload_endpoint.py | 4 +- .../test_collection_asset_upload_model.py | 4 +- .../test_collection_assets_endpoint.py | 2 +- app/tests/tests_10/test_collection_model.py | 2 +- .../tests_10/test_collections_endpoint.py | 6 +- app/tests/tests_10/test_collections_extent.py | 2 +- .../tests_10/test_external_assets_endpoint.py | 2 +- app/tests/tests_10/test_generic_api.py | 2 +- app/tests/tests_10/test_item_model.py | 4 +- app/tests/tests_10/test_items_endpoint.py | 6 +- .../tests_10/test_items_endpoint_bbox.py | 2 +- .../tests_10/test_remove_expired_items.py | 4 +- app/tests/tests_10/test_serializer.py | 2 +- .../tests_10/test_serializer_asset_upload.py | 2 +- scripts/fill_local_db.py | 113 ++++++------------ 72 files changed, 204 insertions(+), 243 deletions(-) create mode 100644 app/stac_api/models/__init__.py rename app/stac_api/{models.py => models/general.py} (100%) diff --git a/app/stac_api/admin.py b/app/stac_api/admin.py index 7b775043..494af29a 100644 --- a/app/stac_api/admin.py +++ b/app/stac_api/admin.py @@ -17,17 +17,17 @@ from django.urls import reverse from django.utils.translation import gettext_lazy as _ -from stac_api.models import BBOX_CH -from stac_api.models import Asset -from stac_api.models import AssetUpload -from stac_api.models import Collection -from stac_api.models import CollectionAsset -from stac_api.models import CollectionLink -from stac_api.models import Item -from stac_api.models import ItemLink -from stac_api.models import LandingPage -from stac_api.models import LandingPageLink -from stac_api.models import Provider +from stac_api.models.general import BBOX_CH +from stac_api.models.general import Asset +from stac_api.models.general import AssetUpload +from stac_api.models.general import Collection +from stac_api.models.general import CollectionAsset +from stac_api.models.general import CollectionLink +from stac_api.models.general import Item +from stac_api.models.general import ItemLink +from stac_api.models.general import LandingPage +from stac_api.models.general import LandingPageLink +from stac_api.models.general import Provider from stac_api.utils import build_asset_href from stac_api.utils import get_query_params from stac_api.validators import validate_href_url diff --git a/app/stac_api/management/commands/calculate_extent.py b/app/stac_api/management/commands/calculate_extent.py index f01104e0..4118d8e9 100644 --- a/app/stac_api/management/commands/calculate_extent.py +++ b/app/stac_api/management/commands/calculate_extent.py @@ -3,7 +3,7 @@ from django.core.management.base import CommandParser from django.db import connection -from stac_api.models import Collection +from stac_api.models.general import Collection from stac_api.utils import CommandHandler from stac_api.utils import CustomBaseCommand diff --git a/app/stac_api/management/commands/dummy_asset_upload.py b/app/stac_api/management/commands/dummy_asset_upload.py index 758f98b8..e79964b6 100644 --- a/app/stac_api/management/commands/dummy_asset_upload.py +++ b/app/stac_api/management/commands/dummy_asset_upload.py @@ -4,9 +4,9 @@ from django.conf import settings from django.core.management.base import BaseCommand -from stac_api.models import Asset -from stac_api.models import AssetUpload -from stac_api.models import BaseAssetUpload +from stac_api.models.general import Asset +from stac_api.models.general import AssetUpload +from stac_api.models.general import BaseAssetUpload from stac_api.s3_multipart_upload import MultipartUpload from stac_api.utils import AVAILABLE_S3_BUCKETS from stac_api.utils import CommandHandler diff --git a/app/stac_api/management/commands/dummy_data.py b/app/stac_api/management/commands/dummy_data.py index bb9d39dc..ac13f159 100644 --- a/app/stac_api/management/commands/dummy_data.py +++ b/app/stac_api/management/commands/dummy_data.py @@ -13,9 +13,9 @@ from django.core.files.uploadedfile import SimpleUploadedFile from django.core.management.base import BaseCommand -from stac_api.models import Asset -from stac_api.models import Collection -from stac_api.models import Item +from stac_api.models.general import Asset +from stac_api.models.general import Collection +from stac_api.models.general import Item from stac_api.utils import CommandHandler from stac_api.validators import MEDIA_TYPES diff --git a/app/stac_api/management/commands/list_asset_uploads.py b/app/stac_api/management/commands/list_asset_uploads.py index 079ac3ec..c0a0f5db 100644 --- a/app/stac_api/management/commands/list_asset_uploads.py +++ b/app/stac_api/management/commands/list_asset_uploads.py @@ -4,7 +4,7 @@ from django.core.management.base import BaseCommand from django.core.serializers.json import DjangoJSONEncoder -from stac_api.models import AssetUpload +from stac_api.models.general import AssetUpload from stac_api.s3_multipart_upload import MultipartUpload from stac_api.serializers.upload import AssetUploadSerializer from stac_api.utils import CommandHandler diff --git a/app/stac_api/management/commands/profile_cursor_paginator.py b/app/stac_api/management/commands/profile_cursor_paginator.py index c4c7cc97..311ecff3 100644 --- a/app/stac_api/management/commands/profile_cursor_paginator.py +++ b/app/stac_api/management/commands/profile_cursor_paginator.py @@ -10,7 +10,7 @@ from rest_framework.request import Request from rest_framework.test import APIRequestFactory -from stac_api.models import Item +from stac_api.models.general import Item from stac_api.utils import CommandHandler logger = logging.getLogger(__name__) diff --git a/app/stac_api/management/commands/profile_item_serializer.py b/app/stac_api/management/commands/profile_item_serializer.py index 47f2ba79..9b43822f 100644 --- a/app/stac_api/management/commands/profile_item_serializer.py +++ b/app/stac_api/management/commands/profile_item_serializer.py @@ -8,7 +8,7 @@ from rest_framework.test import APIRequestFactory -from stac_api.models import Item +from stac_api.models.general import Item from stac_api.utils import CommandHandler logger = logging.getLogger(__name__) diff --git a/app/stac_api/management/commands/profile_serializer_vs_no_drf.py b/app/stac_api/management/commands/profile_serializer_vs_no_drf.py index d4b1ddca..3f2ada21 100644 --- a/app/stac_api/management/commands/profile_serializer_vs_no_drf.py +++ b/app/stac_api/management/commands/profile_serializer_vs_no_drf.py @@ -7,7 +7,7 @@ from rest_framework.test import APIRequestFactory -from stac_api.models import Item +from stac_api.models.general import Item from stac_api.utils import CommandHandler logger = logging.getLogger(__name__) diff --git a/app/stac_api/management/commands/remove_expired_items.py b/app/stac_api/management/commands/remove_expired_items.py index d1eb2668..a1f73a22 100644 --- a/app/stac_api/management/commands/remove_expired_items.py +++ b/app/stac_api/management/commands/remove_expired_items.py @@ -4,9 +4,9 @@ from django.core.management.base import CommandParser from django.utils import timezone -from stac_api.models import AssetUpload -from stac_api.models import BaseAssetUpload -from stac_api.models import Item +from stac_api.models.general import AssetUpload +from stac_api.models.general import BaseAssetUpload +from stac_api.models.general import Item from stac_api.utils import CommandHandler from stac_api.utils import CustomBaseCommand diff --git a/app/stac_api/management/commands/update_asset_file_size.py b/app/stac_api/management/commands/update_asset_file_size.py index a4591c99..7b6742bc 100644 --- a/app/stac_api/management/commands/update_asset_file_size.py +++ b/app/stac_api/management/commands/update_asset_file_size.py @@ -6,8 +6,8 @@ from django.core.management.base import BaseCommand from django.core.management.base import CommandParser -from stac_api.models import Asset -from stac_api.models import CollectionAsset +from stac_api.models.general import Asset +from stac_api.models.general import CollectionAsset from stac_api.utils import CommandHandler from stac_api.utils import get_s3_client from stac_api.utils import select_s3_bucket diff --git a/app/stac_api/migrations/0001_initial.py b/app/stac_api/migrations/0001_initial.py index 529ed67d..9857f8f0 100644 --- a/app/stac_api/migrations/0001_initial.py +++ b/app/stac_api/migrations/0001_initial.py @@ -7,7 +7,7 @@ from django.db import migrations from django.db import models -import stac_api.models +import stac_api.models.general import stac_api.validators @@ -74,7 +74,7 @@ class Migration(migrations.Migration): ( 'summaries', models.JSONField( - default=stac_api.models.get_default_summaries_value, + default=stac_api.models.general.get_default_summaries_value, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder ) @@ -102,7 +102,7 @@ class Migration(migrations.Migration): 'conformsTo', django.contrib.postgres.fields.ArrayField( base_field=models.URLField(), - default=stac_api.models.get_conformance_default_links, + default=stac_api.models.general.get_conformance_default_links, help_text='Comma-separated list of URLs for the value conformsTo', size=None ) @@ -335,7 +335,7 @@ class Migration(migrations.Migration): ( 'file', models.FileField( - max_length=255, upload_to=stac_api.models.upload_asset_to_path_hook + max_length=255, upload_to=stac_api.models.general.upload_asset_to_path_hook ) ), ( diff --git a/app/stac_api/migrations/0005_auto_20210408_0821.py b/app/stac_api/migrations/0005_auto_20210408_0821.py index 7ef461a7..a7079354 100644 --- a/app/stac_api/migrations/0005_auto_20210408_0821.py +++ b/app/stac_api/migrations/0005_auto_20210408_0821.py @@ -6,7 +6,7 @@ from django.db import migrations from django.db import models -import stac_api.models +import stac_api.models.general class Migration(migrations.Migration): @@ -50,7 +50,10 @@ class Migration(migrations.Migration): ('created', models.DateTimeField(auto_now_add=True)), ('ended', models.DateTimeField(blank=True, default=None, null=True)), ('checksum_multihash', models.CharField(max_length=255)), - ('etag', models.CharField(default=stac_api.models.compute_etag, max_length=56)), + ( + 'etag', + models.CharField(default=stac_api.models.general.compute_etag, max_length=56) + ), ( 'asset', models.ForeignKey( diff --git a/app/stac_api/migrations/0014_auto_20210715_1358.py b/app/stac_api/migrations/0014_auto_20210715_1358.py index 032244c7..11493eee 100644 --- a/app/stac_api/migrations/0014_auto_20210715_1358.py +++ b/app/stac_api/migrations/0014_auto_20210715_1358.py @@ -6,7 +6,7 @@ from django.db import migrations from django.db import models -import stac_api.models +import stac_api.models.general class Migration(migrations.Migration): @@ -49,7 +49,7 @@ class Migration(migrations.Migration): model_name='asset', name='etag', field=models.CharField( - default=stac_api.models.compute_etag, editable=False, max_length=56 + default=stac_api.models.general.compute_etag, editable=False, max_length=56 ), ), migrations.AlterField( @@ -61,7 +61,7 @@ class Migration(migrations.Migration): model_name='collection', name='etag', field=models.CharField( - default=stac_api.models.compute_etag, editable=False, max_length=56 + default=stac_api.models.general.compute_etag, editable=False, max_length=56 ), ), migrations.AlterField( @@ -89,7 +89,7 @@ class Migration(migrations.Migration): model_name='item', name='etag', field=models.CharField( - default=stac_api.models.compute_etag, editable=False, max_length=56 + default=stac_api.models.general.compute_etag, editable=False, max_length=56 ), ), migrations.AlterField( diff --git a/app/stac_api/migrations/0032_alter_asset_file.py b/app/stac_api/migrations/0032_alter_asset_file.py index ed8a336d..76164f48 100644 --- a/app/stac_api/migrations/0032_alter_asset_file.py +++ b/app/stac_api/migrations/0032_alter_asset_file.py @@ -2,7 +2,7 @@ from django.db import migrations -import stac_api.models +import stac_api.models.general class Migration(migrations.Migration): @@ -15,8 +15,8 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='asset', name='file', - field=stac_api.models.DynamicStorageFileField( - max_length=255, upload_to=stac_api.models.upload_asset_to_path_hook + field=stac_api.models.general.DynamicStorageFileField( + max_length=255, upload_to=stac_api.models.general.upload_asset_to_path_hook ), ), ] diff --git a/app/stac_api/migrations/0032_delete_conformancepage_landingpage_conformsto_and_more.py b/app/stac_api/migrations/0032_delete_conformancepage_landingpage_conformsto_and_more.py index ff457375..9a19fccc 100644 --- a/app/stac_api/migrations/0032_delete_conformancepage_landingpage_conformsto_and_more.py +++ b/app/stac_api/migrations/0032_delete_conformancepage_landingpage_conformsto_and_more.py @@ -4,7 +4,7 @@ from django.db import migrations from django.db import models -import stac_api.models +import stac_api.models.general import stac_api.validators @@ -21,7 +21,7 @@ class Migration(migrations.Migration): name='conformsTo', field=django.contrib.postgres.fields.ArrayField( base_field=models.URLField(), - default=stac_api.models.get_conformance_default_links, + default=stac_api.models.general.get_conformance_default_links, help_text='Comma-separated list of URLs for the value conformsTo', size=None ), diff --git a/app/stac_api/migrations/0036_collectionasset_and_more.py b/app/stac_api/migrations/0036_collectionasset_and_more.py index efddc860..d69e0a16 100644 --- a/app/stac_api/migrations/0036_collectionasset_and_more.py +++ b/app/stac_api/migrations/0036_collectionasset_and_more.py @@ -9,7 +9,7 @@ from django.db import migrations from django.db import models -import stac_api.models +import stac_api.models.general import stac_api.validators @@ -34,8 +34,8 @@ class Migration(migrations.Migration): ), ( 'file', - stac_api.models.DynamicStorageFileField( - max_length=255, upload_to=stac_api.models.upload_asset_to_path_hook + stac_api.models.general.DynamicStorageFileField( + max_length=255, upload_to=stac_api.models.general.upload_asset_to_path_hook ) ), ( @@ -192,7 +192,7 @@ class Migration(migrations.Migration): ( 'etag', models.CharField( - default=stac_api.models.compute_etag, editable=False, max_length=56 + default=stac_api.models.general.compute_etag, editable=False, max_length=56 ) ), ( diff --git a/app/stac_api/migrations/0050_collectionassetupload_and_more.py b/app/stac_api/migrations/0050_collectionassetupload_and_more.py index 1238a62a..0e01547b 100644 --- a/app/stac_api/migrations/0050_collectionassetupload_and_more.py +++ b/app/stac_api/migrations/0050_collectionassetupload_and_more.py @@ -9,7 +9,7 @@ from django.db import migrations from django.db import models -import stac_api.models +import stac_api.models.general class Migration(migrations.Migration): @@ -69,7 +69,10 @@ class Migration(migrations.Migration): ('created', models.DateTimeField(auto_now_add=True)), ('ended', models.DateTimeField(blank=True, default=None, null=True)), ('checksum_multihash', models.CharField(max_length=255)), - ('etag', models.CharField(default=stac_api.models.compute_etag, max_length=56)), + ( + 'etag', + models.CharField(default=stac_api.models.general.compute_etag, max_length=56) + ), ( 'update_interval', models.IntegerField( diff --git a/app/stac_api/models/__init__.py b/app/stac_api/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/stac_api/models.py b/app/stac_api/models/general.py similarity index 100% rename from app/stac_api/models.py rename to app/stac_api/models/general.py diff --git a/app/stac_api/s3_multipart_upload.py b/app/stac_api/s3_multipart_upload.py index a8983127..14dae036 100644 --- a/app/stac_api/s3_multipart_upload.py +++ b/app/stac_api/s3_multipart_upload.py @@ -12,8 +12,8 @@ from rest_framework import serializers from stac_api.exceptions import UploadNotInProgressError -from stac_api.models import Asset -from stac_api.models import CollectionAsset +from stac_api.models.general import Asset +from stac_api.models.general import CollectionAsset from stac_api.utils import AVAILABLE_S3_BUCKETS from stac_api.utils import get_s3_cache_control_value from stac_api.utils import get_s3_client diff --git a/app/stac_api/sample_data/importer.py b/app/stac_api/sample_data/importer.py index 3194a99b..c308f90e 100644 --- a/app/stac_api/sample_data/importer.py +++ b/app/stac_api/sample_data/importer.py @@ -9,11 +9,11 @@ from django.contrib.gis.geos import GEOSGeometry from django.core.files.uploadedfile import SimpleUploadedFile -from stac_api.models import Asset -from stac_api.models import Collection -from stac_api.models import CollectionLink -from stac_api.models import Item -from stac_api.models import Provider +from stac_api.models.general import Asset +from stac_api.models.general import Collection +from stac_api.models.general import CollectionLink +from stac_api.models.general import Item +from stac_api.models.general import Provider # path definition relative to the directory that contains manage.py DATADIR = settings.BASE_DIR / 'app/stac_api/management/sample_data/' diff --git a/app/stac_api/serializers/collection.py b/app/stac_api/serializers/collection.py index d66547f0..dc28dc10 100644 --- a/app/stac_api/serializers/collection.py +++ b/app/stac_api/serializers/collection.py @@ -4,10 +4,10 @@ from rest_framework import serializers -from stac_api.models import Collection -from stac_api.models import CollectionAsset -from stac_api.models import CollectionLink -from stac_api.models import Provider +from stac_api.models.general import Collection +from stac_api.models.general import CollectionAsset +from stac_api.models.general import CollectionLink +from stac_api.models.general import Provider from stac_api.serializers.utils import AssetsDictSerializer from stac_api.serializers.utils import HrefField from stac_api.serializers.utils import NonNullModelSerializer diff --git a/app/stac_api/serializers/general.py b/app/stac_api/serializers/general.py index 4d1aba7e..876e3342 100644 --- a/app/stac_api/serializers/general.py +++ b/app/stac_api/serializers/general.py @@ -8,8 +8,8 @@ from rest_framework import serializers from rest_framework.validators import UniqueValidator -from stac_api.models import LandingPage -from stac_api.models import LandingPageLink +from stac_api.models.general import LandingPage +from stac_api.models.general import LandingPageLink from stac_api.utils import get_browser_url from stac_api.utils import get_stac_version from stac_api.utils import get_url diff --git a/app/stac_api/serializers/item.py b/app/stac_api/serializers/item.py index f79bad37..93655089 100644 --- a/app/stac_api/serializers/item.py +++ b/app/stac_api/serializers/item.py @@ -8,9 +8,9 @@ from rest_framework import serializers from rest_framework_gis import serializers as gis_serializers -from stac_api.models import Asset -from stac_api.models import Item -from stac_api.models import ItemLink +from stac_api.models.general import Asset +from stac_api.models.general import Item +from stac_api.models.general import ItemLink from stac_api.serializers.utils import AssetsDictSerializer from stac_api.serializers.utils import HrefField from stac_api.serializers.utils import IsoDurationField diff --git a/app/stac_api/serializers/upload.py b/app/stac_api/serializers/upload.py index 9a42887c..691f942c 100644 --- a/app/stac_api/serializers/upload.py +++ b/app/stac_api/serializers/upload.py @@ -5,8 +5,8 @@ from rest_framework import serializers from rest_framework.utils.serializer_helpers import ReturnDict -from stac_api.models import AssetUpload -from stac_api.models import CollectionAssetUpload +from stac_api.models.general import AssetUpload +from stac_api.models.general import CollectionAssetUpload from stac_api.serializers.utils import NonNullModelSerializer from stac_api.utils import is_api_version_1 from stac_api.utils import isoformat diff --git a/app/stac_api/serializers/utils.py b/app/stac_api/serializers/utils.py index 9ac92085..8b54dc9b 100644 --- a/app/stac_api/serializers/utils.py +++ b/app/stac_api/serializers/utils.py @@ -10,9 +10,9 @@ from rest_framework import serializers from rest_framework.utils.serializer_helpers import ReturnDict -from stac_api.models import Collection -from stac_api.models import Item -from stac_api.models import Link +from stac_api.models.general import Collection +from stac_api.models.general import Item +from stac_api.models.general import Link from stac_api.utils import build_asset_href from stac_api.utils import get_browser_url from stac_api.utils import get_url diff --git a/app/stac_api/signals.py b/app/stac_api/signals.py index 0410b70f..1e95fa19 100644 --- a/app/stac_api/signals.py +++ b/app/stac_api/signals.py @@ -4,10 +4,10 @@ from django.db.models.signals import pre_delete from django.dispatch import receiver -from stac_api.models import Asset -from stac_api.models import AssetUpload -from stac_api.models import CollectionAsset -from stac_api.models import CollectionAssetUpload +from stac_api.models.general import Asset +from stac_api.models.general import AssetUpload +from stac_api.models.general import CollectionAsset +from stac_api.models.general import CollectionAssetUpload logger = logging.getLogger(__name__) diff --git a/app/stac_api/validators_view.py b/app/stac_api/validators_view.py index c8bd571d..ce720508 100644 --- a/app/stac_api/validators_view.py +++ b/app/stac_api/validators_view.py @@ -7,10 +7,10 @@ from rest_framework import serializers -from stac_api.models import Asset -from stac_api.models import Collection -from stac_api.models import CollectionAsset -from stac_api.models import Item +from stac_api.models.general import Asset +from stac_api.models.general import Collection +from stac_api.models.general import CollectionAsset +from stac_api.models.general import Item logger = logging.getLogger(__name__) diff --git a/app/stac_api/views/collection.py b/app/stac_api/views/collection.py index 708e0a6c..3955de1a 100644 --- a/app/stac_api/views/collection.py +++ b/app/stac_api/views/collection.py @@ -8,8 +8,8 @@ from rest_framework.response import Response from rest_framework_condition import etag -from stac_api.models import Collection -from stac_api.models import CollectionAsset +from stac_api.models.general import Collection +from stac_api.models.general import CollectionAsset from stac_api.serializers.collection import CollectionAssetSerializer from stac_api.serializers.collection import CollectionSerializer from stac_api.serializers.utils import get_relation_links diff --git a/app/stac_api/views/general.py b/app/stac_api/views/general.py index 662ed813..c0192961 100644 --- a/app/stac_api/views/general.py +++ b/app/stac_api/views/general.py @@ -14,8 +14,8 @@ from rest_framework.permissions import AllowAny from rest_framework.response import Response -from stac_api.models import Item -from stac_api.models import LandingPage +from stac_api.models.general import Item +from stac_api.models.general import LandingPage from stac_api.pagination import GetPostCursorPagination from stac_api.serializers.general import ConformancePageSerializer from stac_api.serializers.general import LandingPageSerializer diff --git a/app/stac_api/views/item.py b/app/stac_api/views/item.py index 8f024d6d..4e1de5ff 100644 --- a/app/stac_api/views/item.py +++ b/app/stac_api/views/item.py @@ -12,9 +12,9 @@ from rest_framework.response import Response from rest_framework_condition import etag -from stac_api.models import Asset -from stac_api.models import Collection -from stac_api.models import Item +from stac_api.models.general import Asset +from stac_api.models.general import Collection +from stac_api.models.general import Item from stac_api.serializers.item import AssetSerializer from stac_api.serializers.item import ItemSerializer from stac_api.serializers.utils import get_relation_links diff --git a/app/stac_api/views/test.py b/app/stac_api/views/test.py index 85b7d1ab..76c5ba4f 100644 --- a/app/stac_api/views/test.py +++ b/app/stac_api/views/test.py @@ -2,7 +2,7 @@ from rest_framework import generics -from stac_api.models import LandingPage +from stac_api.models.general import LandingPage from stac_api.views.collection import CollectionAssetDetail from stac_api.views.collection import CollectionDetail from stac_api.views.item import AssetDetail diff --git a/app/stac_api/views/upload.py b/app/stac_api/views/upload.py index ed84f4a7..b8eb1197 100644 --- a/app/stac_api/views/upload.py +++ b/app/stac_api/views/upload.py @@ -16,11 +16,11 @@ from stac_api.exceptions import UploadInProgressError from stac_api.exceptions import UploadNotInProgressError -from stac_api.models import Asset -from stac_api.models import AssetUpload -from stac_api.models import BaseAssetUpload -from stac_api.models import CollectionAsset -from stac_api.models import CollectionAssetUpload +from stac_api.models.general import Asset +from stac_api.models.general import AssetUpload +from stac_api.models.general import BaseAssetUpload +from stac_api.models.general import CollectionAsset +from stac_api.models.general import CollectionAssetUpload from stac_api.pagination import ExtApiPagination from stac_api.s3_multipart_upload import MultipartUpload from stac_api.serializers.upload import AssetUploadPartsSerializer diff --git a/app/tests/base_test_admin_page.py b/app/tests/base_test_admin_page.py index 3d83ae8e..07e65351 100644 --- a/app/tests/base_test_admin_page.py +++ b/app/tests/base_test_admin_page.py @@ -7,12 +7,12 @@ from django.test import TestCase from django.urls import reverse -from stac_api.models import Asset -from stac_api.models import Collection -from stac_api.models import CollectionLink -from stac_api.models import Item -from stac_api.models import ItemLink -from stac_api.models import Provider +from stac_api.models.general import Asset +from stac_api.models.general import Collection +from stac_api.models.general import CollectionLink +from stac_api.models.general import Item +from stac_api.models.general import ItemLink +from stac_api.models.general import Provider from tests.tests_09.data_factory import Factory diff --git a/app/tests/test_admin_page.py b/app/tests/test_admin_page.py index 4d339ce3..e2570e6b 100644 --- a/app/tests/test_admin_page.py +++ b/app/tests/test_admin_page.py @@ -5,12 +5,12 @@ from django.test import override_settings from django.urls import reverse -from stac_api.models import Asset -from stac_api.models import Collection -from stac_api.models import CollectionLink -from stac_api.models import Item -from stac_api.models import ItemLink -from stac_api.models import Provider +from stac_api.models.general import Asset +from stac_api.models.general import Collection +from stac_api.models.general import CollectionLink +from stac_api.models.general import Item +from stac_api.models.general import ItemLink +from stac_api.models.general import Provider from stac_api.utils import parse_multihash from tests.base_test_admin_page import AdminBaseTestCase diff --git a/app/tests/tests_09/data_factory.py b/app/tests/tests_09/data_factory.py index c9474ab3..57f949a2 100644 --- a/app/tests/tests_09/data_factory.py +++ b/app/tests/tests_09/data_factory.py @@ -97,12 +97,12 @@ from django.core.files.base import File from django.core.files.uploadedfile import SimpleUploadedFile -from stac_api.models import Asset -from stac_api.models import Collection -from stac_api.models import CollectionLink -from stac_api.models import Item -from stac_api.models import ItemLink -from stac_api.models import Provider +from stac_api.models.general import Asset +from stac_api.models.general import Collection +from stac_api.models.general import CollectionLink +from stac_api.models.general import Item +from stac_api.models.general import ItemLink +from stac_api.models.general import Provider from stac_api.utils import get_s3_resource from stac_api.utils import isoformat from stac_api.validators import get_media_type diff --git a/app/tests/tests_09/sample_data/item_samples.py b/app/tests/tests_09/sample_data/item_samples.py index 55babe4b..3395925a 100644 --- a/app/tests/tests_09/sample_data/item_samples.py +++ b/app/tests/tests_09/sample_data/item_samples.py @@ -2,7 +2,7 @@ from django.contrib.gis.geos import GEOSGeometry -from stac_api.models import BBOX_CH +from stac_api.models.general import BBOX_CH from stac_api.utils import fromisoformat geometries = { diff --git a/app/tests/tests_09/test_asset_model.py b/app/tests/tests_09/test_asset_model.py index 17aa9f42..4c52bf46 100644 --- a/app/tests/tests_09/test_asset_model.py +++ b/app/tests/tests_09/test_asset_model.py @@ -2,7 +2,7 @@ from django.core.exceptions import ValidationError -from stac_api.models import Asset +from stac_api.models.general import Asset from tests.tests_09.base_test import StacBaseTransactionTestCase from tests.tests_09.data_factory import Factory diff --git a/app/tests/tests_09/test_asset_upload_endpoint.py b/app/tests/tests_09/test_asset_upload_endpoint.py index 8775a20d..bc50210a 100644 --- a/app/tests/tests_09/test_asset_upload_endpoint.py +++ b/app/tests/tests_09/test_asset_upload_endpoint.py @@ -10,8 +10,8 @@ from django.contrib.auth import get_user_model from django.test import Client -from stac_api.models import Asset -from stac_api.models import AssetUpload +from stac_api.models.general import Asset +from stac_api.models.general import AssetUpload from stac_api.utils import fromisoformat from stac_api.utils import get_asset_path from stac_api.utils import get_s3_client diff --git a/app/tests/tests_09/test_asset_upload_model.py b/app/tests/tests_09/test_asset_upload_model.py index f48f42e9..1c84e291 100644 --- a/app/tests/tests_09/test_asset_upload_model.py +++ b/app/tests/tests_09/test_asset_upload_model.py @@ -6,8 +6,8 @@ from django.test import TestCase from django.test import TransactionTestCase -from stac_api.models import Asset -from stac_api.models import AssetUpload +from stac_api.models.general import Asset +from stac_api.models.general import AssetUpload from stac_api.utils import get_sha256_multihash from stac_api.utils import utc_aware diff --git a/app/tests/tests_09/test_assets_endpoint.py b/app/tests/tests_09/test_assets_endpoint.py index c4dc33a1..980dcebe 100644 --- a/app/tests/tests_09/test_assets_endpoint.py +++ b/app/tests/tests_09/test_assets_endpoint.py @@ -8,7 +8,7 @@ from django.test import Client from django.urls import reverse -from stac_api.models import Asset +from stac_api.models.general import Asset from stac_api.utils import get_asset_path from stac_api.utils import utc_aware diff --git a/app/tests/tests_09/test_collection_model.py b/app/tests/tests_09/test_collection_model.py index cfc83d05..1142c0b2 100644 --- a/app/tests/tests_09/test_collection_model.py +++ b/app/tests/tests_09/test_collection_model.py @@ -3,7 +3,7 @@ from django.core.exceptions import ValidationError -from stac_api.models import Collection +from stac_api.models.general import Collection from tests.tests_09.base_test import StacBaseTransactionTestCase from tests.tests_09.data_factory import Factory diff --git a/app/tests/tests_09/test_collections_endpoint.py b/app/tests/tests_09/test_collections_endpoint.py index 99e21dbf..8a40150a 100644 --- a/app/tests/tests_09/test_collections_endpoint.py +++ b/app/tests/tests_09/test_collections_endpoint.py @@ -5,9 +5,9 @@ from django.test import Client from django.urls import reverse -from stac_api.models import Collection -from stac_api.models import CollectionLink -from stac_api.models import Provider +from stac_api.models.general import Collection +from stac_api.models.general import CollectionLink +from stac_api.models.general import Provider from stac_api.utils import utc_aware from tests.tests_09.base_test import STAC_BASE_V diff --git a/app/tests/tests_09/test_collections_extent.py b/app/tests/tests_09/test_collections_extent.py index 943c24b8..d25ff329 100644 --- a/app/tests/tests_09/test_collections_extent.py +++ b/app/tests/tests_09/test_collections_extent.py @@ -4,7 +4,7 @@ from django.contrib.gis.geos import GEOSGeometry from django.contrib.gis.geos import Polygon -from stac_api.models import Item +from stac_api.models.general import Item from stac_api.utils import utc_aware from tests.tests_09.base_test import StacBaseTransactionTestCase diff --git a/app/tests/tests_09/test_generic_api.py b/app/tests/tests_09/test_generic_api.py index 88544420..c8252897 100644 --- a/app/tests/tests_09/test_generic_api.py +++ b/app/tests/tests_09/test_generic_api.py @@ -5,7 +5,7 @@ from django.test import Client from django.test import override_settings -from stac_api.models import AssetUpload +from stac_api.models.general import AssetUpload from stac_api.utils import get_asset_path from stac_api.utils import get_link from stac_api.utils import get_sha256_multihash diff --git a/app/tests/tests_09/test_item_model.py b/app/tests/tests_09/test_item_model.py index b7c8cfd5..3f84a128 100644 --- a/app/tests/tests_09/test_item_model.py +++ b/app/tests/tests_09/test_item_model.py @@ -6,8 +6,8 @@ from django.core.exceptions import ValidationError from django.test import TestCase -from stac_api.models import Collection -from stac_api.models import Item +from stac_api.models.general import Collection +from stac_api.models.general import Item from stac_api.utils import utc_aware from tests.tests_09.data_factory import CollectionFactory diff --git a/app/tests/tests_09/test_items_endpoint.py b/app/tests/tests_09/test_items_endpoint.py index 3f2aaeeb..37640c0c 100644 --- a/app/tests/tests_09/test_items_endpoint.py +++ b/app/tests/tests_09/test_items_endpoint.py @@ -6,7 +6,7 @@ from django.test import Client from django.urls import reverse -from stac_api.models import Item +from stac_api.models.general import Item from stac_api.utils import fromisoformat from stac_api.utils import get_link from stac_api.utils import isoformat diff --git a/app/tests/tests_09/test_items_endpoint_bbox.py b/app/tests/tests_09/test_items_endpoint_bbox.py index 678d89b3..848d9a67 100644 --- a/app/tests/tests_09/test_items_endpoint_bbox.py +++ b/app/tests/tests_09/test_items_endpoint_bbox.py @@ -3,7 +3,7 @@ from django.contrib.gis.geos.geometry import GEOSGeometry from django.test import Client -from stac_api.models import BBOX_CH +from stac_api.models.general import BBOX_CH from tests.tests_09.base_test import STAC_BASE_V from tests.tests_09.base_test import StacBaseTestCase diff --git a/app/tests/tests_09/test_serializer.py b/app/tests/tests_09/test_serializer.py index 66374304..c6386104 100644 --- a/app/tests/tests_09/test_serializer.py +++ b/app/tests/tests_09/test_serializer.py @@ -12,7 +12,7 @@ from rest_framework.renderers import JSONRenderer from rest_framework.test import APIRequestFactory -from stac_api.models import get_asset_path +from stac_api.models.general import get_asset_path from stac_api.serializers.collection import CollectionSerializer from stac_api.serializers.item import AssetSerializer from stac_api.serializers.item import ItemSerializer diff --git a/app/tests/tests_09/test_serializer_asset_upload.py b/app/tests/tests_09/test_serializer_asset_upload.py index e5a54e43..4dcb25a2 100644 --- a/app/tests/tests_09/test_serializer_asset_upload.py +++ b/app/tests/tests_09/test_serializer_asset_upload.py @@ -6,7 +6,7 @@ from rest_framework import serializers from rest_framework.exceptions import ValidationError -from stac_api.models import AssetUpload +from stac_api.models.general import AssetUpload from stac_api.serializers.upload import AssetUploadSerializer from stac_api.utils import get_sha256_multihash diff --git a/app/tests/tests_10/data_factory.py b/app/tests/tests_10/data_factory.py index 4a522de1..38cbd4bf 100644 --- a/app/tests/tests_10/data_factory.py +++ b/app/tests/tests_10/data_factory.py @@ -97,13 +97,13 @@ from django.core.files.base import File from django.core.files.uploadedfile import SimpleUploadedFile -from stac_api.models import Asset -from stac_api.models import Collection -from stac_api.models import CollectionAsset -from stac_api.models import CollectionLink -from stac_api.models import Item -from stac_api.models import ItemLink -from stac_api.models import Provider +from stac_api.models.general import Asset +from stac_api.models.general import Collection +from stac_api.models.general import CollectionAsset +from stac_api.models.general import CollectionLink +from stac_api.models.general import Item +from stac_api.models.general import ItemLink +from stac_api.models.general import Provider from stac_api.utils import get_s3_resource from stac_api.utils import isoformat from stac_api.validators import get_media_type diff --git a/app/tests/tests_10/sample_data/item_samples.py b/app/tests/tests_10/sample_data/item_samples.py index 5b4dd12a..adfd76e0 100644 --- a/app/tests/tests_10/sample_data/item_samples.py +++ b/app/tests/tests_10/sample_data/item_samples.py @@ -2,7 +2,7 @@ from django.contrib.gis.geos import GEOSGeometry -from stac_api.models import BBOX_CH +from stac_api.models.general import BBOX_CH from stac_api.utils import fromisoformat geometries = { diff --git a/app/tests/tests_10/test_asset_model.py b/app/tests/tests_10/test_asset_model.py index 01f37578..f424dd66 100644 --- a/app/tests/tests_10/test_asset_model.py +++ b/app/tests/tests_10/test_asset_model.py @@ -2,7 +2,7 @@ from django.core.exceptions import ValidationError -from stac_api.models import Asset +from stac_api.models.general import Asset from tests.tests_10.base_test import StacBaseTransactionTestCase from tests.tests_10.data_factory import Factory diff --git a/app/tests/tests_10/test_asset_upload_endpoint.py b/app/tests/tests_10/test_asset_upload_endpoint.py index ef569c76..91465f5a 100644 --- a/app/tests/tests_10/test_asset_upload_endpoint.py +++ b/app/tests/tests_10/test_asset_upload_endpoint.py @@ -10,8 +10,8 @@ from django.contrib.auth import get_user_model from django.test import Client -from stac_api.models import Asset -from stac_api.models import AssetUpload +from stac_api.models.general import Asset +from stac_api.models.general import AssetUpload from stac_api.utils import fromisoformat from stac_api.utils import get_asset_path from stac_api.utils import get_s3_client diff --git a/app/tests/tests_10/test_asset_upload_model.py b/app/tests/tests_10/test_asset_upload_model.py index c8809762..21c4dffd 100644 --- a/app/tests/tests_10/test_asset_upload_model.py +++ b/app/tests/tests_10/test_asset_upload_model.py @@ -6,8 +6,8 @@ from django.test import TestCase from django.test import TransactionTestCase -from stac_api.models import Asset -from stac_api.models import AssetUpload +from stac_api.models.general import Asset +from stac_api.models.general import AssetUpload from stac_api.utils import get_sha256_multihash from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_assets_endpoint.py b/app/tests/tests_10/test_assets_endpoint.py index d2afbd36..987bb470 100644 --- a/app/tests/tests_10/test_assets_endpoint.py +++ b/app/tests/tests_10/test_assets_endpoint.py @@ -10,7 +10,7 @@ from django.urls import reverse from django.utils import timezone -from stac_api.models import Asset +from stac_api.models.general import Asset from stac_api.utils import get_asset_path from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_collection_asset_model.py b/app/tests/tests_10/test_collection_asset_model.py index 21cf9f9a..9cb8f172 100644 --- a/app/tests/tests_10/test_collection_asset_model.py +++ b/app/tests/tests_10/test_collection_asset_model.py @@ -2,7 +2,7 @@ from django.core.exceptions import ValidationError -from stac_api.models import CollectionAsset +from stac_api.models.general import CollectionAsset from tests.tests_10.base_test import StacBaseTransactionTestCase from tests.tests_10.data_factory import Factory diff --git a/app/tests/tests_10/test_collection_asset_upload_endpoint.py b/app/tests/tests_10/test_collection_asset_upload_endpoint.py index 9734ebff..95078dc7 100644 --- a/app/tests/tests_10/test_collection_asset_upload_endpoint.py +++ b/app/tests/tests_10/test_collection_asset_upload_endpoint.py @@ -10,8 +10,8 @@ from django.contrib.auth import get_user_model from django.test import Client -from stac_api.models import CollectionAsset -from stac_api.models import CollectionAssetUpload +from stac_api.models.general import CollectionAsset +from stac_api.models.general import CollectionAssetUpload from stac_api.utils import fromisoformat from stac_api.utils import get_collection_asset_path from stac_api.utils import get_s3_client diff --git a/app/tests/tests_10/test_collection_asset_upload_model.py b/app/tests/tests_10/test_collection_asset_upload_model.py index 9ea016aa..bb6e6c50 100644 --- a/app/tests/tests_10/test_collection_asset_upload_model.py +++ b/app/tests/tests_10/test_collection_asset_upload_model.py @@ -6,8 +6,8 @@ from django.test import TestCase from django.test import TransactionTestCase -from stac_api.models import CollectionAsset -from stac_api.models import CollectionAssetUpload +from stac_api.models.general import CollectionAsset +from stac_api.models.general import CollectionAssetUpload from stac_api.utils import get_sha256_multihash from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_collection_assets_endpoint.py b/app/tests/tests_10/test_collection_assets_endpoint.py index 1596a8d9..e782c243 100644 --- a/app/tests/tests_10/test_collection_assets_endpoint.py +++ b/app/tests/tests_10/test_collection_assets_endpoint.py @@ -8,7 +8,7 @@ from django.test import Client from django.urls import reverse -from stac_api.models import CollectionAsset +from stac_api.models.general import CollectionAsset from stac_api.utils import get_collection_asset_path from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_collection_model.py b/app/tests/tests_10/test_collection_model.py index 6e136b92..b6bb6ef4 100644 --- a/app/tests/tests_10/test_collection_model.py +++ b/app/tests/tests_10/test_collection_model.py @@ -3,7 +3,7 @@ from django.core.exceptions import ValidationError -from stac_api.models import Collection +from stac_api.models.general import Collection from tests.tests_10.base_test import StacBaseTransactionTestCase from tests.tests_10.data_factory import Factory diff --git a/app/tests/tests_10/test_collections_endpoint.py b/app/tests/tests_10/test_collections_endpoint.py index 42306568..5c2108e8 100644 --- a/app/tests/tests_10/test_collections_endpoint.py +++ b/app/tests/tests_10/test_collections_endpoint.py @@ -6,9 +6,9 @@ from django.test import Client from django.urls import reverse -from stac_api.models import Collection -from stac_api.models import CollectionLink -from stac_api.models import Provider +from stac_api.models.general import Collection +from stac_api.models.general import CollectionLink +from stac_api.models.general import Provider from stac_api.utils import utc_aware from tests.tests_10.base_test import STAC_BASE_V diff --git a/app/tests/tests_10/test_collections_extent.py b/app/tests/tests_10/test_collections_extent.py index be6957c3..fc308cef 100644 --- a/app/tests/tests_10/test_collections_extent.py +++ b/app/tests/tests_10/test_collections_extent.py @@ -4,7 +4,7 @@ from django.contrib.gis.geos import GEOSGeometry from django.contrib.gis.geos import Polygon -from stac_api.models import Item +from stac_api.models.general import Item from stac_api.utils import utc_aware from tests.tests_10.base_test import StacBaseTransactionTestCase diff --git a/app/tests/tests_10/test_external_assets_endpoint.py b/app/tests/tests_10/test_external_assets_endpoint.py index 4a8829b9..e8bf2fe4 100644 --- a/app/tests/tests_10/test_external_assets_endpoint.py +++ b/app/tests/tests_10/test_external_assets_endpoint.py @@ -3,7 +3,7 @@ from django.conf import settings from django.test import Client -from stac_api.models import Asset +from stac_api.models.general import Asset from tests.tests_10.base_test import StacBaseTestCase from tests.tests_10.data_factory import Factory diff --git a/app/tests/tests_10/test_generic_api.py b/app/tests/tests_10/test_generic_api.py index e5adc074..cb183187 100644 --- a/app/tests/tests_10/test_generic_api.py +++ b/app/tests/tests_10/test_generic_api.py @@ -5,7 +5,7 @@ from django.test import Client from django.test import override_settings -from stac_api.models import AssetUpload +from stac_api.models.general import AssetUpload from stac_api.utils import get_asset_path from stac_api.utils import get_link from stac_api.utils import get_sha256_multihash diff --git a/app/tests/tests_10/test_item_model.py b/app/tests/tests_10/test_item_model.py index c1704bba..a7391f69 100644 --- a/app/tests/tests_10/test_item_model.py +++ b/app/tests/tests_10/test_item_model.py @@ -7,8 +7,8 @@ from django.core.exceptions import ValidationError from django.test import TestCase -from stac_api.models import Collection -from stac_api.models import Item +from stac_api.models.general import Collection +from stac_api.models.general import Item from stac_api.utils import utc_aware from tests.tests_10.data_factory import CollectionFactory diff --git a/app/tests/tests_10/test_items_endpoint.py b/app/tests/tests_10/test_items_endpoint.py index 38397530..cc7ca6db 100644 --- a/app/tests/tests_10/test_items_endpoint.py +++ b/app/tests/tests_10/test_items_endpoint.py @@ -9,9 +9,9 @@ from django.urls import reverse from django.utils import timezone -from stac_api.models import Collection -from stac_api.models import Item -from stac_api.models import ItemLink +from stac_api.models.general import Collection +from stac_api.models.general import Item +from stac_api.models.general import ItemLink from stac_api.utils import fromisoformat from stac_api.utils import get_link from stac_api.utils import isoformat diff --git a/app/tests/tests_10/test_items_endpoint_bbox.py b/app/tests/tests_10/test_items_endpoint_bbox.py index b327ad0e..47a6e7d4 100644 --- a/app/tests/tests_10/test_items_endpoint_bbox.py +++ b/app/tests/tests_10/test_items_endpoint_bbox.py @@ -3,7 +3,7 @@ from django.contrib.gis.geos.geometry import GEOSGeometry from django.test import Client -from stac_api.models import BBOX_CH +from stac_api.models.general import BBOX_CH from tests.tests_10.base_test import STAC_BASE_V from tests.tests_10.base_test import StacBaseTestCase diff --git a/app/tests/tests_10/test_remove_expired_items.py b/app/tests/tests_10/test_remove_expired_items.py index 505dcd33..c7a80c95 100644 --- a/app/tests/tests_10/test_remove_expired_items.py +++ b/app/tests/tests_10/test_remove_expired_items.py @@ -5,8 +5,8 @@ from django.test import TestCase from django.utils import timezone -from stac_api.models import Asset -from stac_api.models import Item +from stac_api.models.general import Asset +from stac_api.models.general import Item from tests.tests_10.data_factory import Factory from tests.utils import mock_s3_asset_file diff --git a/app/tests/tests_10/test_serializer.py b/app/tests/tests_10/test_serializer.py index e757bcc9..68d9aa43 100644 --- a/app/tests/tests_10/test_serializer.py +++ b/app/tests/tests_10/test_serializer.py @@ -14,7 +14,7 @@ from rest_framework.renderers import JSONRenderer from rest_framework.test import APIRequestFactory -from stac_api.models import get_asset_path +from stac_api.models.general import get_asset_path from stac_api.serializers.collection import CollectionSerializer from stac_api.serializers.item import AssetSerializer from stac_api.serializers.item import ItemSerializer diff --git a/app/tests/tests_10/test_serializer_asset_upload.py b/app/tests/tests_10/test_serializer_asset_upload.py index c5d359a2..34e4ba29 100644 --- a/app/tests/tests_10/test_serializer_asset_upload.py +++ b/app/tests/tests_10/test_serializer_asset_upload.py @@ -6,7 +6,7 @@ from rest_framework import serializers from rest_framework.exceptions import ValidationError -from stac_api.models import AssetUpload +from stac_api.models.general import AssetUpload from stac_api.serializers.upload import AssetUploadSerializer from stac_api.utils import get_sha256_multihash diff --git a/scripts/fill_local_db.py b/scripts/fill_local_db.py index 6db19f8e..47690898 100644 --- a/scripts/fill_local_db.py +++ b/scripts/fill_local_db.py @@ -19,7 +19,7 @@ import io from rest_framework.renderers import JSONRenderer from rest_framework.parsers import JSONParser -from stac_api.models import * +from stac_api.models.general import * from stac_api.serializers.general import CollectionSerializer from stac_api.serializers.general import CollectionSerializer from stac_api.serializers.general import LinkSerializer @@ -28,14 +28,12 @@ # create link instances for testing link_root = CollectionLink.objects.create( - item=self.item, - href="https://data.geo.admin.ch/api/stac/v0.9/", - rel='root', - link_type='root', - title='Root link' - ) - - + item=self.item, + href="https://data.geo.admin.ch/api/stac/v0.9/", + rel='root', + link_type='root', + title='Root link' +) # create provider instances for testing provider1 = Provider( @@ -60,33 +58,19 @@ updated=datetime.now(), description='test', extent={ - "spatial": { - "bbox": [ - [ - 5.685114, - 45.534903, - 10.747775, - 47.982586 - ] - ] + "spatial": { + "bbox": [[5.685114, 45.534903, 10.747775, 47.982586]] + }, + "temporal": { + "interval": [["2019", None]] + } }, - "temporal": { - "interval": [ - [ - "2019", - None - ] - ] - } - }, collection_name='my collection', item_type='Feature', license='test', - summaries = { - "eo:gsd": [10,20], - "geoadmin:variant": ["kgrel", "komb", "krel"], - "proj:epsg": [2056] - }, + summaries={ + "eo:gsd": [10, 20], "geoadmin:variant": ["kgrel", "komb", "krel"], "proj:epsg": [2056] + }, title='testtitel2' ) @@ -104,33 +88,19 @@ updated=datetime.now(), description='test', extent={ - "spatial": { - "bbox": [ - [ - 5.685114, - 45.534903, - 10.747775, - 47.982586 - ] - ] + "spatial": { + "bbox": [[5.685114, 45.534903, 10.747775, 47.982586]] + }, + "temporal": { + "interval": [["2019", None]] + } }, - "temporal": { - "interval": [ - [ - "2019", - None - ] - ] - } - }, collection_name='b_123', item_type='Feature', license='test', - summaries = { - "eo:gsd": [10,20], - "geoadmin:variant": ["kgrel", "komb", "krel"], - "proj:epsg": [2056] - }, + summaries={ + "eo:gsd": [10, 20], "geoadmin:variant": ["kgrel", "komb", "krel"], "proj:epsg": [2056] + }, title='testtitel2' ) collection2.save() @@ -145,33 +115,19 @@ updated=datetime.now(), description='test', extent={ - "spatial": { - "bbox": [ - [ - 5.685114, - 45.534903, - 10.747775, - 47.982586 - ] - ] + "spatial": { + "bbox": [[5.685114, 45.534903, 10.747775, 47.982586]] + }, + "temporal": { + "interval": [["2019", None]] + } }, - "temporal": { - "interval": [ - [ - "2019", - None - ] - ] - } - }, collection_name='c_123', item_type='Feature', license='test', - summaries = { - "eo:gsd": [10,20], - "geoadmin:variant": ["kgrel", "komb", "krel"], - "proj:epsg": [2056] - }, + summaries={ + "eo:gsd": [10, 20], "geoadmin:variant": ["kgrel", "komb", "krel"], "proj:epsg": [2056] + }, title='testtitel2' ) collection3.save() @@ -185,7 +141,6 @@ provider3.save() link_root.save() - # test the serialization process: # translate into Python native serializer = CollectionSerializer(collection1) From e606b082dd8f5b718b64df5164dd21383d82b49b Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Mon, 27 Jan 2025 12:55:14 +0100 Subject: [PATCH 8/8] Refactor models - split into files Move collection related models into separate file. Move item related models into separate file. Fix import statements across all files. --- app/stac_api/admin.py | 14 +- .../management/commands/calculate_extent.py | 2 +- .../management/commands/dummy_asset_upload.py | 4 +- .../management/commands/dummy_data.py | 6 +- .../management/commands/list_asset_uploads.py | 2 +- .../commands/profile_cursor_paginator.py | 2 +- .../commands/profile_item_serializer.py | 2 +- .../commands/profile_serializer_vs_no_drf.py | 2 +- .../commands/remove_expired_items.py | 4 +- .../commands/update_asset_file_size.py | 4 +- app/stac_api/models/collection.py | 264 +++++++++ app/stac_api/models/general.py | 555 ------------------ app/stac_api/models/item.py | 327 +++++++++++ app/stac_api/s3_multipart_upload.py | 4 +- app/stac_api/sample_data/importer.py | 8 +- app/stac_api/serializers/collection.py | 6 +- app/stac_api/serializers/item.py | 6 +- app/stac_api/serializers/upload.py | 4 +- app/stac_api/serializers/utils.py | 4 +- app/stac_api/signals.py | 8 +- app/stac_api/validators_view.py | 8 +- app/stac_api/views/collection.py | 4 +- app/stac_api/views/general.py | 2 +- app/stac_api/views/item.py | 6 +- app/stac_api/views/upload.py | 8 +- app/tests/base_test_admin_page.py | 10 +- app/tests/test_admin_page.py | 10 +- app/tests/tests_09/data_factory.py | 10 +- app/tests/tests_09/test_asset_model.py | 2 +- .../tests_09/test_asset_upload_endpoint.py | 4 +- app/tests/tests_09/test_asset_upload_model.py | 4 +- app/tests/tests_09/test_assets_endpoint.py | 2 +- app/tests/tests_09/test_collection_model.py | 2 +- .../tests_09/test_collections_endpoint.py | 4 +- app/tests/tests_09/test_collections_extent.py | 2 +- app/tests/tests_09/test_generic_api.py | 2 +- app/tests/tests_09/test_item_model.py | 4 +- app/tests/tests_09/test_items_endpoint.py | 2 +- app/tests/tests_09/test_serializer.py | 2 +- .../tests_09/test_serializer_asset_upload.py | 2 +- app/tests/tests_10/data_factory.py | 12 +- app/tests/tests_10/test_asset_model.py | 2 +- .../tests_10/test_asset_upload_endpoint.py | 4 +- app/tests/tests_10/test_asset_upload_model.py | 4 +- app/tests/tests_10/test_assets_endpoint.py | 2 +- .../tests_10/test_collection_asset_model.py | 2 +- .../test_collection_asset_upload_endpoint.py | 4 +- .../test_collection_asset_upload_model.py | 4 +- .../test_collection_assets_endpoint.py | 2 +- app/tests/tests_10/test_collection_model.py | 2 +- .../tests_10/test_collections_endpoint.py | 4 +- app/tests/tests_10/test_collections_extent.py | 2 +- .../tests_10/test_external_assets_endpoint.py | 2 +- app/tests/tests_10/test_generic_api.py | 2 +- app/tests/tests_10/test_item_model.py | 4 +- app/tests/tests_10/test_items_endpoint.py | 6 +- .../tests_10/test_remove_expired_items.py | 4 +- app/tests/tests_10/test_serializer.py | 2 +- .../tests_10/test_serializer_asset_upload.py | 2 +- 59 files changed, 710 insertions(+), 674 deletions(-) create mode 100644 app/stac_api/models/collection.py create mode 100644 app/stac_api/models/item.py diff --git a/app/stac_api/admin.py b/app/stac_api/admin.py index 494af29a..68ec5a60 100644 --- a/app/stac_api/admin.py +++ b/app/stac_api/admin.py @@ -17,17 +17,17 @@ from django.urls import reverse from django.utils.translation import gettext_lazy as _ +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionAsset +from stac_api.models.collection import CollectionLink from stac_api.models.general import BBOX_CH -from stac_api.models.general import Asset -from stac_api.models.general import AssetUpload -from stac_api.models.general import Collection -from stac_api.models.general import CollectionAsset -from stac_api.models.general import CollectionLink -from stac_api.models.general import Item -from stac_api.models.general import ItemLink from stac_api.models.general import LandingPage from stac_api.models.general import LandingPageLink from stac_api.models.general import Provider +from stac_api.models.item import Asset +from stac_api.models.item import AssetUpload +from stac_api.models.item import Item +from stac_api.models.item import ItemLink from stac_api.utils import build_asset_href from stac_api.utils import get_query_params from stac_api.validators import validate_href_url diff --git a/app/stac_api/management/commands/calculate_extent.py b/app/stac_api/management/commands/calculate_extent.py index 4118d8e9..b0d257dc 100644 --- a/app/stac_api/management/commands/calculate_extent.py +++ b/app/stac_api/management/commands/calculate_extent.py @@ -3,7 +3,7 @@ from django.core.management.base import CommandParser from django.db import connection -from stac_api.models.general import Collection +from stac_api.models.collection import Collection from stac_api.utils import CommandHandler from stac_api.utils import CustomBaseCommand diff --git a/app/stac_api/management/commands/dummy_asset_upload.py b/app/stac_api/management/commands/dummy_asset_upload.py index e79964b6..7232e8c2 100644 --- a/app/stac_api/management/commands/dummy_asset_upload.py +++ b/app/stac_api/management/commands/dummy_asset_upload.py @@ -4,9 +4,9 @@ from django.conf import settings from django.core.management.base import BaseCommand -from stac_api.models.general import Asset -from stac_api.models.general import AssetUpload from stac_api.models.general import BaseAssetUpload +from stac_api.models.item import Asset +from stac_api.models.item import AssetUpload from stac_api.s3_multipart_upload import MultipartUpload from stac_api.utils import AVAILABLE_S3_BUCKETS from stac_api.utils import CommandHandler diff --git a/app/stac_api/management/commands/dummy_data.py b/app/stac_api/management/commands/dummy_data.py index ac13f159..a69702db 100644 --- a/app/stac_api/management/commands/dummy_data.py +++ b/app/stac_api/management/commands/dummy_data.py @@ -13,9 +13,9 @@ from django.core.files.uploadedfile import SimpleUploadedFile from django.core.management.base import BaseCommand -from stac_api.models.general import Asset -from stac_api.models.general import Collection -from stac_api.models.general import Item +from stac_api.models.collection import Collection +from stac_api.models.item import Asset +from stac_api.models.item import Item from stac_api.utils import CommandHandler from stac_api.validators import MEDIA_TYPES diff --git a/app/stac_api/management/commands/list_asset_uploads.py b/app/stac_api/management/commands/list_asset_uploads.py index c0a0f5db..24bb58ce 100644 --- a/app/stac_api/management/commands/list_asset_uploads.py +++ b/app/stac_api/management/commands/list_asset_uploads.py @@ -4,7 +4,7 @@ from django.core.management.base import BaseCommand from django.core.serializers.json import DjangoJSONEncoder -from stac_api.models.general import AssetUpload +from stac_api.models.item import AssetUpload from stac_api.s3_multipart_upload import MultipartUpload from stac_api.serializers.upload import AssetUploadSerializer from stac_api.utils import CommandHandler diff --git a/app/stac_api/management/commands/profile_cursor_paginator.py b/app/stac_api/management/commands/profile_cursor_paginator.py index 311ecff3..18c6174c 100644 --- a/app/stac_api/management/commands/profile_cursor_paginator.py +++ b/app/stac_api/management/commands/profile_cursor_paginator.py @@ -10,7 +10,7 @@ from rest_framework.request import Request from rest_framework.test import APIRequestFactory -from stac_api.models.general import Item +from stac_api.models.item import Item from stac_api.utils import CommandHandler logger = logging.getLogger(__name__) diff --git a/app/stac_api/management/commands/profile_item_serializer.py b/app/stac_api/management/commands/profile_item_serializer.py index 9b43822f..8439598d 100644 --- a/app/stac_api/management/commands/profile_item_serializer.py +++ b/app/stac_api/management/commands/profile_item_serializer.py @@ -8,7 +8,7 @@ from rest_framework.test import APIRequestFactory -from stac_api.models.general import Item +from stac_api.models.item import Item from stac_api.utils import CommandHandler logger = logging.getLogger(__name__) diff --git a/app/stac_api/management/commands/profile_serializer_vs_no_drf.py b/app/stac_api/management/commands/profile_serializer_vs_no_drf.py index 3f2ada21..28ed2a2d 100644 --- a/app/stac_api/management/commands/profile_serializer_vs_no_drf.py +++ b/app/stac_api/management/commands/profile_serializer_vs_no_drf.py @@ -7,7 +7,7 @@ from rest_framework.test import APIRequestFactory -from stac_api.models.general import Item +from stac_api.models.item import Item from stac_api.utils import CommandHandler logger = logging.getLogger(__name__) diff --git a/app/stac_api/management/commands/remove_expired_items.py b/app/stac_api/management/commands/remove_expired_items.py index a1f73a22..6b99042f 100644 --- a/app/stac_api/management/commands/remove_expired_items.py +++ b/app/stac_api/management/commands/remove_expired_items.py @@ -4,9 +4,9 @@ from django.core.management.base import CommandParser from django.utils import timezone -from stac_api.models.general import AssetUpload from stac_api.models.general import BaseAssetUpload -from stac_api.models.general import Item +from stac_api.models.item import AssetUpload +from stac_api.models.item import Item from stac_api.utils import CommandHandler from stac_api.utils import CustomBaseCommand diff --git a/app/stac_api/management/commands/update_asset_file_size.py b/app/stac_api/management/commands/update_asset_file_size.py index 7b6742bc..a243ac0d 100644 --- a/app/stac_api/management/commands/update_asset_file_size.py +++ b/app/stac_api/management/commands/update_asset_file_size.py @@ -6,8 +6,8 @@ from django.core.management.base import BaseCommand from django.core.management.base import CommandParser -from stac_api.models.general import Asset -from stac_api.models.general import CollectionAsset +from stac_api.models.collection import CollectionAsset +from stac_api.models.item import Asset from stac_api.utils import CommandHandler from stac_api.utils import get_s3_client from stac_api.utils import select_s3_bucket diff --git a/app/stac_api/models/collection.py b/app/stac_api/models/collection.py new file mode 100644 index 00000000..efb0bf3e --- /dev/null +++ b/app/stac_api/models/collection.py @@ -0,0 +1,264 @@ +import logging + +from django.contrib.gis.db import models +from django.contrib.postgres.fields import ArrayField +from django.core.serializers.json import DjangoJSONEncoder +from django.core.validators import MinValueValidator +from django.db.models import Q +from django.utils.translation import gettext_lazy as _ + +from stac_api.models.general import SEARCH_TEXT_HELP_ITEM +from stac_api.models.general import AssetBase +from stac_api.models.general import BaseAssetUpload +from stac_api.models.general import Link +from stac_api.models.general import compute_etag +from stac_api.pgtriggers import SummaryFields +from stac_api.pgtriggers import child_triggers +from stac_api.pgtriggers import generates_asset_upload_triggers +from stac_api.pgtriggers import generates_collection_asset_triggers +from stac_api.pgtriggers import generates_collection_triggers +from stac_api.pgtriggers import generates_summary_count_triggers +from stac_api.utils import get_collection_asset_path +from stac_api.validators import validate_name + +logger = logging.getLogger(__name__) + + +# For Collections and Items: No primary key will be defined, so that the auto-generated ones +# will be used by Django. For assets, a primary key is defined as "BigAutoField" due the +# expected large number of assets +class Collection(models.Model): + + class Meta: + indexes = [ + models.Index(fields=['name'], name='collection_name_idx'), + models.Index(fields=['published'], name='collection_published_idx') + ] + triggers = generates_collection_triggers() + + published = models.BooleanField( + default=True, + help_text="When not published the collection doesn't appear on the " + "api/stac/v0.9/collections endpoint and its items are not listed in /search endpoint." + "

NOTE: unpublished collections/items can still be accessed by their path.

" + ) + # using "name" instead of "id", as "id" has a default meaning in django + name = models.CharField('id', unique=True, max_length=255, validators=[validate_name]) + created = models.DateTimeField(auto_now_add=True) + # NOTE: the updated field is automatically updated by stac_api.pgtriggers, we use auto_now_add + # only for the initial value. + updated = models.DateTimeField(auto_now_add=True) + description = models.TextField() + # Set to true if the extent needs to be recalculated. + extent_out_of_sync = models.BooleanField(default=False) + extent_geometry = models.GeometryField( + default=None, srid=4326, editable=False, blank=True, null=True + ) + extent_start_datetime = models.DateTimeField(editable=False, null=True, blank=True) + extent_end_datetime = models.DateTimeField(editable=False, null=True, blank=True) + + license = models.CharField(max_length=30) # string + + # DEPRECATED: summaries JSON field is not used anymore and not up to date, it will be removed + # in future. It has been replaced by summaries_proj_epsg, summaries_eo_gsd and + # summaries_geoadmin_variant + summaries = models.JSONField(default=dict, encoder=DjangoJSONEncoder, editable=False) + + # NOTE: the following summaries_* fields are automatically update by the stac_api.pgtriggers + summaries_proj_epsg = ArrayField( + models.IntegerField(), default=list, blank=True, editable=False + ) + summaries_eo_gsd = ArrayField(models.FloatField(), default=list, blank=True, editable=False) + summaries_geoadmin_variant = ArrayField( + models.CharField(max_length=25), default=list, blank=True, editable=False + ) + summaries_geoadmin_lang = ArrayField( + models.CharField(max_length=2), default=list, blank=True, editable=False + ) + + title = models.CharField(blank=True, null=True, max_length=255) + + # NOTE: hidden ETag field, this field is automatically updated by stac_api.pgtriggers + etag = models.CharField( + blank=False, null=False, editable=False, max_length=56, default=compute_etag + ) + + update_interval = models.IntegerField( + default=-1, + null=False, + blank=False, + validators=[MinValueValidator(-1)], + help_text="Minimal update interval in seconds " + "in which the underlying assets data are updated." + ) + + total_data_size = models.BigIntegerField(default=0, null=True, blank=True) + + allow_external_assets = models.BooleanField( + default=False, + help_text=_('Whether this collection can have assets that are hosted externally') + ) + + external_asset_whitelist = ArrayField( + models.CharField(max_length=255), blank=True, default=list, + help_text=_('Provide a comma separated list of ' + 'protocol://domain values for the external asset url validation') + ) + + def __str__(self): + return self.name + + +class CollectionLink(Link): + collection = models.ForeignKey( + Collection, related_name='links', related_query_name='link', on_delete=models.CASCADE + ) + + class Meta: + ordering = ['pk'] + triggers = child_triggers('collection', 'CollectionLink') + + +class CollectionAsset(AssetBase): + + class Meta: + unique_together = (('collection', 'name'),) + ordering = ['id'] + triggers = generates_collection_asset_triggers() + + collection = models.ForeignKey( + Collection, + related_name='assets', + related_query_name='asset', + on_delete=models.PROTECT, + help_text=_(SEARCH_TEXT_HELP_ITEM) + ) + + # CollectionAssets are never external + is_external = False + + def get_collection(self): + return self.collection + + def get_asset_path(self): + return get_collection_asset_path(self.collection, self.name) + + +class CollectionAssetUpload(BaseAssetUpload): + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=['asset', 'upload_id'], + name='unique_asset_upload_collection_asset_upload_id' + ), + # Make sure that there is only one upload in progress per collection asset + models.UniqueConstraint( + fields=['asset', 'status'], + condition=Q(status='in-progress'), + name='unique_asset_upload_in_progress' + ) + ] + triggers = generates_asset_upload_triggers() + + asset = models.ForeignKey(CollectionAsset, related_name='+', on_delete=models.CASCADE) + + def update_asset_from_upload(self): + '''Updating the asset's file:checksum and update_interval from the upload + + When the upload is completed, the new file:checksum and update interval from the upload + is set to its asset parent. + ''' + logger.debug( + 'Updating collection asset %s file:checksum from %s to %s and update_interval ' + 'from %d to %d due to upload complete', + self.asset.name, + self.asset.checksum_multihash, + self.checksum_multihash, + self.asset.update_interval, + self.update_interval, + extra={ + 'upload_id': self.upload_id, + 'asset': self.asset.name, + 'collection': self.asset.collection.name + } + ) + + self.asset.checksum_multihash = self.checksum_multihash + self.asset.update_interval = self.update_interval + self.asset.file_size = self.file_size + self.asset.save() + + +class CountBase(models.Model): + '''CountBase tables are used to help calculate the summary on a collection. + This is only performant if the distinct number of values is small, e.g. we currently only have + 5 possible values for geoadmin_language. + For each assets value we keep a count of how often that value exists, per collection. On + insert/update/delete of an asset we only need to decrease and/or increase the counter value. + Since the number of possible values is small, the aggregate array calculation to update the + collection also stays performant. + ''' + + class Meta: + abstract = True + + id = models.BigAutoField(primary_key=True) + collection = models.ForeignKey( + Collection, + related_name='+', + on_delete=models.CASCADE, + ) + count = models.PositiveIntegerField(null=False) + + +class GSDCount(CountBase): + # Update by asset triggers. + + class Meta: + unique_together = (('collection', 'value'),) + ordering = ['id'] + triggers = generates_summary_count_triggers( + SummaryFields.GSD.value[0], SummaryFields.GSD.value[1] + ) + + value = models.FloatField(null=True, blank=True) + + +class GeoadminLangCount(CountBase): + # Update by asset triggers. + + class Meta: + unique_together = (('collection', 'value'),) + ordering = ['id'] + triggers = generates_summary_count_triggers( + SummaryFields.LANGUAGE.value[0], SummaryFields.LANGUAGE.value[1] + ) + + value = models.CharField(max_length=2, default=None, null=True, blank=True) + + +class GeoadminVariantCount(CountBase): + # Update by asset triggers. + + class Meta: + unique_together = (('collection', 'value'),) + ordering = ['id'] + triggers = generates_summary_count_triggers( + SummaryFields.VARIANT.value[0], SummaryFields.VARIANT.value[1] + ) + + value = models.CharField(max_length=25, null=True, blank=True) + + +class ProjEPSGCount(CountBase): + # Update by asset and collection asset triggers. + + class Meta: + unique_together = (('collection', 'value'),) + ordering = ['id'] + triggers = generates_summary_count_triggers( + SummaryFields.PROJ_EPSG.value[0], SummaryFields.PROJ_EPSG.value[1] + ) + + value = models.IntegerField(null=True, blank=True) diff --git a/app/stac_api/models/general.py b/app/stac_api/models/general.py index c6b7cd8f..6849409a 100644 --- a/app/stac_api/models/general.py +++ b/app/stac_api/models/general.py @@ -16,30 +16,15 @@ from django.core.serializers.json import DjangoJSONEncoder from django.core.validators import MaxValueValidator from django.core.validators import MinValueValidator -from django.db.models import Q from django.db.models.deletion import ProtectedError from django.utils.translation import gettext_lazy as _ from stac_api.managers import AssetUploadManager -from stac_api.managers import ItemManager -from stac_api.pgtriggers import SummaryFields from stac_api.pgtriggers import child_triggers -from stac_api.pgtriggers import generates_asset_triggers -from stac_api.pgtriggers import generates_asset_upload_triggers -from stac_api.pgtriggers import generates_collection_asset_triggers -from stac_api.pgtriggers import generates_collection_triggers -from stac_api.pgtriggers import generates_item_triggers -from stac_api.pgtriggers import generates_summary_count_triggers -from stac_api.utils import get_asset_path -from stac_api.utils import get_collection_asset_path from stac_api.utils import select_s3_bucket from stac_api.validators import MEDIA_TYPES from stac_api.validators import validate_asset_name from stac_api.validators import validate_asset_name_with_media_type -from stac_api.validators import validate_eo_gsd -from stac_api.validators import validate_geoadmin_variant -from stac_api.validators import validate_geometry -from stac_api.validators import validate_item_properties_datetimes from stac_api.validators import validate_link_rel from stac_api.validators import validate_media_type from stac_api.validators import validate_name @@ -88,40 +73,6 @@ ''' -SEARCH_TEXT_HELP_COLLECTION = ''' -
- Search Usage: -
    -
  • - arg will make a non exact search checking if arg is part of - the collection ID -
    • -
  • - Multiple arg can be used, separated by spaces. This will search for all - collections ID containing all arguments. -
    • -
  • - "collectionID" will make an exact search for the specified collection. -
    • -
- Examples : -
    -
  • - Searching for pixelkarte will return all collections which have - pixelkarte as a part of their collection ID -
    • -
  • - Searching for pixelkarte 2016 4 will return all collection - which have pixelkarte, 2016 AND 4 as part of their collection ID -
    • -
  • - Searching for ch.swisstopo.pixelkarte.example will yield only this - collection, if this collection exists. Please note that it would not return - a collection named ch.swisstopo.pixelkarte.example.2. -
    • -
-
''' - def get_conformance_default_links(): '''A helper function of the class Conformance Page @@ -252,101 +203,6 @@ def clean(self): ) -# For Collections and Items: No primary key will be defined, so that the auto-generated ones -# will be used by Django. For assets, a primary key is defined as "BigAutoField" due the -# expected large number of assets -class Collection(models.Model): - - class Meta: - indexes = [ - models.Index(fields=['name'], name='collection_name_idx'), - models.Index(fields=['published'], name='collection_published_idx') - ] - triggers = generates_collection_triggers() - - published = models.BooleanField( - default=True, - help_text="When not published the collection doesn't appear on the " - "api/stac/v0.9/collections endpoint and its items are not listed in /search endpoint." - "

NOTE: unpublished collections/items can still be accessed by their path.

" - ) - # using "name" instead of "id", as "id" has a default meaning in django - name = models.CharField('id', unique=True, max_length=255, validators=[validate_name]) - created = models.DateTimeField(auto_now_add=True) - # NOTE: the updated field is automatically updated by stac_api.pgtriggers, we use auto_now_add - # only for the initial value. - updated = models.DateTimeField(auto_now_add=True) - description = models.TextField() - # Set to true if the extent needs to be recalculated. - extent_out_of_sync = models.BooleanField(default=False) - extent_geometry = models.GeometryField( - default=None, srid=4326, editable=False, blank=True, null=True - ) - extent_start_datetime = models.DateTimeField(editable=False, null=True, blank=True) - extent_end_datetime = models.DateTimeField(editable=False, null=True, blank=True) - - license = models.CharField(max_length=30) # string - - # DEPRECATED: summaries JSON field is not used anymore and not up to date, it will be removed - # in future. It has been replaced by summaries_proj_epsg, summaries_eo_gsd and - # summaries_geoadmin_variant - summaries = models.JSONField(default=dict, encoder=DjangoJSONEncoder, editable=False) - - # NOTE: the following summaries_* fields are automatically update by the stac_api.pgtriggers - summaries_proj_epsg = ArrayField( - models.IntegerField(), default=list, blank=True, editable=False - ) - summaries_eo_gsd = ArrayField(models.FloatField(), default=list, blank=True, editable=False) - summaries_geoadmin_variant = ArrayField( - models.CharField(max_length=25), default=list, blank=True, editable=False - ) - summaries_geoadmin_lang = ArrayField( - models.CharField(max_length=2), default=list, blank=True, editable=False - ) - - title = models.CharField(blank=True, null=True, max_length=255) - - # NOTE: hidden ETag field, this field is automatically updated by stac_api.pgtriggers - etag = models.CharField( - blank=False, null=False, editable=False, max_length=56, default=compute_etag - ) - - update_interval = models.IntegerField( - default=-1, - null=False, - blank=False, - validators=[MinValueValidator(-1)], - help_text="Minimal update interval in seconds " - "in which the underlying assets data are updated." - ) - - total_data_size = models.BigIntegerField(default=0, null=True, blank=True) - - allow_external_assets = models.BooleanField( - default=False, - help_text=_('Whether this collection can have assets that are hosted externally') - ) - - external_asset_whitelist = ArrayField( - models.CharField(max_length=255), blank=True, default=list, - help_text=_('Provide a comma separated list of ' - 'protocol://domain values for the external asset url validation') - ) - - def __str__(self): - return self.name - - -class CollectionLink(Link): - collection = models.ForeignKey( - Collection, related_name='links', related_query_name='link', on_delete=models.CASCADE - ) - - class Meta: - ordering = ['pk'] - triggers = child_triggers('collection', 'CollectionLink') - - ITEM_KEEP_ORIGINAL_FIELDS = [ 'geometry', 'properties_datetime', @@ -355,179 +211,6 @@ class Meta: ] -class Item(models.Model): - - class Meta: - unique_together = (('collection', 'name'),) - indexes = [ - models.Index(fields=['name'], name='item_name_idx'), - # the following 3 indices are used e.g. in collection_temporal_extent - models.Index(fields=['properties_datetime'], name='item_datetime_idx'), - models.Index(fields=['properties_start_datetime'], name='item_start_datetime_idx'), - models.Index(fields=['properties_end_datetime'], name='item_end_datetime_idx'), - # created, updated, and title are "queryable" in the search endpoint - # see: views.py:322 and 323 - models.Index(fields=['created'], name='item_created_idx'), - models.Index(fields=['updated'], name='item_updated_idx'), - models.Index(fields=['properties_title'], name='item_title_idx'), - # forecast properties are "queryable" in the search endpoint - models.Index( - fields=['forecast_reference_datetime'], name='item_fc_reference_datetime_idx' - ), - models.Index(fields=['forecast_horizon'], name='item_fc_horizon_idx'), - models.Index(fields=['forecast_duration'], name='item_fc_duration_idx'), - models.Index(fields=['forecast_variable'], name='item_fc_variable_idx'), - models.Index(fields=['forecast_perturbed'], name='item_fc_perturbed_idx'), - # combination of datetime and start_ and end_datetimes are used in - # managers.py:110 and following - models.Index( - fields=[ - 'properties_datetime', 'properties_start_datetime', 'properties_end_datetime' - ], - name='item_dttme_start_end_dttm_idx' - ), - models.Index(fields=['update_interval'], name='item_update_interval_idx'), - ] - triggers = generates_item_triggers() - - name = models.CharField('id', blank=False, max_length=255, validators=[validate_name]) - collection = models.ForeignKey( - Collection, on_delete=models.PROTECT, help_text=_(SEARCH_TEXT_HELP_COLLECTION) - ) - geometry = models.GeometryField( - null=False, blank=False, default=BBOX_CH, srid=4326, validators=[validate_geometry] - ) - created = models.DateTimeField(auto_now_add=True) - # NOTE: the updated field is automatically updated by stac_api.pgtriggers, we use auto_now_add - # only for the initial value. - updated = models.DateTimeField(auto_now_add=True) - # after discussion with Chris and Tobias: for the moment only support - # proterties: datetime and title (the rest is hence commented out) - properties_datetime = models.DateTimeField( - blank=True, - null=True, - help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format" - ) - properties_start_datetime = models.DateTimeField( - blank=True, - null=True, - help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format" - ) - properties_end_datetime = models.DateTimeField( - blank=True, - null=True, - help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format" - ) - properties_expires = models.DateTimeField( - blank=True, - null=True, - help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format" - ) - # properties_eo_bands = model.TextFields(blank=True) # ? [string]? - # properties_eo_cloud_cover = models.FloatField(blank=True) - # eo_gsd is defined on asset level and will be updated here on ever - # update of an asset inside this item. - # properties_instruments = models.TextField(blank=True) - # properties_license = models.TextField(blank=True) - # properties_platform = models.TextField(blank=True) - # properties_providers = models.ManyToManyField(Provider) - # Although it is discouraged by Django to set blank=True and null=True on a CharField, it is - # here required because this field is optional and having an empty string value is not permitted - # in the serializer and default to None. None value are then not displayed in the serializer. - properties_title = models.CharField(blank=True, null=True, max_length=255) - - # properties_view_off_nadir = models.FloatField(blank=True) - # properties_view_sun_azimuth = models.FloatField(blank=True) - # properties_view_elevation = models.FloatField(blank=True) - - # NOTE: hidden ETag field, this field is automatically updated by stac_api.pgtriggers - etag = models.CharField( - blank=False, null=False, editable=False, max_length=56, default=compute_etag - ) - - update_interval = models.IntegerField( - default=-1, - null=False, - blank=False, - validators=[MinValueValidator(-1)], - help_text="Minimal update interval in seconds " - "in which the underlying assets data are updated." - ) - - total_data_size = models.BigIntegerField(default=0, null=True, blank=True) - - forecast_reference_datetime = models.DateTimeField( - null=True, - blank=True, - help_text="The reference datetime: i.e. predictions for times after " - "this point occur in the future. Predictions prior to this " - "time represent 'hindcasts', predicting states that have " - "already occurred. This must be in UTC. It is formatted like " - "'2022-08-12T00:00:00Z'." - ) - - forecast_horizon = models.DurationField( - null=True, - blank=True, - help_text="The time between the reference datetime and the forecast datetime." - "Formatted as ISO 8601 duration, e.g. 'PT6H' for a 6-hour forecast.", - ) - - forecast_duration = models.DurationField( - null=True, - blank=True, - help_text="If the forecast is not only for a specific instance in time " - "but instead is for a certain period, you can specify the " - "length here. Formatted as ISO 8601 duration, e.g. 'PT3H' for a 3-hour " - "accumulation. If not given, assumes that the forecast is for an " - "instance in time as if this was set to PT0S (0 seconds).", - ) - - forecast_variable = models.CharField( - null=True, - blank=True, - help_text="Name of the model variable that corresponds to the data. The variables " - "should correspond to the CF Standard Names, " - "e.g. `air_temperature` for the air temperature." - ) - - forecast_perturbed = models.BooleanField( - null=True, - blank=True, - default=None, - help_text="Denotes whether the data corresponds to the control run (`false`) or " - "perturbed runs (`true`). The property needs to be specified in both " - "cases as no default value is specified and as such the meaning is " - "\"unknown\" in case it's missing." - ) - - # Custom Manager that preselects the collection - objects = ItemManager() - - def __str__(self): - # This is used in the admin page in the autocomplete_fields of the Asset page - return f"{self.collection.name}/{self.name}" - - def clean(self): - validate_item_properties_datetimes( - self.properties_datetime, - self.properties_start_datetime, - self.properties_end_datetime, - self.properties_expires, - ) - - -class ItemLink(Link): - item = models.ForeignKey( - Item, related_name='links', related_query_name='link', on_delete=models.CASCADE - ) - - class Meta: - unique_together = (('rel', 'item'),) - ordering = ['pk'] - triggers = child_triggers('item', 'ItemLink') - - def upload_asset_to_path_hook(instance, filename=None): '''This returns the asset upload path on S3 and compute the asset file multihash @@ -707,80 +390,6 @@ def clean(self): validate_asset_name_with_media_type(self.name, media_type) -class Asset(AssetBase): - - class Meta: - unique_together = (('item', 'name'),) - ordering = ['id'] - triggers = generates_asset_triggers() - - item = models.ForeignKey( - Item, - related_name='assets', - related_query_name='asset', - on_delete=models.PROTECT, - help_text=_(SEARCH_TEXT_HELP_ITEM) - ) - # From v1 on the json representation of this field changed from "eo:gsd" to "gsd". The two names - # may be used interchangeably for a now. - eo_gsd = models.FloatField(null=True, blank=True, validators=[validate_eo_gsd]) - - class Language(models.TextChoices): - # pylint: disable=invalid-name - GERMAN = 'de', _('German') - ITALIAN = 'it', _('Italian') - FRENCH = 'fr', _('French') - ROMANSH = 'rm', _('Romansh') - ENGLISH = 'en', _('English') - __empty__ = _('') - - # here we need to set blank=True otherwise the field is as required in the admin interface - geoadmin_lang = models.CharField( - max_length=2, choices=Language.choices, default=None, null=True, blank=True - ) - # here we need to set blank=True otherwise the field is as required in the admin interface - geoadmin_variant = models.CharField( - max_length=25, null=True, blank=True, validators=[validate_geoadmin_variant] - ) - - # whether this asset is hosted externally - is_external = models.BooleanField( - default=False, - help_text=_("Whether this asset is hosted externally") - ) - - def get_collection(self): - return self.item.collection - - def get_asset_path(self): - return get_asset_path(self.item, self.name) - - -class CollectionAsset(AssetBase): - - class Meta: - unique_together = (('collection', 'name'),) - ordering = ['id'] - triggers = generates_collection_asset_triggers() - - collection = models.ForeignKey( - Collection, - related_name='assets', - related_query_name='asset', - on_delete=models.PROTECT, - help_text=_(SEARCH_TEXT_HELP_ITEM) - ) - - # CollectionAssets are never external - is_external = False - - def get_collection(self): - return self.collection - - def get_asset_path(self): - return get_collection_asset_path(self.collection, self.name) - - class BaseAssetUpload(models.Model): class Meta: @@ -839,167 +448,3 @@ class ContentEncoding(models.TextChoices): # Custom Manager that preselects the collection objects = AssetUploadManager() - - -class AssetUpload(BaseAssetUpload): - - class Meta: - constraints = [ - models.UniqueConstraint(fields=['asset', 'upload_id'], name='unique_together'), - # Make sure that there is only one asset upload in progress per asset - models.UniqueConstraint( - fields=['asset', 'status'], - condition=Q(status='in-progress'), - name='unique_in_progress' - ) - ] - triggers = generates_asset_upload_triggers() - - asset = models.ForeignKey(Asset, related_name='+', on_delete=models.CASCADE) - - def update_asset_from_upload(self): - '''Updating the asset's file:checksum and update_interval from the upload - - When the upload is completed, the new file:checksum and update interval from the upload - is set to its asset parent. - ''' - logger.debug( - 'Updating asset %s file:checksum from %s to %s and update_interval from %d to %d ' - 'due to upload complete', - self.asset.name, - self.asset.checksum_multihash, - self.checksum_multihash, - self.asset.update_interval, - self.update_interval, - extra={ - 'upload_id': self.upload_id, - 'asset': self.asset.name, - 'item': self.asset.item.name, - 'collection': self.asset.item.collection.name - } - ) - - self.asset.checksum_multihash = self.checksum_multihash - self.asset.update_interval = self.update_interval - self.asset.file_size = self.file_size - self.asset.save() - - -class CollectionAssetUpload(BaseAssetUpload): - - class Meta: - constraints = [ - models.UniqueConstraint( - fields=['asset', 'upload_id'], - name='unique_asset_upload_collection_asset_upload_id' - ), - # Make sure that there is only one upload in progress per collection asset - models.UniqueConstraint( - fields=['asset', 'status'], - condition=Q(status='in-progress'), - name='unique_asset_upload_in_progress' - ) - ] - triggers = generates_asset_upload_triggers() - - asset = models.ForeignKey(CollectionAsset, related_name='+', on_delete=models.CASCADE) - - def update_asset_from_upload(self): - '''Updating the asset's file:checksum and update_interval from the upload - - When the upload is completed, the new file:checksum and update interval from the upload - is set to its asset parent. - ''' - logger.debug( - 'Updating collection asset %s file:checksum from %s to %s and update_interval ' - 'from %d to %d due to upload complete', - self.asset.name, - self.asset.checksum_multihash, - self.checksum_multihash, - self.asset.update_interval, - self.update_interval, - extra={ - 'upload_id': self.upload_id, - 'asset': self.asset.name, - 'collection': self.asset.collection.name - } - ) - - self.asset.checksum_multihash = self.checksum_multihash - self.asset.update_interval = self.update_interval - self.asset.file_size = self.file_size - self.asset.save() - - -class CountBase(models.Model): - '''CountBase tables are used to help calculate the summary on a collection. - This is only performant if the distinct number of values is small, e.g. we currently only have - 5 possible values for geoadmin_language. - For each assets value we keep a count of how often that value exists, per collection. On - insert/update/delete of an asset we only need to decrease and/or increase the counter value. - Since the number of possible values is small, the aggregate array calculation to update the - collection also stays performant. - ''' - - class Meta: - abstract = True - - id = models.BigAutoField(primary_key=True) - collection = models.ForeignKey( - Collection, - related_name='+', - on_delete=models.CASCADE, - ) - count = models.PositiveIntegerField(null=False) - - -class GSDCount(CountBase): - # Update by asset triggers. - - class Meta: - unique_together = (('collection', 'value'),) - ordering = ['id'] - triggers = generates_summary_count_triggers( - SummaryFields.GSD.value[0], SummaryFields.GSD.value[1] - ) - - value = models.FloatField(null=True, blank=True) - - -class GeoadminLangCount(CountBase): - # Update by asset triggers. - - class Meta: - unique_together = (('collection', 'value'),) - ordering = ['id'] - triggers = generates_summary_count_triggers( - SummaryFields.LANGUAGE.value[0], SummaryFields.LANGUAGE.value[1] - ) - - value = models.CharField(max_length=2, default=None, null=True, blank=True) - - -class GeoadminVariantCount(CountBase): - # Update by asset triggers. - - class Meta: - unique_together = (('collection', 'value'),) - ordering = ['id'] - triggers = generates_summary_count_triggers( - SummaryFields.VARIANT.value[0], SummaryFields.VARIANT.value[1] - ) - - value = models.CharField(max_length=25, null=True, blank=True) - - -class ProjEPSGCount(CountBase): - # Update by asset and collection asset triggers. - - class Meta: - unique_together = (('collection', 'value'),) - ordering = ['id'] - triggers = generates_summary_count_triggers( - SummaryFields.PROJ_EPSG.value[0], SummaryFields.PROJ_EPSG.value[1] - ) - - value = models.IntegerField(null=True, blank=True) diff --git a/app/stac_api/models/item.py b/app/stac_api/models/item.py new file mode 100644 index 00000000..7d3aae9d --- /dev/null +++ b/app/stac_api/models/item.py @@ -0,0 +1,327 @@ +import logging + +from django.contrib.gis.db import models +from django.core.validators import MinValueValidator +from django.db.models import Q +from django.utils.translation import gettext_lazy as _ + +from stac_api.managers import ItemManager +from stac_api.models.collection import Collection +from stac_api.models.general import BBOX_CH +from stac_api.models.general import SEARCH_TEXT_HELP_ITEM +from stac_api.models.general import AssetBase +from stac_api.models.general import BaseAssetUpload +from stac_api.models.general import Link +from stac_api.models.general import compute_etag +from stac_api.pgtriggers import child_triggers +from stac_api.pgtriggers import generates_asset_triggers +from stac_api.pgtriggers import generates_asset_upload_triggers +from stac_api.pgtriggers import generates_item_triggers +from stac_api.utils import get_asset_path +from stac_api.validators import validate_eo_gsd +from stac_api.validators import validate_geoadmin_variant +from stac_api.validators import validate_geometry +from stac_api.validators import validate_item_properties_datetimes +from stac_api.validators import validate_name + +logger = logging.getLogger(__name__) + +SEARCH_TEXT_HELP_COLLECTION = ''' +
+ Search Usage: +
    +
  • + arg will make a non exact search checking if arg is part of + the collection ID +
    • +
  • + Multiple arg can be used, separated by spaces. This will search for all + collections ID containing all arguments. +
    • +
  • + "collectionID" will make an exact search for the specified collection. +
    • +
+ Examples : +
    +
  • + Searching for pixelkarte will return all collections which have + pixelkarte as a part of their collection ID +
    • +
  • + Searching for pixelkarte 2016 4 will return all collection + which have pixelkarte, 2016 AND 4 as part of their collection ID +
    • +
  • + Searching for ch.swisstopo.pixelkarte.example will yield only this + collection, if this collection exists. Please note that it would not return + a collection named ch.swisstopo.pixelkarte.example.2. +
    • +
+
''' + + +class Item(models.Model): + + class Meta: + unique_together = (('collection', 'name'),) + indexes = [ + models.Index(fields=['name'], name='item_name_idx'), + # the following 3 indices are used e.g. in collection_temporal_extent + models.Index(fields=['properties_datetime'], name='item_datetime_idx'), + models.Index(fields=['properties_start_datetime'], name='item_start_datetime_idx'), + models.Index(fields=['properties_end_datetime'], name='item_end_datetime_idx'), + # created, updated, and title are "queryable" in the search endpoint + # see: views.py:322 and 323 + models.Index(fields=['created'], name='item_created_idx'), + models.Index(fields=['updated'], name='item_updated_idx'), + models.Index(fields=['properties_title'], name='item_title_idx'), + # forecast properties are "queryable" in the search endpoint + models.Index( + fields=['forecast_reference_datetime'], name='item_fc_reference_datetime_idx' + ), + models.Index(fields=['forecast_horizon'], name='item_fc_horizon_idx'), + models.Index(fields=['forecast_duration'], name='item_fc_duration_idx'), + models.Index(fields=['forecast_variable'], name='item_fc_variable_idx'), + models.Index(fields=['forecast_perturbed'], name='item_fc_perturbed_idx'), + # combination of datetime and start_ and end_datetimes are used in + # managers.py:110 and following + models.Index( + fields=[ + 'properties_datetime', 'properties_start_datetime', 'properties_end_datetime' + ], + name='item_dttme_start_end_dttm_idx' + ), + models.Index(fields=['update_interval'], name='item_update_interval_idx'), + ] + triggers = generates_item_triggers() + + name = models.CharField('id', blank=False, max_length=255, validators=[validate_name]) + collection = models.ForeignKey( + Collection, on_delete=models.PROTECT, help_text=_(SEARCH_TEXT_HELP_COLLECTION) + ) + geometry = models.GeometryField( + null=False, blank=False, default=BBOX_CH, srid=4326, validators=[validate_geometry] + ) + created = models.DateTimeField(auto_now_add=True) + # NOTE: the updated field is automatically updated by stac_api.pgtriggers, we use auto_now_add + # only for the initial value. + updated = models.DateTimeField(auto_now_add=True) + # after discussion with Chris and Tobias: for the moment only support + # proterties: datetime and title (the rest is hence commented out) + properties_datetime = models.DateTimeField( + blank=True, + null=True, + help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format" + ) + properties_start_datetime = models.DateTimeField( + blank=True, + null=True, + help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format" + ) + properties_end_datetime = models.DateTimeField( + blank=True, + null=True, + help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format" + ) + properties_expires = models.DateTimeField( + blank=True, + null=True, + help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format" + ) + # properties_eo_bands = model.TextFields(blank=True) # ? [string]? + # properties_eo_cloud_cover = models.FloatField(blank=True) + # eo_gsd is defined on asset level and will be updated here on ever + # update of an asset inside this item. + # properties_instruments = models.TextField(blank=True) + # properties_license = models.TextField(blank=True) + # properties_platform = models.TextField(blank=True) + # properties_providers = models.ManyToManyField(Provider) + # Although it is discouraged by Django to set blank=True and null=True on a CharField, it is + # here required because this field is optional and having an empty string value is not permitted + # in the serializer and default to None. None value are then not displayed in the serializer. + properties_title = models.CharField(blank=True, null=True, max_length=255) + + # properties_view_off_nadir = models.FloatField(blank=True) + # properties_view_sun_azimuth = models.FloatField(blank=True) + # properties_view_elevation = models.FloatField(blank=True) + + # NOTE: hidden ETag field, this field is automatically updated by stac_api.pgtriggers + etag = models.CharField( + blank=False, null=False, editable=False, max_length=56, default=compute_etag + ) + + update_interval = models.IntegerField( + default=-1, + null=False, + blank=False, + validators=[MinValueValidator(-1)], + help_text="Minimal update interval in seconds " + "in which the underlying assets data are updated." + ) + + total_data_size = models.BigIntegerField(default=0, null=True, blank=True) + + forecast_reference_datetime = models.DateTimeField( + null=True, + blank=True, + help_text="The reference datetime: i.e. predictions for times after " + "this point occur in the future. Predictions prior to this " + "time represent 'hindcasts', predicting states that have " + "already occurred. This must be in UTC. It is formatted like " + "'2022-08-12T00:00:00Z'." + ) + + forecast_horizon = models.DurationField( + null=True, + blank=True, + help_text="The time between the reference datetime and the forecast datetime." + "Formatted as ISO 8601 duration, e.g. 'PT6H' for a 6-hour forecast.", + ) + + forecast_duration = models.DurationField( + null=True, + blank=True, + help_text="If the forecast is not only for a specific instance in time " + "but instead is for a certain period, you can specify the " + "length here. Formatted as ISO 8601 duration, e.g. 'PT3H' for a 3-hour " + "accumulation. If not given, assumes that the forecast is for an " + "instance in time as if this was set to PT0S (0 seconds).", + ) + + forecast_variable = models.CharField( + null=True, + blank=True, + help_text="Name of the model variable that corresponds to the data. The variables " + "should correspond to the CF Standard Names, " + "e.g. `air_temperature` for the air temperature." + ) + + forecast_perturbed = models.BooleanField( + null=True, + blank=True, + default=None, + help_text="Denotes whether the data corresponds to the control run (`false`) or " + "perturbed runs (`true`). The property needs to be specified in both " + "cases as no default value is specified and as such the meaning is " + "\"unknown\" in case it's missing." + ) + + # Custom Manager that preselects the collection + objects = ItemManager() + + def __str__(self): + # This is used in the admin page in the autocomplete_fields of the Asset page + return f"{self.collection.name}/{self.name}" + + def clean(self): + validate_item_properties_datetimes( + self.properties_datetime, + self.properties_start_datetime, + self.properties_end_datetime, + self.properties_expires, + ) + + +class ItemLink(Link): + item = models.ForeignKey( + Item, related_name='links', related_query_name='link', on_delete=models.CASCADE + ) + + class Meta: + unique_together = (('rel', 'item'),) + ordering = ['pk'] + triggers = child_triggers('item', 'ItemLink') + + +class Asset(AssetBase): + + class Meta: + unique_together = (('item', 'name'),) + ordering = ['id'] + triggers = generates_asset_triggers() + + item = models.ForeignKey( + Item, + related_name='assets', + related_query_name='asset', + on_delete=models.PROTECT, + help_text=_(SEARCH_TEXT_HELP_ITEM) + ) + # From v1 on the json representation of this field changed from "eo:gsd" to "gsd". The two names + # may be used interchangeably for a now. + eo_gsd = models.FloatField(null=True, blank=True, validators=[validate_eo_gsd]) + + class Language(models.TextChoices): + # pylint: disable=invalid-name + GERMAN = 'de', _('German') + ITALIAN = 'it', _('Italian') + FRENCH = 'fr', _('French') + ROMANSH = 'rm', _('Romansh') + ENGLISH = 'en', _('English') + __empty__ = _('') + + # here we need to set blank=True otherwise the field is as required in the admin interface + geoadmin_lang = models.CharField( + max_length=2, choices=Language.choices, default=None, null=True, blank=True + ) + # here we need to set blank=True otherwise the field is as required in the admin interface + geoadmin_variant = models.CharField( + max_length=25, null=True, blank=True, validators=[validate_geoadmin_variant] + ) + + # whether this asset is hosted externally + is_external = models.BooleanField( + default=False, + help_text=_("Whether this asset is hosted externally") + ) + + def get_collection(self): + return self.item.collection + + def get_asset_path(self): + return get_asset_path(self.item, self.name) + + +class AssetUpload(BaseAssetUpload): + + class Meta: + constraints = [ + models.UniqueConstraint(fields=['asset', 'upload_id'], name='unique_together'), + # Make sure that there is only one asset upload in progress per asset + models.UniqueConstraint( + fields=['asset', 'status'], + condition=Q(status='in-progress'), + name='unique_in_progress' + ) + ] + triggers = generates_asset_upload_triggers() + + asset = models.ForeignKey(Asset, related_name='+', on_delete=models.CASCADE) + + def update_asset_from_upload(self): + '''Updating the asset's file:checksum and update_interval from the upload + + When the upload is completed, the new file:checksum and update interval from the upload + is set to its asset parent. + ''' + logger.debug( + 'Updating asset %s file:checksum from %s to %s and update_interval from %d to %d ' + 'due to upload complete', + self.asset.name, + self.asset.checksum_multihash, + self.checksum_multihash, + self.asset.update_interval, + self.update_interval, + extra={ + 'upload_id': self.upload_id, + 'asset': self.asset.name, + 'item': self.asset.item.name, + 'collection': self.asset.item.collection.name + } + ) + + self.asset.checksum_multihash = self.checksum_multihash + self.asset.update_interval = self.update_interval + self.asset.file_size = self.file_size + self.asset.save() diff --git a/app/stac_api/s3_multipart_upload.py b/app/stac_api/s3_multipart_upload.py index 14dae036..41950947 100644 --- a/app/stac_api/s3_multipart_upload.py +++ b/app/stac_api/s3_multipart_upload.py @@ -12,8 +12,8 @@ from rest_framework import serializers from stac_api.exceptions import UploadNotInProgressError -from stac_api.models.general import Asset -from stac_api.models.general import CollectionAsset +from stac_api.models.collection import CollectionAsset +from stac_api.models.item import Asset from stac_api.utils import AVAILABLE_S3_BUCKETS from stac_api.utils import get_s3_cache_control_value from stac_api.utils import get_s3_client diff --git a/app/stac_api/sample_data/importer.py b/app/stac_api/sample_data/importer.py index c308f90e..74af6cec 100644 --- a/app/stac_api/sample_data/importer.py +++ b/app/stac_api/sample_data/importer.py @@ -9,11 +9,11 @@ from django.contrib.gis.geos import GEOSGeometry from django.core.files.uploadedfile import SimpleUploadedFile -from stac_api.models.general import Asset -from stac_api.models.general import Collection -from stac_api.models.general import CollectionLink -from stac_api.models.general import Item +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionLink from stac_api.models.general import Provider +from stac_api.models.item import Asset +from stac_api.models.item import Item # path definition relative to the directory that contains manage.py DATADIR = settings.BASE_DIR / 'app/stac_api/management/sample_data/' diff --git a/app/stac_api/serializers/collection.py b/app/stac_api/serializers/collection.py index dc28dc10..406643fe 100644 --- a/app/stac_api/serializers/collection.py +++ b/app/stac_api/serializers/collection.py @@ -4,9 +4,9 @@ from rest_framework import serializers -from stac_api.models.general import Collection -from stac_api.models.general import CollectionAsset -from stac_api.models.general import CollectionLink +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionAsset +from stac_api.models.collection import CollectionLink from stac_api.models.general import Provider from stac_api.serializers.utils import AssetsDictSerializer from stac_api.serializers.utils import HrefField diff --git a/app/stac_api/serializers/item.py b/app/stac_api/serializers/item.py index 93655089..90f299af 100644 --- a/app/stac_api/serializers/item.py +++ b/app/stac_api/serializers/item.py @@ -8,9 +8,9 @@ from rest_framework import serializers from rest_framework_gis import serializers as gis_serializers -from stac_api.models.general import Asset -from stac_api.models.general import Item -from stac_api.models.general import ItemLink +from stac_api.models.item import Asset +from stac_api.models.item import Item +from stac_api.models.item import ItemLink from stac_api.serializers.utils import AssetsDictSerializer from stac_api.serializers.utils import HrefField from stac_api.serializers.utils import IsoDurationField diff --git a/app/stac_api/serializers/upload.py b/app/stac_api/serializers/upload.py index 691f942c..579f2332 100644 --- a/app/stac_api/serializers/upload.py +++ b/app/stac_api/serializers/upload.py @@ -5,8 +5,8 @@ from rest_framework import serializers from rest_framework.utils.serializer_helpers import ReturnDict -from stac_api.models.general import AssetUpload -from stac_api.models.general import CollectionAssetUpload +from stac_api.models.collection import CollectionAssetUpload +from stac_api.models.item import AssetUpload from stac_api.serializers.utils import NonNullModelSerializer from stac_api.utils import is_api_version_1 from stac_api.utils import isoformat diff --git a/app/stac_api/serializers/utils.py b/app/stac_api/serializers/utils.py index 8b54dc9b..650d0bee 100644 --- a/app/stac_api/serializers/utils.py +++ b/app/stac_api/serializers/utils.py @@ -10,9 +10,9 @@ from rest_framework import serializers from rest_framework.utils.serializer_helpers import ReturnDict -from stac_api.models.general import Collection -from stac_api.models.general import Item +from stac_api.models.collection import Collection from stac_api.models.general import Link +from stac_api.models.item import Item from stac_api.utils import build_asset_href from stac_api.utils import get_browser_url from stac_api.utils import get_url diff --git a/app/stac_api/signals.py b/app/stac_api/signals.py index 1e95fa19..ad011ddf 100644 --- a/app/stac_api/signals.py +++ b/app/stac_api/signals.py @@ -4,10 +4,10 @@ from django.db.models.signals import pre_delete from django.dispatch import receiver -from stac_api.models.general import Asset -from stac_api.models.general import AssetUpload -from stac_api.models.general import CollectionAsset -from stac_api.models.general import CollectionAssetUpload +from stac_api.models.collection import CollectionAsset +from stac_api.models.collection import CollectionAssetUpload +from stac_api.models.item import Asset +from stac_api.models.item import AssetUpload logger = logging.getLogger(__name__) diff --git a/app/stac_api/validators_view.py b/app/stac_api/validators_view.py index ce720508..dbfbcbc8 100644 --- a/app/stac_api/validators_view.py +++ b/app/stac_api/validators_view.py @@ -7,10 +7,10 @@ from rest_framework import serializers -from stac_api.models.general import Asset -from stac_api.models.general import Collection -from stac_api.models.general import CollectionAsset -from stac_api.models.general import Item +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionAsset +from stac_api.models.item import Asset +from stac_api.models.item import Item logger = logging.getLogger(__name__) diff --git a/app/stac_api/views/collection.py b/app/stac_api/views/collection.py index 3955de1a..35650356 100644 --- a/app/stac_api/views/collection.py +++ b/app/stac_api/views/collection.py @@ -8,8 +8,8 @@ from rest_framework.response import Response from rest_framework_condition import etag -from stac_api.models.general import Collection -from stac_api.models.general import CollectionAsset +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionAsset from stac_api.serializers.collection import CollectionAssetSerializer from stac_api.serializers.collection import CollectionSerializer from stac_api.serializers.utils import get_relation_links diff --git a/app/stac_api/views/general.py b/app/stac_api/views/general.py index c0192961..cbb6f86c 100644 --- a/app/stac_api/views/general.py +++ b/app/stac_api/views/general.py @@ -14,8 +14,8 @@ from rest_framework.permissions import AllowAny from rest_framework.response import Response -from stac_api.models.general import Item from stac_api.models.general import LandingPage +from stac_api.models.item import Item from stac_api.pagination import GetPostCursorPagination from stac_api.serializers.general import ConformancePageSerializer from stac_api.serializers.general import LandingPageSerializer diff --git a/app/stac_api/views/item.py b/app/stac_api/views/item.py index 4e1de5ff..5bbdd590 100644 --- a/app/stac_api/views/item.py +++ b/app/stac_api/views/item.py @@ -12,9 +12,9 @@ from rest_framework.response import Response from rest_framework_condition import etag -from stac_api.models.general import Asset -from stac_api.models.general import Collection -from stac_api.models.general import Item +from stac_api.models.collection import Collection +from stac_api.models.item import Asset +from stac_api.models.item import Item from stac_api.serializers.item import AssetSerializer from stac_api.serializers.item import ItemSerializer from stac_api.serializers.utils import get_relation_links diff --git a/app/stac_api/views/upload.py b/app/stac_api/views/upload.py index b8eb1197..ff775179 100644 --- a/app/stac_api/views/upload.py +++ b/app/stac_api/views/upload.py @@ -16,11 +16,11 @@ from stac_api.exceptions import UploadInProgressError from stac_api.exceptions import UploadNotInProgressError -from stac_api.models.general import Asset -from stac_api.models.general import AssetUpload +from stac_api.models.collection import CollectionAsset +from stac_api.models.collection import CollectionAssetUpload from stac_api.models.general import BaseAssetUpload -from stac_api.models.general import CollectionAsset -from stac_api.models.general import CollectionAssetUpload +from stac_api.models.item import Asset +from stac_api.models.item import AssetUpload from stac_api.pagination import ExtApiPagination from stac_api.s3_multipart_upload import MultipartUpload from stac_api.serializers.upload import AssetUploadPartsSerializer diff --git a/app/tests/base_test_admin_page.py b/app/tests/base_test_admin_page.py index 07e65351..0e137ca6 100644 --- a/app/tests/base_test_admin_page.py +++ b/app/tests/base_test_admin_page.py @@ -7,12 +7,12 @@ from django.test import TestCase from django.urls import reverse -from stac_api.models.general import Asset -from stac_api.models.general import Collection -from stac_api.models.general import CollectionLink -from stac_api.models.general import Item -from stac_api.models.general import ItemLink +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionLink from stac_api.models.general import Provider +from stac_api.models.item import Asset +from stac_api.models.item import Item +from stac_api.models.item import ItemLink from tests.tests_09.data_factory import Factory diff --git a/app/tests/test_admin_page.py b/app/tests/test_admin_page.py index e2570e6b..fff0b77c 100644 --- a/app/tests/test_admin_page.py +++ b/app/tests/test_admin_page.py @@ -5,12 +5,12 @@ from django.test import override_settings from django.urls import reverse -from stac_api.models.general import Asset -from stac_api.models.general import Collection -from stac_api.models.general import CollectionLink -from stac_api.models.general import Item -from stac_api.models.general import ItemLink +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionLink from stac_api.models.general import Provider +from stac_api.models.item import Asset +from stac_api.models.item import Item +from stac_api.models.item import ItemLink from stac_api.utils import parse_multihash from tests.base_test_admin_page import AdminBaseTestCase diff --git a/app/tests/tests_09/data_factory.py b/app/tests/tests_09/data_factory.py index 57f949a2..f21af465 100644 --- a/app/tests/tests_09/data_factory.py +++ b/app/tests/tests_09/data_factory.py @@ -97,12 +97,12 @@ from django.core.files.base import File from django.core.files.uploadedfile import SimpleUploadedFile -from stac_api.models.general import Asset -from stac_api.models.general import Collection -from stac_api.models.general import CollectionLink -from stac_api.models.general import Item -from stac_api.models.general import ItemLink +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionLink from stac_api.models.general import Provider +from stac_api.models.item import Asset +from stac_api.models.item import Item +from stac_api.models.item import ItemLink from stac_api.utils import get_s3_resource from stac_api.utils import isoformat from stac_api.validators import get_media_type diff --git a/app/tests/tests_09/test_asset_model.py b/app/tests/tests_09/test_asset_model.py index 4c52bf46..587caa98 100644 --- a/app/tests/tests_09/test_asset_model.py +++ b/app/tests/tests_09/test_asset_model.py @@ -2,7 +2,7 @@ from django.core.exceptions import ValidationError -from stac_api.models.general import Asset +from stac_api.models.item import Asset from tests.tests_09.base_test import StacBaseTransactionTestCase from tests.tests_09.data_factory import Factory diff --git a/app/tests/tests_09/test_asset_upload_endpoint.py b/app/tests/tests_09/test_asset_upload_endpoint.py index bc50210a..ca7d2b08 100644 --- a/app/tests/tests_09/test_asset_upload_endpoint.py +++ b/app/tests/tests_09/test_asset_upload_endpoint.py @@ -10,8 +10,8 @@ from django.contrib.auth import get_user_model from django.test import Client -from stac_api.models.general import Asset -from stac_api.models.general import AssetUpload +from stac_api.models.item import Asset +from stac_api.models.item import AssetUpload from stac_api.utils import fromisoformat from stac_api.utils import get_asset_path from stac_api.utils import get_s3_client diff --git a/app/tests/tests_09/test_asset_upload_model.py b/app/tests/tests_09/test_asset_upload_model.py index 1c84e291..c6b874c3 100644 --- a/app/tests/tests_09/test_asset_upload_model.py +++ b/app/tests/tests_09/test_asset_upload_model.py @@ -6,8 +6,8 @@ from django.test import TestCase from django.test import TransactionTestCase -from stac_api.models.general import Asset -from stac_api.models.general import AssetUpload +from stac_api.models.item import Asset +from stac_api.models.item import AssetUpload from stac_api.utils import get_sha256_multihash from stac_api.utils import utc_aware diff --git a/app/tests/tests_09/test_assets_endpoint.py b/app/tests/tests_09/test_assets_endpoint.py index 980dcebe..a3917a81 100644 --- a/app/tests/tests_09/test_assets_endpoint.py +++ b/app/tests/tests_09/test_assets_endpoint.py @@ -8,7 +8,7 @@ from django.test import Client from django.urls import reverse -from stac_api.models.general import Asset +from stac_api.models.item import Asset from stac_api.utils import get_asset_path from stac_api.utils import utc_aware diff --git a/app/tests/tests_09/test_collection_model.py b/app/tests/tests_09/test_collection_model.py index 1142c0b2..cbeb0bd2 100644 --- a/app/tests/tests_09/test_collection_model.py +++ b/app/tests/tests_09/test_collection_model.py @@ -3,7 +3,7 @@ from django.core.exceptions import ValidationError -from stac_api.models.general import Collection +from stac_api.models.collection import Collection from tests.tests_09.base_test import StacBaseTransactionTestCase from tests.tests_09.data_factory import Factory diff --git a/app/tests/tests_09/test_collections_endpoint.py b/app/tests/tests_09/test_collections_endpoint.py index 8a40150a..39f03b77 100644 --- a/app/tests/tests_09/test_collections_endpoint.py +++ b/app/tests/tests_09/test_collections_endpoint.py @@ -5,8 +5,8 @@ from django.test import Client from django.urls import reverse -from stac_api.models.general import Collection -from stac_api.models.general import CollectionLink +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionLink from stac_api.models.general import Provider from stac_api.utils import utc_aware diff --git a/app/tests/tests_09/test_collections_extent.py b/app/tests/tests_09/test_collections_extent.py index d25ff329..09691f8b 100644 --- a/app/tests/tests_09/test_collections_extent.py +++ b/app/tests/tests_09/test_collections_extent.py @@ -4,7 +4,7 @@ from django.contrib.gis.geos import GEOSGeometry from django.contrib.gis.geos import Polygon -from stac_api.models.general import Item +from stac_api.models.item import Item from stac_api.utils import utc_aware from tests.tests_09.base_test import StacBaseTransactionTestCase diff --git a/app/tests/tests_09/test_generic_api.py b/app/tests/tests_09/test_generic_api.py index c8252897..f9345510 100644 --- a/app/tests/tests_09/test_generic_api.py +++ b/app/tests/tests_09/test_generic_api.py @@ -5,7 +5,7 @@ from django.test import Client from django.test import override_settings -from stac_api.models.general import AssetUpload +from stac_api.models.item import AssetUpload from stac_api.utils import get_asset_path from stac_api.utils import get_link from stac_api.utils import get_sha256_multihash diff --git a/app/tests/tests_09/test_item_model.py b/app/tests/tests_09/test_item_model.py index 3f84a128..ad7d4e2e 100644 --- a/app/tests/tests_09/test_item_model.py +++ b/app/tests/tests_09/test_item_model.py @@ -6,8 +6,8 @@ from django.core.exceptions import ValidationError from django.test import TestCase -from stac_api.models.general import Collection -from stac_api.models.general import Item +from stac_api.models.collection import Collection +from stac_api.models.item import Item from stac_api.utils import utc_aware from tests.tests_09.data_factory import CollectionFactory diff --git a/app/tests/tests_09/test_items_endpoint.py b/app/tests/tests_09/test_items_endpoint.py index 37640c0c..1924d54f 100644 --- a/app/tests/tests_09/test_items_endpoint.py +++ b/app/tests/tests_09/test_items_endpoint.py @@ -6,7 +6,7 @@ from django.test import Client from django.urls import reverse -from stac_api.models.general import Item +from stac_api.models.item import Item from stac_api.utils import fromisoformat from stac_api.utils import get_link from stac_api.utils import isoformat diff --git a/app/tests/tests_09/test_serializer.py b/app/tests/tests_09/test_serializer.py index c6386104..09d0f426 100644 --- a/app/tests/tests_09/test_serializer.py +++ b/app/tests/tests_09/test_serializer.py @@ -12,10 +12,10 @@ from rest_framework.renderers import JSONRenderer from rest_framework.test import APIRequestFactory -from stac_api.models.general import get_asset_path from stac_api.serializers.collection import CollectionSerializer from stac_api.serializers.item import AssetSerializer from stac_api.serializers.item import ItemSerializer +from stac_api.utils import get_asset_path from stac_api.utils import get_link from stac_api.utils import isoformat from stac_api.utils import utc_aware diff --git a/app/tests/tests_09/test_serializer_asset_upload.py b/app/tests/tests_09/test_serializer_asset_upload.py index 4dcb25a2..9f11bc72 100644 --- a/app/tests/tests_09/test_serializer_asset_upload.py +++ b/app/tests/tests_09/test_serializer_asset_upload.py @@ -6,7 +6,7 @@ from rest_framework import serializers from rest_framework.exceptions import ValidationError -from stac_api.models.general import AssetUpload +from stac_api.models.item import AssetUpload from stac_api.serializers.upload import AssetUploadSerializer from stac_api.utils import get_sha256_multihash diff --git a/app/tests/tests_10/data_factory.py b/app/tests/tests_10/data_factory.py index 38cbd4bf..de59878f 100644 --- a/app/tests/tests_10/data_factory.py +++ b/app/tests/tests_10/data_factory.py @@ -97,13 +97,13 @@ from django.core.files.base import File from django.core.files.uploadedfile import SimpleUploadedFile -from stac_api.models.general import Asset -from stac_api.models.general import Collection -from stac_api.models.general import CollectionAsset -from stac_api.models.general import CollectionLink -from stac_api.models.general import Item -from stac_api.models.general import ItemLink +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionAsset +from stac_api.models.collection import CollectionLink from stac_api.models.general import Provider +from stac_api.models.item import Asset +from stac_api.models.item import Item +from stac_api.models.item import ItemLink from stac_api.utils import get_s3_resource from stac_api.utils import isoformat from stac_api.validators import get_media_type diff --git a/app/tests/tests_10/test_asset_model.py b/app/tests/tests_10/test_asset_model.py index f424dd66..757dc3ba 100644 --- a/app/tests/tests_10/test_asset_model.py +++ b/app/tests/tests_10/test_asset_model.py @@ -2,7 +2,7 @@ from django.core.exceptions import ValidationError -from stac_api.models.general import Asset +from stac_api.models.item import Asset from tests.tests_10.base_test import StacBaseTransactionTestCase from tests.tests_10.data_factory import Factory diff --git a/app/tests/tests_10/test_asset_upload_endpoint.py b/app/tests/tests_10/test_asset_upload_endpoint.py index 91465f5a..8369a7dc 100644 --- a/app/tests/tests_10/test_asset_upload_endpoint.py +++ b/app/tests/tests_10/test_asset_upload_endpoint.py @@ -10,8 +10,8 @@ from django.contrib.auth import get_user_model from django.test import Client -from stac_api.models.general import Asset -from stac_api.models.general import AssetUpload +from stac_api.models.item import Asset +from stac_api.models.item import AssetUpload from stac_api.utils import fromisoformat from stac_api.utils import get_asset_path from stac_api.utils import get_s3_client diff --git a/app/tests/tests_10/test_asset_upload_model.py b/app/tests/tests_10/test_asset_upload_model.py index 21c4dffd..361fba0e 100644 --- a/app/tests/tests_10/test_asset_upload_model.py +++ b/app/tests/tests_10/test_asset_upload_model.py @@ -6,8 +6,8 @@ from django.test import TestCase from django.test import TransactionTestCase -from stac_api.models.general import Asset -from stac_api.models.general import AssetUpload +from stac_api.models.item import Asset +from stac_api.models.item import AssetUpload from stac_api.utils import get_sha256_multihash from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_assets_endpoint.py b/app/tests/tests_10/test_assets_endpoint.py index 987bb470..2c9ab550 100644 --- a/app/tests/tests_10/test_assets_endpoint.py +++ b/app/tests/tests_10/test_assets_endpoint.py @@ -10,7 +10,7 @@ from django.urls import reverse from django.utils import timezone -from stac_api.models.general import Asset +from stac_api.models.item import Asset from stac_api.utils import get_asset_path from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_collection_asset_model.py b/app/tests/tests_10/test_collection_asset_model.py index 9cb8f172..b75366ae 100644 --- a/app/tests/tests_10/test_collection_asset_model.py +++ b/app/tests/tests_10/test_collection_asset_model.py @@ -2,7 +2,7 @@ from django.core.exceptions import ValidationError -from stac_api.models.general import CollectionAsset +from stac_api.models.collection import CollectionAsset from tests.tests_10.base_test import StacBaseTransactionTestCase from tests.tests_10.data_factory import Factory diff --git a/app/tests/tests_10/test_collection_asset_upload_endpoint.py b/app/tests/tests_10/test_collection_asset_upload_endpoint.py index 95078dc7..1d02c26a 100644 --- a/app/tests/tests_10/test_collection_asset_upload_endpoint.py +++ b/app/tests/tests_10/test_collection_asset_upload_endpoint.py @@ -10,8 +10,8 @@ from django.contrib.auth import get_user_model from django.test import Client -from stac_api.models.general import CollectionAsset -from stac_api.models.general import CollectionAssetUpload +from stac_api.models.collection import CollectionAsset +from stac_api.models.collection import CollectionAssetUpload from stac_api.utils import fromisoformat from stac_api.utils import get_collection_asset_path from stac_api.utils import get_s3_client diff --git a/app/tests/tests_10/test_collection_asset_upload_model.py b/app/tests/tests_10/test_collection_asset_upload_model.py index bb6e6c50..63a0cde3 100644 --- a/app/tests/tests_10/test_collection_asset_upload_model.py +++ b/app/tests/tests_10/test_collection_asset_upload_model.py @@ -6,8 +6,8 @@ from django.test import TestCase from django.test import TransactionTestCase -from stac_api.models.general import CollectionAsset -from stac_api.models.general import CollectionAssetUpload +from stac_api.models.collection import CollectionAsset +from stac_api.models.collection import CollectionAssetUpload from stac_api.utils import get_sha256_multihash from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_collection_assets_endpoint.py b/app/tests/tests_10/test_collection_assets_endpoint.py index e782c243..c2ce980d 100644 --- a/app/tests/tests_10/test_collection_assets_endpoint.py +++ b/app/tests/tests_10/test_collection_assets_endpoint.py @@ -8,7 +8,7 @@ from django.test import Client from django.urls import reverse -from stac_api.models.general import CollectionAsset +from stac_api.models.collection import CollectionAsset from stac_api.utils import get_collection_asset_path from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_collection_model.py b/app/tests/tests_10/test_collection_model.py index b6bb6ef4..75998ade 100644 --- a/app/tests/tests_10/test_collection_model.py +++ b/app/tests/tests_10/test_collection_model.py @@ -3,7 +3,7 @@ from django.core.exceptions import ValidationError -from stac_api.models.general import Collection +from stac_api.models.collection import Collection from tests.tests_10.base_test import StacBaseTransactionTestCase from tests.tests_10.data_factory import Factory diff --git a/app/tests/tests_10/test_collections_endpoint.py b/app/tests/tests_10/test_collections_endpoint.py index 5c2108e8..e5a674bd 100644 --- a/app/tests/tests_10/test_collections_endpoint.py +++ b/app/tests/tests_10/test_collections_endpoint.py @@ -6,8 +6,8 @@ from django.test import Client from django.urls import reverse -from stac_api.models.general import Collection -from stac_api.models.general import CollectionLink +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionLink from stac_api.models.general import Provider from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_collections_extent.py b/app/tests/tests_10/test_collections_extent.py index fc308cef..19b778c0 100644 --- a/app/tests/tests_10/test_collections_extent.py +++ b/app/tests/tests_10/test_collections_extent.py @@ -4,7 +4,7 @@ from django.contrib.gis.geos import GEOSGeometry from django.contrib.gis.geos import Polygon -from stac_api.models.general import Item +from stac_api.models.item import Item from stac_api.utils import utc_aware from tests.tests_10.base_test import StacBaseTransactionTestCase diff --git a/app/tests/tests_10/test_external_assets_endpoint.py b/app/tests/tests_10/test_external_assets_endpoint.py index e8bf2fe4..0711d514 100644 --- a/app/tests/tests_10/test_external_assets_endpoint.py +++ b/app/tests/tests_10/test_external_assets_endpoint.py @@ -3,7 +3,7 @@ from django.conf import settings from django.test import Client -from stac_api.models.general import Asset +from stac_api.models.item import Asset from tests.tests_10.base_test import StacBaseTestCase from tests.tests_10.data_factory import Factory diff --git a/app/tests/tests_10/test_generic_api.py b/app/tests/tests_10/test_generic_api.py index cb183187..d0d42efd 100644 --- a/app/tests/tests_10/test_generic_api.py +++ b/app/tests/tests_10/test_generic_api.py @@ -5,7 +5,7 @@ from django.test import Client from django.test import override_settings -from stac_api.models.general import AssetUpload +from stac_api.models.item import AssetUpload from stac_api.utils import get_asset_path from stac_api.utils import get_link from stac_api.utils import get_sha256_multihash diff --git a/app/tests/tests_10/test_item_model.py b/app/tests/tests_10/test_item_model.py index a7391f69..83776750 100644 --- a/app/tests/tests_10/test_item_model.py +++ b/app/tests/tests_10/test_item_model.py @@ -7,8 +7,8 @@ from django.core.exceptions import ValidationError from django.test import TestCase -from stac_api.models.general import Collection -from stac_api.models.general import Item +from stac_api.models.collection import Collection +from stac_api.models.item import Item from stac_api.utils import utc_aware from tests.tests_10.data_factory import CollectionFactory diff --git a/app/tests/tests_10/test_items_endpoint.py b/app/tests/tests_10/test_items_endpoint.py index cc7ca6db..72dadf19 100644 --- a/app/tests/tests_10/test_items_endpoint.py +++ b/app/tests/tests_10/test_items_endpoint.py @@ -9,9 +9,9 @@ from django.urls import reverse from django.utils import timezone -from stac_api.models.general import Collection -from stac_api.models.general import Item -from stac_api.models.general import ItemLink +from stac_api.models.collection import Collection +from stac_api.models.item import Item +from stac_api.models.item import ItemLink from stac_api.utils import fromisoformat from stac_api.utils import get_link from stac_api.utils import isoformat diff --git a/app/tests/tests_10/test_remove_expired_items.py b/app/tests/tests_10/test_remove_expired_items.py index c7a80c95..cfe1f39f 100644 --- a/app/tests/tests_10/test_remove_expired_items.py +++ b/app/tests/tests_10/test_remove_expired_items.py @@ -5,8 +5,8 @@ from django.test import TestCase from django.utils import timezone -from stac_api.models.general import Asset -from stac_api.models.general import Item +from stac_api.models.item import Asset +from stac_api.models.item import Item from tests.tests_10.data_factory import Factory from tests.utils import mock_s3_asset_file diff --git a/app/tests/tests_10/test_serializer.py b/app/tests/tests_10/test_serializer.py index 68d9aa43..388a350e 100644 --- a/app/tests/tests_10/test_serializer.py +++ b/app/tests/tests_10/test_serializer.py @@ -14,11 +14,11 @@ from rest_framework.renderers import JSONRenderer from rest_framework.test import APIRequestFactory -from stac_api.models.general import get_asset_path from stac_api.serializers.collection import CollectionSerializer from stac_api.serializers.item import AssetSerializer from stac_api.serializers.item import ItemSerializer from stac_api.serializers.item import ItemsPropertiesSerializer +from stac_api.utils import get_asset_path from stac_api.utils import get_link from stac_api.utils import isoformat from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_serializer_asset_upload.py b/app/tests/tests_10/test_serializer_asset_upload.py index 34e4ba29..e557c1f6 100644 --- a/app/tests/tests_10/test_serializer_asset_upload.py +++ b/app/tests/tests_10/test_serializer_asset_upload.py @@ -6,7 +6,7 @@ from rest_framework import serializers from rest_framework.exceptions import ValidationError -from stac_api.models.general import AssetUpload +from stac_api.models.item import AssetUpload from stac_api.serializers.upload import AssetUploadSerializer from stac_api.utils import get_sha256_multihash