From b84ad1681ef7a481d54c285783d48b68daf26bf9 Mon Sep 17 00:00:00 2001 From: Marc Sommerhalder Date: Thu, 23 Jan 2025 13:27:00 +0100 Subject: [PATCH 01/15] PB-1281: Add feature flag for new authentication --- app/config/settings_dev.py | 10 ---------- app/config/settings_prod.py | 10 ++++++++++ app/middleware/apigw.py | 24 ++++++++++++++++++++++-- app/tests/test_admin_page.py | 16 ++++++++++++++++ 4 files changed, 48 insertions(+), 12 deletions(-) diff --git a/app/config/settings_dev.py b/app/config/settings_dev.py index 1434dcb4..96ba123b 100644 --- a/app/config/settings_dev.py +++ b/app/config/settings_dev.py @@ -66,16 +66,6 @@ AWS_SETTINGS['managed']['ACCESS_KEY_ID'] = env("LEGACY_AWS_ACCESS_KEY_ID") AWS_SETTINGS['managed']['SECRET_ACCESS_KEY'] = env("LEGACY_AWS_SECRET_ACCESS_KEY") -# API Gateway integration PB-1009 -AUTHENTICATION_BACKENDS = [ - "django.contrib.auth.backends.RemoteUserBackend", - # We keep ModelBackend as fallback until we have moved all users to Cognito. - "django.contrib.auth.backends.ModelBackend", -] -MIDDLEWARE += [ - "django.contrib.auth.middleware.AuthenticationMiddleware", - "middleware.apigw.ApiGatewayMiddleware", -] # By default sessions expire after two weeks. # Sessions are only useful for user tracking in the admin UI. For security # reason we should expire these sessions as soon as possible. Given the use diff --git a/app/config/settings_prod.py b/app/config/settings_prod.py index 08f2c50f..9d0504b6 100644 --- a/app/config/settings_prod.py +++ b/app/config/settings_prod.py @@ -75,6 +75,9 @@ 'stac_api.apps.StacApiConfig', ] +# API Authentication options +FEATURE_AUTH_ENABLE_APIGW = env('FEATURE_AUTH_ENABLE_APIGW', bool, default=False) + # Middlewares are executed in order, once for the incoming # request top-down, once for the outgoing response bottom up # Note: The prometheus middlewares should always be first and @@ -92,6 +95,7 @@ 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'middleware.apigw.ApiGatewayMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'middleware.cache_headers.CacheHeadersMiddleware', @@ -99,6 +103,12 @@ 'django_prometheus.middleware.PrometheusAfterMiddleware', ] +AUTHENTICATION_BACKENDS = [ + "middleware.apigw.ApiGatewayUserBackend", + # We keep ModelBackend as fallback until we have moved all users to Cognito. + "django.contrib.auth.backends.ModelBackend", +] + ROOT_URLCONF = 'config.urls' API_BASE = 'api' STAC_BASE = f'{API_BASE}/stac' diff --git a/app/middleware/apigw.py b/app/middleware/apigw.py index 7263e684..7b505123 100644 --- a/app/middleware/apigw.py +++ b/app/middleware/apigw.py @@ -1,3 +1,5 @@ +from django.conf import settings +from django.contrib.auth.backends import RemoteUserBackend from django.contrib.auth.middleware import PersistentRemoteUserMiddleware @@ -12,10 +14,10 @@ def process_request(self, request): whether it was able to authenticate the user. If it could not authenticate the user, the value of the header as seen on the wire is a single whitespace. An hexdump looks like this: - + 47 65 6f 61 64 6d 69 6e 5f 75 73 65 72 6e 61 6d 65 3a 20 0d 0a Geoadmin-Username:... - + This doesn't seem possible to reproduce with curl. It is possible to reproduce with wget. It is unclear whether that technically counts as an empty value or a whitespace. It is also possible that AWS change their @@ -26,7 +28,25 @@ def process_request(self, request): Based on discussion in https://code.djangoproject.com/ticket/35971 """ + if not settings.FEATURE_AUTH_ENABLE_APIGW: + return None + apigw_auth = request.META.get("HTTP_GEOADMIN_AUTHENTICATED", "false").lower() == "true" if not apigw_auth and self.header in request.META: del request.META[self.header] return super().process_request(request) + + +class ApiGatewayUserBackend(RemoteUserBackend): + """ This backend is to be used in conjunction with the ``ApiGatewayMiddleware`. + + It is probably not needed to provide a custom remote user backend as our custom remote user + middleware will never call authenticate if the feature is not enabled. But better be safe than + sorry. + """ + + def authenticate(self, request, remote_user): + if not settings.FEATURE_AUTH_ENABLE_APIGW: + return None + + return super().authenticate(request, remote_user) diff --git a/app/tests/test_admin_page.py b/app/tests/test_admin_page.py index dc9f7002..4d339ce3 100644 --- a/app/tests/test_admin_page.py +++ b/app/tests/test_admin_page.py @@ -2,6 +2,7 @@ from io import BytesIO from django.conf import settings +from django.test import override_settings from django.urls import reverse from stac_api.models import Asset @@ -47,6 +48,17 @@ def test_login_failure(self): self.assertEqual(response.status_code, 302) self.assertEqual("/api/stac/admin/login/?next=/api/stac/admin/", response.url) + def test_login_header_disabled(self): + response = self.client.get( + "/api/stac/admin/", + headers={ + "Geoadmin-Username": self.username, "Geoadmin-Authenticated": "true" + } + ) + self.assertEqual(response.status_code, 302) + self.assertEqual("/api/stac/admin/login/?next=/api/stac/admin/", response.url) + + @override_settings(FEATURE_AUTH_ENABLE_APIGW=True) def test_login_header(self): response = self.client.get( "/api/stac/admin/", @@ -56,11 +68,13 @@ def test_login_header(self): ) self.assertEqual(response.status_code, 200, msg="Admin page login with header failed") + @override_settings(FEATURE_AUTH_ENABLE_APIGW=True) def test_login_header_noheader(self): response = self.client.get("/api/stac/admin/") self.assertEqual(response.status_code, 302) self.assertEqual("/api/stac/admin/login/?next=/api/stac/admin/", response.url) + @override_settings(FEATURE_AUTH_ENABLE_APIGW=True) def test_login_header_wronguser(self): response = self.client.get( "/api/stac/admin/", @@ -71,6 +85,7 @@ def test_login_header_wronguser(self): self.assertEqual(response.status_code, 302) self.assertEqual("/api/stac/admin/login/?next=/api/stac/admin/", response.url) + @override_settings(FEATURE_AUTH_ENABLE_APIGW=True) def test_login_header_not_authenticated(self): self.assertNotIn("sessionid", self.client.cookies) response = self.client.get( @@ -82,6 +97,7 @@ def test_login_header_not_authenticated(self): self.assertEqual(response.status_code, 302) self.assertEqual("/api/stac/admin/login/?next=/api/stac/admin/", response.url) + @override_settings(FEATURE_AUTH_ENABLE_APIGW=True) def test_login_header_session(self): self.assertNotIn("sessionid", self.client.cookies) From 7876b73d9db17dc121bd4bfe4ced60763382c0b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=B6cklin?= Date: Thu, 23 Jan 2025 10:03:55 +0100 Subject: [PATCH 02/15] using GET range request instead of HEAD to check external asset reachability --- app/stac_api/validators.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/app/stac_api/validators.py b/app/stac_api/validators.py index a1c10a3c..0ffea86c 100644 --- a/app/stac_api/validators.py +++ b/app/stac_api/validators.py @@ -651,9 +651,19 @@ def _validate_href_configured_pattern(url, collection): def _validate_href_reachability(url, collection): unreachable_error = _('Provided URL is unreachable') try: - response = requests.head(url, timeout=settings.EXTERNAL_URL_REACHABLE_TIMEOUT) + # We change the way how we check reachability for MCH usecase from + # using HTTP HEAD request to HTTP GET with range + # Once we have other use cases that would require using HTTP HEAD request + # we would have to generalize this and make it configurable + # response = requests.head(url, timeout=settings.EXTERNAL_URL_REACHABLE_TIMEOUT) + + # We just wanna check reachability and aren't really interested in the content + headers = {"Range": "bytes=0-2"} + response = requests.get( + url, headers=headers, timeout=settings.EXTERNAL_URL_REACHABLE_TIMEOUT + ) - if response.status_code > 400: + if response.status_code >= 400: logger.warning( "Attempted external asset upload failed the reachability check", extra={ From 7f4486472ab2b7d24f7ec90c2ca678a9d711eb2c Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Thu, 23 Jan 2025 14:35:28 +0100 Subject: [PATCH 03/15] PB-1372: Check content length of external asset Add content length check when validating external asset url. Add test cases. --- app/stac_api/validators.py | 11 ++ .../tests_10/test_external_assets_endpoint.py | 131 ++++++++++++++++++ 2 files changed, 142 insertions(+) diff --git a/app/stac_api/validators.py b/app/stac_api/validators.py index 0ffea86c..ee713ffc 100644 --- a/app/stac_api/validators.py +++ b/app/stac_api/validators.py @@ -650,6 +650,7 @@ def _validate_href_configured_pattern(url, collection): def _validate_href_reachability(url, collection): unreachable_error = _('Provided URL is unreachable') + invalidcontent_error = _('Provided URL returns bad content') try: # We change the way how we check reachability for MCH usecase from # using HTTP HEAD request to HTTP GET with range @@ -673,6 +674,16 @@ def _validate_href_reachability(url, collection): } ) raise ValidationError(unreachable_error) + if response.headers.get("Content-Length") != "3": + logger.warning( + "Attempted external asset upload failed the content length check", + extra={ + 'url': url, + 'collection': collection, # to have the means to know who this might have been + 'response': response, + } + ) + raise ValidationError(invalidcontent_error) except requests.Timeout as exc: logger.warning( "Attempted external asset upload resulted in a timeout", diff --git a/app/tests/tests_10/test_external_assets_endpoint.py b/app/tests/tests_10/test_external_assets_endpoint.py index 883fc7b0..4a8829b9 100644 --- a/app/tests/tests_10/test_external_assets_endpoint.py +++ b/app/tests/tests_10/test_external_assets_endpoint.py @@ -1,3 +1,5 @@ +import responses + from django.conf import settings from django.test import Client @@ -60,6 +62,135 @@ def test_create_asset_with_external_url(self): self.assertEqual(created_asset.file, asset_data['href']) self.assertTrue(created_asset.is_external) + @responses.activate + def test_create_asset_validate_external_url(self): + collection = self.collection + item = self.item + external_test_asset_url = 'https://example.com/api/123.jpeg' + collection.allow_external_assets = True + collection.external_asset_whitelist = ['https://example.com'] + collection.save() + + # Mock response of external asset url + responses.add( + method=responses.GET, + url=external_test_asset_url, + body='som', + status=200, + content_type='application/json', + adding_headers={'Content-Length': '3'}, + match=[responses.matchers.header_matcher({"Range": "bytes=0-2"})] + ) + + asset_data = { + 'id': 'clouds.jpg', + 'description': 'High in the sky', + 'type': 'image/jpeg', # specify the file explicitly + # 'href': settings.EXTERNAL_TEST_ASSET_URL + 'href': external_test_asset_url + } + + # create the asset + response = self.client.put( + reverse_version('asset-detail', args=[collection.name, item.name, asset_data['id']]), + data=asset_data, + content_type="application/json" + ) + + json_data = response.json() + self.assertStatusCode(201, response) + self.assertEqual(json_data['href'], asset_data['href']) + + created_asset = Asset.objects.last() + self.assertEqual(created_asset.file, asset_data['href']) + self.assertTrue(created_asset.is_external) + + @responses.activate + def test_create_asset_validate_external_url_not_found(self): + collection = self.collection + item = self.item + external_test_asset_url = 'https://example.com/api/123.jpeg' + collection.allow_external_assets = True + collection.external_asset_whitelist = ['https://example.com'] + collection.save() + + # Mock response of external asset url returning 404 + responses.add( + method=responses.GET, + url=external_test_asset_url, + body='', + status=404, + content_type='application/json', + adding_headers={'Content-Length': '0'}, + match=[responses.matchers.header_matcher({"Range": "bytes=0-2"})] + ) + + asset_data = { + 'id': 'not_found.jpg', + 'description': 'High in the sky', + 'type': 'image/jpeg', # specify the file explicitly + 'href': external_test_asset_url + } + + # create the asset + response = self.client.put( + reverse_version('asset-detail', args=[collection.name, item.name, asset_data['id']]), + data=asset_data, + content_type="application/json" + ) + + self.assertStatusCode(400, response) + description = response.json()['description'] + self.assertIn('href', description, msg=f'Unexpected field error {description}') + self.assertIn( + 'Provided URL is unreachable', + description['href'], + msg=f'Unexpected field error {description}' + ) + + @responses.activate + def test_create_asset_validate_external_url_bad_content(self): + collection = self.collection + item = self.item + external_test_asset_url = 'https://example.com/api/123.jpeg' + collection.allow_external_assets = True + collection.external_asset_whitelist = ['https://example.com'] + collection.save() + + # Mock response of external asset url returning wrong content length + responses.add( + method=responses.GET, + url=external_test_asset_url, + body='', + status=200, + content_type='application/json', + adding_headers={'Content-Length': '0'}, + match=[responses.matchers.header_matcher({"Range": "bytes=0-2"})] + ) + + asset_data = { + 'id': 'not_found.jpg', + 'description': 'High in the sky', + 'type': 'image/jpeg', # specify the file explicitly + 'href': external_test_asset_url + } + + # create the asset + response = self.client.put( + reverse_version('asset-detail', args=[collection.name, item.name, asset_data['id']]), + data=asset_data, + content_type="application/json" + ) + + self.assertStatusCode(400, response) + description = response.json()['description'] + self.assertIn('href', description, msg=f'Unexpected field error {description}') + self.assertIn( + 'Provided URL returns bad content', + description['href'], + msg=f'Unexpected field error {description}' + ) + def test_update_asset_with_external_url(self): collection = self.collection item = self.item From d90b71fcb6e6ec6ed630533bf8758dfd29f68ea8 Mon Sep 17 00:00:00 2001 From: Adrien Kunysz Date: Thu, 23 Jan 2025 09:47:19 +0100 Subject: [PATCH 04/15] PB-1366: make new auth work for write requests. Django is configured to use CSRF mitigation tokens. This only works for clients that are web browsers. Typical API clients are not web browsers, do not keep track of cookies and fail CSRF mitigation checks. The old token authentication method works around that by disabling CSRF mitigation when that authentication method is used. With this change, we do the same for the new token authentication method. In PB-1009 we implemented the new authentication method for the Admin UI. This change does the same for the API endpoints and add tests exercising it. The API endpoints are rest_framework views that rely on authentication methods defined by the REST framework which is different from the Admin UI authentication plumbing. To support both, we need to have modules for both even though they basically do the same thing. Specifically: * create `api_gateway_authentication` as `rest_framework` authentication module * create `api_gateway_middleware` as generic Django authentication module (for Admin UI) * abstract the common parts into a new `api_gateway` module * configure both authentication modules in the debug build * add tests exercising a PUT API endpoint for both v0.9 and v1, including CSRF checks This is all flag-guarded behind FEATURE_AUTH_ENABLE_APIGW. --- app/config/settings_prod.py | 5 +- app/middleware/api_gateway.py | 30 +++++++++++ app/middleware/api_gateway_authentication.py | 26 ++++++++++ app/middleware/api_gateway_middleware.py | 32 ++++++++++++ app/middleware/apigw.py | 52 ------------------- .../tests_09/test_geoadmin_header_auth.py | 51 ++++++++++++++++++ .../tests_10/test_geoadmin_header_auth.py | 51 ++++++++++++++++++ 7 files changed, 193 insertions(+), 54 deletions(-) create mode 100644 app/middleware/api_gateway.py create mode 100644 app/middleware/api_gateway_authentication.py create mode 100644 app/middleware/api_gateway_middleware.py delete mode 100644 app/middleware/apigw.py create mode 100644 app/tests/tests_09/test_geoadmin_header_auth.py create mode 100644 app/tests/tests_10/test_geoadmin_header_auth.py diff --git a/app/config/settings_prod.py b/app/config/settings_prod.py index 9d0504b6..e9e110dd 100644 --- a/app/config/settings_prod.py +++ b/app/config/settings_prod.py @@ -95,7 +95,7 @@ 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'middleware.apigw.ApiGatewayMiddleware', + 'middleware.api_gateway_middleware.ApiGatewayMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'middleware.cache_headers.CacheHeadersMiddleware', @@ -104,7 +104,7 @@ ] AUTHENTICATION_BACKENDS = [ - "middleware.apigw.ApiGatewayUserBackend", + "middleware.api_gateway_middleware.ApiGatewayUserBackend", # We keep ModelBackend as fallback until we have moved all users to Cognito. "django.contrib.auth.backends.ModelBackend", ] @@ -305,6 +305,7 @@ def get_logging_config(): REST_FRAMEWORK = { 'DEFAULT_RENDERER_CLASSES': ['rest_framework.renderers.JSONRenderer'], 'DEFAULT_AUTHENTICATION_CLASSES': [ + 'middleware.api_gateway_authentication.ApiGatewayAuthentication', 'rest_framework.authentication.BasicAuthentication', 'rest_framework.authentication.TokenAuthentication', 'rest_framework.authentication.SessionAuthentication', diff --git a/app/middleware/api_gateway.py b/app/middleware/api_gateway.py new file mode 100644 index 00000000..245d885f --- /dev/null +++ b/app/middleware/api_gateway.py @@ -0,0 +1,30 @@ +REMOTE_USER_HEADER = "HTTP_GEOADMIN_USERNAME" + + +def validate_username_header(request): + """Drop the Geoadmin-Username header if it's invalid. + + This should be called before making any decision based on the value of the + Geoadmin-Username header. + + API Gateway always sends the Geoadmin-Username header regardless of + whether it was able to authenticate the user. If it could not + authenticate the user, the value of the header as seen on the wire is a + single whitespace. An hexdump looks like this: + + 47 65 6f 61 64 6d 69 6e 5f 75 73 65 72 6e 61 6d 65 3a 20 0d 0a + Geoadmin-Username:... + + This doesn't seem possible to reproduce with curl. It is possible to + reproduce with wget. It is unclear whether that technically counts as an + empty value or a whitespace. It is also possible that AWS change their + implementation later to send something slightly different. Regardless, + we already have a separate signal to tell us whether that value is + valid: Geoadmin-Authenticated. So we only consider Geoadmin-Username if + Geoadmin-Authenticated is set to "true". + + Based on discussion in https://code.djangoproject.com/ticket/35971 + """ + apigw_auth = request.META.get("HTTP_GEOADMIN_AUTHENTICATED", "false").lower() == "true" + if not apigw_auth and REMOTE_USER_HEADER in request.META: + del request.META[REMOTE_USER_HEADER] diff --git a/app/middleware/api_gateway_authentication.py b/app/middleware/api_gateway_authentication.py new file mode 100644 index 00000000..6e2cad5d --- /dev/null +++ b/app/middleware/api_gateway_authentication.py @@ -0,0 +1,26 @@ +from middleware import api_gateway + +from django.conf import settings + +from rest_framework.authentication import RemoteUserAuthentication + + +class ApiGatewayAuthentication(RemoteUserAuthentication): + header = api_gateway.REMOTE_USER_HEADER + + def authenticate(self, request): + if not settings.FEATURE_AUTH_ENABLE_APIGW: + return None + + api_gateway.validate_username_header(request) + return super().authenticate(request) + + def authenticate_header(self, request): + # For this authentication method, users send a "Bearer" token via the + # Authorization header. API Gateway looks up that token in Cognito and + # sets the Geoadmin-Username and Geoadmin-Authenticated headers. In this + # module we only care about the Geoadmin-* headers. But when + # authentication fails with a 401 error we need to hint at the correct + # authentication method from the point of view of the user, which is the + # Authorization/Bearer scheme. + return 'Bearer' diff --git a/app/middleware/api_gateway_middleware.py b/app/middleware/api_gateway_middleware.py new file mode 100644 index 00000000..18854c72 --- /dev/null +++ b/app/middleware/api_gateway_middleware.py @@ -0,0 +1,32 @@ +from middleware import api_gateway + +from django.conf import settings +from django.contrib.auth.backends import RemoteUserBackend +from django.contrib.auth.middleware import PersistentRemoteUserMiddleware + + +class ApiGatewayMiddleware(PersistentRemoteUserMiddleware): + """Persist user authentication based on the API Gateway headers.""" + header = api_gateway.REMOTE_USER_HEADER + + def process_request(self, request): + if not settings.FEATURE_AUTH_ENABLE_APIGW: + return None + + api_gateway.validate_username_header(request) + return super().process_request(request) + + +class ApiGatewayUserBackend(RemoteUserBackend): + """ This backend is to be used in conjunction with the ``ApiGatewayMiddleware`. + + It is probably not needed to provide a custom remote user backend as our custom remote user + middleware will never call authenticate if the feature is not enabled. But better be safe than + sorry. + """ + + def authenticate(self, request, remote_user): + if not settings.FEATURE_AUTH_ENABLE_APIGW: + return None + + return super().authenticate(request, remote_user) diff --git a/app/middleware/apigw.py b/app/middleware/apigw.py deleted file mode 100644 index 7b505123..00000000 --- a/app/middleware/apigw.py +++ /dev/null @@ -1,52 +0,0 @@ -from django.conf import settings -from django.contrib.auth.backends import RemoteUserBackend -from django.contrib.auth.middleware import PersistentRemoteUserMiddleware - - -class ApiGatewayMiddleware(PersistentRemoteUserMiddleware): - """Persist user authentication based on the API Gateway headers.""" - header = "HTTP_GEOADMIN_USERNAME" - - def process_request(self, request): - """Before processing the request, drop the Geoadmin-Username header if it's invalid. - - API Gateway always sends the Geoadmin-Username header regardless of - whether it was able to authenticate the user. If it could not - authenticate the user, the value of the header as seen on the wire is a - single whitespace. An hexdump looks like this: - - 47 65 6f 61 64 6d 69 6e 5f 75 73 65 72 6e 61 6d 65 3a 20 0d 0a - Geoadmin-Username:... - - This doesn't seem possible to reproduce with curl. It is possible to - reproduce with wget. It is unclear whether that technically counts as an - empty value or a whitespace. It is also possible that AWS change their - implementation later to send something slightly different. Regardless, - we already have a separate signal to tell us whether that value is - valid: Geoadmin-Authenticated. So we only consider Geoadmin-Username if - Geoadmin-Authenticated is set to "true". - - Based on discussion in https://code.djangoproject.com/ticket/35971 - """ - if not settings.FEATURE_AUTH_ENABLE_APIGW: - return None - - apigw_auth = request.META.get("HTTP_GEOADMIN_AUTHENTICATED", "false").lower() == "true" - if not apigw_auth and self.header in request.META: - del request.META[self.header] - return super().process_request(request) - - -class ApiGatewayUserBackend(RemoteUserBackend): - """ This backend is to be used in conjunction with the ``ApiGatewayMiddleware`. - - It is probably not needed to provide a custom remote user backend as our custom remote user - middleware will never call authenticate if the feature is not enabled. But better be safe than - sorry. - """ - - def authenticate(self, request, remote_user): - if not settings.FEATURE_AUTH_ENABLE_APIGW: - return None - - return super().authenticate(request, remote_user) diff --git a/app/tests/tests_09/test_geoadmin_header_auth.py b/app/tests/tests_09/test_geoadmin_header_auth.py new file mode 100644 index 00000000..5b81aed2 --- /dev/null +++ b/app/tests/tests_09/test_geoadmin_header_auth.py @@ -0,0 +1,51 @@ +import logging + +from parameterized import parameterized + +from django.contrib.auth import get_user_model +from django.test import Client +from django.test import override_settings + +from tests.tests_09.base_test import STAC_BASE_V +from tests.tests_09.base_test import StacBaseTestCase +from tests.tests_09.data_factory import Factory + +logger = logging.getLogger(__name__) + + +@override_settings(FEATURE_AUTH_ENABLE_APIGW=True) +class GeoadminHeadersAuthForPutEndpointTestCase(StacBaseTestCase): + valid_username = "another_test_user" + + def setUp(self): # pylint: disable=invalid-name + self.client = Client(enforce_csrf_checks=True) + self.factory = Factory() + self.collection = self.factory.create_collection_sample() + get_user_model().objects.create_superuser(self.valid_username) + + @parameterized.expand([ + (valid_username, "true", 201), + (None, None, 401), + (valid_username, "false", 401), + ("wronguser", "true", 403), + ]) + def test_collection_upsert_create_with_geoadmin_header_auth( + self, username_header, authenticated_header, expected_response_code + ): + sample = self.factory.create_collection_sample(sample='collection-2') + + headers = None + if username_header or authenticated_header: + headers = { + "Geoadmin-Username": username_header, + "Geoadmin-Authenticated": authenticated_header, + } + response = self.client.put( + path=f"/{STAC_BASE_V}/collections/{sample['name']}", + data=sample.get_json('put'), + content_type='application/json', + headers=headers, + ) + self.assertStatusCode(expected_response_code, response) + if 200 <= expected_response_code < 300: + self.check_stac_collection(sample.json, response.json()) diff --git a/app/tests/tests_10/test_geoadmin_header_auth.py b/app/tests/tests_10/test_geoadmin_header_auth.py new file mode 100644 index 00000000..93e0478d --- /dev/null +++ b/app/tests/tests_10/test_geoadmin_header_auth.py @@ -0,0 +1,51 @@ +import logging + +from parameterized import parameterized + +from django.contrib.auth import get_user_model +from django.test import Client +from django.test import override_settings + +from tests.tests_10.base_test import STAC_BASE_V +from tests.tests_10.base_test import StacBaseTestCase +from tests.tests_10.data_factory import Factory + +logger = logging.getLogger(__name__) + + +@override_settings(FEATURE_AUTH_ENABLE_APIGW=True) +class GeoadminHeadersAuthForPutEndpointTestCase(StacBaseTestCase): + valid_username = "another_test_user" + + def setUp(self): # pylint: disable=invalid-name + self.client = Client(enforce_csrf_checks=True) + self.factory = Factory() + self.collection = self.factory.create_collection_sample() + get_user_model().objects.create_superuser(self.valid_username) + + @parameterized.expand([ + (valid_username, "true", 201), + (None, None, 401), + (valid_username, "false", 401), + ("wronguser", "true", 403), + ]) + def test_collection_upsert_create_with_geoadmin_header_auth( + self, username_header, authenticated_header, expected_response_code + ): + sample = self.factory.create_collection_sample(sample='collection-2') + + headers = None + if username_header or authenticated_header: + headers = { + "Geoadmin-Username": username_header, + "Geoadmin-Authenticated": authenticated_header, + } + response = self.client.put( + path=f"/{STAC_BASE_V}/collections/{sample['name']}", + data=sample.get_json('put'), + content_type='application/json', + headers=headers, + ) + self.assertStatusCode(expected_response_code, response) + if 200 <= expected_response_code < 300: + self.check_stac_collection(sample.json, response.json()) From 2f1d51dfea81091d092f2bb1a3f06afc5e67b799 Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Wed, 22 Jan 2025 16:12:12 +0100 Subject: [PATCH 05/15] PB-1169: Spec search forecast fields Add forecast properties to search filter for GET and POST. Refactor forecast:variable and forecast:perturbed descriptions into schema properties to be reused. --- spec/components/parameters.yaml | 44 +++++++++ spec/components/schemas.yaml | 59 +++++++++--- spec/openapi.yaml | 5 + spec/static/spec/v1/openapi.yaml | 96 +++++++++++++++++-- spec/static/spec/v1/openapitransactional.yaml | 96 +++++++++++++++++-- 5 files changed, 267 insertions(+), 33 deletions(-) diff --git a/spec/components/parameters.yaml b/spec/components/parameters.yaml index 63ca9bbd..84b8ea24 100644 --- a/spec/components/parameters.yaml +++ b/spec/components/parameters.yaml @@ -117,3 +117,47 @@ components: required: false schema: type: string + forecast_reference_datetime: + explode: false + in: query + name: forecast_reference_datetime + required: false + schema: + $ref: "./schemas.yaml#/components/schemas/datetimeQuery" + example: 2018-02-12T00%3A00%3A00Z%2F2018-03-18T12%3A31%3A12Z + style: form + forecast_horizon: + explode: false + in: query + name: forecast_horizon + required: false + schema: + $ref: "./schemas.yaml#/components/schemas/duration" + example: P3DT6H + style: form + forecast_duration: + explode: false + in: query + name: forecast_duration + required: false + schema: + $ref: "./schemas.yaml#/components/schemas/duration" + example: P3DT6H + style: form + forecast_variable: + explode: false + in: query + name: forecast_variable + required: false + schema: + $ref: "./schemas.yaml#/components/schemas/forecast_variable" + example: air_temperature + style: form + forecast_perturbed: + explode: false + in: query + name: forecast_perturbed + required: false + schema: + $ref: "./schemas.yaml#/components/schemas/forecast_perturbed" + style: form diff --git a/spec/components/schemas.yaml b/spec/components/schemas.yaml index b877cd12..5f3cee07 100644 --- a/spec/components/schemas.yaml +++ b/spec/components/schemas.yaml @@ -1096,22 +1096,9 @@ components: forecast:duration: $ref: "#/components/schemas/duration" forecast:variable: - description: >- - Name of the model variable that corresponds to the data. The variables - should correspond to the - [CF Standard Names](https://cfconventions.org/Data/cf-standard-names/current/build/cf-standard-name-table.html), - e.g. `air_temperature` for the air temperature. - example: air_temperature - type: string - nullable: true + $ref: "#/components/schemas/forecast_variable" forecast:perturbed: - description: >- - Denotes whether the data corresponds to the control run (`false`) or - perturbed runs (`true`). The property needs to be specified in both - cases as no default value is specified and as such the meaning is "unknown" - in case it's missing. - type: boolean - nullable: true + $ref: "#/components/schemas/forecast_perturbed" required: - created - updated @@ -1494,6 +1481,43 @@ components: query: $ref: "#/components/schemas/query" type: object + forecast_reference_datetimeFilter: + properties: + forecast:reference_datetime: + $ref: "#/components/schemas/datetimeQuery" + forecast_horizonFilter: + properties: + forecast:horizon: + $ref: "#/components/schemas/duration" + forecast_durationFilter: + properties: + forecast:duration: + $ref: "#/components/schemas/duration" + forecast_variableFilter: + properties: + forecast:variable: + $ref: "#/components/schemas/forecast_variable" + forecast_variable: + description: >- + Name of the model variable that corresponds to the data. The variables + should correspond to the + [CF Standard Names](https://cfconventions.org/Data/cf-standard-names/current/build/cf-standard-name-table.html), + e.g. `air_temperature` for the air temperature. + example: air_temperature + type: string + nullable: true + forecast_perturbedFilter: + properties: + forecast:perturbed: + $ref: "#/components/schemas/forecast_perturbed" + forecast_perturbed: + description: >- + Denotes whether the data corresponds to the control run (`false`) or + perturbed runs (`true`). The property needs to be specified in both + cases as no default value is specified and as such the meaning is "unknown" + in case it's missing. + type: boolean + nullable: true queryProp: anyOf: - description: >- @@ -1588,6 +1612,11 @@ components: - $ref: "#/components/schemas/collectionsFilter" - $ref: "#/components/schemas/idsFilter" - $ref: "#/components/schemas/limitFilter" + - $ref: "#/components/schemas/forecast_reference_datetimeFilter" + - $ref: "#/components/schemas/forecast_horizonFilter" + - $ref: "#/components/schemas/forecast_durationFilter" + - $ref: "#/components/schemas/forecast_variableFilter" + - $ref: "#/components/schemas/forecast_perturbedFilter" description: The search criteria type: object stac_version: diff --git a/spec/openapi.yaml b/spec/openapi.yaml index 80c8d4ad..befc9641 100644 --- a/spec/openapi.yaml +++ b/spec/openapi.yaml @@ -283,6 +283,11 @@ paths: - $ref: "./components/parameters.yaml#/components/parameters/limit" - $ref: "./components/parameters.yaml#/components/parameters/ids" - $ref: "./components/parameters.yaml#/components/parameters/collectionsArray" + - $ref: "./components/parameters.yaml#/components/parameters/forecast_reference_datetime" + - $ref: "./components/parameters.yaml#/components/parameters/forecast_horizon" + - $ref: "./components/parameters.yaml#/components/parameters/forecast_duration" + - $ref: "./components/parameters.yaml#/components/parameters/forecast_variable" + - $ref: "./components/parameters.yaml#/components/parameters/forecast_perturbed" responses: "200": content: diff --git a/spec/static/spec/v1/openapi.yaml b/spec/static/spec/v1/openapi.yaml index 552bd542..24663687 100644 --- a/spec/static/spec/v1/openapi.yaml +++ b/spec/static/spec/v1/openapi.yaml @@ -115,6 +115,50 @@ components: required: false schema: type: string + forecast_reference_datetime: + explode: false + in: query + name: forecast_reference_datetime + required: false + schema: + $ref: "#/components/schemas/datetimeQuery" + example: 2018-02-12T00%3A00%3A00Z%2F2018-03-18T12%3A31%3A12Z + style: form + forecast_horizon: + explode: false + in: query + name: forecast_horizon + required: false + schema: + $ref: "#/components/schemas/duration" + example: P3DT6H + style: form + forecast_duration: + explode: false + in: query + name: forecast_duration + required: false + schema: + $ref: "#/components/schemas/duration" + example: P3DT6H + style: form + forecast_variable: + explode: false + in: query + name: forecast_variable + required: false + schema: + $ref: "#/components/schemas/forecast_variable" + example: air_temperature + style: form + forecast_perturbed: + explode: false + in: query + name: forecast_perturbed + required: false + schema: + $ref: "#/components/schemas/forecast_perturbed" + style: form responses: Collection: headers: @@ -1331,16 +1375,9 @@ components: forecast:duration: $ref: "#/components/schemas/duration" forecast:variable: - description: >- - Name of the model variable that corresponds to the data. The variables should correspond to the [CF Standard Names](https://cfconventions.org/Data/cf-standard-names/current/build/cf-standard-name-table.html), e.g. `air_temperature` for the air temperature. - example: air_temperature - type: string - nullable: true + $ref: "#/components/schemas/forecast_variable" forecast:perturbed: - description: >- - Denotes whether the data corresponds to the control run (`false`) or perturbed runs (`true`). The property needs to be specified in both cases as no default value is specified and as such the meaning is "unknown" in case it's missing. - type: boolean - nullable: true + $ref: "#/components/schemas/forecast_perturbed" required: - created - updated @@ -1697,6 +1734,37 @@ components: query: $ref: "#/components/schemas/query" type: object + forecast_reference_datetimeFilter: + properties: + forecast:reference_datetime: + $ref: "#/components/schemas/datetimeQuery" + forecast_horizonFilter: + properties: + forecast:horizon: + $ref: "#/components/schemas/duration" + forecast_durationFilter: + properties: + forecast:duration: + $ref: "#/components/schemas/duration" + forecast_variableFilter: + properties: + forecast:variable: + $ref: "#/components/schemas/forecast_variable" + forecast_variable: + description: >- + Name of the model variable that corresponds to the data. The variables should correspond to the [CF Standard Names](https://cfconventions.org/Data/cf-standard-names/current/build/cf-standard-name-table.html), e.g. `air_temperature` for the air temperature. + example: air_temperature + type: string + nullable: true + forecast_perturbedFilter: + properties: + forecast:perturbed: + $ref: "#/components/schemas/forecast_perturbed" + forecast_perturbed: + description: >- + Denotes whether the data corresponds to the control run (`false`) or perturbed runs (`true`). The property needs to be specified in both cases as no default value is specified and as such the meaning is "unknown" in case it's missing. + type: boolean + nullable: true queryProp: anyOf: - description: >- @@ -1783,6 +1851,11 @@ components: - $ref: "#/components/schemas/collectionsFilter" - $ref: "#/components/schemas/idsFilter" - $ref: "#/components/schemas/limitFilter" + - $ref: "#/components/schemas/forecast_reference_datetimeFilter" + - $ref: "#/components/schemas/forecast_horizonFilter" + - $ref: "#/components/schemas/forecast_durationFilter" + - $ref: "#/components/schemas/forecast_variableFilter" + - $ref: "#/components/schemas/forecast_perturbedFilter" description: The search criteria type: object stac_version: @@ -2091,6 +2164,11 @@ paths: - $ref: "#/components/parameters/limit" - $ref: "#/components/parameters/ids" - $ref: "#/components/parameters/collectionsArray" + - $ref: "#/components/parameters/forecast_reference_datetime" + - $ref: "#/components/parameters/forecast_horizon" + - $ref: "#/components/parameters/forecast_duration" + - $ref: "#/components/parameters/forecast_variable" + - $ref: "#/components/parameters/forecast_perturbed" responses: "200": content: diff --git a/spec/static/spec/v1/openapitransactional.yaml b/spec/static/spec/v1/openapitransactional.yaml index 7ab6759e..e3230d2f 100644 --- a/spec/static/spec/v1/openapitransactional.yaml +++ b/spec/static/spec/v1/openapitransactional.yaml @@ -115,6 +115,50 @@ components: required: false schema: type: string + forecast_reference_datetime: + explode: false + in: query + name: forecast_reference_datetime + required: false + schema: + $ref: "#/components/schemas/datetimeQuery" + example: 2018-02-12T00%3A00%3A00Z%2F2018-03-18T12%3A31%3A12Z + style: form + forecast_horizon: + explode: false + in: query + name: forecast_horizon + required: false + schema: + $ref: "#/components/schemas/duration" + example: P3DT6H + style: form + forecast_duration: + explode: false + in: query + name: forecast_duration + required: false + schema: + $ref: "#/components/schemas/duration" + example: P3DT6H + style: form + forecast_variable: + explode: false + in: query + name: forecast_variable + required: false + schema: + $ref: "#/components/schemas/forecast_variable" + example: air_temperature + style: form + forecast_perturbed: + explode: false + in: query + name: forecast_perturbed + required: false + schema: + $ref: "#/components/schemas/forecast_perturbed" + style: form assetId: name: assetId in: path @@ -1435,16 +1479,9 @@ components: forecast:duration: $ref: "#/components/schemas/duration" forecast:variable: - description: >- - Name of the model variable that corresponds to the data. The variables should correspond to the [CF Standard Names](https://cfconventions.org/Data/cf-standard-names/current/build/cf-standard-name-table.html), e.g. `air_temperature` for the air temperature. - example: air_temperature - type: string - nullable: true + $ref: "#/components/schemas/forecast_variable" forecast:perturbed: - description: >- - Denotes whether the data corresponds to the control run (`false`) or perturbed runs (`true`). The property needs to be specified in both cases as no default value is specified and as such the meaning is "unknown" in case it's missing. - type: boolean - nullable: true + $ref: "#/components/schemas/forecast_perturbed" required: - created - updated @@ -1805,6 +1842,37 @@ components: query: $ref: "#/components/schemas/query" type: object + forecast_reference_datetimeFilter: + properties: + forecast:reference_datetime: + $ref: "#/components/schemas/datetimeQuery" + forecast_horizonFilter: + properties: + forecast:horizon: + $ref: "#/components/schemas/duration" + forecast_durationFilter: + properties: + forecast:duration: + $ref: "#/components/schemas/duration" + forecast_variableFilter: + properties: + forecast:variable: + $ref: "#/components/schemas/forecast_variable" + forecast_variable: + description: >- + Name of the model variable that corresponds to the data. The variables should correspond to the [CF Standard Names](https://cfconventions.org/Data/cf-standard-names/current/build/cf-standard-name-table.html), e.g. `air_temperature` for the air temperature. + example: air_temperature + type: string + nullable: true + forecast_perturbedFilter: + properties: + forecast:perturbed: + $ref: "#/components/schemas/forecast_perturbed" + forecast_perturbed: + description: >- + Denotes whether the data corresponds to the control run (`false`) or perturbed runs (`true`). The property needs to be specified in both cases as no default value is specified and as such the meaning is "unknown" in case it's missing. + type: boolean + nullable: true queryProp: anyOf: - description: >- @@ -1891,6 +1959,11 @@ components: - $ref: "#/components/schemas/collectionsFilter" - $ref: "#/components/schemas/idsFilter" - $ref: "#/components/schemas/limitFilter" + - $ref: "#/components/schemas/forecast_reference_datetimeFilter" + - $ref: "#/components/schemas/forecast_horizonFilter" + - $ref: "#/components/schemas/forecast_durationFilter" + - $ref: "#/components/schemas/forecast_variableFilter" + - $ref: "#/components/schemas/forecast_perturbedFilter" description: The search criteria type: object stac_version: @@ -3511,6 +3584,11 @@ paths: - $ref: "#/components/parameters/limit" - $ref: "#/components/parameters/ids" - $ref: "#/components/parameters/collectionsArray" + - $ref: "#/components/parameters/forecast_reference_datetime" + - $ref: "#/components/parameters/forecast_horizon" + - $ref: "#/components/parameters/forecast_duration" + - $ref: "#/components/parameters/forecast_variable" + - $ref: "#/components/parameters/forecast_perturbed" responses: "200": content: From ad90dab334ba4b96b1f59f23183f671bd9e98a19 Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Thu, 23 Jan 2025 10:36:14 +0100 Subject: [PATCH 06/15] PB-1169: Search forecast fields Add filters for forecast properties to search endpoints. Can filter for forecast_reference_datetime either by exact value or by a range. --- app/stac_api/managers.py | 104 +++++++++ app/stac_api/validators_serializer.py | 14 +- app/stac_api/views/general.py | 13 ++ .../tests_10/sample_data/item_samples.py | 52 ++++- app/tests/tests_10/test_search_endpoint.py | 199 ++++++++++++++++++ 5 files changed, 380 insertions(+), 2 deletions(-) diff --git a/app/stac_api/managers.py b/app/stac_api/managers.py index 7ee411e5..8b0df7f1 100644 --- a/app/stac_api/managers.py +++ b/app/stac_api/managers.py @@ -105,6 +105,47 @@ def _filter_by_datetime_range(self, start_datetime, end_datetime): ) ) + def filter_by_forecast_reference_datetime(self, date_time): + '''Filter a queryset by forecast reference datetime + + Args: + queryset: + A django queryset (https://docs.djangoproject.com/en/3.0/ref/models/querysets/) + date_time: + A string + + Returns: + The queryset filtered by date_time + ''' + start, end = self._parse_datetime_query(date_time) + if end is not None: + return self._filter_by_forecast_reference_datetime_range(start, end) + return self.filter(forecast_reference_datetime=start) + + def _filter_by_forecast_reference_datetime_range(self, start_datetime, end_datetime): + '''Filter a queryset by forecast reference datetime range + + Helper function of filter_by_forecast_reference_datetime + + Args: + queryset: + A django queryset (https://docs.djangoproject.com/en/3.0/ref/models/querysets/) + start_datetime: + A string with the start datetime + end_datetime: + A string with the end datetime + Returns: + The queryset filtered by datetime range + ''' + if start_datetime == '..': + # open start range + return self.filter(forecast_reference_datetime__lte=end_datetime) + if end_datetime == '..': + # open end range + return self.filter(forecast_reference_datetime__gte=start_datetime) + # else fixed range + return self.filter(forecast_reference_datetime__range=(start_datetime, end_datetime)) + def _parse_datetime_query(self, date_time): '''Parse the datetime query as specified in the api-spec.md. @@ -189,6 +230,54 @@ def filter_by_intersects(self, intersects): the_geom = GEOSGeometry(intersects) return self.filter(geometry__intersects=the_geom) + def filter_by_forecast_horizon(self, duration): + '''Filter by forecast horizon + + Args: + duration: string + ISO 8601 duration string + + Returns: + queryset filtered by forecast horizon + ''' + return self.filter(forecast_horizon=duration) + + def filter_by_forecast_duration(self, duration): + '''Filter by forecast duration + + Args: + duration: string + ISO 8601 duration string + + Returns: + queryset filtered by forecast duration + ''' + return self.filter(forecast_duration=duration) + + def filter_by_forecast_variable(self, val): + '''Filter by forecast variable + + Args: + val: string + foracast variable value + + Returns: + queryset filtered by forecast variable + ''' + return self.filter(forecast_variable=val) + + def filter_by_forecast_perturbed(self, pert): + '''Filter by forecast perturbed + + Args: + pert: boolean + foracast perturbed + + Returns: + queryset filtered by forecast perturbed + ''' + return self.filter(forecast_perturbed=pert) + def filter_by_query(self, query): '''Filter by the query parameter @@ -236,6 +325,21 @@ def filter_by_item_name(self, item_name_array): def filter_by_query(self, query): return self.get_queryset().filter_by_query(query) + def filter_by_forecast_reference_datetime(self, date_time): + return self.get_queryset().filter_by_forecast_reference_datetime(date_time) + + def filter_by_forecast_horizon(self, duration): + return self.get_queryset().filter_by_forecast_horizon(duration) + + def filter_by_forecast_duration(self, duration): + return self.get_queryset().filter_by_forecast_duration(duration) + + def filter_by_forecast_variable(self, val): + return self.get_queryset().filter_by_forecast_variable(val) + + def filter_by_forecast_perturbed(self, pert): + return self.get_queryset().filter_by_forecast_perturbed(pert) + class AssetUploadQuerySet(models.QuerySet): diff --git a/app/stac_api/validators_serializer.py b/app/stac_api/validators_serializer.py index d0114e6b..56a7d430 100644 --- a/app/stac_api/validators_serializer.py +++ b/app/stac_api/validators_serializer.py @@ -351,7 +351,19 @@ def validate_query_parameters_post_search(self, query_param): Copy of the harmonized QueryDict ''' accepted_query_parameters = [ - "bbox", "collections", "datetime", "ids", "intersects", "limit", "cursor", "query" + "bbox", + "collections", + "datetime", + "ids", + "intersects", + "limit", + "cursor", + "query", + "forecast_reference_datetime", + "forecast_horizon", + "forecast_duration", + "forecast_variable", + "forecast_perturbed" ] wrong_query_parameters = set(query_param.keys()).difference(set(accepted_query_parameters)) if wrong_query_parameters: diff --git a/app/stac_api/views/general.py b/app/stac_api/views/general.py index 3b4d3a71..498766ea 100644 --- a/app/stac_api/views/general.py +++ b/app/stac_api/views/general.py @@ -69,6 +69,7 @@ class SearchList(generics.GenericAPIView, mixins.ListModelMixin): # we must use the pk as ordering attribute, otherwise the cursor pagination will not work ordering = ['pk'] + # pylint: disable=too-many-branches def get_queryset(self): queryset = Item.objects.filter(collection__published=True ).prefetch_related('assets', 'links') @@ -92,6 +93,18 @@ def get_queryset(self): queryset = queryset.filter_by_query(dict_query) if 'intersects' in query_param: queryset = queryset.filter_by_intersects(json.dumps(query_param['intersects'])) + if 'forecast_reference_datetime' in query_param: + queryset = queryset.filter_by_forecast_reference_datetime( + query_param['forecast_reference_datetime'] + ) + if 'forecast_horizon' in query_param: + queryset = queryset.filter_by_forecast_horizon(query_param['forecast_horizon']) + if 'forecast_duration' in query_param: + queryset = queryset.filter_by_forecast_duration(query_param['forecast_duration']) + if 'forecast_variable' in query_param: + queryset = queryset.filter_by_forecast_variable(query_param['forecast_variable']) + if 'forecast_perturbed' in query_param: + queryset = queryset.filter_by_forecast_perturbed(query_param['forecast_perturbed']) if settings.DEBUG_ENABLE_DB_EXPLAIN_ANALYZE: logger.debug( diff --git a/app/tests/tests_10/sample_data/item_samples.py b/app/tests/tests_10/sample_data/item_samples.py index 999696a8..5b4dd12a 100644 --- a/app/tests/tests_10/sample_data/item_samples.py +++ b/app/tests/tests_10/sample_data/item_samples.py @@ -218,5 +218,55 @@ 'name': 'item-covers_switzerland', 'geometry': geometries['covers-switzerland'], 'properties_datetime': fromisoformat('2020-10-28T13:05:10Z') - } + }, + 'item-forecast-1': { + 'name': 'item-forecast-1', + 'geometry': geometries['covers-switzerland'], + 'properties_datetime': fromisoformat('2020-10-28T13:05:10Z'), + 'forecast_reference_datetime': fromisoformat('2025-01-01T13:05:10Z'), + 'forecast_horizon': 'PT6H', + 'forecast_duration': 'PT12H', + 'forecast_variable': 'T', + 'forecast_perturbed': 'False', + }, + 'item-forecast-2': { + 'name': 'item-forecast-2', + 'geometry': geometries['covers-switzerland'], + 'properties_datetime': fromisoformat('2020-10-28T13:05:10Z'), + 'forecast_reference_datetime': fromisoformat('2025-02-01T13:05:10Z'), + 'forecast_horizon': 'PT6H', + 'forecast_duration': 'PT12H', + 'forecast_variable': 'T', + 'forecast_perturbed': 'False', + }, + 'item-forecast-3': { + 'name': 'item-forecast-3', + 'geometry': geometries['covers-switzerland'], + 'properties_datetime': fromisoformat('2020-10-28T13:05:10Z'), + 'forecast_reference_datetime': fromisoformat('2025-02-01T13:05:10Z'), + 'forecast_horizon': 'PT3H', + 'forecast_duration': 'PT6H', + 'forecast_variable': 'T', + 'forecast_perturbed': 'False', + }, + 'item-forecast-4': { + 'name': 'item-forecast-4', + 'geometry': geometries['covers-switzerland'], + 'properties_datetime': fromisoformat('2020-10-28T13:05:10Z'), + 'forecast_reference_datetime': fromisoformat('2025-02-01T13:05:10Z'), + 'forecast_horizon': 'PT6H', + 'forecast_duration': 'PT12H', + 'forecast_variable': 'air_temperature', + 'forecast_perturbed': 'True', + }, + 'item-forecast-5': { + 'name': 'item-forecast-5', + 'geometry': geometries['covers-switzerland'], + 'properties_datetime': fromisoformat('2020-10-28T13:05:10Z'), + 'forecast_reference_datetime': fromisoformat('2025-04-01T13:05:10Z'), + 'forecast_horizon': 'PT6H', + 'forecast_duration': 'PT12H', + 'forecast_variable': 'air_temperature', + 'forecast_perturbed': 'False', + }, } diff --git a/app/tests/tests_10/test_search_endpoint.py b/app/tests/tests_10/test_search_endpoint.py index 63321f08..8043a263 100644 --- a/app/tests/tests_10/test_search_endpoint.py +++ b/app/tests/tests_10/test_search_endpoint.py @@ -580,3 +580,202 @@ def test_post_search_no_cache_setting(self): response.has_header('Cache-Control'), msg="Unexpected Cache-Control header in POST response" ) + + +class SearchEndpointTestForecast(StacBaseTestCase): + + @classmethod + def setUpTestData(cls): + cls.factory = Factory() + cls.collection = cls.factory.create_collection_sample().model + cls.items = cls.factory.create_item_samples( + [ + 'item-forecast-1', + 'item-forecast-2', + 'item-forecast-3', + 'item-forecast-4', + 'item-forecast-5' + ], + cls.collection, + db_create=True, + ) + cls.now = utc_aware(datetime.utcnow()) + cls.yesterday = cls.now - timedelta(days=1) + + def setUp(self): # pylint: disable=invalid-name + self.client = Client() + client_login(self.client) + self.path = f'/{STAC_BASE_V}/search' + self.maxDiff = None # pylint: disable=invalid-name + + def test_reference_datetime_exact(self): + val = '2025-01-01T13:05:10Z' + response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 1) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-1']) + + payload = {"forecast_reference_datetime": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 1) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-1']) + + val = '2025-02-01T13:05:10Z' + response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 3) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) + + payload = {"forecast_reference_datetime": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 3) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) + + def test_reference_datetime_range(self): + val = '2025-02-01T00:00:00Z/2025-02-28T00:00:00Z' + response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 3) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) + + payload = {"forecast_reference_datetime": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 3) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) + + def test_reference_datetime_open_end(self): + val = '2025-02-01T13:05:10Z/..' + response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 4) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4', 'item-5']) + + payload = {"forecast_reference_datetime": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 4) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4', 'item-5']) + + def test_reference_datetime_open_start(self): + val = '../2025-02-01T13:05:10Z' + response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 4) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-1', 'item-2', 'item-3', 'item-4']) + + payload = {"forecast_reference_datetime": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 4) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-1', 'item-2', 'item-3', 'item-4']) + + def test_horizon(self): + val = 'PT3H' + response = self.client.get(f"{self.path}?forecast_horizon={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 1) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-3']) + + payload = {"forecast_horizon": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 1) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-3']) + + def test_duration(self): + val = 'PT12H' + response = self.client.get(f"{self.path}?forecast_duration={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 4) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-1', 'item-2', 'item-4', 'item-5']) + + payload = {"forecast_duration": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 4) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-1', 'item-2', 'item-4', 'item-5']) + + def test_variable(self): + val = 'air_temperature' + response = self.client.get(f"{self.path}?forecast_variable={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 2) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-4', 'item-5']) + + payload = {"forecast_variable": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 2) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-4', 'item-5']) + + def test_perturbed(self): + val = 'True' + response = self.client.get(f"{self.path}?forecast_perturbed={val}") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 1) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-4']) + + payload = {"forecast_perturbed": f"{val}"} + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 1) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-4']) + + def test_multiple(self): + response = self.client.get( + f"{self.path}?forecast_perturbed=False&forecast_horizon=PT6H&forecast_variable=T" + ) + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 2) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-1', 'item-2']) + + payload = { + "forecast_perturbed": "False", "forecast_horizon": "PT6H", "forecast_variable": "T" + } + response = self.client.post(self.path, data=payload, content_type="application/json") + self.assertStatusCode(200, response) + json_data = response.json() + self.assertEqual(len(json_data['features']), 2) + for feature in json_data['features']: + self.assertIn(feature['id'], ['item-1', 'item-2']) From b29937ce49112beb958740148a3cc07ec9c6b55b Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Thu, 23 Jan 2025 15:23:18 +0100 Subject: [PATCH 07/15] PB-1169: Spec remove forecast params from Get search Decided to remove the forecast parameters because the variable name differs to the post variable names (using underscore instead of colon) and colons need url encoding. --- spec/components/parameters.yaml | 44 ---------------- spec/openapi.yaml | 8 +-- spec/static/spec/v1/openapi.yaml | 51 +------------------ spec/static/spec/v1/openapitransactional.yaml | 51 +------------------ 4 files changed, 4 insertions(+), 150 deletions(-) diff --git a/spec/components/parameters.yaml b/spec/components/parameters.yaml index 84b8ea24..63ca9bbd 100644 --- a/spec/components/parameters.yaml +++ b/spec/components/parameters.yaml @@ -117,47 +117,3 @@ components: required: false schema: type: string - forecast_reference_datetime: - explode: false - in: query - name: forecast_reference_datetime - required: false - schema: - $ref: "./schemas.yaml#/components/schemas/datetimeQuery" - example: 2018-02-12T00%3A00%3A00Z%2F2018-03-18T12%3A31%3A12Z - style: form - forecast_horizon: - explode: false - in: query - name: forecast_horizon - required: false - schema: - $ref: "./schemas.yaml#/components/schemas/duration" - example: P3DT6H - style: form - forecast_duration: - explode: false - in: query - name: forecast_duration - required: false - schema: - $ref: "./schemas.yaml#/components/schemas/duration" - example: P3DT6H - style: form - forecast_variable: - explode: false - in: query - name: forecast_variable - required: false - schema: - $ref: "./schemas.yaml#/components/schemas/forecast_variable" - example: air_temperature - style: form - forecast_perturbed: - explode: false - in: query - name: forecast_perturbed - required: false - schema: - $ref: "./schemas.yaml#/components/schemas/forecast_perturbed" - style: form diff --git a/spec/openapi.yaml b/spec/openapi.yaml index befc9641..a49fcec1 100644 --- a/spec/openapi.yaml +++ b/spec/openapi.yaml @@ -274,7 +274,8 @@ paths: get: description: >- Retrieve Items matching filters. Intended as a shorthand API for simple - queries. + queries. To filter by forecast properties please use the + [POST /search](#tag/STAC/operation/postSearchSTAC) request. operationId: getSearchSTAC parameters: @@ -283,11 +284,6 @@ paths: - $ref: "./components/parameters.yaml#/components/parameters/limit" - $ref: "./components/parameters.yaml#/components/parameters/ids" - $ref: "./components/parameters.yaml#/components/parameters/collectionsArray" - - $ref: "./components/parameters.yaml#/components/parameters/forecast_reference_datetime" - - $ref: "./components/parameters.yaml#/components/parameters/forecast_horizon" - - $ref: "./components/parameters.yaml#/components/parameters/forecast_duration" - - $ref: "./components/parameters.yaml#/components/parameters/forecast_variable" - - $ref: "./components/parameters.yaml#/components/parameters/forecast_perturbed" responses: "200": content: diff --git a/spec/static/spec/v1/openapi.yaml b/spec/static/spec/v1/openapi.yaml index 24663687..c6e61d31 100644 --- a/spec/static/spec/v1/openapi.yaml +++ b/spec/static/spec/v1/openapi.yaml @@ -115,50 +115,6 @@ components: required: false schema: type: string - forecast_reference_datetime: - explode: false - in: query - name: forecast_reference_datetime - required: false - schema: - $ref: "#/components/schemas/datetimeQuery" - example: 2018-02-12T00%3A00%3A00Z%2F2018-03-18T12%3A31%3A12Z - style: form - forecast_horizon: - explode: false - in: query - name: forecast_horizon - required: false - schema: - $ref: "#/components/schemas/duration" - example: P3DT6H - style: form - forecast_duration: - explode: false - in: query - name: forecast_duration - required: false - schema: - $ref: "#/components/schemas/duration" - example: P3DT6H - style: form - forecast_variable: - explode: false - in: query - name: forecast_variable - required: false - schema: - $ref: "#/components/schemas/forecast_variable" - example: air_temperature - style: form - forecast_perturbed: - explode: false - in: query - name: forecast_perturbed - required: false - schema: - $ref: "#/components/schemas/forecast_perturbed" - style: form responses: Collection: headers: @@ -2156,7 +2112,7 @@ paths: /search: get: description: >- - Retrieve Items matching filters. Intended as a shorthand API for simple queries. + Retrieve Items matching filters. Intended as a shorthand API for simple queries. To filter by forecast properties please use the [POST /search](#tag/STAC/operation/postSearchSTAC) request. operationId: getSearchSTAC parameters: - $ref: "#/components/parameters/bbox" @@ -2164,11 +2120,6 @@ paths: - $ref: "#/components/parameters/limit" - $ref: "#/components/parameters/ids" - $ref: "#/components/parameters/collectionsArray" - - $ref: "#/components/parameters/forecast_reference_datetime" - - $ref: "#/components/parameters/forecast_horizon" - - $ref: "#/components/parameters/forecast_duration" - - $ref: "#/components/parameters/forecast_variable" - - $ref: "#/components/parameters/forecast_perturbed" responses: "200": content: diff --git a/spec/static/spec/v1/openapitransactional.yaml b/spec/static/spec/v1/openapitransactional.yaml index e3230d2f..4133df4d 100644 --- a/spec/static/spec/v1/openapitransactional.yaml +++ b/spec/static/spec/v1/openapitransactional.yaml @@ -115,50 +115,6 @@ components: required: false schema: type: string - forecast_reference_datetime: - explode: false - in: query - name: forecast_reference_datetime - required: false - schema: - $ref: "#/components/schemas/datetimeQuery" - example: 2018-02-12T00%3A00%3A00Z%2F2018-03-18T12%3A31%3A12Z - style: form - forecast_horizon: - explode: false - in: query - name: forecast_horizon - required: false - schema: - $ref: "#/components/schemas/duration" - example: P3DT6H - style: form - forecast_duration: - explode: false - in: query - name: forecast_duration - required: false - schema: - $ref: "#/components/schemas/duration" - example: P3DT6H - style: form - forecast_variable: - explode: false - in: query - name: forecast_variable - required: false - schema: - $ref: "#/components/schemas/forecast_variable" - example: air_temperature - style: form - forecast_perturbed: - explode: false - in: query - name: forecast_perturbed - required: false - schema: - $ref: "#/components/schemas/forecast_perturbed" - style: form assetId: name: assetId in: path @@ -3576,7 +3532,7 @@ paths: /search: get: description: >- - Retrieve Items matching filters. Intended as a shorthand API for simple queries. + Retrieve Items matching filters. Intended as a shorthand API for simple queries. To filter by forecast properties please use the [POST /search](#tag/STAC/operation/postSearchSTAC) request. operationId: getSearchSTAC parameters: - $ref: "#/components/parameters/bbox" @@ -3584,11 +3540,6 @@ paths: - $ref: "#/components/parameters/limit" - $ref: "#/components/parameters/ids" - $ref: "#/components/parameters/collectionsArray" - - $ref: "#/components/parameters/forecast_reference_datetime" - - $ref: "#/components/parameters/forecast_horizon" - - $ref: "#/components/parameters/forecast_duration" - - $ref: "#/components/parameters/forecast_variable" - - $ref: "#/components/parameters/forecast_perturbed" responses: "200": content: From 757c46dd593228d68fa948fc04ec684854dc00bc Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Thu, 23 Jan 2025 15:52:17 +0100 Subject: [PATCH 08/15] PB-1169: Remove forecast filter from GET search Decided to remove the forecast filtering options from the GET /search request due to colons in the parameter names. Also fixed the POST request body to actually use the parameters that contain a colon. --- app/stac_api/utils.py | 14 +++ app/stac_api/validators_serializer.py | 10 +- app/stac_api/views/general.py | 20 ++-- app/tests/tests_10/test_search_endpoint.py | 114 +++++---------------- 4 files changed, 52 insertions(+), 106 deletions(-) diff --git a/app/stac_api/utils.py b/app/stac_api/utils.py index af354a17..9cff55af 100644 --- a/app/stac_api/utils.py +++ b/app/stac_api/utils.py @@ -309,6 +309,20 @@ def harmonize_post_get_for_search(request): query_param['ids'] = query_param['ids'].split(',') # to array if 'collections' in query_param: query_param['collections'] = query_param['collections'].split(',') # to array + + # Forecast properties can only be filtered with method POST. + # Decision was made as `:` need to be url encoded and (at least for now) we do not need to + # support forecast filtering in the GET request. + if 'forecast:reference_datetime' in query_param: + del query_param['forecast:reference_datetime'] + if 'forecast:horizon' in query_param: + del query_param['forecast:horizon'] + if 'forecast:duration' in query_param: + del query_param['forecast:duration'] + if 'forecast:variable' in query_param: + del query_param['forecast:variable'] + if 'forecast:perturbed' in query_param: + del query_param['forecast:perturbed'] return query_param diff --git a/app/stac_api/validators_serializer.py b/app/stac_api/validators_serializer.py index 56a7d430..509d38e7 100644 --- a/app/stac_api/validators_serializer.py +++ b/app/stac_api/validators_serializer.py @@ -359,11 +359,11 @@ def validate_query_parameters_post_search(self, query_param): "limit", "cursor", "query", - "forecast_reference_datetime", - "forecast_horizon", - "forecast_duration", - "forecast_variable", - "forecast_perturbed" + "forecast:reference_datetime", + "forecast:horizon", + "forecast:duration", + "forecast:variable", + "forecast:perturbed" ] wrong_query_parameters = set(query_param.keys()).difference(set(accepted_query_parameters)) if wrong_query_parameters: diff --git a/app/stac_api/views/general.py b/app/stac_api/views/general.py index 498766ea..662ed813 100644 --- a/app/stac_api/views/general.py +++ b/app/stac_api/views/general.py @@ -93,18 +93,18 @@ def get_queryset(self): queryset = queryset.filter_by_query(dict_query) if 'intersects' in query_param: queryset = queryset.filter_by_intersects(json.dumps(query_param['intersects'])) - if 'forecast_reference_datetime' in query_param: + if 'forecast:reference_datetime' in query_param: queryset = queryset.filter_by_forecast_reference_datetime( - query_param['forecast_reference_datetime'] + query_param['forecast:reference_datetime'] ) - if 'forecast_horizon' in query_param: - queryset = queryset.filter_by_forecast_horizon(query_param['forecast_horizon']) - if 'forecast_duration' in query_param: - queryset = queryset.filter_by_forecast_duration(query_param['forecast_duration']) - if 'forecast_variable' in query_param: - queryset = queryset.filter_by_forecast_variable(query_param['forecast_variable']) - if 'forecast_perturbed' in query_param: - queryset = queryset.filter_by_forecast_perturbed(query_param['forecast_perturbed']) + if 'forecast:horizon' in query_param: + queryset = queryset.filter_by_forecast_horizon(query_param['forecast:horizon']) + if 'forecast:duration' in query_param: + queryset = queryset.filter_by_forecast_duration(query_param['forecast:duration']) + if 'forecast:variable' in query_param: + queryset = queryset.filter_by_forecast_variable(query_param['forecast:variable']) + if 'forecast:perturbed' in query_param: + queryset = queryset.filter_by_forecast_perturbed(query_param['forecast:perturbed']) if settings.DEBUG_ENABLE_DB_EXPLAIN_ANALYZE: logger.debug( diff --git a/app/tests/tests_10/test_search_endpoint.py b/app/tests/tests_10/test_search_endpoint.py index 8043a263..dadb6be3 100644 --- a/app/tests/tests_10/test_search_endpoint.py +++ b/app/tests/tests_10/test_search_endpoint.py @@ -2,6 +2,7 @@ import logging from datetime import datetime from datetime import timedelta +from urllib.parse import quote_plus from django.test import Client from django.test import override_settings @@ -609,15 +610,7 @@ def setUp(self): # pylint: disable=invalid-name self.maxDiff = None # pylint: disable=invalid-name def test_reference_datetime_exact(self): - val = '2025-01-01T13:05:10Z' - response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 1) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-1']) - - payload = {"forecast_reference_datetime": f"{val}"} + payload = {"forecast:reference_datetime": "2025-01-01T13:05:10Z"} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -625,15 +618,7 @@ def test_reference_datetime_exact(self): for feature in json_data['features']: self.assertIn(feature['id'], ['item-1']) - val = '2025-02-01T13:05:10Z' - response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 3) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) - - payload = {"forecast_reference_datetime": f"{val}"} + payload = {"forecast:reference_datetime": "2025-02-01T13:05:10Z"} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -642,15 +627,7 @@ def test_reference_datetime_exact(self): self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) def test_reference_datetime_range(self): - val = '2025-02-01T00:00:00Z/2025-02-28T00:00:00Z' - response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 3) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) - - payload = {"forecast_reference_datetime": f"{val}"} + payload = {"forecast:reference_datetime": "2025-02-01T00:00:00Z/2025-02-28T00:00:00Z"} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -659,15 +636,7 @@ def test_reference_datetime_range(self): self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) def test_reference_datetime_open_end(self): - val = '2025-02-01T13:05:10Z/..' - response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 4) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4', 'item-5']) - - payload = {"forecast_reference_datetime": f"{val}"} + payload = {"forecast:reference_datetime": "2025-02-01T13:05:10Z/.."} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -676,15 +645,7 @@ def test_reference_datetime_open_end(self): self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4', 'item-5']) def test_reference_datetime_open_start(self): - val = '../2025-02-01T13:05:10Z' - response = self.client.get(f"{self.path}?forecast_reference_datetime={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 4) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-1', 'item-2', 'item-3', 'item-4']) - - payload = {"forecast_reference_datetime": f"{val}"} + payload = {"forecast:reference_datetime": "../2025-02-01T13:05:10Z"} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -693,15 +654,7 @@ def test_reference_datetime_open_start(self): self.assertIn(feature['id'], ['item-1', 'item-2', 'item-3', 'item-4']) def test_horizon(self): - val = 'PT3H' - response = self.client.get(f"{self.path}?forecast_horizon={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 1) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-3']) - - payload = {"forecast_horizon": f"{val}"} + payload = {"forecast:horizon": "PT3H"} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -710,15 +663,7 @@ def test_horizon(self): self.assertIn(feature['id'], ['item-3']) def test_duration(self): - val = 'PT12H' - response = self.client.get(f"{self.path}?forecast_duration={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 4) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-1', 'item-2', 'item-4', 'item-5']) - - payload = {"forecast_duration": f"{val}"} + payload = {"forecast:duration": "PT12H"} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -727,15 +672,7 @@ def test_duration(self): self.assertIn(feature['id'], ['item-1', 'item-2', 'item-4', 'item-5']) def test_variable(self): - val = 'air_temperature' - response = self.client.get(f"{self.path}?forecast_variable={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 2) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-4', 'item-5']) - - payload = {"forecast_variable": f"{val}"} + payload = {"forecast:variable": "air_temperature"} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -744,15 +681,7 @@ def test_variable(self): self.assertIn(feature['id'], ['item-4', 'item-5']) def test_perturbed(self): - val = 'True' - response = self.client.get(f"{self.path}?forecast_perturbed={val}") - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 1) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-4']) - - payload = {"forecast_perturbed": f"{val}"} + payload = {"forecast:perturbed": "True"} response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) json_data = response.json() @@ -761,17 +690,8 @@ def test_perturbed(self): self.assertIn(feature['id'], ['item-4']) def test_multiple(self): - response = self.client.get( - f"{self.path}?forecast_perturbed=False&forecast_horizon=PT6H&forecast_variable=T" - ) - self.assertStatusCode(200, response) - json_data = response.json() - self.assertEqual(len(json_data['features']), 2) - for feature in json_data['features']: - self.assertIn(feature['id'], ['item-1', 'item-2']) - payload = { - "forecast_perturbed": "False", "forecast_horizon": "PT6H", "forecast_variable": "T" + "forecast:perturbed": "False", "forecast:horizon": "PT6H", "forecast:variable": "T" } response = self.client.post(self.path, data=payload, content_type="application/json") self.assertStatusCode(200, response) @@ -779,3 +699,15 @@ def test_multiple(self): self.assertEqual(len(json_data['features']), 2) for feature in json_data['features']: self.assertIn(feature['id'], ['item-1', 'item-2']) + + def test_get_request_does_not_filter_forecast(self): + response = self.client.get( + f"{self.path}?" + quote_plus( + "forecast:reference_datetime=2025-01-01T13:05:10Z&" + "forecast:duration=PT12H&" + + "forecast:perturbed=False&" + "forecast:horizon=PT6H&" + "forecast:variable=T" + ) + ) + self.assertStatusCode(200, response) + json_data = response.json() + # As GET request should not filter for forecast expect all 5 features to be returned. + self.assertEqual(len(json_data['features']), 5) From d2492aaad2de46fc621701ccc923b42057e572d3 Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Mon, 27 Jan 2025 08:56:47 +0100 Subject: [PATCH 09/15] Fix for review comments Fix code comments. Refactor create sample data names. --- app/stac_api/managers.py | 10 ++-- app/tests/tests_10/test_search_endpoint.py | 53 ++++++++++++++-------- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/app/stac_api/managers.py b/app/stac_api/managers.py index 8b0df7f1..6075cc0e 100644 --- a/app/stac_api/managers.py +++ b/app/stac_api/managers.py @@ -109,10 +109,8 @@ def filter_by_forecast_reference_datetime(self, date_time): '''Filter a queryset by forecast reference datetime Args: - queryset: - A django queryset (https://docs.djangoproject.com/en/3.0/ref/models/querysets/) date_time: - A string + A string containing datetime like "2020-10-28T13:05:10Z" Returns: The queryset filtered by date_time @@ -128,12 +126,10 @@ def _filter_by_forecast_reference_datetime_range(self, start_datetime, end_datet Helper function of filter_by_forecast_reference_datetime Args: - queryset: - A django queryset (https://docs.djangoproject.com/en/3.0/ref/models/querysets/) start_datetime: - A string with the start datetime + A string with the start datetime or ".." to denote open start end_datetime: - A string with the end datetime + A string with the end datetime or ".." to denote open end Returns: The queryset filtered by datetime range ''' diff --git a/app/tests/tests_10/test_search_endpoint.py b/app/tests/tests_10/test_search_endpoint.py index dadb6be3..b48023a2 100644 --- a/app/tests/tests_10/test_search_endpoint.py +++ b/app/tests/tests_10/test_search_endpoint.py @@ -589,16 +589,20 @@ class SearchEndpointTestForecast(StacBaseTestCase): def setUpTestData(cls): cls.factory = Factory() cls.collection = cls.factory.create_collection_sample().model - cls.items = cls.factory.create_item_samples( - [ - 'item-forecast-1', - 'item-forecast-2', - 'item-forecast-3', - 'item-forecast-4', - 'item-forecast-5' - ], - cls.collection, - db_create=True, + cls.factory.create_item_sample( + cls.collection, 'item-forecast-1', 'item-forecast-1', db_create=True + ) + cls.factory.create_item_sample( + cls.collection, 'item-forecast-2', 'item-forecast-2', db_create=True + ) + cls.factory.create_item_sample( + cls.collection, 'item-forecast-3', 'item-forecast-3', db_create=True + ) + cls.factory.create_item_sample( + cls.collection, 'item-forecast-4', 'item-forecast-4', db_create=True + ) + cls.factory.create_item_sample( + cls.collection, 'item-forecast-5', 'item-forecast-5', db_create=True ) cls.now = utc_aware(datetime.utcnow()) cls.yesterday = cls.now - timedelta(days=1) @@ -616,7 +620,7 @@ def test_reference_datetime_exact(self): json_data = response.json() self.assertEqual(len(json_data['features']), 1) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-1']) + self.assertIn(feature['id'], ['item-forecast-1']) payload = {"forecast:reference_datetime": "2025-02-01T13:05:10Z"} response = self.client.post(self.path, data=payload, content_type="application/json") @@ -624,7 +628,7 @@ def test_reference_datetime_exact(self): json_data = response.json() self.assertEqual(len(json_data['features']), 3) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) + self.assertIn(feature['id'], ['item-forecast-2', 'item-forecast-3', 'item-forecast-4']) def test_reference_datetime_range(self): payload = {"forecast:reference_datetime": "2025-02-01T00:00:00Z/2025-02-28T00:00:00Z"} @@ -633,7 +637,7 @@ def test_reference_datetime_range(self): json_data = response.json() self.assertEqual(len(json_data['features']), 3) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4']) + self.assertIn(feature['id'], ['item-forecast-2', 'item-forecast-3', 'item-forecast-4']) def test_reference_datetime_open_end(self): payload = {"forecast:reference_datetime": "2025-02-01T13:05:10Z/.."} @@ -642,7 +646,10 @@ def test_reference_datetime_open_end(self): json_data = response.json() self.assertEqual(len(json_data['features']), 4) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-2', 'item-3', 'item-4', 'item-5']) + self.assertIn( + feature['id'], + ['item-forecast-2', 'item-forecast-3', 'item-forecast-4', 'item-forecast-5'] + ) def test_reference_datetime_open_start(self): payload = {"forecast:reference_datetime": "../2025-02-01T13:05:10Z"} @@ -651,7 +658,10 @@ def test_reference_datetime_open_start(self): json_data = response.json() self.assertEqual(len(json_data['features']), 4) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-1', 'item-2', 'item-3', 'item-4']) + self.assertIn( + feature['id'], + ['item-forecast-1', 'item-forecast-2', 'item-forecast-3', 'item-forecast-4'] + ) def test_horizon(self): payload = {"forecast:horizon": "PT3H"} @@ -660,7 +670,7 @@ def test_horizon(self): json_data = response.json() self.assertEqual(len(json_data['features']), 1) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-3']) + self.assertIn(feature['id'], ['item-forecast-3']) def test_duration(self): payload = {"forecast:duration": "PT12H"} @@ -669,7 +679,10 @@ def test_duration(self): json_data = response.json() self.assertEqual(len(json_data['features']), 4) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-1', 'item-2', 'item-4', 'item-5']) + self.assertIn( + feature['id'], + ['item-forecast-1', 'item-forecast-2', 'item-forecast-4', 'item-forecast-5'] + ) def test_variable(self): payload = {"forecast:variable": "air_temperature"} @@ -678,7 +691,7 @@ def test_variable(self): json_data = response.json() self.assertEqual(len(json_data['features']), 2) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-4', 'item-5']) + self.assertIn(feature['id'], ['item-forecast-4', 'item-forecast-5']) def test_perturbed(self): payload = {"forecast:perturbed": "True"} @@ -687,7 +700,7 @@ def test_perturbed(self): json_data = response.json() self.assertEqual(len(json_data['features']), 1) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-4']) + self.assertIn(feature['id'], ['item-forecast-4']) def test_multiple(self): payload = { @@ -698,7 +711,7 @@ def test_multiple(self): json_data = response.json() self.assertEqual(len(json_data['features']), 2) for feature in json_data['features']: - self.assertIn(feature['id'], ['item-1', 'item-2']) + self.assertIn(feature['id'], ['item-forecast-1', 'item-forecast-2']) def test_get_request_does_not_filter_forecast(self): response = self.client.get( From e5aa63f400b4b3ad17ef8eae2af2e04fa93e7521 Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Mon, 27 Jan 2025 10:23:49 +0100 Subject: [PATCH 10/15] PB-1169: Index item forecast fields Add index for the item forecast properties as they can be used as filters in the item search. --- ...item_fc_reference_datetime_idx_and_more.py | 36 +++++++++++++++++++ app/stac_api/models.py | 8 +++++ 2 files changed, 44 insertions(+) create mode 100644 app/stac_api/migrations/0062_item_item_fc_reference_datetime_idx_and_more.py diff --git a/app/stac_api/migrations/0062_item_item_fc_reference_datetime_idx_and_more.py b/app/stac_api/migrations/0062_item_item_fc_reference_datetime_idx_and_more.py new file mode 100644 index 00000000..5297a700 --- /dev/null +++ b/app/stac_api/migrations/0062_item_item_fc_reference_datetime_idx_and_more.py @@ -0,0 +1,36 @@ +# Generated by Django 5.0.11 on 2025-01-27 08:22 + +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ('stac_api', '0061_remove_item_forecast_mode_remove_item_forecast_param_and_more'), + ] + + operations = [ + migrations.AddIndex( + model_name='item', + index=models.Index( + fields=['forecast_reference_datetime'], name='item_fc_reference_datetime_idx' + ), + ), + migrations.AddIndex( + model_name='item', + index=models.Index(fields=['forecast_horizon'], name='item_fc_horizon_idx'), + ), + migrations.AddIndex( + model_name='item', + index=models.Index(fields=['forecast_duration'], name='item_fc_duration_idx'), + ), + migrations.AddIndex( + model_name='item', + index=models.Index(fields=['forecast_variable'], name='item_fc_variable_idx'), + ), + migrations.AddIndex( + model_name='item', + index=models.Index(fields=['forecast_perturbed'], name='item_fc_perturbed_idx'), + ), + ] diff --git a/app/stac_api/models.py b/app/stac_api/models.py index 934f613b..c6b7cd8f 100644 --- a/app/stac_api/models.py +++ b/app/stac_api/models.py @@ -370,6 +370,14 @@ class Meta: models.Index(fields=['created'], name='item_created_idx'), models.Index(fields=['updated'], name='item_updated_idx'), models.Index(fields=['properties_title'], name='item_title_idx'), + # forecast properties are "queryable" in the search endpoint + models.Index( + fields=['forecast_reference_datetime'], name='item_fc_reference_datetime_idx' + ), + models.Index(fields=['forecast_horizon'], name='item_fc_horizon_idx'), + models.Index(fields=['forecast_duration'], name='item_fc_duration_idx'), + models.Index(fields=['forecast_variable'], name='item_fc_variable_idx'), + models.Index(fields=['forecast_perturbed'], name='item_fc_perturbed_idx'), # combination of datetime and start_ and end_datetimes are used in # managers.py:110 and following models.Index( From 18d698be3a0d3d716e979e6816162484fa6b56cf Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Mon, 27 Jan 2025 10:57:40 +0100 Subject: [PATCH 11/15] Refactor models - move file Move models file into folder /models and rename file to general.py. Fix all imports that referenced the models file. --- app/stac_api/admin.py | 22 ++-- .../management/commands/calculate_extent.py | 2 +- .../management/commands/dummy_asset_upload.py | 6 +- .../management/commands/dummy_data.py | 6 +- .../management/commands/list_asset_uploads.py | 2 +- .../commands/profile_cursor_paginator.py | 2 +- .../commands/profile_item_serializer.py | 2 +- .../commands/profile_serializer_vs_no_drf.py | 2 +- .../commands/remove_expired_items.py | 6 +- .../commands/update_asset_file_size.py | 4 +- app/stac_api/migrations/0001_initial.py | 8 +- .../migrations/0005_auto_20210408_0821.py | 7 +- .../migrations/0014_auto_20210715_1358.py | 8 +- .../migrations/0032_alter_asset_file.py | 6 +- ...ncepage_landingpage_conformsto_and_more.py | 4 +- .../0036_collectionasset_and_more.py | 8 +- .../0050_collectionassetupload_and_more.py | 7 +- app/stac_api/models/__init__.py | 0 app/stac_api/{models.py => models/general.py} | 0 app/stac_api/s3_multipart_upload.py | 4 +- app/stac_api/sample_data/importer.py | 10 +- app/stac_api/serializers/collection.py | 8 +- app/stac_api/serializers/general.py | 4 +- app/stac_api/serializers/item.py | 6 +- app/stac_api/serializers/upload.py | 4 +- app/stac_api/serializers/utils.py | 6 +- app/stac_api/signals.py | 8 +- app/stac_api/validators_view.py | 8 +- app/stac_api/views/collection.py | 4 +- app/stac_api/views/general.py | 4 +- app/stac_api/views/item.py | 6 +- app/stac_api/views/test.py | 2 +- app/stac_api/views/upload.py | 10 +- app/tests/base_test_admin_page.py | 12 +- app/tests/test_admin_page.py | 12 +- app/tests/tests_09/data_factory.py | 12 +- .../tests_09/sample_data/item_samples.py | 2 +- app/tests/tests_09/test_asset_model.py | 2 +- .../tests_09/test_asset_upload_endpoint.py | 4 +- app/tests/tests_09/test_asset_upload_model.py | 4 +- app/tests/tests_09/test_assets_endpoint.py | 2 +- app/tests/tests_09/test_collection_model.py | 2 +- .../tests_09/test_collections_endpoint.py | 6 +- app/tests/tests_09/test_collections_extent.py | 2 +- app/tests/tests_09/test_generic_api.py | 2 +- app/tests/tests_09/test_item_model.py | 4 +- app/tests/tests_09/test_items_endpoint.py | 2 +- .../tests_09/test_items_endpoint_bbox.py | 2 +- app/tests/tests_09/test_serializer.py | 2 +- .../tests_09/test_serializer_asset_upload.py | 2 +- app/tests/tests_10/data_factory.py | 14 +-- .../tests_10/sample_data/item_samples.py | 2 +- app/tests/tests_10/test_asset_model.py | 2 +- .../tests_10/test_asset_upload_endpoint.py | 4 +- app/tests/tests_10/test_asset_upload_model.py | 4 +- app/tests/tests_10/test_assets_endpoint.py | 2 +- .../tests_10/test_collection_asset_model.py | 2 +- .../test_collection_asset_upload_endpoint.py | 4 +- .../test_collection_asset_upload_model.py | 4 +- .../test_collection_assets_endpoint.py | 2 +- app/tests/tests_10/test_collection_model.py | 2 +- .../tests_10/test_collections_endpoint.py | 6 +- app/tests/tests_10/test_collections_extent.py | 2 +- .../tests_10/test_external_assets_endpoint.py | 2 +- app/tests/tests_10/test_generic_api.py | 2 +- app/tests/tests_10/test_item_model.py | 4 +- app/tests/tests_10/test_items_endpoint.py | 6 +- .../tests_10/test_items_endpoint_bbox.py | 2 +- .../tests_10/test_remove_expired_items.py | 4 +- app/tests/tests_10/test_serializer.py | 2 +- .../tests_10/test_serializer_asset_upload.py | 2 +- scripts/fill_local_db.py | 113 ++++++------------ 72 files changed, 204 insertions(+), 243 deletions(-) create mode 100644 app/stac_api/models/__init__.py rename app/stac_api/{models.py => models/general.py} (100%) diff --git a/app/stac_api/admin.py b/app/stac_api/admin.py index 7b775043..494af29a 100644 --- a/app/stac_api/admin.py +++ b/app/stac_api/admin.py @@ -17,17 +17,17 @@ from django.urls import reverse from django.utils.translation import gettext_lazy as _ -from stac_api.models import BBOX_CH -from stac_api.models import Asset -from stac_api.models import AssetUpload -from stac_api.models import Collection -from stac_api.models import CollectionAsset -from stac_api.models import CollectionLink -from stac_api.models import Item -from stac_api.models import ItemLink -from stac_api.models import LandingPage -from stac_api.models import LandingPageLink -from stac_api.models import Provider +from stac_api.models.general import BBOX_CH +from stac_api.models.general import Asset +from stac_api.models.general import AssetUpload +from stac_api.models.general import Collection +from stac_api.models.general import CollectionAsset +from stac_api.models.general import CollectionLink +from stac_api.models.general import Item +from stac_api.models.general import ItemLink +from stac_api.models.general import LandingPage +from stac_api.models.general import LandingPageLink +from stac_api.models.general import Provider from stac_api.utils import build_asset_href from stac_api.utils import get_query_params from stac_api.validators import validate_href_url diff --git a/app/stac_api/management/commands/calculate_extent.py b/app/stac_api/management/commands/calculate_extent.py index f01104e0..4118d8e9 100644 --- a/app/stac_api/management/commands/calculate_extent.py +++ b/app/stac_api/management/commands/calculate_extent.py @@ -3,7 +3,7 @@ from django.core.management.base import CommandParser from django.db import connection -from stac_api.models import Collection +from stac_api.models.general import Collection from stac_api.utils import CommandHandler from stac_api.utils import CustomBaseCommand diff --git a/app/stac_api/management/commands/dummy_asset_upload.py b/app/stac_api/management/commands/dummy_asset_upload.py index 758f98b8..e79964b6 100644 --- a/app/stac_api/management/commands/dummy_asset_upload.py +++ b/app/stac_api/management/commands/dummy_asset_upload.py @@ -4,9 +4,9 @@ from django.conf import settings from django.core.management.base import BaseCommand -from stac_api.models import Asset -from stac_api.models import AssetUpload -from stac_api.models import BaseAssetUpload +from stac_api.models.general import Asset +from stac_api.models.general import AssetUpload +from stac_api.models.general import BaseAssetUpload from stac_api.s3_multipart_upload import MultipartUpload from stac_api.utils import AVAILABLE_S3_BUCKETS from stac_api.utils import CommandHandler diff --git a/app/stac_api/management/commands/dummy_data.py b/app/stac_api/management/commands/dummy_data.py index bb9d39dc..ac13f159 100644 --- a/app/stac_api/management/commands/dummy_data.py +++ b/app/stac_api/management/commands/dummy_data.py @@ -13,9 +13,9 @@ from django.core.files.uploadedfile import SimpleUploadedFile from django.core.management.base import BaseCommand -from stac_api.models import Asset -from stac_api.models import Collection -from stac_api.models import Item +from stac_api.models.general import Asset +from stac_api.models.general import Collection +from stac_api.models.general import Item from stac_api.utils import CommandHandler from stac_api.validators import MEDIA_TYPES diff --git a/app/stac_api/management/commands/list_asset_uploads.py b/app/stac_api/management/commands/list_asset_uploads.py index 079ac3ec..c0a0f5db 100644 --- a/app/stac_api/management/commands/list_asset_uploads.py +++ b/app/stac_api/management/commands/list_asset_uploads.py @@ -4,7 +4,7 @@ from django.core.management.base import BaseCommand from django.core.serializers.json import DjangoJSONEncoder -from stac_api.models import AssetUpload +from stac_api.models.general import AssetUpload from stac_api.s3_multipart_upload import MultipartUpload from stac_api.serializers.upload import AssetUploadSerializer from stac_api.utils import CommandHandler diff --git a/app/stac_api/management/commands/profile_cursor_paginator.py b/app/stac_api/management/commands/profile_cursor_paginator.py index c4c7cc97..311ecff3 100644 --- a/app/stac_api/management/commands/profile_cursor_paginator.py +++ b/app/stac_api/management/commands/profile_cursor_paginator.py @@ -10,7 +10,7 @@ from rest_framework.request import Request from rest_framework.test import APIRequestFactory -from stac_api.models import Item +from stac_api.models.general import Item from stac_api.utils import CommandHandler logger = logging.getLogger(__name__) diff --git a/app/stac_api/management/commands/profile_item_serializer.py b/app/stac_api/management/commands/profile_item_serializer.py index 47f2ba79..9b43822f 100644 --- a/app/stac_api/management/commands/profile_item_serializer.py +++ b/app/stac_api/management/commands/profile_item_serializer.py @@ -8,7 +8,7 @@ from rest_framework.test import APIRequestFactory -from stac_api.models import Item +from stac_api.models.general import Item from stac_api.utils import CommandHandler logger = logging.getLogger(__name__) diff --git a/app/stac_api/management/commands/profile_serializer_vs_no_drf.py b/app/stac_api/management/commands/profile_serializer_vs_no_drf.py index d4b1ddca..3f2ada21 100644 --- a/app/stac_api/management/commands/profile_serializer_vs_no_drf.py +++ b/app/stac_api/management/commands/profile_serializer_vs_no_drf.py @@ -7,7 +7,7 @@ from rest_framework.test import APIRequestFactory -from stac_api.models import Item +from stac_api.models.general import Item from stac_api.utils import CommandHandler logger = logging.getLogger(__name__) diff --git a/app/stac_api/management/commands/remove_expired_items.py b/app/stac_api/management/commands/remove_expired_items.py index d1eb2668..a1f73a22 100644 --- a/app/stac_api/management/commands/remove_expired_items.py +++ b/app/stac_api/management/commands/remove_expired_items.py @@ -4,9 +4,9 @@ from django.core.management.base import CommandParser from django.utils import timezone -from stac_api.models import AssetUpload -from stac_api.models import BaseAssetUpload -from stac_api.models import Item +from stac_api.models.general import AssetUpload +from stac_api.models.general import BaseAssetUpload +from stac_api.models.general import Item from stac_api.utils import CommandHandler from stac_api.utils import CustomBaseCommand diff --git a/app/stac_api/management/commands/update_asset_file_size.py b/app/stac_api/management/commands/update_asset_file_size.py index a4591c99..7b6742bc 100644 --- a/app/stac_api/management/commands/update_asset_file_size.py +++ b/app/stac_api/management/commands/update_asset_file_size.py @@ -6,8 +6,8 @@ from django.core.management.base import BaseCommand from django.core.management.base import CommandParser -from stac_api.models import Asset -from stac_api.models import CollectionAsset +from stac_api.models.general import Asset +from stac_api.models.general import CollectionAsset from stac_api.utils import CommandHandler from stac_api.utils import get_s3_client from stac_api.utils import select_s3_bucket diff --git a/app/stac_api/migrations/0001_initial.py b/app/stac_api/migrations/0001_initial.py index 529ed67d..9857f8f0 100644 --- a/app/stac_api/migrations/0001_initial.py +++ b/app/stac_api/migrations/0001_initial.py @@ -7,7 +7,7 @@ from django.db import migrations from django.db import models -import stac_api.models +import stac_api.models.general import stac_api.validators @@ -74,7 +74,7 @@ class Migration(migrations.Migration): ( 'summaries', models.JSONField( - default=stac_api.models.get_default_summaries_value, + default=stac_api.models.general.get_default_summaries_value, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder ) @@ -102,7 +102,7 @@ class Migration(migrations.Migration): 'conformsTo', django.contrib.postgres.fields.ArrayField( base_field=models.URLField(), - default=stac_api.models.get_conformance_default_links, + default=stac_api.models.general.get_conformance_default_links, help_text='Comma-separated list of URLs for the value conformsTo', size=None ) @@ -335,7 +335,7 @@ class Migration(migrations.Migration): ( 'file', models.FileField( - max_length=255, upload_to=stac_api.models.upload_asset_to_path_hook + max_length=255, upload_to=stac_api.models.general.upload_asset_to_path_hook ) ), ( diff --git a/app/stac_api/migrations/0005_auto_20210408_0821.py b/app/stac_api/migrations/0005_auto_20210408_0821.py index 7ef461a7..a7079354 100644 --- a/app/stac_api/migrations/0005_auto_20210408_0821.py +++ b/app/stac_api/migrations/0005_auto_20210408_0821.py @@ -6,7 +6,7 @@ from django.db import migrations from django.db import models -import stac_api.models +import stac_api.models.general class Migration(migrations.Migration): @@ -50,7 +50,10 @@ class Migration(migrations.Migration): ('created', models.DateTimeField(auto_now_add=True)), ('ended', models.DateTimeField(blank=True, default=None, null=True)), ('checksum_multihash', models.CharField(max_length=255)), - ('etag', models.CharField(default=stac_api.models.compute_etag, max_length=56)), + ( + 'etag', + models.CharField(default=stac_api.models.general.compute_etag, max_length=56) + ), ( 'asset', models.ForeignKey( diff --git a/app/stac_api/migrations/0014_auto_20210715_1358.py b/app/stac_api/migrations/0014_auto_20210715_1358.py index 032244c7..11493eee 100644 --- a/app/stac_api/migrations/0014_auto_20210715_1358.py +++ b/app/stac_api/migrations/0014_auto_20210715_1358.py @@ -6,7 +6,7 @@ from django.db import migrations from django.db import models -import stac_api.models +import stac_api.models.general class Migration(migrations.Migration): @@ -49,7 +49,7 @@ class Migration(migrations.Migration): model_name='asset', name='etag', field=models.CharField( - default=stac_api.models.compute_etag, editable=False, max_length=56 + default=stac_api.models.general.compute_etag, editable=False, max_length=56 ), ), migrations.AlterField( @@ -61,7 +61,7 @@ class Migration(migrations.Migration): model_name='collection', name='etag', field=models.CharField( - default=stac_api.models.compute_etag, editable=False, max_length=56 + default=stac_api.models.general.compute_etag, editable=False, max_length=56 ), ), migrations.AlterField( @@ -89,7 +89,7 @@ class Migration(migrations.Migration): model_name='item', name='etag', field=models.CharField( - default=stac_api.models.compute_etag, editable=False, max_length=56 + default=stac_api.models.general.compute_etag, editable=False, max_length=56 ), ), migrations.AlterField( diff --git a/app/stac_api/migrations/0032_alter_asset_file.py b/app/stac_api/migrations/0032_alter_asset_file.py index ed8a336d..76164f48 100644 --- a/app/stac_api/migrations/0032_alter_asset_file.py +++ b/app/stac_api/migrations/0032_alter_asset_file.py @@ -2,7 +2,7 @@ from django.db import migrations -import stac_api.models +import stac_api.models.general class Migration(migrations.Migration): @@ -15,8 +15,8 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='asset', name='file', - field=stac_api.models.DynamicStorageFileField( - max_length=255, upload_to=stac_api.models.upload_asset_to_path_hook + field=stac_api.models.general.DynamicStorageFileField( + max_length=255, upload_to=stac_api.models.general.upload_asset_to_path_hook ), ), ] diff --git a/app/stac_api/migrations/0032_delete_conformancepage_landingpage_conformsto_and_more.py b/app/stac_api/migrations/0032_delete_conformancepage_landingpage_conformsto_and_more.py index ff457375..9a19fccc 100644 --- a/app/stac_api/migrations/0032_delete_conformancepage_landingpage_conformsto_and_more.py +++ b/app/stac_api/migrations/0032_delete_conformancepage_landingpage_conformsto_and_more.py @@ -4,7 +4,7 @@ from django.db import migrations from django.db import models -import stac_api.models +import stac_api.models.general import stac_api.validators @@ -21,7 +21,7 @@ class Migration(migrations.Migration): name='conformsTo', field=django.contrib.postgres.fields.ArrayField( base_field=models.URLField(), - default=stac_api.models.get_conformance_default_links, + default=stac_api.models.general.get_conformance_default_links, help_text='Comma-separated list of URLs for the value conformsTo', size=None ), diff --git a/app/stac_api/migrations/0036_collectionasset_and_more.py b/app/stac_api/migrations/0036_collectionasset_and_more.py index efddc860..d69e0a16 100644 --- a/app/stac_api/migrations/0036_collectionasset_and_more.py +++ b/app/stac_api/migrations/0036_collectionasset_and_more.py @@ -9,7 +9,7 @@ from django.db import migrations from django.db import models -import stac_api.models +import stac_api.models.general import stac_api.validators @@ -34,8 +34,8 @@ class Migration(migrations.Migration): ), ( 'file', - stac_api.models.DynamicStorageFileField( - max_length=255, upload_to=stac_api.models.upload_asset_to_path_hook + stac_api.models.general.DynamicStorageFileField( + max_length=255, upload_to=stac_api.models.general.upload_asset_to_path_hook ) ), ( @@ -192,7 +192,7 @@ class Migration(migrations.Migration): ( 'etag', models.CharField( - default=stac_api.models.compute_etag, editable=False, max_length=56 + default=stac_api.models.general.compute_etag, editable=False, max_length=56 ) ), ( diff --git a/app/stac_api/migrations/0050_collectionassetupload_and_more.py b/app/stac_api/migrations/0050_collectionassetupload_and_more.py index 1238a62a..0e01547b 100644 --- a/app/stac_api/migrations/0050_collectionassetupload_and_more.py +++ b/app/stac_api/migrations/0050_collectionassetupload_and_more.py @@ -9,7 +9,7 @@ from django.db import migrations from django.db import models -import stac_api.models +import stac_api.models.general class Migration(migrations.Migration): @@ -69,7 +69,10 @@ class Migration(migrations.Migration): ('created', models.DateTimeField(auto_now_add=True)), ('ended', models.DateTimeField(blank=True, default=None, null=True)), ('checksum_multihash', models.CharField(max_length=255)), - ('etag', models.CharField(default=stac_api.models.compute_etag, max_length=56)), + ( + 'etag', + models.CharField(default=stac_api.models.general.compute_etag, max_length=56) + ), ( 'update_interval', models.IntegerField( diff --git a/app/stac_api/models/__init__.py b/app/stac_api/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/stac_api/models.py b/app/stac_api/models/general.py similarity index 100% rename from app/stac_api/models.py rename to app/stac_api/models/general.py diff --git a/app/stac_api/s3_multipart_upload.py b/app/stac_api/s3_multipart_upload.py index a8983127..14dae036 100644 --- a/app/stac_api/s3_multipart_upload.py +++ b/app/stac_api/s3_multipart_upload.py @@ -12,8 +12,8 @@ from rest_framework import serializers from stac_api.exceptions import UploadNotInProgressError -from stac_api.models import Asset -from stac_api.models import CollectionAsset +from stac_api.models.general import Asset +from stac_api.models.general import CollectionAsset from stac_api.utils import AVAILABLE_S3_BUCKETS from stac_api.utils import get_s3_cache_control_value from stac_api.utils import get_s3_client diff --git a/app/stac_api/sample_data/importer.py b/app/stac_api/sample_data/importer.py index 3194a99b..c308f90e 100644 --- a/app/stac_api/sample_data/importer.py +++ b/app/stac_api/sample_data/importer.py @@ -9,11 +9,11 @@ from django.contrib.gis.geos import GEOSGeometry from django.core.files.uploadedfile import SimpleUploadedFile -from stac_api.models import Asset -from stac_api.models import Collection -from stac_api.models import CollectionLink -from stac_api.models import Item -from stac_api.models import Provider +from stac_api.models.general import Asset +from stac_api.models.general import Collection +from stac_api.models.general import CollectionLink +from stac_api.models.general import Item +from stac_api.models.general import Provider # path definition relative to the directory that contains manage.py DATADIR = settings.BASE_DIR / 'app/stac_api/management/sample_data/' diff --git a/app/stac_api/serializers/collection.py b/app/stac_api/serializers/collection.py index d66547f0..dc28dc10 100644 --- a/app/stac_api/serializers/collection.py +++ b/app/stac_api/serializers/collection.py @@ -4,10 +4,10 @@ from rest_framework import serializers -from stac_api.models import Collection -from stac_api.models import CollectionAsset -from stac_api.models import CollectionLink -from stac_api.models import Provider +from stac_api.models.general import Collection +from stac_api.models.general import CollectionAsset +from stac_api.models.general import CollectionLink +from stac_api.models.general import Provider from stac_api.serializers.utils import AssetsDictSerializer from stac_api.serializers.utils import HrefField from stac_api.serializers.utils import NonNullModelSerializer diff --git a/app/stac_api/serializers/general.py b/app/stac_api/serializers/general.py index 4d1aba7e..876e3342 100644 --- a/app/stac_api/serializers/general.py +++ b/app/stac_api/serializers/general.py @@ -8,8 +8,8 @@ from rest_framework import serializers from rest_framework.validators import UniqueValidator -from stac_api.models import LandingPage -from stac_api.models import LandingPageLink +from stac_api.models.general import LandingPage +from stac_api.models.general import LandingPageLink from stac_api.utils import get_browser_url from stac_api.utils import get_stac_version from stac_api.utils import get_url diff --git a/app/stac_api/serializers/item.py b/app/stac_api/serializers/item.py index f79bad37..93655089 100644 --- a/app/stac_api/serializers/item.py +++ b/app/stac_api/serializers/item.py @@ -8,9 +8,9 @@ from rest_framework import serializers from rest_framework_gis import serializers as gis_serializers -from stac_api.models import Asset -from stac_api.models import Item -from stac_api.models import ItemLink +from stac_api.models.general import Asset +from stac_api.models.general import Item +from stac_api.models.general import ItemLink from stac_api.serializers.utils import AssetsDictSerializer from stac_api.serializers.utils import HrefField from stac_api.serializers.utils import IsoDurationField diff --git a/app/stac_api/serializers/upload.py b/app/stac_api/serializers/upload.py index 9a42887c..691f942c 100644 --- a/app/stac_api/serializers/upload.py +++ b/app/stac_api/serializers/upload.py @@ -5,8 +5,8 @@ from rest_framework import serializers from rest_framework.utils.serializer_helpers import ReturnDict -from stac_api.models import AssetUpload -from stac_api.models import CollectionAssetUpload +from stac_api.models.general import AssetUpload +from stac_api.models.general import CollectionAssetUpload from stac_api.serializers.utils import NonNullModelSerializer from stac_api.utils import is_api_version_1 from stac_api.utils import isoformat diff --git a/app/stac_api/serializers/utils.py b/app/stac_api/serializers/utils.py index 9ac92085..8b54dc9b 100644 --- a/app/stac_api/serializers/utils.py +++ b/app/stac_api/serializers/utils.py @@ -10,9 +10,9 @@ from rest_framework import serializers from rest_framework.utils.serializer_helpers import ReturnDict -from stac_api.models import Collection -from stac_api.models import Item -from stac_api.models import Link +from stac_api.models.general import Collection +from stac_api.models.general import Item +from stac_api.models.general import Link from stac_api.utils import build_asset_href from stac_api.utils import get_browser_url from stac_api.utils import get_url diff --git a/app/stac_api/signals.py b/app/stac_api/signals.py index 0410b70f..1e95fa19 100644 --- a/app/stac_api/signals.py +++ b/app/stac_api/signals.py @@ -4,10 +4,10 @@ from django.db.models.signals import pre_delete from django.dispatch import receiver -from stac_api.models import Asset -from stac_api.models import AssetUpload -from stac_api.models import CollectionAsset -from stac_api.models import CollectionAssetUpload +from stac_api.models.general import Asset +from stac_api.models.general import AssetUpload +from stac_api.models.general import CollectionAsset +from stac_api.models.general import CollectionAssetUpload logger = logging.getLogger(__name__) diff --git a/app/stac_api/validators_view.py b/app/stac_api/validators_view.py index c8bd571d..ce720508 100644 --- a/app/stac_api/validators_view.py +++ b/app/stac_api/validators_view.py @@ -7,10 +7,10 @@ from rest_framework import serializers -from stac_api.models import Asset -from stac_api.models import Collection -from stac_api.models import CollectionAsset -from stac_api.models import Item +from stac_api.models.general import Asset +from stac_api.models.general import Collection +from stac_api.models.general import CollectionAsset +from stac_api.models.general import Item logger = logging.getLogger(__name__) diff --git a/app/stac_api/views/collection.py b/app/stac_api/views/collection.py index 708e0a6c..3955de1a 100644 --- a/app/stac_api/views/collection.py +++ b/app/stac_api/views/collection.py @@ -8,8 +8,8 @@ from rest_framework.response import Response from rest_framework_condition import etag -from stac_api.models import Collection -from stac_api.models import CollectionAsset +from stac_api.models.general import Collection +from stac_api.models.general import CollectionAsset from stac_api.serializers.collection import CollectionAssetSerializer from stac_api.serializers.collection import CollectionSerializer from stac_api.serializers.utils import get_relation_links diff --git a/app/stac_api/views/general.py b/app/stac_api/views/general.py index 662ed813..c0192961 100644 --- a/app/stac_api/views/general.py +++ b/app/stac_api/views/general.py @@ -14,8 +14,8 @@ from rest_framework.permissions import AllowAny from rest_framework.response import Response -from stac_api.models import Item -from stac_api.models import LandingPage +from stac_api.models.general import Item +from stac_api.models.general import LandingPage from stac_api.pagination import GetPostCursorPagination from stac_api.serializers.general import ConformancePageSerializer from stac_api.serializers.general import LandingPageSerializer diff --git a/app/stac_api/views/item.py b/app/stac_api/views/item.py index 8f024d6d..4e1de5ff 100644 --- a/app/stac_api/views/item.py +++ b/app/stac_api/views/item.py @@ -12,9 +12,9 @@ from rest_framework.response import Response from rest_framework_condition import etag -from stac_api.models import Asset -from stac_api.models import Collection -from stac_api.models import Item +from stac_api.models.general import Asset +from stac_api.models.general import Collection +from stac_api.models.general import Item from stac_api.serializers.item import AssetSerializer from stac_api.serializers.item import ItemSerializer from stac_api.serializers.utils import get_relation_links diff --git a/app/stac_api/views/test.py b/app/stac_api/views/test.py index 85b7d1ab..76c5ba4f 100644 --- a/app/stac_api/views/test.py +++ b/app/stac_api/views/test.py @@ -2,7 +2,7 @@ from rest_framework import generics -from stac_api.models import LandingPage +from stac_api.models.general import LandingPage from stac_api.views.collection import CollectionAssetDetail from stac_api.views.collection import CollectionDetail from stac_api.views.item import AssetDetail diff --git a/app/stac_api/views/upload.py b/app/stac_api/views/upload.py index ed84f4a7..b8eb1197 100644 --- a/app/stac_api/views/upload.py +++ b/app/stac_api/views/upload.py @@ -16,11 +16,11 @@ from stac_api.exceptions import UploadInProgressError from stac_api.exceptions import UploadNotInProgressError -from stac_api.models import Asset -from stac_api.models import AssetUpload -from stac_api.models import BaseAssetUpload -from stac_api.models import CollectionAsset -from stac_api.models import CollectionAssetUpload +from stac_api.models.general import Asset +from stac_api.models.general import AssetUpload +from stac_api.models.general import BaseAssetUpload +from stac_api.models.general import CollectionAsset +from stac_api.models.general import CollectionAssetUpload from stac_api.pagination import ExtApiPagination from stac_api.s3_multipart_upload import MultipartUpload from stac_api.serializers.upload import AssetUploadPartsSerializer diff --git a/app/tests/base_test_admin_page.py b/app/tests/base_test_admin_page.py index 3d83ae8e..07e65351 100644 --- a/app/tests/base_test_admin_page.py +++ b/app/tests/base_test_admin_page.py @@ -7,12 +7,12 @@ from django.test import TestCase from django.urls import reverse -from stac_api.models import Asset -from stac_api.models import Collection -from stac_api.models import CollectionLink -from stac_api.models import Item -from stac_api.models import ItemLink -from stac_api.models import Provider +from stac_api.models.general import Asset +from stac_api.models.general import Collection +from stac_api.models.general import CollectionLink +from stac_api.models.general import Item +from stac_api.models.general import ItemLink +from stac_api.models.general import Provider from tests.tests_09.data_factory import Factory diff --git a/app/tests/test_admin_page.py b/app/tests/test_admin_page.py index 4d339ce3..e2570e6b 100644 --- a/app/tests/test_admin_page.py +++ b/app/tests/test_admin_page.py @@ -5,12 +5,12 @@ from django.test import override_settings from django.urls import reverse -from stac_api.models import Asset -from stac_api.models import Collection -from stac_api.models import CollectionLink -from stac_api.models import Item -from stac_api.models import ItemLink -from stac_api.models import Provider +from stac_api.models.general import Asset +from stac_api.models.general import Collection +from stac_api.models.general import CollectionLink +from stac_api.models.general import Item +from stac_api.models.general import ItemLink +from stac_api.models.general import Provider from stac_api.utils import parse_multihash from tests.base_test_admin_page import AdminBaseTestCase diff --git a/app/tests/tests_09/data_factory.py b/app/tests/tests_09/data_factory.py index c9474ab3..57f949a2 100644 --- a/app/tests/tests_09/data_factory.py +++ b/app/tests/tests_09/data_factory.py @@ -97,12 +97,12 @@ from django.core.files.base import File from django.core.files.uploadedfile import SimpleUploadedFile -from stac_api.models import Asset -from stac_api.models import Collection -from stac_api.models import CollectionLink -from stac_api.models import Item -from stac_api.models import ItemLink -from stac_api.models import Provider +from stac_api.models.general import Asset +from stac_api.models.general import Collection +from stac_api.models.general import CollectionLink +from stac_api.models.general import Item +from stac_api.models.general import ItemLink +from stac_api.models.general import Provider from stac_api.utils import get_s3_resource from stac_api.utils import isoformat from stac_api.validators import get_media_type diff --git a/app/tests/tests_09/sample_data/item_samples.py b/app/tests/tests_09/sample_data/item_samples.py index 55babe4b..3395925a 100644 --- a/app/tests/tests_09/sample_data/item_samples.py +++ b/app/tests/tests_09/sample_data/item_samples.py @@ -2,7 +2,7 @@ from django.contrib.gis.geos import GEOSGeometry -from stac_api.models import BBOX_CH +from stac_api.models.general import BBOX_CH from stac_api.utils import fromisoformat geometries = { diff --git a/app/tests/tests_09/test_asset_model.py b/app/tests/tests_09/test_asset_model.py index 17aa9f42..4c52bf46 100644 --- a/app/tests/tests_09/test_asset_model.py +++ b/app/tests/tests_09/test_asset_model.py @@ -2,7 +2,7 @@ from django.core.exceptions import ValidationError -from stac_api.models import Asset +from stac_api.models.general import Asset from tests.tests_09.base_test import StacBaseTransactionTestCase from tests.tests_09.data_factory import Factory diff --git a/app/tests/tests_09/test_asset_upload_endpoint.py b/app/tests/tests_09/test_asset_upload_endpoint.py index 8775a20d..bc50210a 100644 --- a/app/tests/tests_09/test_asset_upload_endpoint.py +++ b/app/tests/tests_09/test_asset_upload_endpoint.py @@ -10,8 +10,8 @@ from django.contrib.auth import get_user_model from django.test import Client -from stac_api.models import Asset -from stac_api.models import AssetUpload +from stac_api.models.general import Asset +from stac_api.models.general import AssetUpload from stac_api.utils import fromisoformat from stac_api.utils import get_asset_path from stac_api.utils import get_s3_client diff --git a/app/tests/tests_09/test_asset_upload_model.py b/app/tests/tests_09/test_asset_upload_model.py index f48f42e9..1c84e291 100644 --- a/app/tests/tests_09/test_asset_upload_model.py +++ b/app/tests/tests_09/test_asset_upload_model.py @@ -6,8 +6,8 @@ from django.test import TestCase from django.test import TransactionTestCase -from stac_api.models import Asset -from stac_api.models import AssetUpload +from stac_api.models.general import Asset +from stac_api.models.general import AssetUpload from stac_api.utils import get_sha256_multihash from stac_api.utils import utc_aware diff --git a/app/tests/tests_09/test_assets_endpoint.py b/app/tests/tests_09/test_assets_endpoint.py index c4dc33a1..980dcebe 100644 --- a/app/tests/tests_09/test_assets_endpoint.py +++ b/app/tests/tests_09/test_assets_endpoint.py @@ -8,7 +8,7 @@ from django.test import Client from django.urls import reverse -from stac_api.models import Asset +from stac_api.models.general import Asset from stac_api.utils import get_asset_path from stac_api.utils import utc_aware diff --git a/app/tests/tests_09/test_collection_model.py b/app/tests/tests_09/test_collection_model.py index cfc83d05..1142c0b2 100644 --- a/app/tests/tests_09/test_collection_model.py +++ b/app/tests/tests_09/test_collection_model.py @@ -3,7 +3,7 @@ from django.core.exceptions import ValidationError -from stac_api.models import Collection +from stac_api.models.general import Collection from tests.tests_09.base_test import StacBaseTransactionTestCase from tests.tests_09.data_factory import Factory diff --git a/app/tests/tests_09/test_collections_endpoint.py b/app/tests/tests_09/test_collections_endpoint.py index 99e21dbf..8a40150a 100644 --- a/app/tests/tests_09/test_collections_endpoint.py +++ b/app/tests/tests_09/test_collections_endpoint.py @@ -5,9 +5,9 @@ from django.test import Client from django.urls import reverse -from stac_api.models import Collection -from stac_api.models import CollectionLink -from stac_api.models import Provider +from stac_api.models.general import Collection +from stac_api.models.general import CollectionLink +from stac_api.models.general import Provider from stac_api.utils import utc_aware from tests.tests_09.base_test import STAC_BASE_V diff --git a/app/tests/tests_09/test_collections_extent.py b/app/tests/tests_09/test_collections_extent.py index 943c24b8..d25ff329 100644 --- a/app/tests/tests_09/test_collections_extent.py +++ b/app/tests/tests_09/test_collections_extent.py @@ -4,7 +4,7 @@ from django.contrib.gis.geos import GEOSGeometry from django.contrib.gis.geos import Polygon -from stac_api.models import Item +from stac_api.models.general import Item from stac_api.utils import utc_aware from tests.tests_09.base_test import StacBaseTransactionTestCase diff --git a/app/tests/tests_09/test_generic_api.py b/app/tests/tests_09/test_generic_api.py index 88544420..c8252897 100644 --- a/app/tests/tests_09/test_generic_api.py +++ b/app/tests/tests_09/test_generic_api.py @@ -5,7 +5,7 @@ from django.test import Client from django.test import override_settings -from stac_api.models import AssetUpload +from stac_api.models.general import AssetUpload from stac_api.utils import get_asset_path from stac_api.utils import get_link from stac_api.utils import get_sha256_multihash diff --git a/app/tests/tests_09/test_item_model.py b/app/tests/tests_09/test_item_model.py index b7c8cfd5..3f84a128 100644 --- a/app/tests/tests_09/test_item_model.py +++ b/app/tests/tests_09/test_item_model.py @@ -6,8 +6,8 @@ from django.core.exceptions import ValidationError from django.test import TestCase -from stac_api.models import Collection -from stac_api.models import Item +from stac_api.models.general import Collection +from stac_api.models.general import Item from stac_api.utils import utc_aware from tests.tests_09.data_factory import CollectionFactory diff --git a/app/tests/tests_09/test_items_endpoint.py b/app/tests/tests_09/test_items_endpoint.py index 3f2aaeeb..37640c0c 100644 --- a/app/tests/tests_09/test_items_endpoint.py +++ b/app/tests/tests_09/test_items_endpoint.py @@ -6,7 +6,7 @@ from django.test import Client from django.urls import reverse -from stac_api.models import Item +from stac_api.models.general import Item from stac_api.utils import fromisoformat from stac_api.utils import get_link from stac_api.utils import isoformat diff --git a/app/tests/tests_09/test_items_endpoint_bbox.py b/app/tests/tests_09/test_items_endpoint_bbox.py index 678d89b3..848d9a67 100644 --- a/app/tests/tests_09/test_items_endpoint_bbox.py +++ b/app/tests/tests_09/test_items_endpoint_bbox.py @@ -3,7 +3,7 @@ from django.contrib.gis.geos.geometry import GEOSGeometry from django.test import Client -from stac_api.models import BBOX_CH +from stac_api.models.general import BBOX_CH from tests.tests_09.base_test import STAC_BASE_V from tests.tests_09.base_test import StacBaseTestCase diff --git a/app/tests/tests_09/test_serializer.py b/app/tests/tests_09/test_serializer.py index 66374304..c6386104 100644 --- a/app/tests/tests_09/test_serializer.py +++ b/app/tests/tests_09/test_serializer.py @@ -12,7 +12,7 @@ from rest_framework.renderers import JSONRenderer from rest_framework.test import APIRequestFactory -from stac_api.models import get_asset_path +from stac_api.models.general import get_asset_path from stac_api.serializers.collection import CollectionSerializer from stac_api.serializers.item import AssetSerializer from stac_api.serializers.item import ItemSerializer diff --git a/app/tests/tests_09/test_serializer_asset_upload.py b/app/tests/tests_09/test_serializer_asset_upload.py index e5a54e43..4dcb25a2 100644 --- a/app/tests/tests_09/test_serializer_asset_upload.py +++ b/app/tests/tests_09/test_serializer_asset_upload.py @@ -6,7 +6,7 @@ from rest_framework import serializers from rest_framework.exceptions import ValidationError -from stac_api.models import AssetUpload +from stac_api.models.general import AssetUpload from stac_api.serializers.upload import AssetUploadSerializer from stac_api.utils import get_sha256_multihash diff --git a/app/tests/tests_10/data_factory.py b/app/tests/tests_10/data_factory.py index 4a522de1..38cbd4bf 100644 --- a/app/tests/tests_10/data_factory.py +++ b/app/tests/tests_10/data_factory.py @@ -97,13 +97,13 @@ from django.core.files.base import File from django.core.files.uploadedfile import SimpleUploadedFile -from stac_api.models import Asset -from stac_api.models import Collection -from stac_api.models import CollectionAsset -from stac_api.models import CollectionLink -from stac_api.models import Item -from stac_api.models import ItemLink -from stac_api.models import Provider +from stac_api.models.general import Asset +from stac_api.models.general import Collection +from stac_api.models.general import CollectionAsset +from stac_api.models.general import CollectionLink +from stac_api.models.general import Item +from stac_api.models.general import ItemLink +from stac_api.models.general import Provider from stac_api.utils import get_s3_resource from stac_api.utils import isoformat from stac_api.validators import get_media_type diff --git a/app/tests/tests_10/sample_data/item_samples.py b/app/tests/tests_10/sample_data/item_samples.py index 5b4dd12a..adfd76e0 100644 --- a/app/tests/tests_10/sample_data/item_samples.py +++ b/app/tests/tests_10/sample_data/item_samples.py @@ -2,7 +2,7 @@ from django.contrib.gis.geos import GEOSGeometry -from stac_api.models import BBOX_CH +from stac_api.models.general import BBOX_CH from stac_api.utils import fromisoformat geometries = { diff --git a/app/tests/tests_10/test_asset_model.py b/app/tests/tests_10/test_asset_model.py index 01f37578..f424dd66 100644 --- a/app/tests/tests_10/test_asset_model.py +++ b/app/tests/tests_10/test_asset_model.py @@ -2,7 +2,7 @@ from django.core.exceptions import ValidationError -from stac_api.models import Asset +from stac_api.models.general import Asset from tests.tests_10.base_test import StacBaseTransactionTestCase from tests.tests_10.data_factory import Factory diff --git a/app/tests/tests_10/test_asset_upload_endpoint.py b/app/tests/tests_10/test_asset_upload_endpoint.py index ef569c76..91465f5a 100644 --- a/app/tests/tests_10/test_asset_upload_endpoint.py +++ b/app/tests/tests_10/test_asset_upload_endpoint.py @@ -10,8 +10,8 @@ from django.contrib.auth import get_user_model from django.test import Client -from stac_api.models import Asset -from stac_api.models import AssetUpload +from stac_api.models.general import Asset +from stac_api.models.general import AssetUpload from stac_api.utils import fromisoformat from stac_api.utils import get_asset_path from stac_api.utils import get_s3_client diff --git a/app/tests/tests_10/test_asset_upload_model.py b/app/tests/tests_10/test_asset_upload_model.py index c8809762..21c4dffd 100644 --- a/app/tests/tests_10/test_asset_upload_model.py +++ b/app/tests/tests_10/test_asset_upload_model.py @@ -6,8 +6,8 @@ from django.test import TestCase from django.test import TransactionTestCase -from stac_api.models import Asset -from stac_api.models import AssetUpload +from stac_api.models.general import Asset +from stac_api.models.general import AssetUpload from stac_api.utils import get_sha256_multihash from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_assets_endpoint.py b/app/tests/tests_10/test_assets_endpoint.py index d2afbd36..987bb470 100644 --- a/app/tests/tests_10/test_assets_endpoint.py +++ b/app/tests/tests_10/test_assets_endpoint.py @@ -10,7 +10,7 @@ from django.urls import reverse from django.utils import timezone -from stac_api.models import Asset +from stac_api.models.general import Asset from stac_api.utils import get_asset_path from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_collection_asset_model.py b/app/tests/tests_10/test_collection_asset_model.py index 21cf9f9a..9cb8f172 100644 --- a/app/tests/tests_10/test_collection_asset_model.py +++ b/app/tests/tests_10/test_collection_asset_model.py @@ -2,7 +2,7 @@ from django.core.exceptions import ValidationError -from stac_api.models import CollectionAsset +from stac_api.models.general import CollectionAsset from tests.tests_10.base_test import StacBaseTransactionTestCase from tests.tests_10.data_factory import Factory diff --git a/app/tests/tests_10/test_collection_asset_upload_endpoint.py b/app/tests/tests_10/test_collection_asset_upload_endpoint.py index 9734ebff..95078dc7 100644 --- a/app/tests/tests_10/test_collection_asset_upload_endpoint.py +++ b/app/tests/tests_10/test_collection_asset_upload_endpoint.py @@ -10,8 +10,8 @@ from django.contrib.auth import get_user_model from django.test import Client -from stac_api.models import CollectionAsset -from stac_api.models import CollectionAssetUpload +from stac_api.models.general import CollectionAsset +from stac_api.models.general import CollectionAssetUpload from stac_api.utils import fromisoformat from stac_api.utils import get_collection_asset_path from stac_api.utils import get_s3_client diff --git a/app/tests/tests_10/test_collection_asset_upload_model.py b/app/tests/tests_10/test_collection_asset_upload_model.py index 9ea016aa..bb6e6c50 100644 --- a/app/tests/tests_10/test_collection_asset_upload_model.py +++ b/app/tests/tests_10/test_collection_asset_upload_model.py @@ -6,8 +6,8 @@ from django.test import TestCase from django.test import TransactionTestCase -from stac_api.models import CollectionAsset -from stac_api.models import CollectionAssetUpload +from stac_api.models.general import CollectionAsset +from stac_api.models.general import CollectionAssetUpload from stac_api.utils import get_sha256_multihash from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_collection_assets_endpoint.py b/app/tests/tests_10/test_collection_assets_endpoint.py index 1596a8d9..e782c243 100644 --- a/app/tests/tests_10/test_collection_assets_endpoint.py +++ b/app/tests/tests_10/test_collection_assets_endpoint.py @@ -8,7 +8,7 @@ from django.test import Client from django.urls import reverse -from stac_api.models import CollectionAsset +from stac_api.models.general import CollectionAsset from stac_api.utils import get_collection_asset_path from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_collection_model.py b/app/tests/tests_10/test_collection_model.py index 6e136b92..b6bb6ef4 100644 --- a/app/tests/tests_10/test_collection_model.py +++ b/app/tests/tests_10/test_collection_model.py @@ -3,7 +3,7 @@ from django.core.exceptions import ValidationError -from stac_api.models import Collection +from stac_api.models.general import Collection from tests.tests_10.base_test import StacBaseTransactionTestCase from tests.tests_10.data_factory import Factory diff --git a/app/tests/tests_10/test_collections_endpoint.py b/app/tests/tests_10/test_collections_endpoint.py index 42306568..5c2108e8 100644 --- a/app/tests/tests_10/test_collections_endpoint.py +++ b/app/tests/tests_10/test_collections_endpoint.py @@ -6,9 +6,9 @@ from django.test import Client from django.urls import reverse -from stac_api.models import Collection -from stac_api.models import CollectionLink -from stac_api.models import Provider +from stac_api.models.general import Collection +from stac_api.models.general import CollectionLink +from stac_api.models.general import Provider from stac_api.utils import utc_aware from tests.tests_10.base_test import STAC_BASE_V diff --git a/app/tests/tests_10/test_collections_extent.py b/app/tests/tests_10/test_collections_extent.py index be6957c3..fc308cef 100644 --- a/app/tests/tests_10/test_collections_extent.py +++ b/app/tests/tests_10/test_collections_extent.py @@ -4,7 +4,7 @@ from django.contrib.gis.geos import GEOSGeometry from django.contrib.gis.geos import Polygon -from stac_api.models import Item +from stac_api.models.general import Item from stac_api.utils import utc_aware from tests.tests_10.base_test import StacBaseTransactionTestCase diff --git a/app/tests/tests_10/test_external_assets_endpoint.py b/app/tests/tests_10/test_external_assets_endpoint.py index 4a8829b9..e8bf2fe4 100644 --- a/app/tests/tests_10/test_external_assets_endpoint.py +++ b/app/tests/tests_10/test_external_assets_endpoint.py @@ -3,7 +3,7 @@ from django.conf import settings from django.test import Client -from stac_api.models import Asset +from stac_api.models.general import Asset from tests.tests_10.base_test import StacBaseTestCase from tests.tests_10.data_factory import Factory diff --git a/app/tests/tests_10/test_generic_api.py b/app/tests/tests_10/test_generic_api.py index e5adc074..cb183187 100644 --- a/app/tests/tests_10/test_generic_api.py +++ b/app/tests/tests_10/test_generic_api.py @@ -5,7 +5,7 @@ from django.test import Client from django.test import override_settings -from stac_api.models import AssetUpload +from stac_api.models.general import AssetUpload from stac_api.utils import get_asset_path from stac_api.utils import get_link from stac_api.utils import get_sha256_multihash diff --git a/app/tests/tests_10/test_item_model.py b/app/tests/tests_10/test_item_model.py index c1704bba..a7391f69 100644 --- a/app/tests/tests_10/test_item_model.py +++ b/app/tests/tests_10/test_item_model.py @@ -7,8 +7,8 @@ from django.core.exceptions import ValidationError from django.test import TestCase -from stac_api.models import Collection -from stac_api.models import Item +from stac_api.models.general import Collection +from stac_api.models.general import Item from stac_api.utils import utc_aware from tests.tests_10.data_factory import CollectionFactory diff --git a/app/tests/tests_10/test_items_endpoint.py b/app/tests/tests_10/test_items_endpoint.py index 38397530..cc7ca6db 100644 --- a/app/tests/tests_10/test_items_endpoint.py +++ b/app/tests/tests_10/test_items_endpoint.py @@ -9,9 +9,9 @@ from django.urls import reverse from django.utils import timezone -from stac_api.models import Collection -from stac_api.models import Item -from stac_api.models import ItemLink +from stac_api.models.general import Collection +from stac_api.models.general import Item +from stac_api.models.general import ItemLink from stac_api.utils import fromisoformat from stac_api.utils import get_link from stac_api.utils import isoformat diff --git a/app/tests/tests_10/test_items_endpoint_bbox.py b/app/tests/tests_10/test_items_endpoint_bbox.py index b327ad0e..47a6e7d4 100644 --- a/app/tests/tests_10/test_items_endpoint_bbox.py +++ b/app/tests/tests_10/test_items_endpoint_bbox.py @@ -3,7 +3,7 @@ from django.contrib.gis.geos.geometry import GEOSGeometry from django.test import Client -from stac_api.models import BBOX_CH +from stac_api.models.general import BBOX_CH from tests.tests_10.base_test import STAC_BASE_V from tests.tests_10.base_test import StacBaseTestCase diff --git a/app/tests/tests_10/test_remove_expired_items.py b/app/tests/tests_10/test_remove_expired_items.py index 505dcd33..c7a80c95 100644 --- a/app/tests/tests_10/test_remove_expired_items.py +++ b/app/tests/tests_10/test_remove_expired_items.py @@ -5,8 +5,8 @@ from django.test import TestCase from django.utils import timezone -from stac_api.models import Asset -from stac_api.models import Item +from stac_api.models.general import Asset +from stac_api.models.general import Item from tests.tests_10.data_factory import Factory from tests.utils import mock_s3_asset_file diff --git a/app/tests/tests_10/test_serializer.py b/app/tests/tests_10/test_serializer.py index e757bcc9..68d9aa43 100644 --- a/app/tests/tests_10/test_serializer.py +++ b/app/tests/tests_10/test_serializer.py @@ -14,7 +14,7 @@ from rest_framework.renderers import JSONRenderer from rest_framework.test import APIRequestFactory -from stac_api.models import get_asset_path +from stac_api.models.general import get_asset_path from stac_api.serializers.collection import CollectionSerializer from stac_api.serializers.item import AssetSerializer from stac_api.serializers.item import ItemSerializer diff --git a/app/tests/tests_10/test_serializer_asset_upload.py b/app/tests/tests_10/test_serializer_asset_upload.py index c5d359a2..34e4ba29 100644 --- a/app/tests/tests_10/test_serializer_asset_upload.py +++ b/app/tests/tests_10/test_serializer_asset_upload.py @@ -6,7 +6,7 @@ from rest_framework import serializers from rest_framework.exceptions import ValidationError -from stac_api.models import AssetUpload +from stac_api.models.general import AssetUpload from stac_api.serializers.upload import AssetUploadSerializer from stac_api.utils import get_sha256_multihash diff --git a/scripts/fill_local_db.py b/scripts/fill_local_db.py index 6db19f8e..47690898 100644 --- a/scripts/fill_local_db.py +++ b/scripts/fill_local_db.py @@ -19,7 +19,7 @@ import io from rest_framework.renderers import JSONRenderer from rest_framework.parsers import JSONParser -from stac_api.models import * +from stac_api.models.general import * from stac_api.serializers.general import CollectionSerializer from stac_api.serializers.general import CollectionSerializer from stac_api.serializers.general import LinkSerializer @@ -28,14 +28,12 @@ # create link instances for testing link_root = CollectionLink.objects.create( - item=self.item, - href="https://data.geo.admin.ch/api/stac/v0.9/", - rel='root', - link_type='root', - title='Root link' - ) - - + item=self.item, + href="https://data.geo.admin.ch/api/stac/v0.9/", + rel='root', + link_type='root', + title='Root link' +) # create provider instances for testing provider1 = Provider( @@ -60,33 +58,19 @@ updated=datetime.now(), description='test', extent={ - "spatial": { - "bbox": [ - [ - 5.685114, - 45.534903, - 10.747775, - 47.982586 - ] - ] + "spatial": { + "bbox": [[5.685114, 45.534903, 10.747775, 47.982586]] + }, + "temporal": { + "interval": [["2019", None]] + } }, - "temporal": { - "interval": [ - [ - "2019", - None - ] - ] - } - }, collection_name='my collection', item_type='Feature', license='test', - summaries = { - "eo:gsd": [10,20], - "geoadmin:variant": ["kgrel", "komb", "krel"], - "proj:epsg": [2056] - }, + summaries={ + "eo:gsd": [10, 20], "geoadmin:variant": ["kgrel", "komb", "krel"], "proj:epsg": [2056] + }, title='testtitel2' ) @@ -104,33 +88,19 @@ updated=datetime.now(), description='test', extent={ - "spatial": { - "bbox": [ - [ - 5.685114, - 45.534903, - 10.747775, - 47.982586 - ] - ] + "spatial": { + "bbox": [[5.685114, 45.534903, 10.747775, 47.982586]] + }, + "temporal": { + "interval": [["2019", None]] + } }, - "temporal": { - "interval": [ - [ - "2019", - None - ] - ] - } - }, collection_name='b_123', item_type='Feature', license='test', - summaries = { - "eo:gsd": [10,20], - "geoadmin:variant": ["kgrel", "komb", "krel"], - "proj:epsg": [2056] - }, + summaries={ + "eo:gsd": [10, 20], "geoadmin:variant": ["kgrel", "komb", "krel"], "proj:epsg": [2056] + }, title='testtitel2' ) collection2.save() @@ -145,33 +115,19 @@ updated=datetime.now(), description='test', extent={ - "spatial": { - "bbox": [ - [ - 5.685114, - 45.534903, - 10.747775, - 47.982586 - ] - ] + "spatial": { + "bbox": [[5.685114, 45.534903, 10.747775, 47.982586]] + }, + "temporal": { + "interval": [["2019", None]] + } }, - "temporal": { - "interval": [ - [ - "2019", - None - ] - ] - } - }, collection_name='c_123', item_type='Feature', license='test', - summaries = { - "eo:gsd": [10,20], - "geoadmin:variant": ["kgrel", "komb", "krel"], - "proj:epsg": [2056] - }, + summaries={ + "eo:gsd": [10, 20], "geoadmin:variant": ["kgrel", "komb", "krel"], "proj:epsg": [2056] + }, title='testtitel2' ) collection3.save() @@ -185,7 +141,6 @@ provider3.save() link_root.save() - # test the serialization process: # translate into Python native serializer = CollectionSerializer(collection1) From e606b082dd8f5b718b64df5164dd21383d82b49b Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Mon, 27 Jan 2025 12:55:14 +0100 Subject: [PATCH 12/15] Refactor models - split into files Move collection related models into separate file. Move item related models into separate file. Fix import statements across all files. --- app/stac_api/admin.py | 14 +- .../management/commands/calculate_extent.py | 2 +- .../management/commands/dummy_asset_upload.py | 4 +- .../management/commands/dummy_data.py | 6 +- .../management/commands/list_asset_uploads.py | 2 +- .../commands/profile_cursor_paginator.py | 2 +- .../commands/profile_item_serializer.py | 2 +- .../commands/profile_serializer_vs_no_drf.py | 2 +- .../commands/remove_expired_items.py | 4 +- .../commands/update_asset_file_size.py | 4 +- app/stac_api/models/collection.py | 264 +++++++++ app/stac_api/models/general.py | 555 ------------------ app/stac_api/models/item.py | 327 +++++++++++ app/stac_api/s3_multipart_upload.py | 4 +- app/stac_api/sample_data/importer.py | 8 +- app/stac_api/serializers/collection.py | 6 +- app/stac_api/serializers/item.py | 6 +- app/stac_api/serializers/upload.py | 4 +- app/stac_api/serializers/utils.py | 4 +- app/stac_api/signals.py | 8 +- app/stac_api/validators_view.py | 8 +- app/stac_api/views/collection.py | 4 +- app/stac_api/views/general.py | 2 +- app/stac_api/views/item.py | 6 +- app/stac_api/views/upload.py | 8 +- app/tests/base_test_admin_page.py | 10 +- app/tests/test_admin_page.py | 10 +- app/tests/tests_09/data_factory.py | 10 +- app/tests/tests_09/test_asset_model.py | 2 +- .../tests_09/test_asset_upload_endpoint.py | 4 +- app/tests/tests_09/test_asset_upload_model.py | 4 +- app/tests/tests_09/test_assets_endpoint.py | 2 +- app/tests/tests_09/test_collection_model.py | 2 +- .../tests_09/test_collections_endpoint.py | 4 +- app/tests/tests_09/test_collections_extent.py | 2 +- app/tests/tests_09/test_generic_api.py | 2 +- app/tests/tests_09/test_item_model.py | 4 +- app/tests/tests_09/test_items_endpoint.py | 2 +- app/tests/tests_09/test_serializer.py | 2 +- .../tests_09/test_serializer_asset_upload.py | 2 +- app/tests/tests_10/data_factory.py | 12 +- app/tests/tests_10/test_asset_model.py | 2 +- .../tests_10/test_asset_upload_endpoint.py | 4 +- app/tests/tests_10/test_asset_upload_model.py | 4 +- app/tests/tests_10/test_assets_endpoint.py | 2 +- .../tests_10/test_collection_asset_model.py | 2 +- .../test_collection_asset_upload_endpoint.py | 4 +- .../test_collection_asset_upload_model.py | 4 +- .../test_collection_assets_endpoint.py | 2 +- app/tests/tests_10/test_collection_model.py | 2 +- .../tests_10/test_collections_endpoint.py | 4 +- app/tests/tests_10/test_collections_extent.py | 2 +- .../tests_10/test_external_assets_endpoint.py | 2 +- app/tests/tests_10/test_generic_api.py | 2 +- app/tests/tests_10/test_item_model.py | 4 +- app/tests/tests_10/test_items_endpoint.py | 6 +- .../tests_10/test_remove_expired_items.py | 4 +- app/tests/tests_10/test_serializer.py | 2 +- .../tests_10/test_serializer_asset_upload.py | 2 +- 59 files changed, 710 insertions(+), 674 deletions(-) create mode 100644 app/stac_api/models/collection.py create mode 100644 app/stac_api/models/item.py diff --git a/app/stac_api/admin.py b/app/stac_api/admin.py index 494af29a..68ec5a60 100644 --- a/app/stac_api/admin.py +++ b/app/stac_api/admin.py @@ -17,17 +17,17 @@ from django.urls import reverse from django.utils.translation import gettext_lazy as _ +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionAsset +from stac_api.models.collection import CollectionLink from stac_api.models.general import BBOX_CH -from stac_api.models.general import Asset -from stac_api.models.general import AssetUpload -from stac_api.models.general import Collection -from stac_api.models.general import CollectionAsset -from stac_api.models.general import CollectionLink -from stac_api.models.general import Item -from stac_api.models.general import ItemLink from stac_api.models.general import LandingPage from stac_api.models.general import LandingPageLink from stac_api.models.general import Provider +from stac_api.models.item import Asset +from stac_api.models.item import AssetUpload +from stac_api.models.item import Item +from stac_api.models.item import ItemLink from stac_api.utils import build_asset_href from stac_api.utils import get_query_params from stac_api.validators import validate_href_url diff --git a/app/stac_api/management/commands/calculate_extent.py b/app/stac_api/management/commands/calculate_extent.py index 4118d8e9..b0d257dc 100644 --- a/app/stac_api/management/commands/calculate_extent.py +++ b/app/stac_api/management/commands/calculate_extent.py @@ -3,7 +3,7 @@ from django.core.management.base import CommandParser from django.db import connection -from stac_api.models.general import Collection +from stac_api.models.collection import Collection from stac_api.utils import CommandHandler from stac_api.utils import CustomBaseCommand diff --git a/app/stac_api/management/commands/dummy_asset_upload.py b/app/stac_api/management/commands/dummy_asset_upload.py index e79964b6..7232e8c2 100644 --- a/app/stac_api/management/commands/dummy_asset_upload.py +++ b/app/stac_api/management/commands/dummy_asset_upload.py @@ -4,9 +4,9 @@ from django.conf import settings from django.core.management.base import BaseCommand -from stac_api.models.general import Asset -from stac_api.models.general import AssetUpload from stac_api.models.general import BaseAssetUpload +from stac_api.models.item import Asset +from stac_api.models.item import AssetUpload from stac_api.s3_multipart_upload import MultipartUpload from stac_api.utils import AVAILABLE_S3_BUCKETS from stac_api.utils import CommandHandler diff --git a/app/stac_api/management/commands/dummy_data.py b/app/stac_api/management/commands/dummy_data.py index ac13f159..a69702db 100644 --- a/app/stac_api/management/commands/dummy_data.py +++ b/app/stac_api/management/commands/dummy_data.py @@ -13,9 +13,9 @@ from django.core.files.uploadedfile import SimpleUploadedFile from django.core.management.base import BaseCommand -from stac_api.models.general import Asset -from stac_api.models.general import Collection -from stac_api.models.general import Item +from stac_api.models.collection import Collection +from stac_api.models.item import Asset +from stac_api.models.item import Item from stac_api.utils import CommandHandler from stac_api.validators import MEDIA_TYPES diff --git a/app/stac_api/management/commands/list_asset_uploads.py b/app/stac_api/management/commands/list_asset_uploads.py index c0a0f5db..24bb58ce 100644 --- a/app/stac_api/management/commands/list_asset_uploads.py +++ b/app/stac_api/management/commands/list_asset_uploads.py @@ -4,7 +4,7 @@ from django.core.management.base import BaseCommand from django.core.serializers.json import DjangoJSONEncoder -from stac_api.models.general import AssetUpload +from stac_api.models.item import AssetUpload from stac_api.s3_multipart_upload import MultipartUpload from stac_api.serializers.upload import AssetUploadSerializer from stac_api.utils import CommandHandler diff --git a/app/stac_api/management/commands/profile_cursor_paginator.py b/app/stac_api/management/commands/profile_cursor_paginator.py index 311ecff3..18c6174c 100644 --- a/app/stac_api/management/commands/profile_cursor_paginator.py +++ b/app/stac_api/management/commands/profile_cursor_paginator.py @@ -10,7 +10,7 @@ from rest_framework.request import Request from rest_framework.test import APIRequestFactory -from stac_api.models.general import Item +from stac_api.models.item import Item from stac_api.utils import CommandHandler logger = logging.getLogger(__name__) diff --git a/app/stac_api/management/commands/profile_item_serializer.py b/app/stac_api/management/commands/profile_item_serializer.py index 9b43822f..8439598d 100644 --- a/app/stac_api/management/commands/profile_item_serializer.py +++ b/app/stac_api/management/commands/profile_item_serializer.py @@ -8,7 +8,7 @@ from rest_framework.test import APIRequestFactory -from stac_api.models.general import Item +from stac_api.models.item import Item from stac_api.utils import CommandHandler logger = logging.getLogger(__name__) diff --git a/app/stac_api/management/commands/profile_serializer_vs_no_drf.py b/app/stac_api/management/commands/profile_serializer_vs_no_drf.py index 3f2ada21..28ed2a2d 100644 --- a/app/stac_api/management/commands/profile_serializer_vs_no_drf.py +++ b/app/stac_api/management/commands/profile_serializer_vs_no_drf.py @@ -7,7 +7,7 @@ from rest_framework.test import APIRequestFactory -from stac_api.models.general import Item +from stac_api.models.item import Item from stac_api.utils import CommandHandler logger = logging.getLogger(__name__) diff --git a/app/stac_api/management/commands/remove_expired_items.py b/app/stac_api/management/commands/remove_expired_items.py index a1f73a22..6b99042f 100644 --- a/app/stac_api/management/commands/remove_expired_items.py +++ b/app/stac_api/management/commands/remove_expired_items.py @@ -4,9 +4,9 @@ from django.core.management.base import CommandParser from django.utils import timezone -from stac_api.models.general import AssetUpload from stac_api.models.general import BaseAssetUpload -from stac_api.models.general import Item +from stac_api.models.item import AssetUpload +from stac_api.models.item import Item from stac_api.utils import CommandHandler from stac_api.utils import CustomBaseCommand diff --git a/app/stac_api/management/commands/update_asset_file_size.py b/app/stac_api/management/commands/update_asset_file_size.py index 7b6742bc..a243ac0d 100644 --- a/app/stac_api/management/commands/update_asset_file_size.py +++ b/app/stac_api/management/commands/update_asset_file_size.py @@ -6,8 +6,8 @@ from django.core.management.base import BaseCommand from django.core.management.base import CommandParser -from stac_api.models.general import Asset -from stac_api.models.general import CollectionAsset +from stac_api.models.collection import CollectionAsset +from stac_api.models.item import Asset from stac_api.utils import CommandHandler from stac_api.utils import get_s3_client from stac_api.utils import select_s3_bucket diff --git a/app/stac_api/models/collection.py b/app/stac_api/models/collection.py new file mode 100644 index 00000000..efb0bf3e --- /dev/null +++ b/app/stac_api/models/collection.py @@ -0,0 +1,264 @@ +import logging + +from django.contrib.gis.db import models +from django.contrib.postgres.fields import ArrayField +from django.core.serializers.json import DjangoJSONEncoder +from django.core.validators import MinValueValidator +from django.db.models import Q +from django.utils.translation import gettext_lazy as _ + +from stac_api.models.general import SEARCH_TEXT_HELP_ITEM +from stac_api.models.general import AssetBase +from stac_api.models.general import BaseAssetUpload +from stac_api.models.general import Link +from stac_api.models.general import compute_etag +from stac_api.pgtriggers import SummaryFields +from stac_api.pgtriggers import child_triggers +from stac_api.pgtriggers import generates_asset_upload_triggers +from stac_api.pgtriggers import generates_collection_asset_triggers +from stac_api.pgtriggers import generates_collection_triggers +from stac_api.pgtriggers import generates_summary_count_triggers +from stac_api.utils import get_collection_asset_path +from stac_api.validators import validate_name + +logger = logging.getLogger(__name__) + + +# For Collections and Items: No primary key will be defined, so that the auto-generated ones +# will be used by Django. For assets, a primary key is defined as "BigAutoField" due the +# expected large number of assets +class Collection(models.Model): + + class Meta: + indexes = [ + models.Index(fields=['name'], name='collection_name_idx'), + models.Index(fields=['published'], name='collection_published_idx') + ] + triggers = generates_collection_triggers() + + published = models.BooleanField( + default=True, + help_text="When not published the collection doesn't appear on the " + "api/stac/v0.9/collections endpoint and its items are not listed in /search endpoint." + "

NOTE: unpublished collections/items can still be accessed by their path.

" + ) + # using "name" instead of "id", as "id" has a default meaning in django + name = models.CharField('id', unique=True, max_length=255, validators=[validate_name]) + created = models.DateTimeField(auto_now_add=True) + # NOTE: the updated field is automatically updated by stac_api.pgtriggers, we use auto_now_add + # only for the initial value. + updated = models.DateTimeField(auto_now_add=True) + description = models.TextField() + # Set to true if the extent needs to be recalculated. + extent_out_of_sync = models.BooleanField(default=False) + extent_geometry = models.GeometryField( + default=None, srid=4326, editable=False, blank=True, null=True + ) + extent_start_datetime = models.DateTimeField(editable=False, null=True, blank=True) + extent_end_datetime = models.DateTimeField(editable=False, null=True, blank=True) + + license = models.CharField(max_length=30) # string + + # DEPRECATED: summaries JSON field is not used anymore and not up to date, it will be removed + # in future. It has been replaced by summaries_proj_epsg, summaries_eo_gsd and + # summaries_geoadmin_variant + summaries = models.JSONField(default=dict, encoder=DjangoJSONEncoder, editable=False) + + # NOTE: the following summaries_* fields are automatically update by the stac_api.pgtriggers + summaries_proj_epsg = ArrayField( + models.IntegerField(), default=list, blank=True, editable=False + ) + summaries_eo_gsd = ArrayField(models.FloatField(), default=list, blank=True, editable=False) + summaries_geoadmin_variant = ArrayField( + models.CharField(max_length=25), default=list, blank=True, editable=False + ) + summaries_geoadmin_lang = ArrayField( + models.CharField(max_length=2), default=list, blank=True, editable=False + ) + + title = models.CharField(blank=True, null=True, max_length=255) + + # NOTE: hidden ETag field, this field is automatically updated by stac_api.pgtriggers + etag = models.CharField( + blank=False, null=False, editable=False, max_length=56, default=compute_etag + ) + + update_interval = models.IntegerField( + default=-1, + null=False, + blank=False, + validators=[MinValueValidator(-1)], + help_text="Minimal update interval in seconds " + "in which the underlying assets data are updated." + ) + + total_data_size = models.BigIntegerField(default=0, null=True, blank=True) + + allow_external_assets = models.BooleanField( + default=False, + help_text=_('Whether this collection can have assets that are hosted externally') + ) + + external_asset_whitelist = ArrayField( + models.CharField(max_length=255), blank=True, default=list, + help_text=_('Provide a comma separated list of ' + 'protocol://domain values for the external asset url validation') + ) + + def __str__(self): + return self.name + + +class CollectionLink(Link): + collection = models.ForeignKey( + Collection, related_name='links', related_query_name='link', on_delete=models.CASCADE + ) + + class Meta: + ordering = ['pk'] + triggers = child_triggers('collection', 'CollectionLink') + + +class CollectionAsset(AssetBase): + + class Meta: + unique_together = (('collection', 'name'),) + ordering = ['id'] + triggers = generates_collection_asset_triggers() + + collection = models.ForeignKey( + Collection, + related_name='assets', + related_query_name='asset', + on_delete=models.PROTECT, + help_text=_(SEARCH_TEXT_HELP_ITEM) + ) + + # CollectionAssets are never external + is_external = False + + def get_collection(self): + return self.collection + + def get_asset_path(self): + return get_collection_asset_path(self.collection, self.name) + + +class CollectionAssetUpload(BaseAssetUpload): + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=['asset', 'upload_id'], + name='unique_asset_upload_collection_asset_upload_id' + ), + # Make sure that there is only one upload in progress per collection asset + models.UniqueConstraint( + fields=['asset', 'status'], + condition=Q(status='in-progress'), + name='unique_asset_upload_in_progress' + ) + ] + triggers = generates_asset_upload_triggers() + + asset = models.ForeignKey(CollectionAsset, related_name='+', on_delete=models.CASCADE) + + def update_asset_from_upload(self): + '''Updating the asset's file:checksum and update_interval from the upload + + When the upload is completed, the new file:checksum and update interval from the upload + is set to its asset parent. + ''' + logger.debug( + 'Updating collection asset %s file:checksum from %s to %s and update_interval ' + 'from %d to %d due to upload complete', + self.asset.name, + self.asset.checksum_multihash, + self.checksum_multihash, + self.asset.update_interval, + self.update_interval, + extra={ + 'upload_id': self.upload_id, + 'asset': self.asset.name, + 'collection': self.asset.collection.name + } + ) + + self.asset.checksum_multihash = self.checksum_multihash + self.asset.update_interval = self.update_interval + self.asset.file_size = self.file_size + self.asset.save() + + +class CountBase(models.Model): + '''CountBase tables are used to help calculate the summary on a collection. + This is only performant if the distinct number of values is small, e.g. we currently only have + 5 possible values for geoadmin_language. + For each assets value we keep a count of how often that value exists, per collection. On + insert/update/delete of an asset we only need to decrease and/or increase the counter value. + Since the number of possible values is small, the aggregate array calculation to update the + collection also stays performant. + ''' + + class Meta: + abstract = True + + id = models.BigAutoField(primary_key=True) + collection = models.ForeignKey( + Collection, + related_name='+', + on_delete=models.CASCADE, + ) + count = models.PositiveIntegerField(null=False) + + +class GSDCount(CountBase): + # Update by asset triggers. + + class Meta: + unique_together = (('collection', 'value'),) + ordering = ['id'] + triggers = generates_summary_count_triggers( + SummaryFields.GSD.value[0], SummaryFields.GSD.value[1] + ) + + value = models.FloatField(null=True, blank=True) + + +class GeoadminLangCount(CountBase): + # Update by asset triggers. + + class Meta: + unique_together = (('collection', 'value'),) + ordering = ['id'] + triggers = generates_summary_count_triggers( + SummaryFields.LANGUAGE.value[0], SummaryFields.LANGUAGE.value[1] + ) + + value = models.CharField(max_length=2, default=None, null=True, blank=True) + + +class GeoadminVariantCount(CountBase): + # Update by asset triggers. + + class Meta: + unique_together = (('collection', 'value'),) + ordering = ['id'] + triggers = generates_summary_count_triggers( + SummaryFields.VARIANT.value[0], SummaryFields.VARIANT.value[1] + ) + + value = models.CharField(max_length=25, null=True, blank=True) + + +class ProjEPSGCount(CountBase): + # Update by asset and collection asset triggers. + + class Meta: + unique_together = (('collection', 'value'),) + ordering = ['id'] + triggers = generates_summary_count_triggers( + SummaryFields.PROJ_EPSG.value[0], SummaryFields.PROJ_EPSG.value[1] + ) + + value = models.IntegerField(null=True, blank=True) diff --git a/app/stac_api/models/general.py b/app/stac_api/models/general.py index c6b7cd8f..6849409a 100644 --- a/app/stac_api/models/general.py +++ b/app/stac_api/models/general.py @@ -16,30 +16,15 @@ from django.core.serializers.json import DjangoJSONEncoder from django.core.validators import MaxValueValidator from django.core.validators import MinValueValidator -from django.db.models import Q from django.db.models.deletion import ProtectedError from django.utils.translation import gettext_lazy as _ from stac_api.managers import AssetUploadManager -from stac_api.managers import ItemManager -from stac_api.pgtriggers import SummaryFields from stac_api.pgtriggers import child_triggers -from stac_api.pgtriggers import generates_asset_triggers -from stac_api.pgtriggers import generates_asset_upload_triggers -from stac_api.pgtriggers import generates_collection_asset_triggers -from stac_api.pgtriggers import generates_collection_triggers -from stac_api.pgtriggers import generates_item_triggers -from stac_api.pgtriggers import generates_summary_count_triggers -from stac_api.utils import get_asset_path -from stac_api.utils import get_collection_asset_path from stac_api.utils import select_s3_bucket from stac_api.validators import MEDIA_TYPES from stac_api.validators import validate_asset_name from stac_api.validators import validate_asset_name_with_media_type -from stac_api.validators import validate_eo_gsd -from stac_api.validators import validate_geoadmin_variant -from stac_api.validators import validate_geometry -from stac_api.validators import validate_item_properties_datetimes from stac_api.validators import validate_link_rel from stac_api.validators import validate_media_type from stac_api.validators import validate_name @@ -88,40 +73,6 @@ ''' -SEARCH_TEXT_HELP_COLLECTION = ''' -
- Search Usage: -
    -
  • - arg will make a non exact search checking if arg is part of - the collection ID -
    • -
  • - Multiple arg can be used, separated by spaces. This will search for all - collections ID containing all arguments. -
    • -
  • - "collectionID" will make an exact search for the specified collection. -
    • -
- Examples : -
    -
  • - Searching for pixelkarte will return all collections which have - pixelkarte as a part of their collection ID -
    • -
  • - Searching for pixelkarte 2016 4 will return all collection - which have pixelkarte, 2016 AND 4 as part of their collection ID -
    • -
  • - Searching for ch.swisstopo.pixelkarte.example will yield only this - collection, if this collection exists. Please note that it would not return - a collection named ch.swisstopo.pixelkarte.example.2. -
    • -
-
''' - def get_conformance_default_links(): '''A helper function of the class Conformance Page @@ -252,101 +203,6 @@ def clean(self): ) -# For Collections and Items: No primary key will be defined, so that the auto-generated ones -# will be used by Django. For assets, a primary key is defined as "BigAutoField" due the -# expected large number of assets -class Collection(models.Model): - - class Meta: - indexes = [ - models.Index(fields=['name'], name='collection_name_idx'), - models.Index(fields=['published'], name='collection_published_idx') - ] - triggers = generates_collection_triggers() - - published = models.BooleanField( - default=True, - help_text="When not published the collection doesn't appear on the " - "api/stac/v0.9/collections endpoint and its items are not listed in /search endpoint." - "

NOTE: unpublished collections/items can still be accessed by their path.

" - ) - # using "name" instead of "id", as "id" has a default meaning in django - name = models.CharField('id', unique=True, max_length=255, validators=[validate_name]) - created = models.DateTimeField(auto_now_add=True) - # NOTE: the updated field is automatically updated by stac_api.pgtriggers, we use auto_now_add - # only for the initial value. - updated = models.DateTimeField(auto_now_add=True) - description = models.TextField() - # Set to true if the extent needs to be recalculated. - extent_out_of_sync = models.BooleanField(default=False) - extent_geometry = models.GeometryField( - default=None, srid=4326, editable=False, blank=True, null=True - ) - extent_start_datetime = models.DateTimeField(editable=False, null=True, blank=True) - extent_end_datetime = models.DateTimeField(editable=False, null=True, blank=True) - - license = models.CharField(max_length=30) # string - - # DEPRECATED: summaries JSON field is not used anymore and not up to date, it will be removed - # in future. It has been replaced by summaries_proj_epsg, summaries_eo_gsd and - # summaries_geoadmin_variant - summaries = models.JSONField(default=dict, encoder=DjangoJSONEncoder, editable=False) - - # NOTE: the following summaries_* fields are automatically update by the stac_api.pgtriggers - summaries_proj_epsg = ArrayField( - models.IntegerField(), default=list, blank=True, editable=False - ) - summaries_eo_gsd = ArrayField(models.FloatField(), default=list, blank=True, editable=False) - summaries_geoadmin_variant = ArrayField( - models.CharField(max_length=25), default=list, blank=True, editable=False - ) - summaries_geoadmin_lang = ArrayField( - models.CharField(max_length=2), default=list, blank=True, editable=False - ) - - title = models.CharField(blank=True, null=True, max_length=255) - - # NOTE: hidden ETag field, this field is automatically updated by stac_api.pgtriggers - etag = models.CharField( - blank=False, null=False, editable=False, max_length=56, default=compute_etag - ) - - update_interval = models.IntegerField( - default=-1, - null=False, - blank=False, - validators=[MinValueValidator(-1)], - help_text="Minimal update interval in seconds " - "in which the underlying assets data are updated." - ) - - total_data_size = models.BigIntegerField(default=0, null=True, blank=True) - - allow_external_assets = models.BooleanField( - default=False, - help_text=_('Whether this collection can have assets that are hosted externally') - ) - - external_asset_whitelist = ArrayField( - models.CharField(max_length=255), blank=True, default=list, - help_text=_('Provide a comma separated list of ' - 'protocol://domain values for the external asset url validation') - ) - - def __str__(self): - return self.name - - -class CollectionLink(Link): - collection = models.ForeignKey( - Collection, related_name='links', related_query_name='link', on_delete=models.CASCADE - ) - - class Meta: - ordering = ['pk'] - triggers = child_triggers('collection', 'CollectionLink') - - ITEM_KEEP_ORIGINAL_FIELDS = [ 'geometry', 'properties_datetime', @@ -355,179 +211,6 @@ class Meta: ] -class Item(models.Model): - - class Meta: - unique_together = (('collection', 'name'),) - indexes = [ - models.Index(fields=['name'], name='item_name_idx'), - # the following 3 indices are used e.g. in collection_temporal_extent - models.Index(fields=['properties_datetime'], name='item_datetime_idx'), - models.Index(fields=['properties_start_datetime'], name='item_start_datetime_idx'), - models.Index(fields=['properties_end_datetime'], name='item_end_datetime_idx'), - # created, updated, and title are "queryable" in the search endpoint - # see: views.py:322 and 323 - models.Index(fields=['created'], name='item_created_idx'), - models.Index(fields=['updated'], name='item_updated_idx'), - models.Index(fields=['properties_title'], name='item_title_idx'), - # forecast properties are "queryable" in the search endpoint - models.Index( - fields=['forecast_reference_datetime'], name='item_fc_reference_datetime_idx' - ), - models.Index(fields=['forecast_horizon'], name='item_fc_horizon_idx'), - models.Index(fields=['forecast_duration'], name='item_fc_duration_idx'), - models.Index(fields=['forecast_variable'], name='item_fc_variable_idx'), - models.Index(fields=['forecast_perturbed'], name='item_fc_perturbed_idx'), - # combination of datetime and start_ and end_datetimes are used in - # managers.py:110 and following - models.Index( - fields=[ - 'properties_datetime', 'properties_start_datetime', 'properties_end_datetime' - ], - name='item_dttme_start_end_dttm_idx' - ), - models.Index(fields=['update_interval'], name='item_update_interval_idx'), - ] - triggers = generates_item_triggers() - - name = models.CharField('id', blank=False, max_length=255, validators=[validate_name]) - collection = models.ForeignKey( - Collection, on_delete=models.PROTECT, help_text=_(SEARCH_TEXT_HELP_COLLECTION) - ) - geometry = models.GeometryField( - null=False, blank=False, default=BBOX_CH, srid=4326, validators=[validate_geometry] - ) - created = models.DateTimeField(auto_now_add=True) - # NOTE: the updated field is automatically updated by stac_api.pgtriggers, we use auto_now_add - # only for the initial value. - updated = models.DateTimeField(auto_now_add=True) - # after discussion with Chris and Tobias: for the moment only support - # proterties: datetime and title (the rest is hence commented out) - properties_datetime = models.DateTimeField( - blank=True, - null=True, - help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format" - ) - properties_start_datetime = models.DateTimeField( - blank=True, - null=True, - help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format" - ) - properties_end_datetime = models.DateTimeField( - blank=True, - null=True, - help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format" - ) - properties_expires = models.DateTimeField( - blank=True, - null=True, - help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format" - ) - # properties_eo_bands = model.TextFields(blank=True) # ? [string]? - # properties_eo_cloud_cover = models.FloatField(blank=True) - # eo_gsd is defined on asset level and will be updated here on ever - # update of an asset inside this item. - # properties_instruments = models.TextField(blank=True) - # properties_license = models.TextField(blank=True) - # properties_platform = models.TextField(blank=True) - # properties_providers = models.ManyToManyField(Provider) - # Although it is discouraged by Django to set blank=True and null=True on a CharField, it is - # here required because this field is optional and having an empty string value is not permitted - # in the serializer and default to None. None value are then not displayed in the serializer. - properties_title = models.CharField(blank=True, null=True, max_length=255) - - # properties_view_off_nadir = models.FloatField(blank=True) - # properties_view_sun_azimuth = models.FloatField(blank=True) - # properties_view_elevation = models.FloatField(blank=True) - - # NOTE: hidden ETag field, this field is automatically updated by stac_api.pgtriggers - etag = models.CharField( - blank=False, null=False, editable=False, max_length=56, default=compute_etag - ) - - update_interval = models.IntegerField( - default=-1, - null=False, - blank=False, - validators=[MinValueValidator(-1)], - help_text="Minimal update interval in seconds " - "in which the underlying assets data are updated." - ) - - total_data_size = models.BigIntegerField(default=0, null=True, blank=True) - - forecast_reference_datetime = models.DateTimeField( - null=True, - blank=True, - help_text="The reference datetime: i.e. predictions for times after " - "this point occur in the future. Predictions prior to this " - "time represent 'hindcasts', predicting states that have " - "already occurred. This must be in UTC. It is formatted like " - "'2022-08-12T00:00:00Z'." - ) - - forecast_horizon = models.DurationField( - null=True, - blank=True, - help_text="The time between the reference datetime and the forecast datetime." - "Formatted as ISO 8601 duration, e.g. 'PT6H' for a 6-hour forecast.", - ) - - forecast_duration = models.DurationField( - null=True, - blank=True, - help_text="If the forecast is not only for a specific instance in time " - "but instead is for a certain period, you can specify the " - "length here. Formatted as ISO 8601 duration, e.g. 'PT3H' for a 3-hour " - "accumulation. If not given, assumes that the forecast is for an " - "instance in time as if this was set to PT0S (0 seconds).", - ) - - forecast_variable = models.CharField( - null=True, - blank=True, - help_text="Name of the model variable that corresponds to the data. The variables " - "should correspond to the CF Standard Names, " - "e.g. `air_temperature` for the air temperature." - ) - - forecast_perturbed = models.BooleanField( - null=True, - blank=True, - default=None, - help_text="Denotes whether the data corresponds to the control run (`false`) or " - "perturbed runs (`true`). The property needs to be specified in both " - "cases as no default value is specified and as such the meaning is " - "\"unknown\" in case it's missing." - ) - - # Custom Manager that preselects the collection - objects = ItemManager() - - def __str__(self): - # This is used in the admin page in the autocomplete_fields of the Asset page - return f"{self.collection.name}/{self.name}" - - def clean(self): - validate_item_properties_datetimes( - self.properties_datetime, - self.properties_start_datetime, - self.properties_end_datetime, - self.properties_expires, - ) - - -class ItemLink(Link): - item = models.ForeignKey( - Item, related_name='links', related_query_name='link', on_delete=models.CASCADE - ) - - class Meta: - unique_together = (('rel', 'item'),) - ordering = ['pk'] - triggers = child_triggers('item', 'ItemLink') - - def upload_asset_to_path_hook(instance, filename=None): '''This returns the asset upload path on S3 and compute the asset file multihash @@ -707,80 +390,6 @@ def clean(self): validate_asset_name_with_media_type(self.name, media_type) -class Asset(AssetBase): - - class Meta: - unique_together = (('item', 'name'),) - ordering = ['id'] - triggers = generates_asset_triggers() - - item = models.ForeignKey( - Item, - related_name='assets', - related_query_name='asset', - on_delete=models.PROTECT, - help_text=_(SEARCH_TEXT_HELP_ITEM) - ) - # From v1 on the json representation of this field changed from "eo:gsd" to "gsd". The two names - # may be used interchangeably for a now. - eo_gsd = models.FloatField(null=True, blank=True, validators=[validate_eo_gsd]) - - class Language(models.TextChoices): - # pylint: disable=invalid-name - GERMAN = 'de', _('German') - ITALIAN = 'it', _('Italian') - FRENCH = 'fr', _('French') - ROMANSH = 'rm', _('Romansh') - ENGLISH = 'en', _('English') - __empty__ = _('') - - # here we need to set blank=True otherwise the field is as required in the admin interface - geoadmin_lang = models.CharField( - max_length=2, choices=Language.choices, default=None, null=True, blank=True - ) - # here we need to set blank=True otherwise the field is as required in the admin interface - geoadmin_variant = models.CharField( - max_length=25, null=True, blank=True, validators=[validate_geoadmin_variant] - ) - - # whether this asset is hosted externally - is_external = models.BooleanField( - default=False, - help_text=_("Whether this asset is hosted externally") - ) - - def get_collection(self): - return self.item.collection - - def get_asset_path(self): - return get_asset_path(self.item, self.name) - - -class CollectionAsset(AssetBase): - - class Meta: - unique_together = (('collection', 'name'),) - ordering = ['id'] - triggers = generates_collection_asset_triggers() - - collection = models.ForeignKey( - Collection, - related_name='assets', - related_query_name='asset', - on_delete=models.PROTECT, - help_text=_(SEARCH_TEXT_HELP_ITEM) - ) - - # CollectionAssets are never external - is_external = False - - def get_collection(self): - return self.collection - - def get_asset_path(self): - return get_collection_asset_path(self.collection, self.name) - - class BaseAssetUpload(models.Model): class Meta: @@ -839,167 +448,3 @@ class ContentEncoding(models.TextChoices): # Custom Manager that preselects the collection objects = AssetUploadManager() - - -class AssetUpload(BaseAssetUpload): - - class Meta: - constraints = [ - models.UniqueConstraint(fields=['asset', 'upload_id'], name='unique_together'), - # Make sure that there is only one asset upload in progress per asset - models.UniqueConstraint( - fields=['asset', 'status'], - condition=Q(status='in-progress'), - name='unique_in_progress' - ) - ] - triggers = generates_asset_upload_triggers() - - asset = models.ForeignKey(Asset, related_name='+', on_delete=models.CASCADE) - - def update_asset_from_upload(self): - '''Updating the asset's file:checksum and update_interval from the upload - - When the upload is completed, the new file:checksum and update interval from the upload - is set to its asset parent. - ''' - logger.debug( - 'Updating asset %s file:checksum from %s to %s and update_interval from %d to %d ' - 'due to upload complete', - self.asset.name, - self.asset.checksum_multihash, - self.checksum_multihash, - self.asset.update_interval, - self.update_interval, - extra={ - 'upload_id': self.upload_id, - 'asset': self.asset.name, - 'item': self.asset.item.name, - 'collection': self.asset.item.collection.name - } - ) - - self.asset.checksum_multihash = self.checksum_multihash - self.asset.update_interval = self.update_interval - self.asset.file_size = self.file_size - self.asset.save() - - -class CollectionAssetUpload(BaseAssetUpload): - - class Meta: - constraints = [ - models.UniqueConstraint( - fields=['asset', 'upload_id'], - name='unique_asset_upload_collection_asset_upload_id' - ), - # Make sure that there is only one upload in progress per collection asset - models.UniqueConstraint( - fields=['asset', 'status'], - condition=Q(status='in-progress'), - name='unique_asset_upload_in_progress' - ) - ] - triggers = generates_asset_upload_triggers() - - asset = models.ForeignKey(CollectionAsset, related_name='+', on_delete=models.CASCADE) - - def update_asset_from_upload(self): - '''Updating the asset's file:checksum and update_interval from the upload - - When the upload is completed, the new file:checksum and update interval from the upload - is set to its asset parent. - ''' - logger.debug( - 'Updating collection asset %s file:checksum from %s to %s and update_interval ' - 'from %d to %d due to upload complete', - self.asset.name, - self.asset.checksum_multihash, - self.checksum_multihash, - self.asset.update_interval, - self.update_interval, - extra={ - 'upload_id': self.upload_id, - 'asset': self.asset.name, - 'collection': self.asset.collection.name - } - ) - - self.asset.checksum_multihash = self.checksum_multihash - self.asset.update_interval = self.update_interval - self.asset.file_size = self.file_size - self.asset.save() - - -class CountBase(models.Model): - '''CountBase tables are used to help calculate the summary on a collection. - This is only performant if the distinct number of values is small, e.g. we currently only have - 5 possible values for geoadmin_language. - For each assets value we keep a count of how often that value exists, per collection. On - insert/update/delete of an asset we only need to decrease and/or increase the counter value. - Since the number of possible values is small, the aggregate array calculation to update the - collection also stays performant. - ''' - - class Meta: - abstract = True - - id = models.BigAutoField(primary_key=True) - collection = models.ForeignKey( - Collection, - related_name='+', - on_delete=models.CASCADE, - ) - count = models.PositiveIntegerField(null=False) - - -class GSDCount(CountBase): - # Update by asset triggers. - - class Meta: - unique_together = (('collection', 'value'),) - ordering = ['id'] - triggers = generates_summary_count_triggers( - SummaryFields.GSD.value[0], SummaryFields.GSD.value[1] - ) - - value = models.FloatField(null=True, blank=True) - - -class GeoadminLangCount(CountBase): - # Update by asset triggers. - - class Meta: - unique_together = (('collection', 'value'),) - ordering = ['id'] - triggers = generates_summary_count_triggers( - SummaryFields.LANGUAGE.value[0], SummaryFields.LANGUAGE.value[1] - ) - - value = models.CharField(max_length=2, default=None, null=True, blank=True) - - -class GeoadminVariantCount(CountBase): - # Update by asset triggers. - - class Meta: - unique_together = (('collection', 'value'),) - ordering = ['id'] - triggers = generates_summary_count_triggers( - SummaryFields.VARIANT.value[0], SummaryFields.VARIANT.value[1] - ) - - value = models.CharField(max_length=25, null=True, blank=True) - - -class ProjEPSGCount(CountBase): - # Update by asset and collection asset triggers. - - class Meta: - unique_together = (('collection', 'value'),) - ordering = ['id'] - triggers = generates_summary_count_triggers( - SummaryFields.PROJ_EPSG.value[0], SummaryFields.PROJ_EPSG.value[1] - ) - - value = models.IntegerField(null=True, blank=True) diff --git a/app/stac_api/models/item.py b/app/stac_api/models/item.py new file mode 100644 index 00000000..7d3aae9d --- /dev/null +++ b/app/stac_api/models/item.py @@ -0,0 +1,327 @@ +import logging + +from django.contrib.gis.db import models +from django.core.validators import MinValueValidator +from django.db.models import Q +from django.utils.translation import gettext_lazy as _ + +from stac_api.managers import ItemManager +from stac_api.models.collection import Collection +from stac_api.models.general import BBOX_CH +from stac_api.models.general import SEARCH_TEXT_HELP_ITEM +from stac_api.models.general import AssetBase +from stac_api.models.general import BaseAssetUpload +from stac_api.models.general import Link +from stac_api.models.general import compute_etag +from stac_api.pgtriggers import child_triggers +from stac_api.pgtriggers import generates_asset_triggers +from stac_api.pgtriggers import generates_asset_upload_triggers +from stac_api.pgtriggers import generates_item_triggers +from stac_api.utils import get_asset_path +from stac_api.validators import validate_eo_gsd +from stac_api.validators import validate_geoadmin_variant +from stac_api.validators import validate_geometry +from stac_api.validators import validate_item_properties_datetimes +from stac_api.validators import validate_name + +logger = logging.getLogger(__name__) + +SEARCH_TEXT_HELP_COLLECTION = ''' +
+ Search Usage: +
    +
  • + arg will make a non exact search checking if arg is part of + the collection ID +
    • +
  • + Multiple arg can be used, separated by spaces. This will search for all + collections ID containing all arguments. +
    • +
  • + "collectionID" will make an exact search for the specified collection. +
    • +
+ Examples : +
    +
  • + Searching for pixelkarte will return all collections which have + pixelkarte as a part of their collection ID +
    • +
  • + Searching for pixelkarte 2016 4 will return all collection + which have pixelkarte, 2016 AND 4 as part of their collection ID +
    • +
  • + Searching for ch.swisstopo.pixelkarte.example will yield only this + collection, if this collection exists. Please note that it would not return + a collection named ch.swisstopo.pixelkarte.example.2. +
    • +
+
''' + + +class Item(models.Model): + + class Meta: + unique_together = (('collection', 'name'),) + indexes = [ + models.Index(fields=['name'], name='item_name_idx'), + # the following 3 indices are used e.g. in collection_temporal_extent + models.Index(fields=['properties_datetime'], name='item_datetime_idx'), + models.Index(fields=['properties_start_datetime'], name='item_start_datetime_idx'), + models.Index(fields=['properties_end_datetime'], name='item_end_datetime_idx'), + # created, updated, and title are "queryable" in the search endpoint + # see: views.py:322 and 323 + models.Index(fields=['created'], name='item_created_idx'), + models.Index(fields=['updated'], name='item_updated_idx'), + models.Index(fields=['properties_title'], name='item_title_idx'), + # forecast properties are "queryable" in the search endpoint + models.Index( + fields=['forecast_reference_datetime'], name='item_fc_reference_datetime_idx' + ), + models.Index(fields=['forecast_horizon'], name='item_fc_horizon_idx'), + models.Index(fields=['forecast_duration'], name='item_fc_duration_idx'), + models.Index(fields=['forecast_variable'], name='item_fc_variable_idx'), + models.Index(fields=['forecast_perturbed'], name='item_fc_perturbed_idx'), + # combination of datetime and start_ and end_datetimes are used in + # managers.py:110 and following + models.Index( + fields=[ + 'properties_datetime', 'properties_start_datetime', 'properties_end_datetime' + ], + name='item_dttme_start_end_dttm_idx' + ), + models.Index(fields=['update_interval'], name='item_update_interval_idx'), + ] + triggers = generates_item_triggers() + + name = models.CharField('id', blank=False, max_length=255, validators=[validate_name]) + collection = models.ForeignKey( + Collection, on_delete=models.PROTECT, help_text=_(SEARCH_TEXT_HELP_COLLECTION) + ) + geometry = models.GeometryField( + null=False, blank=False, default=BBOX_CH, srid=4326, validators=[validate_geometry] + ) + created = models.DateTimeField(auto_now_add=True) + # NOTE: the updated field is automatically updated by stac_api.pgtriggers, we use auto_now_add + # only for the initial value. + updated = models.DateTimeField(auto_now_add=True) + # after discussion with Chris and Tobias: for the moment only support + # proterties: datetime and title (the rest is hence commented out) + properties_datetime = models.DateTimeField( + blank=True, + null=True, + help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format" + ) + properties_start_datetime = models.DateTimeField( + blank=True, + null=True, + help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format" + ) + properties_end_datetime = models.DateTimeField( + blank=True, + null=True, + help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format" + ) + properties_expires = models.DateTimeField( + blank=True, + null=True, + help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format" + ) + # properties_eo_bands = model.TextFields(blank=True) # ? [string]? + # properties_eo_cloud_cover = models.FloatField(blank=True) + # eo_gsd is defined on asset level and will be updated here on ever + # update of an asset inside this item. + # properties_instruments = models.TextField(blank=True) + # properties_license = models.TextField(blank=True) + # properties_platform = models.TextField(blank=True) + # properties_providers = models.ManyToManyField(Provider) + # Although it is discouraged by Django to set blank=True and null=True on a CharField, it is + # here required because this field is optional and having an empty string value is not permitted + # in the serializer and default to None. None value are then not displayed in the serializer. + properties_title = models.CharField(blank=True, null=True, max_length=255) + + # properties_view_off_nadir = models.FloatField(blank=True) + # properties_view_sun_azimuth = models.FloatField(blank=True) + # properties_view_elevation = models.FloatField(blank=True) + + # NOTE: hidden ETag field, this field is automatically updated by stac_api.pgtriggers + etag = models.CharField( + blank=False, null=False, editable=False, max_length=56, default=compute_etag + ) + + update_interval = models.IntegerField( + default=-1, + null=False, + blank=False, + validators=[MinValueValidator(-1)], + help_text="Minimal update interval in seconds " + "in which the underlying assets data are updated." + ) + + total_data_size = models.BigIntegerField(default=0, null=True, blank=True) + + forecast_reference_datetime = models.DateTimeField( + null=True, + blank=True, + help_text="The reference datetime: i.e. predictions for times after " + "this point occur in the future. Predictions prior to this " + "time represent 'hindcasts', predicting states that have " + "already occurred. This must be in UTC. It is formatted like " + "'2022-08-12T00:00:00Z'." + ) + + forecast_horizon = models.DurationField( + null=True, + blank=True, + help_text="The time between the reference datetime and the forecast datetime." + "Formatted as ISO 8601 duration, e.g. 'PT6H' for a 6-hour forecast.", + ) + + forecast_duration = models.DurationField( + null=True, + blank=True, + help_text="If the forecast is not only for a specific instance in time " + "but instead is for a certain period, you can specify the " + "length here. Formatted as ISO 8601 duration, e.g. 'PT3H' for a 3-hour " + "accumulation. If not given, assumes that the forecast is for an " + "instance in time as if this was set to PT0S (0 seconds).", + ) + + forecast_variable = models.CharField( + null=True, + blank=True, + help_text="Name of the model variable that corresponds to the data. The variables " + "should correspond to the CF Standard Names, " + "e.g. `air_temperature` for the air temperature." + ) + + forecast_perturbed = models.BooleanField( + null=True, + blank=True, + default=None, + help_text="Denotes whether the data corresponds to the control run (`false`) or " + "perturbed runs (`true`). The property needs to be specified in both " + "cases as no default value is specified and as such the meaning is " + "\"unknown\" in case it's missing." + ) + + # Custom Manager that preselects the collection + objects = ItemManager() + + def __str__(self): + # This is used in the admin page in the autocomplete_fields of the Asset page + return f"{self.collection.name}/{self.name}" + + def clean(self): + validate_item_properties_datetimes( + self.properties_datetime, + self.properties_start_datetime, + self.properties_end_datetime, + self.properties_expires, + ) + + +class ItemLink(Link): + item = models.ForeignKey( + Item, related_name='links', related_query_name='link', on_delete=models.CASCADE + ) + + class Meta: + unique_together = (('rel', 'item'),) + ordering = ['pk'] + triggers = child_triggers('item', 'ItemLink') + + +class Asset(AssetBase): + + class Meta: + unique_together = (('item', 'name'),) + ordering = ['id'] + triggers = generates_asset_triggers() + + item = models.ForeignKey( + Item, + related_name='assets', + related_query_name='asset', + on_delete=models.PROTECT, + help_text=_(SEARCH_TEXT_HELP_ITEM) + ) + # From v1 on the json representation of this field changed from "eo:gsd" to "gsd". The two names + # may be used interchangeably for a now. + eo_gsd = models.FloatField(null=True, blank=True, validators=[validate_eo_gsd]) + + class Language(models.TextChoices): + # pylint: disable=invalid-name + GERMAN = 'de', _('German') + ITALIAN = 'it', _('Italian') + FRENCH = 'fr', _('French') + ROMANSH = 'rm', _('Romansh') + ENGLISH = 'en', _('English') + __empty__ = _('') + + # here we need to set blank=True otherwise the field is as required in the admin interface + geoadmin_lang = models.CharField( + max_length=2, choices=Language.choices, default=None, null=True, blank=True + ) + # here we need to set blank=True otherwise the field is as required in the admin interface + geoadmin_variant = models.CharField( + max_length=25, null=True, blank=True, validators=[validate_geoadmin_variant] + ) + + # whether this asset is hosted externally + is_external = models.BooleanField( + default=False, + help_text=_("Whether this asset is hosted externally") + ) + + def get_collection(self): + return self.item.collection + + def get_asset_path(self): + return get_asset_path(self.item, self.name) + + +class AssetUpload(BaseAssetUpload): + + class Meta: + constraints = [ + models.UniqueConstraint(fields=['asset', 'upload_id'], name='unique_together'), + # Make sure that there is only one asset upload in progress per asset + models.UniqueConstraint( + fields=['asset', 'status'], + condition=Q(status='in-progress'), + name='unique_in_progress' + ) + ] + triggers = generates_asset_upload_triggers() + + asset = models.ForeignKey(Asset, related_name='+', on_delete=models.CASCADE) + + def update_asset_from_upload(self): + '''Updating the asset's file:checksum and update_interval from the upload + + When the upload is completed, the new file:checksum and update interval from the upload + is set to its asset parent. + ''' + logger.debug( + 'Updating asset %s file:checksum from %s to %s and update_interval from %d to %d ' + 'due to upload complete', + self.asset.name, + self.asset.checksum_multihash, + self.checksum_multihash, + self.asset.update_interval, + self.update_interval, + extra={ + 'upload_id': self.upload_id, + 'asset': self.asset.name, + 'item': self.asset.item.name, + 'collection': self.asset.item.collection.name + } + ) + + self.asset.checksum_multihash = self.checksum_multihash + self.asset.update_interval = self.update_interval + self.asset.file_size = self.file_size + self.asset.save() diff --git a/app/stac_api/s3_multipart_upload.py b/app/stac_api/s3_multipart_upload.py index 14dae036..41950947 100644 --- a/app/stac_api/s3_multipart_upload.py +++ b/app/stac_api/s3_multipart_upload.py @@ -12,8 +12,8 @@ from rest_framework import serializers from stac_api.exceptions import UploadNotInProgressError -from stac_api.models.general import Asset -from stac_api.models.general import CollectionAsset +from stac_api.models.collection import CollectionAsset +from stac_api.models.item import Asset from stac_api.utils import AVAILABLE_S3_BUCKETS from stac_api.utils import get_s3_cache_control_value from stac_api.utils import get_s3_client diff --git a/app/stac_api/sample_data/importer.py b/app/stac_api/sample_data/importer.py index c308f90e..74af6cec 100644 --- a/app/stac_api/sample_data/importer.py +++ b/app/stac_api/sample_data/importer.py @@ -9,11 +9,11 @@ from django.contrib.gis.geos import GEOSGeometry from django.core.files.uploadedfile import SimpleUploadedFile -from stac_api.models.general import Asset -from stac_api.models.general import Collection -from stac_api.models.general import CollectionLink -from stac_api.models.general import Item +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionLink from stac_api.models.general import Provider +from stac_api.models.item import Asset +from stac_api.models.item import Item # path definition relative to the directory that contains manage.py DATADIR = settings.BASE_DIR / 'app/stac_api/management/sample_data/' diff --git a/app/stac_api/serializers/collection.py b/app/stac_api/serializers/collection.py index dc28dc10..406643fe 100644 --- a/app/stac_api/serializers/collection.py +++ b/app/stac_api/serializers/collection.py @@ -4,9 +4,9 @@ from rest_framework import serializers -from stac_api.models.general import Collection -from stac_api.models.general import CollectionAsset -from stac_api.models.general import CollectionLink +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionAsset +from stac_api.models.collection import CollectionLink from stac_api.models.general import Provider from stac_api.serializers.utils import AssetsDictSerializer from stac_api.serializers.utils import HrefField diff --git a/app/stac_api/serializers/item.py b/app/stac_api/serializers/item.py index 93655089..90f299af 100644 --- a/app/stac_api/serializers/item.py +++ b/app/stac_api/serializers/item.py @@ -8,9 +8,9 @@ from rest_framework import serializers from rest_framework_gis import serializers as gis_serializers -from stac_api.models.general import Asset -from stac_api.models.general import Item -from stac_api.models.general import ItemLink +from stac_api.models.item import Asset +from stac_api.models.item import Item +from stac_api.models.item import ItemLink from stac_api.serializers.utils import AssetsDictSerializer from stac_api.serializers.utils import HrefField from stac_api.serializers.utils import IsoDurationField diff --git a/app/stac_api/serializers/upload.py b/app/stac_api/serializers/upload.py index 691f942c..579f2332 100644 --- a/app/stac_api/serializers/upload.py +++ b/app/stac_api/serializers/upload.py @@ -5,8 +5,8 @@ from rest_framework import serializers from rest_framework.utils.serializer_helpers import ReturnDict -from stac_api.models.general import AssetUpload -from stac_api.models.general import CollectionAssetUpload +from stac_api.models.collection import CollectionAssetUpload +from stac_api.models.item import AssetUpload from stac_api.serializers.utils import NonNullModelSerializer from stac_api.utils import is_api_version_1 from stac_api.utils import isoformat diff --git a/app/stac_api/serializers/utils.py b/app/stac_api/serializers/utils.py index 8b54dc9b..650d0bee 100644 --- a/app/stac_api/serializers/utils.py +++ b/app/stac_api/serializers/utils.py @@ -10,9 +10,9 @@ from rest_framework import serializers from rest_framework.utils.serializer_helpers import ReturnDict -from stac_api.models.general import Collection -from stac_api.models.general import Item +from stac_api.models.collection import Collection from stac_api.models.general import Link +from stac_api.models.item import Item from stac_api.utils import build_asset_href from stac_api.utils import get_browser_url from stac_api.utils import get_url diff --git a/app/stac_api/signals.py b/app/stac_api/signals.py index 1e95fa19..ad011ddf 100644 --- a/app/stac_api/signals.py +++ b/app/stac_api/signals.py @@ -4,10 +4,10 @@ from django.db.models.signals import pre_delete from django.dispatch import receiver -from stac_api.models.general import Asset -from stac_api.models.general import AssetUpload -from stac_api.models.general import CollectionAsset -from stac_api.models.general import CollectionAssetUpload +from stac_api.models.collection import CollectionAsset +from stac_api.models.collection import CollectionAssetUpload +from stac_api.models.item import Asset +from stac_api.models.item import AssetUpload logger = logging.getLogger(__name__) diff --git a/app/stac_api/validators_view.py b/app/stac_api/validators_view.py index ce720508..dbfbcbc8 100644 --- a/app/stac_api/validators_view.py +++ b/app/stac_api/validators_view.py @@ -7,10 +7,10 @@ from rest_framework import serializers -from stac_api.models.general import Asset -from stac_api.models.general import Collection -from stac_api.models.general import CollectionAsset -from stac_api.models.general import Item +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionAsset +from stac_api.models.item import Asset +from stac_api.models.item import Item logger = logging.getLogger(__name__) diff --git a/app/stac_api/views/collection.py b/app/stac_api/views/collection.py index 3955de1a..35650356 100644 --- a/app/stac_api/views/collection.py +++ b/app/stac_api/views/collection.py @@ -8,8 +8,8 @@ from rest_framework.response import Response from rest_framework_condition import etag -from stac_api.models.general import Collection -from stac_api.models.general import CollectionAsset +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionAsset from stac_api.serializers.collection import CollectionAssetSerializer from stac_api.serializers.collection import CollectionSerializer from stac_api.serializers.utils import get_relation_links diff --git a/app/stac_api/views/general.py b/app/stac_api/views/general.py index c0192961..cbb6f86c 100644 --- a/app/stac_api/views/general.py +++ b/app/stac_api/views/general.py @@ -14,8 +14,8 @@ from rest_framework.permissions import AllowAny from rest_framework.response import Response -from stac_api.models.general import Item from stac_api.models.general import LandingPage +from stac_api.models.item import Item from stac_api.pagination import GetPostCursorPagination from stac_api.serializers.general import ConformancePageSerializer from stac_api.serializers.general import LandingPageSerializer diff --git a/app/stac_api/views/item.py b/app/stac_api/views/item.py index 4e1de5ff..5bbdd590 100644 --- a/app/stac_api/views/item.py +++ b/app/stac_api/views/item.py @@ -12,9 +12,9 @@ from rest_framework.response import Response from rest_framework_condition import etag -from stac_api.models.general import Asset -from stac_api.models.general import Collection -from stac_api.models.general import Item +from stac_api.models.collection import Collection +from stac_api.models.item import Asset +from stac_api.models.item import Item from stac_api.serializers.item import AssetSerializer from stac_api.serializers.item import ItemSerializer from stac_api.serializers.utils import get_relation_links diff --git a/app/stac_api/views/upload.py b/app/stac_api/views/upload.py index b8eb1197..ff775179 100644 --- a/app/stac_api/views/upload.py +++ b/app/stac_api/views/upload.py @@ -16,11 +16,11 @@ from stac_api.exceptions import UploadInProgressError from stac_api.exceptions import UploadNotInProgressError -from stac_api.models.general import Asset -from stac_api.models.general import AssetUpload +from stac_api.models.collection import CollectionAsset +from stac_api.models.collection import CollectionAssetUpload from stac_api.models.general import BaseAssetUpload -from stac_api.models.general import CollectionAsset -from stac_api.models.general import CollectionAssetUpload +from stac_api.models.item import Asset +from stac_api.models.item import AssetUpload from stac_api.pagination import ExtApiPagination from stac_api.s3_multipart_upload import MultipartUpload from stac_api.serializers.upload import AssetUploadPartsSerializer diff --git a/app/tests/base_test_admin_page.py b/app/tests/base_test_admin_page.py index 07e65351..0e137ca6 100644 --- a/app/tests/base_test_admin_page.py +++ b/app/tests/base_test_admin_page.py @@ -7,12 +7,12 @@ from django.test import TestCase from django.urls import reverse -from stac_api.models.general import Asset -from stac_api.models.general import Collection -from stac_api.models.general import CollectionLink -from stac_api.models.general import Item -from stac_api.models.general import ItemLink +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionLink from stac_api.models.general import Provider +from stac_api.models.item import Asset +from stac_api.models.item import Item +from stac_api.models.item import ItemLink from tests.tests_09.data_factory import Factory diff --git a/app/tests/test_admin_page.py b/app/tests/test_admin_page.py index e2570e6b..fff0b77c 100644 --- a/app/tests/test_admin_page.py +++ b/app/tests/test_admin_page.py @@ -5,12 +5,12 @@ from django.test import override_settings from django.urls import reverse -from stac_api.models.general import Asset -from stac_api.models.general import Collection -from stac_api.models.general import CollectionLink -from stac_api.models.general import Item -from stac_api.models.general import ItemLink +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionLink from stac_api.models.general import Provider +from stac_api.models.item import Asset +from stac_api.models.item import Item +from stac_api.models.item import ItemLink from stac_api.utils import parse_multihash from tests.base_test_admin_page import AdminBaseTestCase diff --git a/app/tests/tests_09/data_factory.py b/app/tests/tests_09/data_factory.py index 57f949a2..f21af465 100644 --- a/app/tests/tests_09/data_factory.py +++ b/app/tests/tests_09/data_factory.py @@ -97,12 +97,12 @@ from django.core.files.base import File from django.core.files.uploadedfile import SimpleUploadedFile -from stac_api.models.general import Asset -from stac_api.models.general import Collection -from stac_api.models.general import CollectionLink -from stac_api.models.general import Item -from stac_api.models.general import ItemLink +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionLink from stac_api.models.general import Provider +from stac_api.models.item import Asset +from stac_api.models.item import Item +from stac_api.models.item import ItemLink from stac_api.utils import get_s3_resource from stac_api.utils import isoformat from stac_api.validators import get_media_type diff --git a/app/tests/tests_09/test_asset_model.py b/app/tests/tests_09/test_asset_model.py index 4c52bf46..587caa98 100644 --- a/app/tests/tests_09/test_asset_model.py +++ b/app/tests/tests_09/test_asset_model.py @@ -2,7 +2,7 @@ from django.core.exceptions import ValidationError -from stac_api.models.general import Asset +from stac_api.models.item import Asset from tests.tests_09.base_test import StacBaseTransactionTestCase from tests.tests_09.data_factory import Factory diff --git a/app/tests/tests_09/test_asset_upload_endpoint.py b/app/tests/tests_09/test_asset_upload_endpoint.py index bc50210a..ca7d2b08 100644 --- a/app/tests/tests_09/test_asset_upload_endpoint.py +++ b/app/tests/tests_09/test_asset_upload_endpoint.py @@ -10,8 +10,8 @@ from django.contrib.auth import get_user_model from django.test import Client -from stac_api.models.general import Asset -from stac_api.models.general import AssetUpload +from stac_api.models.item import Asset +from stac_api.models.item import AssetUpload from stac_api.utils import fromisoformat from stac_api.utils import get_asset_path from stac_api.utils import get_s3_client diff --git a/app/tests/tests_09/test_asset_upload_model.py b/app/tests/tests_09/test_asset_upload_model.py index 1c84e291..c6b874c3 100644 --- a/app/tests/tests_09/test_asset_upload_model.py +++ b/app/tests/tests_09/test_asset_upload_model.py @@ -6,8 +6,8 @@ from django.test import TestCase from django.test import TransactionTestCase -from stac_api.models.general import Asset -from stac_api.models.general import AssetUpload +from stac_api.models.item import Asset +from stac_api.models.item import AssetUpload from stac_api.utils import get_sha256_multihash from stac_api.utils import utc_aware diff --git a/app/tests/tests_09/test_assets_endpoint.py b/app/tests/tests_09/test_assets_endpoint.py index 980dcebe..a3917a81 100644 --- a/app/tests/tests_09/test_assets_endpoint.py +++ b/app/tests/tests_09/test_assets_endpoint.py @@ -8,7 +8,7 @@ from django.test import Client from django.urls import reverse -from stac_api.models.general import Asset +from stac_api.models.item import Asset from stac_api.utils import get_asset_path from stac_api.utils import utc_aware diff --git a/app/tests/tests_09/test_collection_model.py b/app/tests/tests_09/test_collection_model.py index 1142c0b2..cbeb0bd2 100644 --- a/app/tests/tests_09/test_collection_model.py +++ b/app/tests/tests_09/test_collection_model.py @@ -3,7 +3,7 @@ from django.core.exceptions import ValidationError -from stac_api.models.general import Collection +from stac_api.models.collection import Collection from tests.tests_09.base_test import StacBaseTransactionTestCase from tests.tests_09.data_factory import Factory diff --git a/app/tests/tests_09/test_collections_endpoint.py b/app/tests/tests_09/test_collections_endpoint.py index 8a40150a..39f03b77 100644 --- a/app/tests/tests_09/test_collections_endpoint.py +++ b/app/tests/tests_09/test_collections_endpoint.py @@ -5,8 +5,8 @@ from django.test import Client from django.urls import reverse -from stac_api.models.general import Collection -from stac_api.models.general import CollectionLink +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionLink from stac_api.models.general import Provider from stac_api.utils import utc_aware diff --git a/app/tests/tests_09/test_collections_extent.py b/app/tests/tests_09/test_collections_extent.py index d25ff329..09691f8b 100644 --- a/app/tests/tests_09/test_collections_extent.py +++ b/app/tests/tests_09/test_collections_extent.py @@ -4,7 +4,7 @@ from django.contrib.gis.geos import GEOSGeometry from django.contrib.gis.geos import Polygon -from stac_api.models.general import Item +from stac_api.models.item import Item from stac_api.utils import utc_aware from tests.tests_09.base_test import StacBaseTransactionTestCase diff --git a/app/tests/tests_09/test_generic_api.py b/app/tests/tests_09/test_generic_api.py index c8252897..f9345510 100644 --- a/app/tests/tests_09/test_generic_api.py +++ b/app/tests/tests_09/test_generic_api.py @@ -5,7 +5,7 @@ from django.test import Client from django.test import override_settings -from stac_api.models.general import AssetUpload +from stac_api.models.item import AssetUpload from stac_api.utils import get_asset_path from stac_api.utils import get_link from stac_api.utils import get_sha256_multihash diff --git a/app/tests/tests_09/test_item_model.py b/app/tests/tests_09/test_item_model.py index 3f84a128..ad7d4e2e 100644 --- a/app/tests/tests_09/test_item_model.py +++ b/app/tests/tests_09/test_item_model.py @@ -6,8 +6,8 @@ from django.core.exceptions import ValidationError from django.test import TestCase -from stac_api.models.general import Collection -from stac_api.models.general import Item +from stac_api.models.collection import Collection +from stac_api.models.item import Item from stac_api.utils import utc_aware from tests.tests_09.data_factory import CollectionFactory diff --git a/app/tests/tests_09/test_items_endpoint.py b/app/tests/tests_09/test_items_endpoint.py index 37640c0c..1924d54f 100644 --- a/app/tests/tests_09/test_items_endpoint.py +++ b/app/tests/tests_09/test_items_endpoint.py @@ -6,7 +6,7 @@ from django.test import Client from django.urls import reverse -from stac_api.models.general import Item +from stac_api.models.item import Item from stac_api.utils import fromisoformat from stac_api.utils import get_link from stac_api.utils import isoformat diff --git a/app/tests/tests_09/test_serializer.py b/app/tests/tests_09/test_serializer.py index c6386104..09d0f426 100644 --- a/app/tests/tests_09/test_serializer.py +++ b/app/tests/tests_09/test_serializer.py @@ -12,10 +12,10 @@ from rest_framework.renderers import JSONRenderer from rest_framework.test import APIRequestFactory -from stac_api.models.general import get_asset_path from stac_api.serializers.collection import CollectionSerializer from stac_api.serializers.item import AssetSerializer from stac_api.serializers.item import ItemSerializer +from stac_api.utils import get_asset_path from stac_api.utils import get_link from stac_api.utils import isoformat from stac_api.utils import utc_aware diff --git a/app/tests/tests_09/test_serializer_asset_upload.py b/app/tests/tests_09/test_serializer_asset_upload.py index 4dcb25a2..9f11bc72 100644 --- a/app/tests/tests_09/test_serializer_asset_upload.py +++ b/app/tests/tests_09/test_serializer_asset_upload.py @@ -6,7 +6,7 @@ from rest_framework import serializers from rest_framework.exceptions import ValidationError -from stac_api.models.general import AssetUpload +from stac_api.models.item import AssetUpload from stac_api.serializers.upload import AssetUploadSerializer from stac_api.utils import get_sha256_multihash diff --git a/app/tests/tests_10/data_factory.py b/app/tests/tests_10/data_factory.py index 38cbd4bf..de59878f 100644 --- a/app/tests/tests_10/data_factory.py +++ b/app/tests/tests_10/data_factory.py @@ -97,13 +97,13 @@ from django.core.files.base import File from django.core.files.uploadedfile import SimpleUploadedFile -from stac_api.models.general import Asset -from stac_api.models.general import Collection -from stac_api.models.general import CollectionAsset -from stac_api.models.general import CollectionLink -from stac_api.models.general import Item -from stac_api.models.general import ItemLink +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionAsset +from stac_api.models.collection import CollectionLink from stac_api.models.general import Provider +from stac_api.models.item import Asset +from stac_api.models.item import Item +from stac_api.models.item import ItemLink from stac_api.utils import get_s3_resource from stac_api.utils import isoformat from stac_api.validators import get_media_type diff --git a/app/tests/tests_10/test_asset_model.py b/app/tests/tests_10/test_asset_model.py index f424dd66..757dc3ba 100644 --- a/app/tests/tests_10/test_asset_model.py +++ b/app/tests/tests_10/test_asset_model.py @@ -2,7 +2,7 @@ from django.core.exceptions import ValidationError -from stac_api.models.general import Asset +from stac_api.models.item import Asset from tests.tests_10.base_test import StacBaseTransactionTestCase from tests.tests_10.data_factory import Factory diff --git a/app/tests/tests_10/test_asset_upload_endpoint.py b/app/tests/tests_10/test_asset_upload_endpoint.py index 91465f5a..8369a7dc 100644 --- a/app/tests/tests_10/test_asset_upload_endpoint.py +++ b/app/tests/tests_10/test_asset_upload_endpoint.py @@ -10,8 +10,8 @@ from django.contrib.auth import get_user_model from django.test import Client -from stac_api.models.general import Asset -from stac_api.models.general import AssetUpload +from stac_api.models.item import Asset +from stac_api.models.item import AssetUpload from stac_api.utils import fromisoformat from stac_api.utils import get_asset_path from stac_api.utils import get_s3_client diff --git a/app/tests/tests_10/test_asset_upload_model.py b/app/tests/tests_10/test_asset_upload_model.py index 21c4dffd..361fba0e 100644 --- a/app/tests/tests_10/test_asset_upload_model.py +++ b/app/tests/tests_10/test_asset_upload_model.py @@ -6,8 +6,8 @@ from django.test import TestCase from django.test import TransactionTestCase -from stac_api.models.general import Asset -from stac_api.models.general import AssetUpload +from stac_api.models.item import Asset +from stac_api.models.item import AssetUpload from stac_api.utils import get_sha256_multihash from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_assets_endpoint.py b/app/tests/tests_10/test_assets_endpoint.py index 987bb470..2c9ab550 100644 --- a/app/tests/tests_10/test_assets_endpoint.py +++ b/app/tests/tests_10/test_assets_endpoint.py @@ -10,7 +10,7 @@ from django.urls import reverse from django.utils import timezone -from stac_api.models.general import Asset +from stac_api.models.item import Asset from stac_api.utils import get_asset_path from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_collection_asset_model.py b/app/tests/tests_10/test_collection_asset_model.py index 9cb8f172..b75366ae 100644 --- a/app/tests/tests_10/test_collection_asset_model.py +++ b/app/tests/tests_10/test_collection_asset_model.py @@ -2,7 +2,7 @@ from django.core.exceptions import ValidationError -from stac_api.models.general import CollectionAsset +from stac_api.models.collection import CollectionAsset from tests.tests_10.base_test import StacBaseTransactionTestCase from tests.tests_10.data_factory import Factory diff --git a/app/tests/tests_10/test_collection_asset_upload_endpoint.py b/app/tests/tests_10/test_collection_asset_upload_endpoint.py index 95078dc7..1d02c26a 100644 --- a/app/tests/tests_10/test_collection_asset_upload_endpoint.py +++ b/app/tests/tests_10/test_collection_asset_upload_endpoint.py @@ -10,8 +10,8 @@ from django.contrib.auth import get_user_model from django.test import Client -from stac_api.models.general import CollectionAsset -from stac_api.models.general import CollectionAssetUpload +from stac_api.models.collection import CollectionAsset +from stac_api.models.collection import CollectionAssetUpload from stac_api.utils import fromisoformat from stac_api.utils import get_collection_asset_path from stac_api.utils import get_s3_client diff --git a/app/tests/tests_10/test_collection_asset_upload_model.py b/app/tests/tests_10/test_collection_asset_upload_model.py index bb6e6c50..63a0cde3 100644 --- a/app/tests/tests_10/test_collection_asset_upload_model.py +++ b/app/tests/tests_10/test_collection_asset_upload_model.py @@ -6,8 +6,8 @@ from django.test import TestCase from django.test import TransactionTestCase -from stac_api.models.general import CollectionAsset -from stac_api.models.general import CollectionAssetUpload +from stac_api.models.collection import CollectionAsset +from stac_api.models.collection import CollectionAssetUpload from stac_api.utils import get_sha256_multihash from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_collection_assets_endpoint.py b/app/tests/tests_10/test_collection_assets_endpoint.py index e782c243..c2ce980d 100644 --- a/app/tests/tests_10/test_collection_assets_endpoint.py +++ b/app/tests/tests_10/test_collection_assets_endpoint.py @@ -8,7 +8,7 @@ from django.test import Client from django.urls import reverse -from stac_api.models.general import CollectionAsset +from stac_api.models.collection import CollectionAsset from stac_api.utils import get_collection_asset_path from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_collection_model.py b/app/tests/tests_10/test_collection_model.py index b6bb6ef4..75998ade 100644 --- a/app/tests/tests_10/test_collection_model.py +++ b/app/tests/tests_10/test_collection_model.py @@ -3,7 +3,7 @@ from django.core.exceptions import ValidationError -from stac_api.models.general import Collection +from stac_api.models.collection import Collection from tests.tests_10.base_test import StacBaseTransactionTestCase from tests.tests_10.data_factory import Factory diff --git a/app/tests/tests_10/test_collections_endpoint.py b/app/tests/tests_10/test_collections_endpoint.py index 5c2108e8..e5a674bd 100644 --- a/app/tests/tests_10/test_collections_endpoint.py +++ b/app/tests/tests_10/test_collections_endpoint.py @@ -6,8 +6,8 @@ from django.test import Client from django.urls import reverse -from stac_api.models.general import Collection -from stac_api.models.general import CollectionLink +from stac_api.models.collection import Collection +from stac_api.models.collection import CollectionLink from stac_api.models.general import Provider from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_collections_extent.py b/app/tests/tests_10/test_collections_extent.py index fc308cef..19b778c0 100644 --- a/app/tests/tests_10/test_collections_extent.py +++ b/app/tests/tests_10/test_collections_extent.py @@ -4,7 +4,7 @@ from django.contrib.gis.geos import GEOSGeometry from django.contrib.gis.geos import Polygon -from stac_api.models.general import Item +from stac_api.models.item import Item from stac_api.utils import utc_aware from tests.tests_10.base_test import StacBaseTransactionTestCase diff --git a/app/tests/tests_10/test_external_assets_endpoint.py b/app/tests/tests_10/test_external_assets_endpoint.py index e8bf2fe4..0711d514 100644 --- a/app/tests/tests_10/test_external_assets_endpoint.py +++ b/app/tests/tests_10/test_external_assets_endpoint.py @@ -3,7 +3,7 @@ from django.conf import settings from django.test import Client -from stac_api.models.general import Asset +from stac_api.models.item import Asset from tests.tests_10.base_test import StacBaseTestCase from tests.tests_10.data_factory import Factory diff --git a/app/tests/tests_10/test_generic_api.py b/app/tests/tests_10/test_generic_api.py index cb183187..d0d42efd 100644 --- a/app/tests/tests_10/test_generic_api.py +++ b/app/tests/tests_10/test_generic_api.py @@ -5,7 +5,7 @@ from django.test import Client from django.test import override_settings -from stac_api.models.general import AssetUpload +from stac_api.models.item import AssetUpload from stac_api.utils import get_asset_path from stac_api.utils import get_link from stac_api.utils import get_sha256_multihash diff --git a/app/tests/tests_10/test_item_model.py b/app/tests/tests_10/test_item_model.py index a7391f69..83776750 100644 --- a/app/tests/tests_10/test_item_model.py +++ b/app/tests/tests_10/test_item_model.py @@ -7,8 +7,8 @@ from django.core.exceptions import ValidationError from django.test import TestCase -from stac_api.models.general import Collection -from stac_api.models.general import Item +from stac_api.models.collection import Collection +from stac_api.models.item import Item from stac_api.utils import utc_aware from tests.tests_10.data_factory import CollectionFactory diff --git a/app/tests/tests_10/test_items_endpoint.py b/app/tests/tests_10/test_items_endpoint.py index cc7ca6db..72dadf19 100644 --- a/app/tests/tests_10/test_items_endpoint.py +++ b/app/tests/tests_10/test_items_endpoint.py @@ -9,9 +9,9 @@ from django.urls import reverse from django.utils import timezone -from stac_api.models.general import Collection -from stac_api.models.general import Item -from stac_api.models.general import ItemLink +from stac_api.models.collection import Collection +from stac_api.models.item import Item +from stac_api.models.item import ItemLink from stac_api.utils import fromisoformat from stac_api.utils import get_link from stac_api.utils import isoformat diff --git a/app/tests/tests_10/test_remove_expired_items.py b/app/tests/tests_10/test_remove_expired_items.py index c7a80c95..cfe1f39f 100644 --- a/app/tests/tests_10/test_remove_expired_items.py +++ b/app/tests/tests_10/test_remove_expired_items.py @@ -5,8 +5,8 @@ from django.test import TestCase from django.utils import timezone -from stac_api.models.general import Asset -from stac_api.models.general import Item +from stac_api.models.item import Asset +from stac_api.models.item import Item from tests.tests_10.data_factory import Factory from tests.utils import mock_s3_asset_file diff --git a/app/tests/tests_10/test_serializer.py b/app/tests/tests_10/test_serializer.py index 68d9aa43..388a350e 100644 --- a/app/tests/tests_10/test_serializer.py +++ b/app/tests/tests_10/test_serializer.py @@ -14,11 +14,11 @@ from rest_framework.renderers import JSONRenderer from rest_framework.test import APIRequestFactory -from stac_api.models.general import get_asset_path from stac_api.serializers.collection import CollectionSerializer from stac_api.serializers.item import AssetSerializer from stac_api.serializers.item import ItemSerializer from stac_api.serializers.item import ItemsPropertiesSerializer +from stac_api.utils import get_asset_path from stac_api.utils import get_link from stac_api.utils import isoformat from stac_api.utils import utc_aware diff --git a/app/tests/tests_10/test_serializer_asset_upload.py b/app/tests/tests_10/test_serializer_asset_upload.py index 34e4ba29..e557c1f6 100644 --- a/app/tests/tests_10/test_serializer_asset_upload.py +++ b/app/tests/tests_10/test_serializer_asset_upload.py @@ -6,7 +6,7 @@ from rest_framework import serializers from rest_framework.exceptions import ValidationError -from stac_api.models.general import AssetUpload +from stac_api.models.item import AssetUpload from stac_api.serializers.upload import AssetUploadSerializer from stac_api.utils import get_sha256_multihash From d3ef360dc337b6770ecb50993da8eb04be5a0313 Mon Sep 17 00:00:00 2001 From: Adrien Kunysz Date: Mon, 20 Jan 2025 14:22:00 +0100 Subject: [PATCH 13/15] PB-1282: document the new authentication method. This also improves the existing documentation: * explain what session authentication is for; * point to relevant RFC for Basic authentication; * be more explicit about the old Token authentication; * add link from /get-token description to the Authentication section. --- spec/static/spec/v1/openapitransactional.yaml | 87 +++++++++++++++++-- spec/transaction/paths.yaml | 7 +- spec/transaction/tags.yaml | 82 +++++++++++++++-- 3 files changed, 162 insertions(+), 14 deletions(-) diff --git a/spec/static/spec/v1/openapitransactional.yaml b/spec/static/spec/v1/openapitransactional.yaml index 4133df4d..348e1137 100644 --- a/spec/static/spec/v1/openapitransactional.yaml +++ b/spec/static/spec/v1/openapitransactional.yaml @@ -2937,20 +2937,27 @@ tags: - [Example](#section/Example) - name: Authentication description: | - All write requests require authentication. There is currently three type of supported authentications: + All write requests require authentication. There are currently four type of supported authentications: * [Session authentication](#section/Session-authentication) * [Basic authentication](#section/Basic-authentication) - * [Token authentication](#section/Token-authentication) + * [Simple Token authentication](#section/Simple-Token-authentication) + * [JSON Web Token authentication](#section/JSON-Web-Token-authentication) ## Session authentication When using the browsable API the user can simply use the admin interface for logging in. - Once logged in, the browsable API can be used to perform write requests. + The service issues a session cookie which the browser can use to authenticate itself + and perform write requests. This authentication method is only intended + for web browsers users of the admin interface. Non-browser clients and + API endpoints are not guaranteed to work with session authentication. ## Basic authentication - The username and password for authentication can be added to every write request the user wants to perform. + The username and password for authentication can be added to every write + request the user wants to perform using the `Basic` HTTP authentication + scheme as described in [RFC 7614](https://datatracker.ietf.org/doc/html/rfc7617). + Here is an example of posting an asset using curl (_username_="MickeyMouse", _password_="I_love_Minnie_Mouse"): ``` @@ -2971,7 +2978,10 @@ tags: ## Token authentication - A user specific token for authentication can be added to every write request the user wants to perform. + A user-specific token for authentication can be added to every write + request the user wants to perform. The token is passed in the + `Authorization` header with the custom `Token` HTTP authentication scheme. + Here is an example of posting an asset using curl: ``` @@ -3000,6 +3010,68 @@ tags: --header 'Content-Type: application/json' \ --data '{"username": "MickeyMouse", "password": "I_love_Minnie_Mouse"}' ``` + + ## JSON Web Token authentication + + The user authenticates with a JSON Web Token (JWT) passed in the + `Authorization` header with the `Bearer` HTTP authentication scheme as + described in + [RFC 6750](https://datatracker.ietf.org/doc/html/rfc6750#section-2.1). + + Here is an example using curl: + + ``` + curl --request POST \ + --url https://data.geoadmin.ch/api/stac/v1/collections/ch.swisstopo.swisstlmregio/items/swisstlmregio-2020/assets \ + --header 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gU21pdGgiLCJpYXQiOjE1MTYyMzkwMjJ9.Q_w2AVguPRU2KskCXwR7ZHl09TQXEntfEA8Jj2_Jyew' \ + --header 'Content-Type: application/json' \ + --data '{ + "id": "fancy_unique_id", + "item": "swisstlmregio-2020", + "title": "My title", + "type": "application/x.filegdb+zip", + "description": "My description", + "proj:epsg": 2056, + "file:checksum": "12200ADEC47F803A8CF1055ED36750B3BA573C79A3AF7DA6D6F5A2AED03EA16AF3BC" + }' + ``` + + Tokens are obtained by requesting them from the + [Amazon Cognito InitiateAuth API](https://docs.aws.amazon.com/cognito-user-identity-pools/latest/APIReference/API_InitiateAuth.html). + + Here is an example using curl and jq: + + ``` + curl --request POST \ + --url https://cognito-idp.eu-central-1.amazonaws.com/ \ + --header 'Content-Type: application/x-amz-json-1.1' \ + --header 'X-Amz-Target: AWSCognitoIdentityProviderService.InitiateAuth' \ + --data '{ + "AuthFlow": "USER_PASSWORD_AUTH", + "AuthParameters": { + "PASSWORD": "I_love_Minnie_Mouse", + "USERNAME": "MickeyMouse" + }, + "ClientId": "CLIENT_ID" + }' | jq -r .AuthenticationResult.AccessToken + ``` + + The `CLIENT_ID` value needs to be substituted for the correct client + identifier which you should receive along with your username and password. + + Notice the response from `InitiateAuth` is a JSON document. The token used + to authenticate against the STAC API is the `AccessToken`. There are cases + where the response will not contain that token (e.g. if the password must + be updated or a second factor is required for authentication). It is the + responsibility of the client to handle these cases. + [AWS provides an SDK](https://aws.amazon.com/developer/tools/) which may + make this easier. + + The access token is only valid for a certain duration (as per + the `AuthenticationResult.ExpiresIn` field in the response). You need to + refresh it periodically, either by obtaining a new JWT or by + [using the refresh token](https://docs.aws.amazon.com/cognito/latest/developerguide/amazon-cognito-user-pools-using-the-refresh-token.html). + The refresh token is normally valid for a longer time period. paths: /: get: @@ -4483,8 +4555,9 @@ paths: post: tags: - Authentication - summary: >- - Request token for token authentication. + summary: Request token for token authentication. + description: >- + Obtain an authentication token for use in other requests. This is only for the `Token` authorization method. See [Authentication](#tag/Authentication) for usage and context. operationId: getToken requestBody: required: true diff --git a/spec/transaction/paths.yaml b/spec/transaction/paths.yaml index 832f222e..0b26d78e 100644 --- a/spec/transaction/paths.yaml +++ b/spec/transaction/paths.yaml @@ -1231,8 +1231,11 @@ paths: post: tags: - Authentication - summary: >- - Request token for token authentication. + summary: Request token for token authentication. + description: >- + Obtain an authentication token for use in other requests. This is + only for the `Token` authorization method. See + [Authentication](#tag/Authentication) for usage and context. operationId: getToken requestBody: required: true diff --git a/spec/transaction/tags.yaml b/spec/transaction/tags.yaml index e1b61184..0ebd0a1b 100644 --- a/spec/transaction/tags.yaml +++ b/spec/transaction/tags.yaml @@ -299,20 +299,27 @@ tags: - name: Authentication description: | - All write requests require authentication. There is currently three type of supported authentications: + All write requests require authentication. There are currently four type of supported authentications: * [Session authentication](#section/Session-authentication) * [Basic authentication](#section/Basic-authentication) - * [Token authentication](#section/Token-authentication) + * [Simple Token authentication](#section/Simple-Token-authentication) + * [JSON Web Token authentication](#section/JSON-Web-Token-authentication) ## Session authentication When using the browsable API the user can simply use the admin interface for logging in. - Once logged in, the browsable API can be used to perform write requests. + The service issues a session cookie which the browser can use to authenticate itself + and perform write requests. This authentication method is only intended + for web browsers users of the admin interface. Non-browser clients and + API endpoints are not guaranteed to work with session authentication. ## Basic authentication - The username and password for authentication can be added to every write request the user wants to perform. + The username and password for authentication can be added to every write + request the user wants to perform using the `Basic` HTTP authentication + scheme as described in [RFC 7614](https://datatracker.ietf.org/doc/html/rfc7617). + Here is an example of posting an asset using curl (_username_="MickeyMouse", _password_="I_love_Minnie_Mouse"): ``` @@ -333,7 +340,10 @@ tags: ## Token authentication - A user specific token for authentication can be added to every write request the user wants to perform. + A user-specific token for authentication can be added to every write + request the user wants to perform. The token is passed in the + `Authorization` header with the custom `Token` HTTP authentication scheme. + Here is an example of posting an asset using curl: ``` @@ -362,3 +372,65 @@ tags: --header 'Content-Type: application/json' \ --data '{"username": "MickeyMouse", "password": "I_love_Minnie_Mouse"}' ``` + + ## JSON Web Token authentication + + The user authenticates with a JSON Web Token (JWT) passed in the + `Authorization` header with the `Bearer` HTTP authentication scheme as + described in + [RFC 6750](https://datatracker.ietf.org/doc/html/rfc6750#section-2.1). + + Here is an example using curl: + + ``` + curl --request POST \ + --url https://data.geoadmin.ch/api/stac/v1/collections/ch.swisstopo.swisstlmregio/items/swisstlmregio-2020/assets \ + --header 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gU21pdGgiLCJpYXQiOjE1MTYyMzkwMjJ9.Q_w2AVguPRU2KskCXwR7ZHl09TQXEntfEA8Jj2_Jyew' \ + --header 'Content-Type: application/json' \ + --data '{ + "id": "fancy_unique_id", + "item": "swisstlmregio-2020", + "title": "My title", + "type": "application/x.filegdb+zip", + "description": "My description", + "proj:epsg": 2056, + "file:checksum": "12200ADEC47F803A8CF1055ED36750B3BA573C79A3AF7DA6D6F5A2AED03EA16AF3BC" + }' + ``` + + Tokens are obtained by requesting them from the + [Amazon Cognito InitiateAuth API](https://docs.aws.amazon.com/cognito-user-identity-pools/latest/APIReference/API_InitiateAuth.html). + + Here is an example using curl and jq: + + ``` + curl --request POST \ + --url https://cognito-idp.eu-central-1.amazonaws.com/ \ + --header 'Content-Type: application/x-amz-json-1.1' \ + --header 'X-Amz-Target: AWSCognitoIdentityProviderService.InitiateAuth' \ + --data '{ + "AuthFlow": "USER_PASSWORD_AUTH", + "AuthParameters": { + "PASSWORD": "I_love_Minnie_Mouse", + "USERNAME": "MickeyMouse" + }, + "ClientId": "CLIENT_ID" + }' | jq -r .AuthenticationResult.AccessToken + ``` + + The `CLIENT_ID` value needs to be substituted for the correct client + identifier which you should receive along with your username and password. + + Notice the response from `InitiateAuth` is a JSON document. The token used + to authenticate against the STAC API is the `AccessToken`. There are cases + where the response will not contain that token (e.g. if the password must + be updated or a second factor is required for authentication). It is the + responsibility of the client to handle these cases. + [AWS provides an SDK](https://aws.amazon.com/developer/tools/) which may + make this easier. + + The access token is only valid for a certain duration (as per + the `AuthenticationResult.ExpiresIn` field in the response). You need to + refresh it periodically, either by obtaining a new JWT or by + [using the refresh token](https://docs.aws.amazon.com/cognito/latest/developerguide/amazon-cognito-user-pools-using-the-refresh-token.html). + The refresh token is normally valid for a longer time period. From 23f4c82a5d242eca6df7a742e1dce5b6c0c72ffa Mon Sep 17 00:00:00 2001 From: Adrien Kunysz Date: Mon, 27 Jan 2025 13:17:56 +0100 Subject: [PATCH 14/15] Appease GitGuardian. The [GitGuardian Bearer token detector](https://docs.gitguardian.com/secrets-detection/secrets-detection-engine/detectors/generics/bearer_token) does not like it when we add a realistic-looking token. The documentation shows a few ways to make it ignore that token. In this change we use the dummy "123456" value. The previous token was a more realistic dummy but a dummy nonetheless. It has never been a valid token and could not be used to access anything. --- spec/static/spec/v1/openapitransactional.yaml | 4 ++-- spec/transaction/tags.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/static/spec/v1/openapitransactional.yaml b/spec/static/spec/v1/openapitransactional.yaml index 348e1137..8bb1273c 100644 --- a/spec/static/spec/v1/openapitransactional.yaml +++ b/spec/static/spec/v1/openapitransactional.yaml @@ -3018,12 +3018,12 @@ tags: described in [RFC 6750](https://datatracker.ietf.org/doc/html/rfc6750#section-2.1). - Here is an example using curl: + Here is an example using curl, assuming the JWT is `123456`: ``` curl --request POST \ --url https://data.geoadmin.ch/api/stac/v1/collections/ch.swisstopo.swisstlmregio/items/swisstlmregio-2020/assets \ - --header 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gU21pdGgiLCJpYXQiOjE1MTYyMzkwMjJ9.Q_w2AVguPRU2KskCXwR7ZHl09TQXEntfEA8Jj2_Jyew' \ + --header 'Authorization: Bearer 123456' \ --header 'Content-Type: application/json' \ --data '{ "id": "fancy_unique_id", diff --git a/spec/transaction/tags.yaml b/spec/transaction/tags.yaml index 0ebd0a1b..7ddbeba0 100644 --- a/spec/transaction/tags.yaml +++ b/spec/transaction/tags.yaml @@ -380,12 +380,12 @@ tags: described in [RFC 6750](https://datatracker.ietf.org/doc/html/rfc6750#section-2.1). - Here is an example using curl: + Here is an example using curl, assuming the JWT is `123456`: ``` curl --request POST \ --url https://data.geoadmin.ch/api/stac/v1/collections/ch.swisstopo.swisstlmregio/items/swisstlmregio-2020/assets \ - --header 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gU21pdGgiLCJpYXQiOjE1MTYyMzkwMjJ9.Q_w2AVguPRU2KskCXwR7ZHl09TQXEntfEA8Jj2_Jyew' \ + --header 'Authorization: Bearer 123456' \ --header 'Content-Type: application/json' \ --data '{ "id": "fancy_unique_id", From e2924a9d8d58268652a6b96183cb4cec2c4d190d Mon Sep 17 00:00:00 2001 From: Adrien Kunysz Date: Mon, 27 Jan 2025 13:47:54 +0100 Subject: [PATCH 15/15] Remove references to old auth methods. This also removes documentation for the /get-token endpoint as it is only relevant for the old Token authentication method. The old authentication methods (Basic and Token) are only supported for v0.9. We do not update the v0.9 spec via the Makefile any more. --- spec/static/spec/v1/openapitransactional.yaml | 119 +----------------- spec/transaction/paths.yaml | 60 --------- spec/transaction/tags.yaml | 63 +--------- 3 files changed, 2 insertions(+), 240 deletions(-) diff --git a/spec/static/spec/v1/openapitransactional.yaml b/spec/static/spec/v1/openapitransactional.yaml index 8bb1273c..13bb09a7 100644 --- a/spec/static/spec/v1/openapitransactional.yaml +++ b/spec/static/spec/v1/openapitransactional.yaml @@ -2937,11 +2937,9 @@ tags: - [Example](#section/Example) - name: Authentication description: | - All write requests require authentication. There are currently four type of supported authentications: + All write requests require authentication. There are currently two types of supported authentications: * [Session authentication](#section/Session-authentication) - * [Basic authentication](#section/Basic-authentication) - * [Simple Token authentication](#section/Simple-Token-authentication) * [JSON Web Token authentication](#section/JSON-Web-Token-authentication) ## Session authentication @@ -2952,65 +2950,6 @@ tags: for web browsers users of the admin interface. Non-browser clients and API endpoints are not guaranteed to work with session authentication. - ## Basic authentication - - The username and password for authentication can be added to every write - request the user wants to perform using the `Basic` HTTP authentication - scheme as described in [RFC 7614](https://datatracker.ietf.org/doc/html/rfc7617). - - Here is an example of posting an asset using curl (_username_="MickeyMouse", _password_="I_love_Minnie_Mouse"): - - ``` - curl --request POST \ - --user MickeyMouse:I_love_Minnie_Mouse \ - --url https://data.geoadmin.ch/api/stac/v1/collections/ch.swisstopo.swisstlmregio/items/swisstlmregio-2020/assets \ - --header 'Content-Type: application/json' \ - --data '{ - "id": "fancy_unique_id", - "item": "swisstlmregio-2020", - "title": "My title", - "type": "application/x.filegdb+zip", - "description": "My description", - "proj:epsg": 2056, - "file:checksum": "12200ADEC47F803A8CF1055ED36750B3BA573C79A3AF7DA6D6F5A2AED03EA16AF3BC" - }' - ``` - - ## Token authentication - - A user-specific token for authentication can be added to every write - request the user wants to perform. The token is passed in the - `Authorization` header with the custom `Token` HTTP authentication scheme. - - Here is an example of posting an asset using curl: - - ``` - curl --request POST \ - --url https://data.geoadmin.ch/api/stac/v1/collections/ch.swisstopo.swisstlmregio/items/swisstlmregio-2020/assets \ - --header 'Authorization: Token ccecf40693bfc52ba090cd46eb7f19e723fe831f' \ - --header 'Content-Type: application/json' \ - --data '{ - "id": "fancy_unique_id", - "item": "swisstlmregio-2020", - "title": "My title", - "type": "application/x.filegdb+zip", - "description": "My description", - "proj:epsg": 2056, - "file:checksum": "12200ADEC47F803A8CF1055ED36750B3BA573C79A3AF7DA6D6F5A2AED03EA16AF3BC" - }' - ``` - - Tokens can either be generated in the admin interface or existing users can perform a POST request - on the get-token endpoint to request a token (also see [Request token for token authentication](#operation/getToken)). - Here is an example using curl: - - ``` - curl --request POST \ - --url https://data.geoadmin.ch/api/stac/get-token \ - --header 'Content-Type: application/json' \ - --data '{"username": "MickeyMouse", "password": "I_love_Minnie_Mouse"}' - ``` - ## JSON Web Token authentication The user authenticates with a JSON Web Token (JWT) passed in the @@ -4549,59 +4488,3 @@ paths: $ref: "#/components/responses/NotFound" "500": $ref: "#/components/responses/ServerError" - /get-token: - servers: - - url: http://data.geo.admin.ch/api/stac/ - post: - tags: - - Authentication - summary: Request token for token authentication. - description: >- - Obtain an authentication token for use in other requests. This is only for the `Token` authorization method. See [Authentication](#tag/Authentication) for usage and context. - operationId: getToken - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - username: - type: string - description: name of user for whom token is requested - password: - type: string - description: password of user for whom token is requested - required: - - username - - password - example: - username: "Mickey Mouse" - password: "I_love_Minnie_Mouse" - responses: - "200": - description: Returns the token for the specified user - content: - application/json: - schema: - type: object - properties: - token: - type: string - description: generated token for specified user - example: - token: ccecf40693bfc52ba090cd46eb7f19e723fe831f - "400": - description: Wrong credentials were provided. - content: - application/json: - schema: - type: object - properties: - code: - type: string - description: - type: string - example: - code: 400 - description: "Unable to log in with provided credentials." diff --git a/spec/transaction/paths.yaml b/spec/transaction/paths.yaml index 0b26d78e..6cc8691d 100644 --- a/spec/transaction/paths.yaml +++ b/spec/transaction/paths.yaml @@ -1223,63 +1223,3 @@ paths: $ref: "../components/responses.yaml#/components/responses/NotFound" "500": $ref: "../components/responses.yaml#/components/responses/ServerError" - - - "/get-token": - servers: - - url: http://data.geo.admin.ch/api/stac/ - post: - tags: - - Authentication - summary: Request token for token authentication. - description: >- - Obtain an authentication token for use in other requests. This is - only for the `Token` authorization method. See - [Authentication](#tag/Authentication) for usage and context. - operationId: getToken - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - username: - type: string - description: name of user for whom token is requested - password: - type: string - description: password of user for whom token is requested - required: - - username - - password - example: - username: "Mickey Mouse" - password: "I_love_Minnie_Mouse" - responses: - "200": - description: Returns the token for the specified user - content: - application/json: - schema: - type: object - properties: - token: - type: string - description: generated token for specified user - example: - token: ccecf40693bfc52ba090cd46eb7f19e723fe831f - "400": - description: Wrong credentials were provided. - content: - application/json: - schema: - type: object - properties: - code: - type: string - description: - type: string - example: - code: 400 - description: "Unable to log in with provided credentials." diff --git a/spec/transaction/tags.yaml b/spec/transaction/tags.yaml index 7ddbeba0..84907c56 100644 --- a/spec/transaction/tags.yaml +++ b/spec/transaction/tags.yaml @@ -299,11 +299,9 @@ tags: - name: Authentication description: | - All write requests require authentication. There are currently four type of supported authentications: + All write requests require authentication. There are currently two types of supported authentications: * [Session authentication](#section/Session-authentication) - * [Basic authentication](#section/Basic-authentication) - * [Simple Token authentication](#section/Simple-Token-authentication) * [JSON Web Token authentication](#section/JSON-Web-Token-authentication) ## Session authentication @@ -314,65 +312,6 @@ tags: for web browsers users of the admin interface. Non-browser clients and API endpoints are not guaranteed to work with session authentication. - ## Basic authentication - - The username and password for authentication can be added to every write - request the user wants to perform using the `Basic` HTTP authentication - scheme as described in [RFC 7614](https://datatracker.ietf.org/doc/html/rfc7617). - - Here is an example of posting an asset using curl (_username_="MickeyMouse", _password_="I_love_Minnie_Mouse"): - - ``` - curl --request POST \ - --user MickeyMouse:I_love_Minnie_Mouse \ - --url https://data.geoadmin.ch/api/stac/v1/collections/ch.swisstopo.swisstlmregio/items/swisstlmregio-2020/assets \ - --header 'Content-Type: application/json' \ - --data '{ - "id": "fancy_unique_id", - "item": "swisstlmregio-2020", - "title": "My title", - "type": "application/x.filegdb+zip", - "description": "My description", - "proj:epsg": 2056, - "file:checksum": "12200ADEC47F803A8CF1055ED36750B3BA573C79A3AF7DA6D6F5A2AED03EA16AF3BC" - }' - ``` - - ## Token authentication - - A user-specific token for authentication can be added to every write - request the user wants to perform. The token is passed in the - `Authorization` header with the custom `Token` HTTP authentication scheme. - - Here is an example of posting an asset using curl: - - ``` - curl --request POST \ - --url https://data.geoadmin.ch/api/stac/v1/collections/ch.swisstopo.swisstlmregio/items/swisstlmregio-2020/assets \ - --header 'Authorization: Token ccecf40693bfc52ba090cd46eb7f19e723fe831f' \ - --header 'Content-Type: application/json' \ - --data '{ - "id": "fancy_unique_id", - "item": "swisstlmregio-2020", - "title": "My title", - "type": "application/x.filegdb+zip", - "description": "My description", - "proj:epsg": 2056, - "file:checksum": "12200ADEC47F803A8CF1055ED36750B3BA573C79A3AF7DA6D6F5A2AED03EA16AF3BC" - }' - ``` - - Tokens can either be generated in the admin interface or existing users can perform a POST request - on the get-token endpoint to request a token (also see [Request token for token authentication](#operation/getToken)). - Here is an example using curl: - - ``` - curl --request POST \ - --url https://data.geoadmin.ch/api/stac/get-token \ - --header 'Content-Type: application/json' \ - --data '{"username": "MickeyMouse", "password": "I_love_Minnie_Mouse"}' - ``` - ## JSON Web Token authentication The user authenticates with a JSON Web Token (JWT) passed in the