diff --git a/app/stac_api/admin.py b/app/stac_api/admin.py
index 7b775043..68ec5a60 100644
--- a/app/stac_api/admin.py
+++ b/app/stac_api/admin.py
@@ -17,17 +17,17 @@
from django.urls import reverse
from django.utils.translation import gettext_lazy as _
-from stac_api.models import BBOX_CH
-from stac_api.models import Asset
-from stac_api.models import AssetUpload
-from stac_api.models import Collection
-from stac_api.models import CollectionAsset
-from stac_api.models import CollectionLink
-from stac_api.models import Item
-from stac_api.models import ItemLink
-from stac_api.models import LandingPage
-from stac_api.models import LandingPageLink
-from stac_api.models import Provider
+from stac_api.models.collection import Collection
+from stac_api.models.collection import CollectionAsset
+from stac_api.models.collection import CollectionLink
+from stac_api.models.general import BBOX_CH
+from stac_api.models.general import LandingPage
+from stac_api.models.general import LandingPageLink
+from stac_api.models.general import Provider
+from stac_api.models.item import Asset
+from stac_api.models.item import AssetUpload
+from stac_api.models.item import Item
+from stac_api.models.item import ItemLink
from stac_api.utils import build_asset_href
from stac_api.utils import get_query_params
from stac_api.validators import validate_href_url
diff --git a/app/stac_api/management/commands/calculate_extent.py b/app/stac_api/management/commands/calculate_extent.py
index f01104e0..b0d257dc 100644
--- a/app/stac_api/management/commands/calculate_extent.py
+++ b/app/stac_api/management/commands/calculate_extent.py
@@ -3,7 +3,7 @@
from django.core.management.base import CommandParser
from django.db import connection
-from stac_api.models import Collection
+from stac_api.models.collection import Collection
from stac_api.utils import CommandHandler
from stac_api.utils import CustomBaseCommand
diff --git a/app/stac_api/management/commands/dummy_asset_upload.py b/app/stac_api/management/commands/dummy_asset_upload.py
index 758f98b8..7232e8c2 100644
--- a/app/stac_api/management/commands/dummy_asset_upload.py
+++ b/app/stac_api/management/commands/dummy_asset_upload.py
@@ -4,9 +4,9 @@
from django.conf import settings
from django.core.management.base import BaseCommand
-from stac_api.models import Asset
-from stac_api.models import AssetUpload
-from stac_api.models import BaseAssetUpload
+from stac_api.models.general import BaseAssetUpload
+from stac_api.models.item import Asset
+from stac_api.models.item import AssetUpload
from stac_api.s3_multipart_upload import MultipartUpload
from stac_api.utils import AVAILABLE_S3_BUCKETS
from stac_api.utils import CommandHandler
diff --git a/app/stac_api/management/commands/dummy_data.py b/app/stac_api/management/commands/dummy_data.py
index bb9d39dc..a69702db 100644
--- a/app/stac_api/management/commands/dummy_data.py
+++ b/app/stac_api/management/commands/dummy_data.py
@@ -13,9 +13,9 @@
from django.core.files.uploadedfile import SimpleUploadedFile
from django.core.management.base import BaseCommand
-from stac_api.models import Asset
-from stac_api.models import Collection
-from stac_api.models import Item
+from stac_api.models.collection import Collection
+from stac_api.models.item import Asset
+from stac_api.models.item import Item
from stac_api.utils import CommandHandler
from stac_api.validators import MEDIA_TYPES
diff --git a/app/stac_api/management/commands/list_asset_uploads.py b/app/stac_api/management/commands/list_asset_uploads.py
index 079ac3ec..24bb58ce 100644
--- a/app/stac_api/management/commands/list_asset_uploads.py
+++ b/app/stac_api/management/commands/list_asset_uploads.py
@@ -4,7 +4,7 @@
from django.core.management.base import BaseCommand
from django.core.serializers.json import DjangoJSONEncoder
-from stac_api.models import AssetUpload
+from stac_api.models.item import AssetUpload
from stac_api.s3_multipart_upload import MultipartUpload
from stac_api.serializers.upload import AssetUploadSerializer
from stac_api.utils import CommandHandler
diff --git a/app/stac_api/management/commands/profile_cursor_paginator.py b/app/stac_api/management/commands/profile_cursor_paginator.py
index c4c7cc97..18c6174c 100644
--- a/app/stac_api/management/commands/profile_cursor_paginator.py
+++ b/app/stac_api/management/commands/profile_cursor_paginator.py
@@ -10,7 +10,7 @@
from rest_framework.request import Request
from rest_framework.test import APIRequestFactory
-from stac_api.models import Item
+from stac_api.models.item import Item
from stac_api.utils import CommandHandler
logger = logging.getLogger(__name__)
diff --git a/app/stac_api/management/commands/profile_item_serializer.py b/app/stac_api/management/commands/profile_item_serializer.py
index 47f2ba79..8439598d 100644
--- a/app/stac_api/management/commands/profile_item_serializer.py
+++ b/app/stac_api/management/commands/profile_item_serializer.py
@@ -8,7 +8,7 @@
from rest_framework.test import APIRequestFactory
-from stac_api.models import Item
+from stac_api.models.item import Item
from stac_api.utils import CommandHandler
logger = logging.getLogger(__name__)
diff --git a/app/stac_api/management/commands/profile_serializer_vs_no_drf.py b/app/stac_api/management/commands/profile_serializer_vs_no_drf.py
index d4b1ddca..28ed2a2d 100644
--- a/app/stac_api/management/commands/profile_serializer_vs_no_drf.py
+++ b/app/stac_api/management/commands/profile_serializer_vs_no_drf.py
@@ -7,7 +7,7 @@
from rest_framework.test import APIRequestFactory
-from stac_api.models import Item
+from stac_api.models.item import Item
from stac_api.utils import CommandHandler
logger = logging.getLogger(__name__)
diff --git a/app/stac_api/management/commands/remove_expired_items.py b/app/stac_api/management/commands/remove_expired_items.py
index d1eb2668..6b99042f 100644
--- a/app/stac_api/management/commands/remove_expired_items.py
+++ b/app/stac_api/management/commands/remove_expired_items.py
@@ -4,9 +4,9 @@
from django.core.management.base import CommandParser
from django.utils import timezone
-from stac_api.models import AssetUpload
-from stac_api.models import BaseAssetUpload
-from stac_api.models import Item
+from stac_api.models.general import BaseAssetUpload
+from stac_api.models.item import AssetUpload
+from stac_api.models.item import Item
from stac_api.utils import CommandHandler
from stac_api.utils import CustomBaseCommand
diff --git a/app/stac_api/management/commands/update_asset_file_size.py b/app/stac_api/management/commands/update_asset_file_size.py
index a4591c99..a243ac0d 100644
--- a/app/stac_api/management/commands/update_asset_file_size.py
+++ b/app/stac_api/management/commands/update_asset_file_size.py
@@ -6,8 +6,8 @@
from django.core.management.base import BaseCommand
from django.core.management.base import CommandParser
-from stac_api.models import Asset
-from stac_api.models import CollectionAsset
+from stac_api.models.collection import CollectionAsset
+from stac_api.models.item import Asset
from stac_api.utils import CommandHandler
from stac_api.utils import get_s3_client
from stac_api.utils import select_s3_bucket
diff --git a/app/stac_api/managers.py b/app/stac_api/managers.py
index 7ee411e5..6075cc0e 100644
--- a/app/stac_api/managers.py
+++ b/app/stac_api/managers.py
@@ -105,6 +105,43 @@ def _filter_by_datetime_range(self, start_datetime, end_datetime):
)
)
+ def filter_by_forecast_reference_datetime(self, date_time):
+ '''Filter a queryset by forecast reference datetime
+
+ Args:
+ date_time:
+ A string containing datetime like "2020-10-28T13:05:10Z"
+
+ Returns:
+ The queryset filtered by date_time
+ '''
+ start, end = self._parse_datetime_query(date_time)
+ if end is not None:
+ return self._filter_by_forecast_reference_datetime_range(start, end)
+ return self.filter(forecast_reference_datetime=start)
+
+ def _filter_by_forecast_reference_datetime_range(self, start_datetime, end_datetime):
+ '''Filter a queryset by forecast reference datetime range
+
+ Helper function of filter_by_forecast_reference_datetime
+
+ Args:
+ start_datetime:
+ A string with the start datetime or ".." to denote open start
+ end_datetime:
+ A string with the end datetime or ".." to denote open end
+ Returns:
+ The queryset filtered by datetime range
+ '''
+ if start_datetime == '..':
+ # open start range
+ return self.filter(forecast_reference_datetime__lte=end_datetime)
+ if end_datetime == '..':
+ # open end range
+ return self.filter(forecast_reference_datetime__gte=start_datetime)
+ # else fixed range
+ return self.filter(forecast_reference_datetime__range=(start_datetime, end_datetime))
+
def _parse_datetime_query(self, date_time):
'''Parse the datetime query as specified in the api-spec.md.
@@ -189,6 +226,54 @@ def filter_by_intersects(self, intersects):
the_geom = GEOSGeometry(intersects)
return self.filter(geometry__intersects=the_geom)
+ def filter_by_forecast_horizon(self, duration):
+ '''Filter by forecast horizon
+
+ Args:
+ duration: string
+ ISO 8601 duration string
+
+ Returns:
+ queryset filtered by forecast horizon
+ '''
+ return self.filter(forecast_horizon=duration)
+
+ def filter_by_forecast_duration(self, duration):
+ '''Filter by forecast duration
+
+ Args:
+ duration: string
+ ISO 8601 duration string
+
+ Returns:
+ queryset filtered by forecast duration
+ '''
+ return self.filter(forecast_duration=duration)
+
+ def filter_by_forecast_variable(self, val):
+ '''Filter by forecast variable
+
+ Args:
+ val: string
+ foracast variable value
+
+ Returns:
+ queryset filtered by forecast variable
+ '''
+ return self.filter(forecast_variable=val)
+
+ def filter_by_forecast_perturbed(self, pert):
+ '''Filter by forecast perturbed
+
+ Args:
+ pert: boolean
+ foracast perturbed
+
+ Returns:
+ queryset filtered by forecast perturbed
+ '''
+ return self.filter(forecast_perturbed=pert)
+
def filter_by_query(self, query):
'''Filter by the query parameter
@@ -236,6 +321,21 @@ def filter_by_item_name(self, item_name_array):
def filter_by_query(self, query):
return self.get_queryset().filter_by_query(query)
+ def filter_by_forecast_reference_datetime(self, date_time):
+ return self.get_queryset().filter_by_forecast_reference_datetime(date_time)
+
+ def filter_by_forecast_horizon(self, duration):
+ return self.get_queryset().filter_by_forecast_horizon(duration)
+
+ def filter_by_forecast_duration(self, duration):
+ return self.get_queryset().filter_by_forecast_duration(duration)
+
+ def filter_by_forecast_variable(self, val):
+ return self.get_queryset().filter_by_forecast_variable(val)
+
+ def filter_by_forecast_perturbed(self, pert):
+ return self.get_queryset().filter_by_forecast_perturbed(pert)
+
class AssetUploadQuerySet(models.QuerySet):
diff --git a/app/stac_api/migrations/0001_initial.py b/app/stac_api/migrations/0001_initial.py
index 529ed67d..9857f8f0 100644
--- a/app/stac_api/migrations/0001_initial.py
+++ b/app/stac_api/migrations/0001_initial.py
@@ -7,7 +7,7 @@
from django.db import migrations
from django.db import models
-import stac_api.models
+import stac_api.models.general
import stac_api.validators
@@ -74,7 +74,7 @@ class Migration(migrations.Migration):
(
'summaries',
models.JSONField(
- default=stac_api.models.get_default_summaries_value,
+ default=stac_api.models.general.get_default_summaries_value,
editable=False,
encoder=django.core.serializers.json.DjangoJSONEncoder
)
@@ -102,7 +102,7 @@ class Migration(migrations.Migration):
'conformsTo',
django.contrib.postgres.fields.ArrayField(
base_field=models.URLField(),
- default=stac_api.models.get_conformance_default_links,
+ default=stac_api.models.general.get_conformance_default_links,
help_text='Comma-separated list of URLs for the value conformsTo',
size=None
)
@@ -335,7 +335,7 @@ class Migration(migrations.Migration):
(
'file',
models.FileField(
- max_length=255, upload_to=stac_api.models.upload_asset_to_path_hook
+ max_length=255, upload_to=stac_api.models.general.upload_asset_to_path_hook
)
),
(
diff --git a/app/stac_api/migrations/0005_auto_20210408_0821.py b/app/stac_api/migrations/0005_auto_20210408_0821.py
index 7ef461a7..a7079354 100644
--- a/app/stac_api/migrations/0005_auto_20210408_0821.py
+++ b/app/stac_api/migrations/0005_auto_20210408_0821.py
@@ -6,7 +6,7 @@
from django.db import migrations
from django.db import models
-import stac_api.models
+import stac_api.models.general
class Migration(migrations.Migration):
@@ -50,7 +50,10 @@ class Migration(migrations.Migration):
('created', models.DateTimeField(auto_now_add=True)),
('ended', models.DateTimeField(blank=True, default=None, null=True)),
('checksum_multihash', models.CharField(max_length=255)),
- ('etag', models.CharField(default=stac_api.models.compute_etag, max_length=56)),
+ (
+ 'etag',
+ models.CharField(default=stac_api.models.general.compute_etag, max_length=56)
+ ),
(
'asset',
models.ForeignKey(
diff --git a/app/stac_api/migrations/0014_auto_20210715_1358.py b/app/stac_api/migrations/0014_auto_20210715_1358.py
index 032244c7..11493eee 100644
--- a/app/stac_api/migrations/0014_auto_20210715_1358.py
+++ b/app/stac_api/migrations/0014_auto_20210715_1358.py
@@ -6,7 +6,7 @@
from django.db import migrations
from django.db import models
-import stac_api.models
+import stac_api.models.general
class Migration(migrations.Migration):
@@ -49,7 +49,7 @@ class Migration(migrations.Migration):
model_name='asset',
name='etag',
field=models.CharField(
- default=stac_api.models.compute_etag, editable=False, max_length=56
+ default=stac_api.models.general.compute_etag, editable=False, max_length=56
),
),
migrations.AlterField(
@@ -61,7 +61,7 @@ class Migration(migrations.Migration):
model_name='collection',
name='etag',
field=models.CharField(
- default=stac_api.models.compute_etag, editable=False, max_length=56
+ default=stac_api.models.general.compute_etag, editable=False, max_length=56
),
),
migrations.AlterField(
@@ -89,7 +89,7 @@ class Migration(migrations.Migration):
model_name='item',
name='etag',
field=models.CharField(
- default=stac_api.models.compute_etag, editable=False, max_length=56
+ default=stac_api.models.general.compute_etag, editable=False, max_length=56
),
),
migrations.AlterField(
diff --git a/app/stac_api/migrations/0032_alter_asset_file.py b/app/stac_api/migrations/0032_alter_asset_file.py
index ed8a336d..76164f48 100644
--- a/app/stac_api/migrations/0032_alter_asset_file.py
+++ b/app/stac_api/migrations/0032_alter_asset_file.py
@@ -2,7 +2,7 @@
from django.db import migrations
-import stac_api.models
+import stac_api.models.general
class Migration(migrations.Migration):
@@ -15,8 +15,8 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='asset',
name='file',
- field=stac_api.models.DynamicStorageFileField(
- max_length=255, upload_to=stac_api.models.upload_asset_to_path_hook
+ field=stac_api.models.general.DynamicStorageFileField(
+ max_length=255, upload_to=stac_api.models.general.upload_asset_to_path_hook
),
),
]
diff --git a/app/stac_api/migrations/0032_delete_conformancepage_landingpage_conformsto_and_more.py b/app/stac_api/migrations/0032_delete_conformancepage_landingpage_conformsto_and_more.py
index ff457375..9a19fccc 100644
--- a/app/stac_api/migrations/0032_delete_conformancepage_landingpage_conformsto_and_more.py
+++ b/app/stac_api/migrations/0032_delete_conformancepage_landingpage_conformsto_and_more.py
@@ -4,7 +4,7 @@
from django.db import migrations
from django.db import models
-import stac_api.models
+import stac_api.models.general
import stac_api.validators
@@ -21,7 +21,7 @@ class Migration(migrations.Migration):
name='conformsTo',
field=django.contrib.postgres.fields.ArrayField(
base_field=models.URLField(),
- default=stac_api.models.get_conformance_default_links,
+ default=stac_api.models.general.get_conformance_default_links,
help_text='Comma-separated list of URLs for the value conformsTo',
size=None
),
diff --git a/app/stac_api/migrations/0036_collectionasset_and_more.py b/app/stac_api/migrations/0036_collectionasset_and_more.py
index efddc860..d69e0a16 100644
--- a/app/stac_api/migrations/0036_collectionasset_and_more.py
+++ b/app/stac_api/migrations/0036_collectionasset_and_more.py
@@ -9,7 +9,7 @@
from django.db import migrations
from django.db import models
-import stac_api.models
+import stac_api.models.general
import stac_api.validators
@@ -34,8 +34,8 @@ class Migration(migrations.Migration):
),
(
'file',
- stac_api.models.DynamicStorageFileField(
- max_length=255, upload_to=stac_api.models.upload_asset_to_path_hook
+ stac_api.models.general.DynamicStorageFileField(
+ max_length=255, upload_to=stac_api.models.general.upload_asset_to_path_hook
)
),
(
@@ -192,7 +192,7 @@ class Migration(migrations.Migration):
(
'etag',
models.CharField(
- default=stac_api.models.compute_etag, editable=False, max_length=56
+ default=stac_api.models.general.compute_etag, editable=False, max_length=56
)
),
(
diff --git a/app/stac_api/migrations/0050_collectionassetupload_and_more.py b/app/stac_api/migrations/0050_collectionassetupload_and_more.py
index 1238a62a..0e01547b 100644
--- a/app/stac_api/migrations/0050_collectionassetupload_and_more.py
+++ b/app/stac_api/migrations/0050_collectionassetupload_and_more.py
@@ -9,7 +9,7 @@
from django.db import migrations
from django.db import models
-import stac_api.models
+import stac_api.models.general
class Migration(migrations.Migration):
@@ -69,7 +69,10 @@ class Migration(migrations.Migration):
('created', models.DateTimeField(auto_now_add=True)),
('ended', models.DateTimeField(blank=True, default=None, null=True)),
('checksum_multihash', models.CharField(max_length=255)),
- ('etag', models.CharField(default=stac_api.models.compute_etag, max_length=56)),
+ (
+ 'etag',
+ models.CharField(default=stac_api.models.general.compute_etag, max_length=56)
+ ),
(
'update_interval',
models.IntegerField(
diff --git a/app/stac_api/migrations/0062_item_item_fc_reference_datetime_idx_and_more.py b/app/stac_api/migrations/0062_item_item_fc_reference_datetime_idx_and_more.py
new file mode 100644
index 00000000..5297a700
--- /dev/null
+++ b/app/stac_api/migrations/0062_item_item_fc_reference_datetime_idx_and_more.py
@@ -0,0 +1,36 @@
+# Generated by Django 5.0.11 on 2025-01-27 08:22
+
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('stac_api', '0061_remove_item_forecast_mode_remove_item_forecast_param_and_more'),
+ ]
+
+ operations = [
+ migrations.AddIndex(
+ model_name='item',
+ index=models.Index(
+ fields=['forecast_reference_datetime'], name='item_fc_reference_datetime_idx'
+ ),
+ ),
+ migrations.AddIndex(
+ model_name='item',
+ index=models.Index(fields=['forecast_horizon'], name='item_fc_horizon_idx'),
+ ),
+ migrations.AddIndex(
+ model_name='item',
+ index=models.Index(fields=['forecast_duration'], name='item_fc_duration_idx'),
+ ),
+ migrations.AddIndex(
+ model_name='item',
+ index=models.Index(fields=['forecast_variable'], name='item_fc_variable_idx'),
+ ),
+ migrations.AddIndex(
+ model_name='item',
+ index=models.Index(fields=['forecast_perturbed'], name='item_fc_perturbed_idx'),
+ ),
+ ]
diff --git a/app/stac_api/models.py b/app/stac_api/models.py
deleted file mode 100644
index 934f613b..00000000
--- a/app/stac_api/models.py
+++ /dev/null
@@ -1,997 +0,0 @@
-import hashlib
-import logging
-import os
-import time
-from uuid import uuid4
-
-from language_tags import tags
-from multihash import encode as multihash_encode
-from multihash import to_hex_string
-
-from django.conf import settings
-from django.contrib.gis.db import models
-from django.contrib.gis.geos import Polygon
-from django.contrib.postgres.fields import ArrayField
-from django.core.exceptions import ValidationError
-from django.core.serializers.json import DjangoJSONEncoder
-from django.core.validators import MaxValueValidator
-from django.core.validators import MinValueValidator
-from django.db.models import Q
-from django.db.models.deletion import ProtectedError
-from django.utils.translation import gettext_lazy as _
-
-from stac_api.managers import AssetUploadManager
-from stac_api.managers import ItemManager
-from stac_api.pgtriggers import SummaryFields
-from stac_api.pgtriggers import child_triggers
-from stac_api.pgtriggers import generates_asset_triggers
-from stac_api.pgtriggers import generates_asset_upload_triggers
-from stac_api.pgtriggers import generates_collection_asset_triggers
-from stac_api.pgtriggers import generates_collection_triggers
-from stac_api.pgtriggers import generates_item_triggers
-from stac_api.pgtriggers import generates_summary_count_triggers
-from stac_api.utils import get_asset_path
-from stac_api.utils import get_collection_asset_path
-from stac_api.utils import select_s3_bucket
-from stac_api.validators import MEDIA_TYPES
-from stac_api.validators import validate_asset_name
-from stac_api.validators import validate_asset_name_with_media_type
-from stac_api.validators import validate_eo_gsd
-from stac_api.validators import validate_geoadmin_variant
-from stac_api.validators import validate_geometry
-from stac_api.validators import validate_item_properties_datetimes
-from stac_api.validators import validate_link_rel
-from stac_api.validators import validate_media_type
-from stac_api.validators import validate_name
-
-logger = logging.getLogger(__name__)
-
-# We use the WGS84 bounding box as defined here:
-# https://epsg.io/2056
-_BBOX_CH = Polygon.from_bbox((5.96, 45.82, 10.49, 47.81))
-_BBOX_CH.srid = 4326
-# equal to
-# 'SRID=4326;POLYGON ((5.96 45.82, 5.96 47.81, 10.49 47.81, 10.49 45.82, 5.96 45.82))'
-BBOX_CH = str(_BBOX_CH)
-
-SEARCH_TEXT_HELP_ITEM = '''
-
- Search Usage:
-
- -
- arg will make a non exact search checking if >arg
- is part of the Item path
-
- -
- Multiple arg can be used, separated by spaces. This will search
- for all elements containing all arguments in their path
-
- -
- "collectionID/itemID" will make an exact search for the specified item.
-
-
- Examples :
-
- -
- Searching for pixelkarte will return all items which have
- pixelkarte as a part of either their collection ID or their item ID
-
- -
- Searching for pixelkarte 2016 4 will return all items
- which have pixelkarte, 2016 AND 4 as part of their collection ID or
- item ID
-
- -
- Searching for "ch.swisstopo.pixelkarte.example/item2016-4-example"
- will yield only this item, if this item exists.
-
-
-
- Search Usage:
-
- -
- arg will make a non exact search checking if arg is part of
- the collection ID
-
- -
- Multiple arg can be used, separated by spaces. This will search for all
- collections ID containing all arguments.
-
- -
- "collectionID" will make an exact search for the specified collection.
-
-
- Examples :
-
- -
- Searching for pixelkarte will return all collections which have
- pixelkarte as a part of their collection ID
-
- -
- Searching for pixelkarte 2016 4 will return all collection
- which have pixelkarte, 2016 AND 4 as part of their collection ID
-
- -
- Searching for ch.swisstopo.pixelkarte.example will yield only this
- collection, if this collection exists. Please note that it would not return
- a collection named ch.swisstopo.pixelkarte.example.2.
-
-
-
NOTE: unpublished collections/items can still be accessed by their path.
"
- )
- # using "name" instead of "id", as "id" has a default meaning in django
- name = models.CharField('id', unique=True, max_length=255, validators=[validate_name])
- created = models.DateTimeField(auto_now_add=True)
- # NOTE: the updated field is automatically updated by stac_api.pgtriggers, we use auto_now_add
- # only for the initial value.
- updated = models.DateTimeField(auto_now_add=True)
- description = models.TextField()
- # Set to true if the extent needs to be recalculated.
- extent_out_of_sync = models.BooleanField(default=False)
- extent_geometry = models.GeometryField(
- default=None, srid=4326, editable=False, blank=True, null=True
- )
- extent_start_datetime = models.DateTimeField(editable=False, null=True, blank=True)
- extent_end_datetime = models.DateTimeField(editable=False, null=True, blank=True)
-
- license = models.CharField(max_length=30) # string
-
- # DEPRECATED: summaries JSON field is not used anymore and not up to date, it will be removed
- # in future. It has been replaced by summaries_proj_epsg, summaries_eo_gsd and
- # summaries_geoadmin_variant
- summaries = models.JSONField(default=dict, encoder=DjangoJSONEncoder, editable=False)
-
- # NOTE: the following summaries_* fields are automatically update by the stac_api.pgtriggers
- summaries_proj_epsg = ArrayField(
- models.IntegerField(), default=list, blank=True, editable=False
- )
- summaries_eo_gsd = ArrayField(models.FloatField(), default=list, blank=True, editable=False)
- summaries_geoadmin_variant = ArrayField(
- models.CharField(max_length=25), default=list, blank=True, editable=False
- )
- summaries_geoadmin_lang = ArrayField(
- models.CharField(max_length=2), default=list, blank=True, editable=False
- )
-
- title = models.CharField(blank=True, null=True, max_length=255)
-
- # NOTE: hidden ETag field, this field is automatically updated by stac_api.pgtriggers
- etag = models.CharField(
- blank=False, null=False, editable=False, max_length=56, default=compute_etag
- )
-
- update_interval = models.IntegerField(
- default=-1,
- null=False,
- blank=False,
- validators=[MinValueValidator(-1)],
- help_text="Minimal update interval in seconds "
- "in which the underlying assets data are updated."
- )
-
- total_data_size = models.BigIntegerField(default=0, null=True, blank=True)
-
- allow_external_assets = models.BooleanField(
- default=False,
- help_text=_('Whether this collection can have assets that are hosted externally')
- )
-
- external_asset_whitelist = ArrayField(
- models.CharField(max_length=255), blank=True, default=list,
- help_text=_('Provide a comma separated list of '
- 'protocol://domain values for the external asset url validation')
- )
-
- def __str__(self):
- return self.name
-
-
-class CollectionLink(Link):
- collection = models.ForeignKey(
- Collection, related_name='links', related_query_name='link', on_delete=models.CASCADE
- )
-
- class Meta:
- ordering = ['pk']
- triggers = child_triggers('collection', 'CollectionLink')
-
-
-ITEM_KEEP_ORIGINAL_FIELDS = [
- 'geometry',
- 'properties_datetime',
- 'properties_start_datetime',
- 'properties_end_datetime',
-]
-
-
-class Item(models.Model):
-
- class Meta:
- unique_together = (('collection', 'name'),)
- indexes = [
- models.Index(fields=['name'], name='item_name_idx'),
- # the following 3 indices are used e.g. in collection_temporal_extent
- models.Index(fields=['properties_datetime'], name='item_datetime_idx'),
- models.Index(fields=['properties_start_datetime'], name='item_start_datetime_idx'),
- models.Index(fields=['properties_end_datetime'], name='item_end_datetime_idx'),
- # created, updated, and title are "queryable" in the search endpoint
- # see: views.py:322 and 323
- models.Index(fields=['created'], name='item_created_idx'),
- models.Index(fields=['updated'], name='item_updated_idx'),
- models.Index(fields=['properties_title'], name='item_title_idx'),
- # combination of datetime and start_ and end_datetimes are used in
- # managers.py:110 and following
- models.Index(
- fields=[
- 'properties_datetime', 'properties_start_datetime', 'properties_end_datetime'
- ],
- name='item_dttme_start_end_dttm_idx'
- ),
- models.Index(fields=['update_interval'], name='item_update_interval_idx'),
- ]
- triggers = generates_item_triggers()
-
- name = models.CharField('id', blank=False, max_length=255, validators=[validate_name])
- collection = models.ForeignKey(
- Collection, on_delete=models.PROTECT, help_text=_(SEARCH_TEXT_HELP_COLLECTION)
- )
- geometry = models.GeometryField(
- null=False, blank=False, default=BBOX_CH, srid=4326, validators=[validate_geometry]
- )
- created = models.DateTimeField(auto_now_add=True)
- # NOTE: the updated field is automatically updated by stac_api.pgtriggers, we use auto_now_add
- # only for the initial value.
- updated = models.DateTimeField(auto_now_add=True)
- # after discussion with Chris and Tobias: for the moment only support
- # proterties: datetime and title (the rest is hence commented out)
- properties_datetime = models.DateTimeField(
- blank=True,
- null=True,
- help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format"
- )
- properties_start_datetime = models.DateTimeField(
- blank=True,
- null=True,
- help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format"
- )
- properties_end_datetime = models.DateTimeField(
- blank=True,
- null=True,
- help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format"
- )
- properties_expires = models.DateTimeField(
- blank=True,
- null=True,
- help_text="Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format"
- )
- # properties_eo_bands = model.TextFields(blank=True) # ? [string]?
- # properties_eo_cloud_cover = models.FloatField(blank=True)
- # eo_gsd is defined on asset level and will be updated here on ever
- # update of an asset inside this item.
- # properties_instruments = models.TextField(blank=True)
- # properties_license = models.TextField(blank=True)
- # properties_platform = models.TextField(blank=True)
- # properties_providers = models.ManyToManyField(Provider)
- # Although it is discouraged by Django to set blank=True and null=True on a CharField, it is
- # here required because this field is optional and having an empty string value is not permitted
- # in the serializer and default to None. None value are then not displayed in the serializer.
- properties_title = models.CharField(blank=True, null=True, max_length=255)
-
- # properties_view_off_nadir = models.FloatField(blank=True)
- # properties_view_sun_azimuth = models.FloatField(blank=True)
- # properties_view_elevation = models.FloatField(blank=True)
-
- # NOTE: hidden ETag field, this field is automatically updated by stac_api.pgtriggers
- etag = models.CharField(
- blank=False, null=False, editable=False, max_length=56, default=compute_etag
- )
-
- update_interval = models.IntegerField(
- default=-1,
- null=False,
- blank=False,
- validators=[MinValueValidator(-1)],
- help_text="Minimal update interval in seconds "
- "in which the underlying assets data are updated."
- )
-
- total_data_size = models.BigIntegerField(default=0, null=True, blank=True)
-
- forecast_reference_datetime = models.DateTimeField(
- null=True,
- blank=True,
- help_text="The reference datetime: i.e. predictions for times after "
- "this point occur in the future. Predictions prior to this "
- "time represent 'hindcasts', predicting states that have "
- "already occurred. This must be in UTC. It is formatted like "
- "'2022-08-12T00:00:00Z'."
- )
-
- forecast_horizon = models.DurationField(
- null=True,
- blank=True,
- help_text="The time between the reference datetime and the forecast datetime."
- "Formatted as ISO 8601 duration, e.g. 'PT6H' for a 6-hour forecast.",
- )
-
- forecast_duration = models.DurationField(
- null=True,
- blank=True,
- help_text="If the forecast is not only for a specific instance in time "
- "but instead is for a certain period, you can specify the "
- "length here. Formatted as ISO 8601 duration, e.g. 'PT3H' for a 3-hour "
- "accumulation. If not given, assumes that the forecast is for an "
- "instance in time as if this was set to PT0S (0 seconds).",
- )
-
- forecast_variable = models.CharField(
- null=True,
- blank=True,
- help_text="Name of the model variable that corresponds to the data. The variables "
- "should correspond to the CF Standard Names, "
- "e.g. `air_temperature` for the air temperature."
- )
-
- forecast_perturbed = models.BooleanField(
- null=True,
- blank=True,
- default=None,
- help_text="Denotes whether the data corresponds to the control run (`false`) or "
- "perturbed runs (`true`). The property needs to be specified in both "
- "cases as no default value is specified and as such the meaning is "
- "\"unknown\" in case it's missing."
- )
-
- # Custom Manager that preselects the collection
- objects = ItemManager()
-
- def __str__(self):
- # This is used in the admin page in the autocomplete_fields of the Asset page
- return f"{self.collection.name}/{self.name}"
-
- def clean(self):
- validate_item_properties_datetimes(
- self.properties_datetime,
- self.properties_start_datetime,
- self.properties_end_datetime,
- self.properties_expires,
- )
-
-
-class ItemLink(Link):
- item = models.ForeignKey(
- Item, related_name='links', related_query_name='link', on_delete=models.CASCADE
- )
-
- class Meta:
- unique_together = (('rel', 'item'),)
- ordering = ['pk']
- triggers = child_triggers('item', 'ItemLink')
-
-
-def upload_asset_to_path_hook(instance, filename=None):
- '''This returns the asset upload path on S3 and compute the asset file multihash
-
- Args:
- instance: Asset
- Asset instance
- filename: string
- file name of the uploaded asset
-
- Returns:
- Asset file path to use on S3
- '''
- item_name = 'no item on collection asset'
- if hasattr(instance, 'item'):
- item_name = instance.item.name
- logger.debug(
- 'Start computing asset file %s multihash (file size: %.1f MB)',
- filename,
- instance.file.size / 1024**2,
- extra={
- "collection": instance.get_collection().name, "item": item_name, "asset": instance.name
- }
- )
- start = time.time()
- ctx = hashlib.sha256()
- for chunk in instance.file.chunks(settings.UPLOAD_FILE_CHUNK_SIZE):
- ctx.update(chunk)
- mhash = to_hex_string(multihash_encode(ctx.digest(), 'sha2-256'))
- # set the hash to the storage to use it for upload signing, this temporary attribute is
- # then used by storages.S3Storage to set the MetaData.sha256
- instance.file.storage.object_sha256 = ctx.hexdigest()
- # Same here for the update_interval that is used by the storages.S3Storage to set the asset's
- # update_interval
- instance.file.storage.update_interval = instance.update_interval
- logger.debug(
- 'Set uploaded file %s multihash %s to file:checksum; computation done in %.3fs',
- filename,
- mhash,
- time.time() - start,
- extra={
- "collection": instance.get_collection().name, "item": item_name, "asset": instance.name
- }
- )
- instance.checksum_multihash = mhash
- instance.file_size = instance.file.size
- return instance.get_asset_path()
-
-
-class DynamicStorageFileField(models.FileField):
-
- def pre_save(self, model_instance: "AssetBase", add):
- """Determine the storage to use for this file
-
- The storage is determined by the collection's name. See
- settings.MANAGED_BUCKET_COLLECTION_PATTERNS
- """
- collection = model_instance.get_collection()
-
- bucket = select_s3_bucket(collection.name).name
-
- # We need to explicitly instantiate the storage backend here
- # Since the backends are configured as strings in the settings, we take these strings
- # and import them by those string
- # Example is stac_api.storages.LegacyS3Storage
- parts = settings.STORAGES[bucket]['BACKEND'].split(".")
-
- # join the first two parts of the module name together -> stac_api.storages
- storage_module_name = ".".join(parts[:-1])
-
- # the name of the storage class is the last part -> LegacyS3Storage
- storage_cls_name = parts[-1:]
-
- # import the module
- storage_module = __import__(storage_module_name, fromlist=[parts[-2:-1]])
-
- # get the class from the module
- storage_cls = getattr(storage_module, storage_cls_name[0])
-
- # .. and instantiate!
- self.storage = storage_cls()
-
- # we need to specify the storage for the actual
- # file as well
- model_instance.file.storage = self.storage
-
- self.storage.asset_content_type = model_instance.media_type
-
- return super().pre_save(model_instance, add)
-
-
-class AssetBase(models.Model):
-
- class Meta:
- abstract = True
-
- # using BigIntegerField as primary_key to deal with the expected large number of assets.
- id = models.BigAutoField(primary_key=True)
-
- # using "name" instead of "id", as "id" has a default meaning in django
- name = models.CharField('id', max_length=255, validators=[validate_asset_name])
-
- # Disable weird pylint errors, where it doesn't take the inherited constructor
- # into account when linting, somehow
- # pylint: disable=unexpected-keyword-arg
- # pylint: disable=no-value-for-parameter
- file = DynamicStorageFileField(upload_to=upload_asset_to_path_hook, max_length=255)
- roles = ArrayField(
- models.CharField(max_length=255), editable=True, blank=True, null=True, default=None,
- help_text=_("Comma-separated list of roles to describe the purpose of the asset"))
-
- @property
- def filename(self):
- return os.path.basename(self.file.name)
-
- # From v1 on the json representation of this field changed from "checksum:multihash" to
- # "file:checksum". The two names may be used interchangeably for a now.
- checksum_multihash = models.CharField(
- editable=False, max_length=255, blank=True, null=True, default=None
- )
- # here we need to set blank=True otherwise the field is as required in the admin interface
- description = models.TextField(blank=True, null=True, default=None)
-
- proj_epsg = models.IntegerField(null=True, blank=True)
- # here we need to set blank=True otherwise the field is as required in the admin interface
- title = models.CharField(max_length=255, null=True, blank=True)
- media_choices = [
- (x.media_type_str, f'{x.description} ({x.media_type_str})') for x in MEDIA_TYPES
- ]
- media_type = models.CharField(
- choices=media_choices,
- max_length=200,
- blank=False,
- null=False,
- help_text=
- "This media type will be used as Content-Type header for the asset's object upon "
- "upload."
- "WARNING: when updating the Media Type, the asset's object Content-Type header is not "
- "automatically updated, it needs to be uploaded again."
- )
-
- created = models.DateTimeField(auto_now_add=True)
- # NOTE: the updated field is automatically updated by stac_api.pgtriggers, we use auto_now_add
- # only for the initial value.
- updated = models.DateTimeField(auto_now_add=True)
-
- # NOTE: hidden ETag field, this field is automatically updated by stac_api.pgtriggers
- etag = models.CharField(
- blank=False, null=False, editable=False, max_length=56, default=compute_etag
- )
-
- update_interval = models.IntegerField(
- default=-1,
- null=False,
- blank=False,
- validators=[MinValueValidator(-1)],
- help_text="Interval in seconds in which the asset data is updated."
- "-1 means that the data is not on a regular basis updated."
- )
-
- file_size = models.BigIntegerField(default=0, null=True, blank=True)
-
- def __str__(self):
- return self.name
-
- def delete(self, *args, **kwargs): # pylint: disable=signature-differs
- try:
- super().delete(*args, **kwargs)
- except ProtectedError as error:
- logger.error('Cannot delete asset %s: %s', self.name, error)
- raise ValidationError(error.args[0]) from None
-
- def clean(self):
- # Although the media type is already validated, it still needs to be validated a second
- # time as the clean method is run even if the field validation failed and there is no way
- # to check what errors were already raised.
- media_type = validate_media_type(self.media_type)
- validate_asset_name_with_media_type(self.name, media_type)
-
-
-class Asset(AssetBase):
-
- class Meta:
- unique_together = (('item', 'name'),)
- ordering = ['id']
- triggers = generates_asset_triggers()
-
- item = models.ForeignKey(
- Item,
- related_name='assets',
- related_query_name='asset',
- on_delete=models.PROTECT,
- help_text=_(SEARCH_TEXT_HELP_ITEM)
- )
- # From v1 on the json representation of this field changed from "eo:gsd" to "gsd". The two names
- # may be used interchangeably for a now.
- eo_gsd = models.FloatField(null=True, blank=True, validators=[validate_eo_gsd])
-
- class Language(models.TextChoices):
- # pylint: disable=invalid-name
- GERMAN = 'de', _('German')
- ITALIAN = 'it', _('Italian')
- FRENCH = 'fr', _('French')
- ROMANSH = 'rm', _('Romansh')
- ENGLISH = 'en', _('English')
- __empty__ = _('')
-
- # here we need to set blank=True otherwise the field is as required in the admin interface
- geoadmin_lang = models.CharField(
- max_length=2, choices=Language.choices, default=None, null=True, blank=True
- )
- # here we need to set blank=True otherwise the field is as required in the admin interface
- geoadmin_variant = models.CharField(
- max_length=25, null=True, blank=True, validators=[validate_geoadmin_variant]
- )
-
- # whether this asset is hosted externally
- is_external = models.BooleanField(
- default=False,
- help_text=_("Whether this asset is hosted externally")
- )
-
- def get_collection(self):
- return self.item.collection
-
- def get_asset_path(self):
- return get_asset_path(self.item, self.name)
-
-
-class CollectionAsset(AssetBase):
-
- class Meta:
- unique_together = (('collection', 'name'),)
- ordering = ['id']
- triggers = generates_collection_asset_triggers()
-
- collection = models.ForeignKey(
- Collection,
- related_name='assets',
- related_query_name='asset',
- on_delete=models.PROTECT,
- help_text=_(SEARCH_TEXT_HELP_ITEM)
- )
-
- # CollectionAssets are never external
- is_external = False
-
- def get_collection(self):
- return self.collection
-
- def get_asset_path(self):
- return get_collection_asset_path(self.collection, self.name)
-
-
-class BaseAssetUpload(models.Model):
-
- class Meta:
- abstract = True
-
- class Status(models.TextChoices):
- # pylint: disable=invalid-name
- IN_PROGRESS = 'in-progress'
- COMPLETED = 'completed'
- ABORTED = 'aborted'
- __empty__ = ''
-
- class ContentEncoding(models.TextChoices):
- # pylint: disable=invalid-name
- GZIP = 'gzip'
- BR = 'br'
- # DEFLATE = 'deflate'
- # COMPRESS = 'compress'
- __empty__ = ''
-
- # using BigIntegerField as primary_key to deal with the expected large number of assets.
- id = models.BigAutoField(primary_key=True)
- upload_id = models.CharField(max_length=255, blank=False, null=False)
- status = models.CharField(
- choices=Status.choices, max_length=32, default=Status.IN_PROGRESS, blank=False, null=False
- )
- number_parts = models.IntegerField(
- validators=[MinValueValidator(1), MaxValueValidator(100)], null=False, blank=False
- ) # S3 doesn't support more that 10'000 parts
- md5_parts = models.JSONField(encoder=DjangoJSONEncoder, editable=False)
- urls = models.JSONField(default=list, encoder=DjangoJSONEncoder, blank=True)
- created = models.DateTimeField(auto_now_add=True)
- ended = models.DateTimeField(blank=True, null=True, default=None)
- # From v1 on the json representation of this field changed from "checksum:multihash" to
- # "file:checksum". The two names may be used interchangeably for a now.
- checksum_multihash = models.CharField(max_length=255, blank=False, null=False)
-
- # NOTE: hidden ETag field, this field is automatically updated by stac_api.pgtriggers
- etag = models.CharField(blank=False, null=False, max_length=56, default=compute_etag)
-
- update_interval = models.IntegerField(
- default=-1,
- null=False,
- blank=False,
- validators=[MinValueValidator(-1)],
- help_text="Interval in seconds in which the asset data is updated. "
- "-1 means that the data is not on a regular basis updated. "
- "This field can only be set via the API."
- )
-
- file_size = models.BigIntegerField(default=0, null=True, blank=True)
-
- content_encoding = models.CharField(
- choices=ContentEncoding.choices, blank=True, null=False, max_length=32, default=''
- )
-
- # Custom Manager that preselects the collection
- objects = AssetUploadManager()
-
-
-class AssetUpload(BaseAssetUpload):
-
- class Meta:
- constraints = [
- models.UniqueConstraint(fields=['asset', 'upload_id'], name='unique_together'),
- # Make sure that there is only one asset upload in progress per asset
- models.UniqueConstraint(
- fields=['asset', 'status'],
- condition=Q(status='in-progress'),
- name='unique_in_progress'
- )
- ]
- triggers = generates_asset_upload_triggers()
-
- asset = models.ForeignKey(Asset, related_name='+', on_delete=models.CASCADE)
-
- def update_asset_from_upload(self):
- '''Updating the asset's file:checksum and update_interval from the upload
-
- When the upload is completed, the new file:checksum and update interval from the upload
- is set to its asset parent.
- '''
- logger.debug(
- 'Updating asset %s file:checksum from %s to %s and update_interval from %d to %d '
- 'due to upload complete',
- self.asset.name,
- self.asset.checksum_multihash,
- self.checksum_multihash,
- self.asset.update_interval,
- self.update_interval,
- extra={
- 'upload_id': self.upload_id,
- 'asset': self.asset.name,
- 'item': self.asset.item.name,
- 'collection': self.asset.item.collection.name
- }
- )
-
- self.asset.checksum_multihash = self.checksum_multihash
- self.asset.update_interval = self.update_interval
- self.asset.file_size = self.file_size
- self.asset.save()
-
-
-class CollectionAssetUpload(BaseAssetUpload):
-
- class Meta:
- constraints = [
- models.UniqueConstraint(
- fields=['asset', 'upload_id'],
- name='unique_asset_upload_collection_asset_upload_id'
- ),
- # Make sure that there is only one upload in progress per collection asset
- models.UniqueConstraint(
- fields=['asset', 'status'],
- condition=Q(status='in-progress'),
- name='unique_asset_upload_in_progress'
- )
- ]
- triggers = generates_asset_upload_triggers()
-
- asset = models.ForeignKey(CollectionAsset, related_name='+', on_delete=models.CASCADE)
-
- def update_asset_from_upload(self):
- '''Updating the asset's file:checksum and update_interval from the upload
-
- When the upload is completed, the new file:checksum and update interval from the upload
- is set to its asset parent.
- '''
- logger.debug(
- 'Updating collection asset %s file:checksum from %s to %s and update_interval '
- 'from %d to %d due to upload complete',
- self.asset.name,
- self.asset.checksum_multihash,
- self.checksum_multihash,
- self.asset.update_interval,
- self.update_interval,
- extra={
- 'upload_id': self.upload_id,
- 'asset': self.asset.name,
- 'collection': self.asset.collection.name
- }
- )
-
- self.asset.checksum_multihash = self.checksum_multihash
- self.asset.update_interval = self.update_interval
- self.asset.file_size = self.file_size
- self.asset.save()
-
-
-class CountBase(models.Model):
- '''CountBase tables are used to help calculate the summary on a collection.
- This is only performant if the distinct number of values is small, e.g. we currently only have
- 5 possible values for geoadmin_language.
- For each assets value we keep a count of how often that value exists, per collection. On
- insert/update/delete of an asset we only need to decrease and/or increase the counter value.
- Since the number of possible values is small, the aggregate array calculation to update the
- collection also stays performant.
- '''
-
- class Meta:
- abstract = True
-
- id = models.BigAutoField(primary_key=True)
- collection = models.ForeignKey(
- Collection,
- related_name='+',
- on_delete=models.CASCADE,
- )
- count = models.PositiveIntegerField(null=False)
-
-
-class GSDCount(CountBase):
- # Update by asset triggers.
-
- class Meta:
- unique_together = (('collection', 'value'),)
- ordering = ['id']
- triggers = generates_summary_count_triggers(
- SummaryFields.GSD.value[0], SummaryFields.GSD.value[1]
- )
-
- value = models.FloatField(null=True, blank=True)
-
-
-class GeoadminLangCount(CountBase):
- # Update by asset triggers.
-
- class Meta:
- unique_together = (('collection', 'value'),)
- ordering = ['id']
- triggers = generates_summary_count_triggers(
- SummaryFields.LANGUAGE.value[0], SummaryFields.LANGUAGE.value[1]
- )
-
- value = models.CharField(max_length=2, default=None, null=True, blank=True)
-
-
-class GeoadminVariantCount(CountBase):
- # Update by asset triggers.
-
- class Meta:
- unique_together = (('collection', 'value'),)
- ordering = ['id']
- triggers = generates_summary_count_triggers(
- SummaryFields.VARIANT.value[0], SummaryFields.VARIANT.value[1]
- )
-
- value = models.CharField(max_length=25, null=True, blank=True)
-
-
-class ProjEPSGCount(CountBase):
- # Update by asset and collection asset triggers.
-
- class Meta:
- unique_together = (('collection', 'value'),)
- ordering = ['id']
- triggers = generates_summary_count_triggers(
- SummaryFields.PROJ_EPSG.value[0], SummaryFields.PROJ_EPSG.value[1]
- )
-
- value = models.IntegerField(null=True, blank=True)
diff --git a/app/stac_api/models/__init__.py b/app/stac_api/models/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/app/stac_api/models/collection.py b/app/stac_api/models/collection.py
new file mode 100644
index 00000000..efb0bf3e
--- /dev/null
+++ b/app/stac_api/models/collection.py
@@ -0,0 +1,264 @@
+import logging
+
+from django.contrib.gis.db import models
+from django.contrib.postgres.fields import ArrayField
+from django.core.serializers.json import DjangoJSONEncoder
+from django.core.validators import MinValueValidator
+from django.db.models import Q
+from django.utils.translation import gettext_lazy as _
+
+from stac_api.models.general import SEARCH_TEXT_HELP_ITEM
+from stac_api.models.general import AssetBase
+from stac_api.models.general import BaseAssetUpload
+from stac_api.models.general import Link
+from stac_api.models.general import compute_etag
+from stac_api.pgtriggers import SummaryFields
+from stac_api.pgtriggers import child_triggers
+from stac_api.pgtriggers import generates_asset_upload_triggers
+from stac_api.pgtriggers import generates_collection_asset_triggers
+from stac_api.pgtriggers import generates_collection_triggers
+from stac_api.pgtriggers import generates_summary_count_triggers
+from stac_api.utils import get_collection_asset_path
+from stac_api.validators import validate_name
+
+logger = logging.getLogger(__name__)
+
+
+# For Collections and Items: No primary key will be defined, so that the auto-generated ones
+# will be used by Django. For assets, a primary key is defined as "BigAutoField" due the
+# expected large number of assets
+class Collection(models.Model):
+
+ class Meta:
+ indexes = [
+ models.Index(fields=['name'], name='collection_name_idx'),
+ models.Index(fields=['published'], name='collection_published_idx')
+ ]
+ triggers = generates_collection_triggers()
+
+ published = models.BooleanField(
+ default=True,
+ help_text="When not published the collection doesn't appear on the "
+ "api/stac/v0.9/collections endpoint and its items are not listed in /search endpoint."
+ "NOTE: unpublished collections/items can still be accessed by their path.
"
+ )
+ # using "name" instead of "id", as "id" has a default meaning in django
+ name = models.CharField('id', unique=True, max_length=255, validators=[validate_name])
+ created = models.DateTimeField(auto_now_add=True)
+ # NOTE: the updated field is automatically updated by stac_api.pgtriggers, we use auto_now_add
+ # only for the initial value.
+ updated = models.DateTimeField(auto_now_add=True)
+ description = models.TextField()
+ # Set to true if the extent needs to be recalculated.
+ extent_out_of_sync = models.BooleanField(default=False)
+ extent_geometry = models.GeometryField(
+ default=None, srid=4326, editable=False, blank=True, null=True
+ )
+ extent_start_datetime = models.DateTimeField(editable=False, null=True, blank=True)
+ extent_end_datetime = models.DateTimeField(editable=False, null=True, blank=True)
+
+ license = models.CharField(max_length=30) # string
+
+ # DEPRECATED: summaries JSON field is not used anymore and not up to date, it will be removed
+ # in future. It has been replaced by summaries_proj_epsg, summaries_eo_gsd and
+ # summaries_geoadmin_variant
+ summaries = models.JSONField(default=dict, encoder=DjangoJSONEncoder, editable=False)
+
+ # NOTE: the following summaries_* fields are automatically update by the stac_api.pgtriggers
+ summaries_proj_epsg = ArrayField(
+ models.IntegerField(), default=list, blank=True, editable=False
+ )
+ summaries_eo_gsd = ArrayField(models.FloatField(), default=list, blank=True, editable=False)
+ summaries_geoadmin_variant = ArrayField(
+ models.CharField(max_length=25), default=list, blank=True, editable=False
+ )
+ summaries_geoadmin_lang = ArrayField(
+ models.CharField(max_length=2), default=list, blank=True, editable=False
+ )
+
+ title = models.CharField(blank=True, null=True, max_length=255)
+
+ # NOTE: hidden ETag field, this field is automatically updated by stac_api.pgtriggers
+ etag = models.CharField(
+ blank=False, null=False, editable=False, max_length=56, default=compute_etag
+ )
+
+ update_interval = models.IntegerField(
+ default=-1,
+ null=False,
+ blank=False,
+ validators=[MinValueValidator(-1)],
+ help_text="Minimal update interval in seconds "
+ "in which the underlying assets data are updated."
+ )
+
+ total_data_size = models.BigIntegerField(default=0, null=True, blank=True)
+
+ allow_external_assets = models.BooleanField(
+ default=False,
+ help_text=_('Whether this collection can have assets that are hosted externally')
+ )
+
+ external_asset_whitelist = ArrayField(
+ models.CharField(max_length=255), blank=True, default=list,
+ help_text=_('Provide a comma separated list of '
+ 'protocol://domain values for the external asset url validation')
+ )
+
+ def __str__(self):
+ return self.name
+
+
+class CollectionLink(Link):
+ collection = models.ForeignKey(
+ Collection, related_name='links', related_query_name='link', on_delete=models.CASCADE
+ )
+
+ class Meta:
+ ordering = ['pk']
+ triggers = child_triggers('collection', 'CollectionLink')
+
+
+class CollectionAsset(AssetBase):
+
+ class Meta:
+ unique_together = (('collection', 'name'),)
+ ordering = ['id']
+ triggers = generates_collection_asset_triggers()
+
+ collection = models.ForeignKey(
+ Collection,
+ related_name='assets',
+ related_query_name='asset',
+ on_delete=models.PROTECT,
+ help_text=_(SEARCH_TEXT_HELP_ITEM)
+ )
+
+ # CollectionAssets are never external
+ is_external = False
+
+ def get_collection(self):
+ return self.collection
+
+ def get_asset_path(self):
+ return get_collection_asset_path(self.collection, self.name)
+
+
+class CollectionAssetUpload(BaseAssetUpload):
+
+ class Meta:
+ constraints = [
+ models.UniqueConstraint(
+ fields=['asset', 'upload_id'],
+ name='unique_asset_upload_collection_asset_upload_id'
+ ),
+ # Make sure that there is only one upload in progress per collection asset
+ models.UniqueConstraint(
+ fields=['asset', 'status'],
+ condition=Q(status='in-progress'),
+ name='unique_asset_upload_in_progress'
+ )
+ ]
+ triggers = generates_asset_upload_triggers()
+
+ asset = models.ForeignKey(CollectionAsset, related_name='+', on_delete=models.CASCADE)
+
+ def update_asset_from_upload(self):
+ '''Updating the asset's file:checksum and update_interval from the upload
+
+ When the upload is completed, the new file:checksum and update interval from the upload
+ is set to its asset parent.
+ '''
+ logger.debug(
+ 'Updating collection asset %s file:checksum from %s to %s and update_interval '
+ 'from %d to %d due to upload complete',
+ self.asset.name,
+ self.asset.checksum_multihash,
+ self.checksum_multihash,
+ self.asset.update_interval,
+ self.update_interval,
+ extra={
+ 'upload_id': self.upload_id,
+ 'asset': self.asset.name,
+ 'collection': self.asset.collection.name
+ }
+ )
+
+ self.asset.checksum_multihash = self.checksum_multihash
+ self.asset.update_interval = self.update_interval
+ self.asset.file_size = self.file_size
+ self.asset.save()
+
+
+class CountBase(models.Model):
+ '''CountBase tables are used to help calculate the summary on a collection.
+ This is only performant if the distinct number of values is small, e.g. we currently only have
+ 5 possible values for geoadmin_language.
+ For each assets value we keep a count of how often that value exists, per collection. On
+ insert/update/delete of an asset we only need to decrease and/or increase the counter value.
+ Since the number of possible values is small, the aggregate array calculation to update the
+ collection also stays performant.
+ '''
+
+ class Meta:
+ abstract = True
+
+ id = models.BigAutoField(primary_key=True)
+ collection = models.ForeignKey(
+ Collection,
+ related_name='+',
+ on_delete=models.CASCADE,
+ )
+ count = models.PositiveIntegerField(null=False)
+
+
+class GSDCount(CountBase):
+ # Update by asset triggers.
+
+ class Meta:
+ unique_together = (('collection', 'value'),)
+ ordering = ['id']
+ triggers = generates_summary_count_triggers(
+ SummaryFields.GSD.value[0], SummaryFields.GSD.value[1]
+ )
+
+ value = models.FloatField(null=True, blank=True)
+
+
+class GeoadminLangCount(CountBase):
+ # Update by asset triggers.
+
+ class Meta:
+ unique_together = (('collection', 'value'),)
+ ordering = ['id']
+ triggers = generates_summary_count_triggers(
+ SummaryFields.LANGUAGE.value[0], SummaryFields.LANGUAGE.value[1]
+ )
+
+ value = models.CharField(max_length=2, default=None, null=True, blank=True)
+
+
+class GeoadminVariantCount(CountBase):
+ # Update by asset triggers.
+
+ class Meta:
+ unique_together = (('collection', 'value'),)
+ ordering = ['id']
+ triggers = generates_summary_count_triggers(
+ SummaryFields.VARIANT.value[0], SummaryFields.VARIANT.value[1]
+ )
+
+ value = models.CharField(max_length=25, null=True, blank=True)
+
+
+class ProjEPSGCount(CountBase):
+ # Update by asset and collection asset triggers.
+
+ class Meta:
+ unique_together = (('collection', 'value'),)
+ ordering = ['id']
+ triggers = generates_summary_count_triggers(
+ SummaryFields.PROJ_EPSG.value[0], SummaryFields.PROJ_EPSG.value[1]
+ )
+
+ value = models.IntegerField(null=True, blank=True)
diff --git a/app/stac_api/models/general.py b/app/stac_api/models/general.py
new file mode 100644
index 00000000..6849409a
--- /dev/null
+++ b/app/stac_api/models/general.py
@@ -0,0 +1,450 @@
+import hashlib
+import logging
+import os
+import time
+from uuid import uuid4
+
+from language_tags import tags
+from multihash import encode as multihash_encode
+from multihash import to_hex_string
+
+from django.conf import settings
+from django.contrib.gis.db import models
+from django.contrib.gis.geos import Polygon
+from django.contrib.postgres.fields import ArrayField
+from django.core.exceptions import ValidationError
+from django.core.serializers.json import DjangoJSONEncoder
+from django.core.validators import MaxValueValidator
+from django.core.validators import MinValueValidator
+from django.db.models.deletion import ProtectedError
+from django.utils.translation import gettext_lazy as _
+
+from stac_api.managers import AssetUploadManager
+from stac_api.pgtriggers import child_triggers
+from stac_api.utils import select_s3_bucket
+from stac_api.validators import MEDIA_TYPES
+from stac_api.validators import validate_asset_name
+from stac_api.validators import validate_asset_name_with_media_type
+from stac_api.validators import validate_link_rel
+from stac_api.validators import validate_media_type
+from stac_api.validators import validate_name
+
+logger = logging.getLogger(__name__)
+
+# We use the WGS84 bounding box as defined here:
+# https://epsg.io/2056
+_BBOX_CH = Polygon.from_bbox((5.96, 45.82, 10.49, 47.81))
+_BBOX_CH.srid = 4326
+# equal to
+# 'SRID=4326;POLYGON ((5.96 45.82, 5.96 47.81, 10.49 47.81, 10.49 45.82, 5.96 45.82))'
+BBOX_CH = str(_BBOX_CH)
+
+SEARCH_TEXT_HELP_ITEM = '''
+
+ Search Usage:
+
+ -
+ arg will make a non exact search checking if >arg
+ is part of the Item path
+
+ -
+ Multiple arg can be used, separated by spaces. This will search
+ for all elements containing all arguments in their path
+
+ -
+ "collectionID/itemID" will make an exact search for the specified item.
+
+
+ Examples :
+
+ -
+ Searching for pixelkarte will return all items which have
+ pixelkarte as a part of either their collection ID or their item ID
+
+ -
+ Searching for pixelkarte 2016 4 will return all items
+ which have pixelkarte, 2016 AND 4 as part of their collection ID or
+ item ID
+
+ -
+ Searching for "ch.swisstopo.pixelkarte.example/item2016-4-example"
+ will yield only this item, if this item exists.
+
+
+
+ Search Usage:
+
+ -
+ arg will make a non exact search checking if arg is part of
+ the collection ID
+
+ -
+ Multiple arg can be used, separated by spaces. This will search for all
+ collections ID containing all arguments.
+
+ -
+ "collectionID" will make an exact search for the specified collection.
+
+
+ Examples :
+
+ -
+ Searching for pixelkarte will return all collections which have
+ pixelkarte as a part of their collection ID
+
+ -
+ Searching for pixelkarte 2016 4 will return all collection
+ which have pixelkarte, 2016 AND 4 as part of their collection ID
+
+ -
+ Searching for ch.swisstopo.pixelkarte.example will yield only this
+ collection, if this collection exists. Please note that it would not return
+ a collection named ch.swisstopo.pixelkarte.example.2.
+
+
+