Merge pull request #456 from geoadmin/feat-PB-756-api-col-assets

PB-756: CRUD API for collection assets

app/config/urls.py

@@ -38,10 +38,11 @@ def checker(request):
 if settings.DEBUG:
     import debug_toolbar
 
-    from stac_api.views_test import TestAssetUpsertHttp500
-    from stac_api.views_test import TestCollectionUpsertHttp500
-    from stac_api.views_test import TestHttp500
-    from stac_api.views_test import TestItemUpsertHttp500
+    from stac_api.views.test import TestAssetUpsertHttp500
+    from stac_api.views.test import TestCollectionAssetUpsertHttp500
+    from stac_api.views.test import TestCollectionUpsertHttp500
+    from stac_api.views.test import TestHttp500
+    from stac_api.views.test import TestItemUpsertHttp500
 
     urlpatterns = [
         path('__debug__/', include(debug_toolbar.urls)),
@@ -61,6 +62,11 @@ def checker(request):
             TestAssetUpsertHttp500.as_view(),
             name='test-asset-detail-http-500'
         ),
+        path(
+            'tests/test_collection_asset_upsert_http_500/<collection_name>/<asset_name>',
+            TestCollectionAssetUpsertHttp500.as_view(),
+            name='test-collection-asset-detail-http-500'
+        ),
         # Add v0.9 namespace to test routes.
         path(
             'tests/v0.9/test_asset_upsert_http_500/<collection_name>/<item_name>/<asset_name>',

app/stac_api/management/commands/list_asset_uploads.py

@@ -6,7 +6,7 @@
 
 from stac_api.models import AssetUpload
 from stac_api.s3_multipart_upload import MultipartUpload
-from stac_api.serializers import AssetUploadSerializer
+from stac_api.serializers.upload import AssetUploadSerializer
 from stac_api.utils import CommandHandler
 from stac_api.utils import get_asset_path
 

app/stac_api/management/commands/profile_item_serializer.py

@@ -20,7 +20,7 @@ class Handler(CommandHandler):
 
     def profiling(self):
         # pylint: disable=import-outside-toplevel,possibly-unused-variable
-        from stac_api.serializers import ItemSerializer
+        from stac_api.serializers.item import ItemSerializer
         collection_id = self.options["collection"]
         qs = Item.objects.filter(collection__name=collection_id
                                 ).prefetch_related('assets', 'links')[:self.options['limit']]

app/stac_api/management/commands/profile_serializer_vs_no_drf.py

@@ -21,7 +21,7 @@ def profiling(self):
         # pylint: disable=import-outside-toplevel,possibly-unused-variable
 
         self.print('Starting profiling')
-        from stac_api.serializers import ItemSerializer
+        from stac_api.serializers.item import ItemSerializer
 
         def serialize(qs):
             return {

app/stac_api/migrations/0050_collectionassetupload_and_more.py

@@ -0,0 +1,147 @@
+# Generated by Django 5.0.8 on 2024-09-10 12:45
+
+import pgtrigger.compiler
+import pgtrigger.migrations
+
+import django.core.serializers.json
+import django.core.validators
+import django.db.models.deletion
+from django.db import migrations
+from django.db import models
+
+import stac_api.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('stac_api', '0049_item_properties_expires'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='assetupload',
+            name='update_interval',
+            field=models.IntegerField(
+                default=-1,
+                help_text=
+                'Interval in seconds in which the asset data is updated. -1 means that the data is not on a regular basis updated. This field can only be set via the API.',
+                validators=[django.core.validators.MinValueValidator(-1)]
+            ),
+        ),
+        migrations.CreateModel(
+            name='CollectionAssetUpload',
+            fields=[
+                ('id', models.BigAutoField(primary_key=True, serialize=False)),
+                ('upload_id', models.CharField(max_length=255)),
+                (
+                    'status',
+                    models.CharField(
+                        choices=[(None, ''), ('in-progress', 'In Progress'),
+                                 ('completed', 'Completed'), ('aborted', 'Aborted')],
+                        default='in-progress',
+                        max_length=32
+                    )
+                ),
+                (
+                    'number_parts',
+                    models.IntegerField(
+                        validators=[
+                            django.core.validators.MinValueValidator(1),
+                            django.core.validators.MaxValueValidator(100)
+                        ]
+                    )
+                ),
+                (
+                    'md5_parts',
+                    models.JSONField(
+                        editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder
+                    )
+                ),
+                (
+                    'urls',
+                    models.JSONField(
+                        blank=True,
+                        default=list,
+                        encoder=django.core.serializers.json.DjangoJSONEncoder
+                    )
+                ),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('ended', models.DateTimeField(blank=True, default=None, null=True)),
+                ('checksum_multihash', models.CharField(max_length=255)),
+                ('etag', models.CharField(default=stac_api.models.compute_etag, max_length=56)),
+                (
+                    'update_interval',
+                    models.IntegerField(
+                        default=-1,
+                        help_text=
+                        'Interval in seconds in which the asset data is updated. -1 means that the data is not on a regular basis updated. This field can only be set via the API.',
+                        validators=[django.core.validators.MinValueValidator(-1)]
+                    )
+                ),
+                (
+                    'content_encoding',
+                    models.CharField(
+                        blank=True,
+                        choices=[(None, ''), ('gzip', 'Gzip'), ('br', 'Br')],
+                        default='',
+                        max_length=32
+                    )
+                ),
+                (
+                    'asset',
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name='+',
+                        to='stac_api.collectionasset'
+                    )
+                ),
+            ],
+        ),
+        migrations.AddConstraint(
+            model_name='collectionassetupload',
+            constraint=models.UniqueConstraint(
+                fields=('asset', 'upload_id'),
+                name='unique_asset_upload_collection_asset_upload_id'
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name='collectionassetupload',
+            constraint=models.UniqueConstraint(
+                condition=models.Q(('status', 'in-progress')),
+                fields=('asset', 'status'),
+                name='unique_asset_upload_in_progress'
+            ),
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name='collectionassetupload',
+            trigger=pgtrigger.compiler.Trigger(
+                name='add_asset_upload_trigger',
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    func=
+                    '\n    -- update AssetUpload auto variable\n    NEW.etag = public.gen_random_uuid();\n\n    RETURN NEW;\n    ',
+                    hash='5f51ec3c72c4d9fbe6b81d2fd881dd5228dc80bf',
+                    operation='INSERT',
+                    pgid='pgtrigger_add_asset_upload_trigger_8330c',
+                    table='stac_api_collectionassetupload',
+                    when='BEFORE'
+                )
+            ),
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name='collectionassetupload',
+            trigger=pgtrigger.compiler.Trigger(
+                name='update_asset_upload_trigger',
+                sql=pgtrigger.compiler.UpsertTriggerSql(
+                    condition='WHEN (OLD.* IS DISTINCT FROM NEW.*)',
+                    func=
+                    '\n    -- update AssetUpload auto variable\n    NEW.etag = public.gen_random_uuid();\n\n    RETURN NEW;\n    ',
+                    hash='0a7f1aa8f8c0bb2c413a7ce626f75c8da5bf4b6d',
+                    operation='UPDATE',
+                    pgid='pgtrigger_update_asset_upload_trigger_8d012',
+                    table='stac_api_collectionassetupload',
+                    when='BEFORE'
+                )
+            ),
+        ),
+    ]

app/stac_api/models.py

@@ -710,19 +710,10 @@ def get_asset_path(self):
         return get_collection_asset_path(self.collection, self.name)
 
 
-class AssetUpload(models.Model):
+class BaseAssetUpload(models.Model):
 
     class Meta:
-        constraints = [
-            models.UniqueConstraint(fields=['asset', 'upload_id'], name='unique_together'),
-            # Make sure that there is only one asset upload in progress per asset
-            models.UniqueConstraint(
-                fields=['asset', 'status'],
-                condition=Q(status='in-progress'),
-                name='unique_in_progress'
-            )
-        ]
-        triggers = generates_asset_upload_triggers()
+        abstract = True
 
     class Status(models.TextChoices):
         # pylint: disable=invalid-name
@@ -741,7 +732,6 @@ class ContentEncoding(models.TextChoices):
 
     # using BigIntegerField as primary_key to deal with the expected large number of assets.
     id = models.BigAutoField(primary_key=True)
-    asset = models.ForeignKey(Asset, related_name='+', on_delete=models.CASCADE)
     upload_id = models.CharField(max_length=255, blank=False, null=False)
     status = models.CharField(
         choices=Status.choices, max_length=32, default=Status.IN_PROGRESS, blank=False, null=False
@@ -765,8 +755,8 @@ class ContentEncoding(models.TextChoices):
         null=False,
         blank=False,
         validators=[MinValueValidator(-1)],
-        help_text="Interval in seconds in which the asset data is updated."
-        "-1 means that the data is not on a regular basis updated."
+        help_text="Interval in seconds in which the asset data is updated. "
+        "-1 means that the data is not on a regular basis updated. "
         "This field can only be set via the API."
     )
 
@@ -777,6 +767,23 @@ class ContentEncoding(models.TextChoices):
     # Custom Manager that preselects the collection
     objects = AssetUploadManager()
 
+
+class AssetUpload(BaseAssetUpload):
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(fields=['asset', 'upload_id'], name='unique_together'),
+            # Make sure that there is only one asset upload in progress per asset
+            models.UniqueConstraint(
+                fields=['asset', 'status'],
+                condition=Q(status='in-progress'),
+                name='unique_in_progress'
+            )
+        ]
+        triggers = generates_asset_upload_triggers()
+
+    asset = models.ForeignKey(Asset, related_name='+', on_delete=models.CASCADE)
+
     def update_asset_from_upload(self):
         '''Updating the asset's file:checksum and update_interval from the upload
 
@@ -804,6 +811,51 @@ def update_asset_from_upload(self):
         self.asset.save()
 
 
+class CollectionAssetUpload(BaseAssetUpload):
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=['asset', 'upload_id'],
+                name='unique_asset_upload_collection_asset_upload_id'
+            ),
+            # Make sure that there is only one upload in progress per collection asset
+            models.UniqueConstraint(
+                fields=['asset', 'status'],
+                condition=Q(status='in-progress'),
+                name='unique_asset_upload_in_progress'
+            )
+        ]
+        triggers = generates_asset_upload_triggers()
+
+    asset = models.ForeignKey(CollectionAsset, related_name='+', on_delete=models.CASCADE)
+
+    def update_asset_from_upload(self):
+        '''Updating the asset's file:checksum and update_interval from the upload
+
+        When the upload is completed, the new file:checksum and update interval from the upload
+        is set to its asset parent.
+        '''
+        logger.debug(
+            'Updating collection asset %s file:checksum from %s to %s and update_interval '
+            'from %d to %d due to upload complete',
+            self.asset.name,
+            self.asset.checksum_multihash,
+            self.checksum_multihash,
+            self.asset.update_interval,
+            self.update_interval,
+            extra={
+                'upload_id': self.upload_id,
+                'asset': self.asset.name,
+                'collection': self.asset.collection.name
+            }
+        )
+
+        self.asset.checksum_multihash = self.checksum_multihash
+        self.asset.update_interval = self.update_interval
+        self.asset.save()
+
+
 class CountBase(models.Model):
     '''CountBase tables are used to help calculate the summary on a collection.
     This is only performant if the distinct number of values is small, e.g. we currently only have