From 910e63458f324cb6d3b7de100d7633b0ece16d32 Mon Sep 17 00:00:00 2001 From: Brice Schaffner Date: Mon, 5 Jul 2021 13:45:24 +0200 Subject: [PATCH 1/3] BGDIINF_SB-1890: Updated to latest Codebuild image aws/codebuild/standard:5.0 --- buildspec.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index cc0ca483..63a5f7e4 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -13,14 +13,11 @@ env: phases: install: runtime-versions: - docker: 18 + python: 3.7 commands: - echo "Installing necessary dependencies" - - apt-get update && apt-get install -y docker-compose python3-pip python3-venv gdal-bin pass gnupg2 - - echo "Install aws cli v2 for docker login to ECR registry" - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - - unzip awscliv2.zip - - ./aws/install + - apt-get update + - apt-get install -y gdal-bin - aws --version - echo "Login to AWS ECR docker registry" - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${REGISTRY} From a5543f646195fb5b83b16316fc5b1299bc2197f5 Mon Sep 17 00:00:00 2001 From: Brice Schaffner Date: Fri, 9 Jul 2021 10:35:47 +0200 Subject: [PATCH 2/3] BGDIINF_SB-1897: eo:gsd value 0.0 not added to collection summaries The eo:gsd value 0.0 was not added to the collection summaries which did a crash when trying to unset the eo:gsd or change its value. A similar issue was also present for the proj:epsg with value 0 (also value of 0 for espg don't make sense, but if by mistake it would be set, then it would be an issue). The bug has been fixed. Then because having a value of 0.0 for eo:gsd doesn't make sense, we prevent it by raising a 400. --- app/stac_api/collection_summaries.py | 11 +++++---- .../migrations/0012_auto_20210709_0734.py | 23 ++++++++++++++++++ app/stac_api/models.py | 3 ++- app/stac_api/validators.py | 19 +++++++++++++++ app/tests/test_asset_model.py | 24 +++++++++++++++++++ 5 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 app/stac_api/migrations/0012_auto_20210709_0734.py diff --git a/app/stac_api/collection_summaries.py b/app/stac_api/collection_summaries.py index 0b53db91..f8bf90f5 100644 --- a/app/stac_api/collection_summaries.py +++ b/app/stac_api/collection_summaries.py @@ -82,7 +82,8 @@ def _update_summaries_on_asset_delete(self, asset): if ( not assets.filter(**{ attribute: attribute_value - }).exists() and attribute_value is not None + }).exists() and attribute_value is not None and + attribute_value in self.summaries[key] ): logger.info( 'Removing %s %s from collection summaries', @@ -146,7 +147,7 @@ def _update_summaries_on_asset_insert(self, asset): self.summaries["geoadmin:variant"].sort() updated |= True - if asset.proj_epsg and asset.proj_epsg not in self.summaries["proj:epsg"]: + if asset.proj_epsg is not None and asset.proj_epsg not in self.summaries["proj:epsg"]: logger.info( 'Adds proj:epsg %s to collection summaries', asset.proj_epsg, @@ -161,7 +162,7 @@ def _update_summaries_on_asset_insert(self, asset): self.summaries["proj:epsg"].sort() updated |= True - if asset.eo_gsd and not float_in(asset.eo_gsd, self.summaries["eo:gsd"]): + if asset.eo_gsd is not None and not float_in(asset.eo_gsd, self.summaries["eo:gsd"]): logger.info( 'Adds eo:gsd %s to collection summaries', asset.proj_epsg, @@ -262,7 +263,7 @@ def _update_summaries_proj_epsg_on_update(self, assets, asset, proj_epsg, origin ''' updated = False - if proj_epsg and proj_epsg not in self.summaries["proj:epsg"]: + if proj_epsg is not None and proj_epsg not in self.summaries["proj:epsg"]: logger.info( 'Adds proj:epsg value %s from collection summaries', proj_epsg, @@ -319,7 +320,7 @@ def _update_summaries_eo_gsd_on_update(self, assets, asset, eo_gsd, original_eo_ ''' updated = False - if eo_gsd and not float_in(eo_gsd, self.summaries["eo:gsd"]): + if eo_gsd is not None and not float_in(eo_gsd, self.summaries["eo:gsd"]): logger.info( 'Adds eo:gsd value %s from collection summaries', eo_gsd, diff --git a/app/stac_api/migrations/0012_auto_20210709_0734.py b/app/stac_api/migrations/0012_auto_20210709_0734.py new file mode 100644 index 00000000..ed089018 --- /dev/null +++ b/app/stac_api/migrations/0012_auto_20210709_0734.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.10 on 2021-07-09 07:34 + +from django.db import migrations +from django.db import models + +import stac_api.validators + + +class Migration(migrations.Migration): + + dependencies = [ + ('stac_api', '0011_auto_20210623_0521'), + ] + + operations = [ + migrations.AlterField( + model_name='asset', + name='eo_gsd', + field=models.FloatField( + blank=True, null=True, validators=[stac_api.validators.validate_eo_gsd] + ), + ), + ] diff --git a/app/stac_api/models.py b/app/stac_api/models.py index 9dbd73e7..42786931 100644 --- a/app/stac_api/models.py +++ b/app/stac_api/models.py @@ -31,6 +31,7 @@ from stac_api.validators import MEDIA_TYPES from stac_api.validators import validate_asset_name from stac_api.validators import validate_asset_name_with_media_type +from stac_api.validators import validate_eo_gsd from stac_api.validators import validate_geoadmin_variant from stac_api.validators import validate_geometry from stac_api.validators import validate_item_properties_datetimes @@ -591,7 +592,7 @@ def filename(self): ) # here we need to set blank=True otherwise the field is as required in the admin interface description = models.TextField(blank=True, null=True, default=None) - eo_gsd = models.FloatField(null=True, blank=True) + eo_gsd = models.FloatField(null=True, blank=True, validators=[validate_eo_gsd]) class Language(models.TextChoices): # pylint: disable=invalid-name diff --git a/app/stac_api/validators.py b/app/stac_api/validators.py index 7160663e..c5590daf 100644 --- a/app/stac_api/validators.py +++ b/app/stac_api/validators.py @@ -150,6 +150,25 @@ def validate_geoadmin_variant(variant): ) +def validate_eo_gsd(value): + '''Validate eo:gsd + + Args: + value: float + The value to validate + Raise: + ValidationError: When the value is not valid + ''' + if value <= 0: + logger.error("Invalid eo:gsd property \"%f\", value must be > 0", value) + raise ValidationError( + _('Invalid eo:gsd "%(eo_gsd)f", ' + 'value must be a positive number bigger than 0'), + params={'eo_gsd': value}, + code="invalid" + ) + + def validate_link_rel(value): invalid_rel = [ 'self', diff --git a/app/tests/test_asset_model.py b/app/tests/test_asset_model.py index 46ecc9eb..0add152d 100644 --- a/app/tests/test_asset_model.py +++ b/app/tests/test_asset_model.py @@ -55,6 +55,30 @@ def test_create_asset_valid_geoadmin_variant(self): db_create=True, ) + def test_create_asset_invalid_eo_gsd(self): + with self.assertRaises(ValidationError, msg="asset with invalid eo:gsd was accepted."): + self.factory.create_asset_sample( + item=self.item, + eo_gsd=0.0, + db_create=True, + ) + + def test_create_asset_valid_eo_gsd(self): + asset = self.factory.create_asset_sample(item=self.item, eo_gsd=1.33).model + self.collection.refresh_from_db() + self.assertEqual( + self.collection.summaries, { + 'eo:gsd': [1.33, 3.4], 'proj:epsg': [2056], 'geoadmin:variant': ['kgrs'] + } + ) + asset.delete() + self.collection.refresh_from_db() + self.assertEqual( + self.collection.summaries, { + 'eo:gsd': [3.4], 'proj:epsg': [2056], 'geoadmin:variant': ['kgrs'] + } + ) + def test_create_asset_invalid_geoadmin_variant(self): # try to create an asset with invalid geoadmin variant. with self.assertRaises( From 295d00757689e407ec31b9dc0c0572422d85ba2b Mon Sep 17 00:00:00 2001 From: Brice Schaffner Date: Fri, 9 Jul 2021 10:56:01 +0200 Subject: [PATCH 3/3] BGDIINF_SB-1898: Avoid crash if request is missing the content-type header. On prod we had some non authorized requests without content-type header that crashed the service. --- app/stac_api/apps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/stac_api/apps.py b/app/stac_api/apps.py index 38b9f24c..8feddc63 100644 --- a/app/stac_api/apps.py +++ b/app/stac_api/apps.py @@ -47,7 +47,8 @@ def custom_exception_handler(exc, context): if ( context['request']._request.method.upper() in ["PATCH", "POST", "PUT"] and - 'application/json' in context['request']._request.headers['content-type'].lower() + 'application/json' in context['request']._request.headers.get('content-type', + '').lower() ): extra["request.payload"] = context['request'].data