Merge pull request #452 from geoadmin/develop

New Release v1.25.0 - #minor

.env.default

@@ -8,12 +8,29 @@ DB_HOST=localhost
 DB_PORT=15432
 DEBUG=True
 DEBUG_PROPAGATE_API_EXCEPTIONS=False
-AWS_ACCESS_KEY_ID=minioadmin
-AWS_SECRET_ACCESS_KEY=minioadmin
-AWS_STORAGE_BUCKET_NAME=service-stac-local
-AWS_S3_REGION_NAME=eu-central-1
-AWS_S3_ENDPOINT_URL=http://127.0.0.1:9090
-AWS_S3_CUSTOM_DOMAIN=127.0.0.1:9090/service-stac-local
+
+LEGACY_AWS_ACCESS_KEY_ID=minioadmin
+LEGACY_AWS_SECRET_ACCESS_KEY=minioadmin
+
+LEGACY_AWS_S3_BUCKET_NAME=service-stac-local
+LEGACY_AWS_S3_REGION_NAME=eu-central-1
+LEGACY_AWS_S3_ENDPOINT_URL=http://127.0.0.1:9090
+LEGACY_AWS_S3_CUSTOM_DOMAIN=127.0.0.1:9090/service-stac-local
 SECRET_KEY=dummy
 HEALTHCHECK_ENDPOINT=healthcheck
 ALLOWED_HOSTS=*
+MANAGED_BUCKET_COLLECTION_PATTERNS="ch\.meteoschweiz\.ogd-.*,ch\.bgdi-test\..*"
+
+# these are just here for completeness
+AWS_ROLE_ARN=some-arn
+AWS_WEB_IDENTITY_TOKEN_FILE=token
+
+AWS_S3_BUCKET_NAME=service-stac-local-managed
+AWS_S3_REGION_NAME=eu-central-1
+AWS_S3_ENDPOINT_URL=http://127.0.0.1:9090
+AWS_S3_CUSTOM_DOMAIN=127.0.0.1:9090/service-stac-local
+
+# SET THIS value to a URL that points to a jpeg file that is publicly
+# reachable in the web
+# this will be used by the unit tests for external assets
+EXTERNAL_TEST_ASSET_URL=https://prod-swisstopoch-hcms-sdweb.imgix.net/2024/07/04/04d3aafe-99b1-4589-934c-337583eb5564.jpeg

.isort.cfg

@@ -4,8 +4,7 @@ known_third_party=pytest,logging_utilities
 known_django=django
 known_stac_api=stac_api
 known_rest_framework=rest_framework,rest_framework_gis,rest_framework_condition
-known_django_solo=solo
 known_storages=storages
 known_tests=tests
 force_single_line=True
-sections=FUTURE,STDLIB,THIRDPARTY,DJANGO,STORAGES,REST_FRAMEWORK,DJANGO_SOLO,STAC_API,TESTS,FIRSTPARTY,LOCALFOLDER
+sections=FUTURE,STDLIB,THIRDPARTY,DJANGO,STORAGES,REST_FRAMEWORK,STAC_API,TESTS,FIRSTPARTY,LOCALFOLDER

.pylintrc

@@ -19,7 +19,7 @@ ignore-patterns=^[0-9]{4}_.*\.py$
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-init-hook='import sys; sys.path.append("./app")'
+init-hook='import sys; sys.path.append("./app"); from dotenv import load_dotenv; load_dotenv(".env.local")'
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
 # number of processors available to use.
@@ -33,7 +33,7 @@ limit-inference-results=100
 # List of plugins (as comma separated values of python module names) to load,
 # usually to register additional checkers.
 load-plugins=pylint_django
-django-settings-module=stac_api.settings
+django-settings-module=config.settings_dev
 
 # Pickle collected data for later comparisons.
 persistent=yes
@@ -507,7 +507,7 @@ max-branches=12
 max-locals=15
 
 # Maximum number of parents for a class (see R0901).
-max-parents=8
+max-parents=10
 
 # Maximum number of public methods for a class (see R0904).
 max-public-methods=20
@@ -526,6 +526,6 @@ min-public-methods=2
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "BaseException, Exception".
-overgeneral-exceptions=BaseException,
-                       Exception,
-                       StandardError
+overgeneral-exceptions=builtins.BaseException,
+                       builtins.Exception,
+                       builtins.StandardError

Dockerfile

@@ -2,7 +2,7 @@
 # Container that contains basic configurations used by all other containers
 # It should only contain variables that don't change or change very infrequently
 # so that the cache is not needlessly invalidated
-FROM python:3.9-slim-bullseye as base
+FROM python:3.12-slim-bullseye as base
 ENV HTTP_PORT=8080
 ENV USER=geoadmin
 ENV GROUP=geoadmin
@@ -64,7 +64,7 @@ RUN apt-get -qq update > /dev/null \
 COPY Pipfile.lock Pipfile ${INSTALL_DIR}/
 RUN cd ${INSTALL_DIR} && pipenv sync --dev
 
-# this is only used with the docker-compose setup within CI
+# this is only used with the docker compose setup within CI
 # to ensure that the app is only started once the DB container
 # is ready
 COPY ./wait-for-it.sh ${INSTALL_DIR}/app/
@@ -125,9 +125,11 @@ RUN echo "APP_VERSION = '$VERSION'" > ${INSTALL_DIR}/app/config/version.py
 
 # Collect static files, some variables like AWS_ are mandatory so set them to avoid exceptions.
 RUN LOGGING_CFG=0 \
-    AWS_ACCESS_KEY_ID= \
-    AWS_SECRET_ACCESS_KEY= \
-    AWS_STORAGE_BUCKET_NAME= \
+    LEGACY_AWS_ACCESS_KEY_ID= \
+    LEGACY_AWS_SECRET_ACCESS_KEY= \
+    LEGACY_AWS_S3_BUCKET_NAME= \
+    AWS_S3_BUCKET_NAME= \
+    AWS_ROLE_ARN= \
     ${INSTALL_DIR}/app/manage.py collectstatic --noinput
 
 ARG GIT_HASH=unknown
@@ -145,4 +147,4 @@ USER ${USER}
 
 EXPOSE ${HTTP_PORT}
 # entrypoint is the manage command
-ENTRYPOINT ["python"]
+ENTRYPOINT ["python"]

Makefile

@@ -18,6 +18,7 @@ endif
 # Django specific
 APP_SRC_DIR := app
 DJANGO_MANAGER := $(CURRENT_DIR)/$(APP_SRC_DIR)/manage.py
+DJANGO_MANAGER_DEBUG := -m debugpy --listen localhost:5678 --wait-for-client $(CURRENT_DIR)/$(APP_SRC_DIR)/manage.py
 
 # Test options
 ifeq ($(CI),"1")
@@ -84,6 +85,7 @@ help:
 	@echo "- django-checks            Run the django checks"
 	@echo "- django-check-migrations  Check that no django migration file is missing"
 	@echo "- test                     Run the tests"
+	@echo "- test-conformance         Run stac-api-validator, needs a valid collection name, e.g. collection=ch.are.agglomerationsverkehr"
 	@echo -e " \033[1mSPEC TARGETS\033[0m "
 	@echo "- lint-specs               Lint the openapi specs  (openapi.yaml and openapitransactional.yaml)"
 	@echo "- ci-build-check-specs     Checks that the specs have been built"
@@ -98,7 +100,7 @@ help:
 	@echo "- dockerpush-(debug|prod)  Build and push the project localy (with tag := $(DOCKER_IMG_LOCAL_TAG))"
 	@echo "- dockerrun                Run the test container with default manage.py command 'runserver'. Note: ENV is populated from '.env.local'"
 	@echo "                           Other cmds can be invoked with 'make dockerrun CMD'."
-	@echo -e "                           \e[1mNote:\e[0m This will connect to your host Postgres DB. If you wanna test with a containerized DB, run 'docker-compose up'"
+	@echo -e "                           \e[1mNote:\e[0m This will connect to your host Postgres DB. If you wanna test with a containerized DB, run 'docker compose up'"
 	@echo -e " \033[1mCLEANING TARGETS\033[0m "
 	@echo "- clean                    Clean genereated files"
 	@echo "- clean-logs               Clean generated logs files"
@@ -139,8 +141,9 @@ setup-s3-and-db:
 	# Create volume directories for postgres and minio
 	# Note that the '/service_stac_local' part is already the bucket name
 	mkdir -p .volumes/minio/service-stac-local
+	mkdir -p .volumes/minio/service-stac-local-managed
 	mkdir -p .volumes/postgresql
-	docker-compose up -d
+	docker compose up -d
 
 .PHONY: setup
 setup: $(SETTINGS_TIMESTAMP) setup-s3-and-db setup-logs
@@ -196,6 +199,20 @@ test:
 	$(PYTHON) $(DJANGO_MANAGER) collectstatic --noinput
 	$(PYTHON) $(DJANGO_MANAGER) test --verbosity=2 --parallel 20 $(CI_TEST_OPT) $(TEST_DIR)
 
+.PHONY: test-debug
+test-debug:
+	# Collect static first to avoid warning in the test
+	$(PYTHON) $(DJANGO_MANAGER) collectstatic --noinput
+	$(PYTHON) $(DJANGO_MANAGER_DEBUG) test --verbosity=2 $(CI_TEST_OPT) $(TEST_DIR)
+
+
+.PHONY: test-conformance
+test-conformance:
+	stac-api-validator \
+    --root-url http://localhost:$(HTTP_PORT)/api/stac/v1/ \
+    --conformance core \
+    --conformance collections \
+    --collection $(collection)
 
 ###################
 # Specs
@@ -229,6 +246,10 @@ ci-build-check-specs:
 serve: setup-logs
 	$(PYTHON) $(DJANGO_MANAGER) runserver $(HTTP_PORT)
 
+.PHONY: serve-debug
+serve-debug: setup-logs
+	$(PYTHON) $(DJANGO_MANAGER_DEBUG) runserver $(HTTP_PORT)
+
 .PHONY: gunicornserve
 gunicornserve: setup-logs
 	$(PYTHON) $(APP_SRC_DIR)/wsgi.py
@@ -306,7 +327,7 @@ clean-logs:
 
 .PHONY: clean
 clean: clean-logs
-	docker-compose down
+	docker compose down
 	@# clean python cache files
 	find . -path ./.volumes -prune -o -name __pycache__ -type d -print0 | xargs -I {} -0 rm -rf "{}"
 	rm -rf $(TIMESTAMPS)

Pipfile

@@ -4,41 +4,47 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
-yapf = "~=0.32.0"
-isort = "~=5.11.3"
-pylint = "~=2.15.9"
-pylint-django = "~=2.5.3"
-django-extensions = "~=3.2.1"
-django_debug_toolbar = "~=3.8.1"
+yapf = "~=0.40.2"
+isort = "~=5.13.2"
+pylint = "~=3.2.2"
+pylint-django = "~=2.5.5"
+django-extensions = "~=3.2.3"
+django_debug_toolbar = "*"
 pip = "*"
 tblib = "*"  # needed for traceback when running tests in parallel
-mock = "~=4.0.3"
+mock = "~=5.1.0"
 responses = "!=0.12.1" # skip version 0.12.1 which has a bug see https://github.com/getsentry/responses/issues/358
-requests-mock = "~=1.10.0"
+requests-mock = "~=1.12.1"
 moto = {extras = [ "s3",], version = "~=4.0.13"}
+debugpy = "*"
+parameterized = "*"
+stac-api-validator = "~=0.6.2"
+pytest = "*"
+pytest-django = "*"
+pytest-dotenv = "*"
 
 [packages]
-gevent = "~=22.10.2"
-gunicorn = "~=20.1.0"
-psycopg2-binary = "~=2.9.5"
-numpy = "~=1.24.0"
-python-dotenv = "~=0.21.0"
-djangorestframework = "~=3.13.0"
-Django = "~=4.0.0"
-PyYAML = "~=6.0"
-whitenoise = "~=6.2.0"
+gevent = "~=24.2.1"
+gunicorn = "~=22.0.0"
+psycopg2-binary = "~=2.9.9"
+numpy = "~=1.26.4"
+python-dotenv = "~=1.0.1"
+djangorestframework = "~=3.15.1"
+Django = "~=5.0.8"
+PyYAML = "~=6.0.1"
+whitenoise = "~=6.6.0"
 djangorestframework-gis = "~=1.0"
-python-dateutil = "~=2.8.2"
-django-solo = "~=2.0.0"
-django-storages = "~=1.13.1"
-boto3 = "~=1.26.34"
+python-dateutil = "~=2.9.0.post0"
+django-storages = "~=1.14.3"
+boto3 = "~=1.34.115"
 django-rest-framework-condition = "~=0.1.1"
-requests = "~=2.28.1"
+requests = "~=2.32.3"
 py-multihash = "~=2.0.1"
-django-prometheus = "~=2.2.0"
+django-prometheus = "~=2.3.1"
 django-admin-autocomplete-filter = "~=0.7.1"
-django-pgtrigger = "~=4.6.0"
-logging-utilities = "~=4.0.0"
+django-pgtrigger = "~=4.11.1"
+logging-utilities = "~=4.4.1"
+django-environ = "*"
 
 [requires]
-python_version = "3.9"
+python_version = "3.12"