WIP - further sanitize Makefile and env vars

Vizzuality · Dec 20, 2021 · 8c58a53 · 8c58a53
1 parent ee32ab5
commit 8c58a53
Show file tree

Hide file tree

Showing 9 changed files with 26 additions and 137 deletions.
diff --git a/.github/workflows/api-e2e-tests.yml b/.github/workflows/api-e2e-tests.yml
@@ -33,7 +33,7 @@ jobs:
     - name: Checkout envs
       run: docker-compose -f docker-compose-test-e2e.yml -f docker-compose-test-e2e.ci.yml  config
     - name: Run CI tests via make task
-      run: make run-test-e2e-ci
+      run: make test-e2e-backend
   test-api-unit:
     runs-on: ubuntu-18.04
     timeout-minutes: 10

diff --git a/Makefile b/Makefile
@@ -2,24 +2,24 @@
 
 # Read values as needed from .env
 #
-# Optionally pass 'environment=<something>' to load env vars from '.env-something' instead
+# Optionally pass 'environment=<something>' to load env vars from '.env.something' instead
 # Useful to separate i.e. dev from testing databases
 #
 # If using the same variables in recipes that need to use a dotenv file other
 # than .env, remember to check that no values from .env are being used
 # inadvertently.
-ENVFILE := $(if $(environment), .env-$(environment), .env)
+ENVFILE := $(if $(environment),.env.$(environment),.env)
 ifneq (,$(wildcard $(ENVFILE)))
     include $(ENVFILE)
+    DOCKER_COMPOSE_FILE := --env-file $(ENVFILE)
     export
 endif
 
-CIENV := $(if $(filter $(environment), ci), -f docker-compose-test-e2e.ci.yml , -f docker-compose-test-e2e.local.yml)
-API_DB_INSTANCE := $(if $(environment), test-e2e-postgresql-api, postgresql-api)
-GEO_DB_INSTANCE := $(if $(environment), test-e2e-postgresql-geo-api, postgresql-geo-api)
-REDIS_INSTANCE := $(if $(environment), test-e2e-redis, redis)
+API_DB_INSTANCE := $(if $(filter $(IS_DOCKER_E2E_TESTS_RUNTIME), true), test-e2e-postgresql-api, postgresql-api)
+GEO_DB_INSTANCE := $(if $(filter $(IS_DOCKER_E2E_TESTS_RUNTIME), true), test-e2e-postgresql-geo-api, postgresql-geo-api)
+REDIS_INSTANCE := $(if $(filter $(IS_DOCKER_E2E_TESTS_RUNTIME), true), test-e2e-redis, redis)
 
-DOCKER_COMPOSE_FILE := $(if $(environment), -f docker-compose-test-e2e.yml $(CIENV), -f docker-compose.yml )
+DOCKER_COMPOSE_FILE := $(DOCKER_COMPOSE_FILE) $(if $(filter $(IS_DOCKER_E2E_TESTS_RUNTIME), true), -f docker-compose-test-e2e.yml, -f docker-compose.yml )
 DOCKER_CLEAN_VOLUMES := $(if $(environment), , \
 	docker volume rm -f marxan-cloud_marxan-cloud-postgresql-api-data && \
 	docker volume rm -f marxan-cloud_marxan-cloud-postgresql-geo-data && \
@@ -33,7 +33,6 @@ NC :=\033[0m # No Color
 test-commands:
 	@echo $(ENVFILE)
 	@echo $(DOCKER_COMPOSE_FILE)
-	@echo $(CIENV)
 	@echo $(API_POSTGRES_DB)
 	@echo $(GEO_POSTGRES_USER)
 
@@ -128,8 +127,7 @@ test-start-services: clean-slate
 	docker-compose $(DOCKER_COMPOSE_FILE) exec -T api ./apps/api/entrypoint.sh run-migrations-for-e2e-tests && \
 	docker-compose $(DOCKER_COMPOSE_FILE) exec -T geoprocessing ./apps/geoprocessing/entrypoint.sh run-migrations-for-e2e-tests
 
-seed-dbs-e2e: test-start-services
-	$(MAKE) seed-api-with-test-data
+seed-dbs-e2e: test-start-services seed-api-with-test-data
 
 test-e2e-api: seed-dbs-e2e
 	docker-compose $(DOCKER_COMPOSE_FILE) exec -T api ./apps/api/entrypoint.sh test-e2e
@@ -138,13 +136,7 @@ test-e2e-geoprocessing: seed-dbs-e2e
 	docker-compose $(DOCKER_COMPOSE_FILE) exec -T geoprocessing ./apps/geoprocessing/entrypoint.sh test-e2e
 
 test-e2e-backend: test-e2e-api test-e2e-geoprocessing
-	$(MAKE) test-clean-slate
-
-run-test-e2e-local:
-	$(MAKE) --keep-going test-e2e-backend environment=local
-
-run-test-e2e-ci:
-	$(MAKE) --keep-going test-e2e-backend environment=ci
+	$(MAKE) test-clean-slate $(ARGS)
 
 setup-test-unit-backend:
 	# build API and geoprocessing containers
@@ -161,7 +153,7 @@ test-unit-geo:
 test-unit-backend: setup-test-unit-backend test-unit-api test-unit-geo
 
 run-test-unit:
-	$(MAKE) --keep-going test-unit-backend
+	$(MAKE) --keep-going test-unit-backend $(ARGS)
 
 dump-geodb-data:
 	docker-compose exec -T postgresql-geo-api pg_dump -T migrations -a -U "${GEO_POSTGRES_USER}" -F t ${GEO_POSTGRES_DB} | gzip > data/data/processed/db_dumps/geo_db-$$(date +%Y-%m-%d).tar.gz

diff --git a/api/apps/api/config/custom-environment-variables.json b/api/apps/api/config/custom-environment-variables.json
@@ -3,7 +3,6 @@
     "muteAll": "API_LOGGING_MUTE_ALL"
   },
   "postgresApi": {
-    "url": "API_POSTGRES_URL",
     "host": "API_POSTGRES_HOST",
     "username": "API_POSTGRES_USER",
     "password": "API_POSTGRES_PASSWORD",
@@ -12,7 +11,6 @@
     "runMigrationsOnStartup": "API_RUN_MIGRATIONS_ON_STARTUP"
   },
   "postgresGeoApi": {
-    "url": "GEO_POSTGRES_URL",
     "host": "GEO_POSTGRES_HOST",
     "username": "GEO_POSTGRES_USER",
     "password": "GEO_POSTGRES_PASSWORD",

diff --git a/api/apps/geoprocessing/config/custom-environment-variables.json b/api/apps/geoprocessing/config/custom-environment-variables.json
@@ -1,6 +1,5 @@
 {
   "postgresApi": {
-    "url": "API_POSTGRES_URL",
     "host": "API_POSTGRES_HOST",
     "username": "API_POSTGRES_USER",
     "password": "API_POSTGRES_PASSWORD",
@@ -9,7 +8,6 @@
     "runMigrationsOnStartup": "API_RUN_MIGRATIONS_ON_STARTUP"
   },
   "postgresGeoApi": {
-    "url": "GEO_POSTGRES_URL",
     "host": "GEO_POSTGRES_HOST",
     "username": "GEO_POSTGRES_USER",
     "password": "GEO_POSTGRES_PASSWORD",

diff --git a/docker-compose-test-e2e.ci.yml b/docker-compose-test-e2e.ci.yml
diff --git a/docker-compose-test-e2e.local.yml b/docker-compose-test-e2e.local.yml
diff --git a/docker-compose-test-e2e.yml b/docker-compose-test-e2e.yml
@@ -18,12 +18,13 @@ services:
       - ./api/apps:/opt/marxan-api/apps
       - ./api/libs:/opt/marxan-api/libs
       - marxan-cloud-backend-e2e-temp-storage:/tmp/storage
+    env_file: ${ENVFILE}
     environment:
       - NODE_PATH=src
       - NODE_ENV=test
-      - API_POSTGRES_URL=postgres://${API_POSTGRES_USER}:${API_POSTGRES_PASSWORD}@test-e2e-postgresql-api:5432/${API_POSTGRES_DB}
-      - GEO_POSTGRES_URL=postgres://${GEO_POSTGRES_USER}:${GEO_POSTGRES_PASSWORD}@test-e2e-postgresql-geo-api:5432/${GEO_POSTGRES_DB}
       - REDIS_HOST=test-e2e-redis
+      - API_POSTGRES_HOST=test-e2e-postgresql-api
+      - GEO_POSTGRES_HOST=test-e2e-postgresql-geo-api
       - NODE_CONFIG_DIR=apps/api/config
     depends_on:
       - test-e2e-postgresql-api
@@ -46,12 +47,13 @@ services:
       - ./api/apps:/opt/marxan-geoprocessing/apps
       - ./api/libs:/opt/marxan-geoprocessing/libs
       - marxan-cloud-backend-e2e-temp-storage:/tmp/storage
+    env_file: ${ENVFILE}
     environment:
       - NODE_PATH=src
       - NODE_ENV=test
-      - API_POSTGRES_URL=postgres://${API_POSTGRES_USER}:${API_POSTGRES_PASSWORD}@test-e2e-postgresql-api:5432/${API_POSTGRES_DB}
-      - GEO_POSTGRES_URL=postgres://${GEO_POSTGRES_USER}:${GEO_POSTGRES_PASSWORD}@test-e2e-postgresql-geo-api:5432/${GEO_POSTGRES_DB}
       - REDIS_HOST=test-e2e-redis
+      - API_POSTGRES_HOST=test-e2e-postgresql-api
+      - GEO_POSTGRES_HOST=test-e2e-postgresql-geo-api
       - NODE_CONFIG_DIR=apps/geoprocessing/config
     depends_on:
       - test-e2e-postgresql-geo-api

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -17,14 +17,14 @@ services:
       - ./api/apps:/opt/marxan-api/apps
       - ./api/libs:/opt/marxan-api/libs
       - marxan-cloud-backend-temp-storage:/tmp/storage
-    env_file: .env
+    env_file: ${ENVFILE}
     environment:
       - NODE_PATH=src
       - NODE_ENV=development
-      - API_POSTGRES_URL=postgres://${API_POSTGRES_USER}:${API_POSTGRES_PASSWORD}@marxan-postgresql-api:5432/${API_POSTGRES_DB}
-      - GEO_POSTGRES_URL=postgres://${GEO_POSTGRES_USER}:${GEO_POSTGRES_PASSWORD}@marxan-postgresql-geo-api:5432/${GEO_POSTGRES_DB}
       - REDIS_HOST=marxan-redis
       - NODE_CONFIG_DIR=apps/api/config
+      - API_POSTGRES_HOST=postgresql-api
+      - GEO_POSTGRES_HOST=postgresql-geo-api
     depends_on:
       - postgresql-api
       - postgresql-geo-api
@@ -41,7 +41,7 @@ services:
     volumes:
       - ./app/src:/opt/marxan-app/src
       - ./app/test:/opt/marxan-app/test
-    env_file: .env
+    env_file: ${ENVFILE}
     environment:
       - NODE_PATH=src
       - NODE_ENV=development
@@ -62,14 +62,14 @@ services:
       - ./api/apps:/opt/marxan-geoprocessing/apps
       - ./api/libs:/opt/marxan-geoprocessing/libs
       - marxan-cloud-backend-temp-storage:/tmp/storage
-    env_file: .env
+    env_file: ${ENVFILE}
     environment:
       - NODE_PATH=src
       - NODE_ENV=development
-      - API_POSTGRES_URL=postgres://${API_POSTGRES_USER}:${API_POSTGRES_PASSWORD}@marxan-postgresql-api:5432/${API_POSTGRES_DB}
-      - GEO_POSTGRES_URL=postgres://${GEO_POSTGRES_USER}:${GEO_POSTGRES_PASSWORD}@marxan-postgresql-geo-api:5432/${GEO_POSTGRES_DB}
       - REDIS_HOST=marxan-redis
       - NODE_CONFIG_DIR=apps/geoprocessing/config
+      - API_POSTGRES_HOST=postgresql-api
+      - GEO_POSTGRES_HOST=postgresql-geo-api
     depends_on:
       - postgresql-geo-api
       - postgresql-api
@@ -160,8 +160,7 @@ services:
     image: apache/airflow:2.0.0
     restart: on-failure
     entrypoint: ['sh','./scripts/entrypoint.sh']
-    env_file:
-      - .env
+    env_file: ${ENVFILE}
     environment:
       - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgresql-airflow/airflow
       - AIRFLOW__CORE__EXECUTOR=LocalExecutor

diff --git a/env.default b/env.default
@@ -32,3 +32,4 @@ APPLICATION_BASE_URL=http://localhost:3000
 PASSWORD_RESET_TOKEN_PREFIX=/auth/reset-password?token=
 PASSWORD_RESET_EXPIRATION=1800000
 SIGNUP_CONFIRMATION_TOKEN_PREFIX=/auth/sign-up-confirmation?token=
+IS_DOCKER_E2E_TESTS_RUNTIME=false