diff --git a/.github/workflows/api-e2e-tests.yml b/.github/workflows/api-e2e-tests.yml index 9bb52febee..5be93afedb 100644 --- a/.github/workflows/api-e2e-tests.yml +++ b/.github/workflows/api-e2e-tests.yml @@ -33,7 +33,7 @@ jobs: - name: Checkout envs run: docker-compose -f docker-compose-test-e2e.yml -f docker-compose-test-e2e.ci.yml config - name: Run CI tests via make task - run: make run-test-e2e-ci + run: make test-e2e-backend test-api-unit: runs-on: ubuntu-18.04 timeout-minutes: 10 diff --git a/Makefile b/Makefile index bdbc845f74..608409a568 100644 --- a/Makefile +++ b/Makefile @@ -2,24 +2,24 @@ # Read values as needed from .env # -# Optionally pass 'environment=' to load env vars from '.env-something' instead +# Optionally pass 'environment=' to load env vars from '.env.something' instead # Useful to separate i.e. dev from testing databases # # If using the same variables in recipes that need to use a dotenv file other # than .env, remember to check that no values from .env are being used # inadvertently. -ENVFILE := $(if $(environment), .env-$(environment), .env) +ENVFILE := $(if $(environment),.env.$(environment),.env) ifneq (,$(wildcard $(ENVFILE))) include $(ENVFILE) + DOCKER_COMPOSE_FILE := --env-file $(ENVFILE) export endif -CIENV := $(if $(filter $(environment), ci), -f docker-compose-test-e2e.ci.yml , -f docker-compose-test-e2e.local.yml) -API_DB_INSTANCE := $(if $(environment), test-e2e-postgresql-api, postgresql-api) -GEO_DB_INSTANCE := $(if $(environment), test-e2e-postgresql-geo-api, postgresql-geo-api) -REDIS_INSTANCE := $(if $(environment), test-e2e-redis, redis) +API_DB_INSTANCE := $(if $(filter $(IS_DOCKER_E2E_TESTS_RUNTIME), true), test-e2e-postgresql-api, postgresql-api) +GEO_DB_INSTANCE := $(if $(filter $(IS_DOCKER_E2E_TESTS_RUNTIME), true), test-e2e-postgresql-geo-api, postgresql-geo-api) +REDIS_INSTANCE := $(if $(filter $(IS_DOCKER_E2E_TESTS_RUNTIME), true), test-e2e-redis, redis) -DOCKER_COMPOSE_FILE := $(if $(environment), -f docker-compose-test-e2e.yml $(CIENV), -f docker-compose.yml ) +DOCKER_COMPOSE_FILE := $(DOCKER_COMPOSE_FILE) $(if $(filter $(IS_DOCKER_E2E_TESTS_RUNTIME), true), -f docker-compose-test-e2e.yml, -f docker-compose.yml ) DOCKER_CLEAN_VOLUMES := $(if $(environment), , \ docker volume rm -f marxan-cloud_marxan-cloud-postgresql-api-data && \ docker volume rm -f marxan-cloud_marxan-cloud-postgresql-geo-data && \ @@ -33,7 +33,6 @@ NC :=\033[0m # No Color test-commands: @echo $(ENVFILE) @echo $(DOCKER_COMPOSE_FILE) - @echo $(CIENV) @echo $(API_POSTGRES_DB) @echo $(GEO_POSTGRES_USER) @@ -128,8 +127,7 @@ test-start-services: clean-slate docker-compose $(DOCKER_COMPOSE_FILE) exec -T api ./apps/api/entrypoint.sh run-migrations-for-e2e-tests && \ docker-compose $(DOCKER_COMPOSE_FILE) exec -T geoprocessing ./apps/geoprocessing/entrypoint.sh run-migrations-for-e2e-tests -seed-dbs-e2e: test-start-services - $(MAKE) seed-api-with-test-data +seed-dbs-e2e: test-start-services seed-api-with-test-data test-e2e-api: seed-dbs-e2e docker-compose $(DOCKER_COMPOSE_FILE) exec -T api ./apps/api/entrypoint.sh test-e2e @@ -138,13 +136,7 @@ test-e2e-geoprocessing: seed-dbs-e2e docker-compose $(DOCKER_COMPOSE_FILE) exec -T geoprocessing ./apps/geoprocessing/entrypoint.sh test-e2e test-e2e-backend: test-e2e-api test-e2e-geoprocessing - $(MAKE) test-clean-slate - -run-test-e2e-local: - $(MAKE) --keep-going test-e2e-backend environment=local - -run-test-e2e-ci: - $(MAKE) --keep-going test-e2e-backend environment=ci + $(MAKE) test-clean-slate $(ARGS) setup-test-unit-backend: # build API and geoprocessing containers @@ -161,7 +153,7 @@ test-unit-geo: test-unit-backend: setup-test-unit-backend test-unit-api test-unit-geo run-test-unit: - $(MAKE) --keep-going test-unit-backend + $(MAKE) --keep-going test-unit-backend $(ARGS) dump-geodb-data: docker-compose exec -T postgresql-geo-api pg_dump -T migrations -a -U "${GEO_POSTGRES_USER}" -F t ${GEO_POSTGRES_DB} | gzip > data/data/processed/db_dumps/geo_db-$$(date +%Y-%m-%d).tar.gz diff --git a/api/apps/api/config/custom-environment-variables.json b/api/apps/api/config/custom-environment-variables.json index 742a8b2676..8a29b6d216 100644 --- a/api/apps/api/config/custom-environment-variables.json +++ b/api/apps/api/config/custom-environment-variables.json @@ -3,7 +3,6 @@ "muteAll": "API_LOGGING_MUTE_ALL" }, "postgresApi": { - "url": "API_POSTGRES_URL", "host": "API_POSTGRES_HOST", "username": "API_POSTGRES_USER", "password": "API_POSTGRES_PASSWORD", @@ -12,7 +11,6 @@ "runMigrationsOnStartup": "API_RUN_MIGRATIONS_ON_STARTUP" }, "postgresGeoApi": { - "url": "GEO_POSTGRES_URL", "host": "GEO_POSTGRES_HOST", "username": "GEO_POSTGRES_USER", "password": "GEO_POSTGRES_PASSWORD", diff --git a/api/apps/geoprocessing/config/custom-environment-variables.json b/api/apps/geoprocessing/config/custom-environment-variables.json index aa969a724f..39c4926665 100644 --- a/api/apps/geoprocessing/config/custom-environment-variables.json +++ b/api/apps/geoprocessing/config/custom-environment-variables.json @@ -1,6 +1,5 @@ { "postgresApi": { - "url": "API_POSTGRES_URL", "host": "API_POSTGRES_HOST", "username": "API_POSTGRES_USER", "password": "API_POSTGRES_PASSWORD", @@ -9,7 +8,6 @@ "runMigrationsOnStartup": "API_RUN_MIGRATIONS_ON_STARTUP" }, "postgresGeoApi": { - "url": "GEO_POSTGRES_URL", "host": "GEO_POSTGRES_HOST", "username": "GEO_POSTGRES_USER", "password": "GEO_POSTGRES_PASSWORD", diff --git a/docker-compose-test-e2e.ci.yml b/docker-compose-test-e2e.ci.yml deleted file mode 100644 index 96ce4d8a81..0000000000 --- a/docker-compose-test-e2e.ci.yml +++ /dev/null @@ -1,85 +0,0 @@ -version: "3.6" -services: - api: - environment: - - API_SERVICE_PORT - - API_SERVICE_URL - - API_AUTH_JWT_SECRET - - API_RUN_MIGRATIONS_ON_STARTUP - - API_LOGGING_MUTE_ALL - - API_POSTGRES_USER - - API_POSTGRES_PASSWORD - - API_POSTGRES_DB - - POSTGRES_API_SERVICE_PORT - - APP_SERVICE_PORT - - GEOPROCESSING_SERVICE_PORT - - GEOPROCESSING_RUN_MIGRATIONS_ON_STARTUP - - GEO_POSTGRES_USER - - GEO_POSTGRES_PASSWORD - - GEO_POSTGRES_DB - - POSTGRES_GEO_SERVICE_PORT - - REDIS_API_SERVICE_PORT - - API_AUTH_X_API_KEY - - geoprocessing: - environment: - - API_SERVICE_PORT - - API_SERVICE_URL - - API_AUTH_JWT_SECRET - - API_RUN_MIGRATIONS_ON_STARTUP - - API_LOGGING_MUTE_ALL - - API_POSTGRES_USER - - API_POSTGRES_PASSWORD - - API_POSTGRES_DB - - POSTGRES_API_SERVICE_PORT - - APP_SERVICE_PORT - - GEOPROCESSING_SERVICE_PORT - - GEOPROCESSING_RUN_MIGRATIONS_ON_STARTUP - - GEO_POSTGRES_USER - - GEO_POSTGRES_PASSWORD - - GEO_POSTGRES_DB - - POSTGRES_GEO_SERVICE_PORT - - REDIS_API_SERVICE_PORT - - API_AUTH_X_API_KEY - - test-e2e-postgresql-api: - environment: - - API_SERVICE_PORT - - API_SERVICE_URL - - API_AUTH_JWT_SECRET - - API_RUN_MIGRATIONS_ON_STARTUP - - API_POSTGRES_USER - - API_POSTGRES_PASSWORD - - API_POSTGRES_DB - - POSTGRES_API_SERVICE_PORT - - APP_SERVICE_PORT - - GEOPROCESSING_SERVICE_PORT - - GEOPROCESSING_RUN_MIGRATIONS_ON_STARTUP - - GEO_POSTGRES_USER - - GEO_POSTGRES_PASSWORD - - GEO_POSTGRES_DB - - POSTGRES_GEO_SERVICE_PORT - - REDIS_API_SERVICE_PORT - - test-e2e-postgresql-geo-api: - environment: - - API_SERVICE_PORT - - API_SERVICE_URL - - API_AUTH_JWT_SECRET - - API_RUN_MIGRATIONS_ON_STARTUP - - API_POSTGRES_USER - - API_POSTGRES_PASSWORD - - API_POSTGRES_DB - - POSTGRES_API_SERVICE_PORT - - APP_SERVICE_PORT - - GEOPROCESSING_SERVICE_PORT - - GEOPROCESSING_RUN_MIGRATIONS_ON_STARTUP - - GEO_POSTGRES_USER - - GEO_POSTGRES_PASSWORD - - GEO_POSTGRES_DB - - POSTGRES_GEO_SERVICE_PORT - - REDIS_API_SERVICE_PORT - - test-e2e-redis: - environment: - - REDIS_API_SERVICE_PORT diff --git a/docker-compose-test-e2e.local.yml b/docker-compose-test-e2e.local.yml deleted file mode 100644 index f378807ded..0000000000 --- a/docker-compose-test-e2e.local.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: "3.8" -services: - api: - env_file: .env-test-e2e - - geoprocessing: - env_file: .env-test-e2e - - test-e2e-postgresql-api: - env_file: .env-test-e2e - - test-e2e-postgresql-geo-api: - env_file: .env-test-e2e - - test-e2e-redis: - env_file: .env-test-e2e diff --git a/docker-compose-test-e2e.yml b/docker-compose-test-e2e.yml index a9c20b1dfb..a50bb15574 100644 --- a/docker-compose-test-e2e.yml +++ b/docker-compose-test-e2e.yml @@ -18,12 +18,13 @@ services: - ./api/apps:/opt/marxan-api/apps - ./api/libs:/opt/marxan-api/libs - marxan-cloud-backend-e2e-temp-storage:/tmp/storage + env_file: ${ENVFILE} environment: - NODE_PATH=src - NODE_ENV=test - - API_POSTGRES_URL=postgres://${API_POSTGRES_USER}:${API_POSTGRES_PASSWORD}@test-e2e-postgresql-api:5432/${API_POSTGRES_DB} - - GEO_POSTGRES_URL=postgres://${GEO_POSTGRES_USER}:${GEO_POSTGRES_PASSWORD}@test-e2e-postgresql-geo-api:5432/${GEO_POSTGRES_DB} - REDIS_HOST=test-e2e-redis + - API_POSTGRES_HOST=test-e2e-postgresql-api + - GEO_POSTGRES_HOST=test-e2e-postgresql-geo-api - NODE_CONFIG_DIR=apps/api/config depends_on: - test-e2e-postgresql-api @@ -46,12 +47,13 @@ services: - ./api/apps:/opt/marxan-geoprocessing/apps - ./api/libs:/opt/marxan-geoprocessing/libs - marxan-cloud-backend-e2e-temp-storage:/tmp/storage + env_file: ${ENVFILE} environment: - NODE_PATH=src - NODE_ENV=test - - API_POSTGRES_URL=postgres://${API_POSTGRES_USER}:${API_POSTGRES_PASSWORD}@test-e2e-postgresql-api:5432/${API_POSTGRES_DB} - - GEO_POSTGRES_URL=postgres://${GEO_POSTGRES_USER}:${GEO_POSTGRES_PASSWORD}@test-e2e-postgresql-geo-api:5432/${GEO_POSTGRES_DB} - REDIS_HOST=test-e2e-redis + - API_POSTGRES_HOST=test-e2e-postgresql-api + - GEO_POSTGRES_HOST=test-e2e-postgresql-geo-api - NODE_CONFIG_DIR=apps/geoprocessing/config depends_on: - test-e2e-postgresql-geo-api diff --git a/docker-compose.yml b/docker-compose.yml index d3847c2dbb..5e5bf85ea4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,14 +17,14 @@ services: - ./api/apps:/opt/marxan-api/apps - ./api/libs:/opt/marxan-api/libs - marxan-cloud-backend-temp-storage:/tmp/storage - env_file: .env + env_file: ${ENVFILE} environment: - NODE_PATH=src - NODE_ENV=development - - API_POSTGRES_URL=postgres://${API_POSTGRES_USER}:${API_POSTGRES_PASSWORD}@marxan-postgresql-api:5432/${API_POSTGRES_DB} - - GEO_POSTGRES_URL=postgres://${GEO_POSTGRES_USER}:${GEO_POSTGRES_PASSWORD}@marxan-postgresql-geo-api:5432/${GEO_POSTGRES_DB} - REDIS_HOST=marxan-redis - NODE_CONFIG_DIR=apps/api/config + - API_POSTGRES_HOST=postgresql-api + - GEO_POSTGRES_HOST=postgresql-geo-api depends_on: - postgresql-api - postgresql-geo-api @@ -41,7 +41,7 @@ services: volumes: - ./app/src:/opt/marxan-app/src - ./app/test:/opt/marxan-app/test - env_file: .env + env_file: ${ENVFILE} environment: - NODE_PATH=src - NODE_ENV=development @@ -62,14 +62,14 @@ services: - ./api/apps:/opt/marxan-geoprocessing/apps - ./api/libs:/opt/marxan-geoprocessing/libs - marxan-cloud-backend-temp-storage:/tmp/storage - env_file: .env + env_file: ${ENVFILE} environment: - NODE_PATH=src - NODE_ENV=development - - API_POSTGRES_URL=postgres://${API_POSTGRES_USER}:${API_POSTGRES_PASSWORD}@marxan-postgresql-api:5432/${API_POSTGRES_DB} - - GEO_POSTGRES_URL=postgres://${GEO_POSTGRES_USER}:${GEO_POSTGRES_PASSWORD}@marxan-postgresql-geo-api:5432/${GEO_POSTGRES_DB} - REDIS_HOST=marxan-redis - NODE_CONFIG_DIR=apps/geoprocessing/config + - API_POSTGRES_HOST=postgresql-api + - GEO_POSTGRES_HOST=postgresql-geo-api depends_on: - postgresql-geo-api - postgresql-api @@ -160,8 +160,7 @@ services: image: apache/airflow:2.0.0 restart: on-failure entrypoint: ['sh','./scripts/entrypoint.sh'] - env_file: - - .env + env_file: ${ENVFILE} environment: - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgresql-airflow/airflow - AIRFLOW__CORE__EXECUTOR=LocalExecutor diff --git a/env.default b/env.default index 2a7e3090b5..efec3dc11e 100644 --- a/env.default +++ b/env.default @@ -32,3 +32,4 @@ APPLICATION_BASE_URL=http://localhost:3000 PASSWORD_RESET_TOKEN_PREFIX=/auth/reset-password?token= PASSWORD_RESET_EXPIRATION=1800000 SIGNUP_CONFIRMATION_TOKEN_PREFIX=/auth/sign-up-confirmation?token= +IS_DOCKER_E2E_TESTS_RUNTIME=false