diff --git a/.gitignore b/.gitignore index 40b0a6a6e8..556a135d51 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ __pypackages__/ # Editor generated files # .idea/ .vscode/ +.trunk/ [._]*.s[a-v][a-z] [._]*.sw[a-p] [._]s[a-rt-v][a-z] @@ -64,3 +65,6 @@ htmlcov/ # Terragrunt .terragrunt-cache + +# Docker & Docker compose +docker-compose.override.yml diff --git a/Dockerfile b/Dockerfile index ed62c181d0..178c3444a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -89,7 +89,6 @@ CMD ["python", "-m", "debugpy", "--wait-for-client", "--listen", "0.0.0.0:5678", FROM runtime as prod USER root -# Get the necessary bits for the health check RUN apt-get update && \ apt-get install -y curl && \ apt-get clean && \ @@ -97,8 +96,7 @@ RUN apt-get update && \ # Pre-compile packages to .pyc (init speed gains) RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)" RUN python -m compileall . -EXPOSE 8000/tcp -HEALTHCHECK --interval=60s --start-period=15s CMD ["curl", "-f", "http://localhost:8000/api/v2/system/heartbeat/", "||", "exit", "1"] +EXPOSE 5000/tcp USER appuser:appuser CMD ["gunicorn", "-c", "python:backend.gunicorn", "manage:application", \ "--workers", "1", "--log-level", "error"] diff --git a/docker-compose.override.sample.yml b/docker-compose.override.sample.yml new file mode 100644 index 0000000000..c6c01b0c0f --- /dev/null +++ b/docker-compose.override.sample.yml @@ -0,0 +1,27 @@ +version: "3" + +name: tasking-manager-main + +services: + tm-db: + # Database container binds to host's 5433 port, update to use any other port. + ports: + - 5432:5432 + + tm-backend: + # Backend binds to 8001 by default, change to use other ports. + # Make sure to update, TM_APP_API_URL=http://127.0.0.1:8001 in tasking-manager.env accordingly. + ports: + - 5000:5000 + + tm-frontend: + build: + context: . + dockerfile: "./scripts/docker/Dockerfile.frontend_development" + # Frontend development server is binded to host's 8000 port by default, update to use any other port. + # Make sure to update, TM_APP_BASE_URL=http://127.0.0.1:8000 in tasking-manager.env accordingly. + ports: + - 3000:3000 + volumes: + - ./frontend:/usr/src/app + - /usr/src/app/node_modules \ No newline at end of file diff --git a/docker-compose.override.yml b/docker-compose.override.yml deleted file mode 100644 index 3cfcc866c3..0000000000 --- a/docker-compose.override.yml +++ /dev/null @@ -1,30 +0,0 @@ -# To use this file, run `docker-compose up`. -version: '3.4' - -x-backend-config: &backend - build: - context: "." - dockerfile: "./scripts/docker/Dockerfile.backend" - -services: - # Main application - backend: - <<: *backend - volumes: - - .:/usr/src/app - - # Migration service - migration: - <<: *backend - volumes: - - .:/usr/src/app - - frontend: - build: - context: "." - dockerfile: "./scripts/docker/Dockerfile.frontend" - volumes: - - ".:/usr/src/app" - labels: - - traefik.http.routers.frontend.rule=Host(`localhost`) || Host(`127.0.0.1`) - - traefik.http.services.frontend.loadbalancer.server.port=80 diff --git a/docker-compose.yml b/docker-compose.yml index b342c6bbeb..2a908298d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,66 +1,101 @@ -# To use this file, run `docker-compose up`. -version: "3.4" +version: "3" -x-backend-config: &backend - image: hotosm-tasking-manager:backend - env_file: ${ENV_FILE:-tasking-manager.env} - depends_on: - - postgresql - links: - - postgresql - networks: - - tm-web +name: tasking-manager-main + +volumes: + tm_db_data: + name: tm-db-data-main + +networks: + tm-net: + name: tm-net services: - # Main application - backend: - <<: *backend - container_name: tm-backend - restart: always + tm-db: + image: "postgis/postgis:${POSTGIS_TAG:-14-3.3}" + volumes: + - tm_db_data:/var/lib/postgresql/data/ + env_file: + - tasking-manager.env + restart: unless-stopped + healthcheck: + test: pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB} + start_period: 35s + interval: 10s + timeout: 5s + retries: 3 + networks: + - tm-net + + tm-backend: + image: ghcr.io/hotosm/tasking-manager/backend:main + build: + context: . + depends_on: + tm-db: + condition: service_healthy + tm-migration: + condition: service_completed_successfully + env_file: + - tasking-manager.env + restart: unless-stopped + healthcheck: + test: curl --fail http://localhost:5000 || exit 1 + interval: 10s + retries: 5 + start_period: 10s + timeout: 3s + deploy: + replicas: ${API_REPLICAS:-1} + resources: + limits: + cpus: "1" + memory: 1500M + reservations: + cpus: "1" + memory: 100M labels: - traefik.http.routers.backend.rule=(Host(`127.0.0.1`) || Host(`localhost`)) && PathPrefix(`/api/`) - traefik.http.services.backend.loadbalancer.server.port=5000 + networks: + - tm-net - migration: - <<: *backend - container_name: tm-migration - restart: on-failure - command: flask db upgrade - - frontend: - image: hotosm-tasking-manager:frontend - container_name: tm-frontend - restart: always + tm-migration: + image: ghcr.io/hotosm/tasking-manager/backend:main + build: + context: . + entrypoint: ["python", "manage.py", "db", "upgrade"] + depends_on: + tm-db: + condition: service_healthy + env_file: + - tasking-manager.env + deploy: + replicas: ${API_REPLICAS:-1} + resources: + limits: + cpus: "1" + memory: 1500M + reservations: + cpus: "1" + memory: 100M networks: - - tm-web - labels: - - traefik.http.routers.frontend.rule=Host(`127.0.0.1`) || Host(`localhost`) - - traefik.http.services.frontend.loadbalancer.server.port=3000 + - tm-net swagger: image: swaggerapi/swagger-ui:v5.11.10 - container_name: tm-swagger-docs restart: always - networks: - - tm-web environment: - BASE_URL=/docs - SWAGGER_JSON_URL=http://127.0.0.1:${TM_DEV_PORT:-3000}/api/v2/system/docs/json/ labels: - traefik.http.routers.swagger.rule=(Host(`127.0.0.1`) || Host(`localhost`)) && PathPrefix(`/docs/`) - traefik.http.services.swagger.loadbalancer.server.port=8080 - - postgresql: - image: postgis/postgis:14-3.3 - container_name: tm-postgresql - restart: always - env_file: ${ENV_FILE:-tasking-manager.env} networks: - - tm-web + - tm-net traefik: image: traefik:v2.10 - container_name: tm-traefik restart: always ports: - "${TM_DEV_PORT:-3000}:80" @@ -70,8 +105,17 @@ services: - --entrypoints.web.address=:80 - --providers.docker=true networks: - - tm-web + - tm-net -networks: - tm-web: - external: false + tm-frontend: + image: ghcr.io/hotosm/tasking-manager/frontend:main + build: + context: . + dockerfile: "./scripts/docker/Dockerfile.frontend_development" + env_file: + - tasking-manager.env + labels: + - traefik.http.routers.frontend.rule=Host(`127.0.0.1`) || Host(`localhost`) + - traefik.http.services.frontend.loadbalancer.server.port=3000 + networks: + - tm-net diff --git a/docs/developers/development-setup.md b/docs/developers/development-setup.md index 9606b0d874..6585506519 100644 --- a/docs/developers/development-setup.md +++ b/docs/developers/development-setup.md @@ -74,12 +74,16 @@ The easiest option to get started with all components may be using Docker. ### Requirements -[Docker Engine](https://docs.docker.com/engine/install/) must be -available locally. +[Docker Engine](https://docs.docker.com/engine/install/) must be available locally. ### Running Tasking Manager -Once the steps above have been complete, simply run: +Once you have the docke engine running, Quickly generate an environment file from an existing `example.env`. +```bash +cp example.env tasking-manager.env +``` + +Now you can proceed with starting the services. ```bash docker compose pull @@ -92,7 +96,7 @@ Tasking Manager should be available from: #### (Optional) Changing the dev port or dotenv file -You change the default port from 3000 to any other port. +You change the default port from `3000` to any other port. However, you must change your OAuth redirect URL to reflect this, in addition to any variables including a port, e.g. TM_APP_BASE_URL. @@ -102,7 +106,75 @@ The default dotenv file can also be changed. ```bash TM_DEV_PORT=9000 ENV_FILE=.env docker compose up --detach ``` +```bash +docker compose build +docker compose up --detach +``` +#### (Optional) Overriding `docker-compose.yml` +If you want to add custom configuration for the docker services. You can make a copy of `docker-compose.override.sample.yml` which you can edit as per your need. + +Create an override file from sample. +``` +cp docker-compose.override.sample.yml docker-compose.override.yml +``` + +### External or Self Hosted Database + +If you want to use your local postgresql server or some other exter database service. +Find these sets of environment variables in `tasking-manager.env` +```bash +POSTGRES_DB=tasking-manager +POSTGRES_USER=tm +POSTGRES_PASSWORD=tm +POSTGRES_ENDPOINT= +POSTGRES_PORT=5432 +``` +> **_NOTE:_** If database server is self managed on your local machine, Use your machine's ip address. Also make sure it can be reachable from `tm-backend` container. + +Once Updated, recreate containers with +``` +docker compose up -d +``` +### Frontend Only Deployment +If you are looking to deploy only Frontend service with docker, You will need to make sure the following env vars are corrent in `tasking-manager.env` + +``` +TM_APP_API_URL=http://127.0.0.1:5000 +``` +This refers to the backend service that you are going to consume, If you don't have a Tasking Manager backend instance you can use the staging server hosted by hotosm. +``` +TM_APP_API_URL=https://tasking-manager-staging-api.hotosm.org +``` +Then proceed with starting only frontend service with docker. +``` +docker compose up -d tm-frontend +``` + +Check server logs with +``` +docker logs tasking-manager-main-tm-frontend-1 -f + +> TaskingManager-frontend@0.1.0 patch-rapid +> bash -c "cp patch/rapid-imagery.min.json public/static/rapid/data/imagery.min.json" + +ℹ ｢wds｣: Project is running at http://172.22.0.2/ +ℹ ｢wds｣: webpack output is served from +ℹ ｢wds｣: Content not from webpack is served from /usr/src/app/public +ℹ ｢wds｣: 404s will fallback to / +Starting the development server... + +Compiled successfully! + +You can now view TaskingManager-frontend in the browser. + + Local: http://localhost:3000 + On Your Network: http://172.22.0.2:3000 + +Note that the development build is not optimized. +To create a production build, use yarn build. +``` +For OSM related `CLIENT_ID` and `SECRETS` check [OSM AUTH](#osm-auth) section. ## Running Components Standalone ### Frontend diff --git a/example.env b/example.env index 7ba9235e7e..37f9e59302 100644 --- a/example.env +++ b/example.env @@ -134,8 +134,8 @@ TM_SCOPE=read_prefs write_api POSTGRES_DB=tasking-manager POSTGRES_USER=tm POSTGRES_PASSWORD=tm -POSTGRES_ENDPOINT=postgresql -# POSTGRES_PORT=5432 +POSTGRES_ENDPOINT=tm-db +POSTGRES_PORT=5432 # The postgres database name used for testing (required). # All other configurations except the database name are inherited from the main database defined above. diff --git a/scripts/docker/Dockerfile.frontend_development b/scripts/docker/Dockerfile.frontend_development index 75c459df00..fddcd29a6b 100644 --- a/scripts/docker/Dockerfile.frontend_development +++ b/scripts/docker/Dockerfile.frontend_development @@ -1,25 +1,10 @@ -FROM alpine/git as base +FROM node:16 -RUN mkdir -p /usr/src/app WORKDIR /usr/src/app -# Get the Tasking Manager -ARG branch=develop -RUN git clone --depth=1 git://github.com/hotosm/tasking-manager.git \ - --branch $branch /usr/src/app - -RUN rm -rf backend/ migrations/ - -FROM node:16 as build - -WORKDIR /usr/src/app - -COPY tasking-manager.env .. -COPY frontend . - +COPY ./frontend . ## SETUP RUN npm install # SERVE -CMD ["npm", "start"] - +CMD ["npm", "start"] \ No newline at end of file