Merge pull request #6263 from hotosm/feat-dockerize

Update docker dev setup and docs

.gitignore

@@ -35,6 +35,7 @@ __pypackages__/
 # Editor generated files #
 .idea/
 .vscode/
+.trunk/
 [._]*.s[a-v][a-z]
 [._]*.sw[a-p]
 [._]s[a-rt-v][a-z]
@@ -64,3 +65,6 @@ htmlcov/
 
 # Terragrunt
 .terragrunt-cache
+
+# Docker & Docker compose
+docker-compose.override.yml

Dockerfile

@@ -89,16 +89,14 @@ CMD ["python", "-m", "debugpy", "--wait-for-client", "--listen", "0.0.0.0:5678",
 
 FROM runtime as prod
 USER root
-# Get the necessary bits for the health check
 RUN apt-get update && \
 	apt-get install -y curl && \
 	apt-get clean && \
 	rm -rf /var/lib/apt/lists/*
 # Pre-compile packages to .pyc (init speed gains)
 RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)"
 RUN python -m compileall .
-EXPOSE 8000/tcp
-HEALTHCHECK --interval=60s --start-period=15s CMD ["curl", "-f", "http://localhost:8000/api/v2/system/heartbeat/", "||", "exit", "1"]
+EXPOSE 5000/tcp
 USER appuser:appuser
 CMD ["gunicorn", "-c", "python:backend.gunicorn", "manage:application", \
     "--workers", "1", "--log-level", "error"]

docker-compose.override.sample.yml

@@ -0,0 +1,27 @@
+version: "3"
+
+name: tasking-manager-main
+
+services:
+  tm-db:
+    # Database container binds to host's 5433 port, update to use any other port.
+    ports:
+      - 5432:5432
+
+  tm-backend:
+    # Backend binds to 8001 by default, change to use other ports.
+    # Make sure to update, TM_APP_API_URL=http://127.0.0.1:8001 in tasking-manager.env accordingly.
+    ports:
+      - 5000:5000
+
+  tm-frontend:
+    build:
+      context: .
+      dockerfile: "./scripts/docker/Dockerfile.frontend_development"
+    # Frontend development server is binded to host's 8000 port by default, update to use any other port.
+    # Make sure to update, TM_APP_BASE_URL=http://127.0.0.1:8000 in tasking-manager.env accordingly.
+    ports:
+      - 3000:3000
+    volumes:
+      - ./frontend:/usr/src/app
+      - /usr/src/app/node_modules

docker-compose.yml

@@ -1,66 +1,101 @@
-# To use this file, run `docker-compose up`.
-version: "3.4"
+version: "3"
 
-x-backend-config: &backend
-  image: hotosm-tasking-manager:backend
-  env_file: ${ENV_FILE:-tasking-manager.env}
-  depends_on:
-    - postgresql
-  links:
-    - postgresql
-  networks:
-    - tm-web
+name: tasking-manager-main
+
+volumes:
+  tm_db_data:
+    name: tm-db-data-main
+
+networks:
+  tm-net: 
+    name: tm-net
 
 services:
-  # Main application
-  backend:
-    <<: *backend
-    container_name: tm-backend
-    restart: always
+  tm-db:
+    image: "postgis/postgis:${POSTGIS_TAG:-14-3.3}"
+    volumes:
+      - tm_db_data:/var/lib/postgresql/data/
+    env_file:
+      - tasking-manager.env
+    restart: unless-stopped
+    healthcheck:
+      test: pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}
+      start_period: 35s
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    networks:
+      - tm-net
+
+  tm-backend:
+    image: ghcr.io/hotosm/tasking-manager/backend:main
+    build:
+      context: .
+    depends_on:
+      tm-db:
+        condition: service_healthy
+      tm-migration:
+        condition: service_completed_successfully
+    env_file:
+      - tasking-manager.env
+    restart: unless-stopped
+    healthcheck:
+      test: curl --fail http://localhost:5000 || exit 1
+      interval: 10s
+      retries: 5
+      start_period: 10s
+      timeout: 3s
+    deploy:
+      replicas: ${API_REPLICAS:-1}
+      resources:
+        limits:
+          cpus: "1"
+          memory: 1500M
+        reservations:
+          cpus: "1"
+          memory: 100M
     labels:
       - traefik.http.routers.backend.rule=(Host(`127.0.0.1`) || Host(`localhost`)) && PathPrefix(`/api/`)
       - traefik.http.services.backend.loadbalancer.server.port=5000
+    networks:
+      - tm-net
 
-  migration:
-    <<: *backend
-    container_name: tm-migration
-    restart: on-failure
-    command: flask db upgrade
-
-  frontend:
-    image: hotosm-tasking-manager:frontend
-    container_name: tm-frontend
-    restart: always
+  tm-migration:
+    image: ghcr.io/hotosm/tasking-manager/backend:main
+    build:
+      context: .
+    entrypoint: ["python", "manage.py", "db", "upgrade"]
+    depends_on:
+      tm-db:
+        condition: service_healthy
+    env_file:
+      - tasking-manager.env
+    deploy:
+      replicas: ${API_REPLICAS:-1}
+      resources:
+        limits:
+          cpus: "1"
+          memory: 1500M
+        reservations:
+          cpus: "1"
+          memory: 100M
     networks:
-      - tm-web
-    labels:
-      - traefik.http.routers.frontend.rule=Host(`127.0.0.1`) || Host(`localhost`)
-      - traefik.http.services.frontend.loadbalancer.server.port=3000
+      - tm-net
 
   swagger:
     image: swaggerapi/swagger-ui:v5.11.10
-    container_name: tm-swagger-docs
     restart: always
-    networks:
-      - tm-web
     environment:
       - BASE_URL=/docs
       - SWAGGER_JSON_URL=http://127.0.0.1:${TM_DEV_PORT:-3000}/api/v2/system/docs/json/
     labels:
       - traefik.http.routers.swagger.rule=(Host(`127.0.0.1`) || Host(`localhost`)) && PathPrefix(`/docs/`)
       - traefik.http.services.swagger.loadbalancer.server.port=8080
-
-  postgresql:
-    image: postgis/postgis:14-3.3
-    container_name: tm-postgresql
-    restart: always
-    env_file: ${ENV_FILE:-tasking-manager.env}
     networks:
-      - tm-web
+      - tm-net
 
   traefik:
     image: traefik:v2.10
-    container_name: tm-traefik
     restart: always
     ports:
       - "${TM_DEV_PORT:-3000}:80"
@@ -70,8 +105,17 @@ services:
       - --entrypoints.web.address=:80
       - --providers.docker=true
     networks:
-      - tm-web
+      - tm-net
 
-networks:
-  tm-web:
-    external: false
+  tm-frontend:
+    image: ghcr.io/hotosm/tasking-manager/frontend:main
+    build:
+      context: .
+      dockerfile: "./scripts/docker/Dockerfile.frontend_development"
+    env_file:
+      - tasking-manager.env
+    labels:
+      - traefik.http.routers.frontend.rule=Host(`127.0.0.1`) || Host(`localhost`)
+      - traefik.http.services.frontend.loadbalancer.server.port=3000
+    networks:
+      - tm-net

docs/developers/development-setup.md

@@ -74,12 +74,16 @@ The easiest option to get started with all components may be using Docker.
 
 ### Requirements
 
-[Docker Engine](https://docs.docker.com/engine/install/) must be 
-available locally.
+[Docker Engine](https://docs.docker.com/engine/install/) must be available locally.
 
 ### Running Tasking Manager
 
-Once the steps above have been complete, simply run:
+Once you have the docke engine running, Quickly generate an environment file from an existing `example.env`.
+```bash
+cp example.env tasking-manager.env
+```
+
+Now you can proceed with starting the services.
 
 ```bash
 docker compose pull
@@ -92,7 +96,7 @@ Tasking Manager should be available from:
 
 #### (Optional) Changing the dev port or dotenv file
 
-You change the default port from 3000 to any other port.
+You change the default port from `3000` to any other port.
 
 However, you must change your OAuth redirect URL to reflect this,
 in addition to any variables including a port, e.g. TM_APP_BASE_URL.
@@ -102,7 +106,75 @@ The default dotenv file can also be changed.
 ```bash
 TM_DEV_PORT=9000 ENV_FILE=.env docker compose up --detach
 ```
+```bash
+docker compose build
+docker compose up --detach
+```
+#### (Optional) Overriding `docker-compose.yml`
+If you want to add custom configuration for the docker services. You can make a copy of `docker-compose.override.sample.yml` which you can edit as per your need.
+
+Create an override file from sample.
+```
+cp docker-compose.override.sample.yml docker-compose.override.yml
+```
+
+### External or Self Hosted Database
+
+If you want to use your local postgresql server or some other exter database service.
+Find these sets of environment variables in `tasking-manager.env` 
+```bash
+POSTGRES_DB=tasking-manager
+POSTGRES_USER=tm
+POSTGRES_PASSWORD=tm
+POSTGRES_ENDPOINT=<replace-with-your-database-endpoint>
+POSTGRES_PORT=5432
+```
+> **_NOTE:_**  If database server is self managed on your local machine, Use your machine's ip address. Also make sure it can be reachable from `tm-backend` container.
+
+Once Updated, recreate containers with 
+```
+docker compose up -d
+```
 
+### Frontend Only Deployment
+If you are looking to deploy only Frontend service with docker, You will need to make sure the following env vars are corrent in `tasking-manager.env`
+
+```
+TM_APP_API_URL=http://127.0.0.1:5000
+```
+This refers to the backend service that you are going to consume, If you don't have a Tasking Manager backend instance you can use the staging server hosted by hotosm.
+```
+TM_APP_API_URL=https://tasking-manager-staging-api.hotosm.org
+```
+Then proceed with starting only frontend service with docker.
+```
+docker compose up -d tm-frontend
+```
+
+Check server logs with
+```
+docker logs tasking-manager-main-tm-frontend-1 -f
+
+> [email protected] patch-rapid
+> bash -c "cp patch/rapid-imagery.min.json public/static/rapid/data/imagery.min.json"
+
+ℹ ｢wds｣: Project is running at http://172.22.0.2/
+ℹ ｢wds｣: webpack output is served from
+ℹ ｢wds｣: Content not from webpack is served from /usr/src/app/public
+ℹ ｢wds｣: 404s will fallback to /
+Starting the development server...
+
+Compiled successfully!
+
+You can now view TaskingManager-frontend in the browser.
+
+  Local:            http://localhost:3000
+  On Your Network:  http://172.22.0.2:3000
+
+Note that the development build is not optimized.
+To create a production build, use yarn build.
+```
+For OSM related `CLIENT_ID` and `SECRETS` check [OSM AUTH](#osm-auth) section.
 ## Running Components Standalone
 
 ### Frontend

example.env

@@ -134,8 +134,8 @@ TM_SCOPE=read_prefs write_api
 POSTGRES_DB=tasking-manager
 POSTGRES_USER=tm
 POSTGRES_PASSWORD=tm
-POSTGRES_ENDPOINT=postgresql
-# POSTGRES_PORT=5432
+POSTGRES_ENDPOINT=tm-db
+POSTGRES_PORT=5432
 
 # The postgres database name used for testing (required).
 # All other configurations except the database name are inherited from the main database defined above.