update dockerfiles so that it automatically spins up celery and redis…

… in dev env

.github/Architecture.md

@@ -41,6 +41,7 @@
 |  |  |  |  |- 📜 dl_model.py : torch model based on user specifications from drag and drop
 |  |  |  |  |- 📜 dataset.py : read in the dataset through URL or file upload
 |  |  |  |  |- 📜 __init__.py
+|  |  |  |  |- 📜 Dockerfile
 |  |  |  |  |- 📜 worker.py
 |  |  |  |  |- 📜 optimizer.py : what optimizer to use (ie: SGD or Adam for now)
 |  |  |  |- 📜 __init__.py
@@ -63,7 +64,6 @@
 |  |- 📜 manage.py
 |  |- 📜 environment.yml
 |  |- 📜 docker-compose.prod.yml
-|  |- 📜 Dockerfile.prod
 ```
 
 ## Frontend Architecture

frontend/src/pages/train/[train_space_id].tsx

@@ -51,6 +51,7 @@ const TrainSpace = () => {
   const router = useRouter();
   useEffect(() => {
     if (router.isReady && !user) {
+      console.log("redirect to login");
       router.replace({ pathname: "/login" });
     }
   }, [user, router.isReady]);

training/Dockerfile

@@ -1,16 +1,12 @@
-# pull base image
-FROM condaforge/miniforge3
+FROM condaforge/miniforge3 AS base
 
-WORKDIR /usr/src/training
-
-# set environment variables
+WORKDIR /usr/src/build
 
 # install dependencies
 RUN mamba create --name dlp -y
 
 COPY environment.yml pyproject.toml poetry.lock ./
 RUN mamba run --live-stream -n dlp mamba env update --file environment.yml --prune
-RUN mamba run --live-stream -n dlp poetry install
 
 # create directory for the app user
 RUN mkdir -p /home/app
@@ -19,18 +15,39 @@ RUN mkdir -p /home/app
 RUN addgroup --system app && adduser --system --group app
 
 # create the appropriate directories
-ENV HOME=/home/app
-ENV APP_HOME=/home/app/web
+ARG HOME=/home/app
+ARG APP_HOME=/home/app/training
 RUN mkdir $APP_HOME
+
+
+
+######################## development target ######################## 
+FROM base AS development
+ENV ENVIRONMENT development
+
+RUN mamba run --live-stream -n dlp poetry install
+
+
+# we previously mounted training/ to APP_HOME
 WORKDIR $APP_HOME
+RUN chown -R app:app $APP_HOME 
+USER app
+
+CMD mamba run --live-stream -n dlp poetry run python manage.py runserver 0.0.0.0:8000
+
+
+
+######################## production target ######################### 
+FROM base AS production
+ENV ENVIRONMENT production
+
+RUN mamba run --live-stream -n dlp poetry install --without dev
 
 # copy project
+WORKDIR $APP_HOME
 COPY . $APP_HOME
 
-# chown all the files to the app user
 RUN chown -R app:app $APP_HOME
-
-# change to the app user
 USER app
 
-CMD mamba run --live-stream -n dlp poetry run python manage.py runserver 0.0.0.0:8000
+CMD mamba run --live-stream -n dlp gunicorn training.wsgi:application --bind 0.0.0.0:8000

training/README.md

@@ -1,9 +1,8 @@
 ## Run docker
-From the training/ directory
-Dev: docker-compose up
-Production: docker-compose -f docker-compose.prod.yml up
+Ensure you have logged in to AWS, using (`aws configure sso`) then (`aws sso login --profile=DLP`)
 
-## Rebuild and run docker
 From the training/ directory
-Dev: docker-compose up --build
-Production: docker-compose -f docker-compose.prod.yml up --build
+Dev: `AWS_PROFILE=DLP docker-compose up`
+Production: `AWS_PROFILE=DLP docker compose -f docker-compose.prod.yml up`
+
+To rebuild, add a --build flag to the command

training/docker-compose.prod.yml

@@ -1,11 +1,13 @@
-version: '3.8'
+version: "3.8"
 
 services:
   web:
     build:
-      dockerfile: Dockerfile.prod
+      context: .
+      target: production
     volumes:
-      - ./:/usr/src/training/
-      - $HOME/.aws/credentials:/home/app/.aws/credentials:ro
+      - $HOME/.aws:/home/app/.aws:ro
     ports:
       - 8000:8000
+    environment:
+      - AWS_PROFILE=$AWS_PROFILE

training/docker-compose.yml

@@ -2,9 +2,31 @@ version: '3.8'
 
 services:
   web:
-    build: .
+    build:
+      context: .
+      target: development
     volumes:
-      - ./:/usr/src/training/
-      - $HOME/.aws/credentials:/home/app/.aws/credentials:ro
+      - ./:/home/app/training
+      - $HOME/.aws:/home/app/.aws:ro
     ports:
       - 8000:8000
+    environment:
+      - AWS_PROFILE=$AWS_PROFILE
+    depends_on:
+      - celery
+  celery:
+    build:
+      context: .
+      dockerfile: ./training/core/celery/Dockerfile
+      target: development
+    volumes:
+      - ./:/home/app/training
+      - $HOME/.aws:/home/app/.aws:ro
+    environment:
+      - AWS_PROFILE=$AWS_PROFILE
+    depends_on:
+      - redis
+  redis:
+    image: redis:7.2.4
+    expose:
+      - 6379

training/training/celeryconfig.py

@@ -1,11 +1,23 @@
-broker_url = "sqs://"  # "redis://localhost:6379"
+import os
 
-broker_transport_options = {
-    "predefined_queues": {
-        "training-queue.fifo": {
-            "url": "https://sqs.us-east-1.amazonaws.com/521654603461/training-queue.fifo",
+# note, this file is used both in training/ and in the celery worker
+if "ENVIRONMENT" in os.environ and os.environ["ENVIRONMENT"] == "production":
+    print(
+        "ENVIRONMENT env var set to production, setting broker_url to sqs training-queue"
+    )
+    broker_url = "sqs://"
+
+    broker_transport_options = {
+        "predefined_queues": {
+            "training-queue.fifo": {
+                "url": "https://sqs.us-east-1.amazonaws.com/521654603461/training-queue.fifo",
+            }
         }
     }
-}
 
-task_default_queue = "training-queue.fifo"
+    task_default_queue = "training-queue.fifo"
+else:
+    print(
+        "ENVIRONMENT env var either not found or not set to production, setting broker_url to redis://redis:6379"
+    )
+    broker_url = "redis://redis:6379"

training/training/core/celery/Dockerfile

@@ -0,0 +1,51 @@
+FROM condaforge/miniforge3 AS base
+
+WORKDIR /usr/src/build
+
+# install dependencies
+RUN mamba create --name dlp -y
+
+COPY environment.yml pyproject.toml poetry.lock ./
+RUN mamba run --live-stream -n dlp mamba env update --file environment.yml --prune
+
+# create directory for the app user
+RUN mkdir -p /home/app
+
+# create the app user
+RUN addgroup --system app && adduser --system --group app
+
+# create the appropriate directories
+ARG HOME=/home/app
+ARG APP_HOME=/home/app/training
+RUN mkdir $APP_HOME
+
+
+
+######################## development target ########################
+FROM base AS development
+ENV ENVIRONMENT development
+
+RUN mamba run --live-stream -n dlp poetry install
+
+
+# we previously mounted training/ to APP_HOME
+WORKDIR $APP_HOME
+RUN chown -R app:app $APP_HOME 
+USER app
+
+CMD mamba run --live-stream -n dlp celery -A training.core.celery.worker worker --loglevel=INFO
+
+######################## production target ######################### 
+FROM base AS production
+ENV ENVIRONMENT production
+
+RUN mamba run --live-stream -n dlp poetry install --without dev
+
+# copy project
+WORKDIR $APP_HOME
+COPY . $APP_HOME
+
+RUN chown -R app:app $APP_HOME
+USER app
+
+CMD mamba run --live-stream -n dlp celery -A training.core.celery.worker worker --loglevel=INFO