feat(deploy): prod deployment and devops setup

.github/workflows/deploy.yml

@@ -34,17 +34,29 @@ jobs:
           VERSION=v$TAG_NAME
           echo "VERSION=$VERSION" >> $GITHUB_ENV
 
-          docker tag napse_dtk_prod_litestream ghcr.io/${{ github.repository }}/napse_dtk_prod_litestream:$VERSION
           docker tag napse_dtk_prod_django ghcr.io/${{ github.repository }}/napse_dtk_prod_django:$VERSION
-      
-      - name: Push Litestream Docker image
+            
+      - name: Push Django Docker image
         env:
           VERSION: ${{ env.VERSION }}
         run: |
-          docker push ghcr.io/${{ github.repository }}/napse_dtk_prod_litestream:${{ env.VERSION }}
-      
-      - name: Push Django Docker image
+          docker push ghcr.io/${{ github.repository }}/napse_dtk_prod_django:${{ env.VERSION }}
+
+      - name: Push Django Docker image latest
+        run: |
+          docker push ghcr.io/${{ github.repository }}/napse_dtk_prod_django:latest
+
+      - name: Create .zip file
         env:
           VERSION: ${{ env.VERSION }}
         run: |
-          docker push ghcr.io/${{ github.repository }}/napse_dtk_prod_django:${{ env.VERSION }}
+          make setup-prod
+          zip -r deploy-$VERSION.zip provisionEB/*
+      
+      - name: Push .zip file
+        uses: actions/upload-artifact@v2
+        env:
+          VERSION: ${{ env.VERSION }}
+        with:
+          name: deploy-package-aws-eb:$VERSION
+          path: deploy-$VERSION.zip 

backend/.envs/.dev_as_prod/.django

@@ -10,10 +10,9 @@ REDIS_URL="redis://redis:6379/0"
 # Django
 # ------------------------------------------------------------------------------
 DJANGO_SETTINGS_MODULE="config.settings.production"
-SECRET_KEY="faNOpRrPjt59ZE7ZYTdLIYkaBG0YluPfzNFhtkuN1Bo"
+DJANGO_SECRET_KEY="faNOpRrPjt59ZE7ZYTdLIYkaBG0YluPfzNFhtkuN1Bo"
 DJANGO_DEBUG=True
 IS_LOCAL=True
-DB_ENGINE="POSTGRES"
 
 # Flower
 # ------------------------------------------------------------------------------

backend/.envs/.dev_as_prod/.postgres

@@ -4,4 +4,7 @@ POSTGRES_HOST = "postgres"
 POSTGRES_PORT = 5432
 POSTGRES_DB = "napse_developer_toolkit"
 POSTGRES_USER = "f_My4PB9Qig6Ga7sYshUqW_GTgS6oGyoznN1TUjwU1c"
-POSTGRES_PASSWORD = "MJxMbEuFOCDn6sd7IdhauswmW1GGXNtQfPemnGmLFb8j8Uc6R9nU_cCPpxHuZEri34ZJM8mZ7MBFkLOT9uU8iM" # noqa: S105
+POSTGRES_PASSWORD = "MJxMbEuFOCDn6sd7IdhauswmW1GGXNtQfPemnGmLFb8j8Uc6R9nU_cCPpxHuZEri34ZJM8mZ7MBFkLOT9uU8iM"  # noqa: S105
+
+DB_SETUP = "postgres"
+DB_ENGINE = "POSTGRES"

backend/.envs/.development/.django

@@ -9,12 +9,11 @@ REDIS_URL="redis://redis:6379/0"
 
 # Django
 # ------------------------------------------------------------------------------
-SECRET_KEY="faNOpRrPjt59ZE7ZYTdLIYkaBG0YluPfzNFhtkuN1Bo"
+DJANGO_SECRET_KEY="faNOpRrPjt59ZE7ZYTdLIYkaBG0YluPfzNFhtkuN1Bo"
 DJANGO_DEBUG=True
 IS_LOCAL=True
 
 # Flower
 # ------------------------------------------------------------------------------
 CELERY_FLOWER_USER="wRSdgTpIDD8ySZFL14bA6rPPQzMJ0pjXL620YIPUhXk"
 CELERY_FLOWER_PASSWORD="ikQMfq0t0XClQ3MaKaip8ogum0itD0KOMCuX9H-tX60"
-DB_ENGINE="POSTGRES"

backend/.envs/.development/.litestream

@@ -1,4 +1,5 @@
-DB_SETUP=litestream
 LITESTREAM_ACCESS_KEY_ID=minioadmin
 LITESTREAM_SECRET_ACCESS_KEY=minioadmin
-S3_BUCKET_PATH=s3://miniobkt.localhost:9000/db.sqlite3
+S3_BUCKET_PATH=s3://miniobkt.localhost:9000/db.sqlite3
+DB_SETUP="litestream"
+DB_ENGINE="SQLITE"

backend/.envs/.development/.postgres

@@ -5,3 +5,4 @@ POSTGRES_USER = "ouG2p-TRxYcYXH71sA8v8Aqeo2OYjoUC5IdWIz48DmE"
 POSTGRES_PASSWORD = "ePUrk4ffFM34gNEltS-InxEAI9fehVbsK1uaY-2nomAB3ftGIWIJQx3vlLnTPZs1O1BbI4hUDVUdFuiXN_jkjY"  # noqa: S105
 
 DB_SETUP = "postgres"
+DB_ENGINE="POSTGRES"

backend/.envs/.production/.django

@@ -9,12 +9,9 @@ REDIS_URL="redis://redis:6379/0"
 
 # Django
 # ------------------------------------------------------------------------------
-SECRET_KEY="" # TODO : Generate random password
 DJANGO_DEBUG=False
 IS_LOCAL=False
 
-# Flower
+# To fill in in JS
 # ------------------------------------------------------------------------------
-CELERY_FLOWER_USER="" # TODO : Generate random password
-CELERY_FLOWER_PASSWORD="" # TODO : Generate random password
-DB_ENGINE="POSTGRES"
+# DJANGO_SECRET_KEY="" # TODO : Generate random password

backend/.envs/.production/.litestream

@@ -1,4 +1,11 @@
-DB_SETUP=litestream
-LITESTREAM_ACCESS_KEY_ID="" # TODO : Generate random password
-LITESTREAM_SECRET_ACCESS_KEY="" # TODO : Generate random password
-S3_BUCKET_PATH="" # TODO : Create s3 bucket
+# General secrets
+# ------------------------------------------------------------------------------
+DB_SETUP="litestream"
+DB_ENGINE="SQLITE"
+
+# To fill in in JS
+# ------------------------------------------------------------------------------
+# AWS_ACCESS_KEY_ID="" # TODO : Get IAM credentials
+# AWS_SECRET_ACCESS_KEY="" # TODO : Get IAM credentials
+# AWS_S3_BUCKET_URI="" # TODO : Get s3 bucket path
+

backend/config/settings/base.py

@@ -15,7 +15,7 @@
 )
 DEBUG = env.bool("DJANGO_DEBUG", False)
 IS_LOCAL = env.bool("IS_LOCAL", False)
-SECRET_KEY = env("SECRET_KEY")
+SECRET_KEY = env("DJANGO_SECRET_KEY")
 
 
 # TIME ZONE

backend/config/settings/modules/secrets.py

@@ -1 +0,0 @@
-SECRET_KEY = "secret-key"  # noqa: S105

backend/deploy/aws/.ebextensions/01_CloudWatch_Avg_Mem.config

@@ -0,0 +1,41 @@
+files:
+  "/opt/aws/amazon-cloudwatch-agent/bin/config.json":
+    mode: "000600"
+    owner: root
+    group: root
+    content: |
+        {
+            "metrics":{
+                "metrics_collected":{
+                    "mem":{
+                        "measurement":[
+                        "mem_used_percent"
+                        ],
+                        "metrics_collection_interval":60
+                    }
+                },
+                "append_dimensions": {
+                    "InstanceId": "${aws:InstanceId}"
+                }
+            }
+        }
+container_commands:
+  start_cloudwatch_agent:
+    command: /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/bin/config.json -s
+  create_alarm:
+    command: |
+        InstanceId=$(ec2-metadata -i)
+        InstanceId=${InstanceId:13}
+        aws cloudwatch put-metric-alarm \
+            --alarm-name 'MemoryUtilization' \
+            --alarm-description 'Alarm when server Memory exceeds 80 percent' \
+            --metric-name mem_used_percent \
+            --namespace CWAgent \
+            --statistic Average \
+            --period 60 \
+            --threshold 80 \
+            --comparison-operator GreaterThanThreshold \
+            --dimensions Name=InstanceId,Value=$InstanceId \
+            --evaluation-periods 1 \
+            --alarm-actions arn:aws:swf:eu-west-3:073501479403:action/actions/AWS_EC2.InstanceId.Reboot/1.0 \
+            --unit Percent

backend/docker/compose/development/django/celery/worker/start

@@ -3,11 +3,12 @@
 set -o errexit
 set -o nounset
 
+sleep 10
 if [[ ${DB_SETUP} == "postgres" ]]; then 
     /wait-for-it.sh $POSTGRES_HOST:$POSTGRES_PORT --timeout=10 --strict -- echo "postgres is up"
 fi
 
 # local settings
-watchfiles celery.__main__.main --args '-A config.celery_app worker -P prefork -l INFO'
+watchfiles celery.__main__.main --ignore-paths db --args '-A config.celery_app worker -P prefork -l INFO'