👷(project) add support for multiple releases

We need our CI to build every release to prevent possible side-effects. When publishing a new image, we rebuild only a single release corresponding to the release tag.
openfun · Sep 3, 2019 · acd296d · acd296d
1 parent 3634b9b
commit acd296d
Show file tree

Hide file tree

Showing 6 changed files with 98 additions and 135 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -2,126 +2,87 @@
 # Docker containers CI
 version: 2
 
+# Templates
+defaults: &defaults
+  # We use the machine executor, i.e. a VM, not a container
+  machine:
+    # Cache docker layers so that we strongly speed up this job execution
+    # This cache will be available to future jobs (although because jobs run
+    # in parallel, CircleCI does not guarantee that a given job will see a
+    # specific version of the cache. See documentation for details)
+    docker_layer_caching: true
+
+  working_directory: ~/fun
+
+build_steps: &build_steps
+  steps:
+    # Checkout openedx-docker sources
+    - checkout
+
+    # Install a recent docker-compose release
+    - run:
+        name: Upgrade docker-compose
+        command: |
+          curl -L "https://github.com/docker/compose/releases/download/1.24.1/docker-compose-$(uname -s)-$(uname -m)" -o ~/docker-compose
+          chmod +x ~/docker-compose
+          sudo mv ~/docker-compose /usr/local/bin/docker-compose
+
+    # Production image build. It will be tagged as edxapp:latest
+    - run:
+        name: Build production image
+        command: |
+          source releases/${CIRCLE_JOB}/activate
+          make build
+
+    # Development image build. It uses the "development" Dockerfile target
+    # file and will be tagged as edxapp:dev
+    - run:
+        name: Build development image
+        command: |
+          source releases/${CIRCLE_JOB}/activate
+          make dev-build
+
 # List openedx-docker jobs that will be integrated and executed in a workflow
 jobs:
-  # Build job
-  # Build the Docker images for production and development
-  build:
-    # We use the machine executor, i.e. a VM, not a container
-    machine:
-      # Cache docker layers so that we strongly speed up this job execution
-      # This cache will be available to future jobs (although because jobs run
-      # in parallel, CircleCI does not garantee that a given job will see a
-      # specific version of the cache. See documentation for details)
-      docker_layer_caching: true
-
-    working_directory: ~/fun
-
-    steps:
-      # Checkout openedx-docker sources
-      - checkout
-
-      # Restore the ~/fun/src cached repository. If the cache does not exists for
-      # the current .Revision (commit hash), we fall back to the latest cache
-      # with a label matching 'edx-archive-v1-'
-      - restore_cache:
-          keys:
-            - edx-archive-v1-{{ .Revision }}
-            - edx-archive-v1-
-
-      # Install a recent docker-compose release
-      - run:
-          name: Upgrade docker-compose
-          command: |
-            curl -L "https://github.com/docker/compose/releases/download/1.22.0/docker-compose-$(uname -s)-$(uname -m)" -o ~/docker-compose
-            chmod +x ~/docker-compose
-            sudo mv ~/docker-compose /usr/local/bin/docker-compose
-
-      # Production image build. It will be tagged as edxapp:latest
-      - run:
-          name: Build production image
-          command: |
-            make build
-
-      # Development image build. It uses the "development" Dockerfile target
-      # file and will be tagged as edxapp:dev
-      - run:
-          name: Build development image
-          command: |
-            make dev-build
+  # Build jobs
+  #
+  # Note that the job name should match the EDX_RELEASE value
+  master/bare:
+    <<: [*defaults, *build_steps]
 
-      # Cache Open edX repository (cloned in ~/fun/src) as the checkout is
-      # rather time consuming for this project
-      - save_cache:
-          paths:
-            - ~/fun/releases/master/bare/src/edx-platform
-          key: edx-archive-v1-{{ .Revision }}
-
-      # Save and cache the built images to filesystem so that they will be
-      # available to test and push them to DockerHub in subsequent jobs
-      - run:
-          name: Save docker images to filesystem
-          command: |
-            docker save -o edxapp.tar edxapp:latest edxapp:dev
-      - save_cache:
-          paths:
-            - ~/fun/edxapp.tar
-          key: edx-image-v1-{{ .Revision }}
+  hawthorn/1/bare:
+    <<: [*defaults, *build_steps]
 
   # Hub job
-  # Load and tag production/development images to push them to Dockerhub
-  # public registry
-  # These images are now the latest for this branch so we will publish
-  # both under their tag and under the `latest` tag so that our `latest`
-  # images are always up-to-date
   hub:
-    # We use the machine executor, i.e. a VM, not a container
-    machine: true
-
-    working_directory: ~/fun
+    <<: *defaults
 
     steps:
-      # First, check that the BRANCH name is included in the TAG name. This is important
-      # because we handle several important branches (master, funmooc, funwb, etc.) and
-      # we must make sure that tag names are explicitly linked to a branch in order to
-      # avoid conflicts
+      # Thanks to docker layer caching, rebuilding the image should be blazing
+      # fast!
       - run:
-          name: Check that the BRANCH name is included in the TAG name
+          name: Rebuild production image
           command: |
-            if ! echo ${CIRCLE_TAG} | grep "${CIRCLE_BRANCH}" &> /dev/null; then
-              # Stop the step without failing
-              circleci step halt
-            fi
-
-      # Load the docker images from our cache to the docker engine and check that they
-      # have been effectively loaded
-      - restore_cache:
-          keys:
-            - edx-image-v1-{{ .Revision }}
-      - run:
-          name: Load images to docker engine
-          command: |
-            docker load < edxapp.tar
+            source releases/$(echo ${CIRCLE_TAG} | \
+              sed -r 's|^([a-z]*)\.?([0-9]*)-?([a-z]*)-?([0-9.]+)$|\1/\2/\3|g' | \
+              sed -r 's|//|/|g' | \
+              sed -r 's|/$|/bare|g')/activate
+            make build
+
+      # Tag images with our DockerHub namespace (fundocker/), and list images to
+      # check that they have been properly tagged.
       - run:
-          name: Check docker image tags
+          name: Tag production image
           command: |
-            docker images edxapp:latest
-            docker images edxapp:dev
+            docker tag edxapp:latest fundocker/edxapp:${CIRCLE_TAG}
+            docker images fundocker/edxapp:${CIRCLE_TAG}
 
       # Login to DockerHub with encrypted credentials stored as secret
       # environment variables (set in CircleCI project settings)
       - run:
           name: Login to DockerHub
           command: echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin
 
-      # Tag images with our DockerHub namespace (fundocker/), and list
-      # images to check that they have been properly tagged
-      - run:
-          name: Tag production image
-          command: |
-            docker tag edxapp:latest fundocker/edxapp:${CIRCLE_TAG}
-            docker images fundocker/edxapp:${CIRCLE_TAG}
-
       # Publish the production image to DockerHub
       - run:
           name: Publish production image
@@ -136,12 +97,8 @@ workflows:
   edxapp:
     jobs:
       # The build job has no required jobs, hence this will be our first job
-      - build:
-          # Filtering rule to run this job: none (we accept all tags; this job
-          # will always run).
-          filters:
-            tags:
-              only: /.*/
+      - master/bare
+      - hawthorn/1/bare
 
       # We are pushing to Docker only images that are the result of a tag respecting the pattern:
       #    **{branch-name}-x.y.z**
@@ -156,7 +113,8 @@ workflows:
       #   - eucalyptus-funwb-2.3.19
       - hub:
           requires:
-            - build
+            - master/bare
+            - hawthorn/1/bare
           filters:
             branches:
               ignore: /.*/

diff --git a/Makefile b/Makefile
@@ -11,7 +11,12 @@ UID              = $(shell id -u)
 GID              = $(shell id -g)
 
 # Docker
-COMPOSE          = UID=$(UID) GID=$(GID) EDX_RELEASE_PATH=$(EDX_RELEASE_PATH) docker-compose
+COMPOSE          = \
+  UID=$(UID) \
+  GID=$(GID) \
+  EDX_RELEASE_PATH="$(EDX_RELEASE_PATH)" \
+  EDX_RELEASE_REF="$(EDX_RELEASE_REF)" \
+  docker-compose
 COMPOSE_RUN      = $(COMPOSE) run --rm -e HOME="/tmp"
 COMPOSE_EXEC     = $(COMPOSE) exec
 
@@ -27,6 +32,8 @@ COLOR_RESET   = \033[0m
 COLOR_SUCCESS = \033[0;32m
 COLOR_WARNING = \033[0;33m
 
+default: help
+
 # Target release expected tree
 $(EDX_RELEASE_PATH)/data/static/production/.keep:
 	mkdir -p $(EDX_RELEASE_PATH)/data/static/production
@@ -67,8 +74,6 @@ $(EDX_RELEASE_PATH)/src/edx-demo-course/README.md:
 	curl -Lo /tmp/edx-demo.tgz https://github.com/edx/edx-demo-course/archive/$(EDX_DEMO_RELEASE_REF).tar.gz
 	tar xzf /tmp/edx-demo.tgz -C $(EDX_RELEASE_PATH)/src/edx-demo-course --strip-components=1
 
-default: help
-
 bootstrap: \
   tree \
   build \

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -28,7 +28,7 @@ services:
       context: ./${EDX_RELEASE_PATH:-releases/master/bare}
       target: production
       args:
-        EDXAPP_RELEASE: ${EDX_RELEASE_REF}
+        EDX_RELEASE_REF: ${EDX_RELEASE_REF:-release-2018-08-29-14.14}
     image: edxapp:latest
     env_file: env.d/development
     environment:
@@ -53,7 +53,7 @@ services:
       args:
         UID: ${UID}
         GID: ${GID}
-        EDXAPP_RELEASE: ${EDX_RELEASE_REF}
+        EDX_RELEASE_REF: ${EDX_RELEASE_REF:-release-2018-08-29-14.14}
     image: edxapp:dev
     env_file: env.d/development
     ports:
@@ -73,7 +73,7 @@ services:
       - memcached
 
   cms:
-    image: edxapp:latest
+    image: edxapp:${EDX_RELEASE_REF:-release-2018-08-29-14.14}
     env_file: env.d/development
     environment:
       SERVICE_VARIANT: cms
@@ -88,7 +88,7 @@ services:
     user: ${UID}:${GID}
 
   cms-dev:
-    image: edxapp:dev
+    image: edxapp:${EDX_RELEASE_REF:-release-2018-08-29-14.14}-dev
     env_file: env.d/development
     ports:
       - "8082:8000"

diff --git a/env.d/development b/env.d/development
@@ -1,6 +1,6 @@
 # Django
-SERVICE_VARIANT: lms
-DJANGO_SETTINGS_MODULE: lms.envs.fun.docker_run
+SERVICE_VARIANT=lms
+DJANGO_SETTINGS_MODULE=lms.envs.fun.docker_run
 
 # Database
 MYSQL_ROOT_PASSWORD=

diff --git a/releases/hawthorn/1/bare/Dockerfile b/releases/hawthorn/1/bare/Dockerfile
@@ -1,13 +1,13 @@
 # EDX-PLATFORM multi-stage docker build
 
-# Change release to build, by providing the EDXAPP_RELEASE build argument to
+# Change release to build, by providing the EDX_RELEASE_REF build argument to
 # your build command:
 #
 # $ docker build \
-#     --build-arg EDXAPP_RELEASE="open-release/hawthorn.1" \
+#     --build-arg EDX_RELEASE_REF="open-release/hawthorn.1" \
 #     -t edxapp:hawthorn.1 \
 #     .
-ARG EDXAPP_RELEASE=open-release/hawthorn.1
+ARG EDX_RELEASE_REF=open-release/hawthorn.1
 
 # === BASE ===
 FROM ubuntu:16.04 as base
@@ -35,9 +35,9 @@ RUN apt-get update && \
 RUN curl -sLo get-pip.py https://bootstrap.pypa.io/get-pip.py
 
 # Download edxapp release
-# Get default EDXAPP_RELEASE value (defined on top)
-ARG EDXAPP_RELEASE
-RUN curl -sLo edxapp.tgz https://github.com/edx/edx-platform/archive/$EDXAPP_RELEASE.tar.gz && \
+# Get default EDX_RELEASE_REF value (defined on top)
+ARG EDX_RELEASE_REF
+RUN curl -sLo edxapp.tgz https://github.com/edx/edx-platform/archive/$EDX_RELEASE_REF.tar.gz && \
     tar xzf edxapp.tgz
 
 
@@ -52,8 +52,8 @@ RUN apt-get update && \
 
 WORKDIR /edx/app/edxapp/edx-platform
 
-# Get default EDXAPP_RELEASE value (defined on top)
-ARG EDXAPP_RELEASE
+# Get default EDX_RELEASE_REF value (defined on top)
+ARG EDX_RELEASE_REF
 COPY --from=downloads /downloads/edx-platform-* .
 
 # We copy default configuration files to "/config" and we point to them via
@@ -117,7 +117,7 @@ FROM builder as development
 
 ARG UID=1000
 ARG GID=1000
-ARG EDXAPP_RELEASE
+ARG EDX_RELEASE_REF
 
 # Install system dependencies
 RUN apt-get update && \

diff --git a/releases/master/bare/Dockerfile b/releases/master/bare/Dockerfile
@@ -1,13 +1,13 @@
 # EDX-PLATFORM multi-stage docker build
 
-# Change release to build, by providing the EDXAPP_RELEASE build argument to
+# Change release to build, by providing the EDX_RELEASE_REF build argument to
 # your build command:
 #
 # $ docker build \
-#     --build-arg EDXAPP_RELEASE="open-release/hawthorn.1" \
+#     --build-arg EDX_RELEASE_REF="open-release/hawthorn.1" \
 #     -t edxapp:hawthorn.1 \
 #     .
-ARG EDXAPP_RELEASE=release-2018-08-29-14.14
+ARG EDX_RELEASE_REF=release-2018-08-29-14.14
 
 # === BASE ===
 FROM ubuntu:16.04 as base
@@ -35,9 +35,9 @@ RUN apt-get update && \
 RUN curl -sLo get-pip.py https://bootstrap.pypa.io/get-pip.py
 
 # Download edxapp release
-# Get default EDXAPP_RELEASE value (defined on top)
-ARG EDXAPP_RELEASE
-RUN curl -sLo edxapp.tgz https://github.com/edx/edx-platform/archive/$EDXAPP_RELEASE.tar.gz && \
+# Get default EDX_RELEASE_REF value (defined on top)
+ARG EDX_RELEASE_REF
+RUN curl -sLo edxapp.tgz https://github.com/edx/edx-platform/archive/$EDX_RELEASE_REF.tar.gz && \
     tar xzf edxapp.tgz
 
 
@@ -52,8 +52,8 @@ RUN apt-get update && \
 
 WORKDIR /edx/app/edxapp/edx-platform
 
-# Get default EDXAPP_RELEASE value (defined on top)
-ARG EDXAPP_RELEASE
+# Get default EDX_RELEASE_REF value (defined on top)
+ARG EDX_RELEASE_REF
 COPY --from=downloads /downloads/edx-platform-* .
 
 # We copy default configuration files to "/config" and we point to them via
@@ -117,7 +117,7 @@ FROM builder as development
 
 ARG UID=1000
 ARG GID=1000
-ARG EDXAPP_RELEASE
+ARG EDX_RELEASE_REF
 
 # Install system dependencies
 RUN apt-get update && \