diff --git a/.circleci/config.yml b/.circleci/config.yml index 46443ac6..7da5effa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,111 +2,80 @@ # Docker containers CI version: 2 +# Templates +defaults: &defaults + # We use the machine executor, i.e. a VM, not a container + machine: + # Cache docker layers so that we strongly speed up this job execution + # This cache will be available to future jobs (although because jobs run + # in parallel, CircleCI does not guarantee that a given job will see a + # specific version of the cache. See documentation for details) + docker_layer_caching: true + + working_directory: ~/fun + +build_steps: &build_steps + steps: + # Checkout openedx-docker sources + - checkout + + # Install a recent docker-compose release + - run: + name: Upgrade docker-compose + command: | + curl -L "https://github.com/docker/compose/releases/download/1.24.1/docker-compose-$(uname -s)-$(uname -m)" -o ~/docker-compose + chmod +x ~/docker-compose + sudo mv ~/docker-compose /usr/local/bin/docker-compose + + # Production image build. It will be tagged as edxapp:latest + - run: + name: Build production image + command: | + source releases/${CIRCLE_JOB}/activate + make build + + # Development image build. It uses the "development" Dockerfile target + # file and will be tagged as edxapp:dev + - run: + name: Build development image + command: | + source releases/${CIRCLE_JOB}/activate + make dev-build + # List openedx-docker jobs that will be integrated and executed in a workflow jobs: - # Build job - # Build the Docker images for production and development - build: - # We use the machine executor, i.e. a VM, not a container - machine: - # Cache docker layers so that we strongly speed up this job execution - # This cache will be available to future jobs (although because jobs run - # in parallel, CircleCI does not garantee that a given job will see a - # specific version of the cache. See documentation for details) - docker_layer_caching: true - - working_directory: ~/fun - - steps: - # Checkout openedx-docker sources - - checkout - - # Restore the ~/fun/src cached repository. If the cache does not exists for - # the current .Revision (commit hash), we fall back to the latest cache - # with a label matching 'edx-archive-v1-' - - restore_cache: - keys: - - edx-archive-v1-{{ .Revision }} - - edx-archive-v1- - - # Install a recent docker-compose release - - run: - name: Upgrade docker-compose - command: | - curl -L "https://github.com/docker/compose/releases/download/1.22.0/docker-compose-$(uname -s)-$(uname -m)" -o ~/docker-compose - chmod +x ~/docker-compose - sudo mv ~/docker-compose /usr/local/bin/docker-compose - - # Production image build. It will be tagged as edxapp:latest - - run: - name: Build production image - command: | - make build - - # Development image build. It uses the "development" Dockerfile target - # file and will be tagged as edxapp:dev - - run: - name: Build development image - command: | - make dev-build + # Build jobs + # + # Note that the job name should match the EDX_RELEASE value + master/bare: + <<: [*defaults, *build_steps] - # Cache Open edX repository (cloned in ~/fun/src) as the checkout is - # rather time consuming for this project - - save_cache: - paths: - - ~/fun/releases/master/bare/src/edx-platform - key: edx-archive-v1-{{ .Revision }} - - # Save and cache the built images to filesystem so that they will be - # available to test and push them to DockerHub in subsequent jobs - - run: - name: Save docker images to filesystem - command: | - docker save -o edxapp.tar edxapp:latest edxapp:dev - - save_cache: - paths: - - ~/fun/edxapp.tar - key: edx-image-v1-{{ .Revision }} + hawthorn/1/bare: + <<: [*defaults, *build_steps] # Hub job - # Load and tag production/development images to push them to Dockerhub - # public registry - # These images are now the latest for this branch so we will publish - # both under their tag and under the `latest` tag so that our `latest` - # images are always up-to-date hub: - # We use the machine executor, i.e. a VM, not a container - machine: true - - working_directory: ~/fun + <<: *defaults steps: - # First, check that the BRANCH name is included in the TAG name. This is important - # because we handle several important branches (master, funmooc, funwb, etc.) and - # we must make sure that tag names are explicitly linked to a branch in order to - # avoid conflicts + # Thanks to docker layer caching, rebuilding the image should be blazing + # fast! - run: - name: Check that the BRANCH name is included in the TAG name + name: Rebuild production image command: | - if ! echo ${CIRCLE_TAG} | grep "${CIRCLE_BRANCH}" &> /dev/null; then - # Stop the step without failing - circleci step halt - fi - - # Load the docker images from our cache to the docker engine and check that they - # have been effectively loaded - - restore_cache: - keys: - - edx-image-v1-{{ .Revision }} - - run: - name: Load images to docker engine - command: | - docker load < edxapp.tar + source releases/$(echo ${CIRCLE_TAG} | \ + sed -r 's|^([a-z]*)\.?([0-9]*)-?([a-z]*)-?([0-9.]+)$|\1/\2/\3|g' | \ + sed -r 's|//|/|g' | \ + sed -r 's|/$|/bare|g')/activate + make build + + # Tag images with our DockerHub namespace (fundocker/), and list images to + # check that they have been properly tagged. - run: - name: Check docker image tags + name: Tag production image command: | - docker images edxapp:latest - docker images edxapp:dev + docker tag edxapp:latest fundocker/edxapp:${CIRCLE_TAG} + docker images fundocker/edxapp:${CIRCLE_TAG} # Login to DockerHub with encrypted credentials stored as secret # environment variables (set in CircleCI project settings) @@ -114,14 +83,6 @@ jobs: name: Login to DockerHub command: echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin - # Tag images with our DockerHub namespace (fundocker/), and list - # images to check that they have been properly tagged - - run: - name: Tag production image - command: | - docker tag edxapp:latest fundocker/edxapp:${CIRCLE_TAG} - docker images fundocker/edxapp:${CIRCLE_TAG} - # Publish the production image to DockerHub - run: name: Publish production image @@ -136,12 +97,8 @@ workflows: edxapp: jobs: # The build job has no required jobs, hence this will be our first job - - build: - # Filtering rule to run this job: none (we accept all tags; this job - # will always run). - filters: - tags: - only: /.*/ + - master/bare + - hawthorn/1/bare # We are pushing to Docker only images that are the result of a tag respecting the pattern: # **{branch-name}-x.y.z** @@ -156,7 +113,8 @@ workflows: # - eucalyptus-funwb-2.3.19 - hub: requires: - - build + - master/bare + - hawthorn/1/bare filters: branches: ignore: /.*/ diff --git a/Makefile b/Makefile index df4f1cd9..385ce8ba 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,12 @@ UID = $(shell id -u) GID = $(shell id -g) # Docker -COMPOSE = UID=$(UID) GID=$(GID) EDX_RELEASE_PATH=$(EDX_RELEASE_PATH) docker-compose +COMPOSE = \ + UID=$(UID) \ + GID=$(GID) \ + EDX_RELEASE_PATH="$(EDX_RELEASE_PATH)" \ + EDX_RELEASE_REF="$(EDX_RELEASE_REF)" \ + docker-compose COMPOSE_RUN = $(COMPOSE) run --rm -e HOME="/tmp" COMPOSE_EXEC = $(COMPOSE) exec @@ -27,6 +32,8 @@ COLOR_RESET = \033[0m COLOR_SUCCESS = \033[0;32m COLOR_WARNING = \033[0;33m +default: help + # Target release expected tree $(EDX_RELEASE_PATH)/data/static/production/.keep: mkdir -p $(EDX_RELEASE_PATH)/data/static/production @@ -67,8 +74,6 @@ $(EDX_RELEASE_PATH)/src/edx-demo-course/README.md: curl -Lo /tmp/edx-demo.tgz https://github.com/edx/edx-demo-course/archive/$(EDX_DEMO_RELEASE_REF).tar.gz tar xzf /tmp/edx-demo.tgz -C $(EDX_RELEASE_PATH)/src/edx-demo-course --strip-components=1 -default: help - bootstrap: \ tree \ build \ diff --git a/docker-compose.yml b/docker-compose.yml index 63253b28..1f791b8e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: context: ./${EDX_RELEASE_PATH:-releases/master/bare} target: production args: - EDXAPP_RELEASE: ${EDX_RELEASE_REF} + EDX_RELEASE_REF: ${EDX_RELEASE_REF:-release-2018-08-29-14.14} image: edxapp:latest env_file: env.d/development environment: @@ -53,7 +53,7 @@ services: args: UID: ${UID} GID: ${GID} - EDXAPP_RELEASE: ${EDX_RELEASE_REF} + EDX_RELEASE_REF: ${EDX_RELEASE_REF:-release-2018-08-29-14.14} image: edxapp:dev env_file: env.d/development ports: @@ -73,7 +73,7 @@ services: - memcached cms: - image: edxapp:latest + image: edxapp:${EDX_RELEASE_REF:-release-2018-08-29-14.14} env_file: env.d/development environment: SERVICE_VARIANT: cms @@ -88,7 +88,7 @@ services: user: ${UID}:${GID} cms-dev: - image: edxapp:dev + image: edxapp:${EDX_RELEASE_REF:-release-2018-08-29-14.14}-dev env_file: env.d/development ports: - "8082:8000" diff --git a/env.d/development b/env.d/development index d91203f9..f1e878e5 100644 --- a/env.d/development +++ b/env.d/development @@ -1,6 +1,6 @@ # Django -SERVICE_VARIANT: lms -DJANGO_SETTINGS_MODULE: lms.envs.fun.docker_run +SERVICE_VARIANT=lms +DJANGO_SETTINGS_MODULE=lms.envs.fun.docker_run # Database MYSQL_ROOT_PASSWORD= diff --git a/releases/hawthorn/1/bare/Dockerfile b/releases/hawthorn/1/bare/Dockerfile index 0c0a6b9c..025e8e61 100644 --- a/releases/hawthorn/1/bare/Dockerfile +++ b/releases/hawthorn/1/bare/Dockerfile @@ -1,13 +1,13 @@ # EDX-PLATFORM multi-stage docker build -# Change release to build, by providing the EDXAPP_RELEASE build argument to +# Change release to build, by providing the EDX_RELEASE_REF build argument to # your build command: # # $ docker build \ -# --build-arg EDXAPP_RELEASE="open-release/hawthorn.1" \ +# --build-arg EDX_RELEASE_REF="open-release/hawthorn.1" \ # -t edxapp:hawthorn.1 \ # . -ARG EDXAPP_RELEASE=open-release/hawthorn.1 +ARG EDX_RELEASE_REF=open-release/hawthorn.1 # === BASE === FROM ubuntu:16.04 as base @@ -35,9 +35,9 @@ RUN apt-get update && \ RUN curl -sLo get-pip.py https://bootstrap.pypa.io/get-pip.py # Download edxapp release -# Get default EDXAPP_RELEASE value (defined on top) -ARG EDXAPP_RELEASE -RUN curl -sLo edxapp.tgz https://github.com/edx/edx-platform/archive/$EDXAPP_RELEASE.tar.gz && \ +# Get default EDX_RELEASE_REF value (defined on top) +ARG EDX_RELEASE_REF +RUN curl -sLo edxapp.tgz https://github.com/edx/edx-platform/archive/$EDX_RELEASE_REF.tar.gz && \ tar xzf edxapp.tgz @@ -52,8 +52,8 @@ RUN apt-get update && \ WORKDIR /edx/app/edxapp/edx-platform -# Get default EDXAPP_RELEASE value (defined on top) -ARG EDXAPP_RELEASE +# Get default EDX_RELEASE_REF value (defined on top) +ARG EDX_RELEASE_REF COPY --from=downloads /downloads/edx-platform-* . # We copy default configuration files to "/config" and we point to them via @@ -117,7 +117,7 @@ FROM builder as development ARG UID=1000 ARG GID=1000 -ARG EDXAPP_RELEASE +ARG EDX_RELEASE_REF # Install system dependencies RUN apt-get update && \ diff --git a/releases/master/bare/Dockerfile b/releases/master/bare/Dockerfile index f5934401..5cc8bbdc 100644 --- a/releases/master/bare/Dockerfile +++ b/releases/master/bare/Dockerfile @@ -1,13 +1,13 @@ # EDX-PLATFORM multi-stage docker build -# Change release to build, by providing the EDXAPP_RELEASE build argument to +# Change release to build, by providing the EDX_RELEASE_REF build argument to # your build command: # # $ docker build \ -# --build-arg EDXAPP_RELEASE="open-release/hawthorn.1" \ +# --build-arg EDX_RELEASE_REF="open-release/hawthorn.1" \ # -t edxapp:hawthorn.1 \ # . -ARG EDXAPP_RELEASE=release-2018-08-29-14.14 +ARG EDX_RELEASE_REF=release-2018-08-29-14.14 # === BASE === FROM ubuntu:16.04 as base @@ -35,9 +35,9 @@ RUN apt-get update && \ RUN curl -sLo get-pip.py https://bootstrap.pypa.io/get-pip.py # Download edxapp release -# Get default EDXAPP_RELEASE value (defined on top) -ARG EDXAPP_RELEASE -RUN curl -sLo edxapp.tgz https://github.com/edx/edx-platform/archive/$EDXAPP_RELEASE.tar.gz && \ +# Get default EDX_RELEASE_REF value (defined on top) +ARG EDX_RELEASE_REF +RUN curl -sLo edxapp.tgz https://github.com/edx/edx-platform/archive/$EDX_RELEASE_REF.tar.gz && \ tar xzf edxapp.tgz @@ -52,8 +52,8 @@ RUN apt-get update && \ WORKDIR /edx/app/edxapp/edx-platform -# Get default EDXAPP_RELEASE value (defined on top) -ARG EDXAPP_RELEASE +# Get default EDX_RELEASE_REF value (defined on top) +ARG EDX_RELEASE_REF COPY --from=downloads /downloads/edx-platform-* . # We copy default configuration files to "/config" and we point to them via @@ -117,7 +117,7 @@ FROM builder as development ARG UID=1000 ARG GID=1000 -ARG EDXAPP_RELEASE +ARG EDX_RELEASE_REF # Install system dependencies RUN apt-get update && \