Refactor project tree to handle multiple OpenEdx releases

.circleci/config.yml

@@ -2,132 +2,152 @@
 # Docker containers CI
 version: 2
 
+# Templates
+defaults: &defaults # We use the machine executor, i.e. a VM, not a container
+  machine:
+    # Cache docker layers so that we strongly speed up this job execution
+    # This cache will be available to future jobs (although because jobs run
+    # in parallel, CircleCI does not guarantee that a given job will see a
+    # specific version of the cache. See documentation for details)
+    docker_layer_caching: true
+
+  working_directory: ~/fun
+
+build_steps: &build_steps
+  steps:
+    # Checkout openedx-docker sources
+    - checkout
+
+    # Install a recent docker-compose release
+    - run:
+        name: Upgrade docker-compose
+        command: |
+          curl -L "https://github.com/docker/compose/releases/download/1.24.1/docker-compose-$(uname -s)-$(uname -m)" -o ~/docker-compose
+          chmod +x ~/docker-compose
+          sudo mv ~/docker-compose /usr/local/bin/docker-compose
+
+    # Production image build. It will be tagged as edxapp:latest
+    - run:
+        name: Build production image
+        command: |
+          source releases/${CIRCLE_JOB}/activate
+          make build
+
+    # Development image build. It uses the "development" Dockerfile target
+    # file and will be tagged as edxapp:dev
+    - run:
+        name: Build development image
+        command: |
+          source releases/${CIRCLE_JOB}/activate
+          make dev-build
+
 # List openedx-docker jobs that will be integrated and executed in a workflow
 jobs:
-  # Build job
-  # Build the Docker images for production and development
-  build:
-    # We use the machine executor, i.e. a VM, not a container
-    machine:
-      # Cache docker layers so that we strongly speed up this job execution
-      # This cache will be available to future jobs (although because jobs run
-      # in parallel, CircleCI does not garantee that a given job will see a
-      # specific version of the cache. See documentation for details)
-      docker_layer_caching: true
+  # Quality jobs
 
+  # Check that the git history is clean and complies with our expectations
+  lint-git:
+    docker:
+      - image: circleci/python:3.7-stretch
     working_directory: ~/fun
-
     steps:
-      # Checkout openedx-docker sources
       - checkout
-
-      # Restore the ~/fun/src cached repository. If the cache does not exists for
-      # the current .Revision (commit hash), we fall back to the latest cache
-      # with a label matching 'edx-repository-v1-'
-      - restore_cache:
-          keys:
-            - edx-repository-v1-{{ .Revision }}
-            - edx-repository-v1-
-
-      # Install a recent docker-compose release
+      # Make sure the changes don't add a "print" statement to the code base.
+      # We should exclude the ".circleci" folder from the search as the very command that checks
+      # the absence of "print" is including a "print(" itself.
       - run:
-          name: Upgrade docker-compose
+          name: enforce absence of print statements in code
           command: |
-            curl -L "https://github.com/docker/compose/releases/download/1.22.0/docker-compose-$(uname -s)-$(uname -m)" -o ~/docker-compose
-            chmod +x ~/docker-compose
-            sudo mv ~/docker-compose /usr/local/bin/docker-compose
-
-      # Clone Open edX sources
+            ! git diff origin/master..HEAD -- . ':(exclude).circleci' | grep "print("
       - run:
-          name: Clone Open edX platform
+          name: Check absence of fixup commits
           command: |
-            make clone
-
-      # Production image build. It will be tagged as edxapp:latest
+            ! git log | grep 'fixup!'
       - run:
-          name: Build production image
+          name: Install gitlint
           command: |
-            make build
-
-      # Development image build. It uses the Dockerfile_dev file and will
-      # be tagged as edxapp:dev
+            pip install --user gitlint
       - run:
-          name: Build development image
+          name: lint commit messages added to master
           command: |
-            make dev-build
+            ~/.local/bin/gitlint --commits origin/master..HEAD
 
-      # Cache Open edX repository (cloned in ~/fun/src) as the checkout is
-      # rather time consuming for this project
-      - save_cache:
-          paths:
-            - ~/fun/src/edx-platform/.git
-          key: edx-repository-v1-{{ .Revision }}
+  # Check that the CHANGELOG has been updated in the current branch
+  check-changelog:
+    docker:
+      - image: circleci/buildpack-deps:stretch-scm
+    working_directory: ~/fun
+    steps:
+      - checkout
+      - run:
+          name: Check that the CHANGELOG has been modified in the current branch
+          command: |
+            git whatchanged --name-only --pretty="" origin..HEAD | grep CHANGELOG
 
-      # Save and cache the built images to filesystem so that they will be
-      # available to test and push them to DockerHub in subsequent jobs
+  # Check that the CHANGELOG max line length does not exceed 80 characters
+  lint-changelog:
+    docker:
+      - image: debian:stretch
+    working_directory: ~/fun
+    steps:
+      - checkout
       - run:
-          name: Save docker images to filesystem
+          name: Check CHANGELOG max line length
           command: |
-            docker save -o edxapp.tar edxapp:latest edxapp:dev
-      - save_cache:
-          paths:
-            - ~/fun/edxapp.tar
-          key: edx-image-v1-{{ .Revision }}
+            # Get the longuest line width (ignoring release links)
+            test $(cat CHANGELOG.md | grep -Ev "^\[.*\]: https://github.com/openfun" | wc -L) -le 80
+
+  # Build jobs
+  #
+  # Note that the job name should match the EDX_RELEASE value
+  master/bare:
+    <<: [*defaults, *build_steps]
+
+  hawthorn/1/bare:
+    <<: [*defaults, *build_steps]
+
+  hawthorn/1/oee:
+    <<: [*defaults, *build_steps]
 
   # Hub job
-  # Load and tag production/development images to push them to Dockerhub
-  # public registry
-  # These images are now the latest for this branch so we will publish
-  # both under their tag and under the `latest` tag so that our `latest`
-  # images are always up-to-date
   hub:
-    # We use the machine executor, i.e. a VM, not a container
-    machine: true
-
-    working_directory: ~/fun
+    <<: *defaults
 
     steps:
-      # First, check that the BRANCH name is included in the TAG name. This is important
-      # because we handle several important branches (master, funmooc, funwb, etc.) and
-      # we must make sure that tag names are explicitly linked to a branch in order to
-      # avoid conflicts
-      - run:
-          name: Check that the BRANCH name is included in the TAG name
-          command: |
-            if ! echo ${CIRCLE_TAG} | grep "${CIRCLE_BRANCH}" &> /dev/null; then
-              # Stop the step without failing
-              circleci step halt
-            fi
-
-      # Load the docker images from our cache to the docker engine and check that they
-      # have been effectively loaded
-      - restore_cache:
-          keys:
-            - edx-image-v1-{{ .Revision }}
+      # Thanks to docker layer caching, rebuilding the image should be blazing
+      # fast!
+      #
+      # Note that we need to convert the CIRCLE_TAG (_e.g._ something like
+      # hawthorn.1-1.0.3) to a flavored release path (e.g. something like
+      # hawthorn/1/bare for the later CIRCLE_TAG example). In the following, we
+      # have a three-steps pipeline to do so: i. get the release name, number and
+      # optionally a flavor from the CIRCLE_TAG, ii. in case of empty release
+      # number and/or flavor, our sed substitution will generate duplicated slashes
+      # that should be fixed, and iii. the default flavor (empty third group
+      # from our regular expression) should be named "bare".
       - run:
-          name: Load images to docker engine
+          name: Rebuild production image
           command: |
-            docker load < edxapp.tar
+            source releases/$(echo ${CIRCLE_TAG} | \
+              sed -r 's|^([a-z]*)\.?([0-9]*)-?([a-z]*)-?([0-9.]+)$|\1/\2/\3|g' | \
+              sed -r 's|//|/|g' | \
+              sed -r 's|/$|/bare|g')/activate
+            make build
+
+      # Tag images with our DockerHub namespace (fundocker/), and list images to
+      # check that they have been properly tagged.
       - run:
-          name: Check docker image tags
+          name: Tag production image
           command: |
-            docker images edxapp:latest
-            docker images edxapp:dev
+            docker tag edxapp:latest fundocker/edxapp:${CIRCLE_TAG}
+            docker images fundocker/edxapp:${CIRCLE_TAG}
 
       # Login to DockerHub with encrypted credentials stored as secret
       # environment variables (set in CircleCI project settings)
       - run:
           name: Login to DockerHub
           command: echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin
 
-      # Tag images with our DockerHub namespace (fundocker/), and list
-      # images to check that they have been properly tagged
-      - run:
-          name: Tag production image
-          command: |
-            docker tag edxapp:latest fundocker/edxapp:${CIRCLE_TAG}
-            docker images fundocker/edxapp:${CIRCLE_TAG}
-
       # Publish the production image to DockerHub
       - run:
           name: Publish production image
@@ -141,13 +161,30 @@ workflows:
   # We have a single workflow
   edxapp:
     jobs:
-      # The build job has no required jobs, hence this will be our first job
-      - build:
-          # Filtering rule to run this job: none (we accept all tags; this job
-          # will always run).
+      # Quality
+      - lint-git:
+          filters:
+            branches:
+              ignore: master
+            tags:
+              only: /.*/
+      - check-changelog:
           filters:
+            branches:
+              ignore: master
             tags:
               only: /.*/
+      - lint-changelog:
+          filters:
+            branches:
+              ignore: master
+            tags:
+              only: /.*/
+
+      # Build jobs
+      - master/bare
+      - hawthorn/1/bare
+      - hawthorn/1/oee
 
       # We are pushing to Docker only images that are the result of a tag respecting the pattern:
       #    **{branch-name}-x.y.z**
@@ -162,7 +199,12 @@ workflows:
       #   - eucalyptus-funwb-2.3.19
       - hub:
           requires:
-            - build
+            - lint-git
+            - check-changelog
+            - lint-changelog
+            - master/bare
+            - hawthorn/1/bare
+            - hawthorn/1/oee
           filters:
             branches:
               ignore: /.*/

.dockerignore

@@ -1,2 +1,2 @@
-data/
-src/
+releases/**/data/
+releases/**/src/

.gitignore

@@ -6,10 +6,10 @@ __pycache__/
 *.py[cod]
 
 # Site media
-/data/
+data/
 
 # Sources
-/src/
+src/
 
 # Unit test / coverage reports
 htmlcov/
@@ -57,4 +57,3 @@ coverage.xml
 
 # Swap files
 *.sw[po]
-

.gitlint

@@ -0,0 +1,78 @@
+# All these sections are optional, edit this file as you like.
+[general]
+# Ignore certain rules, you can reference them by their id or by their full name
+# ignore=title-trailing-punctuation, T3
+
+# verbosity should be a value between 1 and 3, the commandline -v flags take precedence over this
+# verbosity = 2
+
+# By default gitlint will ignore merge commits. Set to 'false' to disable.
+# ignore-merge-commits=true
+
+# By default gitlint will ignore fixup commits. Set to 'false' to disable.
+# ignore-fixup-commits=true
+
+# By default gitlint will ignore squash commits. Set to 'false' to disable.
+# ignore-squash-commits=true
+
+# Enable debug mode (prints more output). Disabled by default.
+# debug=true
+
+# Set the extra-path where gitlint will search for user defined rules
+# See http://jorisroovers.github.io/gitlint/user_defined_rules for details
+extra-path=gitlint/
+
+# [title-max-length]
+# line-length=80
+
+[title-must-not-contain-word]
+# Comma-separated list of words that should not occur in the title. Matching is case
+# insensitive. It's fine if the keyword occurs as part of a larger word (so "WIPING"
+# will not cause a violation, but "WIP: my title" will.
+words=wip
+
+#[title-match-regex]
+# python like regex (https://docs.python.org/2/library/re.html) that the
+# commit-msg title must be matched to.
+# Note that the regex can contradict with other rules if not used correctly
+# (e.g. title-must-not-contain-word).
+#regex=
+
+# [B1]
+# B1 = body-max-line-length
+# line-length=120
+# [body-min-length]
+# min-length=5
+
+# [body-is-missing]
+# Whether to ignore this rule on merge commits (which typically only have a title)
+# default = True
+# ignore-merge-commits=false
+
+# [body-changed-file-mention]
+# List of files that need to be explicitly mentioned in the body when they are changed
+# This is useful for when developers often erroneously edit certain files or git submodules.
+# By specifying this rule, developers can only change the file when they explicitly reference
+# it in the commit message.
+# files=gitlint/rules.py,README.md
+
+# [author-valid-email]
+# python like regex (https://docs.python.org/2/library/re.html) that the
+# commit author email address should be matched to
+# For example, use the following regex if you only want to allow email addresses from foo.com
+# regex=[^@][email protected]
+
+[ignore-by-title]
+# Allow empty body & wrong title pattern only when bots (pyup/greenkeeper)
+# upgrade dependencies
+regex=^(⬆️.*|Update (.*) from (.*) to (.*)|(chore|fix)\(package\): update .*)$
+ignore=B6,UC1
+
+# [ignore-by-body]
+# Ignore certain rules for commits of which the body has a line that matches a regex
+# E.g. Match bodies that have a line that that contain "release"
+# regex=(.*)release(.*)
+#
+# Ignore certain rules, you can reference them by their id or by their full name
+# Use 'all' to ignore all rules
+# ignore=T1,body-min-length