From 35f907ace5ec4c66460fc5cf4a668b7ae75cdb42 Mon Sep 17 00:00:00 2001 From: Julien Maupetit Date: Fri, 30 Mar 2018 10:45:52 +0200 Subject: [PATCH] =?UTF-8?q?[draft]=20=F0=9F=91=B7(circle)=20move=20a=20ric?= =?UTF-8?q?hie's=20container=20based=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .circleci/config.yml | 479 +++++++++++++++++++++++---- docker/compose/ci/docker-compose.yml | 20 ++ docker/images/alpine/dev/Dockerfile | 9 +- docker/images/dev/Dockerfile | 9 +- env.d/ci | 14 + 5 files changed, 456 insertions(+), 75 deletions(-) create mode 100644 docker/compose/ci/docker-compose.yml create mode 100644 env.d/ci diff --git a/.circleci/config.yml b/.circleci/config.yml index c50a6cbc45..25d82fc51f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,84 +1,375 @@ version: 2 jobs: - build-back: - docker: - - image: python:3.6-stretch - working_directory: ~/richie + # Docker/back-end jobs + # Build job + # Build the Docker container ready for production + build: + + # We use the machine executor, i.e. a VM, not a container + machine: + # Cache docker layers so that we strongly speed up this job execution + docker_layer_caching: true + + working_directory: ~/fun + steps: + # Checkout repository sources - checkout - # Download and cache dependencies - - restore_cache: - keys: - - v3-back-dependencies-{{ checksum "requirements/base.txt" }}-{{ checksum "requirements/dev.txt" }} - # fallback to using the latest cache if no exact match is found - - v3-back-dependencies- + + # Each image is tagged with the current git commit sha1 to avoid + # collisions in parallel builds. + - run: + name: Build production container + command: | + docker build \ + -t richie:${CIRCLE_SHA1} \ + . + + - run: + name: List available images + command: docker images richie + + # Since we cannot rely on CircleCI's Docker layers cache (for obscure + # reasons some subsequent jobs will benefit from a previous job cache and + # some others won't), we choose to save built docker images in cached + # directories. This ensures that we will be able to load built docker + # images in subsequent jobs. - run: - name: Install back-end dependencies + name: Store docker image in cache command: | - python3 -m venv venv - . venv/bin/activate - pip install -r requirements/dev.txt + docker save \ + -o docker/images/richie.tar \ + richie:${CIRCLE_SHA1} + - save_cache: paths: - - ./venv - key: v3-back-dependencies-{{ checksum "requirements/base.txt" }}-{{ checksum "requirements/dev.txt" }} + - ~/fun/docker/images/ + key: docker-images-v1-{{ .Revision }} + + # Build dev job + # Build the Docker container ready for development + build-dev: + + machine: + docker_layer_caching: true + + working_directory: ~/fun - build-front: - docker: - - image: circleci/node:9-browsers - working_directory: ~/richie steps: - checkout + - restore_cache: keys: - - v3-front-dependencies-{{ checksum "package.json" }} - - v3-front-dependencies- + - docker-images-v1-{{ .Revision }} + - run: - name: Install front-end dependencies - command: yarn install --frozen-lockfile + name: Load images to docker engine + command: | + docker load < docker/images/richie.tar + - run: - name: Build front-end application - command: yarn build + name: Check available images + command: docker images richie + + # Development container build, it uses the dev Dockerfile file - run: - name: Build application styles - command: yarn sass + name: Build development container + command: | + docker build \ + -t richie:${CIRCLE_SHA1}-dev \ + -f docker/images/dev/Dockerfile \ + --build-arg BASE_TAG=${CIRCLE_SHA1} \ + . + + - run: + name: List available images + command: docker images richie + + - run: + name: Store docker image in cache + command: | + docker save \ + -o docker/images/richie.tar \ + richie:${CIRCLE_SHA1} \ + richie:${CIRCLE_SHA1}-dev + - save_cache: paths: - - ./node_modules - key: v3-front-dependencies-{{ checksum "package.json" }} + - ~/fun/docker/images/ + key: docker-images-v1-{{ .Revision }}-dev lint-back-flake8: - docker: - - image: python:3.6-stretch - working_directory: ~/richie + + machine: + docker_layer_caching: true + + working_directory: ~/fun + steps: - checkout + - restore_cache: keys: - - v3-back-dependencies-{{ checksum "requirements/base.txt" }}-{{ checksum "requirements/dev.txt" }} + - docker-images-v1-{{ .Revision }}-dev + + - run: + name: Load images to docker engine + command: | + docker load < docker/images/richie.tar + - run: name: Lint code with flake8 command: | - . venv/bin/activate - flake8 + docker-compose \ + -p richie-test \ + -f docker/compose/ci/docker-compose.yml \ + --project-directory . \ + run --rm --no-deps app \ + flake8 lint-back-pylint: - docker: - - image: python:3.6-stretch - working_directory: ~/richie + + machine: + docker_layer_caching: true + + working_directory: ~/fun + steps: - checkout + - restore_cache: keys: - - v3-back-dependencies-{{ checksum "requirements/base.txt" }}-{{ checksum "requirements/dev.txt" }} + - docker-images-v1-{{ .Revision }}-dev + + - run: + name: Load images to docker engine + command: | + docker load < docker/images/richie.tar + - run: name: Lint code with pylint command: | - . venv/bin/activate - pylint apps plugins richie + docker-compose \ + -p richie-test \ + -f docker/compose/ci/docker-compose.yml \ + --project-directory . \ + run --rm --no-deps app \ + pylint apps plugins richie - lint-front: + test-back: + + machine: + docker_layer_caching: true + + working_directory: ~/fun + + steps: + - checkout + + - restore_cache: + keys: + - docker-images-v1-{{ .Revision }}-dev + + - run: + name: Load images to docker engine + command: | + docker load < docker/images/richie.tar + + # ElasticSearch configuration + # + # We need to increase the VM max memory size, or else, ElasticSearch (ES) + # service won't bootstrap. + # + # Source: + # https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-cli-run-prod-mode + - run: + name: Increase VM max memory size for ES + command: | + sudo sysctl -w vm.max_map_count=262144 + sudo sysctl vm.max_map_count + + # Run back-end (Django) test suite + # + # Nota bene: to run the django test suite, we need to ensure that both + # PostgreSQL and ElasticSearch services are up and ready. To achieve this, + # we wrap the pytest command execution with dockerize, a tiny tool + # installed in the development container. In our case, dockerize will wait + # up to one minute that both the database and elastisearch containers + # opened their expected tcp port (5432 and 9200 resp.). + - run: + name: Run tests + command: | + docker-compose \ + -p richie-test \ + -f docker/compose/ci/docker-compose.yml \ + --project-directory . \ + run --rm app \ + dockerize \ + -wait tcp://db:5432 \ + -wait tcp://elasticsearch:9200 \ + -timeout 60s \ + pytest + + # ---- Alpine jobs ---- + build-alpine: + + machine: + docker_layer_caching: true + + working_directory: ~/fun + + steps: + - checkout + + - run: + name: Build alpine production container + command: | + docker build \ + -t richie:${CIRCLE_SHA1}-alpine \ + -f docker/images/alpine/Dockerfile \ + . + + - run: + name: Build alpine development container + command: | + docker build \ + -t richie:${CIRCLE_SHA1}-alpine-dev \ + -f docker/images/alpine/dev/Dockerfile \ + --build-arg BASE_TAG=${CIRCLE_SHA1}-alpine \ + . + + - run: + name: List available images + command: docker images richie + + - run: + name: Store docker image in cache + command: | + docker save \ + -o docker/images/richie-alpine.tar \ + richie:${CIRCLE_SHA1}-alpine \ + richie:${CIRCLE_SHA1}-alpine-dev + + - save_cache: + paths: + - ~/fun/docker/images/ + key: docker-images-v1-{{ .Revision }}-alpine + + test-alpine: + + machine: + docker_layer_caching: true + + working_directory: ~/fun + + steps: + - checkout + + - restore_cache: + keys: + - docker-images-v1-{{ .Revision }}-alpine + + - run: + name: Load images to docker engine + command: | + docker load < docker/images/richie-alpine.tar + + - run: + name: Increase VM max memory size + command: | + sudo sysctl -w vm.max_map_count=262144 + sudo sysctl vm.max_map_count + + - run: + name: Run tests + command: | + IMAGE_SUFFIX="-alpine" \ + docker-compose \ + -p richie-test \ + -f docker/compose/ci/docker-compose.yml \ + --project-directory . \ + run --rm app \ + dockerize \ + -wait tcp://db:5432 \ + -wait tcp://elasticsearch:9200 \ + -timeout 60s \ + pytest + + # ---- DockerHub publication job ---- + hub: + + machine: + docker_layer_caching: true + + working_directory: ~/fun + + steps: + - checkout + + - restore_cache: + keys: + - docker-images-v1-{{ .Revision }}-dev + + - restore_cache: + keys: + - docker-images-v1-{{ .Revision }}-alpine + + # Load all built images in all flavors + - run: + name: Load images to docker engine + command: | + docker load < docker/images/richie.tar + docker load < docker/images/richie-alpine.tar + + # Login to DockerHub to Publish new containers + # + # Nota bene: you'll need to define the following secrets environment vars + # in CircleCI interface: + # + # - DOCKER_USER + # - DOCKER_PASS + - run: + name: Login to DockerHub + command: echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin + + # Tag docker images with the same pattern used in Git (Semantic Versioning) + # + # Git tag: v1.0.1 + # Docker tag: 1.0.1(-alpine)(-dev) + - run: + name: Tag images + command: | + docker images fundocker/richie + DOCKER_TAG=$(echo ${CIRCLE_TAG} | sed 's/^v//') + echo "DOCKER_TAG: ${DOCKER_TAG} (from Git tag: ${CIRCLE_TAG})" + docker tag richie:${CIRCLE_SHA1} fundocker/richie:latest + docker tag richie:${CIRCLE_SHA1} fundocker/richie:${DOCKER_TAG} + docker tag richie:${CIRCLE_SHA1}-dev fundocker/richie:${DOCKER_TAG}-dev + docker tag richie:${CIRCLE_SHA1}-alpine fundocker/richie:alpine + docker tag richie:${CIRCLE_SHA1}-alpine fundocker/richie:${DOCKER_TAG}-alpine + docker tag richie:${CIRCLE_SHA1}-alpine-dev fundocker/richie:${DOCKER_TAG}-alpine-dev + docker images "fundocker/richie:${DOCKER_TAG}*" + + # Publish images to DockerHub + # + # Nota bene: logged user (see "Login to DockerHub" step) must have write + # permission for the project's repository; this also implies that the + # DockerHub repository already exists. + - run: + name: Publish images + command: | + DOCKER_TAG=$(echo ${CIRCLE_TAG} | sed 's/^v//') + echo "DOCKER_TAG: ${DOCKER_TAG} (from Git tag: ${CIRCLE_TAG})" + docker push fundocker/richie:latest + docker push fundocker/richie:${DOCKER_TAG} + docker push fundocker/richie:${DOCKER_TAG}-dev + docker push fundocker/richie:alpine + docker push fundocker/richie:${DOCKER_TAG}-alpine + docker push fundocker/richie:${DOCKER_TAG}-alpine-dev + + # ---- Front-end jobs ---- + build-front: docker: - image: circleci/node:9-browsers working_directory: ~/richie @@ -87,34 +378,33 @@ jobs: - restore_cache: keys: - v3-front-dependencies-{{ checksum "package.json" }} + - v3-front-dependencies- - run: - name: Lint code with tslint - command: yarn lint + name: Install front-end dependencies + command: yarn install --frozen-lockfile + - run: + name: Build front-end application + command: yarn build + - run: + name: Build application styles + command: yarn sass + - save_cache: + paths: + - ./node_modules + key: v3-front-dependencies-{{ checksum "package.json" }} - test-back: + lint-front: docker: - - image: python:3.6-stretch - environment: - DJANGO_SETTINGS_MODULE: 'richie.settings' - DJANGO_CONFIGURATION: 'ContinuousIntegration' - DJANGO_SECRET_KEY: 'ThisIsAnExampleKeyForCIPurposeOnly' - POSTGRES_HOST: '127.0.0.1' - POSTGRES_DB: 'postgres' - POSTGRES_USER: 'postgres' - POSTGRES_PASSWORD: '' - - image: postgres:9.6 - - image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.1.2 + - image: circleci/node:9-browsers working_directory: ~/richie steps: - checkout - restore_cache: keys: - - v3-back-dependencies-{{ checksum "requirements/base.txt" }}-{{ checksum "requirements/dev.txt" }} + - v3-front-dependencies-{{ checksum "package.json" }} - run: - name: Run tests - command: | - . venv/bin/activate - pytest + name: Lint code with tslint + command: yarn lint test-front: docker: @@ -134,41 +424,84 @@ workflows: richie: jobs: - - build-back: + # Front-end jobs + # + # Build, lint and test the front-end apps + - build-front: filters: tags: only: /.*/ - - build-front: + - lint-front: + requires: + - build-front filters: tags: only: /.*/ - - lint-back-flake8: + - test-front: requires: - - build-back + - lint-front filters: tags: only: /.*/ - - lint-back-pylint: + + # Docker jobs + # + # Build, lint and test production and development Docker images + # (debian-based) + - build: + filters: + tags: + only: /.*/ + - build-dev: requires: - - build-back + - build filters: tags: only: /.*/ - - lint-front: + - lint-back-flake8: requires: - - build-front + - build-dev + filters: + tags: + only: /.*/ + - lint-back-pylint: + requires: + - build-dev filters: tags: only: /.*/ - test-back: requires: - - build-back + - build-dev filters: tags: only: /.*/ - - test-front: + + # Docker alpine jobs + # + # Build and run tests in alpine based images + - build-alpine: + filters: + tags: + only: /.*/ + - test-alpine: requires: - - build-front + - build-alpine filters: tags: only: /.*/ + + # DockerHub publication. + # + # Publish docker images only if all build, lint and test jobs succeed and + # it has been tagged with a tag starting with the letter v + - hub: + requires: + - test-front + - test-back + - test-alpine + filters: + branches: + ignore: /.*/ + tags: + only: /^v.*/ diff --git a/docker/compose/ci/docker-compose.yml b/docker/compose/ci/docker-compose.yml new file mode 100644 index 0000000000..a1cc3eb306 --- /dev/null +++ b/docker/compose/ci/docker-compose.yml @@ -0,0 +1,20 @@ +version: "3" + +services: + db: + image: postgres:9.6 + env_file: env.d/ci + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.1.2 + ulimits: + memlock: + soft: -1 + hard: -1 + + app: + image: "richie:${CIRCLE_SHA1}${IMAGE_SUFFIX}-dev" + env_file: env.d/ci + depends_on: + - "db" + - "elasticsearch" diff --git a/docker/images/alpine/dev/Dockerfile b/docker/images/alpine/dev/Dockerfile index 193ed9ab37..e5ce56eae1 100644 --- a/docker/images/alpine/dev/Dockerfile +++ b/docker/images/alpine/dev/Dockerfile @@ -1,4 +1,11 @@ -FROM richie:alpine +# The base image we inherit from is richie:alpine, but you can override this by +# passing a build argument to your build command, e.g.: +# +# docker build --build-arg BASE_TAG=${CIRCLE_SHA1}-alpine . +# +ARG BASE_TAG=alpine + +FROM richie:${BASE_TAG} # Switch back to the root user to install development dependencies USER root:root diff --git a/docker/images/dev/Dockerfile b/docker/images/dev/Dockerfile index 5d9b0d1443..9ff21f61a9 100644 --- a/docker/images/dev/Dockerfile +++ b/docker/images/dev/Dockerfile @@ -1,4 +1,11 @@ -FROM richie:latest +# The base image we inherit from is richie:latest, but you can override this by +# passing a build argument to your build command, e.g.: +# +# docker build --build-arg BASE_TAG=${CIRCLE_SHA1} . +# +ARG BASE_TAG=latest + +FROM richie:${BASE_TAG} # Switch back to the root user to install development dependencies USER root:root diff --git a/env.d/ci b/env.d/ci new file mode 100644 index 0000000000..9e8aca2c02 --- /dev/null +++ b/env.d/ci @@ -0,0 +1,14 @@ +# Django +DJANGO_SETTINGS_MODULE=richie.settings +DJANGO_CONFIGURATION=Test +DJANGO_SECRET_KEY=ThisIsAnExampleKeyForTestPurposeOnly + +# Database +POSTGRES_HOST=db +POSTGRES_DB=richie +POSTGRES_USER=fun +POSTGRES_PASSWORD=pass + +# Elastic search +ES_CLIENT=elasticsearch +bootstrap.memory_lock=true