Merge pull request #3705 from GeotrekCE/ci/cache_and_docker

Improve CI, cache and docker

.env-dev.dist → .env.dist

@@ -6,5 +6,6 @@ POSTGRES_USER=geotrek
 POSTGRES_PASSWORD=geotrek
 POSTGRES_DB=geotrekdb
 WEB_PORT=8000
+SPHINX_PORT=8800
 UID=1000
 GID=1000

.github/actions/python-cache-requirements/action.yml

@@ -0,0 +1,19 @@
+name: 'Python requirements - get or set in cache'
+description: 'Get pip cache from cache else generate it'
+inputs:
+  requirements:
+    description: 'Cache key to use'
+    required: true
+    default: requirements.txt
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        cache: 'pip'
+        cache-dependency-path: '**/*requirements.txt'
+    - run: |
+        pip install -r ${{ inputs.requirements }}
+      shell: bash

.github/workflows/dependencies.yml

@@ -35,9 +35,9 @@ jobs:
             ~/.cache/pip
             ~/.wheel_dir
 
-          key: pip-${{ matrix.python-version }}
+          key: pip-${{ matrix.python-version }}-${{ hashFiles('**/*requirements.txt') }}
           restore-keys: |
-            pip-${{ matrix.python-version }}
+            pip-${{ matrix.python-version }}-${{ hashFiles('**/*requirements.txt') }}
 
       - name: Install dependencies
         run: |

.github/workflows/doc.yml

@@ -26,8 +26,15 @@ jobs:
         LANG: C.UTF-8
     steps:
       - uses: actions/checkout@v3
-      - name: Documentation
+      - uses: ./.github/actions/python-cache-requirements
+        with:
+          requirements: docs/requirements.txt
+
+      - name: Install dependencies
+        run: |
+          pip3 install -r docs/requirements.txt
+
+      - name: Build doc
         run: |
           cd docs/
-          pip install -r ./requirements.txt -U
           make html SPHINXOPTS="-W"

.github/workflows/lint.yml

@@ -12,41 +12,26 @@ env:
   DEBIAN_FRONTEND: noninteractive
 
 jobs:
-  flake8:
-    name: Checking Flake8 rules
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: ['ubuntu-20.04']
-        python-version: ['3.8']
-
+  migrations:
+    name: Checking SRID in migrations files
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-
-      - name: Not evaluated values in migration files
-        run: |
+      - run: |
           test $(ls geotrek/*/migrations/*.py | xargs grep -l srid | xargs grep -L SRID | wc -l) -eq 0
 
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - uses: actions/cache@v3
+  flake8:
+    name: Checking Flake8 rules
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/python-cache-requirements
         with:
-          path: |
-            ~/.cache/pip
-            ~/.wheel_dir
-
-          key: pip-${{ matrix.python-version }}
-          restore-keys: |
-            pip-${{ matrix.python-version }}
+          requirements: dev-requirements.txt
 
       - name: Install dependencies
         run: |
-          pip3 wheel --wheel-dir=~/.wheel_dir flake8 -c dev-requirements.txt
-          pip3 install --find-links=~/.wheel_dir --upgrade flake8 -c dev-requirements.txt
+          pip3 install -c dev-requirements.txt flake8
 
       - name: Flake8
         run: |

.github/workflows/test.yml

@@ -74,11 +74,13 @@ jobs:
     - uses: actions/cache@v3
       with:
         path: |
+          ~/.cache/pip
+          ~/.wheel_dir
           ./venv
 
-        key: pip-${{ matrix.python-version }}
+        key: pip-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'dev-requirements.txt', 'docs/requirements.txt') }}
         restore-keys: |
-          pip-${{ matrix.python-version }}
+          pip-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'dev-requirements.txt', 'docs/requirements.txt') }}
 
     - name: Prepare test env
       run: |
@@ -162,7 +164,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
       - name: Build Docker image
         run: |
-          docker build -t geotrek:latest .
+          docker build -f ./docker/Dockerfile -t geotrek:latest .
       - name: Upload image
         uses: ishworkh/docker-image-artifact-upload@v1
         with:
@@ -237,7 +239,8 @@ jobs:
       - name: Launch service
         run: |
           docker tag geotrek:latest geotrekce/admin:latest
-          cp .env-prod.dist .env; cp docker-compose-prod.yml docker-compose.yml
+          cp docker/install/.env.dist .env
+          cp docker/install/docker-compose.yml docker-compose.yml
           echo 'LANGUAGES=en fr' >> .env
           echo 'LANGUAGE_CODE=en' >> .env
           sudo sed -i 's/localhost/geotrek.local/' .env

.gitignore

@@ -43,7 +43,6 @@ docs/_build/
 /env/
 /pyvenv.cfg
 /.env
-/docker-compose.yml
 /build/
 
 /cypress/videos/

Makefile

@@ -32,7 +32,7 @@ serve:
 	$(docker_compose) up
 
 deps:
-	$(docker_compose) run --rm web bash -c "pip-compile -q && pip-compile -q dev-requirements.in"
+	$(docker_compose) run --rm web bash -c "pip-compile -q && pip-compile -q dev-requirements.in && pip-compile -q docs/requirements.in"
 
 flake8:
 	$(docker_compose) run --rm web flake8 geotrek

debian/rules

@@ -8,7 +8,9 @@ export DH_VIRTUALENV_INSTALL_ROOT=/opt
 override_dh_virtualenv:
 	dh_virtualenv \
 	--python /usr/bin/python3.8 \
-	--upgrade-pip 23.1\
+	--upgrade-pip \
+	--preinstall wheel \
+	--preinstall setuptools \
 	--builtin-venv \
 	--extra-pip-arg --no-cache-dir \
 	--extra-pip-arg --quiet

docker-compose-dev.yml → docker-compose.yml

@@ -1,4 +1,18 @@
 version: "3"
+
+x-images:
+  django: &django
+    image: geotrek
+    build:
+      context: .
+      dockerfile: ./docker/Dockerfile
+      target: dev
+    user: ${UID:-1000}:${GID:-1000}
+    env_file:
+      - .env
+    volumes:
+      - .:/opt/geotrek-admin
+
 services:
   postgres:
     image: postgis/postgis:11-2.5
@@ -21,45 +35,37 @@ services:
     image: makinacorpus/screamshotter:v2
 
   celery:
-    image: geotrek
-    build:
-      context: .
-    env_file:
-      - .env
-    volumes:
-        - .:/opt/geotrek-admin
+    <<: [ *django ]
     depends_on:
       - postgres
       - redis
-    user: ${UID:-1000}:${GID:-1000}
     command: celery -A geotrek worker -c 1
 
   web:
-    image: geotrek
-    env_file:
-      - .env
+    <<: [ *django ]
+
     ports:
       - "${WEB_PORT:-8000}:8000"
-    volumes:
-      - ./:/opt/geotrek-admin/
+
     depends_on:
       - postgres
       - redis
       - screamshotter
       - convertit
-    user: ${UID:-1000}:${GID:-1000}
-    command: ./manage.py runserver 0.0.0.0:8000
+      - celery
+
     networks:
       default:
         aliases:
           - ${SERVER_NAME}
 
   sphinx:
-    build: docs
-    volumes:
-      - ./docs:/docs
+    <<: [ *django ]
+    working_dir: /opt/geotrek-admin/docs
     ports:
-      - "8800:8800"
+      - "${SPHINX_PORT:-8800}:8800"
+
+    command: sphinx-autobuild -b html --host 0.0.0.0 --port 8800 ./ ./_build/html
 
 volumes:
   postgres:

Dockerfile → docker/Dockerfile

@@ -1,31 +1,41 @@
-ARG BASE_IMAGE_TAG=focal-3.8
-FROM makinacorpus/geodjango:${BASE_IMAGE_TAG}
+FROM ubuntu:focal as base
 
-ENV ENV=prod
-ENV SERVER_NAME="localhost"
+ENV PYTHONBUFFERED 1
+ENV DEBIAN_FRONTEND noninteractive
+ENV ENV prod
+ENV SERVER_NAME "localhost"
 # If POSTGRES_HOST is empty, entrypoint will set it to the IP of the docker host in the container
-ENV POSTGRES_HOST=""
-ENV POSTGRES_PORT="5432"
-ENV POSTGRES_USER="geotrek"
-ENV POSTGRES_PASSWORD="geotrek"
-ENV POSTGRES_DB="geotrekdb"
-ENV REDIS_HOST="redis"
-ENV CONVERSION_HOST="convertit"
-ENV CAPTURE_HOST="screamshotter"
-ENV CUSTOM_SETTINGS_FILE="/opt/geotrek-admin/var/conf/custom.py"
+ENV POSTGRES_HOST ""
+ENV POSTGRES_PORT "5432"
+ENV POSTGRES_USER "geotrek"
+ENV POSTGRES_PASSWORD "geotrek"
+ENV POSTGRES_DB "geotrekdb"
+ENV REDIS_HOST "redis"
+ENV CONVERSION_HOST "convertit"
+ENV CAPTURE_HOST "screamshotter"
+ENV CUSTOM_SETTINGS_FILE "/opt/geotrek-admin/var/conf/custom.py"
+ENV TZ UTC
 
 WORKDIR /opt/geotrek-admin
 RUN mkdir -p /opt/geotrek-admin/var/log /opt/geotrek-admin/var/cache
 RUN adduser geotrek --disabled-password && chown geotrek:geotrek -R /opt
+
 # Install postgis because raster2pgsl is required by manage.py loaddem
 RUN apt-get update -qq && apt-get install -y -qq  \
+    python3.8 \
+    ca-certificates \
+    gettext \
+    postgresql-client \
+    tzdata \
+    netcat \
+    gdal-bin \
+    binutils \
+    libproj-dev \
     unzip \
-    sudo \
     less \
+    iproute2 \
     nano \
     curl \
-    git \
-    iproute2 \
     software-properties-common \
     shared-mime-info \
     fonts-liberation \
@@ -42,23 +52,54 @@ RUN apt-get update -qq && apt-get install -y -qq  \
     apt-get install -y --no-install-recommends postgis && \
     apt-get clean all && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/apt/*
 
+USER geotrek
+COPY --chown=geotrek:geotrek docker/scripts/* /usr/local/bin/
+ENTRYPOINT ["/bin/sh", "-e", "/usr/local/bin/entrypoint.sh"]
+EXPOSE 8000
+
+FROM base as build
+
+USER root
+
+RUN apt-get update -qq && apt-get install -y -qq  \
+    python3.8-dev \
+    python3.8-venv \
+    build-essential \
+    libpq-dev &&\
+    apt-get clean all && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/apt/*
+
 USER geotrek
 RUN python3.8 -m venv /opt/venv
 RUN /opt/venv/bin/pip install --no-cache-dir -U pip setuptools wheel
 COPY requirements.txt requirements.txt
 RUN /opt/venv/bin/pip install --no-cache-dir -r requirements.txt -U
+
+FROM build as dev
+
 COPY dev-requirements.txt dev-requirements.txt
-RUN /opt/venv/bin/pip install --no-cache-dir -r dev-requirements.txt -U
+RUN /opt/venv/bin/pip install --no-cache-dir -r dev-requirements.txt
+COPY ./docs/requirements.txt doc-requirements.txt
+RUN /opt/venv/bin/pip install --no-cache-dir -r doc-requirements.txt
+
+CMD ["./manage.py", "runserver", "0.0.0.0:8000"]
 
+FROM base as prod
+
+ENV ENV prod
+
+COPY --chown=geotrek:geotrek --from=build /opt/venv /opt/venv
 COPY --chown=geotrek:geotrek geotrek/ geotrek/
 COPY --chown=geotrek:geotrek manage.py manage.py
 COPY --chown=geotrek:geotrek VERSION VERSION
 COPY --chown=geotrek:geotrek setup.cfg setup.cfg
-COPY --chown=geotrek:geotrek docker/* /usr/local/bin/
 
-ENTRYPOINT ["/bin/sh", "-e", "/usr/local/bin/entrypoint.sh"]
-EXPOSE 8000
+RUN CUSTOM_SETTINGS_FILE= SECRET_KEY=tmp /opt/venv/bin/python ./manage.py compilemessages
+
+USER root
 
-RUN ENV=dev CUSTOM_SETTINGS_FILE= SECRET_KEY=tmp /opt/venv/bin/python ./manage.py compilemessages
+RUN apt-get update -qq && apt-get -yqq full-upgrade && \
+    apt-get clean all && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/apt/*
+
+USER geotrek
 
 CMD ["gunicorn", "geotrek.wsgi:application", "--bind=0.0.0.0:8000"]