.github/workflows/test.yml

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Test CI

on:
  pull_request:
    paths-ignore:
      - 'docs/**'
  push:
    branches:
      - master

    paths-ignore:
      - 'docs/**'
  release:
    types:
      - created

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  DEBIAN_FRONTEND: noninteractive
  POSTGRES_DB: ci_test
  POSTGRES_PASSWORD: ci_test
  POSTGRES_USER: ci_test
  POSTGRES_HOST: 127.0.0.1
  SERVER_NAME: geotrek.local
  SECRET_KEY: test-test-test
  CONVERSION_HOST: localhost
  CAPTURE_HOST: localhost
  REDIS_HOST: 127.0.0.1
  VAR_DIR: /home/runner/work/Geotrek-admin/Geotrek-admin/var
  LANG: C.UTF-8
  LANGUAGES: en fr
  LANGUAGE_CODE: en
  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true

jobs:
  test:
    name: Python Unittests
    runs-on: ubuntu-latest
    container: makinacorpus/geodjango:${{ matrix.os }}
    
    permissions:
      packages: write  # required to publish docker image

    strategy:
      matrix:
        os: ['focal-3.8', 'jammy-3.10']
        tests-env: ['tests', 'tests_nds']

    env:
      ENV: ${{ matrix.tests-env }}
      OS: ${{ matrix.os }}
      POSTGRES_HOST: postgres
      REDIS_HOST: redis

    services:
      postgres:
        image: postgis/postgis:12-2.5
        env:
          POSTGRES_DB: ci_test
          POSTGRES_PASSWORD: ci_test
          POSTGRES_USER: ci_test
        ports:
          - 5432:5432
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready
      redis:
        image: redis:5.0-alpine
        ports:
          - 6379:6379

    steps:
    - uses: actions/checkout@v4
    - uses: actions/cache@v4
      with:
        path: |
          ./venv

        key: pip-${{ matrix.os }}-${{ hashFiles('./requirements.txt') }}-${{ hashFiles('./dev-requirements.txt') }}-${{ hashFiles('./docs/requirements.txt') }}
        restore-keys: |
          pip-${{ matrix.os }}-${{ hashFiles('./requirements.txt') }}-${{ hashFiles('./dev-requirements.txt') }}-${{ hashFiles('./docs/requirements.txt') }}
          pip-${{ matrix.os }}-${{ hashFiles('./requirements.txt') }}-${{ hashFiles('./dev-requirements.txt') }}
          pip-${{ matrix.os }}-${{ hashFiles('./requirements.txt') }}
          pip-${{ matrix.os }}

    - name: Prepare test env
      run: |
        apt-get update -q
        apt-get -qq -y install \
           unzip \
           gettext \
           binutils \
           libproj-dev \
           gdal-bin \
           sudo \
           less \
           curl \
           git \
           iproute2 \
           software-properties-common \
           shared-mime-info \
           fonts-liberation \
           libssl-dev \
           libfreetype6-dev \
           libxml2-dev \
           libxslt-dev \
           libcairo2 \
           libpango1.0-0 \
           libpangocairo-1.0-0 \
           libgdk-pixbuf2.0-dev \
           libffi-dev \
           libvips
        apt-get install -y --no-install-recommends postgis

    - name: Install python venv
      run: |
          python3 -m venv ./venv

    - name: Install python dependencies
      run: |
        ./venv/bin/pip3 install --upgrade pip wheel setuptools pip-tools -c requirements.txt -c dev-requirements.txt
        ./venv/bin/pip-sync requirements.txt dev-requirements.txt

    - name: Create test required directories
      run: |
        mkdir -p ./var/log ./var/cache/sessions

    - name: Prepare environment
      run: |
        mkdir -p $VAR_DIR/conf/extra_static $VAR_DIR/tmp
        ./venv/bin/python3 ./manage.py collectstatic --clear --noinput --verbosity=0
        ./venv/bin/python3 ./manage.py compilemessages

    - name: Check missing migrations
      run: |
        ENV=dev ./venv/bin/python3 ./manage.py makemigrations --check

    - name: Check running migrations
      run: |
        ENV=dev ./venv/bin/python3 ./manage.py migrate

    - name: Launch Celery
      run: |
        ./venv/bin/celery -A geotrek worker -c 1 &

    - name: Unit test
      run: |
        ./venv/bin/coverage run --parallel-mode --concurrency=multiprocessing ./manage.py test --parallel -v 3
        ./venv/bin/coverage combine
        ./venv/bin/coverage xml -o coverage.xml

    - name: Save coverage
      uses: actions/upload-artifact@v4
      with:
        name: coverage_${{ matrix.os }}_${{ matrix.tests-env }}
        path: coverage.xml

  upload_coverage:
    name: Upload coverage to codecov
    runs-on: ubuntu-latest
    needs: [ test ]
    strategy:
      matrix:
        os: ['focal-3.8', 'jammy-3.10']
        tests-env: ['tests', 'tests_nds']

    env:
      ENV: ${{ matrix.tests-env }}
      OS: ${{ matrix.os }}

    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
            name: coverage_${{ matrix.os }}_${{ matrix.tests-env }}
            path: .

      - uses: codecov/codecov-action@v5
        with:
            files: ./coverage.xml
            env_vars: OS,ENV
            token: ${{ secrets.CODECOV_TOKEN }} # not usually required for public repos
            fail_ci_if_error: true # optional (default = false)

  build_docker_image:
    name: Build docker image
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3

      - name: Build and push
        id: docker_build
        uses: docker/build-push-action@v6
        with:
            push: false
            builder: ${{ steps.buildx.outputs.name }}
            tags: "geotrek:latest"
            load: true
            file: ./docker/Dockerfile

      - name: Upload image
        uses: ishworkh/container-image-artifact-upload@v2.0.0
        with:
          image: "geotrek:latest"

  build_deb:
      name: Build debian package
      runs-on: ubuntu-latest
      strategy:
          matrix:
              os: ['bionic', 'focal', 'jammy']
              include:
                  - os: 'bionic'
                    code: '18.04'
                  - os: 'focal'
                    code: '20.04'
                  - os: 'jammy'
                    code: '22.04'

      env:
          OS: ${{ matrix.os }}
          CODE: ${{ matrix.code }}
          DISTRO: ubuntu:${{ matrix.os }}

      steps:
          - uses: actions/checkout@v4
          - name: Prepare versioning
            run: |
                grep '^[0-9]\+\.[0-9]\+\.[0-9]\+$' VERSION || sed -i 's/+dev/.ubuntu'$CODE'~dev'$GITHUB_RUN_ID'/' debian/changelog
                sed -i 's/geotrek-admin (\([0-9]\+\.[0-9]\+\.[0-9]\+\)\(.*\)) RELEASED;/geotrek-admin (\1.ubuntu'$CODE'\2) '$OS';/' debian/changelog

          - name: Building package
            run: |
              make build_deb
              cp ./dpkg/*.deb /home/runner/work/Geotrek-admin/

          - name: Archive package artifact
            uses: actions/upload-artifact@v4
            with:
              name: ${{ matrix.os }}
              path: |
                /home/runner/work/Geotrek-admin/*.deb

  e2e_docker_image:
    name: Tests E2E docker
    runs-on: ubuntu-20.04
    needs: [build_docker_image]
    env:
      CYPRESS_BASE_URL: http://geotrek.local
      CYPRESS_CACHE_FOLDER: ~/cypress/cache

    services:
      postgres:
        image: postgis/postgis:12-2.5
        env:
          POSTGRES_DB: ci_test
          POSTGRES_PASSWORD: ci_test
          POSTGRES_USER: ci_test
        ports:
          - 5432:5432
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready

    steps:
      - uses: actions/checkout@v4

      - name: Download image
        uses: ishworkh/container-image-artifact-download@v2.0.0
        with:
          image: "geotrek:latest"

      - name: Launch service
        run: |
          docker tag geotrek:latest geotrekce/admin:latest
          cp docker/install/.env.dist .env
          cp docker/install/docker-compose.yml docker-compose.yml
          echo 'LANGUAGES=en fr' >> .env
          echo 'LANGUAGE_CODE=en' >> .env
          sudo sed -i 's/localhost/geotrek.local/' .env
          sudo sed -i 's/127.0.0.1 localhost/127.0.0.1 localhost geotrek.local/' /etc/hosts
          mkdir -p var/conf/extra_static
          cp cypress/custom.py var/conf/custom.py
          echo 'POSTGRES_USER=ci_test' >> .env
          echo 'POSTGRES_PASSWORD=ci_test' >> .env
          echo 'POSTGRES_DB=ci_test' >> .env
          docker compose run web update.sh
          make load_data
          make load_test_integration
          docker compose up -d

      - name: Prepare nginx
        run: |
          sudo service apache2 stop
          sudo apt-get purge apache2 apache2-utils apache2.2-bin
          sudo apt-get autoremove
          sudo apt-get install -q -y nginx
          sudo unlink /etc/nginx/sites-enabled/default
          sudo mkdir /var/www/geotrek -p
          ls -als
          pwd
          sudo ln -s /home/runner/work/Geotrek-admin/Geotrek-admin/var/media /var/www/geotrek/media
          sudo ln -s /home/runner/work/Geotrek-admin/Geotrek-admin/var/static /var/www/geotrek/static
          sudo ln -s /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/nginx.conf /etc/nginx/sites-enabled/geotrek.conf
          sudo systemctl restart nginx

      - name: install Cypress
        run: |
          cd /home/runner/work/Geotrek-admin/Geotrek-admin/cypress
          npm ci
          npm list
          echo "Waiting for container..."
          while ! nc -z "127.0.0.1" "8001"; do
            sleep 0.1
          done

      - name: E2E test
        run: |
          /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/node_modules/.bin/cypress run -P /home/runner/work/Geotrek-admin/Geotrek-admin/cypress --record --key 64a5a9b3-9869-4a2f-91e4-e3cd27c2f564
        continue-on-error: false

      - name: Archive results as artifacts
        uses: actions/upload-artifact@v4
        with:
          name: cypress-docker
          path: |
            /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/videos/*.mp4
            /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/screenshots/*.png

  e2e_deb_20_04:
    name: Tests E2E 20.04
    runs-on: ubuntu-20.04
    needs: [ build_deb ]

    env:
      CYPRESS_BASE_URL: http://geotrek.local
      CYPRESS_CACHE_FOLDER: ~/cypress/cache

    services:
      postgres:
        image: postgis/postgis:12-2.5
        env:
          POSTGRES_DB: ci_test
          POSTGRES_PASSWORD: ci_test
          POSTGRES_USER: ci_test
        ports:
          - 5432:5432
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready

    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'focal'

      - name: Simulate install.sh procedure
        run: |
          sudo service apache2 stop
          sudo apt-get purge apache2 apache2-utils apache2.2-bin
          sudo apt-get autoremove
          echo "deb [arch=amd64] https://packages.geotrek.fr/ubuntu $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/geotrek.list
          wget -O- "https://packages.geotrek.fr/geotrek.gpg.key" | sudo apt-key add -
          sudo apt-get -q update
          echo "geotrek-admin geotrek-admin/LANGUAGES string en fr" >> .debian_settings
          echo "geotrek-admin geotrek-admin/POSTGRES_USER string $POSTGRES_USER" >> .debian_settings
          echo "geotrek-admin geotrek-admin/POSTGRES_DB string $POSTGRES_DB" >> .debian_settings
          echo "geotrek-admin geotrek-admin/POSTGRES_PASSWORD string $POSTGRES_PASSWORD" >> .debian_settings
          echo "geotrek-admin geotrek-admin/SERVER_NAME string geotrek.local" >> .debian_settings
          mkdir -p /opt/geotrek-admin/var/conf/
          cp /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/custom.py /opt/geotrek-admin/var/conf/custom.py
          sudo sed -i 's/127.0.0.1 localhost/127.0.0.1 localhost geotrek.local/' /etc/hosts
          sudo debconf-set-selections .debian_settings

      - name: Install package
        run: |
          sudo unlink /etc/nginx/sites-enabled/default
          sudo apt-get install /home/runner/work/Geotrek-admin/Geotrek-admin/*.deb || exit 0;
          sudo systemctl restart nginx

      - name: Load Data
        run: |
          sudo geotrek loaddata minimal
          sudo geotrek loaddata cirkwi
          sudo geotrek loaddata basic
          sudo geotrek loaddata test-integration
          for dir in `ls -d /opt/geotrek-admin/lib/python3*/site-packages/geotrek/*/fixtures/upload`; do
          	cd $dir > /dev/null
          	sudo cp -r * /opt/geotrek-admin/var/media/upload/
          done

      - name: Install cypress
        run: |
          cd /home/runner/work/Geotrek-admin/Geotrek-admin/cypress
          npm ci
          npm list

      - name: E2E test
        run: |
          /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/node_modules/.bin/cypress run -P /home/runner/work/Geotrek-admin/Geotrek-admin/cypress --record --key 64a5a9b3-9869-4a2f-91e4-e3cd27c2f564
        continue-on-error: false

      - name: Archive results as artifacts
        uses: actions/upload-artifact@v4
        with:
          name: cypress-20.04
          path: |
            /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/videos/*.mp4
            /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/screenshots/*.png

  deploy:
    name: Publish (on release only)
    runs-on: ubuntu-latest
    needs: [ test, e2e_docker_image, build_deb, e2e_deb_20_04 ]
    if: ${{ github.event_name != 'pull_request' }}
    steps:
      - uses: actions/checkout@v4
      - name: Download 18.04 debian artifact
        uses: actions/download-artifact@v4
        if: ${{ github.event_name == 'release' && github.event.action == 'created' }}
        with:
          name: bionic

      - name: Download 20.04 debian artifact
        uses: actions/download-artifact@v4
        if: ${{ github.event_name == 'release' && github.event.action == 'created' }}
        with:
          name: focal

      - name: Download docker image
        uses: ishworkh/container-image-artifact-download@v2.0.0
        with:
          image: 'geotrek:latest'

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_LOGIN }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3

      - name: Extract metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images:  |
            geotrekce/admin
            ghcr.io/geotrekce/geotrek-admin

      - name: Build and push image
        uses: docker/build-push-action@v6
        with:
          push: true
          provenance: mode=max
          sbom: true
          builder: ${{ steps.buildx.outputs.name }}
          tags: ${{ steps.meta.outputs.tags }}
          file: ./docker/Dockerfile

      - name: Install SSH key
        uses: shimataro/ssh-key-action@v2
        if: ${{ github.event_name == 'release' && github.event.action == 'created' }}
        with:
          key: ${{ secrets.SSH_PRIVATE_KEY }}
          known_hosts: ${{ secrets.SSH_KNOWN_HOSTS }}

      - name: Publish debian packages
        if: ${{ github.event_name == 'release' && github.event.action == 'created' }}
        run: |
          if [[ "${{ github.ref }}" == *"dev"* ]]; then
            export DEB_COMPONENT=dev
          else
            export DEB_COMPONENT=main
          fi
          echo "${{ github.ref }} : Publishing as $DEB_COMPONENT package"
          scp -P ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no /home/runner/work/Geotrek-admin/Geotrek-admin/geotrek-admin_*_amd64.deb ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }}:/srv/packages/incoming/$DEB_COMPONENT/
          if [[ "${{ github.ref }}" == *"dev"* ]]; then
            ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make bionic_dev -C /srv/packages
            ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make focal_dev -C /srv/packages
          else
            ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make bionic_main -C /srv/packages
            ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make focal_main -C /srv/packages
          fi

      - name: Download 22.04 debian artifact
        if: ${{ github.event_name == 'release' && github.event.action == 'created' }}
        uses: actions/download-artifact@v4
        with:
          name: jammy

      - name: Attach debian packages as release binaries
        if: ${{ github.event_name == 'release' && github.event.action == 'created' }}
        uses: skx/github-action-publish-binaries@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          args: '*.deb'