test.yml

name: Tests

on:
  pull_request:
    paths:
      - src/**
      - tests/**
      - noxfile.py
      - poetry.lock
      - pyproject.toml
      - .github/workflows/test.yml
      - .github/workflows/resources/constraints.txt
  push:
    branches: [main]
    paths:
      - src/**
      - tests/**
      - noxfile.py
      - poetry.lock
      - pyproject.toml
      - .github/workflows/test.yml
      - .github/workflows/resources/constraints.txt
  workflow_dispatch:
    inputs: {}

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  tests:
    strategy:
      matrix:
        include:
        - { id: '01', python-version: '3.8',  os: ubuntu-latest, backend-db: sqlite }
        - { id: '02', python-version: '3.9',  os: ubuntu-latest, backend-db: sqlite }
        - { id: '03', python-version: '3.10', os: ubuntu-latest, backend-db: sqlite }
        - { id: '04', python-version: '3.11', os: ubuntu-latest, backend-db: sqlite }
        - { id: '05', python-version: '3.8',  os: ubuntu-latest, backend-db: postgresql }
        - { id: '06', python-version: '3.9',  os: ubuntu-latest, backend-db: postgresql }
        - { id: '07', python-version: '3.10', os: ubuntu-latest, backend-db: postgresql }
        - { id: '08', python-version: '3.11', os: ubuntu-latest, backend-db: postgresql }
        - { id: '09', python-version: '3.8',  os: ubuntu-latest, backend-db: mssql }
        - { id: '10', python-version: '3.9',  os: ubuntu-latest, backend-db: mssql }
        - { id: '11', python-version: '3.10', os: ubuntu-latest, backend-db: mssql }
        - { id: '12', python-version: '3.11', os: ubuntu-latest, backend-db: mssql }
        # We'd like to run Windows tests for all backend-dbs see https://github.com/meltano/meltano/issues/6281
        - { id: '13', python-version: '3.8',  os: windows-2022,  backend-db: sqlite }
        - { id: '14', python-version: '3.9',  os: windows-2022,  backend-db: sqlite }
        - { id: '15', python-version: '3.10', os: windows-2022,  backend-db: sqlite }
        - { id: '16', python-version: '3.11', os: windows-2022,  backend-db: sqlite }
        - { id: '17', python-version: '3.8',  os: ubuntu-latest, backend-db: postgresql_psycopg3 }
        - { id: '18', python-version: '3.9',  os: ubuntu-latest, backend-db: postgresql_psycopg3 }
        - { id: '19', python-version: '3.10', os: ubuntu-latest, backend-db: postgresql_psycopg3 }
        - { id: '20', python-version: '3.11', os: ubuntu-latest, backend-db: postgresql_psycopg3 }
      fail-fast: false

    # GitHub doesn't handle matrix outputs well: https://stackoverflow.com/questions/70287603
    outputs:
      pytest-results-row-01: ${{ steps.append-results.outputs.pytest-results-row-01 }}
      pytest-results-row-02: ${{ steps.append-results.outputs.pytest-results-row-02 }}
      pytest-results-row-03: ${{ steps.append-results.outputs.pytest-results-row-03 }}
      pytest-results-row-04: ${{ steps.append-results.outputs.pytest-results-row-04 }}
      pytest-results-row-05: ${{ steps.append-results.outputs.pytest-results-row-05 }}
      pytest-results-row-06: ${{ steps.append-results.outputs.pytest-results-row-06 }}
      pytest-results-row-07: ${{ steps.append-results.outputs.pytest-results-row-07 }}
      pytest-results-row-08: ${{ steps.append-results.outputs.pytest-results-row-08 }}
      pytest-results-row-09: ${{ steps.append-results.outputs.pytest-results-row-09 }}
      pytest-results-row-10: ${{ steps.append-results.outputs.pytest-results-row-10 }}
      pytest-results-row-11: ${{ steps.append-results.outputs.pytest-results-row-11 }}
      pytest-results-row-12: ${{ steps.append-results.outputs.pytest-results-row-12 }}
      pytest-results-row-13: ${{ steps.append-results.outputs.pytest-results-row-13 }}
      pytest-results-row-14: ${{ steps.append-results.outputs.pytest-results-row-14 }}
      pytest-results-row-15: ${{ steps.append-results.outputs.pytest-results-row-15 }}
      pytest-results-row-16: ${{ steps.append-results.outputs.pytest-results-row-16 }}
      pytest-results-row-17: ${{ steps.append-results.outputs.pytest-results-row-17 }}
      pytest-results-row-18: ${{ steps.append-results.outputs.pytest-results-row-18 }}
      pytest-results-row-19: ${{ steps.append-results.outputs.pytest-results-row-19 }}
      pytest-results-row-20: ${{ steps.append-results.outputs.pytest-results-row-20 }}

    name: "Pytest on py${{ matrix.python-version }} (OS: ${{ matrix.os }}, DB: ${{ matrix.backend-db }})"
    runs-on: ${{ matrix.os }}
    env:
      PYTEST_MARKERS: not concurrent
      FORCE_COLOR: 1
      NOXPYTHON: ${{ matrix.python-version }}

    steps:
    - name: Check out the repository
      uses: actions/checkout@v4.0.0

    - name: Install Poetry
      env:
        PIP_CONSTRAINT: .github/workflows/resources/constraints.txt
      run: |
        pipx install poetry
        poetry --version

    - name: Setup Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4.7.0
      with:
        python-version: ${{ matrix.python-version }}
        architecture: x64
        # Nox uses pip, so we cache with pip
        cache: 'pip'
        cache-dependency-path: 'poetry.lock'
        check-latest: true

    - name: Set up Docker Buildx
      if: always() && (matrix.backend-db == 'mssql')
      uses: docker/setup-buildx-action@v2.10.0

    - name: Get Docker
      if: always() && (matrix.backend-db == 'mssql')
      uses: actions-hub/docker/cli@v1.0.3
      env:
        SKIP_LOGIN: true

    - name: Start Postgres Container
      if: always() && (matrix.backend-db == 'postgresql' || matrix.backend-db == 'postgresql_psycopg3')
      run: >
        docker run -d
        -p "5432:5432"
        -e "POSTGRES_PASSWORD=postgres"
        --name postgres
        --health-cmd "pg_isready -d postgres -U postgres"
        --health-interval 10s
        --health-timeout 5s
        --health-retries 5
        postgres:11

    - name: Start MSSQL Container
      if: always() && (matrix.backend-db == 'mssql')
      run: |
        docker compose -f .github/workflows/resources/docker-compose.mssql.yaml up -d --wait --quiet-pull

    - name: Check running containers
      run: |
        docker ps -a

    - name: Install Nox
      env:
        PIP_CONSTRAINT: .github/workflows/resources/constraints.txt
      run: |
        pipx install nox
        pipx inject nox nox-poetry

    - name: Run pytest
      env:
        SQLALCHEMY_WARN_20: 1
        COLUMNS: 160
        PYTEST_BACKEND: ${{ matrix.backend-db }}
        # Postgres backend
        POSTGRES_HOST_AUTH_METHOD: trust
        POSTGRES_ADDRESS: localhost
        POSTGRES_PORT: 5432
        POSTGRES_USER: postgres
        POSTGRES_PASSWORD: postgres
        POSTGRES_DB: pytest_warehouse
        # MSSQL backend
        MSSQL_ADDRESS: localhost
        MSSQL_PORT: 1433
        MSSQL_USER: sa
        MSSQL_PASSWORD: Meltan0admin
        MSSQL_DB: pytest_warehouse
      shell: bash
      run: |
        nox -t test -- -m "${{ env.PYTEST_MARKERS }}" 2>&1 | \
          tee >( \
            tail | sed -r "s/[[:cntrl:]]\[([0-9]{1,3};)*[0-9]{1,3}m//g" | grep -E '^=+ [0-9].+ =+$' | \
            { read pytest_results; echo "pytest_results='${pytest_results}'" >> ${GITHUB_ENV}; } )

    - name: Append row to workflow summary table
      id: append-results
      shell: bash
      run: |
        echo "pytest_results='${{ env.pytest_results }}'"
        pytest_results_count () { pattern="([0-9]+) $1" && [[ ${{ env.pytest_results }} =~ $pattern ]] && echo "${BASH_REMATCH[1]}" || echo 0; }
        echo "pytest-results-row-${{ matrix.id }}=| \
        ${{ matrix.python-version }} | \
        ${RUNNER_OS} ${RUNNER_ARCH,,} | \
        ${{ matrix.backend-db }} | \
        $( pytest_results_count passed ) | \
        $( pytest_results_count failed ) | \
        $( pytest_results_count xpassed ) | \
        $( pytest_results_count xfailed ) | \
        $( pytest_results_count skipped ) | \
        $( pytest_results_count deselected ) | \
        $( pytest_results_count warning ) | \
        $( pytest_results_count error ) | \
        $( pattern='in ([0-9]+)\.[0-9]+s' && [[ ${{ env.pytest_results }} =~ $pattern ]] && echo "${BASH_REMATCH[1]}" )s |" >> $GITHUB_OUTPUT

    - name: Upload coverage data
      if: always()
      uses: actions/upload-artifact@v3.1.2
      with:
        name: coverage-data
        path: ".coverage.*"

    - name: Upload test logs
      if: always()
      uses: actions/upload-artifact@v3.1.2
      with:
        name: test-logs-${{ matrix.id }}
        path: pytest.log

  summary:
    runs-on: ubuntu-latest
    needs: tests
    steps:
    - name: Summarize workflow
      run: |
        echo '## Test results' >> ${GITHUB_STEP_SUMMARY}
        echo '' >> ${GITHUB_STEP_SUMMARY}
        echo '| PYTHON | OS     | DB     | PASSED | FAILED | XPASSED | XFAILED | SKIPPED | DESELECTED | WARNINGS | ERRORS | DURATION |' >> ${GITHUB_STEP_SUMMARY}
        echo '|--------|--------|--------|--------|--------|---------|---------|---------|------------|----------|--------|----------|' >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-01 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-02 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-03 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-04 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-05 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-06 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-07 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-08 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-09 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-10 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-11 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-12 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-13 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-14 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-15 }}" >> ${GITHUB_STEP_SUMMARY}
        # echo "${{ needs.tests.outputs.pytest-results-row-16 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-17 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-18 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-19 }}" >> ${GITHUB_STEP_SUMMARY}
        echo "${{ needs.tests.outputs.pytest-results-row-20 }}" >> ${GITHUB_STEP_SUMMARY}
        echo '' >> ${GITHUB_STEP_SUMMARY}
        echo 'Please address any tests which have errored, failed, or xpassed, and any warnings emitted.' >> ${GITHUB_STEP_SUMMARY}

  coverage:
    runs-on: ubuntu-latest
    needs: tests
    steps:
    - name: Check out the repository
      uses: actions/checkout@v4.0.0

    - name: Install Poetry
      env:
        PIP_CONSTRAINT: .github/workflows/resources/constraints.txt
      run: |
        pipx install poetry
        poetry --version

    - name: Set up Python 3.10
      uses: actions/setup-python@v4.7.0
      with:
        python-version: '3.10'
        architecture: x64
        # Nox uses pip, so we cache with pip
        cache: 'pip'
        cache-dependency-path: 'poetry.lock'

    - name: Download coverage data
      uses: actions/download-artifact@v3.0.2
      with:
        name: coverage-data

    - name: Install Nox
      env:
        PIP_CONSTRAINT: .github/workflows/resources/constraints.txt
      run: |
        pipx install nox
        pipx inject nox nox-poetry

    - name: Combine coverage data and display human readable report
      run: |
        nox -rs coverage -- combine
        nox -rs coverage -- report --show-missing --ignore-errors

    - name: Create coverage report
      run: |
        nox -rs coverage -- xml --ignore-errors

    - name: Upload coverage report
      uses: codecov/codecov-action@v3.1.4

  mypy:
    name: "Static type checking"
    runs-on: ubuntu-latest
    env:
      FORCE_COLOR: 1

    steps:
    - name: Check out the repository
      uses: actions/checkout@v4.0.0

    - name: Install Poetry
      env:
        PIP_CONSTRAINT: .github/workflows/resources/constraints.txt
      run: |
        pipx install poetry
        poetry --version

    - name: Setup Python 3.10
      uses: actions/setup-python@v4.7.0
      with:
        python-version: '3.10'
        architecture: x64
        # Nox uses pip, so we cache with pip
        cache: 'pip'
        cache-dependency-path: 'poetry.lock'

    - name: Install Nox
      env:
        PIP_CONSTRAINT: .github/workflows/resources/constraints.txt
      run: |
        pipx install nox
        pipx inject nox nox-poetry

    - name: Run mypy
      run: |
        nox -rs mypy