PGQueuer CI Pipeline #2000
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PGQueuer CI Pipeline | |
| on: | |
| pull_request: | |
| branches: [main] | |
| push: | |
| branches: [main] | |
| schedule: | |
| - cron: "0 */6 * * *" | |
| jobs: | |
| run-example: | |
| if: github.event_name != 'schedule' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.12"] | |
| os: [ubuntu-24.04] | |
| postgres-version: ["16"] | |
| env: | |
| PGUSER: pgquser | |
| PGDATABASE: pgqdb | |
| PGPASSWORD: pgqpw | |
| PGHOST: localhost | |
| name: Run PGQueuer Example | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 5 | |
| services: | |
| postgres: | |
| image: postgres:${{ matrix.postgres-version }} | |
| env: | |
| POSTGRES_USER: ${{ env.PGUSER }} | |
| POSTGRES_DB: ${{ env.PGDATABASE }} | |
| POSTGRES_PASSWORD: ${{ env.PGPASSWORD }} | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| --tmpfs /var/lib/postgresql/data:rw | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "uv.lock" | |
| - name: Set up Python | |
| run: | | |
| uv sync --group asyncpg --frozen --python ${{ matrix.python-version }} | |
| uv run --frozen pgq install | |
| - name: Test pgq CLI Command | |
| run: | | |
| source .venv/bin/activate | |
| pgq --help | |
| - name: Run Example with SIGINT | |
| run: | | |
| source .venv/bin/activate | |
| PYTHONUNBUFFERED=1 pgq run examples.consumer:main & | |
| pid=$! | |
| sleep 5 | |
| kill -SIGINT $pid | |
| wait $pid | |
| run-benchmark: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.12"] | |
| os: [ubuntu-24.04] | |
| pgdriver: ["apg", "apgpool", "psy"] | |
| postgres-version: ["16"] | |
| env: | |
| PGUSER: pgquser | |
| PGDATABASE: pgqdb | |
| PGPASSWORD: pgqpw | |
| PGHOST: localhost | |
| name: Run PGQueuer Benchmark | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 5 | |
| services: | |
| postgres: | |
| image: postgres:${{ matrix.postgres-version }} | |
| env: | |
| POSTGRES_USER: ${{ env.PGUSER }} | |
| POSTGRES_DB: ${{ env.PGDATABASE }} | |
| POSTGRES_PASSWORD: ${{ env.PGPASSWORD }} | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| --tmpfs /var/lib/postgresql/data:rw | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "uv.lock" | |
| - name: Set up Python | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Install PGQueuer | |
| run: | | |
| uv sync --all-extras --dev | |
| uv run --frozen pgq install | |
| - name: Set ISO Timestamp | |
| id: timestamp | |
| run: echo "TIMESTAMP=$(date -u +%s)" >> $GITHUB_ENV | |
| - name: Set Branch or Tag Name | |
| id: get_ref | |
| run: | | |
| if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then | |
| echo "REF_NAME=${GITHUB_HEAD_REF}" >> $GITHUB_ENV | |
| else | |
| echo "REF_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV | |
| fi | |
| - name: Benchmark | |
| run: | | |
| PYTHONUNBUFFERED=1 uv run python3 tools/benchmark.py \ | |
| --driver ${{ matrix.pgdriver }} \ | |
| --output-json benchmark.json | |
| - name: Commit and Push Benchmark to Artifact Repository | |
| env: | |
| ARTIFACTS_REPO_URL: https://github.com/janbjorge/artifacts-storage.git | |
| ARTIFACTS_REPO_PAT: ${{ secrets.ARTIFACTS_STORAGE_PAT }} | |
| run: | | |
| git clone https://x-access-token:${ARTIFACTS_REPO_PAT}@github.com/janbjorge/artifacts-storage.git artifacts-repo | |
| mkdir -p artifacts-repo/pgqueuer/benchmark/${{ matrix.pgdriver }} | |
| cp benchmark.json artifacts-repo/pgqueuer/benchmark/${{ matrix.pgdriver }}/benchmark-${{ env.TIMESTAMP }}.json | |
| cd artifacts-repo | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add . | |
| git commit -m "Add benchmark-${{ matrix.pgdriver }}-${{ env.TIMESTAMP }}.json" | |
| push_with_retry() { | |
| local retries=5 | |
| local count=0 | |
| until git push origin main; do | |
| count=$((count + 1)) | |
| if [ $count -ge $retries ]; then | |
| echo "Push failed after $retries attempts." | |
| exit 1 | |
| fi | |
| echo "Push failed, attempting to pull and retry... ($count/$retries)" | |
| git fetch origin main | |
| git rebase origin/main --autosquash | |
| done | |
| } | |
| push_with_retry | |
| validate-benchmark: | |
| needs: [run-benchmark] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.12"] | |
| os: [ubuntu-24.04] | |
| name: Validate Benchmark Results | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "uv.lock" | |
| - name: Set up Python | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Set Branch or Tag Name | |
| id: get_ref | |
| run: | | |
| if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then | |
| echo "REF_NAME=${GITHUB_HEAD_REF}" >> $GITHUB_ENV | |
| else | |
| echo "REF_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV | |
| fi | |
| - name: Clone Artifact Repository | |
| run: git clone https://github.com/janbjorge/artifacts-storage.git | |
| - name: Install Dependencies | |
| run: uv venv && uv pip install pydantic | |
| - name: Validate RPS Against Main | |
| run: | | |
| cd artifacts-storage | |
| PYTHONUNBUFFERED=1 uv run python3 pgqueuer/compare_rps_main.py | |
| run-install-upgrade: | |
| if: github.event_name != 'schedule' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.12"] | |
| os: [ubuntu-24.04] | |
| postgres-version: ["11", "13", "14", "15", "16", "17"] | |
| env: | |
| PGUSER: pgquser | |
| PGDATABASE: pgqdb | |
| PGPASSWORD: pgqpw | |
| PGHOST: localhost | |
| name: Run PGQueuer install and upgrade | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 5 | |
| services: | |
| postgres: | |
| image: postgres:${{ matrix.postgres-version }} | |
| env: | |
| POSTGRES_USER: ${{ env.PGUSER }} | |
| POSTGRES_DB: ${{ env.PGDATABASE }} | |
| POSTGRES_PASSWORD: ${{ env.PGPASSWORD }} | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| --tmpfs /var/lib/postgresql/data:rw | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "uv.lock" | |
| - name: PGQueuer install and upgrade | |
| run: | | |
| uv sync --group asyncpg --frozen --python ${{ matrix.python-version }} | |
| uv run --frozen pgq install | |
| uv run --frozen pgq upgrade | |
| run-tests: | |
| if: github.event_name != 'schedule' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12", "3.13"] | |
| postgres-version: ["13", "14", "15", "16", "17"] | |
| os: [ubuntu-24.04] | |
| env: | |
| PGUSER: pgquser | |
| PGDATABASE: pgqdb | |
| PGPASSWORD: pgqpw | |
| PGHOST: localhost | |
| name: Test PGQueuer with Multiple Python and Postgres Versions | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 5 | |
| services: | |
| postgres: | |
| image: postgres:${{ matrix.postgres-version }} | |
| env: | |
| POSTGRES_USER: ${{ env.PGUSER }} | |
| POSTGRES_DB: ${{ env.PGDATABASE }} | |
| POSTGRES_PASSWORD: ${{ env.PGPASSWORD }} | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| --tmpfs /var/lib/postgresql/data:rw | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "uv.lock" | |
| - name: Set up Python | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Install PGQueuer | |
| run: | | |
| uv sync --dev --frozen | |
| uv run --frozen pgq install | |
| - name: Run Full Test Suite | |
| run: PYTHONASYNCIODEBUG=1 PYTHONDEBUG=1 PYTHONFAULTHANDLER=1 uv run pytest -v | |
| run-ruff: | |
| if: github.event_name != 'schedule' | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "uv.lock" | |
| - name: Set up Python | |
| run: uv python install 3.12 | |
| - name: Install Dev Environment | |
| run: uv sync --all-extras --dev | |
| - name: Run Ruff Linter | |
| if: ${{ always() }} | |
| run: uv run ruff check . | |
| - name: Apply Ruff Formatting | |
| if: ${{ always() }} | |
| run: uv run ruff format . --check | |
| run-mypy: | |
| if: github.event_name != 'schedule' | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "uv.lock" | |
| - name: Set up Python | |
| run: uv python install 3.12 | |
| - name: Install Dev Environment | |
| run: uv sync --all-extras --dev | |
| - name: Run Mypy Type Check | |
| run: uv run mypy . | |
| complete-pipeline: | |
| if: github.event_name != 'schedule' | |
| needs: [validate-benchmark, run-tests, run-ruff, run-mypy, run-example] | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Confirm All Tests Completed | |
| run: echo "All tests completed successfully." |