Meltano Pipeline #877
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Meltano Pipeline | |
| on: | |
| pull_request: | |
| types: [opened, reopened, labeled, synchronize] | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: Meltano Environment | |
| required: true | |
| default: prod | |
| logConfig: | |
| description: 'Logging configuration file' | |
| required: true | |
| default: 'logging/ci.yaml' | |
| type: choice | |
| options: | |
| - logging/ci.yaml | |
| - logging/dev.yaml | |
| schedule: | |
| - cron: '0 12 * * *' # Once a day around 6am CST (12am UTC) | |
| env: | |
| MELTANO_DATABASE_URI: ${{ secrets.MELTANO_DATABASE_URI }} | |
| DUCKDB_PATH: ${{ secrets.DUCKDB_PATH }} | |
| MELTANO_CLI_LOG_CONFIG: ${{ github.event.inputs.environment || 'logging/ci.yaml' }} | |
| MELTANO_ENVIRONMENT: ${{ github.event.inputs.environment || 'prod' }} | |
| MOTHERDUCK_TOKEN: ${{ secrets.MOTHERDUCK_TOKEN }} | |
| jobs: | |
| meltano-run: | |
| if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'Test/ELTP') }} | |
| name: "Meltano Job: ${{ matrix.job }}" | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/meltano/meltano:latest-python3.11 | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GHCR_IO_TOKEN }} | |
| env: | |
| TAP_GETPOCKET_CONSUMER_KEY: ${{ secrets.TAP_GETPOCKET_CONSUMER_KEY }} | |
| TAP_GETPOCKET_ACCESS_TOKEN: ${{ secrets.TAP_GETPOCKET_ACCESS_TOKEN }} | |
| TAP_GITHUB_AUTH_TOKEN: ${{ secrets.TAP_GITHUB_AUTH_TOKEN }} | |
| TAP_READTHEDOCS_TOKEN: ${{ secrets.TAP_READTHEDOCS_TOKEN }} | |
| TAP_STACKEXCHANGE_KEY: ${{ secrets.TAP_STACKEXCHANGE_KEY }} | |
| TARGET_DUCKDB_PATH: ${{ secrets.TARGET_DUCKDB_PATH }} | |
| strategy: | |
| matrix: | |
| include: | |
| - job: "pocket-to-duckdb" | |
| schedule: "daily_pocket" | |
| - job: "stackoverflow-to-duckdb" | |
| schedule: "daily_stackoverflow" | |
| - job: "rtd-to-duckdb" | |
| schedule: "daily_readthedocs" | |
| - job: "gh-to-duckdb" | |
| schedule: "daily_github" | |
| fail-fast: false | |
| steps: | |
| - name: Checkout code | |
| uses: actions/[email protected] | |
| - name: Check Meltano | |
| run: | | |
| meltano --version | |
| - name: Install Plugins | |
| run: | | |
| meltano install --schedule ${{ matrix.schedule }} | |
| - name: Run | |
| env: | |
| MELTANO_FF_STRICT_ENV_VAR_MODE: true | |
| run: | | |
| meltano run ${{ matrix.job }} | |
| - name: Rename metrics log | |
| run: | | |
| mv logs/singer_metrics.log logs/singer_metrics_${{ matrix.job }}.log | |
| - name: Upload metrics log | |
| uses: actions/[email protected] | |
| with: | |
| name: singer-metrics | |
| path: logs/singer_metrics_${{ matrix.job }}.log | |
| lint: | |
| name: Lint models | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/meltano/meltano:latest-python3.9 | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GHCR_IO_TOKEN }} | |
| env: | |
| DUCKDB_PATH: ${{ secrets.DUCKDB_PATH }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/[email protected] | |
| - name: Check Meltano | |
| run: | | |
| meltano --version | |
| - name: Install SQLFluff | |
| run: | | |
| meltano install utility sqlfluff | |
| - name: Lint | |
| env: | |
| MELTANO_CLI_LOG_LEVEL: info | |
| run: | | |
| meltano invoke sqlfluff:lint --format github-annotation-native -v transform/models evidence/sources | |
| transform: | |
| name: Data Transformation with dbt-duckdb | |
| runs-on: ubuntu-latest | |
| needs: [meltano-run, lint] | |
| container: | |
| image: ghcr.io/meltano/meltano:latest-python3.9 | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GHCR_IO_TOKEN }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/[email protected] | |
| - name: Check Meltano | |
| run: | | |
| meltano --version | |
| - name: Install dbt-duckdb | |
| run: | | |
| meltano install transformer dbt-duckdb | |
| - name: Transform and Test | |
| run: | | |
| meltano run dbt-duckdb:build | |
| build-evidence: | |
| name: Build Evidence site locally | |
| needs: [transform] | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/meltano/meltano:latest-python3.9 | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GHCR_IO_TOKEN }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/[email protected] | |
| - name: Check Meltano | |
| run: | | |
| meltano --version | |
| - name: Install evidence | |
| run: | | |
| meltano install transform dbt-duckdb | |
| meltano install utility evidence | |
| - name: Build | |
| run: | | |
| meltano invoke evidence build --strict | |
| - name: Upload Evidence build | |
| uses: actions/[email protected] | |
| with: | |
| name: evidence-build | |
| path: evidence/build | |
| deploy-evidence: | |
| name: Deploy Evidence site | |
| needs: [build-evidence] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/[email protected] | |
| - name: Set Git config | |
| run: | | |
| git config --global --add safe.directory /__w/meltano-dataops/meltano-dataops | |
| - name: Download Evidence build | |
| uses: actions/[email protected] | |
| with: | |
| name: evidence-build | |
| path: evidence/build | |
| - name: Deploy | |
| uses: amondnet/[email protected] | |
| with: | |
| github-comment: ${{ github.event_name == 'pull_request' && 'true' || 'false' }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| vercel-token: ${{ secrets.VERCEL_TOKEN }} | |
| vercel-org-id: ${{ secrets.VERCEL_ORG_ID }} | |
| vercel-project-id: ${{ secrets.VERCEL_PROJECT_ID }} | |
| vercel-args: ${{ github.event_name == 'pull_request' && '' || '--prod' }} |