meltano-run.yml

name: Meltano Run

on:
  pull_request: {}
  push:
    branches: [main]
  schedule:
  # Daily at 12:00 PM UTC
  - cron: '0 12 * * *'
  workflow_dispatch:

jobs:
  meltano-run:
    name: ${{ matrix.mapping }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
        - tap: tap-csv
          mapping: hash_email
          output_db: tap_csv.db
          query: "select * from customers limit 20"
        - tap: tap-smoke-test
          mapping: whitelist
          output_db: tap_smoke_test.db
          query: "select * from animals limit 20"
        - tap: tap-smoke-test
          mapping: json_field
          output_db: tap_smoke_test.db
          query: "select * from animals limit 20"
        - tap: people
          mapping: flatten
          output_db: people.db
          install_extractor: false
          query: "select * from people limit 20"
        - tap: nested
          mapping: comprehension
          output_db: nested.db
          install_extractor: false
          query: "select * from users limit 20"
        - tap: tap-csv
          mapping: fake
          output_db: tap_csv.db
          install_extractor: false
          query: "select * from customers limit 20"
        - tap: tap-csv--multi
          mapping: glob
          output_db: tap_csv__multi.db
          query: |
            with
              cust as (select *, 'CUSTOMER' as type from customers limit 10),
              empl as (select *, 'EMPLOYEE' as type from employees limit 10)

            select * from cust
            union
            select * from empl
        - tap: tap-csv--multi
          mapping: single_field
          output_db: tap_csv__multi.db
          query: |
            with
              cust as (select *, 'CUSTOMER' as type from customers limit 10),
              empl as (select *, 'EMPLOYEE' as type from employees limit 10)

            select * from cust
            union
            select * from empl
        - tap: tap-csv--multi
          mapping: alias_property_all_streams
          output_db: tap_csv__multi.db
          query: |
            with
              cust as (select *, 'CUSTOMER' as type from customers limit 10),
              empl as (select *, 'EMPLOYEE' as type from employees limit 10)

            select * from cust
            union
            select * from empl
    steps:
    - uses: actions/checkout@v4

    - uses: actions/setup-python@v5
      id: setup-python
      with:
        python-version: 3.x
        cache: pip
        cache-dependency-path: |
          meltano.yml
          poetry.lock

    - name: Install Meltano
      env:
        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
      run: |
        pipx install --python '${{ steps.setup-python.outputs.python-path }}' meltano

    - name: Check Meltano version
      run: |
        meltano --version

    - name: Run
      run: |
        meltano run --install ${{ matrix.tap }} ${{ matrix.mapping }} target-sqlite

    - name: Check output
      run: |
        sqlite3 -markdown output/${{ matrix.output_db }} "${{ matrix.query }}" >> $GITHUB_STEP_SUMMARY