meltano.yml

version: 1
send_anonymous_usage_stats: false
project_id: 324fc8e1-67e5-4d15-a810-e2814fdb0d79
plugins:
  extractors:
    - name: tap-mongodb
      variant: singer-io
      pip_url: tap-mongodb
      config:
        user: datateer
        password: $TAP_MONGODB_PASSWORD
        host: localhost
        port: 27017
        database: sample_airbnb
      metadata:
        sample_airbnb-listingsAndReviews:
          replication-key: _id
          replication-method: INCREMENTAL
      catalog: ./catalog.json
  loaders:
    - name: target-jsonl
      namespace: target_jsonl
      pip_url: git+ssh://git@github.com/Datateer/target-jsonl.git@master
      executable: target-jsonl
      settings:
        - name: destination_path
          value: ./output
    - name: target-s3-jsonl
      namespace: target_s3_jsonl
      pip_url: git+ssh://git@github.com/datateer/target-s3-jsonl.git@main
      executable: target-s3-jsonl
      settings:
        - name: aws_access_key_id
          env: TARGET_S3_AWS_ACCESS_KEY_ID
        - name: aws_secret_access_key
          env: TARGET_S3_AWS_SECRET_ACCESS_KEY
        - name: s3_bucket
          env: S3_RAW_BUCKET
        - name: naming_convention
        - name: temp_dir
        - name: date_format
          value: "%Y-%m-%d"
      config:
        naming_convention: sample/tap-mongodb/{stream}/export_date={date}/{stream}_{timestamp}.jsonl