From d65de2d5697db45f60d4660b78aaea6c91245749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Burwash?= Date: Thu, 7 Mar 2024 11:48:33 -0500 Subject: [PATCH 1/4] WIP --- .../workflows/ci_validate_meltano_install.yml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/ci_validate_meltano_install.yml diff --git a/.github/workflows/ci_validate_meltano_install.yml b/.github/workflows/ci_validate_meltano_install.yml new file mode 100644 index 0000000..d82c7a1 --- /dev/null +++ b/.github/workflows/ci_validate_meltano_install.yml @@ -0,0 +1,22 @@ +name: Validate Meltano Install +on: + pull_request: + types: + - opened + - reopened + - synchronize + +jobs: + setup-meltano: + name: Validate meltano configurations + runs-on: ubuntu-latest + timeout-minutes: 20 + + + steps: + - uses: actions/checkout@v4 + - run: pip install meltano boto3 + - name: Install dependencies + run: meltano install --clean + - name: Validate meltano install + run: meltano schedule list From f1364607fece19213697c99af1217c0309c5ab77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Burwash?= Date: Thu, 7 Mar 2024 11:48:39 -0500 Subject: [PATCH 2/4] Functional --- .github/workflows/ci_validate_meltano_install.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci_validate_meltano_install.yml b/.github/workflows/ci_validate_meltano_install.yml index d82c7a1..5676606 100644 --- a/.github/workflows/ci_validate_meltano_install.yml +++ b/.github/workflows/ci_validate_meltano_install.yml @@ -12,7 +12,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 - steps: - uses: actions/checkout@v4 - run: pip install meltano boto3 From 75e600748b115d4c7ff69380f3a33096de828636 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Burwash?= Date: Thu, 7 Mar 2024 11:50:35 -0500 Subject: [PATCH 3/4] Updated locks --- .../loaders/target-bigquery--adswerve.lock | 111 ++++++++++++++++++ plugins/loaders/target-jsonl--andyh1203.lock | 34 ++++++ 2 files changed, 145 insertions(+) create mode 100644 plugins/loaders/target-bigquery--adswerve.lock create mode 100644 plugins/loaders/target-jsonl--andyh1203.lock diff --git a/plugins/loaders/target-bigquery--adswerve.lock b/plugins/loaders/target-bigquery--adswerve.lock new file mode 100644 index 0000000..a4ebfcf --- /dev/null +++ b/plugins/loaders/target-bigquery--adswerve.lock @@ -0,0 +1,111 @@ +{ + "plugin_type": "loaders", + "name": "target-bigquery", + "namespace": "target_bigquery", + "variant": "adswerve", + "label": "Google BigQuery", + "docs": "https://hub.meltano.com/loaders/target-bigquery--adswerve", + "repo": "https://github.com/adswerve/target-bigquery", + "pip_url": "git+https://github.com/adswerve/target-bigquery.git@0.12.0", + "description": "BigQuery loader", + "logo_url": "https://hub.meltano.com/assets/logos/loaders/bigquery.png", + "settings_group_validation": [ + [ + "project_id", + "dataset_id", + "location", + "credentials_path" + ] + ], + "settings": [ + { + "name": "project_id", + "label": "Project ID", + "description": "BigQuery project" + }, + { + "name": "dataset_id", + "value": "$MELTANO_EXTRACT__LOAD_SCHEMA", + "label": "Dataset ID", + "description": "BigQuery dataset.\n\nThe default value [will expand to](https://docs.meltano.com/guide/configuration.html#expansion-in-setting-values) the value of the [`load_schema` extra](https://docs.meltano.com/concepts/plugins#load-schema-extra) for the extractor used in the pipeline, which defaults to the extractor's namespace, e.g. `tap_gitlab` for [`tap-gitlab`](/extractors/gitlab).\n" + }, + { + "name": "location", + "value": "us-central1", + "label": "Region", + "description": "Dataset Location. See https://cloud.google.com/bigquery/docs/locations.", + "placeholder": "us-central1" + }, + { + "name": "credentials_path", + "env": "GOOGLE_APPLICATION_CREDENTIALS", + "value": "$MELTANO_PROJECT_ROOT/client_secrets.json", + "label": "Credentials Path", + "description": "Fully qualified path to `client_secrets.json` for your service account.\n\nSee the [\"Activate the Google BigQuery API\" section of the repository's README](https://github.com/adswerve/target-bigquery#step-1-activate-the-google-bigquery-api) and .\n\nBy default, this file is expected to be at the root of your project directory.\n" + }, + { + "name": "validate_records", + "kind": "boolean", + "value": false, + "label": "Validate Records" + }, + { + "name": "add_metadata_columns", + "kind": "boolean", + "value": false, + "label": "Add Metadata Columns", + "description": "Add `_time_extracted` and `_time_loaded` metadata columns" + }, + { + "name": "replication_method", + "kind": "options", + "value": "append", + "label": "Replication Method", + "description": "The Replication method can be:\n\n`append`: Adding new rows to the table (Default value)\n`truncate`: Deleting all previous rows and uploading the new ones to the table\n`incremental`: Upserting new rows into the table, using the primary key given by the tap connector (if it finds an old row with same key, updates it. Otherwise it inserts the new row)\n\nWARNING: It is not recommended to use the `incremental` option (which uses MERGE SQL statement). It might result in loss of production data, because historical records get updated. Instead, we recommend using the `append` replication method, which will preserve historical data.\n", + "options": [ + { + "label": "Append", + "value": "append" + }, + { + "label": "Truncate", + "value": "truncate" + }, + { + "label": "Incremental", + "value": "incremental" + } + ] + }, + { + "name": "table_prefix", + "label": "Table Prefix", + "description": "Add prefix to table name" + }, + { + "name": "table_suffix", + "label": "Table Suffix", + "description": "Add suffix to table name" + }, + { + "name": "max_cache", + "value": 50, + "label": "Max Cache", + "description": "Maximum cache size in MB" + }, + { + "name": "merge_state_messages", + "kind": "boolean", + "value": false, + "label": "Merge State Messages", + "description": "Whether to merge multiple state messages from the tap into the state file or uses the last state message as the state file. Note that it is not recommended to set this to true when using with Meltano as the merge behavior conflicts with Meltano\u2019s merge process." + }, + { + "name": "table_config", + "label": "Table Config", + "description": "A path to a file containing the definition of partitioning and clustering." + } + ], + "dialect": "bigquery", + "target_schema": "$TARGET_BIGQUERY_DATASET_ID" +} \ No newline at end of file diff --git a/plugins/loaders/target-jsonl--andyh1203.lock b/plugins/loaders/target-jsonl--andyh1203.lock new file mode 100644 index 0000000..5825fc4 --- /dev/null +++ b/plugins/loaders/target-jsonl--andyh1203.lock @@ -0,0 +1,34 @@ +{ + "plugin_type": "loaders", + "name": "target-jsonl", + "namespace": "target_jsonl", + "variant": "andyh1203", + "label": "JSON Lines (JSONL)", + "docs": "https://hub.meltano.com/loaders/target-jsonl--andyh1203", + "repo": "https://github.com/andyh1203/target-jsonl", + "pip_url": "target-jsonl", + "description": "JSONL loader", + "logo_url": "https://hub.meltano.com/assets/logos/loaders/jsonl.png", + "settings": [ + { + "name": "destination_path", + "kind": "string", + "value": "output", + "label": "Destination Path", + "description": "Sets the destination path the JSONL files are written to, relative\nto the project root.\n\nThe directory needs to exist already, it will not be created\nautomatically.\n\nTo write JSONL files to the project root, set an empty string (`\"\"`).\n" + }, + { + "name": "do_timestamp_file", + "kind": "boolean", + "value": false, + "label": "Include Timestamp in File Names", + "description": "Specifies if the files should get timestamped.\n\nBy default, the resulting file will not have a timestamp in the file name (i.e. `exchange_rate.jsonl`).\n\nIf this option gets set to `true`, the resulting file will have a timestamp associated with it (i.e. `exchange_rate-{timestamp}.jsonl`).\n" + }, + { + "name": "custom_name", + "kind": "string", + "label": "Custom File Name Override", + "description": "Specifies a custom name for the filename, instead of the stream name.\n\nThe file name will be `{custom_name}-{timestamp}.jsonl`, if `do_timestamp_file` is `true`.\nOtherwise the file name will be `{custom_name}.jsonl`.\n\nIf custom name is not provided, the stream name will be used.\n" + } + ] +} \ No newline at end of file From 4491bae4f08057ffde9a7507be8707436bdc855f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Burwash?= Date: Thu, 7 Mar 2024 11:55:25 -0500 Subject: [PATCH 4/4] Updated tap-hibob --- meltano.yml | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/meltano.yml b/meltano.yml index a636076..55f8195 100644 --- a/meltano.yml +++ b/meltano.yml @@ -1,7 +1,9 @@ version: 1 send_anonymous_usage_stats: true -project_id: fadb6424-0b8b-4e32-9dab-c46ea2634ba8 -default_environment: dev +project_id: tap-potloc-app +default_environment: test +environments: + - name: test plugins: extractors: - name: tap-hibob @@ -42,22 +44,3 @@ plugins: - name: target-jsonl variant: andyh1203 pip_url: target-jsonl - - name: target-bigquery - variant: adswerve - pip_url: git+https://github.com/adswerve/target-bigquery.git@0.12.3 - config: - project_id: target_bigquery_project_id - credentials_path: .secrets/gbq-client_secrets-dev.json - replication_method: truncate - add_metadata_columns: true -environments: -- name: dev - config: - plugins: - loaders: - - name: target-bigquery - config: - project_id: potloc-dev - credentials_path: .secrets/gbq-client_secrets-dev.json - env: - GOOGLE_APPLICATION_CREDENTIALS: .secrets/gbq-client_secrets-dev.json