DE-2945 Maintenance - Implement dependabot as package manager - Add basic meltano CI

.github/workflows/ci_validate_meltano_install.yml

@@ -0,0 +1,21 @@
+name: Validate Meltano Install
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+jobs:
+  setup-meltano:
+    name: Validate meltano configurations
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      - uses: actions/checkout@v4
+      - run: pip install meltano boto3
+      - name: Install dependencies
+        run: meltano install --clean
+      - name: Validate meltano install
+        run: meltano schedule list

meltano.yml

@@ -1,7 +1,9 @@
 version: 1
 send_anonymous_usage_stats: true
-project_id: fadb6424-0b8b-4e32-9dab-c46ea2634ba8
-default_environment: dev
+project_id: tap-potloc-app
+default_environment: test
+environments:
+  - name: test
 plugins:
   extractors:
   - name: tap-hibob
@@ -42,22 +44,3 @@ plugins:
   - name: target-jsonl
     variant: andyh1203
     pip_url: target-jsonl
-  - name: target-bigquery
-    variant: adswerve
-    pip_url: git+https://github.com/adswerve/[email protected]
-    config:
-      project_id: target_bigquery_project_id
-      credentials_path: .secrets/gbq-client_secrets-dev.json
-      replication_method: truncate
-      add_metadata_columns: true
-environments:
-- name: dev
-  config:
-    plugins:
-      loaders:
-      - name: target-bigquery
-        config:
-          project_id: potloc-dev
-          credentials_path: .secrets/gbq-client_secrets-dev.json
-  env:
-    GOOGLE_APPLICATION_CREDENTIALS: .secrets/gbq-client_secrets-dev.json

plugins/loaders/target-bigquery--adswerve.lock

@@ -0,0 +1,111 @@
+{
+  "plugin_type": "loaders",
+  "name": "target-bigquery",
+  "namespace": "target_bigquery",
+  "variant": "adswerve",
+  "label": "Google BigQuery",
+  "docs": "https://hub.meltano.com/loaders/target-bigquery--adswerve",
+  "repo": "https://github.com/adswerve/target-bigquery",
+  "pip_url": "git+https://github.com/adswerve/[email protected]",
+  "description": "BigQuery loader",
+  "logo_url": "https://hub.meltano.com/assets/logos/loaders/bigquery.png",
+  "settings_group_validation": [
+    [
+      "project_id",
+      "dataset_id",
+      "location",
+      "credentials_path"
+    ]
+  ],
+  "settings": [
+    {
+      "name": "project_id",
+      "label": "Project ID",
+      "description": "BigQuery project"
+    },
+    {
+      "name": "dataset_id",
+      "value": "$MELTANO_EXTRACT__LOAD_SCHEMA",
+      "label": "Dataset ID",
+      "description": "BigQuery dataset.\n\nThe default value [will expand to](https://docs.meltano.com/guide/configuration.html#expansion-in-setting-values) the value of the [`load_schema` extra](https://docs.meltano.com/concepts/plugins#load-schema-extra) for the extractor used in the pipeline, which defaults to the extractor's namespace, e.g. `tap_gitlab` for [`tap-gitlab`](/extractors/gitlab).\n"
+    },
+    {
+      "name": "location",
+      "value": "us-central1",
+      "label": "Region",
+      "description": "Dataset Location. See https://cloud.google.com/bigquery/docs/locations.",
+      "placeholder": "us-central1"
+    },
+    {
+      "name": "credentials_path",
+      "env": "GOOGLE_APPLICATION_CREDENTIALS",
+      "value": "$MELTANO_PROJECT_ROOT/client_secrets.json",
+      "label": "Credentials Path",
+      "description": "Fully qualified path to `client_secrets.json` for your service account.\n\nSee the [\"Activate the Google BigQuery API\" section of the repository's README](https://github.com/adswerve/target-bigquery#step-1-activate-the-google-bigquery-api) and <https://cloud.google.com/docs/authentication/production>.\n\nBy default, this file is expected to be at the root of your project directory.\n"
+    },
+    {
+      "name": "validate_records",
+      "kind": "boolean",
+      "value": false,
+      "label": "Validate Records"
+    },
+    {
+      "name": "add_metadata_columns",
+      "kind": "boolean",
+      "value": false,
+      "label": "Add Metadata Columns",
+      "description": "Add `_time_extracted` and `_time_loaded` metadata columns"
+    },
+    {
+      "name": "replication_method",
+      "kind": "options",
+      "value": "append",
+      "label": "Replication Method",
+      "description": "The Replication method can be:\n\n`append`: Adding new rows to the table (Default value)\n`truncate`: Deleting all previous rows and uploading the new ones to the table\n`incremental`: Upserting new rows into the table, using the primary key given by the tap connector (if it finds an old row with same key, updates it. Otherwise it inserts the new row)\n\nWARNING: It is not recommended to use the `incremental` option (which uses MERGE SQL statement). It might result in loss of production data, because historical records get updated. Instead, we recommend using the `append` replication method, which will preserve historical data.\n",
+      "options": [
+        {
+          "label": "Append",
+          "value": "append"
+        },
+        {
+          "label": "Truncate",
+          "value": "truncate"
+        },
+        {
+          "label": "Incremental",
+          "value": "incremental"
+        }
+      ]
+    },
+    {
+      "name": "table_prefix",
+      "label": "Table Prefix",
+      "description": "Add prefix to table name"
+    },
+    {
+      "name": "table_suffix",
+      "label": "Table Suffix",
+      "description": "Add suffix to table name"
+    },
+    {
+      "name": "max_cache",
+      "value": 50,
+      "label": "Max Cache",
+      "description": "Maximum cache size in MB"
+    },
+    {
+      "name": "merge_state_messages",
+      "kind": "boolean",
+      "value": false,
+      "label": "Merge State Messages",
+      "description": "Whether to merge multiple state messages from the tap into the state file or uses the last state message as the state file. Note that it is not recommended to set this to true when using with Meltano as the merge behavior conflicts with Meltano\u2019s merge process."
+    },
+    {
+      "name": "table_config",
+      "label": "Table Config",
+      "description": "A path to a file containing the definition of partitioning and clustering."
+    }
+  ],
+  "dialect": "bigquery",
+  "target_schema": "$TARGET_BIGQUERY_DATASET_ID"
+}

plugins/loaders/target-jsonl--andyh1203.lock

@@ -0,0 +1,34 @@
+{
+  "plugin_type": "loaders",
+  "name": "target-jsonl",
+  "namespace": "target_jsonl",
+  "variant": "andyh1203",
+  "label": "JSON Lines (JSONL)",
+  "docs": "https://hub.meltano.com/loaders/target-jsonl--andyh1203",
+  "repo": "https://github.com/andyh1203/target-jsonl",
+  "pip_url": "target-jsonl",
+  "description": "JSONL loader",
+  "logo_url": "https://hub.meltano.com/assets/logos/loaders/jsonl.png",
+  "settings": [
+    {
+      "name": "destination_path",
+      "kind": "string",
+      "value": "output",
+      "label": "Destination Path",
+      "description": "Sets the destination path the JSONL files are written to, relative\nto the project root.\n\nThe directory needs to exist already, it will not be created\nautomatically.\n\nTo write JSONL files to the project root, set an empty string (`\"\"`).\n"
+    },
+    {
+      "name": "do_timestamp_file",
+      "kind": "boolean",
+      "value": false,
+      "label": "Include Timestamp in File Names",
+      "description": "Specifies if the files should get timestamped.\n\nBy default, the resulting file will not have a timestamp in the file name (i.e. `exchange_rate.jsonl`).\n\nIf this option gets set to `true`, the resulting file will have a timestamp associated with it (i.e. `exchange_rate-{timestamp}.jsonl`).\n"
+    },
+    {
+      "name": "custom_name",
+      "kind": "string",
+      "label": "Custom File Name Override",
+      "description": "Specifies a custom name for the filename, instead of the stream name.\n\nThe file name will be `{custom_name}-{timestamp}.jsonl`, if `do_timestamp_file` is `true`.\nOtherwise the file name will be `{custom_name}.jsonl`.\n\nIf custom name is not provided, the stream name will be used.\n"
+    }
+  ]
+}