From 7b4fc16c444fb7996f1caf6c0aa3df9943f29586 Mon Sep 17 00:00:00 2001 From: Josh lloyd Date: Thu, 20 Jun 2024 13:54:49 -0600 Subject: [PATCH] flattened the code object in the customer_recipes stream --- meltano.yml | 6 +++ plugins/loaders/target-jsonl--andyh1203.lock | 34 +++++++++++++++ pyproject.toml | 2 +- tap_workato/streams.py | 46 ++++++++++++++++---- 4 files changed, 79 insertions(+), 9 deletions(-) create mode 100644 plugins/loaders/target-jsonl--andyh1203.lock diff --git a/meltano.yml b/meltano.yml index 5a0ebd5..9ad5517 100644 --- a/meltano.yml +++ b/meltano.yml @@ -16,7 +16,13 @@ plugins: kind: password - name: user_email kind: password +# select: +# - 'customer_recipes.*' +# - '!api_clients.*' loaders: - name: target-jsonl variant: andyh1203 pip_url: target-jsonl +environments: + - name: default +default_environment: default diff --git a/plugins/loaders/target-jsonl--andyh1203.lock b/plugins/loaders/target-jsonl--andyh1203.lock new file mode 100644 index 0000000..5825fc4 --- /dev/null +++ b/plugins/loaders/target-jsonl--andyh1203.lock @@ -0,0 +1,34 @@ +{ + "plugin_type": "loaders", + "name": "target-jsonl", + "namespace": "target_jsonl", + "variant": "andyh1203", + "label": "JSON Lines (JSONL)", + "docs": "https://hub.meltano.com/loaders/target-jsonl--andyh1203", + "repo": "https://github.com/andyh1203/target-jsonl", + "pip_url": "target-jsonl", + "description": "JSONL loader", + "logo_url": "https://hub.meltano.com/assets/logos/loaders/jsonl.png", + "settings": [ + { + "name": "destination_path", + "kind": "string", + "value": "output", + "label": "Destination Path", + "description": "Sets the destination path the JSONL files are written to, relative\nto the project root.\n\nThe directory needs to exist already, it will not be created\nautomatically.\n\nTo write JSONL files to the project root, set an empty string (`\"\"`).\n" + }, + { + "name": "do_timestamp_file", + "kind": "boolean", + "value": false, + "label": "Include Timestamp in File Names", + "description": "Specifies if the files should get timestamped.\n\nBy default, the resulting file will not have a timestamp in the file name (i.e. `exchange_rate.jsonl`).\n\nIf this option gets set to `true`, the resulting file will have a timestamp associated with it (i.e. `exchange_rate-{timestamp}.jsonl`).\n" + }, + { + "name": "custom_name", + "kind": "string", + "label": "Custom File Name Override", + "description": "Specifies a custom name for the filename, instead of the stream name.\n\nThe file name will be `{custom_name}-{timestamp}.jsonl`, if `do_timestamp_file` is `true`.\nOtherwise the file name will be `{custom_name}.jsonl`.\n\nIf custom name is not provided, the stream name will be used.\n" + } + ] +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 01f5076..3d52825 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tap-workato" -version = "0.0.3" +version = "0.1.0" description = "`tap-workato` is a Singer tap for Workato, built with the Meltano SDK for Singer Taps." authors = ["Josh Lloyd"] keywords = [ diff --git a/tap_workato/streams.py b/tap_workato/streams.py index 4e448d3..cc90f4b 100644 --- a/tap_workato/streams.py +++ b/tap_workato/streams.py @@ -1,5 +1,5 @@ """Stream type classes for tap-workato.""" - +import json from typing import Optional from singer_sdk import typing as th @@ -377,7 +377,22 @@ class CustomerRecipesStream(CustomerChildStreams): th.Property("version_no", th.IntegerType), th.Property("webhook_url", th.StringType), th.Property("stop_cause", th.StringType), - th.Property("code", th.StringType), + th.Property("code_number", th.IntegerType), + th.Property("code_provider", th.StringType), + th.Property("code_name", th.StringType), + th.Property("code_as", th.StringType), + th.Property("code_title", th.StringType), + th.Property("code_description", th.StringType), + th.Property("code_keyword", th.StringType), + th.Property("code_dynamicPickListSelection", th.StringType), + th.Property("code_toggleCfg", th.StringType), + th.Property("code_input", th.StringType), + th.Property("code_extended_output_schema", th.StringType), + th.Property("code_extended_input_schema", th.StringType), + th.Property("code_visible_config_fields", th.StringType), + th.Property("code_block", th.StringType), + th.Property("code_uuid", th.StringType), + th.Property("code_unfinished", th.BooleanType), th.Property( "config", th.ArrayType( @@ -392,12 +407,27 @@ class CustomerRecipesStream(CustomerChildStreams): ), ).to_dict() - def get_child_context(self, record: dict, context: Optional[dict]) -> dict: - """Return a context dictionary for child streams.""" - return { - "customer_account_id": record["customer_account_id"], - "recipe_id": record["id"], - } + def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]: + """As needed, append or transform raw data to match expected structure.""" + new_row = row.copy() + + # flatten the code object because it can get to large for target systems + code = json.loads(row["code"]) + del new_row["code"] + for key, value in code.items(): + if isinstance(value, dict) or isinstance(value, list): + new_row[f"code_{key}"] = json.dumps(value) + else: + new_row[f"code_{key}"] = value + + return new_row + + # def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + # """Return a context dictionary for child streams.""" + # return { + # "customer_account_id": record["customer_account_id"], + # "recipe_id": record["id"], + # } # not currently possible to hit this endpoint for managed users' recipes