From 63aa0a0caa7243c092a84ecec80f5cadc5928e1d Mon Sep 17 00:00:00 2001 From: Josh lloyd Date: Mon, 26 Feb 2024 10:59:13 -0700 Subject: [PATCH] stringified all values --- tap_orca/client.py | 8 ++++++++ tap_orca/tap.py | 18 ++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/tap_orca/client.py b/tap_orca/client.py index 6236cb1..9f7e0f8 100644 --- a/tap_orca/client.py +++ b/tap_orca/client.py @@ -64,3 +64,11 @@ def get_new_paginator(self) -> BaseAPIPaginator: A paginator instance. """ return BaseOffsetPaginator(0, self.page_size) + + def post_process( + self, + row: dict, + context: dict | None = None, # noqa: ARG002 + ) -> dict | None: + """Post-process a record before writing it.""" + return {k: str(v) for k, v in row.items()} diff --git a/tap_orca/tap.py b/tap_orca/tap.py index 4eccd64..36693b9 100644 --- a/tap_orca/tap.py +++ b/tap_orca/tap.py @@ -32,7 +32,13 @@ class TapOrca(Tap): th.ArrayType(th.StringType), required=False, description="The primary keys for the custom stream." - ) + ), + th.Property( + "schema", + th.ArrayType(th.StringType), + required=False, + description="A simple schema for the custom stream." + ), ) config_jsonschema = th.PropertiesList( @@ -101,17 +107,17 @@ def discover_schema(self, query: str) -> dict: th.CustomType(jsonschema_type_dict={ "anyOf": [ {"type": "null"}, - {"type": "object"}, - {"type": "array"}, + # {"type": "object"}, + # {"type": "array"}, {"type": "string"}, - {"type": "number"}, + # {"type": "number"}, # {"type": "boolean"}, # causes all values to be a boolean - {"type": "integer"}, + # {"type": "integer"}, ] }) ), ) - + self.logger.info("Schema discovery complete.") return schema.to_dict()