From dc2755a0fbbd28a96868146396f5c723ca59bfd5 Mon Sep 17 00:00:00 2001 From: Niels <77604434+nielsmai@users.noreply.github.com> Date: Tue, 30 May 2023 15:05:33 -0400 Subject: [PATCH 1/3] removed post_process --- tap_hubspot/client.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tap_hubspot/client.py b/tap_hubspot/client.py index 6ad7892..43a3103 100644 --- a/tap_hubspot/client.py +++ b/tap_hubspot/client.py @@ -85,13 +85,13 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]: """Parse the response and return an iterator of result rows.""" yield from extract_jsonpath(self.records_jsonpath, input=response.json()) - def post_process(self, row: dict, context: Optional[dict]) -> dict: - """As needed, append or transform raw data to match expected structure. - Returns row, or None if row is to be excluded""" - if self.replication_key: - if strptime_to_utc(row[self.replication_key]) <= self.get_starting_timestamp(context).astimezone(pytz.utc): - return None - return row + # def post_process(self, row: dict, context: Optional[dict]) -> dict: + # """As needed, append or transform raw data to match expected structure. + # Returns row, or None if row is to be excluded""" + # if self.replication_key: + # if strptime_to_utc(row[self.replication_key]) <= self.get_starting_timestamp(context).astimezone(pytz.utc): + # return None + # return row def get_json_schema(self, from_type: str) -> dict: """Return the JSON Schema dict that describes the sql type. From 10770984d991de1d53a5a02b3312657b2cf2667c Mon Sep 17 00:00:00 2001 From: Niels <77604434+nielsmai@users.noreply.github.com> Date: Tue, 30 May 2023 15:43:12 -0400 Subject: [PATCH 2/3] testing fix for deals --- tap_hubspot/client.py | 14 +++++++------- tap_hubspot/streams.py | 5 +++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tap_hubspot/client.py b/tap_hubspot/client.py index 43a3103..6ad7892 100644 --- a/tap_hubspot/client.py +++ b/tap_hubspot/client.py @@ -85,13 +85,13 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]: """Parse the response and return an iterator of result rows.""" yield from extract_jsonpath(self.records_jsonpath, input=response.json()) - # def post_process(self, row: dict, context: Optional[dict]) -> dict: - # """As needed, append or transform raw data to match expected structure. - # Returns row, or None if row is to be excluded""" - # if self.replication_key: - # if strptime_to_utc(row[self.replication_key]) <= self.get_starting_timestamp(context).astimezone(pytz.utc): - # return None - # return row + def post_process(self, row: dict, context: Optional[dict]) -> dict: + """As needed, append or transform raw data to match expected structure. + Returns row, or None if row is to be excluded""" + if self.replication_key: + if strptime_to_utc(row[self.replication_key]) <= self.get_starting_timestamp(context).astimezone(pytz.utc): + return None + return row def get_json_schema(self, from_type: str) -> dict: """Return the JSON Schema dict that describes the sql type. diff --git a/tap_hubspot/streams.py b/tap_hubspot/streams.py index 96db20c..6325274 100644 --- a/tap_hubspot/streams.py +++ b/tap_hubspot/streams.py @@ -128,6 +128,11 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: "deal_id": record["id"], } + def post_process(self, row: dict, context: Optional[dict]) -> dict: + """As needed, append or transform raw data to match expected structure. + Returns row""" + return row + class ContactsStream(HubspotStream): """Define custom stream.""" From a58e694491a3df955169a7dfe1cd204abf3b319b Mon Sep 17 00:00:00 2001 From: Niels <77604434+nielsmai@users.noreply.github.com> Date: Tue, 30 May 2023 15:50:21 -0400 Subject: [PATCH 3/3] post_process fix Added the post_process fix to companies and contacts --- tap_hubspot/streams.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tap_hubspot/streams.py b/tap_hubspot/streams.py index 6325274..7b4d03e 100644 --- a/tap_hubspot/streams.py +++ b/tap_hubspot/streams.py @@ -98,6 +98,11 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: """Return a context dictionary for child streams.""" return {"archived": record["archived"], "company_id": record["id"]} + def post_process(self, row: dict, context: Optional[dict]) -> dict: + """As needed, append or transform raw data to match expected structure. + Returns row""" + return row + class DealsStream(HubspotStream): """Define custom stream.""" @@ -160,6 +165,11 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: """Return a context dictionary for child streams.""" return {"archived": record["archived"], "contact_id": record["id"]} + def post_process(self, row: dict, context: Optional[dict]) -> dict: + """As needed, append or transform raw data to match expected structure. + Returns row""" + return row + class PropertiesStream(HubspotStream): """Define custom stream."""