From 0d38daee6d8dc8837fe1e5db4de659e39531a644 Mon Sep 17 00:00:00 2001 From: Prratek Ramchandani Date: Mon, 28 Feb 2022 17:28:01 -0500 Subject: [PATCH] fix tests and linting errors --- tap_instagram/streams.py | 43 ++++++++++++++++++++++++++++++---------- tap_instagram/tap.py | 2 +- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/tap_instagram/streams.py b/tap_instagram/streams.py index 357d602..1aec4a1 100644 --- a/tap_instagram/streams.py +++ b/tap_instagram/streams.py @@ -188,7 +188,12 @@ class MediaStream(InstagramStream): def make_since_param(self, context: Optional[dict]) -> datetime: state_ts = self.get_starting_timestamp(context) - return pendulum.instance(state_ts).subtract(days=self.config["media_insights_lookback_days"]) + if state_ts: + return pendulum.instance(state_ts).subtract( + days=self.config["media_insights_lookback_days"] + ) + else: + return state_ts def get_url_params( self, context: Optional[dict], next_page_token: Optional[Any] @@ -210,7 +215,9 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: def parse_response(self, response: requests.Response) -> Iterable[dict]: for row in extract_jsonpath(self.records_jsonpath, input=response.json()): if "timestamp" in row: - row["timestamp"] = pendulum.parse(row["timestamp"]).format("YYYY-MM-DD HH:mm:ss") + row["timestamp"] = pendulum.parse(row["timestamp"]).format( + "YYYY-MM-DD HH:mm:ss" + ) yield row @@ -361,7 +368,9 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: def parse_response(self, response: requests.Response) -> Iterable[dict]: for row in extract_jsonpath(self.records_jsonpath, input=response.json()): if "timestamp" in row: - row["timestamp"] = pendulum.parse(row["timestamp"]).format("YYYY-MM-DD HH:mm:ss") + row["timestamp"] = pendulum.parse(row["timestamp"]).format( + "YYYY-MM-DD HH:mm:ss" + ) yield row @@ -392,7 +401,9 @@ class MediaChildrenStream(MediaStream): def parse_response(self, response: requests.Response) -> Iterable[dict]: for row in extract_jsonpath(self.records_jsonpath, input=response.json()): if "timestamp" in row: - row["timestamp"] = pendulum.parse(row["timestamp"]).format("YYYY-MM-DD HH:mm:ss") + row["timestamp"] = pendulum.parse(row["timestamp"]).format( + "YYYY-MM-DD HH:mm:ss" + ) yield row @@ -528,14 +539,18 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]: item = { "context": key, "value": value, - "end_time": pendulum.parse(values["end_time"]).format("YYYY-MM-DD HH:mm:ss"), + "end_time": pendulum.parse(values["end_time"]).format( + "YYYY-MM-DD HH:mm:ss" + ), } item.update(base_item) yield item else: values.update(base_item) if "end_time" in values: - values["end_time"] = pendulum.parse(values["end_time"]).format("YYYY-MM-DD HH:mm:ss") + values["end_time"] = pendulum.parse( + values["end_time"] + ).format("YYYY-MM-DD HH:mm:ss") yield values @@ -687,14 +702,18 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]: item = { "context": key, "value": value, - "end_time": pendulum.parse(values["end_time"]).format("YYYY-MM-DD HH:mm:ss"), + "end_time": pendulum.parse(values["end_time"]).format( + "YYYY-MM-DD HH:mm:ss" + ), } item.update(base_item) yield item else: values.update(base_item) if "end_time" in values: - values["end_time"] = pendulum.parse(values["end_time"]).format("YYYY-MM-DD HH:mm:ss") + values["end_time"] = pendulum.parse( + values["end_time"] + ).format("YYYY-MM-DD HH:mm:ss") yield values @@ -827,14 +846,18 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]: item = { "context": key, "value": value, - "end_time": pendulum.parse(values["end_time"]).format("YYYY-MM-DD HH:mm:ss"), + "end_time": pendulum.parse(values["end_time"]).format( + "YYYY-MM-DD HH:mm:ss" + ), } item.update(base_item) yield item else: values.update(base_item) if "end_time" in values: - values["end_time"] = pendulum.parse(values["end_time"]).format("YYYY-MM-DD HH:mm:ss") + values["end_time"] = pendulum.parse( + values["end_time"] + ).format("YYYY-MM-DD HH:mm:ss") yield values diff --git a/tap_instagram/tap.py b/tap_instagram/tap.py index aadabba..d524ae2 100644 --- a/tap_instagram/tap.py +++ b/tap_instagram/tap.py @@ -64,7 +64,7 @@ class TapInstagram(Tap): th.IntegerType, default=60, description="The tap fetches media insights for Media objects posted in the last `insights_lookback_days` " - "days - defaults to 14 days if not provided" + "days - defaults to 14 days if not provided", ), th.Property( "start_date",