Merge pull request #12 from voxmedia/CDT-441-error-handling

handle error in validate_response

tap_instagram/client.py

@@ -65,10 +65,21 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]:
         yield from extract_jsonpath(self.records_jsonpath, input=response.json())
 
     def validate_response(self, response: requests.Response) -> None:
-        if 400 <= response.status_code < 500:
+        if response.status_code == 400 and "Unsupported get request" in str(
+            response.json().get("error", {}).get("message")
+        ):
             msg = (
                 f"{response.status_code} Client Error: "
-                f"{response.reason} - {response.json()['error']['message']} for path: {self.path}"
+                f"{response.reason} - {response.json()['error']['message']}"
+                f" for path: {self.path}"
+            )
+            raise UnsupportedGetRequestError(msg)
+
+        elif 400 <= response.status_code < 500:
+            msg = (
+                f"{response.status_code} Client Error: "
+                f"{response.reason} - {response.json()['error']['message']}"
+                f" for path: {self.path}"
             )
             raise FatalAPIError(msg)
 
@@ -78,3 +89,32 @@ def validate_response(self, response: requests.Response) -> None:
                 f"{response.reason} for path: {self.path}"
             )
             raise RetriableAPIError(msg)
+
+    def get_records(self, context: Optional[dict]) -> Iterable[Dict[str, Any]]:
+        """Return a generator of row-type dictionary objects.
+
+        Each row emitted should be a dictionary of property names to their values.
+
+        Args:
+            context: Stream partition or context dictionary.
+
+        Yields:
+            One item per (possibly processed) record in the API.
+        """
+        try:
+            for record in self.request_records(context):
+                transformed_record = self.post_process(record, context)
+                if transformed_record is None:
+                    # Record filtered out during post_process()
+                    continue
+                yield transformed_record
+        except UnsupportedGetRequestError as e:
+            self.logger.warning(e)
+
+
+class UnsupportedGetRequestError(Exception):
+    """
+    Error object to facilitate skipping IDs that cause trouble
+    with the API but aren't themselves grounds for ending the
+    entire ingestion process.
+    """

tap_instagram/streams.py

@@ -504,12 +504,10 @@ def get_url_params(
         return params
 
     def validate_response(self, response: requests.Response) -> None:
-        if (
-            response.json().get("error", {}).get("error_user_title")
-            == "Media posted before business account conversion"
-            or
-            "(#10) Not enough viewers for the media to show insights"
-            in str(response.json().get("error", {}).get("message"))
+        if response.json().get("error", {}).get(
+            "error_user_title"
+        ) == "Media posted before business account conversion" or "(#10) Not enough viewers for the media to show insights" in str(
+            response.json().get("error", {}).get("message")
         ):
             self.logger.warning(f"Skipping: {response.json()['error']}")
             return
@@ -519,12 +517,10 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]:
         resp_json = response.json()
         # Handle the specific case where FB returns error because media was posted before business acct creation
         # TODO: Refactor to raise a specific error in validate_response and handle that instead
-        if (
-            resp_json.get("error", {}).get("error_user_title")
-            == "Media posted before business account conversion"
-            or
-            "(#10) Not enough viewers for the media to show insights"
-            in str(resp_json.get("error", {}).get("message"))
+        if resp_json.get("error", {}).get(
+            "error_user_title"
+        ) == "Media posted before business account conversion" or "(#10) Not enough viewers for the media to show insights" in str(
+            resp_json.get("error", {}).get("message")
         ):
             return
         for row in resp_json["data"]:
@@ -674,12 +670,10 @@ def get_url_params(
         return params
 
     def validate_response(self, response: requests.Response) -> None:
-        if (
-            response.json().get("error", {}).get("error_user_title")
-            == "Media posted before business account conversion"
-            or
-            "(#10) Not enough viewers for the media to show insights"
-            in str(response.json().get("error", {}).get("message"))
+        if response.json().get("error", {}).get(
+            "error_user_title"
+        ) == "Media posted before business account conversion" or "(#10) Not enough viewers for the media to show insights" in str(
+            response.json().get("error", {}).get("message")
         ):
             self.logger.warning(f"Skipping: {response.json()['error']}")
             return
@@ -689,12 +683,10 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]:
         resp_json = response.json()
         # Handle the specific case where FB returns error because media was posted before business acct creation
         # TODO: Refactor to raise a specific error in validate_response and handle that instead
-        if (
-            resp_json.get("error", {}).get("error_user_title")
-            == "Media posted before business account conversion"
-            or
-            "(#10) Not enough viewers for the media to show insights"
-            in str(resp_json.get("error", {}).get("message"))
+        if resp_json.get("error", {}).get(
+            "error_user_title"
+        ) == "Media posted before business account conversion" or "(#10) Not enough viewers for the media to show insights" in str(
+            resp_json.get("error", {}).get("message")
         ):
             return
         for row in resp_json["data"]: