From e1c4cbc49c912b035d8893a821885d829b0f4d29 Mon Sep 17 00:00:00 2001 From: jlloyd-widen <82222659+jlloyd-widen@users.noreply.github.com> Date: Fri, 19 Jan 2024 16:08:18 -0600 Subject: [PATCH] better handling of 503 errors (#2) --- pyproject.toml | 2 +- tap_salesforce_connect/client.py | 28 +++++++++------------------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dc6c7de..60d0a48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tap-salesforce-connect" -version = "0.1.1" +version = "0.1.2" description = "`tap-salesforce-connect` is a Singer tap for SalesforceConnect, built with the Meltano Singer SDK." readme = "README.md" authors = ["Josh Lloyd"] diff --git a/tap_salesforce_connect/client.py b/tap_salesforce_connect/client.py index 9d700f8..25dd130 100644 --- a/tap_salesforce_connect/client.py +++ b/tap_salesforce_connect/client.py @@ -7,7 +7,6 @@ from pathlib import Path from typing import Any, Callable, Generator, Iterable -import backoff import requests from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.streams import RESTStream @@ -110,24 +109,15 @@ def post_process(self, row: dict, context: dict | None = None) -> dict | None: stringified_row[k] = v return stringified_row - # there is a one-hour rate limit on the SalesforceConnect API - def backoff_wait_generator(self) -> Generator[float, None, None]: - """Use wait generator for the backoff decorator on request failure. - - See for options: - https://github.com/litl/backoff/blob/master/backoff/_wait_gen.py + def get_wait_time_based_on_response(self, exception): + """Return the number of seconds to wait before retrying. - And see for examples: `Code Samples <../code_samples.html#custom-backoff>`_ - - Returns: - The wait generator + Salesforce Connect API has a rate limit scoped to an hour """ - return backoff.expo(factor=3) # type: ignore # ignore 'Returning Any' + if exception.response.status_code == 503: + return 60 * 60 + return exception.response.headers.get("Retry-After", 0) - def backoff_max_tries(self) -> int: - """Provide the number of attempts before giving up when retrying requests. - - Returns: - Number of max retries. - """ - return 20 + def backoff_wait_generator(self) -> Generator[float, None, None]: + """Return a generator of wait times between retries.""" + return self.backoff_runtime(value=self.get_wait_time_based_on_response)