From 97560e41283381c4fc08d779fc01a3ffe1ab4d85 Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Date: Sun, 16 Jun 2024 15:14:13 +0200 Subject: [PATCH 1/4] :wrench: add on_predicate decorator to handle 429 --- tap_gitlab/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_gitlab/__init__.py b/tap_gitlab/__init__.py index eede72c..23cbb66 100644 --- a/tap_gitlab/__init__.py +++ b/tap_gitlab/__init__.py @@ -248,11 +248,11 @@ def get_start(entity): STATE[entity] = CONFIG['start_date'] return STATE[entity] - +@backoff.on_predicate(backoff.runtime, predicate=lambda r: r.status_code == 429, vale=lambda r: int(r.headers.get("Retry-After")), jitter=None) @backoff.on_exception(backoff.expo, (requests.exceptions.RequestException), max_tries=5, - giveup=lambda e: e.response is not None and 400 <= e.response.status_code < 500, # pylint: disable=line-too-long + giveup=lambda e: e.response is not None and e.response.status_code != 429 and 400 <= e.response.status_code < 500, # pylint: disable=line-too-long factor=2) def request(url, params=None): params = params or {} From 0e70e1a296b14614efba04227333cb842e30e27c Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Date: Sun, 16 Jun 2024 15:22:52 +0200 Subject: [PATCH 2/4] :wrench: add max_tries 5 on_predicate to match other backoff methods --- tap_gitlab/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tap_gitlab/__init__.py b/tap_gitlab/__init__.py index 23cbb66..26f32ef 100644 --- a/tap_gitlab/__init__.py +++ b/tap_gitlab/__init__.py @@ -248,7 +248,11 @@ def get_start(entity): STATE[entity] = CONFIG['start_date'] return STATE[entity] -@backoff.on_predicate(backoff.runtime, predicate=lambda r: r.status_code == 429, vale=lambda r: int(r.headers.get("Retry-After")), jitter=None) +@backoff.on_predicate(backoff.runtime, + predicate=lambda r: r.status_code == 429, + max_tries = 5, + vale=lambda r: int(r.headers.get("Retry-After")), + jitter=None) @backoff.on_exception(backoff.expo, (requests.exceptions.RequestException), max_tries=5, From 7789c9ff4d825e0c0916ae3e50f5fc46c084a2e7 Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Date: Wed, 19 Jun 2024 16:27:45 +0200 Subject: [PATCH 3/4] :wrench: fix typo on decorator --- tap_gitlab/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_gitlab/__init__.py b/tap_gitlab/__init__.py index 26f32ef..02b9611 100644 --- a/tap_gitlab/__init__.py +++ b/tap_gitlab/__init__.py @@ -251,7 +251,7 @@ def get_start(entity): @backoff.on_predicate(backoff.runtime, predicate=lambda r: r.status_code == 429, max_tries = 5, - vale=lambda r: int(r.headers.get("Retry-After")), + value=lambda r: int(r.headers.get("Retry-After")), jitter=None) @backoff.on_exception(backoff.expo, (requests.exceptions.RequestException), From 0a2b05c1b09843ad3454276900cb7749cee24c94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Wed, 19 Jun 2024 09:08:17 -0600 Subject: [PATCH 4/4] Update tap_gitlab/__init__.py --- tap_gitlab/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_gitlab/__init__.py b/tap_gitlab/__init__.py index 02b9611..1e98095 100644 --- a/tap_gitlab/__init__.py +++ b/tap_gitlab/__init__.py @@ -250,7 +250,7 @@ def get_start(entity): @backoff.on_predicate(backoff.runtime, predicate=lambda r: r.status_code == 429, - max_tries = 5, + max_tries=5, value=lambda r: int(r.headers.get("Retry-After")), jitter=None) @backoff.on_exception(backoff.expo,