Skip to content

Commit

Permalink
Increase download timeout and improve error messages
Browse files Browse the repository at this point in the history
  • Loading branch information
Cito committed Dec 14, 2023
1 parent bfb36c6 commit a3bbe2a
Show file tree
Hide file tree
Showing 3 changed files with 16 additions and 6 deletions.
8 changes: 5 additions & 3 deletions src/ghga_connector/core/api_calls/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from ghga_connector.core import exceptions
from ghga_connector.core.api_calls.work_package import WorkPackageAccessor
from ghga_connector.core.client import httpx_client
from ghga_connector.core.constants import TIMEOUT
from ghga_connector.core.constants import TIMEOUT, TIMEOUT_LONG
from ghga_connector.core.http_translation import ResponseExceptionTranslator
from ghga_connector.core.message_display import AbstractMessageDisplay

Expand Down Expand Up @@ -66,7 +66,9 @@ def get_download_url(
# Make function call to get download url
try:
with httpx_client() as client:
response = client.get(url=url, headers=headers, timeout=TIMEOUT)
# us a longer timeout since this operation accesses the object storage
# where in some cases responses appear after a longer delay
response = client.get(url=url, headers=headers, timeout=TIMEOUT_LONG)
except httpx.RequestError as request_error:
exceptions.raise_if_connection_failed(request_error=request_error, url=url)
raise exceptions.RequestFailedError(url=url) from request_error
Expand Down Expand Up @@ -186,7 +188,7 @@ def get_file_header_envelope(
}
)

# Make function call to get download url
# Make function call to get envelope
try:
with httpx_client() as client:
response = client.get(url=url, headers=headers, timeout=TIMEOUT)
Expand Down
3 changes: 2 additions & 1 deletion src/ghga_connector/core/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,11 @@
# limitations under the License.
#

"""Constants used throught the core."""
"""Constants used throughout the core."""

DEFAULT_PART_SIZE = 16 * 1024 * 1024
TIMEOUT = 60
TIMEOUT_LONG = 5 * TIMEOUT + 10
MAX_PART_NUMBER = 10000
MAX_RETRIES = 5
MAX_WAIT_TIME = 60 * 60
11 changes: 9 additions & 2 deletions src/ghga_connector/core/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -265,8 +265,15 @@ def __init__(self):


def raise_if_connection_failed(request_error: httpx.RequestError, url: str):
"""Check if request exception is caused by hitting max retries and raise accordingly"""
if isinstance(request_error, (httpx.ConnectError, httpx.ConnectTimeout)):
"""Check if request exception is caused by a connection error and raise accordingly.
This is usually a timeout (response takes too long or number of retries exceeded).
"""
# Note that NetworkErrors and ProtocolErrors can also be caused by remote timeouts
if isinstance(
request_error,
(httpx.TimeoutException, httpx.NetworkError, httpx.ProtocolError),
):
connection_failure = str(request_error.args[0])
raise ConnectionFailedError(url=url, reason=connection_failure)

Expand Down

0 comments on commit a3bbe2a

Please sign in to comment.