Saner package exports

.devcontainer/devcontainer.json

@@ -52,7 +52,8 @@
 				"visualstudioexptteam.vscodeintellicode",
 				"ymotongpoo.licenser",
 				"charliermarsh.ruff",
-				"ms-python.mypy-type-checker"
+				"ms-python.mypy-type-checker",
+				"-ms-python.autopep8"
 			]
 		}
 	},
@@ -71,4 +72,4 @@
 		// details can be found here: https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker
 		"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}
 	}
-}
+}

src/ghga_connector/cli.py

@@ -28,31 +28,42 @@
 import typer
 from ghga_service_commons.utils import crypt
 
-from ghga_connector import core
-from ghga_connector.config import Config
-from ghga_connector.core.client import HttpxClientConfigurator, async_client
+from ghga_connector.config import CONFIG
+from ghga_connector.core import (
+    AbstractMessageDisplay,
+    HttpxClientConfigurator,
+    MessageColors,
+    WorkPackageAccessor,
+    async_client,
+    exceptions,
+)
+from ghga_connector.core.api_calls import WKVSCaller
 from ghga_connector.core.downloading.batch_processing import FileStager
+from ghga_connector.core.main import (
+    decrypt_file,
+    download_files,
+    get_wps_token,
+    upload_file,
+)
 
-CONFIG = Config()
 
-
-class CLIMessageDisplay(core.AbstractMessageDisplay):
+class CLIMessageDisplay(AbstractMessageDisplay):
     """
     Command line writer message display implementation,
     using different color based on information type
     """
 
     def display(self, message: str):
         """Write message with default color to stdout"""
-        typer.secho(message, fg=core.MessageColors.DEFAULT)
+        typer.secho(message, fg=MessageColors.DEFAULT)
 
     def success(self, message: str):
         """Write message to stdout representing information about a successful operation"""
-        typer.secho(message, fg=core.MessageColors.SUCCESS)
+        typer.secho(message, fg=MessageColors.SUCCESS)
 
     def failure(self, message: str):
         """Write message to stderr representing information about a failed operation"""
-        typer.secho(message, fg=core.MessageColors.FAILURE, err=True)
+        typer.secho(message, fg=MessageColors.FAILURE, err=True)
 
 
 @dataclass
@@ -61,7 +72,7 @@ class DownloadParameters:
 
     dcs_api_url: str
     file_ids_with_extension: dict[str, str]
-    work_package_accessor: core.WorkPackageAccessor
+    work_package_accessor: WorkPackageAccessor
 
 
 @dataclass
@@ -116,7 +127,7 @@ def init_message_display(debug: bool = False) -> CLIMessageDisplay:
 
 async def retrieve_upload_parameters(client: httpx.AsyncClient) -> UploadParameters:
     """Configure httpx client and retrieve necessary parameters from WKVS"""
-    wkvs_caller = core.WKVSCaller(client=client, wkvs_url=CONFIG.wkvs_api_url)
+    wkvs_caller = WKVSCaller(client=client, wkvs_url=CONFIG.wkvs_api_url)
     ucs_api_url = await wkvs_caller.get_ucs_api_url()
     server_pubkey = await wkvs_caller.get_server_pubkey()
 
@@ -131,11 +142,11 @@ async def retrieve_download_parameters(
     work_package_information: WorkPackageInformation,
 ) -> DownloadParameters:
     """Run necessary API calls to configure file download"""
-    wkvs_caller = core.WKVSCaller(client=client, wkvs_url=CONFIG.wkvs_api_url)
+    wkvs_caller = WKVSCaller(client=client, wkvs_url=CONFIG.wkvs_api_url)
     dcs_api_url = await wkvs_caller.get_dcs_api_url()
     wps_api_url = await wkvs_caller.get_wps_api_url()
 
-    work_package_accessor = core.WorkPackageAccessor(
+    work_package_accessor = WorkPackageAccessor(
         access_token=work_package_information.decrypted_token,
         api_url=wps_api_url,
         client=client,
@@ -154,11 +165,11 @@ async def retrieve_download_parameters(
 
 
 def get_work_package_information(
-    my_private_key: bytes, message_display: core.AbstractMessageDisplay
+    my_private_key: bytes, message_display: AbstractMessageDisplay
 ):
     """Fetch a work package id and work package token and decrypt the token"""
     # get work package access token and id from user input
-    work_package_id, work_package_token = core.get_wps_token(
+    work_package_id, work_package_token = get_wps_token(
         max_tries=3, message_display=message_display
     )
     decrypted_token = crypt.decrypt(data=work_package_token, key=my_private_key)
@@ -197,7 +208,7 @@ async def upload(
     )
     async with async_client() as client:
         parameters = await retrieve_upload_parameters(client)
-        await core.upload(
+        await upload_file(
             api_url=parameters.ucs_api_url,
             client=client,
             file_id=file_id,
@@ -238,12 +249,10 @@ async def download(
 ):
     """Command to download files"""
     if not my_public_key_path.is_file():
-        raise core.exceptions.PubKeyFileDoesNotExistError(
-            public_key_path=my_public_key_path
-        )
+        raise exceptions.PubKeyFileDoesNotExistError(public_key_path=my_public_key_path)
 
     if not output_dir.is_dir():
-        raise core.exceptions.DirectoryDoesNotExistError(directory=output_dir)
+        raise exceptions.DirectoryDoesNotExistError(directory=output_dir)
 
     my_public_key = crypt4gh.keys.get_public_key(filepath=my_public_key_path)
     my_private_key = crypt4gh.keys.get_private_key(
@@ -283,7 +292,7 @@ async def download(
             staged_files = await stager.get_staged_files()
             for file_id in staged_files:
                 message_display.display(f"Downloading file with id '{file_id}'...")
-                await core.download(
+                await download_files(
                     api_url=parameters.dcs_api_url,
                     client=client,
                     file_id=file_id,
@@ -325,13 +334,13 @@ def decrypt(  # noqa: PLR0912, C901
     message_display = init_message_display(debug=debug)
 
     if not input_dir.is_dir():
-        raise core.exceptions.DirectoryDoesNotExistError(directory=input_dir)
+        raise exceptions.DirectoryDoesNotExistError(directory=input_dir)
 
     if not output_dir:
         output_dir = input_dir
 
     if output_dir.exists() and not output_dir.is_dir():
-        raise core.exceptions.OutputPathIsNotDirectory(directory=output_dir)
+        raise exceptions.OutputPathIsNotDirectory(directory=output_dir)
 
     if not output_dir.exists():
         message_display.display(f"Creating output directory '{output_dir}'")
@@ -358,7 +367,7 @@ def decrypt(  # noqa: PLR0912, C901
 
         try:
             message_display.display(f"Decrypting file with id '{input_file}'...")
-            core.decrypt_file(
+            decrypt_file(
                 input_file=input_file,
                 output_file=output_file,
                 decryption_private_key_path=my_private_key_path,

src/ghga_connector/config.py

@@ -20,7 +20,7 @@
 from pydantic import Field, NonNegativeInt, PositiveInt
 from pydantic_settings import BaseSettings
 
-from ghga_connector.core.constants import DEFAULT_PART_SIZE, MAX_RETRIES, MAX_WAIT_TIME
+from ghga_connector.constants import DEFAULT_PART_SIZE, MAX_RETRIES, MAX_WAIT_TIME
 
 
 @config_from_yaml(prefix="ghga_connector")
@@ -52,3 +52,6 @@ class Config(BaseSettings):
         default=[408, 500, 502, 503, 504],
         description="List of status codes that should trigger retrying a request.",
     )
+
+
+CONFIG = Config()

src/ghga_connector/core/__init__.py

@@ -18,7 +18,14 @@
 It should not contain any service API-related code.
 """
 
-from . import exceptions  # noqa: F401
-from .api_calls import WKVSCaller, WorkPackageAccessor  # noqa: F401
-from .main import decrypt_file, download, get_wps_token, upload  # noqa: F401
+from .client import HttpxClientConfigurator, async_client, httpx_client  # noqa: F401
+from .file_operations import (  # noqa: F401
+    calc_part_ranges,
+    get_segments,
+    is_file_encrypted,
+    read_file_parts,
+)
+from .http_translation import ResponseExceptionTranslator  # noqa: F401
 from .message_display import AbstractMessageDisplay, MessageColors  # noqa: F401
+from .structs import PartRange  # noqa: F401
+from .work_package import WorkPackageAccessor  # noqa: F401

src/ghga_connector/core/api_calls/__init__.py

@@ -17,6 +17,5 @@
 This sub-package contains the api calls, this service makes for various purposes
 """
 
-from .utils import is_service_healthy  # noqa: F401
+from .utils import check_url, is_service_healthy  # noqa: F401
 from .well_knowns import WKVSCaller  # noqa: F401
-from .work_package import WorkPackageAccessor  # noqa: F401

src/ghga_connector/core/client.py

@@ -25,7 +25,7 @@
     wait_exponential_jitter,
 )
 
-from ghga_connector.core.constants import TIMEOUT
+from ghga_connector.constants import TIMEOUT
 
 
 class HttpxClientConfigurator:
@@ -41,9 +41,6 @@ def configure(
         retry_status_codes: list[int],
     ):
         """Configure client retry handler with exponential backoff"""
-        # clamp to client timeout, else large values might be bothersome on retries
-        if exponential_backoff_max > TIMEOUT:
-            exponential_backoff_max = int(TIMEOUT)
         cls.retry_handler = AsyncRetrying(
             reraise=True,
             retry=(

src/ghga_connector/core/crypt/__init__.py

@@ -16,5 +16,6 @@
 """Subpackage containing all encryption/decryption related functionality"""
 
 from .abstract_bases import Decryptor, Encryptor  # noqa: F401
+from .checksums import Checksums  # noqa: F401
 from .decryption import Crypt4GHDecryptor  # noqa: F401
 from .encryption import Crypt4GHEncryptor  # noqa: F401

src/ghga_connector/core/crypt/decryption.py

@@ -20,7 +20,7 @@
 import crypt4gh.keys
 import crypt4gh.lib
 
-from ghga_connector.core.crypt.abstract_bases import Decryptor
+from .abstract_bases import Decryptor
 
 
 class Crypt4GHDecryptor(Decryptor):

src/ghga_connector/core/crypt/encryption.py

@@ -27,9 +27,10 @@
 import crypt4gh.lib
 from nacl.bindings import crypto_aead_chacha20poly1305_ietf_encrypt
 
-from ghga_connector.core.crypt.abstract_bases import Encryptor
-from ghga_connector.core.crypt.checksums import Checksums
-from ghga_connector.core.file_operations import get_segments, read_file_parts
+from ghga_connector.core import get_segments, read_file_parts
+
+from .abstract_bases import Encryptor
+from .checksums import Checksums
 
 
 class Crypt4GHEncryptor(Encryptor):

src/ghga_connector/core/downloading/__init__.py

@@ -15,5 +15,5 @@
 #
 """This subpackage contains functionality needed to download files from GHGA"""
 
-from .abstract_downloader import DownloaderBase  # noqa: F401
-from .downloader import Downloader  # noqa: F401
+from .progress_bar import ProgressBar  # noqa: F401
+from .structs import RetryResponse, UrlAndHeaders, URLResponse  # noqa: F401

src/ghga_connector/core/downloading/abstract_downloader.py

@@ -21,8 +21,9 @@
 from pathlib import Path
 from typing import Any
 
-from ghga_connector.core.downloading.structs import URLResponse
-from ghga_connector.core.structs import PartRange
+from ghga_connector.core import PartRange
+
+from .structs import URLResponse
 
 
 class DownloaderBase(ABC):

src/ghga_connector/core/downloading/api_calls.py

@@ -19,10 +19,10 @@
 
 import httpx
 
-from ghga_connector.core import exceptions
-from ghga_connector.core.api_calls.work_package import WorkPackageAccessor
-from ghga_connector.core.constants import TIMEOUT_LONG
-from ghga_connector.core.downloading.structs import (
+from ghga_connector.constants import TIMEOUT_LONG
+from ghga_connector.core import WorkPackageAccessor, exceptions
+
+from .structs import (
     RetryResponse,
     UrlAndHeaders,
     URLResponse,

src/ghga_connector/core/downloading/batch_processing.py

@@ -22,15 +22,18 @@
 import httpx
 
 from ghga_connector.config import Config
-from ghga_connector.core import exceptions
-from ghga_connector.core.api_calls import WorkPackageAccessor, is_service_healthy
-from ghga_connector.core.downloading.api_calls import (
-    RetryResponse,
-    URLResponse,
+from ghga_connector.core import (
+    AbstractMessageDisplay,
+    WorkPackageAccessor,
+    exceptions,
+)
+from ghga_connector.core.api_calls import is_service_healthy
+
+from .api_calls import (
     get_download_url,
     get_file_authorization,
 )
-from ghga_connector.core.message_display import AbstractMessageDisplay
+from .structs import RetryResponse, URLResponse
 
 
 class InputHandler(ABC):

src/ghga_connector/core/downloading/downloader.py

@@ -26,24 +26,24 @@
 import httpx
 from tenacity import RetryError
 
-from ghga_connector.core import exceptions
-from ghga_connector.core.api_calls import WorkPackageAccessor
-from ghga_connector.core.client import HttpxClientConfigurator
-from ghga_connector.core.downloading.abstract_downloader import DownloaderBase
-from ghga_connector.core.downloading.api_calls import (
+from ghga_connector.core import (
+    AbstractMessageDisplay,
+    HttpxClientConfigurator,
+    PartRange,
+    ResponseExceptionTranslator,
+    WorkPackageAccessor,
+    calc_part_ranges,
+    exceptions,
+)
+
+from .abstract_downloader import DownloaderBase
+from .api_calls import (
     get_download_url,
     get_envelope_authorization,
     get_file_authorization,
 )
-from ghga_connector.core.downloading.progress_bar import ProgressBar
-from ghga_connector.core.downloading.structs import (
-    RetryResponse,
-    URLResponse,
-)
-from ghga_connector.core.file_operations import calc_part_ranges
-from ghga_connector.core.http_translation import ResponseExceptionTranslator
-from ghga_connector.core.message_display import AbstractMessageDisplay
-from ghga_connector.core.structs import PartRange
+from .progress_bar import ProgressBar
+from .structs import RetryResponse, URLResponse
 
 
 class TaskHandler:

src/ghga_connector/core/exceptions.py

@@ -20,7 +20,7 @@
 
 import httpx
 
-from ghga_connector.core.constants import MAX_PART_NUMBER
+from ghga_connector.constants import MAX_PART_NUMBER
 
 
 class AbortBatchProcessError(RuntimeError):

src/ghga_connector/core/file_operations.py

@@ -22,7 +22,7 @@
 from pathlib import Path
 from typing import Any
 
-from ghga_connector.core.structs import PartRange
+from .structs import PartRange
 
 
 def is_file_encrypted(file_path: Path):