diff --git a/app/configs/app_config.py b/app/configs/app_config.py index bc6b307..7c902ef 100644 --- a/app/configs/app_config.py +++ b/app/configs/app_config.py @@ -13,7 +13,7 @@ class Env: # NOTE: there is a limitation on minio that # the multipart number is 10000. so we set # the chunk_size as 20MB -> total 200GB - chunk_size = 1024 * 1024 * 20 # MB + chunk_size = ConfigClass.upload_chunk_size # MB resilient_retry = 3 resilient_backoff = 1 resilient_retry_interval = 1 # seconds @@ -31,7 +31,7 @@ class Env: # set hard limit for pending jobs, otherwise cli will consume all memory # to cache jobs. If later on the speed of chunk deliver become faster, we # can increase the concurrency number. - num_of_jobs = 20 + num_of_jobs = ConfigClass.concurrent_job_limit github_url = 'PilotDataPlatform/cli' diff --git a/app/configs/config.py b/app/configs/config.py index 6f375e5..977bc27 100644 --- a/app/configs/config.py +++ b/app/configs/config.py @@ -37,6 +37,8 @@ class Settings(BaseSettings): apikey_endpoint: str = 'api-key' upload_batch_size: int = 100 + concurrent_job_limit: int = 10 + upload_chunk_size: int = 1024 * 1024 * 20 # 20MB def __init__(self, **data): super().__init__(**data) diff --git a/pyproject.toml b/pyproject.toml index c123ee4..448ad45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "3.12.0" +version = "3.12.1" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"]