Merge pull request ClickHouse#63857 from ClickHouse/chesema-less-back…

…off-timeout S3: reduce retires time for queries, increase retries count for backups
Alex-Cheng · Jun 11, 2024 · f52ca21 · f52ca21
2 parents 7f50316 + 2df9ab6
commit f52ca21
Show file tree

Hide file tree

Showing 5 changed files with 7 additions and 4 deletions.
diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp
@@ -54,9 +54,9 @@ namespace
         S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration(
             settings.auth_settings.region,
             context->getRemoteHostFilter(),
-            static_cast<unsigned>(global_settings.s3_max_redirects),
-            static_cast<unsigned>(global_settings.s3_retry_attempts),
-            global_settings.enable_s3_requests_logging,
+            static_cast<unsigned>(local_settings.s3_max_redirects),
+            static_cast<unsigned>(local_settings.backup_restore_s3_retry_attempts),
+            local_settings.enable_s3_requests_logging,
             /* for_disk_s3 = */ false,
             request_settings.get_request_throttler,
             request_settings.put_request_throttler,

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
@@ -517,6 +517,7 @@ class IColumn;
     M(UInt64, backup_restore_keeper_value_max_size, 1048576, "Maximum size of data of a [Zoo]Keeper's node during backup", 0) \
     M(UInt64, backup_restore_batch_size_for_keeper_multiread, 10000, "Maximum size of batch for multiread request to [Zoo]Keeper during backup or restore", 0) \
     M(UInt64, backup_restore_batch_size_for_keeper_multi, 1000, "Maximum size of batch for multi request to [Zoo]Keeper during backup or restore", 0) \
+    M(UInt64, backup_restore_s3_retry_attempts, 1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore.", 0) \
     M(UInt64, max_backup_bandwidth, 0, "The maximum read speed in bytes per second for particular backup on server. Zero means unlimited.", 0) \
     \
     M(Bool, log_profile_events, true, "Log query performance statistics into the query_log, query_thread_log and query_views_log.", 0) \

diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h
@@ -113,6 +113,7 @@ static const std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges
               {"http_max_chunk_size", 0, 0, "Internal limitation"},
               {"prefer_external_sort_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging."},
               {"input_format_force_null_for_omitted_fields", false, false, "Disable type-defaults for omitted fields when needed"},
+              {"backup_restore_s3_retry_attempts", 0, 1000, "A new setting."},
               {"cast_string_to_dynamic_use_inference", false, false, "Add setting to allow converting String to Dynamic through parsing"},
               {"allow_experimental_dynamic_type", false, false, "Add new experimental Dynamic type"},
               {"azure_max_blocks_in_multipart_upload", 50000, 50000, "Maximum number of blocks in multipart upload for Azure."},

diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h
@@ -162,7 +162,7 @@ class Client : private Aws::S3::S3Client
     class RetryStrategy : public Aws::Client::RetryStrategy
     {
     public:
-        explicit RetryStrategy(uint32_t maxRetries_ = 10, uint32_t scaleFactor_ = 25, uint32_t maxDelayMs_ = 90000);
+        explicit RetryStrategy(uint32_t maxRetries_ = 10, uint32_t scaleFactor_ = 25, uint32_t maxDelayMs_ = 5000);
 
         /// NOLINTNEXTLINE(google-runtime-int)
         bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override;

diff --git a/tests/integration/test_mask_sensitive_info/configs/users.xml b/tests/integration/test_mask_sensitive_info/configs/users.xml
@@ -2,6 +2,7 @@
     <profiles>
         <default>
             <s3_retry_attempts>5</s3_retry_attempts>
+            <backup_restore_s3_retry_attempts>5</backup_restore_s3_retry_attempts>
         </default>
     </profiles>
     <users>