diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 29ec06db65989..00533e362b993 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -2097,20 +2097,6 @@ - - fs.s3a.connection.request.timeout - 0s - - Time out on HTTP requests to the AWS service; 0 means no timeout. - - Important: this is the maximum duration of any AWS service call, - including upload and copy operations. If non-zero, it must be larger - than the time to upload multi-megabyte blocks to S3 from the client, - and to rename many-GB files. Use with care. - - - - fs.s3a.etag.checksum.enabled false diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md index 4c03cca17161f..44c8086d4f02a 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md @@ -231,7 +231,7 @@ The choice is yours. Generally recovery is better, but sometimes fail-fast is mo | `fs.s3a.connection.acquisition.timeout` | `60s` | `*` | Timeout for waiting for a connection from the pool. | | `fs.s3a.connection.establish.timeout` | `30s` | | Time to establish the TCP/TLS connection | | `fs.s3a.connection.idle.time` | `60s` | `*` | Maximum time for idle HTTP connections in the pool | -| `fs.s3a.connection.request.timeout` | `0` | | If greater than zero, maximum duration of any request | +| `fs.s3a.connection.request.timeout` | `60s` | | If greater than zero, maximum time for a response | | `fs.s3a.connection.timeout` | `200s` | | Timeout for socket problems on a TCP channel | | `fs.s3a.connection.ttl` | `5m` | | Lifetime of HTTP connections from the pool | diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md index c1b499e3da95d..e53e4a002265a 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md @@ -1510,7 +1510,7 @@ It is possible to configure a global timeout for AWS service calls using followi ```xml fs.s3a.connection.request.timeout - 0 + 5m Time out on HTTP requests to the AWS service; 0 means no timeout. Measured in seconds; the usual time suffixes are all supported diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestAwsClientConfig.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestAwsClientConfig.java index 2a9e05755ce6f..eacff90ea4c8a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestAwsClientConfig.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestAwsClientConfig.java @@ -40,6 +40,7 @@ import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_CONNECTION_TTL_DURATION; import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ESTABLISH_TIMEOUT_DURATION; import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAXIMUM_CONNECTIONS; +import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_REQUEST_TIMEOUT_DURATION; import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_SOCKET_TIMEOUT_DURATION; import static org.apache.hadoop.fs.s3a.Constants.ESTABLISH_TIMEOUT; import static org.apache.hadoop.fs.s3a.Constants.MAXIMUM_CONNECTIONS; @@ -175,6 +176,21 @@ public void testCreateApiConnectionSettings() { .describedAs("%s in %s", REQUEST_TIMEOUT, settings) .isEqualTo(Duration.ofHours(1)); } + /** + * Verify that the timeout from {@link org.apache.hadoop.fs.s3a.Constants#DEFAULT_REQUEST_TIMEOUT_DURATION} + * makes it all the way through and that nothing in in core-default or core-site is setting it. + * This test will fail if someone does set it in core-site.xml + */ + @Test + public void testCreateApiConnectionSettingsDefault() { + final Configuration conf = new Configuration(); + Assertions.assertThat(conf.get(REQUEST_TIMEOUT)) + .describedAs("Request timeout %s", REQUEST_TIMEOUT) + .isNull(); + + assertDuration(REQUEST_TIMEOUT, DEFAULT_REQUEST_TIMEOUT_DURATION, + createApiConnectionSettings(conf).getApiCallTimeout()); + } /** * Set a list of keys to the same value. diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml index 70b87ee275406..30e54ca0b87db 100644 --- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml +++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml @@ -201,11 +201,6 @@ true - - fs.s3a.connection.request.timeout - 10s - - fs.s3a.attempts.maximum 1