diff --git a/awscli/botocore/args.py b/awscli/botocore/args.py
index 7fb4721781a1..24ecb2db9236 100644
--- a/awscli/botocore/args.py
+++ b/awscli/botocore/args.py
@@ -36,6 +36,14 @@
# values result in a warning-level log message.
USERAGENT_APPID_MAXLEN = 50
+VALID_REQUEST_CHECKSUM_CALCULATION_CONFIG = (
+ "when_supported",
+ "when_required",
+)
+VALID_RESPONSE_CHECKSUM_VALIDATION_CONFIG = (
+ "when_supported",
+ "when_required",
+)
class ClientArgsCreator(object):
def __init__(self, event_emitter, user_agent, response_parser_factory,
@@ -216,11 +224,18 @@ def compute_client_args(self, service_model, client_config,
sigv4a_signing_region_set=(
client_config.sigv4a_signing_region_set
),
+ request_checksum_calculation=(
+ client_config.request_checksum_calculation
+ ),
+ response_checksum_validation=(
+ client_config.response_checksum_validation
+ ),
)
self._compute_retry_config(config_kwargs)
self._compute_request_compression_config(config_kwargs)
self._compute_user_agent_appid_config(config_kwargs)
self._compute_sigv4a_signing_region_set_config(config_kwargs)
+ self._compute_checksum_config(config_kwargs)
s3_config = self.compute_s3_config(client_config)
is_s3_service = self._is_s3_service(service_name)
@@ -588,4 +603,40 @@ def _compute_sigv4a_signing_region_set_config(self, config_kwargs):
sigv4a_signing_region_set = self._config_store.get_config_variable(
'sigv4a_signing_region_set'
)
- config_kwargs['sigv4a_signing_region_set'] = sigv4a_signing_region_set
\ No newline at end of file
+ config_kwargs['sigv4a_signing_region_set'] = sigv4a_signing_region_set
+
+ def _compute_checksum_config(self, config_kwargs):
+ self._handle_checksum_config(
+ config_kwargs,
+ config_key="request_checksum_calculation",
+ default_value="when_supported",
+ valid_options=VALID_REQUEST_CHECKSUM_CALCULATION_CONFIG,
+ )
+ self._handle_checksum_config(
+ config_kwargs,
+ config_key="response_checksum_validation",
+ default_value="when_supported",
+ valid_options=VALID_RESPONSE_CHECKSUM_VALIDATION_CONFIG,
+ )
+
+ def _handle_checksum_config(
+ self,
+ config_kwargs,
+ config_key,
+ default_value,
+ valid_options,
+ ):
+ value = config_kwargs.get(config_key)
+ if value is None:
+ value = (
+ self._config_store.get_config_variable(config_key)
+ or default_value
+ )
+ value = value.lower()
+ if value not in valid_options:
+ raise botocore.exceptions.InvalidChecksumConfigError(
+ config_key=config_key,
+ config_value=value,
+ valid_options=valid_options,
+ )
+ config_kwargs[config_key] = value
\ No newline at end of file
diff --git a/awscli/botocore/config.py b/awscli/botocore/config.py
index 8fe5662b1fd9..ece4cb820e92 100644
--- a/awscli/botocore/config.py
+++ b/awscli/botocore/config.py
@@ -194,6 +194,37 @@ class Config(object):
:param sigv4a_signing_region_set: A set of AWS regions to apply the signature for
when using SigV4a for signing. Set to ``*`` to represent all regions.
Defaults to None.
+
+ :type request_checksum_calculation: str
+ :param request_checksum_calculation: Determines when a checksum will be
+ calculated for request payloads. Valid values are:
+
+ * ``when_supported`` -- When set, a checksum will be calculated for
+ all request payloads of operations modeled with the ``httpChecksum``
+ trait where ``requestChecksumRequired`` is ``true`` and/or a
+ ``requestAlgorithmMember`` is modeled.
+
+ * ``when_required`` -- When set, a checksum will only be calculated
+ for request payloads of operations modeled with the ``httpChecksum``
+ trait where ``requestChecksumRequired`` is ``true`` or where a
+ ``requestAlgorithmMember`` is modeled and supplied.
+
+ Defaults to None.
+
+ :type response_checksum_validation: str
+ :param response_checksum_validation: Determines when checksum validation
+ will be performed on response payloads. Valid values are:
+
+ * ``when_supported`` -- When set, checksum validation is performed on
+ all response payloads of operations modeled with the ``httpChecksum``
+ trait where ``responseAlgorithms`` is modeled, except when no modeled
+ checksum algorithms are supported.
+
+ * ``when_required`` -- When set, checksum validation is not performed
+ on response payloads of operations unless the checksum algorithm is
+ supported and the ``requestValidationModeMember`` member is set to ``ENABLED``.
+
+ Defaults to None.
"""
OPTION_DEFAULTS = OrderedDict([
('region_name', None),
@@ -218,6 +249,8 @@ class Config(object):
('request_min_compression_size_bytes', None),
('disable_request_compression', None),
('sigv4a_signing_region_set', None),
+ ('request_checksum_calculation', None),
+ ('response_checksum_validation', None),
])
def __init__(self, *args, **kwargs):
diff --git a/awscli/botocore/configprovider.py b/awscli/botocore/configprovider.py
index 63c75c27f796..86fea6bd1cdc 100644
--- a/awscli/botocore/configprovider.py
+++ b/awscli/botocore/configprovider.py
@@ -137,6 +137,18 @@
None,
None,
),
+ 'request_checksum_calculation': (
+ 'request_checksum_calculation',
+ 'AWS_REQUEST_CHECKSUM_CALCULATION',
+ "when_supported",
+ None,
+ ),
+ 'response_checksum_validation': (
+ 'response_checksum_validation',
+ 'AWS_RESPONSE_CHECKSUM_VALIDATION',
+ "when_supported",
+ None,
+ ),
}
# A mapping for the s3 specific configuration vars. These are the configuration
# vars that typically go in the s3 section of the config file. This mapping
@@ -336,7 +348,7 @@ def __copy__(self):
def get_config_variable(self, logical_name):
"""
- Retrieve the value associeated with the specified logical_name
+ Retrieve the value associated with the specified logical_name
from the corresponding provider. If no value is found None will
be returned.
diff --git a/awscli/botocore/exceptions.py b/awscli/botocore/exceptions.py
index 9adbf32070ef..b5c7ba3807a3 100644
--- a/awscli/botocore/exceptions.py
+++ b/awscli/botocore/exceptions.py
@@ -742,3 +742,12 @@ class EndpointResolutionError(EndpointProviderError):
class UnknownEndpointResolutionBuiltInName(EndpointProviderError):
fmt = 'Unknown builtin variable name: {name}'
+
+
+class InvalidChecksumConfigError(BotoCoreError):
+ """Error when invalid value supplied for checksum config"""
+
+ fmt = (
+ 'Unsupported configuration value for {config_key}. '
+ 'Expected one of {valid_options} but got {config_value}.'
+ )
diff --git a/awscli/botocore/handlers.py b/awscli/botocore/handlers.py
index 8ba860957312..199366a01df1 100644
--- a/awscli/botocore/handlers.py
+++ b/awscli/botocore/handlers.py
@@ -60,8 +60,6 @@
from botocore.utils import (
SAFE_CHARS,
ArnParser,
- conditionally_calculate_checksum,
- conditionally_calculate_md5,
hyphenize_service_id,
is_global_accesspoint,
percent_encode,
@@ -1151,6 +1149,17 @@ def _update_status_code(response, **kwargs):
http_response.status_code = parsed_status_code
+def handle_request_validation_mode_member(params, model, **kwargs):
+ client_config = kwargs.get("context", {}).get("client_config")
+ if client_config is None:
+ return
+ response_checksum_validation = client_config.response_checksum_validation
+ http_checksum = model.http_checksum
+ mode_member = http_checksum.get("requestValidationModeMember")
+ if mode_member and response_checksum_validation == "when_supported":
+ params.setdefault(mode_member, "ENABLED")
+
+
# This is a list of (event_name, handler).
# When a Session is created, everything in this list will be
# automatically registered with that Session.
@@ -1177,7 +1186,7 @@ def _update_status_code(response, **kwargs):
('before-parse.s3.*', handle_expires_header),
('before-parse.s3.*', _handle_200_error, REGISTER_FIRST),
('before-parameter-build', generate_idempotent_uuid),
-
+ ('before-parameter-build', handle_request_validation_mode_member),
('before-parameter-build.s3', validate_bucket_name),
('before-parameter-build.s3', remove_bucket_from_url_paths_from_model),
@@ -1205,10 +1214,7 @@ def _update_status_code(response, **kwargs):
('before-call.s3', add_expect_header),
('before-call.glacier', add_glacier_version),
('before-call.api-gateway', add_accept_header),
- ('before-call.s3.PutObject', conditionally_calculate_checksum),
- ('before-call.s3.UploadPart', conditionally_calculate_md5),
('before-call.s3.DeleteObjects', escape_xml_payload),
- ('before-call.s3.DeleteObjects', conditionally_calculate_checksum),
('before-call.s3.PutBucketLifecycleConfiguration', escape_xml_payload),
('before-call.glacier.UploadArchive', add_glacier_checksums),
('before-call.glacier.UploadMultipartPart', add_glacier_checksums),
diff --git a/awscli/botocore/httpchecksum.py b/awscli/botocore/httpchecksum.py
index 4804483f7272..8b20f76d6b73 100644
--- a/awscli/botocore/httpchecksum.py
+++ b/awscli/botocore/httpchecksum.py
@@ -25,15 +25,15 @@
from hashlib import sha1, sha256
from awscrt import checksums as crt_checksums
+from botocore.compat import urlparse
from botocore.exceptions import AwsChunkedWrapperError, FlexibleChecksumError
from botocore.response import StreamingBody
-from botocore.utils import (
- conditionally_calculate_md5,
- determine_content_length,
-)
+from botocore.utils import determine_content_length, has_checksum_header
logger = logging.getLogger(__name__)
+DEFAULT_CHECKSUM_ALGORITHM = "CRC32"
+
class BaseChecksum:
_CHUNK_SIZE = 1024 * 1024
@@ -246,7 +246,18 @@ def resolve_checksum_context(request, operation_model, params):
def resolve_request_checksum_algorithm(
request, operation_model, params, supported_algorithms=None,
):
+ # If the header is already set by the customer, skip calculation
+ if has_checksum_header(request):
+ return
+
+ request_checksum_calculation = request["context"][
+ "client_config"
+ ].request_checksum_calculation
http_checksum = operation_model.http_checksum
+ request_checksum_required = (
+ operation_model.http_checksum_required
+ or http_checksum.get("requestChecksumRequired")
+ )
algorithm_member = http_checksum.get("requestAlgorithmMember")
if algorithm_member and algorithm_member in params:
# If the client has opted into using flexible checksums and the
@@ -259,35 +270,32 @@ def resolve_request_checksum_algorithm(
raise FlexibleChecksumError(
error_msg="Unsupported checksum algorithm: %s" % algorithm_name
)
+ elif request_checksum_required or (
+ algorithm_member and request_checksum_calculation == "when_supported"
+ ):
+ algorithm_name = DEFAULT_CHECKSUM_ALGORITHM.lower()
+ else:
+ return
- location_type = "header"
- if operation_model.has_streaming_input:
- # Operations with streaming input must support trailers.
- if request["url"].startswith("https:"):
- # We only support unsigned trailer checksums currently. As this
- # disables payload signing we'll only use trailers over TLS.
- location_type = "trailer"
-
- algorithm = {
- "algorithm": algorithm_name,
- "in": location_type,
- "name": "x-amz-checksum-%s" % algorithm_name,
- }
+ location_type = "header"
+ if (
+ operation_model.has_streaming_input
+ and urlparse(request["url"]).scheme == "https"
+ ):
+ # Operations with streaming input must support trailers.
+ # We only support unsigned trailer checksums currently. As this
+ # disables payload signing we'll only use trailers over TLS.
+ location_type = "trailer"
- if algorithm["name"] in request["headers"]:
- # If the header is already set by the customer, skip calculation
- return
+ algorithm = {
+ "algorithm": algorithm_name,
+ "in": location_type,
+ "name": f"x-amz-checksum-{algorithm_name}",
+ }
- checksum_context = request["context"].get("checksum", {})
- checksum_context["request_algorithm"] = algorithm
- request["context"]["checksum"] = checksum_context
- elif operation_model.http_checksum_required or http_checksum.get(
- "requestChecksumRequired"
- ):
- # Otherwise apply the old http checksum behavior via Content-MD5
- checksum_context = request["context"].get("checksum", {})
- checksum_context["request_algorithm"] = "conditional-md5"
- request["context"]["checksum"] = checksum_context
+ checksum_context = request["context"].get("checksum", {})
+ checksum_context["request_algorithm"] = algorithm
+ request["context"]["checksum"] = checksum_context
def apply_request_checksum(request):
@@ -297,10 +305,7 @@ def apply_request_checksum(request):
if not algorithm:
return
- if algorithm == "conditional-md5":
- # Special case to handle the http checksum required trait
- conditionally_calculate_md5(request)
- elif algorithm["in"] == "header":
+ if algorithm["in"] == "header":
_apply_request_header_checksum(request)
elif algorithm["in"] == "trailer":
_apply_request_trailer_checksum(request)
diff --git a/awscli/botocore/utils.py b/awscli/botocore/utils.py
index 2f4e1ef4fd5e..c48ae7d82a2b 100644
--- a/awscli/botocore/utils.py
+++ b/awscli/botocore/utils.py
@@ -2962,6 +2962,7 @@ def get_encoding_from_headers(headers, default='ISO-8859-1'):
def calculate_md5(body, **kwargs):
+ """This function has been deprecated, but is kept for backwards compatibility."""
if isinstance(body, (bytes, bytearray)):
binary_md5 = _calculate_md5_from_bytes(body)
else:
@@ -2970,11 +2971,13 @@ def calculate_md5(body, **kwargs):
def _calculate_md5_from_bytes(body_bytes):
+ """This function has been deprecated, but is kept for backwards compatibility."""
md5 = get_md5(body_bytes)
return md5.digest()
def _calculate_md5_from_file(fileobj):
+ """This function has been deprecated, but is kept for backwards compatibility."""
start_position = fileobj.tell()
md5 = get_md5()
for chunk in iter(lambda: fileobj.read(1024 * 1024), b''):
@@ -2990,15 +2993,17 @@ def _is_s3express_request(params):
return endpoint_properties.get('backend') == 'S3Express'
-def _has_checksum_header(params):
+def has_checksum_header(params):
+ """
+ Checks if a header starting with "x-amz-checksum-" is provided in a request.
+
+ This class is considered private and subject to abrupt breaking changes or
+ removal without prior announcement. Please do not use it directly.
+ """
headers = params['headers']
- # If a user provided Content-MD5 is present,
- # don't try to compute a new one.
- if 'Content-MD5' in headers:
- return True
# If a header matching the x-amz-checksum-* pattern is present, we
- # assume a checksum has already been provided and an md5 is not needed
+ # assume a checksum has already been provided by the user.
for header in headers:
if CHECKSUM_HEADER_PATTERN.match(header):
return True
@@ -3007,12 +3012,14 @@ def _has_checksum_header(params):
def conditionally_calculate_checksum(params, **kwargs):
- if not _has_checksum_header(params):
+ """This function has been deprecated, but is kept for backwards compatibility."""
+ if not has_checksum_header(params):
conditionally_calculate_md5(params, **kwargs)
conditionally_enable_crc32(params, **kwargs)
def conditionally_enable_crc32(params, **kwargs):
+ """This function has been deprecated, but is kept for backwards compatibility."""
checksum_context = params.get('context', {}).get('checksum', {})
checksum_algorithm = checksum_context.get('request_algorithm')
if (
@@ -3030,7 +3037,11 @@ def conditionally_enable_crc32(params, **kwargs):
def conditionally_calculate_md5(params, **kwargs):
- """Only add a Content-MD5 if the system supports it."""
+ """
+ This function has been deprecated, but is kept for backwards compatibility.
+
+ Only add a Content-MD5 if the system supports it.
+ """
body = params['body']
checksum_context = params.get('context', {}).get('checksum', {})
checksum_algorithm = checksum_context.get('request_algorithm')
@@ -3038,7 +3049,7 @@ def conditionally_calculate_md5(params, **kwargs):
# Skip for requests that will have a flexible checksum applied
return
- if _has_checksum_header(params):
+ if has_checksum_header(params):
# Don't add a new header if one is already available.
return
diff --git a/tests/functional/botocore/test_httpchecksum.py b/tests/functional/botocore/test_httpchecksum.py
new file mode 100644
index 000000000000..5b56da39b5bb
--- /dev/null
+++ b/tests/functional/botocore/test_httpchecksum.py
@@ -0,0 +1,482 @@
+# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+
+import pytest
+
+from botocore.exceptions import FlexibleChecksumError
+from tests import ClientHTTPStubber, patch_load_service_model
+
+TEST_CHECKSUM_SERVICE_MODEL = {
+ "version": "2.0",
+ "documentation": "This is a test service.",
+ "metadata": {
+ "apiVersion": "2023-01-01",
+ "endpointPrefix": "test",
+ "protocol": "rest-json",
+ "jsonVersion": "1.1",
+ "serviceFullName": "Test Service",
+ "serviceId": "Test Service",
+ "signatureVersion": "v4",
+ "signingName": "testservice",
+ "uid": "testservice-2023-01-01",
+ },
+ "operations": {
+ "HttpChecksumOperation": {
+ "name": "HttpChecksumOperation",
+ "http": {"method": "POST", "requestUri": "/HttpChecksumOperation"},
+ "input": {"shape": "SomeInput"},
+ "output": {"shape": "SomeOutput"},
+ "httpChecksum": {
+ "requestChecksumRequired": True,
+ "requestAlgorithmMember": "checksumAlgorithm",
+ "requestValidationModeMember": "validationMode",
+ "responseAlgorithms": [
+ "CRC32",
+ "CRC32C",
+ "CRC64NVME",
+ "SHA1",
+ "SHA256",
+ ],
+ },
+ },
+ "HttpChecksumStreamingOperation": {
+ "name": "HttpChecksumStreamingOperation",
+ "http": {
+ "method": "POST",
+ "requestUri": "/HttpChecksumStreamingOperation",
+ },
+ "input": {"shape": "SomeStreamingInput"},
+ "output": {"shape": "SomeStreamingOutput"},
+ "httpChecksum": {
+ "requestChecksumRequired": True,
+ "requestAlgorithmMember": "checksumAlgorithm",
+ "requestValidationModeMember": "validationMode",
+ "responseAlgorithms": [
+ "CRC32",
+ "CRC32C",
+ "CRC64NVME",
+ "SHA1",
+ "SHA256",
+ ],
+ },
+ },
+ },
+ "shapes": {
+ "ChecksumAlgorithm": {
+ "type": "string",
+ "enum": {"CRC32", "CRC32C", "CRC64NVME", "SHA1", "SHA256"},
+ "member": {"shape": "MockOpParam"},
+ },
+ "ValidationMode": {"type": "string", "enum": {"ENABLE"}},
+ "String": {"type": "string"},
+ "Blob": {"type": "blob"},
+ "SomeStreamingOutput": {
+ "type": "structure",
+ "members": {"body": {"shape": "Blob", "streaming": True}},
+ "payload": "body",
+ },
+ "SomeStreamingInput": {
+ "type": "structure",
+ "required": ["body"],
+ "members": {
+ "body": {
+ "shape": "Blob",
+ "streaming": True,
+ },
+ "validationMode": {
+ "shape": "ValidationMode",
+ "location": "header",
+ "locationName": "x-amz-response-validation-mode",
+ },
+ "checksumAlgorithm": {
+ "shape": "ChecksumAlgorithm",
+ "location": "header",
+ "locationName": "x-amz-request-algorithm",
+ },
+ },
+ "payload": "body",
+ },
+ "SomeInput": {
+ "type": "structure",
+ "required": ["body"],
+ "members": {
+ "body": {"shape": "String"},
+ "validationMode": {
+ "shape": "ValidationMode",
+ "location": "header",
+ "locationName": "x-amz-response-validation-mode",
+ },
+ "checksumAlgorithm": {
+ "shape": "ChecksumAlgorithm",
+ "location": "header",
+ "locationName": "x-amz-request-algorithm",
+ },
+ },
+ "payload": "body",
+ },
+ "SomeOutput": {
+ "type": "structure",
+ },
+ },
+}
+
+TEST_CHECKSUM_RULESET = {
+ "version": "1.0",
+ "parameters": {},
+ "rules": [
+ {
+ "conditions": [],
+ "type": "endpoint",
+ "endpoint": {
+ "url": "https://foo.bar",
+ "properties": {},
+ "headers": {},
+ },
+ }
+ ],
+}
+
+
+def _request_checksum_calculation_cases():
+ request_payload = "Hello world"
+ return [
+ (
+ "CRC32",
+ request_payload,
+ {
+ "x-amz-request-algorithm": "CRC32",
+ "x-amz-checksum-crc32": "i9aeUg==",
+ },
+ ),
+ (
+ "SHA1",
+ request_payload,
+ {
+ "x-amz-request-algorithm": "SHA1",
+ "x-amz-checksum-sha1": "e1AsOh9IyGCa4hLN+2Od7jlnP14=",
+ },
+ ),
+ (
+ "SHA256",
+ request_payload,
+ {
+ "x-amz-request-algorithm": "SHA256",
+ "x-amz-checksum-sha256": "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=",
+ },
+ ),
+ (
+ "CRC32C",
+ request_payload,
+ {
+ "x-amz-request-algorithm": "CRC32C",
+ "x-amz-checksum-crc32c": "crUfeA==",
+ },
+ ),
+ (
+ "CRC64NVME",
+ request_payload,
+ {
+ "x-amz-request-algorithm": "CRC64NVME",
+ "x-amz-checksum-crc64nvme": "OOJZ0D8xKts=",
+ },
+ ),
+ ]
+
+
+@pytest.mark.parametrize(
+ "checksum_algorithm, request_payload, expected_headers",
+ _request_checksum_calculation_cases(),
+)
+def test_request_checksum_calculation(
+ patched_session,
+ monkeypatch,
+ checksum_algorithm,
+ request_payload,
+ expected_headers,
+):
+ patch_load_service_model(
+ patched_session,
+ monkeypatch,
+ TEST_CHECKSUM_SERVICE_MODEL,
+ TEST_CHECKSUM_RULESET,
+ )
+ client = patched_session.create_client(
+ "testservice",
+ region_name="us-west-2",
+ )
+ with ClientHTTPStubber(client, strict=True) as http_stubber:
+ http_stubber.add_response(status=200, body=b"")
+ client.http_checksum_operation(
+ body=request_payload, checksumAlgorithm=checksum_algorithm
+ )
+ actual_headers = http_stubber.requests[0].headers
+ for key, val in expected_headers.items():
+ assert key in actual_headers
+ assert actual_headers[key].decode() == val
+
+
+def _streaming_request_checksum_calculation_cases():
+ request_payload = "Hello world"
+ return [
+ (
+ "CRC32",
+ request_payload,
+ {
+ "content-encoding": "aws-chunked",
+ "x-amz-trailer": "x-amz-checksum-crc32",
+ },
+ {"x-amz-checksum-crc32": "i9aeUg=="},
+ ),
+ (
+ "SHA1",
+ request_payload,
+ {
+ "content-encoding": "aws-chunked",
+ "x-amz-trailer": "x-amz-checksum-sha1",
+ },
+ {"x-amz-checksum-sha1": "e1AsOh9IyGCa4hLN+2Od7jlnP14="},
+ ),
+ (
+ "SHA256",
+ request_payload,
+ {
+ "content-encoding": "aws-chunked",
+ "x-amz-trailer": "x-amz-checksum-sha256",
+ },
+ {
+ "x-amz-checksum-sha256": "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw="
+ },
+ ),
+ (
+ "CRC32C",
+ request_payload,
+ {
+ "content-encoding": "aws-chunked",
+ "x-amz-trailer": "x-amz-checksum-crc32c",
+ },
+ {"x-amz-checksum-crc32c": "crUfeA=="},
+ ),
+ (
+ "CRC64NVME",
+ request_payload,
+ {
+ "content-encoding": "aws-chunked",
+ "x-amz-trailer": "x-amz-checksum-crc64nvme",
+ },
+ {"x-amz-checksum-crc64nvme": "OOJZ0D8xKts="},
+ ),
+ ]
+
+
+@pytest.mark.parametrize(
+ "checksum_algorithm, request_payload, expected_headers, expected_trailers",
+ _streaming_request_checksum_calculation_cases(),
+)
+def test_streaming_request_checksum_calculation(
+ patched_session,
+ monkeypatch,
+ checksum_algorithm,
+ request_payload,
+ expected_headers,
+ expected_trailers,
+):
+ patch_load_service_model(
+ patched_session,
+ monkeypatch,
+ TEST_CHECKSUM_SERVICE_MODEL,
+ TEST_CHECKSUM_RULESET,
+ )
+ client = patched_session.create_client(
+ "testservice",
+ region_name="us-west-2",
+ )
+ with ClientHTTPStubber(client, strict=True) as http_stubber:
+ http_stubber.add_response(status=200, body=b"")
+ client.http_checksum_streaming_operation(
+ body=request_payload, checksumAlgorithm=checksum_algorithm
+ )
+ request = http_stubber.requests[0]
+ actual_headers = request.headers
+ for key, val in expected_headers.items():
+ assert key in actual_headers
+ assert actual_headers[key].decode() == val
+ read_body = request.body.read()
+ for key, val in expected_trailers.items():
+ assert f"{key}:{val}".encode() in read_body
+
+
+def _response_checksum_validation_cases():
+ response_payload = "Hello world"
+ return [
+ (
+ "CRC32",
+ response_payload,
+ {"x-amz-checksum-crc32": "i9aeUg=="},
+ {"kind": "success"},
+ ),
+ (
+ "CRC32",
+ response_payload,
+ {"x-amz-checksum-crc32": "bm90LWEtY2hlY2tzdW0="},
+ {"kind": "failure", "calculatedChecksum": "i9aeUg=="},
+ ),
+ (
+ "SHA1",
+ response_payload,
+ {"x-amz-checksum-sha1": "e1AsOh9IyGCa4hLN+2Od7jlnP14="},
+ {"kind": "success"},
+ ),
+ (
+ "SHA1",
+ response_payload,
+ {"x-amz-checksum-sha1": "bm90LWEtY2hlY2tzdW0="},
+ {
+ "kind": "failure",
+ "calculatedChecksum": "e1AsOh9IyGCa4hLN+2Od7jlnP14=",
+ },
+ ),
+ (
+ "SHA256",
+ response_payload,
+ {
+ "x-amz-checksum-sha256": "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw="
+ },
+ {"kind": "success"},
+ ),
+ (
+ "SHA256",
+ response_payload,
+ {"x-amz-checksum-sha256": "bm90LWEtY2hlY2tzdW0="},
+ {
+ "kind": "failure",
+ "calculatedChecksum": "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=",
+ },
+ ),
+ (
+ "CRC32C",
+ response_payload,
+ {"x-amz-checksum-crc32c": "crUfeA=="},
+ {"kind": "success"},
+ ),
+ (
+ "CRC32C",
+ response_payload,
+ {"x-amz-checksum-crc32c": "bm90LWEtY2hlY2tzdW0="},
+ {"kind": "failure", "calculatedChecksum": "crUfeA=="},
+ ),
+ (
+ "CRC64NVME",
+ response_payload,
+ {"x-amz-checksum-crc64nvme": "OOJZ0D8xKts="},
+ {"kind": "success"},
+ ),
+ (
+ "CRC64NVME",
+ response_payload,
+ {"x-amz-checksum-crc64nvme": "bm90LWEtY2hlY2tzdW0="},
+ {"kind": "failure", "calculatedChecksum": "OOJZ0D8xKts="},
+ ),
+ ]
+
+
+@pytest.mark.parametrize(
+ "checksum_algorithm, response_payload, response_headers, expected",
+ _response_checksum_validation_cases(),
+)
+def test_response_checksum_validation(
+ patched_session,
+ monkeypatch,
+ checksum_algorithm,
+ response_payload,
+ response_headers,
+ expected,
+):
+ patch_load_service_model(
+ patched_session,
+ monkeypatch,
+ TEST_CHECKSUM_SERVICE_MODEL,
+ TEST_CHECKSUM_RULESET,
+ )
+ client = patched_session.create_client(
+ "testservice",
+ region_name="us-west-2",
+ )
+ with ClientHTTPStubber(client, strict=True) as http_stubber:
+ http_stubber.add_response(
+ status=200,
+ body=response_payload.encode(),
+ headers=response_headers,
+ )
+ operation_kwargs = {
+ "body": response_payload,
+ "checksumAlgorithm": checksum_algorithm,
+ }
+ if expected["kind"] == "failure":
+ with pytest.raises(FlexibleChecksumError) as expected_error:
+ client.http_checksum_operation(**operation_kwargs)
+ error_msg = "Expected checksum {} did not match calculated checksum: {}".format(
+ response_headers[
+ f'x-amz-checksum-{checksum_algorithm.lower()}'
+ ],
+ expected['calculatedChecksum'],
+ )
+ assert str(expected_error.value) == error_msg
+ else:
+ client.http_checksum_operation(**operation_kwargs)
+
+
+@pytest.mark.parametrize(
+ "checksum_algorithm, response_payload, response_headers, expected",
+ _response_checksum_validation_cases(),
+)
+def test_streaming_response_checksum_validation(
+ patched_session,
+ monkeypatch,
+ checksum_algorithm,
+ response_payload,
+ response_headers,
+ expected,
+):
+ patch_load_service_model(
+ patched_session,
+ monkeypatch,
+ TEST_CHECKSUM_SERVICE_MODEL,
+ TEST_CHECKSUM_RULESET,
+ )
+ client = patched_session.create_client(
+ "testservice",
+ region_name="us-west-2",
+ )
+ with ClientHTTPStubber(client, strict=True) as http_stubber:
+ http_stubber.add_response(
+ status=200,
+ body=response_payload.encode(),
+ headers=response_headers,
+ )
+ response = client.http_checksum_streaming_operation(
+ body=response_payload,
+ checksumAlgorithm=checksum_algorithm,
+ )
+ if expected["kind"] == "failure":
+ with pytest.raises(FlexibleChecksumError) as expected_error:
+ response["body"].read()
+ error_msg = "Expected checksum {} did not match calculated checksum: {}".format(
+ response_headers[
+ f'x-amz-checksum-{checksum_algorithm.lower()}'
+ ],
+ expected['calculatedChecksum'],
+ )
+ assert str(expected_error.value) == error_msg
+ else:
+ response["body"].read()
\ No newline at end of file
diff --git a/tests/functional/botocore/test_s3.py b/tests/functional/botocore/test_s3.py
index c51981c3ada2..c369803b3a85 100644
--- a/tests/functional/botocore/test_s3.py
+++ b/tests/functional/botocore/test_s3.py
@@ -10,20 +10,24 @@
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
-import base64
import re
import pytest
from dateutil.tz import tzutc
from tests import (
- create_session, mock, temporary_file, unittest,
- BaseSessionTest, ClientHTTPStubber, FreezeTime
+ BaseSessionTest,
+ ClientHTTPStubber,
+ FreezeTime,
+ create_session,
+ mock,
+ temporary_file, unittest,
)
+from tests.utils.botocore import get_checksum_cls
import botocore.session
from botocore.config import Config
-from botocore.compat import datetime, urlsplit, parse_qs, get_md5
+from botocore.compat import datetime, parse_qs, urlsplit
from botocore.exceptions import (
ParamValidationError, ClientError,
UnsupportedS3ConfigurationError,
@@ -1389,31 +1393,67 @@ def setUp(self):
def get_sent_headers(self):
return self.http_stubber.requests[0].headers
- def test_content_md5_set(self):
+ def test_trailing_checksum_set(self):
with self.http_stubber:
self.client.put_object(Bucket='foo', Key='bar', Body='baz')
- self.assertIn('content-md5', self.get_sent_headers())
+ sent_headers = self.get_sent_headers()
+ self.assertEqual(sent_headers["Content-Encoding"], b"aws-chunked")
+ self.assertEqual(sent_headers["Transfer-Encoding"], b"chunked")
+ self.assertEqual(
+ sent_headers["X-Amz-Trailer"], b"x-amz-checksum-crc32"
+ )
+ self.assertEqual(sent_headers["X-Amz-Decoded-Content-Length"], b"3")
+ self.assertEqual(
+ sent_headers["x-amz-content-sha256"],
+ b"STREAMING-UNSIGNED-PAYLOAD-TRAILER",
+ )
+ body = self.http_stubber.requests[0].body.read()
+ self.assertIn(b"x-amz-checksum-crc32", body)
- def test_content_md5_set_empty_body(self):
+ def test_trailing_checksum_set_empty_body(self):
with self.http_stubber:
self.client.put_object(Bucket='foo', Key='bar', Body='')
- self.assertIn('content-md5', self.get_sent_headers())
+ sent_headers = self.get_sent_headers()
+ self.assertEqual(sent_headers["Content-Encoding"], b"aws-chunked")
+ self.assertEqual(sent_headers["Transfer-Encoding"], b"chunked")
+ self.assertEqual(
+ sent_headers["X-Amz-Trailer"], b"x-amz-checksum-crc32"
+ )
+ self.assertEqual(sent_headers["X-Amz-Decoded-Content-Length"], b"0")
+ self.assertEqual(
+ sent_headers["x-amz-content-sha256"],
+ b"STREAMING-UNSIGNED-PAYLOAD-TRAILER",
+ )
+ body = self.http_stubber.requests[0].body.read()
+ self.assertIn(b"x-amz-checksum-crc32", body)
- def test_content_md5_set_empty_file(self):
+ def test_trailing_checksum_set_empty_file(self):
with self.http_stubber:
with temporary_file('rb') as f:
assert f.read() == b''
self.client.put_object(Bucket='foo', Key='bar', Body=f)
- self.assertIn('content-md5', self.get_sent_headers())
-
- def test_content_sha256_set_if_config_value_is_true(self):
- # By default, put_object() does not include an x-amz-content-sha256
- # header because it also includes a `Content-MD5` header. The
- # `payload_signing_enabled` config overrides this logic and forces the
- # header.
- config = Config(signature_version='s3v4', s3={
- 'payload_signing_enabled': True
- })
+ body = self.http_stubber.requests[0].body.read()
+ sent_headers = self.get_sent_headers()
+ self.assertEqual(sent_headers["Content-Encoding"], b"aws-chunked")
+ self.assertEqual(sent_headers["Transfer-Encoding"], b"chunked")
+ self.assertEqual(
+ sent_headers["X-Amz-Trailer"], b"x-amz-checksum-crc32"
+ )
+ self.assertEqual(sent_headers["X-Amz-Decoded-Content-Length"], b"0")
+ self.assertEqual(
+ sent_headers["x-amz-content-sha256"],
+ b"STREAMING-UNSIGNED-PAYLOAD-TRAILER",
+ )
+ self.assertIn(b"x-amz-checksum-crc32", body)
+
+ def test_content_sha256_not_set_if_config_value_is_true(self):
+ # By default, put_object() provides a trailing checksum and includes the
+ # x-amz-content-sha256 header with a value of "STREAMING-UNSIGNED-PAYLOAD-TRAILER".
+ # We do not support payload signing for streaming so the `payload_signing_enabled`
+ # config has no effect here.
+ config = Config(
+ signature_version='s3v4',
+ s3={'payload_signing_enabled': True})
self.client = self.session.create_client(
's3', self.region, config=config)
self.http_stubber = ClientHTTPStubber(self.client)
@@ -1422,12 +1462,17 @@ def test_content_sha256_set_if_config_value_is_true(self):
self.client.put_object(Bucket='foo', Key='bar', Body='baz')
sent_headers = self.get_sent_headers()
sha_header = sent_headers.get('x-amz-content-sha256')
- self.assertNotEqual(sha_header, b'UNSIGNED-PAYLOAD')
+ self.assertEqual(sha_header, b"STREAMING-UNSIGNED-PAYLOAD-TRAILER")
def test_content_sha256_not_set_if_config_value_is_false(self):
- config = Config(signature_version='s3v4', s3={
- 'payload_signing_enabled': False
- })
+ # By default, put_object() provides a trailing checksum and includes the
+ # x-amz-content-sha256 header with a value of "STREAMING-UNSIGNED-PAYLOAD-TRAILER".
+ # We do not support payload signing for streaming so the `payload_signing_enabled`
+ # config has no effect here.
+ config = Config(
+ signature_version='s3v4',
+ s3={'payload_signing_enabled': False},
+ )
self.client = self.session.create_client(
's3', self.region, config=config)
self.http_stubber = ClientHTTPStubber(self.client)
@@ -1436,15 +1481,18 @@ def test_content_sha256_not_set_if_config_value_is_false(self):
self.client.put_object(Bucket='foo', Key='bar', Body='baz')
sent_headers = self.get_sent_headers()
sha_header = sent_headers.get('x-amz-content-sha256')
- self.assertEqual(sha_header, b'UNSIGNED-PAYLOAD')
+ self.assertEqual(sha_header, b"STREAMING-UNSIGNED-PAYLOAD-TRAILER")
- def test_content_sha256_set_if_config_value_not_set_put_object(self):
- # The default behavior matches payload_signing_enabled=False. For
- # operations where the `Content-MD5` is present this means that
- # `x-amz-content-sha256` is present but not set.
+ def test_content_sha256_not_set_if_config_value_not_set_put_object(self):
+ # By default, put_object() provides a trailing checksum and includes the
+ # x-amz-content-sha256 header with a value of "STREAMING-UNSIGNED-PAYLOAD-TRAILER".
+ # We do not support payload signing for streaming so the `payload_signing_enabled`
+ # config has no effect here.
config = Config(signature_version='s3v4')
self.client = self.session.create_client(
- 's3', self.region, config=config
+ "s3",
+ self.region,
+ config=config,
)
self.http_stubber = ClientHTTPStubber(self.client)
self.http_stubber.add_response()
@@ -1452,7 +1500,7 @@ def test_content_sha256_set_if_config_value_not_set_put_object(self):
self.client.put_object(Bucket='foo', Key='bar', Body='baz')
sent_headers = self.get_sent_headers()
sha_header = sent_headers.get('x-amz-content-sha256')
- self.assertEqual(sha_header, b'UNSIGNED-PAYLOAD')
+ self.assertEqual(sha_header, b"STREAMING-UNSIGNED-PAYLOAD-TRAILER")
def test_content_sha256_set_if_config_value_not_set_list_objects(self):
# The default behavior matches payload_signing_enabled=False. For
@@ -1488,16 +1536,6 @@ def test_content_sha256_set_s3_on_outpost(self):
sha_header = sent_headers.get('x-amz-content-sha256')
self.assertNotEqual(sha_header, b'UNSIGNED-PAYLOAD')
- def test_content_sha256_set_if_md5_is_unavailable(self):
- with mock.patch('botocore.auth.MD5_AVAILABLE', False):
- with mock.patch('botocore.utils.MD5_AVAILABLE', False):
- with self.http_stubber:
- self.client.put_object(Bucket='foo', Key='bar', Body='baz')
- sent_headers = self.get_sent_headers()
- unsigned = 'UNSIGNED-PAYLOAD'
- self.assertNotEqual(sent_headers['x-amz-content-sha256'], unsigned)
- self.assertNotIn('content-md5', sent_headers)
-
class TestCanSendIntegerHeaders(BaseSessionTest):
@@ -2027,7 +2065,7 @@ def test_checksums_included_in_expected_operations(operation, operation_kwargs):
stub.add_response()
call = getattr(client, operation)
call(**operation_kwargs)
- assert 'Content-MD5' in stub.requests[-1].headers
+ assert "x-amz-checksum-crc32" in stub.requests[-1].headers
@pytest.mark.parametrize(
@@ -2992,10 +3030,12 @@ def test_can_provide_request_payer_get_tagging(self):
class TestS3XMLPayloadEscape(BaseS3OperationTest):
- def assert_correct_content_md5(self, request):
- content_md5_bytes = get_md5(request.body).digest()
- content_md5 = base64.b64encode(content_md5_bytes)
- self.assertEqual(content_md5, request.headers['Content-MD5'])
+ def assert_correct_crc32_checksum(self, request):
+ checksum = get_checksum_cls()()
+ crc32_checksum = checksum.handle(request.body).encode()
+ self.assertEqual(
+ crc32_checksum, request.headers["x-amz-checksum-crc32"]
+ )
def test_escape_keys_in_xml_delete_objects(self):
self.http_stubber.add_response()
@@ -3009,7 +3049,7 @@ def test_escape_keys_in_xml_delete_objects(self):
request = self.http_stubber.requests[0]
self.assertNotIn(b'\r\n\r', request.body)
self.assertIn(b'
', request.body)
- self.assert_correct_content_md5(request)
+ self.assert_correct_crc32_checksum(request)
def test_escape_keys_in_xml_put_bucket_lifecycle_configuration(self):
self.http_stubber.add_response()
@@ -3026,7 +3066,7 @@ def test_escape_keys_in_xml_put_bucket_lifecycle_configuration(self):
request = self.http_stubber.requests[0]
self.assertNotIn(b'my\r\n\rprefix', request.body)
self.assertIn(b'my
prefix', request.body)
- self.assert_correct_content_md5(request)
+ self.assert_correct_crc32_checksum(request)
@pytest.mark.parametrize(
diff --git a/tests/functional/s3/test_cp_command.py b/tests/functional/s3/test_cp_command.py
index 683a426aad1c..13661d84d2bc 100644
--- a/tests/functional/s3/test_cp_command.py
+++ b/tests/functional/s3/test_cp_command.py
@@ -2207,7 +2207,7 @@ def test_streaming_upload_using_crt_client(self):
expected_type=S3RequestType.PUT_OBJECT,
expected_host=self.get_virtual_s3_host('bucket'),
expected_path='/key',
- expected_body_content=b'foo',
+ expected_body_content=b'3\r\nfoo\r\n0\r\nx-amz-checksum-crc32:jHNlIQ==\r\n\r\n',
)
def test_streaming_download_using_crt_client(self):
diff --git a/tests/functional/s3api/test_get_object.py b/tests/functional/s3api/test_get_object.py
index f5be0844600b..462934264d5c 100644
--- a/tests/functional/s3api/test_get_object.py
+++ b/tests/functional/s3api/test_get_object.py
@@ -38,6 +38,7 @@ def test_simple(self):
cmdline += ' outfile'
self.addCleanup(self.remove_file_if_exists, 'outfile')
self.assert_params_for_cmd(cmdline, {'Bucket': 'mybucket',
+ 'ChecksumMode': 'ENABLED',
'Key': 'mykey'})
def test_range(self):
@@ -48,6 +49,7 @@ def test_range(self):
cmdline += ' outfile'
self.addCleanup(self.remove_file_if_exists, 'outfile')
self.assert_params_for_cmd(cmdline, {'Bucket': 'mybucket',
+ 'ChecksumMode': 'ENABLED',
'Key': 'mykey',
'Range': 'bytes=0-499'})
@@ -61,7 +63,9 @@ def test_response_headers(self):
self.addCleanup(self.remove_file_if_exists, 'outfile')
self.assert_params_for_cmd(
cmdline, {
- 'Bucket': 'mybucket', 'Key': 'mykey',
+ 'Bucket': 'mybucket',
+ 'ChecksumMode': 'ENABLED',
+ 'Key': 'mykey',
'ResponseCacheControl': 'No-cache',
'ResponseContentEncoding': 'x-gzip'
}
@@ -83,7 +87,7 @@ def test_streaming_output_arg_with_error_response(self):
cmdline += ' outfile'
self.addCleanup(self.remove_file_if_exists, 'outfile')
self.assert_params_for_cmd(
- cmdline, {'Bucket': 'mybucket', 'Key': 'mykey'})
+ cmdline, {'Bucket': 'mybucket', 'ChecksumMode': 'ENABLED', 'Key': 'mykey'})
if __name__ == "__main__":
diff --git a/tests/unit/botocore/test_args.py b/tests/unit/botocore/test_args.py
index ac6ada14f51d..9a55474b942a 100644
--- a/tests/unit/botocore/test_args.py
+++ b/tests/unit/botocore/test_args.py
@@ -450,6 +450,70 @@ def test_bad_value_disable_request_compression(self):
config = client_args['client_config']
self.assertFalse(config.disable_request_compression)
+ def test_checksum_default_client_config(self):
+ input_config = Config()
+ client_args = self.call_get_client_args(client_config=input_config)
+ config = client_args["client_config"]
+ self.assertEqual(config.request_checksum_calculation, "when_supported")
+ self.assertEqual(config.response_checksum_validation, "when_supported")
+
+ def test_checksum_client_config(self):
+ input_config = Config(
+ request_checksum_calculation="when_required",
+ response_checksum_validation="when_required",
+ )
+ client_args = self.call_get_client_args(client_config=input_config)
+ config = client_args['client_config']
+ self.assertEqual(config.request_checksum_calculation, "when_required")
+ self.assertEqual(config.response_checksum_validation, "when_required")
+
+ def test_checksum_config_store(self):
+ self.config_store.set_config_variable(
+ "request_checksum_calculation", "when_required"
+ )
+ self.config_store.set_config_variable(
+ "response_checksum_validation", "when_required"
+ )
+ config = self.call_get_client_args()['client_config']
+ self.assertEqual(config.request_checksum_calculation, "when_required")
+ self.assertEqual(config.response_checksum_validation, "when_required")
+
+ def test_checksum_client_config_overrides_config_store(self):
+ self.config_store.set_config_variable(
+ "request_checksum_calculation", "when_supported"
+ )
+ self.config_store.set_config_variable(
+ "response_checksum_validation", "when_supported"
+ )
+ input_config = Config(
+ request_checksum_calculation="when_required",
+ response_checksum_validation="when_required",
+ )
+ client_args = self.call_get_client_args(client_config=input_config)
+ config = client_args['client_config']
+ self.assertEqual(config.request_checksum_calculation, "when_required")
+ self.assertEqual(config.response_checksum_validation, "when_required")
+
+ def test_request_checksum_calculation_invalid_client_config(self):
+ with self.assertRaises(exceptions.InvalidChecksumConfigError):
+ config = Config(request_checksum_calculation="invalid_config")
+ self.call_get_client_args(client_config=config)
+ self.config_store.set_config_variable(
+ 'request_checksum_calculation', "invalid_config"
+ )
+ with self.assertRaises(exceptions.InvalidChecksumConfigError):
+ self.call_get_client_args()
+
+ def test_response_checksum_validation_invalid_client_config(self):
+ with self.assertRaises(exceptions.InvalidChecksumConfigError):
+ config = Config(response_checksum_validation="invalid_config")
+ self.call_get_client_args(client_config=config)
+ self.config_store.set_config_variable(
+ 'response_checksum_validation', "invalid_config"
+ )
+ with self.assertRaises(exceptions.InvalidChecksumConfigError):
+ self.call_get_client_args()
+
class TestEndpointResolverBuiltins(unittest.TestCase):
def setUp(self):
diff --git a/tests/unit/botocore/test_handlers.py b/tests/unit/botocore/test_handlers.py
index 564abdbf0ad2..7008fe03b319 100644
--- a/tests/unit/botocore/test_handlers.py
+++ b/tests/unit/botocore/test_handlers.py
@@ -1698,4 +1698,57 @@ def test_document_response_params_without_expires(document_expires_mocks):
)
mocks['section'].get_section.assert_not_called()
mocks['param_section'].add_new_section.assert_not_called()
- mocks['doc_section'].write.assert_not_called()
\ No newline at end of file
+ mocks['doc_section'].write.assert_not_called()
+
+
+@pytest.fixture()
+def checksum_operation_model():
+ operation_model = mock.Mock(spec=OperationModel)
+ operation_model.http_checksum = {
+ "requestValidationModeMember": "ChecksumMode",
+ }
+ return operation_model
+
+
+def create_checksum_context(
+ request_checksum_calculation="when_supported",
+ response_checksum_validation="when_supported",
+):
+ context = {
+ "client_config": Config(
+ request_checksum_calculation=request_checksum_calculation,
+ response_checksum_validation=response_checksum_validation,
+ )
+ }
+ return context
+
+
+def test_request_validation_mode_member_default(checksum_operation_model):
+ params = {}
+ handlers.handle_request_validation_mode_member(
+ params, checksum_operation_model, context=create_checksum_context()
+ )
+ assert params["ChecksumMode"] == "ENABLED"
+
+
+def test_request_validation_mode_member_when_required(
+ checksum_operation_model,
+):
+ params = {}
+ context = create_checksum_context(
+ response_checksum_validation="when_required"
+ )
+ handlers.handle_request_validation_mode_member(
+ params, checksum_operation_model, context=context
+ )
+ assert "ChecksumMode" not in params
+
+
+def test_request_validation_mode_member_is_not_enabled(
+ checksum_operation_model,
+):
+ params = {"ChecksumMode": "FAKE_VALUE"}
+ handlers.handle_request_validation_mode_member(
+ params, checksum_operation_model, context=create_checksum_context()
+ )
+ assert params["ChecksumMode"] == "FAKE_VALUE"
\ No newline at end of file
diff --git a/tests/unit/botocore/test_httpchecksum.py b/tests/unit/botocore/test_httpchecksum.py
index 1df140242d84..2b999fad28cf 100644
--- a/tests/unit/botocore/test_httpchecksum.py
+++ b/tests/unit/botocore/test_httpchecksum.py
@@ -14,9 +14,11 @@
from io import BytesIO
from tests import mock
+from tests.utils.botocore import get_checksum_cls
from botocore.awsrequest import AWSResponse
from botocore.model import OperationModel
+from botocore.config import Config
from botocore.exceptions import AwsChunkedWrapperError
from botocore.exceptions import FlexibleChecksumError
from botocore.httpchecksum import AwsChunkedWrapper
@@ -26,14 +28,13 @@
Sha256Checksum,
CrtCrc64NvmeChecksum,
Sha1Checksum,
- CrtCrc32Checksum,
CrtCrc32cChecksum,
-)
-from botocore.httpchecksum import (
+ CrtCrc32Checksum,
+ CrtCrc64NvmeChecksum,
apply_request_checksum,
+ handle_checksum_body,
resolve_request_checksum_algorithm,
resolve_response_checksum_algorithms,
- handle_checksum_body,
)
@@ -83,7 +84,11 @@ def _build_request(self, body):
request = {
"headers": {},
"body": body,
- "context": {},
+ "context": {
+ "client_config": Config(
+ request_checksum_calculation="when_supported",
+ )
+ },
"url": "https://example.com",
}
return request
@@ -95,25 +100,31 @@ def test_request_checksum_algorithm_no_model(self):
resolve_request_checksum_algorithm(request, operation_model, params)
self.assertNotIn("checksum", request["context"])
- def test_request_checksum_algorithm_model_opt_in(self):
+ def test_request_checksum_algorithm_model_default(self):
operation_model = self._make_operation_model(
http_checksum={"requestAlgorithmMember": "Algorithm"}
)
- # Param is not present, no checksum will be set
+ # Param is not present, crc32 checksum will be set by default
params = {}
request = self._build_request(b"")
resolve_request_checksum_algorithm(request, operation_model, params)
- self.assertNotIn("checksum", request["context"])
+ expected_algorithm = {
+ "algorithm": "crc32",
+ "in": "header",
+ "name": "x-amz-checksum-crc32",
+ }
+ actual_algorithm = request["context"]["checksum"]["request_algorithm"]
+ self.assertEqual(actual_algorithm, expected_algorithm)
- # Param is present, crc32 checksum will be set
- params = {"Algorithm": "crc32"}
+ # Param is present, sha256 checksum will be set
+ params = {"Algorithm": "sha256"}
request = self._build_request(b"")
resolve_request_checksum_algorithm(request, operation_model, params)
expected_algorithm = {
- "algorithm": "crc32",
+ "algorithm": "sha256",
"in": "header",
- "name": "x-amz-checksum-crc32",
+ "name": "x-amz-checksum-sha256",
}
actual_algorithm = request["context"]["checksum"]["request_algorithm"]
self.assertEqual(actual_algorithm, expected_algorithm)
@@ -125,21 +136,16 @@ def test_request_checksum_algorithm_model_opt_in(self):
resolve_request_checksum_algorithm(request, operation_model, params)
self.assertNotIn("checksum", request["context"])
- def test_request_checksum_algorithm_model_opt_in_streaming(self):
+ def test_request_checksum_algorithm_model_default_streaming(self):
request = self._build_request(b"")
operation_model = self._make_operation_model(
http_checksum={"requestAlgorithmMember": "Algorithm"},
streaming_input=True,
)
- # Param is not present, no checksum will be set
+ # Param is not present, crc32 checksum will be set
params = {}
resolve_request_checksum_algorithm(request, operation_model, params)
- self.assertNotIn("checksum", request["context"])
-
- # Param is present, crc32 checksum will be set in the trailer
- params = {"Algorithm": "crc32"}
- resolve_request_checksum_algorithm(request, operation_model, params)
expected_algorithm = {
"algorithm": "crc32",
"in": "trailer",
@@ -148,14 +154,25 @@ def test_request_checksum_algorithm_model_opt_in_streaming(self):
actual_algorithm = request["context"]["checksum"]["request_algorithm"]
self.assertEqual(actual_algorithm, expected_algorithm)
+ # Param is present, sha256 checksum will be set in the trailer
+ params = {"Algorithm": "sha256"}
+ resolve_request_checksum_algorithm(request, operation_model, params)
+ expected_algorithm = {
+ "algorithm": "sha256",
+ "in": "trailer",
+ "name": "x-amz-checksum-sha256",
+ }
+ actual_algorithm = request["context"]["checksum"]["request_algorithm"]
+ self.assertEqual(actual_algorithm, expected_algorithm)
+
# Trailer should not be used for http endpoints
request = self._build_request(b"")
request["url"] = "http://example.com"
resolve_request_checksum_algorithm(request, operation_model, params)
expected_algorithm = {
- "algorithm": "crc32",
+ "algorithm": "sha256",
"in": "header",
- "name": "x-amz-checksum-crc32",
+ "name": "x-amz-checksum-sha256",
}
actual_algorithm = request["context"]["checksum"]["request_algorithm"]
self.assertEqual(actual_algorithm, expected_algorithm)
@@ -172,17 +189,21 @@ def test_request_checksum_algorithm_model_unsupported_algorithm(self):
request, operation_model, params, supported_algorithms=[]
)
- def test_request_checksum_algorithm_model_legacy_md5(self):
+ def test_request_checksum_algorithm_model_legacy_crc32(self):
request = self._build_request(b"")
operation_model = self._make_operation_model(required=True)
params = {}
resolve_request_checksum_algorithm(request, operation_model, params)
+ expected_algorithm = {
+ "algorithm": "crc32",
+ "in": "header",
+ "name": "x-amz-checksum-crc32",
+ }
actual_algorithm = request["context"]["checksum"]["request_algorithm"]
- expected_algorithm = "conditional-md5"
self.assertEqual(actual_algorithm, expected_algorithm)
- def test_request_checksum_algorithm_model_new_md5(self):
+ def test_request_checksum_algorithm_model_new_crc32(self):
request = self._build_request(b"")
operation_model = self._make_operation_model(
http_checksum={"requestChecksumRequired": True}
@@ -191,7 +212,11 @@ def test_request_checksum_algorithm_model_new_md5(self):
resolve_request_checksum_algorithm(request, operation_model, params)
actual_algorithm = request["context"]["checksum"]["request_algorithm"]
- expected_algorithm = "conditional-md5"
+ expected_algorithm = {
+ "algorithm": "crc32",
+ "in": "header",
+ "name": "x-amz-checksum-crc32",
+ }
self.assertEqual(actual_algorithm, expected_algorithm)
def test_apply_request_checksum_handles_no_checksum_context(self):
@@ -199,7 +224,9 @@ def test_apply_request_checksum_handles_no_checksum_context(self):
apply_request_checksum(request)
# Build another request and assert the original request is the same
expected_request = self._build_request(b"")
- self.assertEqual(request, expected_request)
+ self.assertEqual(request["headers"], expected_request["headers"])
+ self.assertEqual(request["body"], expected_request["body"])
+ self.assertEqual(request["url"], expected_request["url"])
def test_apply_request_checksum_handles_invalid_context(self):
request = self._build_request(b"")
@@ -213,14 +240,6 @@ def test_apply_request_checksum_handles_invalid_context(self):
with self.assertRaises(FlexibleChecksumError):
apply_request_checksum(request)
- def test_apply_request_checksum_conditional_md5(self):
- request = self._build_request(b"")
- request["context"]["checksum"] = {
- "request_algorithm": "conditional-md5"
- }
- apply_request_checksum(request)
- self.assertIn("Content-MD5", request["headers"])
-
def test_apply_request_checksum_flex_header_bytes(self):
request = self._build_request(b"")
request["context"]["checksum"] = {
@@ -336,7 +355,7 @@ def test_response_checksum_algorithm_no_model(self):
resolve_response_checksum_algorithms(request, operation_model, params)
self.assertNotIn("checksum", request["context"])
- def test_response_checksum_algorithm_model_opt_in(self):
+ def test_response_checksum_algorithm_model_default(self):
request = self._build_request(b"")
operation_model = self._make_operation_model(
http_checksum={
diff --git a/tests/utils/botocore/__init__.py b/tests/utils/botocore/__init__.py
index 106736f3e7b7..07714439685a 100644
--- a/tests/utils/botocore/__init__.py
+++ b/tests/utils/botocore/__init__.py
@@ -39,10 +39,10 @@
import botocore.loaders
import botocore.session
from botocore.awsrequest import AWSResponse
-from botocore.compat import urlparse
-from botocore.compat import parse_qs
+from botocore.compat import parse_qs, urlparse
from botocore import utils
from botocore import credentials
+from botocore.httpchecksum import _CHECKSUM_CLS, DEFAULT_CHECKSUM_ALGORITHM
from botocore.stub import Stubber
@@ -590,3 +590,13 @@ def mock_load_service_model(service_name, type_name, api_version=None):
loader = session.get_component('data_loader')
monkeypatch.setattr(loader, 'load_service_model', mock_load_service_model)
+
+
+def get_checksum_cls(algorithm=DEFAULT_CHECKSUM_ALGORITHM.lower()):
+ """
+ This pass through is grabbing our internally supported list of checksums
+ to ensure we stay in sync, while not exposing them publicly.
+
+ Returns the default checksum algorithm class if none is specified.
+ """
+ return _CHECKSUM_CLS[algorithm]
\ No newline at end of file