[v2] Botocore request checksum calculation

awscli/botocore/args.py

@@ -36,6 +36,14 @@
 # values result in a warning-level log message.
 USERAGENT_APPID_MAXLEN = 50
 
+VALID_REQUEST_CHECKSUM_CALCULATION_CONFIG = (
+    "when_supported",
+    "when_required",
+)
+VALID_RESPONSE_CHECKSUM_VALIDATION_CONFIG = (
+    "when_supported",
+    "when_required",
+)
 
 class ClientArgsCreator(object):
     def __init__(self, event_emitter, user_agent, response_parser_factory,
@@ -216,11 +224,18 @@ def compute_client_args(self, service_model, client_config,
                 sigv4a_signing_region_set=(
                     client_config.sigv4a_signing_region_set
                 ),
+                request_checksum_calculation=(
+                    client_config.request_checksum_calculation
+                ),
+                response_checksum_validation=(
+                    client_config.response_checksum_validation
+                ),
             )
         self._compute_retry_config(config_kwargs)
         self._compute_request_compression_config(config_kwargs)
         self._compute_user_agent_appid_config(config_kwargs)
         self._compute_sigv4a_signing_region_set_config(config_kwargs)
+        self._compute_checksum_config(config_kwargs)
         s3_config = self.compute_s3_config(client_config)
 
         is_s3_service = self._is_s3_service(service_name)
@@ -588,4 +603,40 @@ def _compute_sigv4a_signing_region_set_config(self, config_kwargs):
             sigv4a_signing_region_set = self._config_store.get_config_variable(
                 'sigv4a_signing_region_set'
             )
-        config_kwargs['sigv4a_signing_region_set'] = sigv4a_signing_region_set
+        config_kwargs['sigv4a_signing_region_set'] = sigv4a_signing_region_set
+
+    def _compute_checksum_config(self, config_kwargs):
+        self._handle_checksum_config(
+            config_kwargs,
+            config_key="request_checksum_calculation",
+            default_value="when_supported",
+            valid_options=VALID_REQUEST_CHECKSUM_CALCULATION_CONFIG,
+        )
+        self._handle_checksum_config(
+            config_kwargs,
+            config_key="response_checksum_validation",
+            default_value="when_supported",
+            valid_options=VALID_RESPONSE_CHECKSUM_VALIDATION_CONFIG,
+        )
+
+    def _handle_checksum_config(
+        self,
+        config_kwargs,
+        config_key,
+        default_value,
+        valid_options,
+    ):
+        value = config_kwargs.get(config_key)
+        if value is None:
+            value = (
+                self._config_store.get_config_variable(config_key)
+                or default_value
+            )
+        value = value.lower()
+        if value not in valid_options:
+            raise botocore.exceptions.InvalidChecksumConfigError(
+                config_key=config_key,
+                config_value=value,
+                valid_options=valid_options,
+            )
+        config_kwargs[config_key] = value

awscli/botocore/config.py

@@ -194,6 +194,37 @@ class Config(object):
     :param sigv4a_signing_region_set: A set of AWS regions to apply the signature for
         when using SigV4a for signing. Set to ``*`` to represent all regions.
         Defaults to None.
+
+    :type request_checksum_calculation: str
+    :param request_checksum_calculation: Determines when a checksum will be
+        calculated for request payloads. Valid values are:
+
+        * ``when_supported`` -- When set, a checksum will be calculated for
+            all request payloads of operations modeled with the ``httpChecksum``
+            trait where ``requestChecksumRequired`` is ``true`` and/or a
+            ``requestAlgorithmMember`` is modeled.
+
+        * ``when_required`` -- When set, a checksum will only be calculated
+            for request payloads of operations modeled with the ``httpChecksum``
+            trait where ``requestChecksumRequired`` is ``true`` or where a
+            ``requestAlgorithmMember`` is modeled and supplied.
+
+        Defaults to None.
+
+    :type response_checksum_validation: str
+    :param response_checksum_validation: Determines when checksum validation
+        will be performed on response payloads. Valid values are:
+
+        * ``when_supported`` -- When set, checksum validation is performed on
+            all response payloads of operations modeled with the ``httpChecksum``
+            trait where ``responseAlgorithms`` is modeled, except when no modeled
+            checksum algorithms are supported.
+
+        * ``when_required`` -- When set, checksum validation is not performed
+            on response payloads of operations unless the checksum algorithm is
+            supported and the ``requestValidationModeMember`` member is set to ``ENABLED``.
+
+        Defaults to None.
     """
     OPTION_DEFAULTS = OrderedDict([
         ('region_name', None),
@@ -218,6 +249,8 @@ class Config(object):
         ('request_min_compression_size_bytes', None),
         ('disable_request_compression', None),
         ('sigv4a_signing_region_set', None),
+        ('request_checksum_calculation', None),
+        ('response_checksum_validation', None),
     ])
 
     def __init__(self, *args, **kwargs):

awscli/botocore/configprovider.py

@@ -137,6 +137,18 @@
         None,
         None,
     ),
+    'request_checksum_calculation': (
+        'request_checksum_calculation',
+        'AWS_REQUEST_CHECKSUM_CALCULATION',
+        "when_supported",
+        None,
+    ),
+    'response_checksum_validation': (
+        'response_checksum_validation',
+        'AWS_RESPONSE_CHECKSUM_VALIDATION',
+        "when_supported",
+        None,
+    ),
 }
 # A mapping for the s3 specific configuration vars. These are the configuration
 # vars that typically go in the s3 section of the config file. This mapping
@@ -336,7 +348,7 @@ def __copy__(self):
 
     def get_config_variable(self, logical_name):
         """
-        Retrieve the value associeated with the specified logical_name
+        Retrieve the value associated with the specified logical_name
         from the corresponding provider. If no value is found None will
         be returned.
 

awscli/botocore/exceptions.py

@@ -742,3 +742,12 @@ class EndpointResolutionError(EndpointProviderError):
 
 class UnknownEndpointResolutionBuiltInName(EndpointProviderError):
     fmt = 'Unknown builtin variable name: {name}'
+
+
+class InvalidChecksumConfigError(BotoCoreError):
+    """Error when invalid value supplied for checksum config"""
+
+    fmt = (
+        'Unsupported configuration value for {config_key}. '
+        'Expected one of {valid_options} but got {config_value}.'
+    )

awscli/botocore/handlers.py

@@ -60,8 +60,6 @@
 from botocore.utils import (
     SAFE_CHARS,
     ArnParser,
-    conditionally_calculate_checksum,
-    conditionally_calculate_md5,
     hyphenize_service_id,
     is_global_accesspoint,
     percent_encode,
@@ -1151,6 +1149,17 @@ def _update_status_code(response, **kwargs):
         http_response.status_code = parsed_status_code
 
 
+def handle_request_validation_mode_member(params, model, **kwargs):
+    client_config = kwargs.get("context", {}).get("client_config")
+    if client_config is None:
+        return
+    response_checksum_validation = client_config.response_checksum_validation
+    http_checksum = model.http_checksum
+    mode_member = http_checksum.get("requestValidationModeMember")
+    if mode_member and response_checksum_validation == "when_supported":
+        params.setdefault(mode_member, "ENABLED")
+
+
 # This is a list of (event_name, handler).
 # When a Session is created, everything in this list will be
 # automatically registered with that Session.
@@ -1177,7 +1186,7 @@ def _update_status_code(response, **kwargs):
     ('before-parse.s3.*', handle_expires_header),
     ('before-parse.s3.*', _handle_200_error, REGISTER_FIRST),
     ('before-parameter-build', generate_idempotent_uuid),
-
+    ('before-parameter-build', handle_request_validation_mode_member),
     ('before-parameter-build.s3', validate_bucket_name),
     ('before-parameter-build.s3', remove_bucket_from_url_paths_from_model),
 
@@ -1205,10 +1214,7 @@ def _update_status_code(response, **kwargs):
     ('before-call.s3', add_expect_header),
     ('before-call.glacier', add_glacier_version),
     ('before-call.api-gateway', add_accept_header),
-    ('before-call.s3.PutObject', conditionally_calculate_checksum),
-    ('before-call.s3.UploadPart', conditionally_calculate_md5),
     ('before-call.s3.DeleteObjects', escape_xml_payload),
-    ('before-call.s3.DeleteObjects', conditionally_calculate_checksum),
     ('before-call.s3.PutBucketLifecycleConfiguration', escape_xml_payload),
     ('before-call.glacier.UploadArchive', add_glacier_checksums),
     ('before-call.glacier.UploadMultipartPart', add_glacier_checksums),

awscli/botocore/httpchecksum.py

@@ -25,15 +25,15 @@
 from hashlib import sha1, sha256
 
 from awscrt import checksums as crt_checksums
+from botocore.compat import urlparse
 from botocore.exceptions import AwsChunkedWrapperError, FlexibleChecksumError
 from botocore.response import StreamingBody
-from botocore.utils import (
-    conditionally_calculate_md5,
-    determine_content_length,
-)
+from botocore.utils import determine_content_length, has_checksum_header
 
 logger = logging.getLogger(__name__)
 
+DEFAULT_CHECKSUM_ALGORITHM = "CRC32"
+
 
 class BaseChecksum:
     _CHUNK_SIZE = 1024 * 1024
@@ -246,7 +246,18 @@ def resolve_checksum_context(request, operation_model, params):
 def resolve_request_checksum_algorithm(
     request, operation_model, params, supported_algorithms=None,
 ):
+    # If the header is already set by the customer, skip calculation
+    if has_checksum_header(request):
+        return
+
+    request_checksum_calculation = request["context"][
+        "client_config"
+    ].request_checksum_calculation
     http_checksum = operation_model.http_checksum
+    request_checksum_required = (
+            operation_model.http_checksum_required
+            or http_checksum.get("requestChecksumRequired")
+    )
     algorithm_member = http_checksum.get("requestAlgorithmMember")
     if algorithm_member and algorithm_member in params:
         # If the client has opted into using flexible checksums and the
@@ -259,35 +270,32 @@ def resolve_request_checksum_algorithm(
             raise FlexibleChecksumError(
                 error_msg="Unsupported checksum algorithm: %s" % algorithm_name
             )
+    elif request_checksum_required or (
+            algorithm_member and request_checksum_calculation == "when_supported"
+    ):
+        algorithm_name = DEFAULT_CHECKSUM_ALGORITHM.lower()
+    else:
+        return
 
-        location_type = "header"
-        if operation_model.has_streaming_input:
-            # Operations with streaming input must support trailers.
-            if request["url"].startswith("https:"):
-                # We only support unsigned trailer checksums currently. As this
-                # disables payload signing we'll only use trailers over TLS.
-                location_type = "trailer"
-
-        algorithm = {
-            "algorithm": algorithm_name,
-            "in": location_type,
-            "name": "x-amz-checksum-%s" % algorithm_name,
-        }
+    location_type = "header"
+    if (
+            operation_model.has_streaming_input
+            and urlparse(request["url"]).scheme == "https"
+    ):
+        # Operations with streaming input must support trailers.
+        # We only support unsigned trailer checksums currently. As this
+        # disables payload signing we'll only use trailers over TLS.
+        location_type = "trailer"
 
-        if algorithm["name"] in request["headers"]:
-            # If the header is already set by the customer, skip calculation
-            return
+    algorithm = {
+        "algorithm": algorithm_name,
+        "in": location_type,
+        "name": f"x-amz-checksum-{algorithm_name}",
+    }
 
-        checksum_context = request["context"].get("checksum", {})
-        checksum_context["request_algorithm"] = algorithm
-        request["context"]["checksum"] = checksum_context
-    elif operation_model.http_checksum_required or http_checksum.get(
-        "requestChecksumRequired"
-    ):
-        # Otherwise apply the old http checksum behavior via Content-MD5
-        checksum_context = request["context"].get("checksum", {})
-        checksum_context["request_algorithm"] = "conditional-md5"
-        request["context"]["checksum"] = checksum_context
+    checksum_context = request["context"].get("checksum", {})
+    checksum_context["request_algorithm"] = algorithm
+    request["context"]["checksum"] = checksum_context
 
 
 def apply_request_checksum(request):
@@ -297,10 +305,7 @@ def apply_request_checksum(request):
     if not algorithm:
         return
 
-    if algorithm == "conditional-md5":
-        # Special case to handle the http checksum required trait
-        conditionally_calculate_md5(request)
-    elif algorithm["in"] == "header":
+    if algorithm["in"] == "header":
         _apply_request_header_checksum(request)
     elif algorithm["in"] == "trailer":
         _apply_request_trailer_checksum(request)