❇️ Added complete support for Informational Response whether it's an …

…early response or not (#151)

CHANGES.rst

@@ -1,3 +1,11 @@
+2.10.900 (2024-09-??)
+=====================
+
+- Added complete support for Informational Response whether it's an early response or not. We introduced a callback named
+  ``on_early_response`` that takes exactly one parameter, namely a ``HTTPResponse``. You may start leveraging Early Hints!
+  This works regardless of the negotiated protocol: HTTP/1.1, HTTP/2 or HTTP/3! As always, you may use that feature
+  in a synchronous or asynchronous context.
+
 2.9.900 (2024-09-24)
 ====================
 

README.md

@@ -28,6 +28,7 @@
 - Helpers for retrying requests and dealing with HTTP redirects.
 - Support for gzip, deflate, brotli, and zstd encoding.
 - Support for Python/PyPy 3.7+, no compromise.
+- Early (Informational) Responses / Hints.
 - HTTP/1.1, HTTP/2 and HTTP/3 support.
 - Proxy support for HTTP and SOCKS.
 - Post-Quantum Security with QUIC.

docs/advanced-usage.rst

@@ -1517,3 +1517,42 @@ Here is a simple example::
 .. note:: The property ``trailers`` return either ``None`` or a fully constructed ``HTTPHeaderDict``.
 
 .. warning:: ``None`` means we did not receive trailer headers (yet). If ``preload_content`` is set to False, you will need to consume the entire body before reaching the ``trailers`` property.
+
+Informational / Early Responses
+-------------------------------
+
+.. note:: Available since version 2.10+
+
+Sometimes, thanks to HTTP standards, a webserver may send intermediary responses before the main one for
+a particular request.
+
+All others HTTP client swallow them silently and you cannot see them. Thanks to urllib3-future, that
+issue is a thing of the past.
+
+You may now inspect early responses like https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/100 (Continue) or
+https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/103 (Early Hint) with a mere callback.
+
+Fastly provide a test server that can helps you see that feature in action::
+
+    from urllib3 import PoolManager, HTTPResponse
+
+    early_response: HTTPResponse | None = None
+
+    def my_callback(caught_response: HTTPResponse) -> None:
+        nonlocal early_response
+        early_response = caught_response
+
+    with PoolManager() as pm:
+        response = pm.urlopen("GET", "https://early-hints.fastlylabs.com/", on_early_response=my_callback)
+
+    print(early_response.status)  # 103
+    print(early_response.headers) # HTTPHeaderDict({"Link": "...
+    print(response.status)  # 200
+
+This example works whether you enable manual multiplexing or using asyncio.
+
+.. warning:: Some webservers may enable that feature on HTTP/2+ and disable it in HTTP/1.1. In urllib3-future case, that feature is available no matter the used protocol.
+
+.. note:: The status 101 (Switching Protocol) is never considered "early", therefor "response" will be the 101 one and "early_response" will be worth "None".
+
+.. note:: Any responses yielded through the "on_early_response" callback will never have a body per standards.

src/urllib3/_async/connection.py

@@ -21,6 +21,7 @@
         _TYPE_ASYNC_BODY,
     )
     from ..util._async.traffic_police import AsyncTrafficPolice
+    from ..backend._async._base import AsyncLowLevelResponse
 
 from .._constant import DEFAULT_BLOCKSIZE
 from ..util.timeout import _DEFAULT_TIMEOUT, Timeout
@@ -530,6 +531,10 @@ async def getresponse(  # type: ignore[override]
         *,
         promise: ResponsePromise | None = None,
         police_officer: AsyncTrafficPolice[AsyncHTTPConnection] | None = None,
+        early_response_callback: typing.Callable[
+            [AsyncHTTPResponse], typing.Awaitable[None]
+        ]
+        | None = None,
     ) -> AsyncHTTPResponse:
         """
         Get the response from the server.
@@ -546,11 +551,58 @@ async def getresponse(  # type: ignore[override]
         # we need to set the timeout on the socket.
         self.sock.settimeout(self.timeout)
 
+        async def early_response_handler(
+            early_low_response: AsyncLowLevelResponse,
+        ) -> None:
+            """Handle unexpected early response. Notify the upper stack!"""
+            nonlocal promise, early_response_callback
+
+            _promise = None
+
+            if promise is None:
+                _promise = early_low_response.from_promise
+            else:
+                _promise = promise
+
+            if _promise is None:
+                raise OSError
+
+            if early_response_callback is None:
+                early_response_callback = _promise.get_parameter("on_early_response")
+
+            if early_response_callback is None:
+                return
+
+            early_resp_options: _ResponseOptions = _promise.get_parameter(  # type: ignore[assignment]
+                "response_options"
+            )
+
+            early_response = AsyncHTTPResponse(
+                body=b"",
+                headers=early_low_response.msg,
+                status=early_low_response.status,
+                version=early_low_response.version,
+                reason=early_low_response.reason,
+                preload_content=False,
+                decode_content=early_resp_options.decode_content,
+                original_response=early_low_response,
+                enforce_content_length=False,
+                request_method=early_resp_options.request_method,
+                request_url=early_resp_options.request_url,
+                connection=None,
+                police_officer=None,
+            )
+
+            await early_response_callback(early_response)
+
         # This is needed here to avoid circular import errors
         from .response import AsyncHTTPResponse
 
         # Get the response from backend._base.BaseBackend
-        low_response = await super().getresponse(promise=promise)
+        low_response = await super().getresponse(
+            promise=promise,
+            early_response_callback=early_response_handler,
+        )
 
         if promise is None:
             promise = low_response.from_promise

src/urllib3/_async/connectionpool.py

@@ -871,6 +871,8 @@ async def _make_request(
         on_upload_body: typing.Callable[
             [int, int | None, bool, bool], typing.Awaitable[None]
         ] = ...,
+        on_early_response: typing.Callable[[AsyncHTTPResponse], typing.Awaitable[None]]
+        | None = ...,
         *,
         multiplexed: Literal[True],
     ) -> ResponsePromise:
@@ -896,6 +898,8 @@ async def _make_request(
         on_upload_body: typing.Callable[
             [int, int | None, bool, bool], typing.Awaitable[None]
         ] = ...,
+        on_early_response: typing.Callable[[AsyncHTTPResponse], typing.Awaitable[None]]
+        | None = ...,
         *,
         multiplexed: Literal[False] = ...,
     ) -> AsyncHTTPResponse:
@@ -921,6 +925,8 @@ async def _make_request(
             [int, int | None, bool, bool], typing.Awaitable[None]
         ]
         | None = None,
+        on_early_response: typing.Callable[[AsyncHTTPResponse], typing.Awaitable[None]]
+        | None = None,
         multiplexed: Literal[False] | Literal[True] = False,
     ) -> AsyncHTTPResponse | ResponsePromise:
         """
@@ -1073,6 +1079,7 @@ async def _make_request(
             if rp is None:
                 raise OSError
             rp.set_parameter("read_timeout", read_timeout)
+            rp.set_parameter("on_early_response", on_early_response)
             return rp
 
         if not conn.is_closed:
@@ -1093,7 +1100,9 @@ async def _make_request(
 
         # Receive the response from the server
         try:
-            response = await conn.getresponse(police_officer=self.pool)
+            response = await conn.getresponse(
+                police_officer=self.pool, early_response_callback=on_early_response
+            )
         except (BaseSSLError, OSError) as e:
             self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
             raise
@@ -1178,6 +1187,8 @@ async def urlopen(
         on_upload_body: typing.Callable[
             [int, int | None, bool, bool], typing.Awaitable[None]
         ] = ...,
+        on_early_response: typing.Callable[[AsyncHTTPResponse], typing.Awaitable[None]]
+        | None = ...,
         *,
         multiplexed: Literal[False] = ...,
         **response_kw: typing.Any,
@@ -1206,6 +1217,8 @@ async def urlopen(
         on_upload_body: typing.Callable[
             [int, int | None, bool, bool], typing.Awaitable[None]
         ] = ...,
+        on_early_response: typing.Callable[[AsyncHTTPResponse], typing.Awaitable[None]]
+        | None = ...,
         *,
         multiplexed: Literal[True],
         **response_kw: typing.Any,
@@ -1234,6 +1247,8 @@ async def urlopen(
             [int, int | None, bool, bool], typing.Awaitable[None]
         ]
         | None = None,
+        on_early_response: typing.Callable[[AsyncHTTPResponse], typing.Awaitable[None]]
+        | None = None,
         multiplexed: bool = False,
         **response_kw: typing.Any,
     ) -> AsyncHTTPResponse | ResponsePromise:
@@ -1342,6 +1357,12 @@ async def urlopen(
             available, thus set to None. In order, arguments are:
             (total_sent, total_to_be_sent, completed, any_error)
 
+        :param on_early_response:
+            Callable that will be invoked upon early responses, can be invoked one or several times.
+            All informational responses except HTTP/102 (Switching Protocol) are concerned here.
+            The callback takes only one positional argument, the fully constructed HTTPResponse.
+            Those responses never have bodies, only headers.
+
         :param multiplexed:
             Dispatch the request in a non-blocking way, this means that the
             response will be retrieved in the future with the get_response()
@@ -1448,6 +1469,7 @@ async def urlopen(
                 enforce_content_length=True,
                 on_post_connection=on_post_connection,
                 on_upload_body=on_upload_body,
+                on_early_response=on_early_response,
                 multiplexed=multiplexed,
             )
 

src/urllib3/_version.py

@@ -1,4 +1,4 @@
 # This file is protected via CODEOWNERS
 from __future__ import annotations
 
-__version__ = "2.9.900"
+__version__ = "2.10.900"

src/urllib3/backend/_async/hface.py

@@ -30,6 +30,7 @@
 from ...contrib.hface.events import (
     ConnectionTerminated,
     DataReceived,
+    EarlyHeadersReceived,
     Event,
     HandshakeCompleted,
     HeadersReceived,
@@ -1075,19 +1076,31 @@ async def __read_st(
         )
 
     async def getresponse(  # type: ignore[override]
-        self, *, promise: ResponsePromise | None = None
+        self,
+        *,
+        promise: ResponsePromise | None = None,
+        early_response_callback: typing.Callable[
+            [AsyncLowLevelResponse], typing.Awaitable[None]
+        ]
+        | None = None,
     ) -> AsyncLowLevelResponse:
         if self.sock is None or self._protocol is None or not self._promises:
             raise ResponseNotReady()  # Defensive: Comply with http.client, actually tested but not reported?
 
         headers = HTTPHeaderDict()
         status: int | None = None
 
-        head_event: HeadersReceived = (
+        head_event: HeadersReceived | EarlyHeadersReceived = (
             await self.__exchange_until(  # type: ignore[assignment]
-                HeadersReceived,
+                (
+                    HeadersReceived,
+                    EarlyHeadersReceived,
+                ),
                 receive_first=True,
-                event_type_collectable=(HeadersReceived,),
+                event_type_collectable=(
+                    HeadersReceived,
+                    EarlyHeadersReceived,
+                ),
                 respect_end_stream_signal=False,
                 stream_id=promise.stream_id if promise else None,
             )
@@ -1101,15 +1114,24 @@ async def getresponse(  # type: ignore[override]
             else:
                 headers.add(raw_header.decode("ascii"), raw_value.decode("iso-8859-1"))
 
+        # 101 = Switching Protocol! It's our final HTTP response, but the stream remains open!
+        is_early_response = (
+            isinstance(head_event, EarlyHeadersReceived) and status != 101
+        )
+
         if promise is None:
             try:
-                promise = self._promises_per_stream.pop(head_event.stream_id)
-            except KeyError:
+                if is_early_response:
+                    promise = self._promises_per_stream[head_event.stream_id]
+                else:
+                    promise = self._promises_per_stream.pop(head_event.stream_id)
+            except KeyError as e:
                 raise ProtocolError(
                     f"Response received (stream: {head_event.stream_id}) but no promise in-flight"
-                )
+                ) from e
         else:
-            del self._promises_per_stream[promise.stream_id]
+            if not is_early_response:
+                del self._promises_per_stream[promise.stream_id]
 
         # this should be unreachable
         if status is None:
@@ -1130,6 +1152,27 @@ async def getresponse(  # type: ignore[override]
         except KeyError:
             reason = "Unknown"
 
+        if is_early_response:
+            if early_response_callback is not None:
+                early_response = AsyncLowLevelResponse(
+                    http_verb.decode("ascii"),
+                    status,
+                    self._http_vsn,
+                    reason,
+                    headers,
+                    body=None,
+                    authority=self.host,
+                    port=self.port,
+                    stream_id=promise.stream_id,
+                )
+                early_response.from_promise = promise
+
+                await early_response_callback(early_response)
+
+            return await AsyncHfaceBackend.getresponse(
+                self, promise=promise, early_response_callback=early_response_callback
+            )
+
         self._response = AsyncLowLevelResponse(
             http_verb.decode("ascii"),
             status,
@@ -1196,7 +1239,9 @@ async def send(  # type: ignore[override]
                     self._protocol.bytes_received(await self.sock.recv(self.blocksize))
 
                     # this is a bad sign. we should stop sending and instead retrieve the response.
-                    if self._protocol.has_pending_event(stream_id=self._stream_id):
+                    if self._protocol.has_pending_event(
+                        stream_id=self._stream_id, excl_event=(EarlyHeadersReceived,)
+                    ):
                         if self._start_last_request and self.conn_info:
                             self.conn_info.request_sent_latency = (
                                 datetime.now(tz=timezone.utc) - self._start_last_request