From 6174c38b69937575cb928b6f14b7b1740d76480d Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 24 Mar 2025 16:24:37 -0700 Subject: [PATCH 1/4] Updating simple_httpclient.py to support asynchronous streaming_callback cases. --- tornado/httpclient.py | 4 ++-- tornado/simple_httpclient.py | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tornado/httpclient.py b/tornado/httpclient.py index 3a45ffd041..c37e16f2c0 100644 --- a/tornado/httpclient.py +++ b/tornado/httpclient.py @@ -53,7 +53,7 @@ from tornado.ioloop import IOLoop from tornado.util import Configurable -from typing import Type, Any, Union, Dict, Callable, Optional, cast +from typing import Type, Any, Union, Dict, Callable, Optional, Awaitable, cast class HTTPClient: @@ -372,7 +372,7 @@ def __init__( user_agent: Optional[str] = None, use_gzip: Optional[bool] = None, network_interface: Optional[str] = None, - streaming_callback: Optional[Callable[[bytes], None]] = None, + streaming_callback: Optional[Callable[[bytes], Optional[Awaitable[None]]]] = None, header_callback: Optional[Callable[[str], None]] = None, prepare_curl_callback: Optional[Callable[[Any], None]] = None, proxy_host: Optional[str] = None, diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py index cc16376133..5705152666 100644 --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py @@ -33,7 +33,7 @@ from io import BytesIO import urllib.parse -from typing import Dict, Any, Callable, Optional, Type, Union +from typing import Dict, Any, Callable, Optional, Type, Union, Awaitable from types import TracebackType import typing @@ -687,14 +687,15 @@ def finish(self) -> None: def _on_end_request(self) -> None: self.stream.close() - def data_received(self, chunk: bytes) -> None: + def data_received(self, chunk: bytes) -> Optional[Awaitable[None]]: if self._should_follow_redirect(): # We're going to follow a redirect so just discard the body. - return - if self.request.streaming_callback is not None: - self.request.streaming_callback(chunk) + return None + if self.request.streaming_callback is not None: + return self.request.streaming_callback(chunk) else: self.chunks.append(chunk) + return None if __name__ == "__main__": From 8249608c9b7b7a3dfac51ef9c26f023c058ec0d6 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 24 Mar 2025 16:38:50 -0700 Subject: [PATCH 2/4] Add test to verify async streaming_callback signatures. --- tornado/test/simple_httpclient_test.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tornado/test/simple_httpclient_test.py b/tornado/test/simple_httpclient_test.py index a40435e812..82620a49e9 100644 --- a/tornado/test/simple_httpclient_test.py +++ b/tornado/test/simple_httpclient_test.py @@ -539,6 +539,26 @@ def test_streaming_follow_redirects(self): num_start_lines = len([h for h in headers if h.startswith("HTTP/")]) self.assertEqual(num_start_lines, 1) + def test_streaming_callback_coroutine(self: typing.Any): + headers = [] + chunk_bytes = [] + + @gen.coroutine + def _put_chunk(chunk): + chunk_bytes.append(chunk) + yield gen.moment + + self.fetch( + "/hello", + header_callback=headers.append, + streaming_callback=_put_chunk, + ) + chunks = list(map(to_unicode, chunk_bytes)) + self.assertEqual(chunks, ["Hello world!"]) + # Make sure we only got one set of headers. + num_start_lines = len([h for h in headers if h.startswith("HTTP/")]) + self.assertEqual(num_start_lines, 1) + class SimpleHTTPClientTestCase(AsyncHTTPTestCase, SimpleHTTPClientTestMixin): def setUp(self): From fe2588224bd0ea8e4429ec3a464ba15de2106caa Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 31 Mar 2025 14:35:20 -0700 Subject: [PATCH 3/4] Formatting fixes to shrink longer lines. --- tornado/httpclient.py | 4 +++- tornado/simple_httpclient.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tornado/httpclient.py b/tornado/httpclient.py index c37e16f2c0..488fe6de0b 100644 --- a/tornado/httpclient.py +++ b/tornado/httpclient.py @@ -372,7 +372,9 @@ def __init__( user_agent: Optional[str] = None, use_gzip: Optional[bool] = None, network_interface: Optional[str] = None, - streaming_callback: Optional[Callable[[bytes], Optional[Awaitable[None]]]] = None, + streaming_callback: Optional[ + Callable[[bytes], Optional[Awaitable[None]]] + ] = None, header_callback: Optional[Callable[[str], None]] = None, prepare_curl_callback: Optional[Callable[[Any], None]] = None, proxy_host: Optional[str] = None, diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py index 5705152666..5ed273db3e 100644 --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py @@ -691,7 +691,7 @@ def data_received(self, chunk: bytes) -> Optional[Awaitable[None]]: if self._should_follow_redirect(): # We're going to follow a redirect so just discard the body. return None - if self.request.streaming_callback is not None: + if self.request.streaming_callback is not None: return self.request.streaming_callback(chunk) else: self.chunks.append(chunk) From f9306ef9db153ddd8656cc9dd3e8902a4f878bd2 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 1 Apr 2025 20:52:04 -0700 Subject: [PATCH 4/4] Add typing annotations for new test. --- tornado/test/simple_httpclient_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tornado/test/simple_httpclient_test.py b/tornado/test/simple_httpclient_test.py index 82620a49e9..c1ee49c841 100644 --- a/tornado/test/simple_httpclient_test.py +++ b/tornado/test/simple_httpclient_test.py @@ -540,8 +540,8 @@ def test_streaming_follow_redirects(self): self.assertEqual(num_start_lines, 1) def test_streaming_callback_coroutine(self: typing.Any): - headers = [] - chunk_bytes = [] + headers = [] # type: typing.List[str] + chunk_bytes = [] # type: typing.List[bytes] @gen.coroutine def _put_chunk(chunk):