diff --git a/scrapy_playwright/_utils.py b/scrapy_playwright/_utils.py index 07984ace..a511dff7 100644 --- a/scrapy_playwright/_utils.py +++ b/scrapy_playwright/_utils.py @@ -7,6 +7,7 @@ import scrapy from playwright.async_api import Error, Page, Request, Response from scrapy.http.headers import Headers +from scrapy.settings import Settings from scrapy.utils.python import to_unicode from twisted.internet.defer import Deferred from w3lib.encoding import html_body_declared_encoding, http_content_type_encoding @@ -85,6 +86,13 @@ async def _get_page_content( raise +def _get_float_setting(settings: Settings, key: str) -> Optional[float]: + try: + return float(settings[key]) + except Exception: + return None + + async def _get_header_value( resource: Union[Request, Response], header_name: str, diff --git a/scrapy_playwright/handler.py b/scrapy_playwright/handler.py index 1287a22c..84d4ac3a 100644 --- a/scrapy_playwright/handler.py +++ b/scrapy_playwright/handler.py @@ -36,6 +36,7 @@ _ThreadedLoopAdapter, _deferred_from_coro, _encode_body, + _get_float_setting, _get_header_value, _get_page_content, _is_safe_close_error, @@ -73,7 +74,7 @@ class Config: max_pages_per_context: int max_contexts: Optional[int] startup_context_kwargs: dict - navigation_timeout: Optional[float] = None + navigation_timeout: Optional[float] @classmethod def from_settings(cls, settings: Settings) -> "Config": @@ -85,15 +86,15 @@ def from_settings(cls, settings: Settings) -> "Config": max_pages_per_context=settings.getint("PLAYWRIGHT_MAX_PAGES_PER_CONTEXT"), max_contexts=settings.getint("PLAYWRIGHT_MAX_CONTEXTS") or None, startup_context_kwargs=settings.getdict("PLAYWRIGHT_CONTEXTS"), + navigation_timeout=_get_float_setting( + settings, "PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT" + ), ) cfg.cdp_kwargs.pop("endpoint_url", None) if not cfg.max_pages_per_context: cfg.max_pages_per_context = settings.getint("CONCURRENT_REQUESTS") if cfg.cdp_url and cfg.launch_options: logger.warning("PLAYWRIGHT_CDP_URL is set, ignoring PLAYWRIGHT_LAUNCH_OPTIONS") - if "PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT" in settings: - with suppress(TypeError, ValueError): - cfg.navigation_timeout = float(settings["PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT"]) return cfg diff --git a/tests/tests_asyncio/test_utils.py b/tests/tests_asyncio/test_utils.py index 6187aa22..09ce238f 100644 --- a/tests/tests_asyncio/test_utils.py +++ b/tests/tests_asyncio/test_utils.py @@ -1,4 +1,5 @@ import logging +from decimal import Decimal from unittest import IsolatedAsyncioTestCase from unittest.mock import AsyncMock @@ -6,10 +7,11 @@ from playwright.async_api import Error as PlaywrightError from scrapy import Spider from scrapy.http.headers import Headers - +from scrapy.settings import Settings from scrapy_playwright._utils import ( _NAVIGATION_ERROR_MSG, _encode_body, + _get_float_setting, _get_header_value, _get_page_content, _maybe_await, @@ -122,20 +124,19 @@ async def test_encode_mismatch(self): class TestHeaderValue(IsolatedAsyncioTestCase): - async def test_get_header_ok(self): + async def test_get_header_value(self): async def _identity(x): return x - resource = AsyncMock() - resource.header_value = _identity - assert "asdf" == await _get_header_value(resource, "asdf") - assert "qwerty" == await _get_header_value(resource, "qwerty") + res1 = AsyncMock() + res1.header_value = _identity + assert "asdf" == await _get_header_value(res1, "asdf") + assert "qwerty" == await _get_header_value(res1, "qwerty") - async def test_get_header_exception(self): - resource = AsyncMock() - resource.header_value.side_effect = Exception("nope") - assert await _get_header_value(resource, "asdf") is None - assert await _get_header_value(resource, "qwerty") is None + res2 = AsyncMock() + res2.header_value.side_effect = Exception("nope") + assert await _get_header_value(res2, "asdf") is None + assert await _get_header_value(res2, "qwerty") is None class TestMaybeAwait(IsolatedAsyncioTestCase): @@ -149,3 +150,28 @@ async def _awaitable_identity(x): assert await _maybe_await("foo") == "foo" assert await _maybe_await("bar") == "bar" assert await _maybe_await(1234) == 1234 + + +class TestGetFloatSetting(IsolatedAsyncioTestCase): + async def test_get_float_setting(self): + settings = Settings( + { + "ZERO": 0, + "FLOAT": 1.5, + "DECIMAL": Decimal("2.5"), + "INT": 3, + "NUMERIC_STRING": "123", + "NON_NUMERIC_STRING": "asdf", + "NONE": None, + "LIST": [1, 2, 3], + } + ) + assert _get_float_setting(settings, "ZERO") == 0.0 + assert _get_float_setting(settings, "FLOAT") == 1.5 + assert _get_float_setting(settings, "DECIMAL") == 2.5 + assert _get_float_setting(settings, "INT") == 3.0 + assert _get_float_setting(settings, "NUMERIC_STRING") == 123 + assert _get_float_setting(settings, "NON_NUMERIC_STRING") is None + assert _get_float_setting(settings, "NONE") is None + assert _get_float_setting(settings, "LIST") is None + assert _get_float_setting(settings, "MISSING_KEY") is None