diff --git a/Makefile b/Makefile index 545b70d9..98299272 100644 --- a/Makefile +++ b/Makefile @@ -22,13 +22,13 @@ for line in sys.stdin: endef export PRINT_HELP_PYSCRIPT BROWSER := python -c "$$BROWSER_PYSCRIPT" +TESTS ?= tests help: @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts - clean-build: ## remove build artifacts rm -fr build/ rm -fr dist/ @@ -51,9 +51,8 @@ lint: ## check style with ruff and black pdm run ruff check src/ tests bench pdm run black --check src tests docs/conf.py -test: ## run tests quickly with the default Python - pdm run pytest -x --ff -n auto tests - +test: ## run tests quickly with the default Python; pass TESTS= for specific path + pdm run pytest -x --ff $(if $(filter $(TESTS),tests),-n auto ,)$(TESTS) test-all: ## run tests on every Python version with tox tox diff --git a/pyproject.toml b/pyproject.toml index fb165065..a7bed01e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,17 +139,18 @@ select = [ "I", # isort ] ignore = [ - "E501", # line length is handled by black - "RUF001", # leave my smart characters alone - "S101", # assert - "S307", # hands off my eval - "SIM300", # Yoda rocks in asserts - "PGH003", # leave my type: ignores alone - "B006", # mutable argument defaults - "DTZ001", # datetimes in tests - "DTZ006", # datetimes in tests - "UP006", # We support old typing constructs at runtime - "UP035", # We support old typing constructs at runtime + "B006", # mutable argument defaults + "DTZ001", # datetimes in tests + "DTZ006", # datetimes in tests + "E501", # line length is handled by black + "PGH003", # leave my type: ignores alone + "PLC0414", # redundant import aliases indicate exported names + "RUF001", # leave my smart characters alone + "S101", # assert + "S307", # hands off my eval + "SIM300", # Yoda rocks in asserts + "UP006", # We support old typing constructs at runtime + "UP035", # We support old typing constructs at runtime ] [tool.ruff.lint.pyupgrade] diff --git a/src/cattrs/preconf/__init__.py b/src/cattrs/preconf/__init__.py index 1b12ef93..adfd0e0c 100644 --- a/src/cattrs/preconf/__init__.py +++ b/src/cattrs/preconf/__init__.py @@ -6,6 +6,9 @@ from .._compat import is_subclass from ..converters import Converter, UnstructureHook from ..fns import identity +from ._all import ConverterFormat as ConverterFormat +from ._all import PreconfiguredConverter as PreconfiguredConverter +from ._all import has_format as has_format if sys.version_info[:2] < (3, 10): from typing_extensions import ParamSpec diff --git a/src/cattrs/preconf/_all.py b/src/cattrs/preconf/_all.py new file mode 100644 index 00000000..75756e24 --- /dev/null +++ b/src/cattrs/preconf/_all.py @@ -0,0 +1,151 @@ +from collections.abc import Sequence +from typing import TYPE_CHECKING, Literal, TypeAlias, TypeIs, Union, overload + +from ..converters import Converter +from ..types import Unavailable + +if TYPE_CHECKING: + try: + from cattrs.preconf.bson import BsonConverter + except ModuleNotFoundError: + BsonConverter = Unavailable + + try: + from cattrs.preconf.cbor2 import Cbor2Converter + except ModuleNotFoundError: + Cbor2Converter = Unavailable + + from cattrs.preconf.json import JsonConverter + + try: + from cattrs.preconf.msgpack import MsgpackConverter + except ModuleNotFoundError: + MsgpackConverter = Unavailable + + try: + from cattrs.preconf.msgspec import MsgspecJsonConverter + except ModuleNotFoundError: + MsgspecJsonConverter = Unavailable + + try: + from cattrs.preconf.orjson import OrjsonConverter + except ModuleNotFoundError: + OrjsonConverter = Unavailable + + try: + from cattrs.preconf.pyyaml import PyyamlConverter + except ModuleNotFoundError: + PyyamlConverter = Unavailable + + try: + from cattrs.preconf.tomlkit import TomlkitConverter + except ModuleNotFoundError: + TomlkitConverter = Unavailable + + try: + from cattrs.preconf.ujson import UjsonConverter + except ModuleNotFoundError: + UjsonConverter = Unavailable + + PreconfiguredConverter: TypeAlias = Union[ + BsonConverter, + Cbor2Converter, + JsonConverter, + MsgpackConverter, + MsgspecJsonConverter, + OrjsonConverter, + PyyamlConverter, + TomlkitConverter, + UjsonConverter, + ] + +else: + PreconfiguredConverter: TypeAlias = Converter + +ConverterFormat: TypeAlias = Literal[ + "bson", + "cbor2", + "json", + "msgpack", + "msgspec-json", + "orjson", + "pyyaml", + "tomlkit", + "ujson", +] + +C: TypeAlias = Converter | Unavailable + + +@overload +def has_format(converter: C, fmt: Literal["bson"]) -> TypeIs["BsonConverter"]: ... +@overload +def has_format(converter: C, fmt: Literal["cbor2"]) -> TypeIs["Cbor2Converter"]: ... +@overload +def has_format(converter: C, fmt: Literal["json"]) -> TypeIs["JsonConverter"]: ... +@overload +def has_format(converter: C, fmt: Literal["msgpack"]) -> TypeIs["MsgpackConverter"]: ... +@overload +def has_format( + converter: C, fmt: Literal["msgspec-json"] +) -> TypeIs["MsgspecJsonConverter"]: ... +@overload +def has_format(converter: C, fmt: Literal["orjson"]) -> TypeIs["OrjsonConverter"]: ... +@overload +def has_format(converter: C, fmt: Literal["pyyaml"]) -> TypeIs["PyyamlConverter"]: ... +@overload +def has_format(converter: C, fmt: Literal["tomlkit"]) -> TypeIs["TomlkitConverter"]: ... +@overload +def has_format(converter: C, fmt: Literal["ujson"]) -> TypeIs["UjsonConverter"]: ... +def has_format( + converter: C, fmt: ConverterFormat | str | Sequence[ConverterFormat] +) -> bool: + if isinstance(fmt, str): + fmt = (fmt,) + + if "bson" in fmt and converter.__class__.__name__ == "BsonConverter": + from .bson import BsonConverter + + return isinstance(converter, BsonConverter) + + if "cbor2" in fmt and converter.__class__.__name__ == "Cbor2Converter": + from .cbor2 import Cbor2Converter + + return isinstance(converter, Cbor2Converter) + + if "json" in fmt and converter.__class__.__name__ == "JsonConverter": + from .json import JsonConverter + + return isinstance(converter, JsonConverter) + + if "msgpack" in fmt and converter.__class__.__name__ == "MsgpackConverter": + from .msgpack import MsgpackConverter + + return isinstance(converter, MsgpackConverter) + + if "msgspec-json" in fmt and converter.__class__.__name__ == "MsgspecJsonConverter": + from .msgspec import MsgspecJsonConverter + + return isinstance(converter, MsgspecJsonConverter) + + if "orjson" in fmt and converter.__class__.__name__ == "OrjsonConverter": + from .orjson import OrjsonConverter + + return isinstance(converter, OrjsonConverter) + + if "pyyaml" in fmt and converter.__class__.__name__ == "PyyamlConverter": + from .pyyaml import PyyamlConverter + + return isinstance(converter, PyyamlConverter) + + if "tomlkit" in fmt and converter.__class__.__name__ == "TomlkitConverter": + from .tomlkit import TomlkitConverter + + return isinstance(converter, TomlkitConverter) + + if "ujson" in fmt and converter.__class__.__name__ == "UjsonConverter": + from .ujson import UjsonConverter + + return isinstance(converter, UjsonConverter) + + return False diff --git a/src/cattrs/strategies/__init__.py b/src/cattrs/strategies/__init__.py index 9caf0732..a95a2396 100644 --- a/src/cattrs/strategies/__init__.py +++ b/src/cattrs/strategies/__init__.py @@ -1,6 +1,7 @@ """High level strategies for converters.""" from ._class_methods import use_class_methods +from ._extra_types import register_extra_types from ._subclasses import include_subclasses from ._unions import configure_tagged_union, configure_union_passthrough @@ -8,5 +9,6 @@ "configure_tagged_union", "configure_union_passthrough", "include_subclasses", + "register_extra_types", "use_class_methods", ] diff --git a/src/cattrs/strategies/_extra_types/__init__.py b/src/cattrs/strategies/_extra_types/__init__.py new file mode 100644 index 00000000..4ac46fc7 --- /dev/null +++ b/src/cattrs/strategies/_extra_types/__init__.py @@ -0,0 +1,53 @@ +from functools import cache, wraps +from importlib import import_module +from types import ModuleType +from typing import NoReturn + +from ...converters import Converter +from ...dispatch import StructuredValue, StructureHook, TargetType, UnstructuredValue + + +def register_extra_types(converter: Converter, *classes: type) -> None: + """ + TODO: Add docs + """ + for cl in classes: + if not isinstance(cl, type): + raise TypeError("Type required instead of object") + + struct_hook = get_module(cl).gen_structure_hook(cl, converter) + if struct_hook is None: + raise_unsupported(cl) + converter.register_structure_hook(cl, bypass(cl, struct_hook)) + + unstruct_hook = get_module(cl).gen_unstructure_hook(cl, converter) + if unstruct_hook is None: + raise_unsupported(cl) + converter.register_unstructure_hook(cl, unstruct_hook) + + +def bypass(target: type, structure_hook: StructureHook) -> StructureHook: + """Bypass structure hook when given object of target type.""" + + @wraps(structure_hook) + def wrapper(obj: UnstructuredValue, cl: TargetType) -> StructuredValue: + return obj if type(obj) is target else structure_hook(obj, cl) + + return wrapper + + +@cache +def get_module(cl: type) -> ModuleType: + modname = getattr(cl, "__module__", "builtins") + try: + return import_module(f"cattrs.strategies._extra_types._{modname}") + except ModuleNotFoundError: + raise_unsupported(cl) + + +def raise_unexpected_structure(target: type, cl: type) -> NoReturn: + raise TypeError(f"Unable to structure registered extra type {target} from {cl}") + + +def raise_unsupported(cl: type) -> NoReturn: + raise ValueError(f"Type {cl} is not supported by register_extra_types strategy") diff --git a/src/cattrs/strategies/_extra_types/_builtins.py b/src/cattrs/strategies/_extra_types/_builtins.py new file mode 100644 index 00000000..b35b6afb --- /dev/null +++ b/src/cattrs/strategies/_extra_types/_builtins.py @@ -0,0 +1,54 @@ +from collections.abc import Sequence +from contextlib import suppress +from functools import cache, partial +from numbers import Real + +from ...converters import Converter +from ...dispatch import StructureHook, UnstructureHook +from ...preconf import has_format +from . import raise_unexpected_structure + +MISSING_SPECIAL_FLOATS = ("msgspec-json", "orjson") + +SPECIAL = (float("inf"), float("-inf"), float("nan")) +SPECIAL_STR = ("inf", "+inf", "-inf", "infinity", "+infinity", "-infinity", "nan") + + +@cache +def gen_structure_hook(cl: type, _) -> StructureHook | None: + if cl is complex: + return structure_complex + return None + + +@cache +def gen_unstructure_hook(cl: type, converter: Converter) -> UnstructureHook | None: + if cl is complex: + if has_format(converter, MISSING_SPECIAL_FLOATS): + return partial(unstructure_complex, special_as_string=True) + return unstructure_complex + return None + + +def structure_complex(obj: object, _) -> complex: + if ( + isinstance(obj, Sequence) + and len(obj) == 2 + and all(isinstance(x, (Real, str)) for x in obj) + ): + with suppress(ValueError): + obj = [ # for all converters, string inf and nan are allowed + float(x) if (isinstance(x, str) and x.lower() in SPECIAL_STR) else x + for x in obj + ] + return complex(*obj) + raise_unexpected_structure(complex, type(obj)) # noqa: RET503 # NoReturn + + +def unstructure_complex( + value: complex, special_as_string: bool = False +) -> list[float | str]: + return [ + str(x) if (x in SPECIAL and special_as_string) else x + for x in [value.real, value.imag] + ] diff --git a/src/cattrs/strategies/_extra_types/_uuid.py b/src/cattrs/strategies/_extra_types/_uuid.py new file mode 100644 index 00000000..09894303 --- /dev/null +++ b/src/cattrs/strategies/_extra_types/_uuid.py @@ -0,0 +1,34 @@ +from functools import cache +from uuid import UUID + +from ...converters import Converter +from ...dispatch import StructureHook, UnstructureHook +from ...fns import identity +from ...preconf import has_format +from . import raise_unexpected_structure + +SUPPORTS_UUID = ("bson", "cbor", "msgspec-json", "orjson") + + +@cache +def gen_structure_hook(cl: type, _) -> StructureHook | None: + if issubclass(cl, UUID): + return structure_uuid + return None + + +@cache +def gen_unstructure_hook(cl: type, converter: Converter) -> UnstructureHook | None: + if issubclass(cl, UUID): + return identity if has_format(converter, SUPPORTS_UUID) else lambda v: str(v) + return None + + +def structure_uuid(value: bytes | int | str, _) -> UUID: + if isinstance(value, bytes): + return UUID(bytes=value) + if isinstance(value, int): + return UUID(int=value) + if isinstance(value, str): + return UUID(value) + raise_unexpected_structure(UUID, type(value)) # noqa: RET503 # NoReturn diff --git a/src/cattrs/strategies/_extra_types/_zoneinfo.py b/src/cattrs/strategies/_extra_types/_zoneinfo.py new file mode 100644 index 00000000..60484f5b --- /dev/null +++ b/src/cattrs/strategies/_extra_types/_zoneinfo.py @@ -0,0 +1,18 @@ +from functools import cache +from zoneinfo import ZoneInfo + +from ...dispatch import StructureHook, UnstructureHook + + +@cache +def gen_structure_hook(cl: type, _) -> StructureHook | None: + if issubclass(cl, ZoneInfo): + return lambda v, _: ZoneInfo(v) + return None + + +@cache +def gen_unstructure_hook(cl: type, _) -> UnstructureHook | None: + if issubclass(cl, ZoneInfo): + return lambda v: str(v) + return None diff --git a/src/cattrs/types.py b/src/cattrs/types.py index a864cb90..22f35c5d 100644 --- a/src/cattrs/types.py +++ b/src/cattrs/types.py @@ -1,6 +1,6 @@ from typing import Protocol, TypeVar -__all__ = ["SimpleStructureHook"] +__all__ = ["SimpleStructureHook", "Unavailable"] In = TypeVar("In") T = TypeVar("T") @@ -10,3 +10,7 @@ class SimpleStructureHook(Protocol[In, T]): """A structure hook with an optional (ignored) second argument.""" def __call__(self, _: In, /, cl=...) -> T: ... + + +class Unavailable: + """Placeholder class to substitute missing class on import.""" diff --git a/tests/strategies/test_extra_types.py b/tests/strategies/test_extra_types.py new file mode 100644 index 00000000..d38f317d --- /dev/null +++ b/tests/strategies/test_extra_types.py @@ -0,0 +1,186 @@ +import uuid +import zoneinfo +from functools import partial +from platform import python_implementation + +from attrs import define, fields +from bson import UuidRepresentation +from hypothesis import given +from hypothesis.strategies import ( + DrawFn, + builds, + complex_numbers, + composite, + timezone_keys, + uuids, +) +from pytest import fixture, mark, skip + +from cattrs import Converter +from cattrs.preconf import has_format +from cattrs.strategies import register_extra_types + +# converters + +# isort: off +from cattrs.preconf import bson +from cattrs.preconf import cbor2 +from cattrs.preconf import json +from cattrs.preconf import msgpack +from cattrs.preconf import pyyaml +from cattrs.preconf import tomlkit +from cattrs.preconf import ujson + +if python_implementation() != "PyPy": + from cattrs.preconf import msgspec + from cattrs.preconf import orjson +else: + msgspec = "msgspec" + orjson = "orjson" +# isort: on + +PRECONF_MODULES = [bson, cbor2, json, msgpack, msgspec, orjson, pyyaml, tomlkit, ujson] + + +@define +class Extras: + complex: complex + uuid: uuid.UUID + zoneinfo: zoneinfo.ZoneInfo + + +EXTRA_TYPES = {attr.name: attr.type for attr in fields(Extras)} + + +@composite +def extras(draw: DrawFn): + return Extras( + complex=draw(complex_numbers(allow_infinity=True, allow_nan=False)), + uuid=draw(uuids(allow_nil=True)), + zoneinfo=draw(builds(zoneinfo.ZoneInfo, timezone_keys())), + ) + + +# converters + + +@fixture(scope="session") +def raw_converter(converter_cls) -> Converter: + """Raw BaseConverter and Converter.""" + conv = converter_cls() + register_extra_types(conv, *EXTRA_TYPES.values()) + return conv + + +@fixture(scope="session", params=PRECONF_MODULES) +def preconf_converter(request) -> Converter: + """All preconfigured converters.""" + if isinstance(request.param, str): + skip(f'Converter "{request.param}" is unavailable for current implementation') + + conv = request.param.make_converter() + register_extra_types(conv, *EXTRA_TYPES.values()) + return conv + + +@fixture(scope="session", params=[None, *PRECONF_MODULES]) +def any_converter(request) -> Converter: + """Global converter and all preconfigured converters.""" + if isinstance(request.param, str): + skip(f'Converter "{request.param}" is unavailable for current implementation') + + conv = request.param.make_converter() if request.param else Converter() + register_extra_types(conv, *EXTRA_TYPES.values()) + return conv + + +# common tests + + +@given(extras()) +def test_restructure_attrs(any_converter, item: Extras): + """Extra types as attributes can be unstructured and restructured.""" + assert any_converter.structure(any_converter.unstructure(item), Extras) == item + + +@given(extras()) +def test_restructure_values(any_converter, item: Extras): + """Extra types as standalone values can be unstructured and restructured.""" + for attr, cl in EXTRA_TYPES.items(): + value = getattr(item, attr) + assert any_converter.structure(any_converter.unstructure(value), cl) == value + + +@given(extras()) +def test_restructure_optional(any_converter, item: Extras): + """Extra types as optional standalone values can be structured.""" + for attr, cl in EXTRA_TYPES.items(): + value = getattr(item, attr) + assert any_converter.structure(None, cl | None) is None + assert ( + any_converter.structure(any_converter.unstructure(value), cl | None) + == value + ) + + +@given(extras()) +def test_dumpload_attrs(preconf_converter, item: Extras): + """Extra types as attributes can be dumped/loaded by preconfigured converters.""" + if has_format(preconf_converter, "bson"): + # BsonConverter requires explicit UUID representation + codec_options = bson.DEFAULT_CODEC_OPTIONS.with_options( + uuid_representation=UuidRepresentation.STANDARD + ) + dumps = partial(preconf_converter.dumps, codec_options=codec_options) + loads = partial(preconf_converter.loads, codec_options=codec_options) + elif has_format(preconf_converter, "msgspec-json"): + # MsgspecJsonConverter can be used with dumps/loads factories for extra types + dumps = preconf_converter.get_dumps_hook(Extras) + loads = lambda v, cl: preconf_converter.get_loads_hook(cl)(v) # noqa: E731 + else: + dumps = preconf_converter.dumps + loads = preconf_converter.loads + # test + assert loads(dumps(item), Extras) == item + + +# builtins.complex + + +@mark.parametrize("unstructured,structured", [([1.0, 0.0], complex(1, 0))]) +def test_specific_complex(raw_converter, unstructured, structured) -> None: + """Raw converter structures complex.""" + assert raw_converter.structure(unstructured, complex) == structured + + +# uuid.UUID + +UUID_NIL = uuid.UUID(bytes=b"\x00" * 16) + + +@mark.parametrize( + "value", + ( + UUID_NIL, # passthrough + b"\x00" * 16, + 0, + "00000000000000000000000000000000", + "00000000-0000-0000-0000-000000000000", + "{00000000000000000000000000000000}", + "{00000000-0000-0000-0000-000000000000}", + "urn:uuid:00000000000000000000-000000000000", + "urn:uuid:00000000-0000-0000-0000-000000000000", + ), +) +def test_specific_uuid(raw_converter, value) -> None: + """Raw converter structures from all formats supported by uuid.UUID.""" + assert raw_converter.structure(value, uuid.UUID) == UUID_NIL + + +# zoneinfo.ZoneInfo + + +@mark.parametrize("value", ("EET", "Europe/Kiev")) +def test_specific_zoneinfo(raw_converter, value) -> None: + """Raw converter structures zoneinfo.ZoneInfo.""" + assert raw_converter.structure(value, zoneinfo.ZoneInfo) == zoneinfo.ZoneInfo(value) diff --git a/tests/test_generics.py b/tests/test_generics.py index 466c4134..374d2e32 100644 --- a/tests/test_generics.py +++ b/tests/test_generics.py @@ -161,7 +161,7 @@ class TClass2(Generic[T]): def test_raises_if_no_generic_params_supplied( - converter: Union[Converter, BaseConverter] + converter: Union[Converter, BaseConverter], ): data = TClass(1, "a") diff --git a/tests/typed.py b/tests/typed.py index 5ff4ea6f..96533fd9 100644 --- a/tests/typed.py +++ b/tests/typed.py @@ -275,7 +275,7 @@ def key(t): def _create_hyp_class_and_strat( - attrs_and_strategy: list[tuple[_CountingAttr, SearchStrategy[PosArg]]] + attrs_and_strategy: list[tuple[_CountingAttr, SearchStrategy[PosArg]]], ) -> SearchStrategy[tuple[type, SearchStrategy[PosArgs], SearchStrategy[KwArgs]]]: def key(t): return (t[0].default is not NOTHING, t[0].kw_only) diff --git a/tests/typeddicts.py b/tests/typeddicts.py index ba488010..36a87510 100644 --- a/tests/typeddicts.py +++ b/tests/typeddicts.py @@ -202,7 +202,7 @@ def simple_typeddicts_with_extra_keys( # The normal attributes are 2 characters or less. extra_keys = draw(sets(text(ascii_lowercase, min_size=3, max_size=3))) - success.update({k: 1 for k in extra_keys}) + success.update(dict.fromkeys(extra_keys, 1)) return cls, success, extra_keys