From ff67a40acfb600bafae2355ed28f23ac30a62487 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Sat, 30 Nov 2024 14:08:23 -0600 Subject: [PATCH] feat(taps,targets): Support the `singer.decimal` JSON Schema extension https://github.com/meltano/sdk/issues/1890 --- singer_sdk/connectors/sql.py | 33 +++++++++++++++++++++++++++++--- singer_sdk/typing.py | 6 ++++++ tests/core/test_connector_sql.py | 8 ++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index e7ba63435..1be71849c 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -124,10 +124,21 @@ class SQLToJSONSchema: .. versionchanged:: 0.43.0 Added the :meth:`singer_sdk.connectors.sql.SQLToJSONSchema.from_config` class method. + .. versionchanged:: 0.43.0 + Added support for the `use_singer_decimal` option. """ + def __init__(self, *, use_singer_decimal: bool) -> None: + """Initialize the SQL to JSON Schema converter. + + Args: + use_singer_decimal: Whether to represent numbers as `string` with + the `singer.decimal` format instead of as `number`. + """ + self.use_singer_decimal = use_singer_decimal + @classmethod - def from_config(cls: type[SQLToJSONSchema], config: dict) -> SQLToJSONSchema: # noqa: ARG003 + def from_config(cls: type[SQLToJSONSchema], config: dict) -> SQLToJSONSchema: """Create a new instance from a configuration dictionary. Override this to instantiate this converter with values from the tap's @@ -146,11 +157,13 @@ def from_config(cls, config): Args: config: The configuration dictionary. + use_singer_decimal: Whether to represent numbers as `string` with + the `singer.decimal` format instead of as `number`. Returns: A new instance of the class. """ - return cls() + return cls(use_singer_decimal=config.get("use_singer_decimal", False)) @functools.singledispatchmethod def to_jsonschema(self, column_type: sa.types.TypeEngine) -> dict: # noqa: ARG002, D102, PLR6301 @@ -193,12 +206,14 @@ def integer_to_jsonschema(self, column_type: sa.types.Integer) -> dict: # noqa: return th.IntegerType.type_dict # type: ignore[no-any-return] @to_jsonschema.register - def float_to_jsonschema(self, column_type: sa.types.Numeric) -> dict: # noqa: ARG002, PLR6301 + def float_to_jsonschema(self, column_type: sa.types.Numeric) -> dict: # noqa: ARG002 """Return a JSON Schema representation of a generic number type. Args: column_type (:column_type:`Numeric`): The column type. """ + if self.use_singer_decimal: + return th.SingerDecimalType.type_dict # type: ignore[no-any-return] return th.NumberType.type_dict # type: ignore[no-any-return] @to_jsonschema.register @@ -272,6 +287,7 @@ def __init__(self, *, max_varchar_length: int | None = None) -> None: "hostname": lambda _: sa.types.VARCHAR(253), # RFC 1035 "ipv4": lambda _: sa.types.VARCHAR(15), "ipv6": lambda _: sa.types.VARCHAR(45), + "singer.decimal": self._handle_singer_decimal, } self._fallback_type: type[sa.types.TypeEngine] = sa.types.VARCHAR @@ -323,6 +339,17 @@ def _invoke_handler( # noqa: PLR6301 return handler() # type: ignore[no-any-return] return handler(schema) + def _handle_singer_decimal(self, schema: dict) -> sa.types.TypeEngine: # noqa: PLR6301 + """Handle a singer.decimal format. + + Args: + schema: The JSON Schema object. + + Returns: + The appropriate SQLAlchemy type. + """ + return sa.types.DECIMAL(schema.get("precision"), schema.get("scale")) + @property def fallback_type(self) -> type[sa.types.TypeEngine]: """Return the fallback type. diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index fb22f4e82..849925398 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -448,6 +448,12 @@ class RegexType(StringType): string_format = "regex" +class SingerDecimalType(StringType): + """Decimal type.""" + + string_format = "singer.decimal" + + class BooleanType(JSONTypeHelper[bool]): """Boolean type. diff --git a/tests/core/test_connector_sql.py b/tests/core/test_connector_sql.py index 059c29a7f..0ab13c0f8 100644 --- a/tests/core/test_connector_sql.py +++ b/tests/core/test_connector_sql.py @@ -482,6 +482,14 @@ def my_type_to_jsonschema(self, column_type) -> dict: # noqa: ARG002 assert m.to_jsonschema(sa.types.BOOLEAN()) == {"type": ["boolean"]} +def test_numeric_to_singer_decimal(): + converter = SQLToJSONSchema(use_singer_decimal=True) + assert converter.to_jsonschema(sa.types.NUMERIC()) == { + "type": ["string"], + "format": "singer.decimal", + } + + class TestJSONSchemaToSQL: # noqa: PLR0904 @pytest.fixture def json_schema_to_sql(self) -> JSONSchemaToSQL: