diff --git a/poetry.lock b/poetry.lock index a616ff9..13aec18 100644 --- a/poetry.lock +++ b/poetry.lock @@ -735,6 +735,8 @@ optional = false python-versions = "*" files = [ {file = "jsonpath-ng-1.7.0.tar.gz", hash = "sha256:f6f5f7fd4e5ff79c785f1573b394043b39849fb2bb47bcead935d12b00beab3c"}, + {file = "jsonpath_ng-1.7.0-py2-none-any.whl", hash = "sha256:898c93fc173f0c336784a3fa63d7434297544b7198124a68f9a3ef9597b0ae6e"}, + {file = "jsonpath_ng-1.7.0-py3-none-any.whl", hash = "sha256:f3d7f9e848cba1b6da28c55b1c26ff915dc9e0b1ba7e752a53d6da8d5cbd00b6"}, ] [package.dependencies] @@ -1574,23 +1576,21 @@ files = [ [[package]] name = "singer-sdk" -version = "0.42.1" +version = "0.43.0a2.post5.dev0+4be6949c" description = "A framework for building Singer taps" optional = false -python-versions = ">=3.8" -files = [ - {file = "singer_sdk-0.42.1-py3-none-any.whl", hash = "sha256:cc588927ff87b7aea0087c57c37c85bbe95f9b7c52ebb265abb50a184cec6ad8"}, - {file = "singer_sdk-0.42.1.tar.gz", hash = "sha256:92019c3f2e476f5c17eaa381e35d3c0ff428814587f8424149a4b28904f75ca0"}, -] +python-versions = ">=3.9" +files = [] +develop = false [package.dependencies] backoff = {version = ">=2.0.0", markers = "python_version < \"4\""} backports-datetime-fromisoformat = {version = ">=2.0.1", markers = "python_version < \"3.11\""} -click = ">=8.0,<9.0" +click = "~=8.0" fs = ">=2.4.16" fsspec = ">=2024.9.0" importlib-metadata = {version = "<9.0.0", markers = "python_version < \"3.12\""} -importlib-resources = {version = ">=5.12.0,<6.2.0 || >6.2.0,<6.3.0 || >6.3.0,<6.3.1 || >6.3.1", markers = "python_version < \"3.10\""} +importlib-resources = {version = ">=5.12.0,!=6.2.0,!=6.3.0,!=6.3.1", markers = "python_version < \"3.10\""} inflection = ">=0.5.1" joblib = ">=1.3.0" jsonpath-ng = ">=1.5.3" @@ -1601,7 +1601,7 @@ PyYAML = ">=6.0" referencing = ">=0.30.0" requests = ">=2.25.1" setuptools = "<=70.3.0" -simpleeval = {version = ">=0.9.13,<1.0.1 || >1.0.1", markers = "python_version >= \"3.9\""} +simpleeval = ">=0.9.13,!=1.0.1" simplejson = ">=3.17.6" sqlalchemy = ">=1.4,<3.0" typing-extensions = ">=4.5.0" @@ -1610,11 +1610,17 @@ typing-extensions = ">=4.5.0" docs = ["furo (>=2024.5.6)", "myst-parser (>=3)", "pytest (>=7.2.1)", "sphinx (>=7)", "sphinx-copybutton (>=0.5.2)", "sphinx-inline-tabs (>=2023.4.21)", "sphinx-notfound-page (>=1.0.0)", "sphinx-reredirects (>=0.1.5)"] faker = ["faker (>=22.5)"] jwt = ["PyJWT (>=2.4,<3.0)", "cryptography (>=3.4.6)"] -parquet = ["numpy (>=1.22)", "numpy (>=1.22,<1.25)", "numpy (>=1.22,<2.1)", "pyarrow (>=13)", "pyarrow (>=13,<18)"] +parquet = ["numpy (>=1.22)", "numpy (>=1.22,<2.1)", "pyarrow (>=13)"] s3 = ["fs-s3fs (>=1.1.1)", "s3fs (>=2024.9.0)"] ssh = ["paramiko (>=3.3.0)"] testing = ["pytest (>=7.2.1)"] +[package.source] +type = "git" +url = "https://github.com/meltano/sdk.git" +reference = "edgarrmondragon/refactor/jsonschematosql-fromconfig" +resolved_reference = "4be6949c60a011307cd2498d9ceea0e256a7b67c" + [[package]] name = "six" version = "1.16.0" @@ -1912,4 +1918,4 @@ faker = ["faker"] [metadata] lock-version = "2.0" python-versions = ">=3.9" -content-hash = "cada27dbb6bcee2496a4310cd954e33c1ee3de76f59009bff01e963b0b100ee5" +content-hash = "57fe3547c7d73bd9bee69074600be7349eb44a34efe5ee35da6cf4cc17448823" diff --git a/pyproject.toml b/pyproject.toml index d2144ad..b0ab7b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,8 @@ sqlalchemy = "~=2.0" sshtunnel = "0.4.0" [tool.poetry.dependencies.singer-sdk] -version = "~=0.42.0b1" +git = "https://github.com/meltano/sdk.git" +branch = "edgarrmondragon/refactor/jsonschematosql-fromconfig" [tool.poetry.extras] faker = ["faker"] diff --git a/target_postgres/connector.py b/target_postgres/connector.py index f627876..1c50788 100644 --- a/target_postgres/connector.py +++ b/target_postgres/connector.py @@ -41,11 +41,24 @@ class JSONSchemaToPostgres(JSONSchemaToSQL): """Convert JSON Schema types to Postgres types.""" - def __init__(self, *, content_encoding: bool = True) -> None: + def __init__(self, *, content_encoding: bool = True, **kwargs): """Initialize the JSONSchemaToPostgres instance.""" - super().__init__() + super().__init__(**kwargs) self.content_encoding = content_encoding + @classmethod + def from_config( + cls: type[JSONSchemaToPostgres], + config: dict, + *, + max_varchar_length: int | None = None, + ) -> JSONSchemaToPostgres: + """Create a JSONSchemaToPostgres instance from a configuration.""" + return cls( + content_encoding=config.get("interpret_content_encoding", False), + max_varchar_length=max_varchar_length, + ) + def handle_raw_string(self, schema): """Handle a raw string type.""" if self.content_encoding and schema.get("contentEncoding") == "base16": @@ -63,6 +76,8 @@ class PostgresConnector(SQLConnector): allow_merge_upsert: bool = True # Whether MERGE UPSERT is supported. allow_temp_tables: bool = True # Whether temp tables are supported. + jsonschema_to_sql_converter = JSONSchemaToPostgres + def __init__(self, config: dict) -> None: """Initialize a connector to a Postgres database. @@ -100,18 +115,6 @@ def __init__(self, config: dict) -> None: sqlalchemy_url=url.render_as_string(hide_password=False), ) - @cached_property - def interpret_content_encoding(self) -> bool: - """Whether to interpret schema contentEncoding to set the column type. - - It is an opt-in feature because it might result in data loss if the - actual data does not match the schema's advertised encoding. - - Returns: - True if the feature is enabled, False otherwise. - """ - return self.config.get("interpret_content_encoding", False) - def prepare_table( # type: ignore[override] # noqa: PLR0913 self, full_table_name: str | FullyQualifiedName, @@ -258,7 +261,10 @@ def _handle_array_type(self, jsonschema: dict) -> ARRAY | JSONB: @cached_property def jsonschema_to_sql(self) -> JSONSchemaToSQL: """Return a JSONSchemaToSQL instance with custom type handling.""" - to_sql = JSONSchemaToPostgres(content_encoding=self.interpret_content_encoding) + to_sql = JSONSchemaToPostgres.from_config( + self.config, + max_varchar_length=self.max_varchar_length, + ) to_sql.fallback_type = TEXT to_sql.register_type_handler("integer", BIGINT) to_sql.register_type_handler("object", JSONB)