From 7a0478d682d7257e9ec4d62604b3bf579af1cdd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Fri, 26 Jan 2024 17:42:41 -0600 Subject: [PATCH 1/2] fix: Replace use of deprecated `jsonschema._RefResolver` with recommended `referencing` library (#2187) * refactor: Replace use of deprecated `jsonschema._RefResolver` with recommended `referencing` library * Fix types * Continue on error --- .github/workflows/test.yml | 1 + poetry.lock | 2 +- pyproject.toml | 1 + singer_sdk/_singerlib/schema.py | 22 +++++++++++++++++----- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 845d12ae1..7ad37ef34 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,6 +39,7 @@ jobs: tests: name: "Test on ${{ matrix.python-version }} (${{ matrix.session }}) / ${{ matrix.os }} / SQLAlchemy: ${{ matrix.sqlalchemy }}" runs-on: ${{ matrix.os }} + continue-on-error: true env: NOXPYTHON: ${{ matrix.python-version }} NOXSESSION: ${{ matrix.session }} diff --git a/poetry.lock b/poetry.lock index 22c8242c7..77ce8dff3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2620,4 +2620,4 @@ testing = ["pytest", "pytest-durations"] [metadata] lock-version = "2.0" python-versions = ">=3.8" -content-hash = "52eac3bc3846a549ece9b6923901f1d2e9f808ad52fc60721e565092b08cf087" +content-hash = "3308a0e70f8f097b0a678989aa8d25133ff05e8d9ea37232d739a9664b05f361" diff --git a/pyproject.toml b/pyproject.toml index abc09835c..a01fe5500 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ PyJWT = "~=2.4" python-dateutil = ">=2.8.2" python-dotenv = ">=0.20" PyYAML = ">=6.0" +referencing = ">=0.30.0" requests = ">=2.25.1" simpleeval = ">=0.9.13" simplejson = ">=3.17.6" diff --git a/singer_sdk/_singerlib/schema.py b/singer_sdk/_singerlib/schema.py index 1be527a97..d12d04561 100644 --- a/singer_sdk/_singerlib/schema.py +++ b/singer_sdk/_singerlib/schema.py @@ -5,7 +5,11 @@ import typing as t from dataclasses import dataclass -from jsonschema import RefResolver +from referencing import Registry +from referencing.jsonschema import DRAFT202012 + +if t.TYPE_CHECKING: + from referencing._core import Resolver # These are keys defined in the JSON Schema spec that do not themselves contain # schemas (or lists of schemas) @@ -148,17 +152,25 @@ def resolve_schema_references( A schema dict with all $refs replaced with the appropriate dict. """ refs = refs or {} - return _resolve_schema_references(schema, RefResolver("", schema, store=refs)) + registry: Registry = Registry() + schema_resource = DRAFT202012.create_resource(schema) + registry = registry.with_resource("", schema_resource) + registry = registry.with_resources( + [(k, DRAFT202012.create_resource(v)) for k, v in refs.items()] + ) + + resolver = registry.resolver() + return _resolve_schema_references(schema, resolver) def _resolve_schema_references( schema: dict[str, t.Any], - resolver: RefResolver, + resolver: Resolver, ) -> dict[str, t.Any]: if _SchemaKey.ref in schema: reference_path = schema.pop(_SchemaKey.ref, None) - resolved = resolver.resolve(reference_path)[1] - schema.update(resolved) + resolved = resolver.lookup(reference_path) + schema.update(resolved.contents) return _resolve_schema_references(schema, resolver) if _SchemaKey.properties in schema: From f798f46f23ff720f9bf8bf67c3ce005c729c6778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Fri, 26 Jan 2024 18:01:28 -0600 Subject: [PATCH 2/2] =?UTF-8?q?refactor:=20Remove=20unused=20`logger`?= =?UTF-8?q?=C2=A0parameter=20from=20private=20catalog=20helper=20functions?= =?UTF-8?q?=20(#2188)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- singer_sdk/helpers/_catalog.py | 9 +-------- singer_sdk/mapper.py | 1 - singer_sdk/streams/core.py | 2 +- singer_sdk/streams/sql.py | 1 - tests/core/test_catalog_selection.py | 8 +------- tests/samples/test_tap_countries.py | 3 --- 6 files changed, 3 insertions(+), 21 deletions(-) diff --git a/singer_sdk/helpers/_catalog.py b/singer_sdk/helpers/_catalog.py index 49ea2f1cc..39df3f0f1 100644 --- a/singer_sdk/helpers/_catalog.py +++ b/singer_sdk/helpers/_catalog.py @@ -10,8 +10,6 @@ from singer_sdk.helpers._typing import is_object_type if t.TYPE_CHECKING: - from logging import Logger - from singer_sdk._singerlib import Catalog, SelectionMask _MAX_LRU_CACHE = 500 @@ -22,11 +20,10 @@ def get_selected_schema( stream_name: str, schema: dict, mask: SelectionMask, - logger: Logger, ) -> dict: """Return a copy of the provided JSON schema, dropping any fields not selected.""" new_schema = deepcopy(schema) - _pop_deselected_schema(new_schema, mask, stream_name, (), logger) + _pop_deselected_schema(new_schema, mask, stream_name, ()) return new_schema @@ -35,7 +32,6 @@ def _pop_deselected_schema( mask: SelectionMask, stream_name: str, breadcrumb: tuple[str, ...], - logger: Logger, ) -> None: """Remove anything from schema that is not selected. @@ -75,7 +71,6 @@ def _pop_deselected_schema( mask, stream_name, property_breadcrumb, - logger, ) @@ -83,7 +78,6 @@ def pop_deselected_record_properties( record: dict[str, t.Any], schema: dict, mask: SelectionMask, - logger: Logger, breadcrumb: tuple[str, ...] = (), ) -> None: """Remove anything from record properties that is not selected. @@ -104,7 +98,6 @@ def pop_deselected_record_properties( val, schema, mask, - logger, property_breadcrumb, ) diff --git a/singer_sdk/mapper.py b/singer_sdk/mapper.py index aeea46812..d3cd9400d 100644 --- a/singer_sdk/mapper.py +++ b/singer_sdk/mapper.py @@ -657,7 +657,6 @@ def register_raw_streams_from_catalog(self, catalog: Catalog) -> None: catalog_entry.stream or catalog_entry.tap_stream_id, catalog_entry.schema.to_dict(), catalog_entry.metadata.resolve_selection(), - self.logger, ), catalog_entry.key_properties, ) diff --git a/singer_sdk/streams/core.py b/singer_sdk/streams/core.py index 36af0c125..064b78361 100644 --- a/singer_sdk/streams/core.py +++ b/singer_sdk/streams/core.py @@ -827,7 +827,7 @@ def _generate_record_messages( Yields: Record message objects. """ - pop_deselected_record_properties(record, self.schema, self.mask, self.logger) + pop_deselected_record_properties(record, self.schema, self.mask) record = conform_record_data_types( stream_name=self.name, record=record, diff --git a/singer_sdk/streams/sql.py b/singer_sdk/streams/sql.py index 29946e78c..47d015f9d 100644 --- a/singer_sdk/streams/sql.py +++ b/singer_sdk/streams/sql.py @@ -154,7 +154,6 @@ def get_selected_schema(self) -> dict: stream_name=self.name, schema=self.schema, mask=self.mask, - logger=self.logger, ) # Get records from stream diff --git a/tests/core/test_catalog_selection.py b/tests/core/test_catalog_selection.py index f6653c25e..d4b30cecb 100644 --- a/tests/core/test_catalog_selection.py +++ b/tests/core/test_catalog_selection.py @@ -197,12 +197,7 @@ def test_schema_selection( stream_name: str, ): """Test that schema selection rules are correctly applied to SCHEMA messages.""" - selected_schema = get_selected_schema( - stream_name, - schema, - mask, - logging.getLogger(), - ) + selected_schema = get_selected_schema(stream_name, schema, mask) assert ( selected_schema["properties"] == PropertiesList( @@ -247,7 +242,6 @@ def test_record_property_pop( record=record_pop, schema=schema, mask=mask, - logger=logging.getLogger(), breadcrumb=(), ) diff --git a/tests/samples/test_tap_countries.py b/tests/samples/test_tap_countries.py index 31e0b8666..a302e646a 100644 --- a/tests/samples/test_tap_countries.py +++ b/tests/samples/test_tap_countries.py @@ -5,7 +5,6 @@ import copy import io import json -import logging import typing as t from contextlib import redirect_stdout @@ -89,7 +88,6 @@ def test_with_catalog_entry(): record=copied_record, schema=stream.schema, mask=stream.mask, - logger=logging.getLogger(), ) assert copied_record == record @@ -97,7 +95,6 @@ def test_with_catalog_entry(): stream_name=stream.name, schema=stream.schema, mask=stream.metadata.resolve_selection(), - logger=logging.getLogger(), ) assert new_schema == stream.schema