From 759c77beb764be49b9a207635349b6d8ed420290 Mon Sep 17 00:00:00 2001 From: Dan Norman Date: Fri, 28 Jul 2023 12:17:35 -0600 Subject: [PATCH 01/30] feat: SQLTarget connector instance shared with sinks (#1864) Co-authored-by: Edgar R. M --- singer_sdk/target_base.py | 133 ++++++++++++++++++++++++++++++++- tests/conftest.py | 65 +++++++++++++++- tests/core/test_target_base.py | 72 +++++++++++++++++- 3 files changed, 264 insertions(+), 6 deletions(-) diff --git a/singer_sdk/target_base.py b/singer_sdk/target_base.py index d62bbbfd84..cf10388317 100644 --- a/singer_sdk/target_base.py +++ b/singer_sdk/target_base.py @@ -28,8 +28,9 @@ if t.TYPE_CHECKING: from pathlib import PurePath + from singer_sdk.connectors import SQLConnector from singer_sdk.mapper import PluginMapper - from singer_sdk.sinks import Sink + from singer_sdk.sinks import Sink, SQLSink _MAX_PARALLELISM = 8 @@ -48,7 +49,7 @@ class Target(PluginBase, SingerReader, metaclass=abc.ABCMeta): # Default class to use for creating new sink objects. # Required if `Target.get_sink_class()` is not defined. - default_sink_class: type[Sink] | None = None + default_sink_class: type[Sink] def __init__( self, @@ -574,6 +575,23 @@ def get_singer_command(cls: type[Target]) -> click.Command: class SQLTarget(Target): """Target implementation for SQL destinations.""" + _target_connector: SQLConnector | None = None + + default_sink_class: type[SQLSink] + + @property + def target_connector(self) -> SQLConnector: + """The connector object. + + Returns: + The connector object. + """ + if self._target_connector is None: + self._target_connector = self.default_sink_class.connector_class( + dict(self.config), + ) + return self._target_connector + @classproperty def capabilities(self) -> list[CapabilitiesEnum]: """Get target capabilities. @@ -617,3 +635,114 @@ def _merge_missing(source_jsonschema: dict, target_jsonschema: dict) -> None: super().append_builtin_config(config_jsonschema) pass + + @final + def add_sqlsink( + self, + stream_name: str, + schema: dict, + key_properties: list[str] | None = None, + ) -> Sink: + """Create a sink and register it. + + This method is internal to the SDK and should not need to be overridden. + + Args: + stream_name: Name of the stream. + schema: Schema of the stream. + key_properties: Primary key of the stream. + + Returns: + A new sink for the stream. + """ + self.logger.info("Initializing '%s' target sink...", self.name) + sink_class = self.get_sink_class(stream_name=stream_name) + sink = sink_class( + target=self, + stream_name=stream_name, + schema=schema, + key_properties=key_properties, + connector=self.target_connector, + ) + sink.setup() + self._sinks_active[stream_name] = sink + + return sink + + def get_sink_class(self, stream_name: str) -> type[SQLSink]: + """Get sink for a stream. + + Developers can override this method to return a custom Sink type depending + on the value of `stream_name`. Optional when `default_sink_class` is set. + + Args: + stream_name: Name of the stream. + + Raises: + ValueError: If no :class:`singer_sdk.sinks.Sink` class is defined. + + Returns: + The sink class to be used with the stream. + """ + if self.default_sink_class: + return self.default_sink_class + + msg = ( + f"No sink class defined for '{stream_name}' and no default sink class " + "available." + ) + raise ValueError(msg) + + def get_sink( + self, + stream_name: str, + *, + record: dict | None = None, + schema: dict | None = None, + key_properties: list[str] | None = None, + ) -> Sink: + """Return a sink for the given stream name. + + A new sink will be created if `schema` is provided and if either `schema` or + `key_properties` has changed. If so, the old sink becomes archived and held + until the next drain_all() operation. + + Developers only need to override this method if they want to provide a different + sink depending on the values within the `record` object. Otherwise, please see + `default_sink_class` property and/or the `get_sink_class()` method. + + Raises :class:`singer_sdk.exceptions.RecordsWithoutSchemaException` if sink does + not exist and schema is not sent. + + Args: + stream_name: Name of the stream. + record: Record being processed. + schema: Stream schema. + key_properties: Primary key of the stream. + + Returns: + The sink used for this target. + """ + _ = record # Custom implementations may use record in sink selection. + if schema is None: + self._assert_sink_exists(stream_name) + return self._sinks_active[stream_name] + + existing_sink = self._sinks_active.get(stream_name, None) + if not existing_sink: + return self.add_sqlsink(stream_name, schema, key_properties) + + if ( + existing_sink.schema != schema + or existing_sink.key_properties != key_properties + ): + self.logger.info( + "Schema or key properties for '%s' stream have changed. " + "Initializing a new '%s' sink...", + stream_name, + stream_name, + ) + self._sinks_to_clear.append(self._sinks_active.pop(stream_name)) + return self.add_sqlsink(stream_name, schema, key_properties) + + return existing_sink diff --git a/tests/conftest.py b/tests/conftest.py index 25319c015b..cf64e28dd2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,9 +10,10 @@ import pytest from sqlalchemy import __version__ as sqlalchemy_version +from singer_sdk import SQLConnector from singer_sdk import typing as th -from singer_sdk.sinks import BatchSink -from singer_sdk.target_base import Target +from singer_sdk.sinks import BatchSink, SQLSink +from singer_sdk.target_base import SQLTarget, Target if t.TYPE_CHECKING: from _pytest.config import Config @@ -116,3 +117,63 @@ def _write_state_message(self, state: dict): """Emit the stream's latest state.""" super()._write_state_message(state) self.state_messages_written.append(state) + + +class SQLConnectorMock(SQLConnector): + """A Mock SQLConnector class.""" + + +class SQLSinkMock(SQLSink): + """A mock Sink class.""" + + name = "sql-sink-mock" + connector_class = SQLConnectorMock + + def __init__( + self, + target: SQLTargetMock, + stream_name: str, + schema: dict, + key_properties: list[str] | None, + connector: SQLConnector | None = None, + ): + """Create the Mock batch-based sink.""" + self._connector: SQLConnector + self._connector = connector or self.connector_class(dict(target.config)) + super().__init__(target, stream_name, schema, key_properties, connector) + self.target = target + + def process_record(self, record: dict, context: dict) -> None: + """Tracks the count of processed records.""" + self.target.num_records_processed += 1 + super().process_record(record, context) + + def process_batch(self, context: dict) -> None: + """Write to mock trackers.""" + self.target.records_written.extend(context["records"]) + self.target.num_batches_processed += 1 + + @property + def key_properties(self) -> list[str]: + return [key.upper() for key in super().key_properties] + + +class SQLTargetMock(SQLTarget): + """A mock Target class.""" + + name = "sql-target-mock" + config_jsonschema = th.PropertiesList().to_dict() + default_sink_class = SQLSinkMock + + def __init__(self, *args, **kwargs): + """Create the Mock target sync.""" + super().__init__(*args, **kwargs) + self.state_messages_written: list[dict] = [] + self.records_written: list[dict] = [] + self.num_records_processed: int = 0 + self.num_batches_processed: int = 0 + + def _write_state_message(self, state: dict): + """Emit the stream's latest state.""" + super()._write_state_message(state) + self.state_messages_written.append(state) diff --git a/tests/core/test_target_base.py b/tests/core/test_target_base.py index 1fd6b9a93b..ee00d35ebb 100644 --- a/tests/core/test_target_base.py +++ b/tests/core/test_target_base.py @@ -4,8 +4,11 @@ import pytest -from singer_sdk.exceptions import MissingKeyPropertiesError -from tests.conftest import BatchSinkMock, TargetMock +from singer_sdk.exceptions import ( + MissingKeyPropertiesError, + RecordsWithoutSchemaException, +) +from tests.conftest import BatchSinkMock, SQLSinkMock, SQLTargetMock, TargetMock def test_get_sink(): @@ -53,3 +56,68 @@ def test_validate_record(): # Test invalid record with pytest.raises(MissingKeyPropertiesError): sink._singer_validate_message({"name": "test"}) + + +def test_sql_get_sink(): + input_schema_1 = { + "properties": { + "id": { + "type": ["string", "null"], + }, + "col_ts": { + "format": "date-time", + "type": ["string", "null"], + }, + }, + } + input_schema_2 = copy.deepcopy(input_schema_1) + key_properties = [] + target = SQLTargetMock(config={"sqlalchemy_url": "sqlite:///"}) + sink = SQLSinkMock( + target=target, + stream_name="foo", + schema=input_schema_1, + key_properties=key_properties, + connector=target.target_connector, + ) + target._sinks_active["foo"] = sink + sink_returned = target.get_sink( + "foo", + schema=input_schema_2, + key_properties=key_properties, + ) + assert sink_returned is sink + + +def test_add_sqlsink_and_get_sink(): + input_schema_1 = { + "properties": { + "id": { + "type": ["string", "null"], + }, + "col_ts": { + "format": "date-time", + "type": ["string", "null"], + }, + }, + } + input_schema_2 = copy.deepcopy(input_schema_1) + key_properties = [] + target = SQLTargetMock(config={"sqlalchemy_url": "sqlite:///"}) + sink = target.add_sqlsink( + "foo", + schema=input_schema_2, + key_properties=key_properties, + ) + + sink_returned = target.get_sink( + "foo", + ) + + assert sink_returned is sink + + # Test invalid call + with pytest.raises(RecordsWithoutSchemaException): + target.get_sink( + "bar", + ) From c3a8f90ac4d79eb5c655740d0fd88f393af7f306 Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Mon, 31 Jul 2023 07:33:02 -0600 Subject: [PATCH 02/30] fix: Append batch config if target supports the batch capability (#1880) --- singer_sdk/target_base.py | 33 +++++++++++++++++++++++++++++++++ tests/conftest.py | 8 ++++++++ tests/core/test_target_base.py | 19 +++++++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/singer_sdk/target_base.py b/singer_sdk/target_base.py index cf10388317..91bce6526f 100644 --- a/singer_sdk/target_base.py +++ b/singer_sdk/target_base.py @@ -17,6 +17,7 @@ from singer_sdk.helpers._classproperty import classproperty from singer_sdk.helpers._compat import final from singer_sdk.helpers.capabilities import ( + BATCH_CONFIG, TARGET_SCHEMA_CONFIG, CapabilitiesEnum, PluginCapabilities, @@ -571,6 +572,38 @@ def get_singer_command(cls: type[Target]) -> click.Command: return command + @classmethod + def append_builtin_config(cls: type[Target], config_jsonschema: dict) -> None: + """Appends built-in config to `config_jsonschema` if not already set. + + To customize or disable this behavior, developers may either override this class + method or override the `capabilities` property to disabled any unwanted + built-in capabilities. + + For all except very advanced use cases, we recommend leaving these + implementations "as-is", since this provides the most choice to users and is + the most "future proof" in terms of taking advantage of built-in capabilities + which may be added in the future. + + Args: + config_jsonschema: [description] + """ + + def _merge_missing(source_jsonschema: dict, target_jsonschema: dict) -> None: + # Append any missing properties in the target with those from source. + for k, v in source_jsonschema["properties"].items(): + if k not in target_jsonschema["properties"]: + target_jsonschema["properties"][k] = v + + capabilities = cls.capabilities + + if PluginCapabilities.BATCH in capabilities: + _merge_missing(BATCH_CONFIG, config_jsonschema) + + super().append_builtin_config(config_jsonschema) + + pass + class SQLTarget(Target): """Target implementation for SQL destinations.""" diff --git a/tests/conftest.py b/tests/conftest.py index cf64e28dd2..7e7c39958f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,12 +12,16 @@ from singer_sdk import SQLConnector from singer_sdk import typing as th +from singer_sdk.helpers.capabilities import PluginCapabilities from singer_sdk.sinks import BatchSink, SQLSink from singer_sdk.target_base import SQLTarget, Target if t.TYPE_CHECKING: from _pytest.config import Config + from singer_sdk.helpers.capabilities import CapabilitiesEnum + + SYSTEMS = {"linux", "darwin", "windows"} pytest_plugins = ("singer_sdk.testing.pytest_plugin",) @@ -104,6 +108,10 @@ class TargetMock(Target): name = "target-mock" config_jsonschema = th.PropertiesList().to_dict() default_sink_class = BatchSinkMock + capabilities: t.ClassVar[list[CapabilitiesEnum]] = [ + *Target.capabilities, + PluginCapabilities.BATCH, + ] def __init__(self, *args, **kwargs): """Create the Mock target sync.""" diff --git a/tests/core/test_target_base.py b/tests/core/test_target_base.py index ee00d35ebb..f9b342d395 100644 --- a/tests/core/test_target_base.py +++ b/tests/core/test_target_base.py @@ -8,6 +8,7 @@ MissingKeyPropertiesError, RecordsWithoutSchemaException, ) +from singer_sdk.helpers.capabilities import PluginCapabilities from tests.conftest import BatchSinkMock, SQLSinkMock, SQLTargetMock, TargetMock @@ -58,6 +59,24 @@ def test_validate_record(): sink._singer_validate_message({"name": "test"}) +def test_target_about_info(): + target = TargetMock() + about = target._get_about_info() + + assert about.capabilities == [ + PluginCapabilities.ABOUT, + PluginCapabilities.STREAM_MAPS, + PluginCapabilities.FLATTENING, + PluginCapabilities.BATCH, + ] + + assert "stream_maps" in about.settings["properties"] + assert "stream_map_config" in about.settings["properties"] + assert "flattening_enabled" in about.settings["properties"] + assert "flattening_max_depth" in about.settings["properties"] + assert "batch_config" in about.settings["properties"] + + def test_sql_get_sink(): input_schema_1 = { "properties": { From 611d50dad6b51b6b6df66d3fd7d0698e9b0c9f56 Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Mon, 31 Jul 2023 13:31:18 -0600 Subject: [PATCH 03/30] fix: Expose `add_record_metadata` as a builtin target setting (#1881) * fix: Append batch config if target supports the batch capability * Test about info * fix: Expose `add_record_metadata` as a builtin target setting * Fix types * Fix more types --- singer_sdk/helpers/capabilities.py | 7 +++++++ singer_sdk/target_base.py | 3 +++ tests/core/test_target_base.py | 1 + 3 files changed, 11 insertions(+) diff --git a/singer_sdk/helpers/capabilities.py b/singer_sdk/helpers/capabilities.py index 55aff8ce3c..f5b5fa305c 100644 --- a/singer_sdk/helpers/capabilities.py +++ b/singer_sdk/helpers/capabilities.py @@ -99,6 +99,13 @@ description="The default target database schema name to use for all streams.", ), ).to_dict() +ADD_RECORD_METADATA_CONFIG = PropertiesList( + Property( + "add_record_metadata", + BooleanType(), + description="Add metadata to records.", + ), +).to_dict() class DeprecatedEnum(Enum): diff --git a/singer_sdk/target_base.py b/singer_sdk/target_base.py index 91bce6526f..a5386199fe 100644 --- a/singer_sdk/target_base.py +++ b/singer_sdk/target_base.py @@ -17,6 +17,7 @@ from singer_sdk.helpers._classproperty import classproperty from singer_sdk.helpers._compat import final from singer_sdk.helpers.capabilities import ( + ADD_RECORD_METADATA_CONFIG, BATCH_CONFIG, TARGET_SCHEMA_CONFIG, CapabilitiesEnum, @@ -595,6 +596,8 @@ def _merge_missing(source_jsonschema: dict, target_jsonschema: dict) -> None: if k not in target_jsonschema["properties"]: target_jsonschema["properties"][k] = v + _merge_missing(ADD_RECORD_METADATA_CONFIG, config_jsonschema) + capabilities = cls.capabilities if PluginCapabilities.BATCH in capabilities: diff --git a/tests/core/test_target_base.py b/tests/core/test_target_base.py index f9b342d395..de344c7e3f 100644 --- a/tests/core/test_target_base.py +++ b/tests/core/test_target_base.py @@ -75,6 +75,7 @@ def test_target_about_info(): assert "flattening_enabled" in about.settings["properties"] assert "flattening_max_depth" in about.settings["properties"] assert "batch_config" in about.settings["properties"] + assert "add_record_metadata" in about.settings["properties"] def test_sql_get_sink(): From 4e92424a7b51f829d2a49b8bd4bbe782c4411ef0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Aug 2023 11:25:38 -0600 Subject: [PATCH 04/30] chore: pre-commit autoupdate (#1889) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Edgar Ramírez Mondragón --- .pre-commit-config.yaml | 4 ++-- .../{{cookiecutter.tap_id}}/.pre-commit-config.yaml | 8 ++++---- ...EST' == cookiecutter.stream_type %}client.py{%endif%} | 4 ++-- ...QL' != cookiecutter.stream_type %}streams.py{%endif%} | 9 +++++---- .../{{cookiecutter.target_id}}/.pre-commit-config.yaml | 8 ++++---- noxfile.py | 7 ++++--- singer_sdk/testing/runners.py | 4 +++- 7 files changed, 24 insertions(+), 20 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 95bc2bad76..0bb8c393b0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,7 +43,7 @@ repos: - id: check-readthedocs - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.280 + rev: v0.0.282 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] @@ -64,7 +64,7 @@ repos: )$ - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 additional_dependencies: diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml index fe3a4d7ca2..12c29e27b0 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml @@ -14,24 +14,24 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.23.0 + rev: 0.23.3 hooks: - id: check-dependabot - id: check-github-workflows - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.269 + rev: v0.0.282 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.7.0 hooks: - id: black - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.3.0 + rev: v1.4.1 hooks: - id: mypy additional_dependencies: diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} index f777e6d008..dae2269dff 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} @@ -159,7 +159,7 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) def get_url_params( self, context: dict | None, # noqa: ARG002 - next_page_token: Any | None, + next_page_token: Any | None, # noqa: ANN401 ) -> dict[str, Any]: """Return a dictionary of values to be used in URL parameterization. @@ -181,7 +181,7 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) def prepare_request_payload( self, context: dict | None, # noqa: ARG002 - next_page_token: Any | None, # noqa: ARG002 + next_page_token: Any | None, # noqa: ARG002, ANN401 ) -> dict | None: """Prepare the data payload for the REST API request. diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} index 4200179509..8272cbc24a 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} @@ -2,6 +2,7 @@ from __future__ import annotations +import typing as t from pathlib import Path from singer_sdk import typing as th # JSON Schema typing helpers @@ -54,7 +55,7 @@ class UsersStream({{ cookiecutter.source_name }}Stream): ), ), ).to_dict() - primary_keys = ["id"] + primary_keys: t.ClassVar[list[str]] = ["id"] replication_key = None graphql_query = """ users { @@ -81,7 +82,7 @@ class GroupsStream({{ cookiecutter.source_name }}Stream): th.Property("id", th.StringType), th.Property("modified", th.DateTimeType), ).to_dict() - primary_keys = ["id"] + primary_keys: t.ClassVar[list[str]] = ["id"] replication_key = "modified" graphql_query = """ groups { @@ -104,7 +105,7 @@ class UsersStream({{ cookiecutter.source_name }}Stream): {%- if cookiecutter.stream_type == "REST" %} path = "/users" {%- endif %} - primary_keys = ["id"] + primary_keys: t.ClassVar[list[str]] = ["id"] replication_key = None # Optionally, you may also use `schema_filepath` in place of `schema`: # schema_filepath = SCHEMAS_DIR / "users.json" # noqa: ERA001 @@ -143,7 +144,7 @@ class GroupsStream({{ cookiecutter.source_name }}Stream): {%- if cookiecutter.stream_type == "REST" %} path = "/groups" {%- endif %} - primary_keys = ["id"] + primary_keys: t.ClassVar[list[str]] = ["id"] replication_key = "modified" schema = th.PropertiesList( th.Property("name", th.StringType), diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml b/cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml index 8d4c83feae..d8bad86cf2 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml @@ -14,24 +14,24 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.23.0 + rev: 0.23.3 hooks: - id: check-dependabot - id: check-github-workflows - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.269 + rev: v0.0.282 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.7.0 hooks: - id: black - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.3.0 + rev: v1.4.1 hooks: - id: mypy additional_dependencies: diff --git a/noxfile.py b/noxfile.py index 7f65b8f218..2838a45036 100644 --- a/noxfile.py +++ b/noxfile.py @@ -2,7 +2,6 @@ from __future__ import annotations -import glob import os import shutil import sys @@ -23,9 +22,11 @@ RUFF_OVERRIDES = """\ extend = "./pyproject.toml" -extend-ignore = ["TD002", "TD003"] +extend-ignore = ["TD002", "TD003", "FIX002"] """ +COOKIECUTTER_REPLAY_FILES = list(Path("./e2e-tests/cookiecutters").glob("*.json")) + package = "singer_sdk" python_versions = ["3.11", "3.10", "3.9", "3.8", "3.7"] main_python_version = "3.10" @@ -185,7 +186,7 @@ def docs_serve(session: Session) -> None: session.run("sphinx-autobuild", *args) -@nox.parametrize("replay_file_path", glob.glob("./e2e-tests/cookiecutters/*.json")) +@nox.parametrize("replay_file_path", COOKIECUTTER_REPLAY_FILES) @session(python=main_python_version) def test_cookiecutter(session: Session, replay_file_path) -> None: """Uses the tap template to build an empty cookiecutter. diff --git a/singer_sdk/testing/runners.py b/singer_sdk/testing/runners.py index 96416de95e..71f2943358 100644 --- a/singer_sdk/testing/runners.py +++ b/singer_sdk/testing/runners.py @@ -242,7 +242,9 @@ def target_input(self) -> t.IO[str]: if self.input_io: self._input = self.input_io elif self.input_filepath: - self._input = Path(self.input_filepath).open(encoding="utf8") + self._input = Path(self.input_filepath).open( # noqa: SIM115 + encoding="utf8", + ) return t.cast(t.IO[str], self._input) @target_input.setter From 6ce7eabcf7dd29c1aebf6d97bdd04d63b8105a4c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Aug 2023 02:24:49 +0000 Subject: [PATCH 05/30] chore(deps): bump cryptography from 41.0.2 to 41.0.3 (#1891) --- poetry.lock | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/poetry.lock b/poetry.lock index 02cfef9476..fe7f32b89c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -569,34 +569,34 @@ toml = ["tomli"] [[package]] name = "cryptography" -version = "41.0.2" +version = "41.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-41.0.2-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:01f1d9e537f9a15b037d5d9ee442b8c22e3ae11ce65ea1f3316a41c78756b711"}, - {file = "cryptography-41.0.2-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:079347de771f9282fbfe0e0236c716686950c19dee1b76240ab09ce1624d76d7"}, - {file = "cryptography-41.0.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:439c3cc4c0d42fa999b83ded80a9a1fb54d53c58d6e59234cfe97f241e6c781d"}, - {file = "cryptography-41.0.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f14ad275364c8b4e525d018f6716537ae7b6d369c094805cae45300847e0894f"}, - {file = "cryptography-41.0.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:84609ade00a6ec59a89729e87a503c6e36af98ddcd566d5f3be52e29ba993182"}, - {file = "cryptography-41.0.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:49c3222bb8f8e800aead2e376cbef687bc9e3cb9b58b29a261210456a7783d83"}, - {file = "cryptography-41.0.2-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:d73f419a56d74fef257955f51b18d046f3506270a5fd2ac5febbfa259d6c0fa5"}, - {file = "cryptography-41.0.2-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:2a034bf7d9ca894720f2ec1d8b7b5832d7e363571828037f9e0c4f18c1b58a58"}, - {file = "cryptography-41.0.2-cp37-abi3-win32.whl", hash = "sha256:d124682c7a23c9764e54ca9ab5b308b14b18eba02722b8659fb238546de83a76"}, - {file = "cryptography-41.0.2-cp37-abi3-win_amd64.whl", hash = "sha256:9c3fe6534d59d071ee82081ca3d71eed3210f76ebd0361798c74abc2bcf347d4"}, - {file = "cryptography-41.0.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a719399b99377b218dac6cf547b6ec54e6ef20207b6165126a280b0ce97e0d2a"}, - {file = "cryptography-41.0.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:182be4171f9332b6741ee818ec27daff9fb00349f706629f5cbf417bd50e66fd"}, - {file = "cryptography-41.0.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7a9a3bced53b7f09da251685224d6a260c3cb291768f54954e28f03ef14e3766"}, - {file = "cryptography-41.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f0dc40e6f7aa37af01aba07277d3d64d5a03dc66d682097541ec4da03cc140ee"}, - {file = "cryptography-41.0.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:674b669d5daa64206c38e507808aae49904c988fa0a71c935e7006a3e1e83831"}, - {file = "cryptography-41.0.2-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7af244b012711a26196450d34f483357e42aeddb04128885d95a69bd8b14b69b"}, - {file = "cryptography-41.0.2-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:9b6d717393dbae53d4e52684ef4f022444fc1cce3c48c38cb74fca29e1f08eaa"}, - {file = "cryptography-41.0.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:192255f539d7a89f2102d07d7375b1e0a81f7478925b3bc2e0549ebf739dae0e"}, - {file = "cryptography-41.0.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f772610fe364372de33d76edcd313636a25684edb94cee53fd790195f5989d14"}, - {file = "cryptography-41.0.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b332cba64d99a70c1e0836902720887fb4529ea49ea7f5462cf6640e095e11d2"}, - {file = "cryptography-41.0.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:9a6673c1828db6270b76b22cc696f40cde9043eb90373da5c2f8f2158957f42f"}, - {file = "cryptography-41.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:342f3767e25876751e14f8459ad85e77e660537ca0a066e10e75df9c9e9099f0"}, - {file = "cryptography-41.0.2.tar.gz", hash = "sha256:7d230bf856164de164ecb615ccc14c7fc6de6906ddd5b491f3af90d3514c925c"}, + {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507"}, + {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922"}, + {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81"}, + {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd"}, + {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47"}, + {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116"}, + {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c"}, + {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae"}, + {file = "cryptography-41.0.3-cp37-abi3-win32.whl", hash = "sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306"}, + {file = "cryptography-41.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574"}, + {file = "cryptography-41.0.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087"}, + {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858"}, + {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906"}, + {file = "cryptography-41.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e"}, + {file = "cryptography-41.0.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd"}, + {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207"}, + {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84"}, + {file = "cryptography-41.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7"}, + {file = "cryptography-41.0.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d"}, + {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de"}, + {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1"}, + {file = "cryptography-41.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4"}, + {file = "cryptography-41.0.3.tar.gz", hash = "sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34"}, ] [package.dependencies] From 65d9dbacb6a000c7ff2058b7b578f57fec56b725 Mon Sep 17 00:00:00 2001 From: Pat Nadolny Date: Thu, 3 Aug 2023 11:19:07 -0400 Subject: [PATCH 06/30] feat: Add a mapper cookiecutter template (#1892) --- .github/workflows/cookiecutter-e2e.yml | 3 + cookiecutter/mapper-template/README.md | 24 +++ .../mapper-template/cookiecutter.json | 9 + ...e_ci_files == 'GitHub' %}test.yml{%endif%} | 30 +++ ...iles == 'GitHub' %}dependabot.yml{%endif%} | 26 +++ .../{{cookiecutter.mapper_id}}/.gitignore | 136 ++++++++++++ .../.pre-commit-config.yaml | 36 ++++ .../.secrets/.gitignore | 10 + .../{{cookiecutter.mapper_id}}/README.md | 128 +++++++++++ .../{{cookiecutter.mapper_id}}/meltano.yml | 31 +++ .../output/.gitignore | 4 + .../{{cookiecutter.mapper_id}}/pyproject.toml | 64 ++++++ .../tests/__init__.py | 1 + .../tests/conftest.py | 3 + .../{{cookiecutter.mapper_id}}/tox.ini | 19 ++ ...== cookiecutter.license %}LICENSE{%endif%} | 202 ++++++++++++++++++ .../{{cookiecutter.library_name}}/__init__.py | 1 + .../{{cookiecutter.library_name}}/mapper.py | 96 +++++++++ e2e-tests/cookiecutters/mapper-base.json | 13 ++ noxfile.py | 19 +- 20 files changed, 846 insertions(+), 9 deletions(-) create mode 100644 cookiecutter/mapper-template/README.md create mode 100644 cookiecutter/mapper-template/cookiecutter.json create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/{% if cookiecutter.include_ci_files == 'GitHub' %}dependabot.yml{%endif%} create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.gitignore create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.secrets/.gitignore create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/README.md create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/meltano.yml create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/output/.gitignore create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/__init__.py create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/conftest.py create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{%if 'Apache-2.0' == cookiecutter.license %}LICENSE{%endif%} create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/__init__.py create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/mapper.py create mode 100644 e2e-tests/cookiecutters/mapper-base.json diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index a88fd6bddf..a33989df45 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -74,7 +74,10 @@ jobs: path: | /tmp/tap-* /tmp/target-* + /tmp/mapper-* !/tmp/tap-*/.mypy_cache/ !/tmp/target-*/.mypy_cache/ + !/tmp/mapper-*/.mypy_cache/ !/tmp/tap-*/.tox/ !/tmp/target-*/.tox/ + !/tmp/mapper-*/.tox/ diff --git a/cookiecutter/mapper-template/README.md b/cookiecutter/mapper-template/README.md new file mode 100644 index 0000000000..70e2e47e84 --- /dev/null +++ b/cookiecutter/mapper-template/README.md @@ -0,0 +1,24 @@ +# Singer Mapper Template + +To use this cookie cutter template: + +```bash +pip3 install pipx +pipx ensurepath +# You may need to reopen your shell at this point +pipx install cookiecutter +``` + +Initialize Cookiecutter template directly from Git: + +```bash +cookiecutter https://github.com/meltano/sdk --directory="cookiecutter/mapper-template" +``` + +Or locally from an already-cloned `sdk` repo: + +```bash +cookiecutter ./sdk/cookiecutter/mapper-template +``` + +See the [dev guide](https://sdk.meltano.com/en/latest/dev_guide.html). diff --git a/cookiecutter/mapper-template/cookiecutter.json b/cookiecutter/mapper-template/cookiecutter.json new file mode 100644 index 0000000000..267e45fb91 --- /dev/null +++ b/cookiecutter/mapper-template/cookiecutter.json @@ -0,0 +1,9 @@ +{ + "name": "MyMapperName", + "admin_name": "FirstName LastName", + "mapper_id": "mapper-{{ cookiecutter.name.lower() }}", + "library_name": "{{ cookiecutter.mapper_id.replace('-', '_') }}", + "variant": "None (Skip)", + "include_ci_files": ["GitHub", "None (Skip)"], + "license": ["Apache-2.0"] +} diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} new file mode 100644 index 0000000000..0cfc81005d --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} @@ -0,0 +1,30 @@ +### A CI workflow template that runs linting and python testing +### TODO: Modify as needed or as desired. + +name: Test {{cookiecutter.mapper_id}} + +on: [push] + +jobs: + pytest: + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: {{ '${{secrets.GITHUB_TOKEN}}' }} + strategy: + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python {{ '${{ matrix.python-version }}' }} + uses: actions/setup-python@v4 + with: + python-version: {{ '${{ matrix.python-version }}' }} + - name: Install Poetry + run: | + pip install poetry + - name: Install dependencies + run: | + poetry install + - name: Test with pytest + run: | + poetry run pytest diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/{% if cookiecutter.include_ci_files == 'GitHub' %}dependabot.yml{%endif%} b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/{% if cookiecutter.include_ci_files == 'GitHub' %}dependabot.yml{%endif%} new file mode 100644 index 0000000000..933e6b1c26 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/{% if cookiecutter.include_ci_files == 'GitHub' %}dependabot.yml{%endif%} @@ -0,0 +1,26 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: "daily" + commit-message: + prefix: "chore(deps): " + prefix-development: "chore(deps-dev): " + - package-ecosystem: pip + directory: "/.github/workflows" + schedule: + interval: daily + commit-message: + prefix: "ci: " + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "ci: " diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.gitignore b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.gitignore new file mode 100644 index 0000000000..475019c316 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.gitignore @@ -0,0 +1,136 @@ +# Secrets and internal config files +**/.secrets/* + +# Ignore meltano internal cache and sqlite systemdb + +.meltano/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml new file mode 100644 index 0000000000..6d9bbbfd57 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml @@ -0,0 +1,36 @@ +ci: + autofix_prs: true + autoupdate_schedule: weekly + autoupdate_commit_msg: 'chore: pre-commit autoupdate' + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-json + - id: check-toml + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + +- repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.23.3 + hooks: + - id: check-dependabot + - id: check-github-workflows + +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.282 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix, --show-fixes] + +- repo: https://github.com/psf/black + rev: 23.7.0 + hooks: + - id: black + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.4.1 + hooks: + - id: mypy diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.secrets/.gitignore b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.secrets/.gitignore new file mode 100644 index 0000000000..33c6acd03e --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.secrets/.gitignore @@ -0,0 +1,10 @@ +# IMPORTANT! This folder is hidden from git - if you need to store config files or other secrets, +# make sure those are never staged for commit into your git repo. You can store them here or another +# secure location. +# +# Note: This may be redundant with the global .gitignore for, and is provided +# for redundancy. If the `.secrets` folder is not needed, you may delete it +# from the project. + +* +!.gitignore diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/README.md b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/README.md new file mode 100644 index 0000000000..ded365fb2c --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/README.md @@ -0,0 +1,128 @@ +# {{ cookiecutter.mapper_id }} + +`{{ cookiecutter.mapper_id }}` is a Singer mapper for {{ cookiecutter.name }}. + +Built with the [Meltano Mapper SDK](https://sdk.meltano.com) for Singer Mappers. + + + +## Configuration + +### Accepted Config Options + + + +A full list of supported settings and capabilities for this +mapper is available by running: + +```bash +{{ cookiecutter.mapper_id }} --about +``` + +### Configure using environment variables + +This Singer mapper will automatically import any environment variables within the working directory's +`.env` if the `--config=ENV` is provided, such that config values will be considered if a matching +environment variable is set either in the terminal context or in the `.env` file. + +### Source Authentication and Authorization + + + +## Usage + +You can easily run `{{ cookiecutter.mapper_id }}` by itself or in a pipeline using [Meltano](https://meltano.com/). + +### Executing the Mapper Directly + +```bash +{{ cookiecutter.mapper_id }} --version +{{ cookiecutter.mapper_id }} --help +``` + +## Developer Resources + +Follow these instructions to contribute to this project. + +### Initialize your Development Environment + +```bash +pipx install poetry +poetry install +``` + +### Create and Run Tests + +Create tests within the `tests` subfolder and + then run: + +```bash +poetry run pytest +``` + +You can also test the `{{cookiecutter.mapper_id}}` CLI interface directly using `poetry run`: + +```bash +poetry run {{cookiecutter.mapper_id}} --help +``` + +### Testing with [Meltano](https://www.meltano.com) + +_**Note:** This mapper will work in any Singer environment and does not require Meltano. +Examples here are for convenience and to streamline end-to-end orchestration scenarios._ + + + +Next, install Meltano (if you haven't already) and any needed plugins: + +```bash +# Install meltano +pipx install meltano +# Initialize meltano within this directory +cd {{ cookiecutter.mapper_id }} +meltano install +``` + +Now you can test and orchestrate using Meltano: + +```bash +# Run a test `run` pipeline: +meltano run tap-smoke-test {{ cookiecutter.mapper_id }} target-jsonl +``` + +### SDK Dev Guide + +See the [dev guide](https://sdk.meltano.com/en/latest/dev_guide.html) for more instructions on how to use the SDK to +develop your own taps, targets, and mappers. diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/meltano.yml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/meltano.yml new file mode 100644 index 0000000000..019015d06e --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/meltano.yml @@ -0,0 +1,31 @@ +version: 1 +send_anonymous_usage_stats: true +project_id: "{{cookiecutter.mapper_id}}" +default_environment: test +environments: +- name: test +plugins: + extractors: + - name: tap-smoke-test + variant: meltano + pip_url: git+https://github.com/meltano/tap-smoke-test.git + config: + streams: + - stream_name: animals + input_filename: https://raw.githubusercontent.com/meltano/tap-smoke-test/main/demo-data/animals-data.jsonl + loaders: + - name: target-jsonl + variant: andyh1203 + pip_url: target-jsonl + mappers: + - name: "{{cookiecutter.mapper_id}}" + pip_url: -e . + namespace: "{{cookiecutter.library_name}}" + # TODO: replace these with the actual settings + settings: + - name: example_config + kind: string + mappings: + - name: example + config: + example_config: foo diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/output/.gitignore b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/output/.gitignore new file mode 100644 index 0000000000..80ff9d2a61 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/output/.gitignore @@ -0,0 +1,4 @@ +# This directory is used as a target by target-jsonl, so ignore all files + +* +!.gitignore diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml new file mode 100644 index 0000000000..9947e314eb --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml @@ -0,0 +1,64 @@ +[tool.poetry] +{%- if cookiecutter.variant != "None (Skip)" %} +name = "{{cookiecutter.variant}}-{{cookiecutter.mapper_id}}" +{%- else %} +name = "{{cookiecutter.mapper_id}}" +{%- endif %} +version = "0.0.1" +description = "`{{cookiecutter.mapper_id}}` is a Singer mapper {{cookiecutter.name}}, built with the Meltano Singer SDK." +readme = "README.md" +authors = ["{{ cookiecutter.admin_name }}"] +keywords = [ + "ELT", + "Mapper", + "{{cookiecutter.name}}", +] +license = "Apache-2.0" +{%- if cookiecutter.variant != "None (Skip)" %} +packages = [ + { include = "{{cookiecutter.library_name}}" }, +] +{%- endif %} + +[tool.poetry.dependencies] +python = "<3.12,>=3.7.1" +singer-sdk = { version="^0.30.0" } +fs-s3fs = { version = "^1.1.1", optional = true } + +[tool.poetry.group.dev.dependencies] +pytest = "^7.2.1" +singer-sdk = { version="^0.30.0", extras = ["testing"] } + +[tool.poetry.extras] +s3 = ["fs-s3fs"] + +[tool.mypy] +python_version = "3.9" +warn_unused_configs = true + +[tool.ruff] +ignore = [ + "ANN101", # missing-type-self + "ANN102", # missing-type-cls +] +select = ["ALL"] +src = ["{{cookiecutter.library_name}}"] +target-version = "py37" + + +[tool.ruff.flake8-annotations] +allow-star-arg-any = true + +[tool.ruff.isort] +known-first-party = ["{{cookiecutter.library_name}}"] + +[tool.ruff.pydocstyle] +convention = "google" + +[build-system] +requires = ["poetry-core>=1.0.8"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.scripts] +# CLI declaration +{{cookiecutter.mapper_id}} = '{{cookiecutter.library_name}}.mapper:{{cookiecutter.name}}Mapper.cli' diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/__init__.py b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/__init__.py new file mode 100644 index 0000000000..7caba56f78 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for {{ cookiecutter.mapper_id }}.""" diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/conftest.py b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/conftest.py new file mode 100644 index 0000000000..6bb3ec2d7a --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/conftest.py @@ -0,0 +1,3 @@ +"""Test Configuration.""" + +pytest_plugins = ("singer_sdk.testing.pytest_plugin",) diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini new file mode 100644 index 0000000000..70b9e4ac7e --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini @@ -0,0 +1,19 @@ +# This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy + +[tox] +envlist = py37, py38, py39, py310, py311 +isolated_build = true + +[testenv] +allowlist_externals = poetry +commands = + poetry install -v + poetry run pytest + +[testenv:pytest] +# Run the python tests. +# To execute, run `tox -e pytest` +envlist = py37, py38, py39, py310, py311 +commands = + poetry install -v + poetry run pytest diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{%if 'Apache-2.0' == cookiecutter.license %}LICENSE{%endif%} b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{%if 'Apache-2.0' == cookiecutter.license %}LICENSE{%endif%} new file mode 100644 index 0000000000..62913ff3af --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{%if 'Apache-2.0' == cookiecutter.license %}LICENSE{%endif%} @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + + Copyright {% now 'utc', '%Y' %} {{ cookiecutter.admin_name }} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/__init__.py b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/__init__.py new file mode 100644 index 0000000000..5781fbbc43 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/__init__.py @@ -0,0 +1 @@ +"""{{ cookiecutter.name }} Mapper.""" diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/mapper.py b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/mapper.py new file mode 100644 index 0000000000..c8c3d23ec8 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/mapper.py @@ -0,0 +1,96 @@ +"""{{ cookiecutter.name }} mapper class.""" + +from __future__ import annotations + +import typing as t +from typing import TYPE_CHECKING + +import singer_sdk.typing as th +from singer_sdk import _singerlib as singer +from singer_sdk.mapper import PluginMapper +from singer_sdk.mapper_base import InlineMapper + +if TYPE_CHECKING: + from pathlib import PurePath + + +class {{ cookiecutter.name }}Mapper(InlineMapper): + """Sample mapper for {{ cookiecutter.name }}.""" + + name = "{{ cookiecutter.mapper_id }}" + + config_jsonschema = th.PropertiesList( + # TODO: Replace or remove this example config based on your needs + th.Property( + "example_config", + th.StringType, + description="An example config, replace or remove based on your needs.", + ), + ).to_dict() + + def __init__( + self, + *, + config: dict | PurePath | str | list[PurePath | str] | None = None, + parse_env_config: bool = False, + validate_config: bool = True, + ) -> None: + """Create a new inline mapper. + + Args: + config: Mapper configuration. Can be a dictionary, a single path to a + configuration file, or a list of paths to multiple configuration + files. + parse_env_config: Whether to look for configuration values in environment + variables. + validate_config: True to require validation of config settings. + """ + super().__init__( + config=config, + parse_env_config=parse_env_config, + validate_config=validate_config, + ) + + self.mapper = PluginMapper(plugin_config=dict(self.config), logger=self.logger) + + def map_schema_message(self, message_dict: dict) -> t.Iterable[singer.Message]: + """Map a schema message to zero or more new messages. + + Args: + message_dict: A SCHEMA message JSON dictionary. + """ + yield singer.SchemaMessage.from_dict(message_dict) + + def map_record_message( + self, + message_dict: dict, + ) -> t.Iterable[singer.RecordMessage]: + """Map a record message to zero or more new messages. + + Args: + message_dict: A RECORD message JSON dictionary. + """ + yield singer.RecordMessage.from_dict(message_dict) + + def map_state_message(self, message_dict: dict) -> t.Iterable[singer.Message]: + """Map a state message to zero or more new messages. + + Args: + message_dict: A STATE message JSON dictionary. + """ + yield singer.StateMessage.from_dict(message_dict) + + def map_activate_version_message( + self, + message_dict: dict, + ) -> t.Iterable[singer.Message]: + """Map a version message to zero or more new messages. + + Args: + message_dict: An ACTIVATE_VERSION message JSON dictionary. + """ + yield singer.ActivateVersionMessage.from_dict(message_dict) + + +if __name__ == "__main__": + {{ cookiecutter.name }}Mapper.cli() diff --git a/e2e-tests/cookiecutters/mapper-base.json b/e2e-tests/cookiecutters/mapper-base.json new file mode 100644 index 0000000000..25d40c6085 --- /dev/null +++ b/e2e-tests/cookiecutters/mapper-base.json @@ -0,0 +1,13 @@ +{ + "cookiecutter": { + "name": "MyMapperName", + "admin_name": "Automatic Tester", + "mapper_id": "mapper-base", + "library_name": "mapper_base", + "variant": "None (Skip)", + "include_ci_files": "None (Skip)", + "license": "Apache-2.0", + "_template": "../mapper-template/", + "_output_dir": "." + } +} diff --git a/noxfile.py b/noxfile.py index 2838a45036..cbb331faf0 100644 --- a/noxfile.py +++ b/noxfile.py @@ -196,21 +196,22 @@ def test_cookiecutter(session: Session, replay_file_path) -> None: cc_build_path = tempfile.gettempdir() folder_base_path = "./cookiecutter" - target_folder = ( - "tap-template" - if Path(replay_file_path).name.startswith("tap") - else "target-template" - ) - tap_template = Path(folder_base_path + "/" + target_folder).resolve() + if Path(replay_file_path).name.startswith("tap"): + folder = "tap-template" + elif Path(replay_file_path).name.startswith("target"): + folder = "target-template" + else: + folder = "mapper-template" + template = Path(folder_base_path + "/" + folder).resolve() replay_file = Path(replay_file_path).resolve() - if not Path(tap_template).exists(): + if not Path(template).exists(): return if not Path(replay_file).is_file(): return - sdk_dir = Path(Path(tap_template).parent).parent + sdk_dir = Path(Path(template).parent).parent cc_output_dir = Path(replay_file_path).name.replace(".json", "") cc_test_output = cc_build_path + "/" + cc_output_dir @@ -224,7 +225,7 @@ def test_cookiecutter(session: Session, replay_file_path) -> None: "cookiecutter", "--replay-file", str(replay_file), - str(tap_template), + str(template), "-o", cc_build_path, ) From bb56a1a8b53f3a90d145b1691f6a709772578dda Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Aug 2023 16:48:32 +0000 Subject: [PATCH 07/30] chore(deps-dev): bump cookiecutter from 2.2.2 to 2.3.0 (#1895) --- .../{{cookiecutter.mapper_id}}/pyproject.toml | 2 +- .../{{cookiecutter.tap_id}}/pyproject.toml | 2 +- .../{{cookiecutter.target_id}}/pyproject.toml | 2 +- poetry.lock | 34 +++++++++++++++---- pyproject.toml | 2 +- 5 files changed, 31 insertions(+), 11 deletions(-) diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml index 9947e314eb..59c484df15 100644 --- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml @@ -26,7 +26,7 @@ singer-sdk = { version="^0.30.0" } fs-s3fs = { version = "^1.1.1", optional = true } [tool.poetry.group.dev.dependencies] -pytest = "^7.2.1" +pytest = "^7.4.0" singer-sdk = { version="^0.30.0", extras = ["testing"] } [tool.poetry.extras] diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml index 3eaed335bd..4be0637197 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml @@ -31,7 +31,7 @@ cached-property = "^1" # Remove after Python 3.7 support is dropped {%- endif %} [tool.poetry.group.dev.dependencies] -pytest = "^7.2.1" +pytest = "^7.4.0" singer-sdk = { version="^0.30.0", extras = ["testing"] } [tool.poetry.extras] diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml index b612331112..16f3f80c86 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml @@ -28,7 +28,7 @@ requests = "^2.31.0" {%- endif %} [tool.poetry.dev-dependencies] -pytest = "^7.2.1" +pytest = "^7.4.0" singer-sdk = { version="^0.30.0", extras = ["testing"] } [tool.poetry.extras] diff --git a/poetry.lock b/poetry.lock index fe7f32b89c..b3b461ea2c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -474,13 +474,13 @@ resolved_reference = "e2e6d5d13d39eae1f37e3a275c0d3d3e38c18439" [[package]] name = "cookiecutter" -version = "2.2.2" +version = "2.3.0" description = "A command-line utility that creates projects from project templates, e.g. creating a Python package project from a Python package project template." optional = false python-versions = ">=3.7" files = [ - {file = "cookiecutter-2.2.2-py3-none-any.whl", hash = "sha256:4feb7485520dd7453e3094d8f3955601156a0fab0d0b90a2c8c74f6dc2cbaac6"}, - {file = "cookiecutter-2.2.2.tar.gz", hash = "sha256:3b475d17573a7785b4a22fab693be249840e235a92c93c0fa088b39e9193f194"}, + {file = "cookiecutter-2.3.0-py3-none-any.whl", hash = "sha256:7e87944757c6e9f8729cf89a4139b6a35ab4d6dcbc6ae3e7d6360d44ad3ad383"}, + {file = "cookiecutter-2.3.0.tar.gz", hash = "sha256:942a794981747f6d7f439d6e49d39dc91a9a641283614160c93c474c72c29621"}, ] [package.dependencies] @@ -491,6 +491,7 @@ Jinja2 = ">=2.7,<4.0.0" python-slugify = ">=4.0.0" pyyaml = ">=5.3.1" requests = ">=2.23.0" +rich = "*" [[package]] name = "coverage" @@ -1054,7 +1055,7 @@ tornado = {version = "*", markers = "python_version > \"2.7\""} name = "markdown-it-py" version = "2.2.0" description = "Python port of markdown-it. Markdown parsing, done right!" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "markdown-it-py-2.2.0.tar.gz", hash = "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"}, @@ -1168,7 +1169,7 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, @@ -1584,7 +1585,7 @@ requests = ">=2.14.0" name = "pygments" version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "Pygments-2.15.1-py3-none-any.whl", hash = "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"}, @@ -1901,6 +1902,25 @@ six = "*" fixture = ["fixtures"] test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] +[[package]] +name = "rich" +version = "13.5.2" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "rich-13.5.2-py3-none-any.whl", hash = "sha256:146a90b3b6b47cac4a73c12866a499e9817426423f57c5a66949c086191a8808"}, + {file = "rich-13.5.2.tar.gz", hash = "sha256:fb9d6c0a0f643c99eed3875b5377a184132ba9be4d61516a55273d3554d75a39"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + [[package]] name = "s3transfer" version = "0.6.1" @@ -2702,4 +2722,4 @@ testing = ["pytest", "pytest-durations"] [metadata] lock-version = "2.0" python-versions = "<3.12,>=3.7.1" -content-hash = "9ac1398cdb5cd011452f152cbd95412139f320881bb2d27855dc52253b313332" +content-hash = "ce7485cc6dffcd5b2fbb8f8f4c3cf0bd7e21d91aad64f6c019683c127f3be952" diff --git a/pyproject.toml b/pyproject.toml index f70055d1de..bceced1d86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,7 +101,7 @@ testing = [ commitizen-version-bump = { git = "https://github.com/meltano/commitizen-version-bump.git", branch = "main" } xdoctest = "^1.1.1" mypy = "^1.0" -cookiecutter = ">=2.1.1,<2.2.3" +cookiecutter = ">=2.1.1,<2.3.1" PyYAML = "^6.0" freezegun = "^1.2.2" numpy = [ From af2e4889c5b2e8fd925507dab5473c52fd1e576e Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Mon, 7 Aug 2023 14:59:47 -0600 Subject: [PATCH 08/30] fix(targets): Correctly serialize `decimal.Decimal` in JSON fields of SQL targets (#1898) --- pyproject.toml | 6 ++--- singer_sdk/connectors/sql.py | 42 +++++++++++++++++++++++++++++++- tests/core/test_connector_sql.py | 25 +++++++++++++++++++ 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bceced1d86..a795820d92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,8 +2,8 @@ name = "singer-sdk" version = "0.30.0" description = "A framework for building Singer taps" -authors = ["Meltano Team and Contributors"] -maintainers = ["Meltano Team and Contributors"] +authors = ["Meltano Team and Contributors "] +maintainers = ["Meltano Team and Contributors "] readme = "README.md" homepage = "https://sdk.meltano.com/en/latest/" repository = "https://github.com/meltano/sdk" @@ -144,7 +144,7 @@ name = "cz_version_bump" version = "0.30.0" tag_format = "v$major.$minor.$patch$prerelease" version_files = [ - "docs/conf.py", + "docs/conf.py:^release =", "pyproject.toml:^version =", "cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml:singer-sdk", "cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml:singer-sdk", diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index e9a65cf80d..e05e359dae 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -2,6 +2,8 @@ from __future__ import annotations +import decimal +import json import logging import typing as t import warnings @@ -9,6 +11,7 @@ from datetime import datetime from functools import lru_cache +import simplejson import sqlalchemy from sqlalchemy.engine import Engine @@ -316,7 +319,12 @@ def create_engine(self) -> Engine: Returns: A new SQLAlchemy Engine. """ - return sqlalchemy.create_engine(self.sqlalchemy_url, echo=False) + return sqlalchemy.create_engine( + self.sqlalchemy_url, + echo=False, + json_serializer=self.serialize_json, + json_deserializer=self.deserialize_json, + ) def quote(self, name: str) -> str: """Quote a name if it needs quoting, using '.' as a name-part delimiter. @@ -1124,3 +1132,35 @@ def _adapt_column_type( ) with self._connect() as conn: conn.execute(alter_column_ddl) + + def serialize_json(self, obj: object) -> str: + """Serialize an object to a JSON string. + + Target connectors may override this method to provide custom serialization logic + for JSON types. + + Args: + obj: The object to serialize. + + Returns: + The JSON string. + + .. versionadded:: 0.31.0 + """ + return simplejson.dumps(obj, use_decimal=True) + + def deserialize_json(self, json_str: str) -> object: + """Deserialize a JSON string to an object. + + Tap connectors may override this method to provide custom deserialization + logic for JSON types. + + Args: + json_str: The JSON string to deserialize. + + Returns: + The deserialized object. + + .. versionadded:: 0.31.0 + """ + return json.loads(json_str, parse_float=decimal.Decimal) diff --git a/tests/core/test_connector_sql.py b/tests/core/test_connector_sql.py index 1c04dbcdd6..58ba59ec7f 100644 --- a/tests/core/test_connector_sql.py +++ b/tests/core/test_connector_sql.py @@ -1,5 +1,6 @@ from __future__ import annotations +from decimal import Decimal from unittest import mock import pytest @@ -258,3 +259,27 @@ def test_merge_generic_sql_types( ): merged_type = connector.merge_sql_types(types) assert isinstance(merged_type, expected_type) + + def test_engine_json_serialization(self, connector: SQLConnector): + engine = connector._engine + meta = sqlalchemy.MetaData() + table = sqlalchemy.Table( + "test_table", + meta, + sqlalchemy.Column("id", sqlalchemy.Integer, primary_key=True), + sqlalchemy.Column("attrs", sqlalchemy.JSON), + ) + meta.create_all(engine) + with engine.connect() as conn: + conn.execute( + table.insert(), + [ + {"attrs": {"x": Decimal("1.0")}}, + {"attrs": {"x": Decimal("2.0"), "y": [1, 2, 3]}}, + ], + ) + result = conn.execute(table.select()) + assert result.fetchall() == [ + (1, {"x": Decimal("1.0")}), + (2, {"x": Decimal("2.0"), "y": [1, 2, 3]}), + ] From f4af551e6d1fea84a195beb8c5f787671057b283 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 7 Aug 2023 15:56:03 -0600 Subject: [PATCH 09/30] chore: Release v0.31.0 (#1899) chore: Bump package version Co-authored-by: edgarrmondragon --- .github/ISSUE_TEMPLATE/bug.yml | 2 +- CHANGELOG.md | 30 +++++++++++++++++++ .../{{cookiecutter.tap_id}}/pyproject.toml | 4 +-- .../{{cookiecutter.target_id}}/pyproject.toml | 4 +-- docs/conf.py | 2 +- pyproject.toml | 4 +-- 6 files changed, 38 insertions(+), 8 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 7d3b1a8974..45d9aabf1b 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -15,7 +15,7 @@ body: attributes: label: Singer SDK Version description: Version of the library you are using - placeholder: "0.30.0" + placeholder: "0.31.0" validations: required: true - type: checkboxes diff --git a/CHANGELOG.md b/CHANGELOG.md index 56198b5973..b7d772080f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,36 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v0.31.0 (2023-08-07) + +### ✨ New + +- [#1892](https://github.com/meltano/sdk/issues/1892) Add a mapper cookiecutter template +- [#1864](https://github.com/meltano/sdk/issues/1864) SQLTarget connector instance shared with sinks -- _**Thanks @BuzzCutNorman!**_ +- [#1878](https://github.com/meltano/sdk/issues/1878) Add `_sdc_sync_started_at` metadata column to indicate the start of the target process +- [#1484](https://github.com/meltano/sdk/issues/1484) Bump latest supported sqlalchemy from `1.*` to `2.*` + +### 🐛 Fixes + +- [#1898](https://github.com/meltano/sdk/issues/1898) Correctly serialize `decimal.Decimal` in JSON fields of SQL targets +- [#1881](https://github.com/meltano/sdk/issues/1881) Expose `add_record_metadata` as a builtin target setting +- [#1880](https://github.com/meltano/sdk/issues/1880) Append batch config if target supports the batch capability +- [#1865](https://github.com/meltano/sdk/issues/1865) Handle missing record properties in SQL sinks +- [#1838](https://github.com/meltano/sdk/issues/1838) Add deprecation warning when importing legacy testing helpers +- [#1842](https://github.com/meltano/sdk/issues/1842) Ensure all expected tap parameters are passed to `SQLTap` initializer +- [#1853](https://github.com/meltano/sdk/issues/1853) Check against the unconformed key properties when validating record keys +- [#1843](https://github.com/meltano/sdk/issues/1843) Target template should not reference `tap_id` +- [#1708](https://github.com/meltano/sdk/issues/1708) Finalize and write last state message with dedupe +- [#1835](https://github.com/meltano/sdk/issues/1835) Avoid setting up mapper in discovery mode + +### ⚙️ Under the Hood + +- [#1877](https://github.com/meltano/sdk/issues/1877) Use `importlib.resources` instead of `__file__` to retrieve sample Singer output files + +### 📚 Documentation Improvements + +- [#1852](https://github.com/meltano/sdk/issues/1852) Fix stale `pip_url` example that uses shell script workaround for editable installation + ## v0.30.0 (2023-07-10) ### ✨ New diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml index 4be0637197..6c1e88fbfb 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml @@ -21,7 +21,7 @@ packages = [ [tool.poetry.dependencies] python = "<3.12,>=3.7.1" -singer-sdk = { version="^0.30.0" } +singer-sdk = { version="^0.31.0" } fs-s3fs = { version = "^1.1.1", optional = true } {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %} requests = "^2.31.0" @@ -32,7 +32,7 @@ cached-property = "^1" # Remove after Python 3.7 support is dropped [tool.poetry.group.dev.dependencies] pytest = "^7.4.0" -singer-sdk = { version="^0.30.0", extras = ["testing"] } +singer-sdk = { version="^0.31.0", extras = ["testing"] } [tool.poetry.extras] s3 = ["fs-s3fs"] diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml index 16f3f80c86..ee8a65192d 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml @@ -21,7 +21,7 @@ packages = [ [tool.poetry.dependencies] python = "<3.12,>=3.7.1" -singer-sdk = { version="^0.30.0" } +singer-sdk = { version="^0.31.0" } fs-s3fs = { version = "^1.1.1", optional = true } {%- if cookiecutter.serialization_method != "SQL" %} requests = "^2.31.0" @@ -29,7 +29,7 @@ requests = "^2.31.0" [tool.poetry.dev-dependencies] pytest = "^7.4.0" -singer-sdk = { version="^0.30.0", extras = ["testing"] } +singer-sdk = { version="^0.31.0", extras = ["testing"] } [tool.poetry.extras] s3 = ["fs-s3fs"] diff --git a/docs/conf.py b/docs/conf.py index 61ac4b0717..14562fdc02 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,7 +26,7 @@ author = "Meltano Core Team and Contributors" # The full version, including alpha/beta/rc tags -release = "0.30.0" +release = "0.31.0" # -- General configuration --------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index a795820d92..eb691dd8ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "singer-sdk" -version = "0.30.0" +version = "0.31.0" description = "A framework for building Singer taps" authors = ["Meltano Team and Contributors "] maintainers = ["Meltano Team and Contributors "] @@ -141,7 +141,7 @@ norecursedirs = "cookiecutter" [tool.commitizen] name = "cz_version_bump" -version = "0.30.0" +version = "0.31.0" tag_format = "v$major.$minor.$patch$prerelease" version_files = [ "docs/conf.py:^release =", From 7f2df99148e28a65a5dd53f915041fe8f652b03f Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Tue, 8 Aug 2023 07:35:14 -0600 Subject: [PATCH 10/30] chore: Bump SDK version in mapper template (#1900) --- .../mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml index 59c484df15..62fd97284f 100644 --- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml @@ -22,7 +22,7 @@ packages = [ [tool.poetry.dependencies] python = "<3.12,>=3.7.1" -singer-sdk = { version="^0.30.0" } +singer-sdk = { version="^0.31.0" } fs-s3fs = { version = "^1.1.1", optional = true } [tool.poetry.group.dev.dependencies] diff --git a/pyproject.toml b/pyproject.toml index eb691dd8ca..4a3601198b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -148,6 +148,7 @@ version_files = [ "pyproject.toml:^version =", "cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml:singer-sdk", "cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml:singer-sdk", + "cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml:singer-sdk", ".github/ISSUE_TEMPLATE/bug.yml:^ placeholder:", ] From 17ab44afa3cc6205748d5c3ad0e34bf55be4edfe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 13:44:00 +0000 Subject: [PATCH 11/30] chore(deps): bump actions/dependency-review-action from 3.0.6 to 3.0.7 (#1908) --- .github/workflows/dependency-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 8048f9cad2..c676791a8d 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -20,7 +20,7 @@ jobs: - name: GitHub dependency vulnerability check if: ${{ github.event_name == 'pull_request_target' }} - uses: actions/dependency-review-action@v3.0.6 + uses: actions/dependency-review-action@v3.0.7 with: fail-on-severity: high From 619bc3dc56d460277c9324ae03262868cf2f6ddd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 13:56:43 +0000 Subject: [PATCH 12/30] chore(deps-dev): bump types-pytz from 2023.3.0.0 to 2023.3.0.1 (#1906) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index b3b461ea2c..8538594a2b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2494,13 +2494,13 @@ files = [ [[package]] name = "types-pytz" -version = "2023.3.0.0" +version = "2023.3.0.1" description = "Typing stubs for pytz" optional = false python-versions = "*" files = [ - {file = "types-pytz-2023.3.0.0.tar.gz", hash = "sha256:ecdc70d543aaf3616a7e48631543a884f74205f284cefd6649ddf44c6a820aac"}, - {file = "types_pytz-2023.3.0.0-py3-none-any.whl", hash = "sha256:4fc2a7fbbc315f0b6630e0b899fd6c743705abe1094d007b0e612d10da15e0f3"}, + {file = "types-pytz-2023.3.0.1.tar.gz", hash = "sha256:1a7b8d4aac70981cfa24478a41eadfcd96a087c986d6f150d77e3ceb3c2bdfab"}, + {file = "types_pytz-2023.3.0.1-py3-none-any.whl", hash = "sha256:65152e872137926bb67a8fe6cc9cfd794365df86650c5d5fdc7b167b0f38892e"}, ] [[package]] From 36f74363f0eaa9c2e2114af6a0d7de2790a20133 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 14:17:58 +0000 Subject: [PATCH 13/30] chore(deps): bump pypa/gh-action-pypi-publish from 1.8.8 to 1.8.10 (#1907) --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 73eea59534..9f85a1a7cb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,4 +56,4 @@ jobs: file_glob: true - name: Publish - uses: pypa/gh-action-pypi-publish@v1.8.8 + uses: pypa/gh-action-pypi-publish@v1.8.10 From e65e3e59979da43f54c92045144bbcddea81e05b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:38:43 -0600 Subject: [PATCH 14/30] chore: pre-commit autoupdate (#1909) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/python-jsonschema/check-jsonschema: 0.23.3 → 0.24.1](https://github.com/python-jsonschema/check-jsonschema/compare/0.23.3...0.24.1) - [github.com/astral-sh/ruff-pre-commit: v0.0.282 → v0.0.284](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.282...v0.0.284) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0bb8c393b0..fc1e804d08 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,14 +36,14 @@ repos: )$ - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.23.3 + rev: 0.24.1 hooks: - id: check-dependabot - id: check-github-workflows - id: check-readthedocs - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.282 + rev: v0.0.284 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] From 59c3e3dfc6c40a88c7cb504e399c612314b54ddf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 12:52:36 +0000 Subject: [PATCH 15/30] chore(deps): bump joblib from 1.3.1 to 1.3.2 (#1910) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8538594a2b..964a6b899d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -987,13 +987,13 @@ files = [ [[package]] name = "joblib" -version = "1.3.1" +version = "1.3.2" description = "Lightweight pipelining with Python functions" optional = false python-versions = ">=3.7" files = [ - {file = "joblib-1.3.1-py3-none-any.whl", hash = "sha256:89cf0529520e01b3de7ac7b74a8102c90d16d54c64b5dd98cafcd14307fdf915"}, - {file = "joblib-1.3.1.tar.gz", hash = "sha256:1f937906df65329ba98013dc9692fe22a4c5e4a648112de500508b18a21b41e3"}, + {file = "joblib-1.3.2-py3-none-any.whl", hash = "sha256:ef4331c65f239985f3f2220ecc87db222f08fd22097a3dd5698f693875f8cbb9"}, + {file = "joblib-1.3.2.tar.gz", hash = "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1"}, ] [[package]] From 1f62ae45888e908bc93afd1b8e047288deb2f964 Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Wed, 16 Aug 2023 08:56:36 -0600 Subject: [PATCH 16/30] feat(templates): Add email field and use human-readable questions in templates (#1905) --- cookiecutter/mapper-template/cookiecutter.json | 12 +++++++++++- .../{{cookiecutter.mapper_id}}/pyproject.toml | 2 +- cookiecutter/tap-template/cookiecutter.json | 14 +++++++++++++- .../{{cookiecutter.tap_id}}/pyproject.toml | 2 +- cookiecutter/target-template/cookiecutter.json | 13 ++++++++++++- .../{{cookiecutter.target_id}}/pyproject.toml | 2 +- docs/dev_guide.md | 18 ++++++++++++++++++ e2e-tests/cookiecutters/mapper-base.json | 1 + e2e-tests/cookiecutters/tap-graphql-jwt.json | 1 + e2e-tests/cookiecutters/tap-other-custom.json | 1 + .../cookiecutters/tap-rest-api_key-github.json | 1 + .../cookiecutters/tap-rest-basic_auth.json | 1 + .../cookiecutters/tap-rest-bearer_token.json | 1 + e2e-tests/cookiecutters/tap-rest-custom.json | 1 + e2e-tests/cookiecutters/tap-rest-jwt.json | 1 + e2e-tests/cookiecutters/tap-rest-oauth2.json | 1 + e2e-tests/cookiecutters/tap-sql-custom.json | 1 + e2e-tests/cookiecutters/target-per_record.json | 1 + e2e-tests/cookiecutters/target-sql.json | 1 + 19 files changed, 69 insertions(+), 6 deletions(-) diff --git a/cookiecutter/mapper-template/cookiecutter.json b/cookiecutter/mapper-template/cookiecutter.json index 267e45fb91..c42b1cf06f 100644 --- a/cookiecutter/mapper-template/cookiecutter.json +++ b/cookiecutter/mapper-template/cookiecutter.json @@ -1,9 +1,19 @@ { "name": "MyMapperName", "admin_name": "FirstName LastName", + "admin_email": "firstname.lastname@example.com", "mapper_id": "mapper-{{ cookiecutter.name.lower() }}", "library_name": "{{ cookiecutter.mapper_id.replace('-', '_') }}", "variant": "None (Skip)", "include_ci_files": ["GitHub", "None (Skip)"], - "license": ["Apache-2.0"] + "license": ["Apache-2.0"], + "__prompts__": { + "name": "The name of the mapper, in CamelCase", + "admin_name": "Provide your [bold yellow]full name[/]", + "admin_email": "Provide your [bold yellow]email[/]", + "mapper_id": "The ID of the tap, in kebab-case", + "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.", + "include_ci_files": "Whether to include CI files for a common CI services", + "license": "The license for the project" + } } diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml index 62fd97284f..36c541fcab 100644 --- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml @@ -7,7 +7,7 @@ name = "{{cookiecutter.mapper_id}}" version = "0.0.1" description = "`{{cookiecutter.mapper_id}}` is a Singer mapper {{cookiecutter.name}}, built with the Meltano Singer SDK." readme = "README.md" -authors = ["{{ cookiecutter.admin_name }}"] +authors = ["{{ cookiecutter.admin_name }} <{{ cookiecutter.admin_email }}>"] keywords = [ "ELT", "Mapper", diff --git a/cookiecutter/tap-template/cookiecutter.json b/cookiecutter/tap-template/cookiecutter.json index 8da7a099b1..e297aae540 100644 --- a/cookiecutter/tap-template/cookiecutter.json +++ b/cookiecutter/tap-template/cookiecutter.json @@ -1,6 +1,7 @@ { "source_name": "MySourceName", "admin_name": "FirstName LastName", + "admin_email": "firstname.lastname@example.com", "tap_id": "tap-{{ cookiecutter.source_name.lower() }}", "library_name": "{{ cookiecutter.tap_id.replace('-', '_') }}", "variant": "None (Skip)", @@ -14,5 +15,16 @@ "Custom or N/A" ], "include_ci_files": ["GitHub", "None (Skip)"], - "license": ["Apache-2.0"] + "license": ["Apache-2.0"], + "__prompts__": { + "source_name": "The name of the source, in CamelCase", + "admin_name": "Provide your [bold yellow]full name[/]", + "admin_email": "Provide your [bold yellow]email[/]", + "tap_id": "The ID of the tap, in kebab-case", + "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.", + "stream_type": "The type of stream the source provides", + "auth_method": "The [bold red]authentication[/] method used by the source, for REST and GraphQL sources", + "include_ci_files": "Whether to include CI files for a common CI services", + "license": "The license for the project" + } } diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml index 6c1e88fbfb..46ffb61f75 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml @@ -7,7 +7,7 @@ name = "{{cookiecutter.tap_id}}" version = "0.0.1" description = "`{{cookiecutter.tap_id}}` is a Singer tap for {{cookiecutter.source_name}}, built with the Meltano Singer SDK." readme = "README.md" -authors = ["{{ cookiecutter.admin_name }}"] +authors = ["{{ cookiecutter.admin_name }} <{{ cookiecutter.admin_email }}>"] keywords = [ "ELT", "{{cookiecutter.source_name}}", diff --git a/cookiecutter/target-template/cookiecutter.json b/cookiecutter/target-template/cookiecutter.json index 4816b54aa2..c7c31835ab 100644 --- a/cookiecutter/target-template/cookiecutter.json +++ b/cookiecutter/target-template/cookiecutter.json @@ -1,10 +1,21 @@ { "destination_name": "MyDestinationName", "admin_name": "FirstName LastName", + "admin_email": "firstname.lastname@example.com", "target_id": "target-{{ cookiecutter.destination_name.lower() }}", "library_name": "{{ cookiecutter.target_id.replace('-', '_') }}", "variant": "None (Skip)", "serialization_method": ["Per record", "Per batch", "SQL"], "include_ci_files": ["GitHub", "None (Skip)"], - "license": ["Apache-2.0"] + "license": ["Apache-2.0"], + "__prompts__": { + "name": "The name of the mapper, in CamelCase", + "admin_name": "Provide your [bold yellow]full name[/]", + "admin_email": "Provide your [bold yellow]email[/]", + "mapper_id": "The ID of the tap, in kebab-case", + "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.", + "serialization_method": "The serialization method to use for loading data", + "include_ci_files": "Whether to include CI files for a common CI services", + "license": "The license for the project" + } } diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml index ee8a65192d..63ef787856 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml @@ -7,7 +7,7 @@ name = "{{cookiecutter.target_id}}" version = "0.0.1" description = "`{{cookiecutter.target_id}}` is a Singer target for {{cookiecutter.destination_name}}, built with the Meltano Singer SDK." readme = "README.md" -authors = ["{{ cookiecutter.admin_name }}"] +authors = ["{{ cookiecutter.admin_name }} <{{ cookiecutter.admin_email }}>"] keywords = [ "ELT", "{{cookiecutter.destination_name}}", diff --git a/docs/dev_guide.md b/docs/dev_guide.md index b93eacedd3..5a118adf0e 100644 --- a/docs/dev_guide.md +++ b/docs/dev_guide.md @@ -76,6 +76,24 @@ generated `README.md` file to complete your new tap or target. You can also refe [Meltano Tutorial](https://docs.meltano.com/tutorials/custom-extractor) for a more detailed guide. +````{admonition} Avoid repeating yourself + If you find yourself repeating the same inputs to the cookiecutter, you can create a + `cookiecutterrc` file in your home directory to set default values for the prompts. + + For example, if you want to set the default value for your name and email, and the + default stream type and authentication method, you can add the following to your + `~/.cookiecutterrc` file: + + ```yaml + # ~/.cookiecutterrc + default_context: + admin_name: Johnny B. Goode + admin_email: jbg@example.com + stream_type: REST + auth_method: Bearer Token + ``` +```` + ### Using an existing library In some cases, there may already be a library that connects to the API and all you need the SDK for diff --git a/e2e-tests/cookiecutters/mapper-base.json b/e2e-tests/cookiecutters/mapper-base.json index 25d40c6085..390e8a7bad 100644 --- a/e2e-tests/cookiecutters/mapper-base.json +++ b/e2e-tests/cookiecutters/mapper-base.json @@ -2,6 +2,7 @@ "cookiecutter": { "name": "MyMapperName", "admin_name": "Automatic Tester", + "admin_email": "auto.tester@example.com", "mapper_id": "mapper-base", "library_name": "mapper_base", "variant": "None (Skip)", diff --git a/e2e-tests/cookiecutters/tap-graphql-jwt.json b/e2e-tests/cookiecutters/tap-graphql-jwt.json index 0c322e06f2..5daf4ab8fb 100644 --- a/e2e-tests/cookiecutters/tap-graphql-jwt.json +++ b/e2e-tests/cookiecutters/tap-graphql-jwt.json @@ -2,6 +2,7 @@ "cookiecutter": { "source_name": "GraphQLJWTTemplateTest", "admin_name": "Automatic Tester", + "admin_email": "auto.tester@example.com", "tap_id": "tap-graphql-jwt", "library_name": "tap_graphql_jwt", "variant": "None (Skip)", diff --git a/e2e-tests/cookiecutters/tap-other-custom.json b/e2e-tests/cookiecutters/tap-other-custom.json index ac3816774e..3ea01eaf4c 100644 --- a/e2e-tests/cookiecutters/tap-other-custom.json +++ b/e2e-tests/cookiecutters/tap-other-custom.json @@ -2,6 +2,7 @@ "cookiecutter": { "source_name": "AutomaticTestTap", "admin_name": "Automatic Tester", + "admin_email": "auto.tester@example.com", "tap_id": "tap-other-custom", "library_name": "tap_other_custom", "variant": "None (Skip)", diff --git a/e2e-tests/cookiecutters/tap-rest-api_key-github.json b/e2e-tests/cookiecutters/tap-rest-api_key-github.json index e659819408..01570aba82 100644 --- a/e2e-tests/cookiecutters/tap-rest-api_key-github.json +++ b/e2e-tests/cookiecutters/tap-rest-api_key-github.json @@ -2,6 +2,7 @@ "cookiecutter": { "source_name": "AutomaticTestTap", "admin_name": "Automatic Tester", + "admin_email": "auto.tester@example.com", "tap_id": "tap-rest-api_key-github", "library_name": "tap_rest_api_key_github", "variant": "None (Skip)", diff --git a/e2e-tests/cookiecutters/tap-rest-basic_auth.json b/e2e-tests/cookiecutters/tap-rest-basic_auth.json index 33eb7b625e..6c7d7fa190 100644 --- a/e2e-tests/cookiecutters/tap-rest-basic_auth.json +++ b/e2e-tests/cookiecutters/tap-rest-basic_auth.json @@ -2,6 +2,7 @@ "cookiecutter": { "source_name": "AutomaticTestTap", "admin_name": "Automatic Tester", + "admin_email": "auto.tester@example.com", "tap_id": "tap-rest-basic_auth", "library_name": "tap_rest_basic_auth", "variant": "None (Skip)", diff --git a/e2e-tests/cookiecutters/tap-rest-bearer_token.json b/e2e-tests/cookiecutters/tap-rest-bearer_token.json index f506061dda..1574574629 100644 --- a/e2e-tests/cookiecutters/tap-rest-bearer_token.json +++ b/e2e-tests/cookiecutters/tap-rest-bearer_token.json @@ -2,6 +2,7 @@ "cookiecutter": { "source_name": "AutomaticTestTap", "admin_name": "Automatic Tester", + "admin_email": "auto.tester@example.com", "tap_id": "tap-rest-bearer_token", "library_name": "tap_rest_bearer_token", "variant": "None (Skip)", diff --git a/e2e-tests/cookiecutters/tap-rest-custom.json b/e2e-tests/cookiecutters/tap-rest-custom.json index 5d68d60bf7..831135b7a7 100644 --- a/e2e-tests/cookiecutters/tap-rest-custom.json +++ b/e2e-tests/cookiecutters/tap-rest-custom.json @@ -2,6 +2,7 @@ "cookiecutter": { "source_name": "AutomaticTestTap", "admin_name": "Automatic Tester", + "admin_email": "auto.tester@example.com", "tap_id": "tap-rest-custom", "library_name": "tap_rest_custom", "variant": "None (Skip)", diff --git a/e2e-tests/cookiecutters/tap-rest-jwt.json b/e2e-tests/cookiecutters/tap-rest-jwt.json index 80837f2441..b46807d491 100644 --- a/e2e-tests/cookiecutters/tap-rest-jwt.json +++ b/e2e-tests/cookiecutters/tap-rest-jwt.json @@ -2,6 +2,7 @@ "cookiecutter": { "source_name": "AutomaticTestTap", "admin_name": "Automatic Tester", + "admin_email": "auto.tester@example.com", "tap_id": "tap-rest-jwt", "library_name": "tap_rest_jwt", "variant": "None (Skip)", diff --git a/e2e-tests/cookiecutters/tap-rest-oauth2.json b/e2e-tests/cookiecutters/tap-rest-oauth2.json index 27c7c39df5..4a41b80e3e 100644 --- a/e2e-tests/cookiecutters/tap-rest-oauth2.json +++ b/e2e-tests/cookiecutters/tap-rest-oauth2.json @@ -2,6 +2,7 @@ "cookiecutter": { "source_name": "AutomaticTestTap", "admin_name": "Automatic Tester", + "admin_email": "auto.tester@example.com", "tap_id": "tap-rest-oauth2", "library_name": "tap_rest_oauth2", "variant": "None (Skip)", diff --git a/e2e-tests/cookiecutters/tap-sql-custom.json b/e2e-tests/cookiecutters/tap-sql-custom.json index 96fa379d74..3c59968607 100644 --- a/e2e-tests/cookiecutters/tap-sql-custom.json +++ b/e2e-tests/cookiecutters/tap-sql-custom.json @@ -2,6 +2,7 @@ "cookiecutter": { "source_name": "AutomaticTestTap", "admin_name": "Automatic Tester", + "admin_email": "auto.tester@example.com", "tap_id": "tap-sql-custom", "library_name": "tap_sql_custom", "variant": "None (Skip)", diff --git a/e2e-tests/cookiecutters/target-per_record.json b/e2e-tests/cookiecutters/target-per_record.json index 9e0047af17..f5dde1cef0 100644 --- a/e2e-tests/cookiecutters/target-per_record.json +++ b/e2e-tests/cookiecutters/target-per_record.json @@ -2,6 +2,7 @@ "cookiecutter": { "destination_name": "MyDestinationName", "admin_name": "FirstName LastName", + "admin_email": "firstname.lastname@example.com", "target_id": "target-per_record", "library_name": "target_per_record", "variant": "None (Skip)", diff --git a/e2e-tests/cookiecutters/target-sql.json b/e2e-tests/cookiecutters/target-sql.json index 5802e5edee..63691d7188 100644 --- a/e2e-tests/cookiecutters/target-sql.json +++ b/e2e-tests/cookiecutters/target-sql.json @@ -2,6 +2,7 @@ "cookiecutter": { "destination_name": "MyDestinationName", "admin_name": "FirstName LastName", + "admin_email": "firstname.lastname@example.com", "target_id": "target-sql", "library_name": "target_sql", "variant": "None (Skip)", From c9eda5a03cf2ef404a2a55b45f5958978efc0734 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Aug 2023 14:10:57 +0000 Subject: [PATCH 17/30] chore(deps): bump sqlalchemy from 2.0.19 to 2.0.20 (#1914) --- poetry.lock | 86 ++++++++++++++++++++++++++--------------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/poetry.lock b/poetry.lock index 964a6b899d..63049a472c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2293,52 +2293,52 @@ test = ["pytest"] [[package]] name = "sqlalchemy" -version = "2.0.19" +version = "2.0.20" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" files = [ - {file = "SQLAlchemy-2.0.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9deaae357edc2091a9ed5d25e9ee8bba98bcfae454b3911adeaf159c2e9ca9e3"}, - {file = "SQLAlchemy-2.0.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0bf0fd65b50a330261ec7fe3d091dfc1c577483c96a9fa1e4323e932961aa1b5"}, - {file = "SQLAlchemy-2.0.19-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d90ccc15ba1baa345796a8fb1965223ca7ded2d235ccbef80a47b85cea2d71a"}, - {file = "SQLAlchemy-2.0.19-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb4e688f6784427e5f9479d1a13617f573de8f7d4aa713ba82813bcd16e259d1"}, - {file = "SQLAlchemy-2.0.19-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:584f66e5e1979a7a00f4935015840be627e31ca29ad13f49a6e51e97a3fb8cae"}, - {file = "SQLAlchemy-2.0.19-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2c69ce70047b801d2aba3e5ff3cba32014558966109fecab0c39d16c18510f15"}, - {file = "SQLAlchemy-2.0.19-cp310-cp310-win32.whl", hash = "sha256:96f0463573469579d32ad0c91929548d78314ef95c210a8115346271beeeaaa2"}, - {file = "SQLAlchemy-2.0.19-cp310-cp310-win_amd64.whl", hash = "sha256:22bafb1da60c24514c141a7ff852b52f9f573fb933b1e6b5263f0daa28ce6db9"}, - {file = "SQLAlchemy-2.0.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d6894708eeb81f6d8193e996257223b6bb4041cb05a17cd5cf373ed836ef87a2"}, - {file = "SQLAlchemy-2.0.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8f2afd1aafded7362b397581772c670f20ea84d0a780b93a1a1529da7c3d369"}, - {file = "SQLAlchemy-2.0.19-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b15afbf5aa76f2241184c1d3b61af1a72ba31ce4161013d7cb5c4c2fca04fd6e"}, - {file = "SQLAlchemy-2.0.19-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fc05b59142445a4efb9c1fd75c334b431d35c304b0e33f4fa0ff1ea4890f92e"}, - {file = "SQLAlchemy-2.0.19-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5831138f0cc06b43edf5f99541c64adf0ab0d41f9a4471fd63b54ae18399e4de"}, - {file = "SQLAlchemy-2.0.19-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3afa8a21a9046917b3a12ffe016ba7ebe7a55a6fc0c7d950beb303c735c3c3ad"}, - {file = "SQLAlchemy-2.0.19-cp311-cp311-win32.whl", hash = "sha256:c896d4e6ab2eba2afa1d56be3d0b936c56d4666e789bfc59d6ae76e9fcf46145"}, - {file = "SQLAlchemy-2.0.19-cp311-cp311-win_amd64.whl", hash = "sha256:024d2f67fb3ec697555e48caeb7147cfe2c08065a4f1a52d93c3d44fc8e6ad1c"}, - {file = "SQLAlchemy-2.0.19-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:89bc2b374ebee1a02fd2eae6fd0570b5ad897ee514e0f84c5c137c942772aa0c"}, - {file = "SQLAlchemy-2.0.19-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd4d410a76c3762511ae075d50f379ae09551d92525aa5bb307f8343bf7c2c12"}, - {file = "SQLAlchemy-2.0.19-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f469f15068cd8351826df4080ffe4cc6377c5bf7d29b5a07b0e717dddb4c7ea2"}, - {file = "SQLAlchemy-2.0.19-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:cda283700c984e699e8ef0fcc5c61f00c9d14b6f65a4f2767c97242513fcdd84"}, - {file = "SQLAlchemy-2.0.19-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:43699eb3f80920cc39a380c159ae21c8a8924fe071bccb68fc509e099420b148"}, - {file = "SQLAlchemy-2.0.19-cp37-cp37m-win32.whl", hash = "sha256:61ada5831db36d897e28eb95f0f81814525e0d7927fb51145526c4e63174920b"}, - {file = "SQLAlchemy-2.0.19-cp37-cp37m-win_amd64.whl", hash = "sha256:57d100a421d9ab4874f51285c059003292433c648df6abe6c9c904e5bd5b0828"}, - {file = "SQLAlchemy-2.0.19-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:16a310f5bc75a5b2ce7cb656d0e76eb13440b8354f927ff15cbaddd2523ee2d1"}, - {file = "SQLAlchemy-2.0.19-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cf7b5e3856cbf1876da4e9d9715546fa26b6e0ba1a682d5ed2fc3ca4c7c3ec5b"}, - {file = "SQLAlchemy-2.0.19-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e7b69d9ced4b53310a87117824b23c509c6fc1f692aa7272d47561347e133b6"}, - {file = "SQLAlchemy-2.0.19-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f9eb4575bfa5afc4b066528302bf12083da3175f71b64a43a7c0badda2be365"}, - {file = "SQLAlchemy-2.0.19-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6b54d1ad7a162857bb7c8ef689049c7cd9eae2f38864fc096d62ae10bc100c7d"}, - {file = "SQLAlchemy-2.0.19-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5d6afc41ca0ecf373366fd8e10aee2797128d3ae45eb8467b19da4899bcd1ee0"}, - {file = "SQLAlchemy-2.0.19-cp38-cp38-win32.whl", hash = "sha256:430614f18443b58ceb9dedec323ecddc0abb2b34e79d03503b5a7579cd73a531"}, - {file = "SQLAlchemy-2.0.19-cp38-cp38-win_amd64.whl", hash = "sha256:eb60699de43ba1a1f77363f563bb2c652f7748127ba3a774f7cf2c7804aa0d3d"}, - {file = "SQLAlchemy-2.0.19-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a752b7a9aceb0ba173955d4f780c64ee15a1a991f1c52d307d6215c6c73b3a4c"}, - {file = "SQLAlchemy-2.0.19-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7351c05db355da112e056a7b731253cbeffab9dfdb3be1e895368513c7d70106"}, - {file = "SQLAlchemy-2.0.19-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa51ce4aea583b0c6b426f4b0563d3535c1c75986c4373a0987d84d22376585b"}, - {file = "SQLAlchemy-2.0.19-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae7473a67cd82a41decfea58c0eac581209a0aa30f8bc9190926fbf628bb17f7"}, - {file = "SQLAlchemy-2.0.19-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:851a37898a8a39783aab603c7348eb5b20d83c76a14766a43f56e6ad422d1ec8"}, - {file = "SQLAlchemy-2.0.19-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:539010665c90e60c4a1650afe4ab49ca100c74e6aef882466f1de6471d414be7"}, - {file = "SQLAlchemy-2.0.19-cp39-cp39-win32.whl", hash = "sha256:f82c310ddf97b04e1392c33cf9a70909e0ae10a7e2ddc1d64495e3abdc5d19fb"}, - {file = "SQLAlchemy-2.0.19-cp39-cp39-win_amd64.whl", hash = "sha256:8e712cfd2e07b801bc6b60fdf64853bc2bd0af33ca8fa46166a23fe11ce0dbb0"}, - {file = "SQLAlchemy-2.0.19-py3-none-any.whl", hash = "sha256:314145c1389b021a9ad5aa3a18bac6f5d939f9087d7fc5443be28cba19d2c972"}, - {file = "SQLAlchemy-2.0.19.tar.gz", hash = "sha256:77a14fa20264af73ddcdb1e2b9c5a829b8cc6b8304d0f093271980e36c200a3f"}, + {file = "SQLAlchemy-2.0.20-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:759b51346aa388c2e606ee206c0bc6f15a5299f6174d1e10cadbe4530d3c7a98"}, + {file = "SQLAlchemy-2.0.20-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1506e988ebeaaf316f183da601f24eedd7452e163010ea63dbe52dc91c7fc70e"}, + {file = "SQLAlchemy-2.0.20-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5768c268df78bacbde166b48be788b83dddaa2a5974b8810af422ddfe68a9bc8"}, + {file = "SQLAlchemy-2.0.20-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3f0dd6d15b6dc8b28a838a5c48ced7455c3e1fb47b89da9c79cc2090b072a50"}, + {file = "SQLAlchemy-2.0.20-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:243d0fb261f80a26774829bc2cee71df3222587ac789b7eaf6555c5b15651eed"}, + {file = "SQLAlchemy-2.0.20-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6eb6d77c31e1bf4268b4d61b549c341cbff9842f8e115ba6904249c20cb78a61"}, + {file = "SQLAlchemy-2.0.20-cp310-cp310-win32.whl", hash = "sha256:bcb04441f370cbe6e37c2b8d79e4af9e4789f626c595899d94abebe8b38f9a4d"}, + {file = "SQLAlchemy-2.0.20-cp310-cp310-win_amd64.whl", hash = "sha256:d32b5ffef6c5bcb452723a496bad2d4c52b346240c59b3e6dba279f6dcc06c14"}, + {file = "SQLAlchemy-2.0.20-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dd81466bdbc82b060c3c110b2937ab65ace41dfa7b18681fdfad2f37f27acdd7"}, + {file = "SQLAlchemy-2.0.20-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6fe7d61dc71119e21ddb0094ee994418c12f68c61b3d263ebaae50ea8399c4d4"}, + {file = "SQLAlchemy-2.0.20-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4e571af672e1bb710b3cc1a9794b55bce1eae5aed41a608c0401885e3491179"}, + {file = "SQLAlchemy-2.0.20-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3364b7066b3c7f4437dd345d47271f1251e0cfb0aba67e785343cdbdb0fff08c"}, + {file = "SQLAlchemy-2.0.20-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1be86ccea0c965a1e8cd6ccf6884b924c319fcc85765f16c69f1ae7148eba64b"}, + {file = "SQLAlchemy-2.0.20-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1d35d49a972649b5080557c603110620a86aa11db350d7a7cb0f0a3f611948a0"}, + {file = "SQLAlchemy-2.0.20-cp311-cp311-win32.whl", hash = "sha256:27d554ef5d12501898d88d255c54eef8414576f34672e02fe96d75908993cf53"}, + {file = "SQLAlchemy-2.0.20-cp311-cp311-win_amd64.whl", hash = "sha256:411e7f140200c02c4b953b3dbd08351c9f9818d2bd591b56d0fa0716bd014f1e"}, + {file = "SQLAlchemy-2.0.20-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3c6aceebbc47db04f2d779db03afeaa2c73ea3f8dcd3987eb9efdb987ffa09a3"}, + {file = "SQLAlchemy-2.0.20-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7d3f175410a6db0ad96b10bfbb0a5530ecd4fcf1e2b5d83d968dd64791f810ed"}, + {file = "SQLAlchemy-2.0.20-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea8186be85da6587456c9ddc7bf480ebad1a0e6dcbad3967c4821233a4d4df57"}, + {file = "SQLAlchemy-2.0.20-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c3d99ba99007dab8233f635c32b5cd24fb1df8d64e17bc7df136cedbea427897"}, + {file = "SQLAlchemy-2.0.20-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:76fdfc0f6f5341987474ff48e7a66c3cd2b8a71ddda01fa82fedb180b961630a"}, + {file = "SQLAlchemy-2.0.20-cp37-cp37m-win32.whl", hash = "sha256:d3793dcf5bc4d74ae1e9db15121250c2da476e1af8e45a1d9a52b1513a393459"}, + {file = "SQLAlchemy-2.0.20-cp37-cp37m-win_amd64.whl", hash = "sha256:79fde625a0a55220d3624e64101ed68a059c1c1f126c74f08a42097a72ff66a9"}, + {file = "SQLAlchemy-2.0.20-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:599ccd23a7146e126be1c7632d1d47847fa9f333104d03325c4e15440fc7d927"}, + {file = "SQLAlchemy-2.0.20-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1a58052b5a93425f656675673ef1f7e005a3b72e3f2c91b8acca1b27ccadf5f4"}, + {file = "SQLAlchemy-2.0.20-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79543f945be7a5ada9943d555cf9b1531cfea49241809dd1183701f94a748624"}, + {file = "SQLAlchemy-2.0.20-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63e73da7fb030ae0a46a9ffbeef7e892f5def4baf8064786d040d45c1d6d1dc5"}, + {file = "SQLAlchemy-2.0.20-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3ce5e81b800a8afc870bb8e0a275d81957e16f8c4b62415a7b386f29a0cb9763"}, + {file = "SQLAlchemy-2.0.20-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cb0d3e94c2a84215532d9bcf10229476ffd3b08f481c53754113b794afb62d14"}, + {file = "SQLAlchemy-2.0.20-cp38-cp38-win32.whl", hash = "sha256:8dd77fd6648b677d7742d2c3cc105a66e2681cc5e5fb247b88c7a7b78351cf74"}, + {file = "SQLAlchemy-2.0.20-cp38-cp38-win_amd64.whl", hash = "sha256:6f8a934f9dfdf762c844e5164046a9cea25fabbc9ec865c023fe7f300f11ca4a"}, + {file = "SQLAlchemy-2.0.20-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:26a3399eaf65e9ab2690c07bd5cf898b639e76903e0abad096cd609233ce5208"}, + {file = "SQLAlchemy-2.0.20-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4cde2e1096cbb3e62002efdb7050113aa5f01718035ba9f29f9d89c3758e7e4e"}, + {file = "SQLAlchemy-2.0.20-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1b09ba72e4e6d341bb5bdd3564f1cea6095d4c3632e45dc69375a1dbe4e26ec"}, + {file = "SQLAlchemy-2.0.20-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b74eeafaa11372627ce94e4dc88a6751b2b4d263015b3523e2b1e57291102f0"}, + {file = "SQLAlchemy-2.0.20-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:77d37c1b4e64c926fa3de23e8244b964aab92963d0f74d98cbc0783a9e04f501"}, + {file = "SQLAlchemy-2.0.20-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:eefebcc5c555803065128401a1e224a64607259b5eb907021bf9b175f315d2a6"}, + {file = "SQLAlchemy-2.0.20-cp39-cp39-win32.whl", hash = "sha256:3423dc2a3b94125094897118b52bdf4d37daf142cbcf26d48af284b763ab90e9"}, + {file = "SQLAlchemy-2.0.20-cp39-cp39-win_amd64.whl", hash = "sha256:5ed61e3463021763b853628aef8bc5d469fe12d95f82c74ef605049d810f3267"}, + {file = "SQLAlchemy-2.0.20-py3-none-any.whl", hash = "sha256:63a368231c53c93e2b67d0c5556a9836fdcd383f7e3026a39602aad775b14acf"}, + {file = "SQLAlchemy-2.0.20.tar.gz", hash = "sha256:ca8a5ff2aa7f3ade6c498aaafce25b1eaeabe4e42b73e25519183e4566a16fc6"}, ] [package.dependencies] @@ -2347,7 +2347,7 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} typing-extensions = ">=4.2.0" [package.extras] -aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] +aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing-extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] From 2e45606eea7ed9473ff5fb45d04b8145d39d36b9 Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Thu, 17 Aug 2023 08:23:48 -0600 Subject: [PATCH 18/30] fix: Fix tap tests for multiple test classes with different input catalogs (#1913) * fix: Fix tap tests for multiple test classes with different input catalogs * Fix mro mess * Test the test class mro --- singer_sdk/testing/factory.py | 18 +++++++++++++++--- tests/core/test_testing.py | 22 ++++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/singer_sdk/testing/factory.py b/singer_sdk/testing/factory.py index c7611955fa..1c15f48448 100644 --- a/singer_sdk/testing/factory.py +++ b/singer_sdk/testing/factory.py @@ -21,8 +21,20 @@ class BaseTestClass: """Base test class.""" - params: t.ClassVar[dict] = {} - param_ids: t.ClassVar[dict] = {} + params: dict[str, t.Any] + param_ids: dict[str, list[str]] + + def __init_subclass__(cls, **kwargs: t.Any) -> None: + """Initialize a subclass. + + Args: + **kwargs: Keyword arguments. + """ + # Add empty params and param_ids attributes to a direct subclass but not to + # subclasses of subclasses + if cls.__base__ == BaseTestClass: + cls.params = {} + cls.param_ids = {} class TapTestClassFactory: @@ -183,7 +195,7 @@ def _annotate_test_class( # noqa: C901 test = test_class() test_name = f"test_{suite.kind}_{test.name}" test_params = [] - test_ids = [] + test_ids: list[str] = [] for stream in streams: test_params.extend( [ diff --git a/tests/core/test_testing.py b/tests/core/test_testing.py index 02672d9adb..5715cd1e1d 100644 --- a/tests/core/test_testing.py +++ b/tests/core/test_testing.py @@ -4,6 +4,8 @@ import pytest +from singer_sdk.testing.factory import BaseTestClass + def test_module_deprecations(): with pytest.deprecated_call(): @@ -19,3 +21,23 @@ def test_module_deprecations(): match="module singer_sdk.testing has no attribute", ): testing.foo # noqa: B018 + + +def test_test_class_mro(): + class PluginTestClass(BaseTestClass): + pass + + PluginTestClass.params["x"] = 1 + + class AnotherPluginTestClass(BaseTestClass): + pass + + AnotherPluginTestClass.params["x"] = 2 + AnotherPluginTestClass.params["y"] = 3 + + class SubPluginTestClass(PluginTestClass): + pass + + assert PluginTestClass.params == {"x": 1} + assert AnotherPluginTestClass.params == {"x": 2, "y": 3} + assert SubPluginTestClass.params == {"x": 1} From efca24f7a439d934f6655eb08cb99d2cbee5e3a7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 17 Aug 2023 09:07:48 -0600 Subject: [PATCH 19/30] chore: Release v0.31.1 (#1915) chore: Bump package version Co-authored-by: edgarrmondragon --- .github/ISSUE_TEMPLATE/bug.yml | 2 +- CHANGELOG.md | 10 ++++++++++ .../{{cookiecutter.mapper_id}}/pyproject.toml | 2 +- .../{{cookiecutter.tap_id}}/pyproject.toml | 4 ++-- .../{{cookiecutter.target_id}}/pyproject.toml | 4 ++-- docs/conf.py | 2 +- pyproject.toml | 4 ++-- 7 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 45d9aabf1b..808c357b6e 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -15,7 +15,7 @@ body: attributes: label: Singer SDK Version description: Version of the library you are using - placeholder: "0.31.0" + placeholder: "0.31.1" validations: required: true - type: checkboxes diff --git a/CHANGELOG.md b/CHANGELOG.md index b7d772080f..62a31673c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v0.31.1 (2023-08-17) + +### ✨ New + +- [#1905](https://github.com/meltano/sdk/issues/1905) Add email field and use human-readable questions in templates + +### 🐛 Fixes + +- [#1913](https://github.com/meltano/sdk/issues/1913) Fix tap tests for multiple test classes with different input catalogs + ## v0.31.0 (2023-08-07) ### ✨ New diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml index 36c541fcab..f63f07b02d 100644 --- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml @@ -22,7 +22,7 @@ packages = [ [tool.poetry.dependencies] python = "<3.12,>=3.7.1" -singer-sdk = { version="^0.31.0" } +singer-sdk = { version="^0.31.1" } fs-s3fs = { version = "^1.1.1", optional = true } [tool.poetry.group.dev.dependencies] diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml index 46ffb61f75..c5381bdaf7 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml @@ -21,7 +21,7 @@ packages = [ [tool.poetry.dependencies] python = "<3.12,>=3.7.1" -singer-sdk = { version="^0.31.0" } +singer-sdk = { version="^0.31.1" } fs-s3fs = { version = "^1.1.1", optional = true } {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %} requests = "^2.31.0" @@ -32,7 +32,7 @@ cached-property = "^1" # Remove after Python 3.7 support is dropped [tool.poetry.group.dev.dependencies] pytest = "^7.4.0" -singer-sdk = { version="^0.31.0", extras = ["testing"] } +singer-sdk = { version="^0.31.1", extras = ["testing"] } [tool.poetry.extras] s3 = ["fs-s3fs"] diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml index 63ef787856..523561b8dd 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml @@ -21,7 +21,7 @@ packages = [ [tool.poetry.dependencies] python = "<3.12,>=3.7.1" -singer-sdk = { version="^0.31.0" } +singer-sdk = { version="^0.31.1" } fs-s3fs = { version = "^1.1.1", optional = true } {%- if cookiecutter.serialization_method != "SQL" %} requests = "^2.31.0" @@ -29,7 +29,7 @@ requests = "^2.31.0" [tool.poetry.dev-dependencies] pytest = "^7.4.0" -singer-sdk = { version="^0.31.0", extras = ["testing"] } +singer-sdk = { version="^0.31.1", extras = ["testing"] } [tool.poetry.extras] s3 = ["fs-s3fs"] diff --git a/docs/conf.py b/docs/conf.py index 14562fdc02..cea1081143 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,7 +26,7 @@ author = "Meltano Core Team and Contributors" # The full version, including alpha/beta/rc tags -release = "0.31.0" +release = "0.31.1" # -- General configuration --------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 4a3601198b..851eee3dc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "singer-sdk" -version = "0.31.0" +version = "0.31.1" description = "A framework for building Singer taps" authors = ["Meltano Team and Contributors "] maintainers = ["Meltano Team and Contributors "] @@ -141,7 +141,7 @@ norecursedirs = "cookiecutter" [tool.commitizen] name = "cz_version_bump" -version = "0.31.0" +version = "0.31.1" tag_format = "v$major.$minor.$patch$prerelease" version_files = [ "docs/conf.py:^release =", From ce4b9a7fd4b0217d74ce3e94f7445c990417412a Mon Sep 17 00:00:00 2001 From: Dan Norman Date: Thu, 17 Aug 2023 09:40:44 -0600 Subject: [PATCH 20/30] feat: SQLTap connector instance shared with streams (#1861) Co-authored-by: Edgar R. M --- singer_sdk/tap_base.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/singer_sdk/tap_base.py b/singer_sdk/tap_base.py index 08ce6f3adb..8b025a1f8a 100644 --- a/singer_sdk/tap_base.py +++ b/singer_sdk/tap_base.py @@ -30,6 +30,7 @@ if t.TYPE_CHECKING: from pathlib import PurePath + from singer_sdk.connectors import SQLConnector from singer_sdk.mapper import PluginMapper from singer_sdk.streams import SQLStream, Stream @@ -612,6 +613,8 @@ class SQLTap(Tap): # Stream class used to initialize new SQL streams from their catalog declarations. default_stream_class: type[SQLStream] + _tap_connector: SQLConnector | None = None + def __init__(self, *args: t.Any, **kwargs: t.Any) -> None: """Initialize the SQL tap. @@ -624,6 +627,19 @@ def __init__(self, *args: t.Any, **kwargs: t.Any) -> None: self._catalog_dict: dict | None = None super().__init__(*args, **kwargs) + @property + def tap_connector(self) -> SQLConnector: + """The connector object. + + Returns: + The connector object. + """ + if self._tap_connector is None: + self._tap_connector = self.default_stream_class.connector_class( + dict(self.config), + ) + return self._tap_connector + @property def catalog_dict(self) -> dict: """Get catalog dictionary. @@ -637,7 +653,7 @@ def catalog_dict(self) -> dict: if self.input_catalog: return self.input_catalog.to_dict() - connector = self.default_stream_class.connector_class(dict(self.config)) + connector = self.tap_connector result: dict[str, list[dict]] = {"streams": []} result["streams"].extend(connector.discover_catalog_entries()) @@ -653,6 +669,12 @@ def discover_streams(self) -> list[Stream]: """ result: list[Stream] = [] for catalog_entry in self.catalog_dict["streams"]: - result.append(self.default_stream_class(self, catalog_entry)) + result.append( + self.default_stream_class( + tap=self, + catalog_entry=catalog_entry, + connector=self.tap_connector, + ), + ) return result From 92e8fec98df16fe89d5254a94930c55eea2a18de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 12:57:05 +0000 Subject: [PATCH 21/30] chore(deps): bump click from 8.1.6 to 8.1.7 (#1919) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 63049a472c..a5ecfc05d8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -404,13 +404,13 @@ files = [ [[package]] name = "click" -version = "8.1.6" +version = "8.1.7" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, - {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] [package.dependencies] From dd03e4a831d3162b8909f8f18a300262e844eb38 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 13:16:25 +0000 Subject: [PATCH 22/30] chore(deps): bump poetry from 1.5.1 to 1.6.0 in /.github/workflows (#1920) --- .github/workflows/constraints.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt index 26aef2cc72..a8ff53ca0e 100644 --- a/.github/workflows/constraints.txt +++ b/.github/workflows/constraints.txt @@ -1,5 +1,5 @@ pip==23.2.1 -poetry==1.5.1 +poetry==1.6.0 pre-commit==3.3.3 nox==2023.4.22 nox-poetry==1.0.3 From 3ad8c8dc09c8bfcf5acf1570a1410df1cf4d7494 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 14:01:56 +0000 Subject: [PATCH 23/30] chore(deps): bump actions/dependency-review-action from 3.0.7 to 3.0.8 (#1921) --- .github/workflows/dependency-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index c676791a8d..03ed97a046 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -20,7 +20,7 @@ jobs: - name: GitHub dependency vulnerability check if: ${{ github.event_name == 'pull_request_target' }} - uses: actions/dependency-review-action@v3.0.7 + uses: actions/dependency-review-action@v3.0.8 with: fail-on-severity: high From 2aa2327643d310d1aa4eed26d2a4fb15694d5009 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Aug 2023 13:25:00 +0000 Subject: [PATCH 24/30] chore(deps): bump poetry from 1.6.0 to 1.6.1 in /.github/workflows (#1923) --- .github/workflows/constraints.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt index a8ff53ca0e..0065729109 100644 --- a/.github/workflows/constraints.txt +++ b/.github/workflows/constraints.txt @@ -1,5 +1,5 @@ pip==23.2.1 -poetry==1.6.0 +poetry==1.6.1 pre-commit==3.3.3 nox==2023.4.22 nox-poetry==1.0.3 From 910163a4f3aa2691eb164e13bfd734c53c1b9eef Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 22 Aug 2023 07:35:51 -0600 Subject: [PATCH 25/30] chore: pre-commit autoupdate (#1922) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.284 → v0.0.285](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.284...v0.0.285) - [github.com/python-poetry/poetry: 1.5.0 → 1.6.0](https://github.com/python-poetry/poetry/compare/1.5.0...1.6.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Edgar R. M --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc1e804d08..51bf0571e3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,7 +43,7 @@ repos: - id: check-readthedocs - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.284 + rev: v0.0.285 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] @@ -76,6 +76,6 @@ repos: )$ - repo: https://github.com/python-poetry/poetry - rev: 1.5.0 + rev: 1.6.0 hooks: - id: poetry-check From f9c2dad3d6a5ac0ca94bd6e0834eb25e808230f3 Mon Sep 17 00:00:00 2001 From: mjsqu Date: Fri, 25 Aug 2023 02:46:18 +1200 Subject: [PATCH 26/30] docs: Add viztracer command for testing targets (#1925) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add viztracer command for testing targets When it comes to testing targets it took me a long while to work out I could use the `--input` switch to pass a file of records to the target rather than trying to work out how to run `viztracer` on a command containing a pipe 🤯. If the added line can be proven to work by a reviewer and it's OK to be added to the documentation then it would have saved me heaps of time yesterday 😄 * Update poetry command --------- Co-authored-by: Edgar Ramírez Mondragón --- docs/dev_guide.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/dev_guide.md b/docs/dev_guide.md index 5a118adf0e..316cca8ad0 100644 --- a/docs/dev_guide.md +++ b/docs/dev_guide.md @@ -257,13 +257,14 @@ We've had success using [`viztracer`](https://github.com/gaogaotiantian/viztrace You can start doing the same in your package. Start by installing `viztracer`. ```console -$ poetry add --dev viztracer +$ poetry add --group dev viztracer ``` Then simply run your package's CLI as normal, preceded by the `viztracer` command ```console $ poetry run viztracer my-tap +$ poetry run viztracer -- my-target --config=config.json --input=messages.json ``` That command will produce a `result.json` file which you can explore with the `vizviewer` tool. From 712ce9a8cac1502390ef0f81376d73264f5f600d Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Thu, 24 Aug 2023 16:44:20 -0600 Subject: [PATCH 27/30] chore: Enable Ruff's PERF checks and apply minor refactorings (#1926) --- pyproject.toml | 5 ++++- singer_sdk/tap_base.py | 19 ++++++++----------- singer_sdk/testing/tap_tests.py | 8 ++++---- singer_sdk/typing.py | 5 ++++- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 851eee3dc6..ea5884d676 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -243,7 +243,8 @@ select = [ "C4", # flake8-comprehensions "DTZ", # flake8-datetimezs "T10", # flake8-debugger - "EM", # flake8-error-message + "EM", # flake8-errmsg + "FA", # flake8-future-annotations "ISC", # flake8-implicit-str-concat "ICN", # flake8-import-conventions "G", # flake8-logging-format @@ -254,6 +255,7 @@ select = [ "Q", # flake8-quotes "RSE", # flake8-raise "RET", # flake8-return + # "SLF", # flake8-self "SIM", # flake8-simplify "TID", # flake8-tidy-imports "TCH", # flake8-type-checking @@ -265,6 +267,7 @@ select = [ "PLE", # pylint (error) "PLR", # pylint (refactor) "PLW", # pylint (warning) + "PERF", # perflint "RUF", # ruff ] src = ["samples", "singer_sdk", "tests"] diff --git a/singer_sdk/tap_base.py b/singer_sdk/tap_base.py index 8b025a1f8a..d4b5a8dc30 100644 --- a/singer_sdk/tap_base.py +++ b/singer_sdk/tap_base.py @@ -408,7 +408,7 @@ def load_state(self, state: dict[str, t.Any]) -> None: def _reset_state_progress_markers(self) -> None: """Clear prior jobs' progress markers at beginning of sync.""" - for _, state in self.state.get("bookmarks", {}).items(): + for state in self.state.get("bookmarks", {}).values(): _state.reset_state_progress_markers(state) for partition_state in state.get("partitions", []): _state.reset_state_progress_markers(partition_state) @@ -667,14 +667,11 @@ def discover_streams(self) -> list[Stream]: Returns: List of discovered Stream objects. """ - result: list[Stream] = [] - for catalog_entry in self.catalog_dict["streams"]: - result.append( - self.default_stream_class( - tap=self, - catalog_entry=catalog_entry, - connector=self.tap_connector, - ), + return [ + self.default_stream_class( + tap=self, + catalog_entry=catalog_entry, + connector=self.tap_connector, ) - - return result + for catalog_entry in self.catalog_dict["streams"] + ] diff --git a/singer_sdk/testing/tap_tests.py b/singer_sdk/testing/tap_tests.py index a95720d571..ecb7eb811f 100644 --- a/singer_sdk/testing/tap_tests.py +++ b/singer_sdk/testing/tap_tests.py @@ -185,12 +185,12 @@ def test(self) -> None: Raises: AssertionError: if value cannot be parsed as a datetime. """ - for v in self.non_null_attribute_values: - try: + try: + for v in self.non_null_attribute_values: error_message = f"Unable to parse value ('{v}') with datetime parser." assert parser.parse(v), error_message - except parser.ParserError as e: - raise AssertionError(error_message) from e + except parser.ParserError as e: + raise AssertionError(error_message) from e @classmethod def evaluate( diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index 850e30e8eb..fd17b9d3d9 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -58,7 +58,10 @@ import typing as t import sqlalchemy -from jsonschema import ValidationError, Validator, validators +from jsonschema import ValidationError, validators + +if t.TYPE_CHECKING: + from jsonschema.protocols import Validator from singer_sdk.helpers._typing import ( JSONSCHEMA_ANNOTATION_SECRET, From 4eac6f4ae0fb8b9eda959da3d74e61ef270f6b4d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 13:27:38 +0000 Subject: [PATCH 28/30] chore(deps): bump actions/checkout from 3.5.3 to 3.6.0 (#1928) --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/cookiecutter-e2e.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 6 +++--- .github/workflows/version_bump.yml | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 75b4dc276a..cb512b15c8 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index a33989df45..b86e16e9ee 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Upgrade pip env: diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 03ed97a046..9a8f71a4b3 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout the repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: GitHub dependency vulnerability check if: ${{ github.event_name == 'pull_request_target' }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9f85a1a7cb..ecf4a57a36 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Set up Python uses: actions/setup-python@v4.7.0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 71a5adf301..23d91ae8aa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,7 +55,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Install Poetry env: @@ -115,7 +115,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Install Poetry env: @@ -157,7 +157,7 @@ jobs: needs: tests steps: - name: Check out the repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Install Poetry run: | diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml index 7503b030fa..30e13fbbf1 100644 --- a/.github/workflows/version_bump.yml +++ b/.github/workflows/version_bump.yml @@ -35,7 +35,7 @@ jobs: pull-requests: write # to create and update PRs steps: - - uses: actions/checkout@v3.5.3 + - uses: actions/checkout@v3.6.0 with: fetch-depth: 0 From b1b97b06231378f95c0530cb5fbd854ad67624b8 Mon Sep 17 00:00:00 2001 From: mjsqu Date: Tue, 29 Aug 2023 07:15:12 +1200 Subject: [PATCH 29/30] fix: Handle replication key not found in stream schema (#1927) * Handle replication key not found in stream schema * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add missing Raises: to is_timestamp_replication_key docstring * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Split raise docstring over to new line for is_timestamp_replication_key docstring * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Correct indentation for raises docstring * Add test_stream_invalid_replication_key * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Import InvalidReplicationKeyException name * pre-commit check updates - line length and 'useless' expression * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Rename 'unused' variable with underscore prefix - variable is used to intentionally raise and exception * Improve error message Co-authored-by: Edgar R. M. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update exception message in invalid replication key test Co-authored-by: Edgar R. M. * Make linter happy --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Edgar R. M Co-authored-by: Edgar R. M. --- singer_sdk/exceptions.py | 4 ++++ singer_sdk/streams/core.py | 8 ++++++++ tests/core/test_streams.py | 21 +++++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/singer_sdk/exceptions.py b/singer_sdk/exceptions.py index 23325aa2ae..351776291a 100644 --- a/singer_sdk/exceptions.py +++ b/singer_sdk/exceptions.py @@ -17,6 +17,10 @@ class FatalAPIError(Exception): """Exception raised when a failed request should not be considered retriable.""" +class InvalidReplicationKeyException(Exception): + """Exception to raise if the replication key is not in the stream properties.""" + + class InvalidStreamSortException(Exception): """Exception to raise if sorting errors are found while syncing the records.""" diff --git a/singer_sdk/streams/core.py b/singer_sdk/streams/core.py index 53e9533ff9..4c3adb2254 100644 --- a/singer_sdk/streams/core.py +++ b/singer_sdk/streams/core.py @@ -19,6 +19,7 @@ from singer_sdk.exceptions import ( AbortedSyncFailedException, AbortedSyncPausedException, + InvalidReplicationKeyException, InvalidStreamSortException, MaxRecordsLimitException, ) @@ -211,10 +212,17 @@ def is_timestamp_replication_key(self) -> bool: Returns: True if the stream uses a timestamp-based replication key. + + Raises: + InvalidReplicationKeyException: If the schema does not contain the + replication key. """ if not self.replication_key: return False type_dict = self.schema.get("properties", {}).get(self.replication_key) + if type_dict is None: + msg = f"Field '{self.replication_key}' is not in schema for stream '{self.name}'" # noqa: E501 + raise InvalidReplicationKeyException(msg) return is_datetime_type(type_dict) def get_starting_replication_key_value( diff --git a/tests/core/test_streams.py b/tests/core/test_streams.py index 34bbc75141..a3a451086d 100644 --- a/tests/core/test_streams.py +++ b/tests/core/test_streams.py @@ -10,6 +10,9 @@ import requests from singer_sdk._singerlib import Catalog, MetadataMapping +from singer_sdk.exceptions import ( + InvalidReplicationKeyException, +) from singer_sdk.helpers._classproperty import classproperty from singer_sdk.helpers.jsonpath import _compile_jsonpath, extract_jsonpath from singer_sdk.pagination import first @@ -275,6 +278,24 @@ def test_stream_starting_timestamp( assert get_starting_value(None) == expected_starting_value +def test_stream_invalid_replication_key(tap: SimpleTestTap): + """Validate an exception is raised if replication_key not in schema.""" + + class InvalidReplicationKeyStream(SimpleTestStream): + replication_key = "INVALID" + + stream = InvalidReplicationKeyStream(tap) + + with pytest.raises( + InvalidReplicationKeyException, + match=( + f"Field '{stream.replication_key}' is not in schema for stream " + f"'{stream.name}'" + ), + ): + _check = stream.is_timestamp_replication_key + + @pytest.mark.parametrize( "path,content,result", [ From fdd35a8f3584081e477d2bf0c20729b7cdf8ab52 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 29 Aug 2023 08:22:09 -0600 Subject: [PATCH 30/30] chore: pre-commit autoupdate (#1929) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/python-jsonschema/check-jsonschema: 0.24.1 → 0.26.3](https://github.com/python-jsonschema/check-jsonschema/compare/0.24.1...0.26.3) - [github.com/astral-sh/ruff-pre-commit: v0.0.285 → v0.0.286](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.285...v0.0.286) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 51bf0571e3..a1cf1e839f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,14 +36,14 @@ repos: )$ - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.24.1 + rev: 0.26.3 hooks: - id: check-dependabot - id: check-github-workflows - id: check-readthedocs - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.285 + rev: v0.0.286 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes]