Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Support validating configuration for any tap with a dynamic catalog #937

Merged
merged 12 commits into from
Jan 19, 2024
Merged
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@

.. autoclass:: APIAuthenticatorBase
:members:
:special-members: __init__, __call__
:special-members: __init__, __call__
39 changes: 31 additions & 8 deletions singer_sdk/tap_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,11 @@

from singer_sdk._singerlib import Catalog, StateMessage
from singer_sdk.configuration._dict_config import merge_missing_config_jsonschema
from singer_sdk.exceptions import AbortedSyncFailedException, AbortedSyncPausedException
from singer_sdk.exceptions import (
AbortedSyncFailedException,
AbortedSyncPausedException,
ConfigValidationError,
)
from singer_sdk.helpers import _state
from singer_sdk.helpers._classproperty import classproperty
from singer_sdk.helpers._compat import final
Expand Down Expand Up @@ -53,6 +57,10 @@ class Tap(PluginBase, SingerWriter, metaclass=abc.ABCMeta):
plugins.
"""

dynamic_catalog: bool = False
"""Whether the tap's catalog is dynamic. Set to True if the catalog is
generated dynamically (e.g. by querying a database's system tables)."""

# Constructor

def __init__(
Expand Down Expand Up @@ -520,12 +528,17 @@ def cb_discover(

config_args = ctx.params.get("config", ())
config_files, parse_env_config = cls.config_from_cli_args(*config_args)
tap = cls(
config=config_files, # type: ignore[arg-type]
parse_env_config=parse_env_config,
validate_config=False,
setup_mapper=False,
)
try:
tap = cls(
config=config_files, # type: ignore[arg-type]
parse_env_config=parse_env_config,
validate_config=cls.dynamic_catalog,
setup_mapper=False,
)
except ConfigValidationError as exc: # pragma: no cover
for error in exc.errors:
cls.logger.error("Config validation error: %s", error)
ctx.exit(1)
tap.run_discovery()
ctx.exit()

Expand Down Expand Up @@ -611,8 +624,18 @@ def get_singer_command(cls: type[Tap]) -> click.Command:
class SQLTap(Tap):
"""A specialized Tap for extracting from SQL streams."""

# Stream class used to initialize new SQL streams from their catalog declarations.
default_stream_class: type[SQLStream]
"""
The default stream class used to initialize new SQL streams from their catalog
entries.
"""

dynamic_catalog: bool = True
"""
Whether the tap's catalog is dynamic, enabling configuration validation in
discovery mode. Set to True if the catalog is generated dynamically (e.g. by
querying a database's system tables).
"""

_tap_connector: SQLConnector | None = None

Expand Down