diff --git a/sample_logging.conf b/sample_logging.conf new file mode 100644 index 0000000..b9a0b7a --- /dev/null +++ b/sample_logging.conf @@ -0,0 +1,25 @@ +[loggers] +keys=root + +[handlers] +keys=stderr + +[formatters] +keys=child + +[logger_root] +level=INFO +handlers=stderr +formatter=child +propagate=0 + +[handler_stderr] +level=INFO +class=StreamHandler +formatter=child +args=(sys.stderr,) + +[formatter_child] +class=logging.Formatter +format=time=%(asctime)s name=%(name)s level=%(levelname)s message=%(message)s +datefmt=%Y-%m-%d %H:%M:%S diff --git a/setup.py b/setup.py index b342839..909dbe8 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ ], py_modules=['tap_snowflake'], install_requires=[ - 'singer-python==5.9.0', + 'pipelinewise-singer-python==1.*', 'snowflake-connector-python==2.0.4', 'backoff==1.8.0', 'pendulum==1.2.0', diff --git a/tap_snowflake/__init__.py b/tap_snowflake/__init__.py index 9846817..4a76dc6 100644 --- a/tap_snowflake/__init__.py +++ b/tap_snowflake/__init__.py @@ -1,31 +1,28 @@ #!/usr/bin/env python3 # pylint: disable=missing-docstring,not-an-iterable,too-many-locals,too-many-arguments,too-many-branches,invalid-name,duplicate-code,too-many-statements -import datetime import collections -import itertools -from itertools import dropwhile import copy - +import itertools +import logging import singer import singer.metrics as metrics import singer.schema - -from singer import bookmarks from singer import metadata from singer import utils -from singer.schema import Schema from singer.catalog import Catalog, CatalogEntry +from singer.schema import Schema import tap_snowflake.sync_strategies.common as common import tap_snowflake.sync_strategies.full_table as full_table import tap_snowflake.sync_strategies.incremental as incremental - from tap_snowflake.connection import SnowflakeConnection +LOGGER = singer.get_logger('tap_snowflake') -LOGGER = singer.get_logger() +# Tone down snowflake connector logs noise +logging.getLogger('snowflake.connector').setLevel(logging.WARNING) Column = collections.namedtuple('Column', [ "table_catalog", diff --git a/tap_snowflake/connection.py b/tap_snowflake/connection.py index 4562f3d..acf9a7c 100644 --- a/tap_snowflake/connection.py +++ b/tap_snowflake/connection.py @@ -4,7 +4,7 @@ import singer import snowflake.connector -LOGGER = singer.get_logger() +LOGGER = singer.get_logger('tap_snowflake') def retry_pattern(): diff --git a/tap_snowflake/sync_strategies/common.py b/tap_snowflake/sync_strategies/common.py index 5523bcd..22c3182 100644 --- a/tap_snowflake/sync_strategies/common.py +++ b/tap_snowflake/sync_strategies/common.py @@ -10,7 +10,7 @@ from singer import metadata from singer import utils -LOGGER = singer.get_logger() +LOGGER = singer.get_logger('tap_snowflake') def escape(string): if '"' in string: diff --git a/tap_snowflake/sync_strategies/full_table.py b/tap_snowflake/sync_strategies/full_table.py index bbc59fc..1ebb1ea 100644 --- a/tap_snowflake/sync_strategies/full_table.py +++ b/tap_snowflake/sync_strategies/full_table.py @@ -8,7 +8,7 @@ import tap_snowflake.sync_strategies.common as common from tap_snowflake.connection import SnowflakeConnection -LOGGER = singer.get_logger() +LOGGER = singer.get_logger('tap_snowflake') BOOKMARK_KEYS = {'last_pk_fetched', 'max_pk_values', 'version', 'initial_full_table_complete'} diff --git a/tap_snowflake/sync_strategies/incremental.py b/tap_snowflake/sync_strategies/incremental.py index f6c420e..9f852d0 100644 --- a/tap_snowflake/sync_strategies/incremental.py +++ b/tap_snowflake/sync_strategies/incremental.py @@ -8,7 +8,7 @@ from tap_snowflake.connection import SnowflakeConnection import tap_snowflake.sync_strategies.common as common -LOGGER = singer.get_logger() +LOGGER = singer.get_logger('tap_snowflake') BOOKMARK_KEYS = {'replication_key', 'replication_key_value', 'version'} diff --git a/tests/test_tap_snowflake.py b/tests/test_tap_snowflake.py index c8a4f19..b79d3a6 100644 --- a/tests/test_tap_snowflake.py +++ b/tests/test_tap_snowflake.py @@ -13,7 +13,7 @@ except ImportError: import utils as test_utils -LOGGER = singer.get_logger() +LOGGER = singer.get_logger('tap_snowflake_tests') SCHEMA_NAME='tap_snowflake_test' diff --git a/tests/utils.py b/tests/utils.py index 145b8f1..a71d12a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,6 @@ import os + import singer -import snowflake.connector import tap_snowflake from tap_snowflake.connection import SnowflakeConnection