Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update schema mapping version #19

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Update schema mapping version #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
dc84949
Update schema mapping version
brose7230 Dec 15, 2021
b72cf5c
test logger
brose7230 Dec 15, 2021
65ff218
removed unneccessary table_stream and updating catalog_entry.stream w…
brose7230 Dec 15, 2021
07be2cc
updated full_table once more
brose7230 Dec 15, 2021
53a56cf
updated write_schema_message to always include the schema_mapping
brose7230 Dec 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 13 additions & 9 deletions tap_mssql/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -464,14 +464,12 @@ def get_binlog_streams(mssql_conn, catalog, config, state):
return resolve_catalog(discovered, binlog_streams)


def write_schema_message(config, catalog_entry, bookmark_properties=[]):
def write_schema_message(catalog_entry, bookmark_properties=[]):
key_properties = common.get_key_properties(catalog_entry)

table_stream = common.set_schema_mapping(config, catalog_entry.stream)

singer.write_message(
singer.SchemaMessage(
stream=table_stream,
stream=catalog_entry.stream,
schema=catalog_entry.schema.to_dict(),
key_properties=key_properties,
bookmark_properties=bookmark_properties,
Expand All @@ -482,21 +480,24 @@ def write_schema_message(config, catalog_entry, bookmark_properties=[]):
def do_sync_incremental(mssql_conn, config, catalog_entry, state, columns):
md_map = metadata.to_map(catalog_entry.metadata)
stream_version = common.get_stream_version(catalog_entry.tap_stream_id, state)

catalog_entry.stream = common.set_schema_mapping(config, catalog_entry.stream)

replication_key = md_map.get((), {}).get("replication-key")
write_schema_message(
config, catalog_entry=catalog_entry, bookmark_properties=[replication_key]
catalog_entry=catalog_entry, bookmark_properties=[replication_key]
)
LOGGER.info("Schema written")
incremental.sync_table(mssql_conn, config, catalog_entry, state, columns)
incremental.sync_table(mssql_conn, catalog_entry, state, columns)

singer.write_message(singer.StateMessage(value=copy.deepcopy(state)))


def do_sync_full_table(mssql_conn, config, catalog_entry, state, columns):
key_properties = common.get_key_properties(catalog_entry)

write_schema_message(config, catalog_entry)
catalog_entry.stream = common.set_schema_mapping(config, catalog_entry.stream)

write_schema_message(catalog_entry)

stream_version = common.get_stream_version(catalog_entry.tap_stream_id, state)

Expand All @@ -518,7 +519,10 @@ def do_sync_log_based_table(mssql_conn, config, catalog_entry, state, columns):

key_properties = common.get_key_properties(catalog_entry)
state = singer.set_currently_syncing(state, catalog_entry.tap_stream_id)
write_schema_message(config, catalog_entry)

catalog_entry.stream = common.set_schema_mapping(config, catalog_entry.stream)

write_schema_message(catalog_entry)

stream_version = common.get_stream_version(catalog_entry.tap_stream_id, state)

Expand Down
12 changes: 5 additions & 7 deletions tap_mssql/sync_strategies/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,7 @@ def escape(string):
return '"' + string + '"'

def set_schema_mapping(config, stream):
schema_mapping = config.get("include_schemas_in_destination_stream_name")

if schema_mapping:
if config.get("include_schemas_in_destination_stream_name"):
stream = stream.replace('-', '_')
return stream

Expand Down Expand Up @@ -100,7 +98,7 @@ def generate_select_sql(catalog_entry, columns):
return select_sql


def row_to_singer_record(catalog_entry, version, table_stream, row, columns, time_extracted):
def row_to_singer_record(catalog_entry, version, row, columns, time_extracted):
row_to_persist = ()
md_map = metadata.to_map(catalog_entry.metadata)
md_map[("properties", "_sdc_deleted_at")] = {
Expand Down Expand Up @@ -145,7 +143,7 @@ def row_to_singer_record(catalog_entry, version, table_stream, row, columns, tim
rec = dict(zip(columns, row_to_persist))

return singer.RecordMessage(
stream=table_stream,
stream=catalog_entry.stream,
record=rec,
version=version,
time_extracted=time_extracted,
Expand All @@ -164,7 +162,7 @@ def whitelist_bookmark_keys(bookmark_key_set, tap_stream_id, state):


def sync_query(
cursor, catalog_entry, state, select_sql, columns, stream_version, table_stream, params
cursor, catalog_entry, state, select_sql, columns, stream_version, params
):
replication_key = singer.get_bookmark(
state, catalog_entry.tap_stream_id, "replication_key"
Expand All @@ -190,7 +188,7 @@ def sync_query(
counter.increment()
rows_saved += 1
record_message = row_to_singer_record(
catalog_entry, stream_version, table_stream, row, columns, time_extracted
catalog_entry, stream_version, row, columns, time_extracted
)
singer.write_message(record_message)
md_map = metadata.to_map(catalog_entry.metadata)
Expand Down
8 changes: 4 additions & 4 deletions tap_mssql/sync_strategies/full_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,11 @@ def sync_table(mssql_conn, config, catalog_entry, state, columns, stream_version
)

state_version = singer.get_bookmark(state, catalog_entry.tap_stream_id, "version")

table_stream = common.set_schema_mapping(config, catalog_entry.stream)
catalog_entry.stream = common.set_schema_mapping(config, catalog_entry.stream)

activate_version_message = singer.ActivateVersionMessage(
stream=table_stream, version=stream_version
stream=catalog_entry.stream, version=stream_version
)

# For the initial replication, emit an ACTIVATE_VERSION message
Expand All @@ -67,7 +67,7 @@ def sync_table(mssql_conn, config, catalog_entry, state, columns, stream_version
params = {}

common.sync_query(
open_conn, catalog_entry, state, select_sql, columns, stream_version, table_stream, params
open_conn, catalog_entry, state, select_sql, columns, stream_version, params
)

# clear max pk value and last pk fetched upon successful sync
Expand Down
7 changes: 3 additions & 4 deletions tap_mssql/sync_strategies/incremental.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,9 @@ def sync_table(mssql_conn, config, catalog_entry, state, columns):
state, catalog_entry.tap_stream_id, "version", stream_version
)

table_stream = common.set_schema_mapping(config, catalog_entry.stream)

catalog_entry.stream = common.set_schema_mapping(config, catalog_entry.stream)
activate_version_message = singer.ActivateVersionMessage(
stream=table_stream, version=stream_version
stream=catalog_entry.stream, version=stream_version
)

singer.write_message(activate_version_message)
Expand All @@ -77,5 +76,5 @@ def sync_table(mssql_conn, config, catalog_entry, state, columns):
select_sql += ' ORDER BY "{}" ASC'.format(replication_key_metadata)

common.sync_query(
open_conn, catalog_entry, state, select_sql, columns, stream_version, table_stream, params
open_conn, catalog_entry, state, select_sql, columns, stream_version, params
)
6 changes: 3 additions & 3 deletions tap_mssql/sync_strategies/logical.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,12 +267,12 @@ def execute_log_based_sync(self):

desired_columns.append("_sdc_deleted_at")
ordered_row.append(None)

table_stream = self.catalog_entry.stream.replace('-', '_')

self.catalog_entry.stream = common.set_schema_mapping(self.config, self.catalog_entry.stream)

record_message = common.row_to_singer_record(
self.catalog_entry,
stream_version,
table_stream,
ordered_row,
desired_columns,
time_extracted,
Expand Down