Skip to content
This repository has been archived by the owner on Sep 23, 2024. It is now read-only.

Fixes https://github.com/transferwise/pipelinewise-tap-postgres/issues/107 #130

Closed
wants to merge 1 commit into from
Closed

Fixes https://github.com/transferwise/pipelinewise-tap-postgres/issues/107 #130

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions tap_postgres/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,8 @@ def sync_traditional_stream(conn_config, stream, state, sync_method, end_lsn):
return state


def sync_logical_streams(conn_config, logical_streams, state, end_lsn, state_file):
# pylint: disable=too-many-arguments
def sync_logical_streams(conn_config, logical_streams, traditional_streams, state, end_lsn, state_file):
"""
Sync streams that use LOG_BASED method
"""
Expand All @@ -212,10 +213,20 @@ def sync_logical_streams(conn_config, logical_streams, state, end_lsn, state_fil
selected_streams.add("{}".format(stream['tap_stream_id']))

new_state = dict(currently_syncing=state['currently_syncing'], bookmarks={})
traditional_stream_ids = [s['tap_stream_id'] for s in traditional_streams]

for stream, bookmark in state['bookmarks'].items():
if bookmark == {} or bookmark['last_replication_method'] != 'LOG_BASED' or stream in selected_streams:
if (
bookmark == {}
or bookmark['last_replication_method'] != 'LOG_BASED'
or stream in selected_streams
# The first time a LOG_BASED stream runs it needs to do an
# initial full table sync, and so will be treated as a
# traditional stream.
or (stream in traditional_stream_ids and bookmark['last_replication_method'] == 'LOG_BASED')
):
new_state['bookmarks'][stream] = bookmark

state = new_state

state = logical_replication.sync_tables(conn_config, logical_streams, state, end_lsn, state_file)
Expand Down Expand Up @@ -319,7 +330,7 @@ def do_sync(conn_config, catalog, default_replication_method, state, state_file=
for dbname, streams in itertools.groupby(logical_streams,
lambda s: metadata.to_map(s['metadata']).get(()).get('database-name')):
conn_config['dbname'] = dbname
state = sync_logical_streams(conn_config, list(streams), state, end_lsn, state_file)
state = sync_logical_streams(conn_config, list(streams), traditional_streams, state, end_lsn, state_file)
return state


Expand Down