Skip to content

Commit

Permalink
feat: Use sorted API requests to support crash tolerant incremental s…
Browse files Browse the repository at this point in the history
…ync (#32)

The streams changed support sorted responses

Ex: https://developers.klaviyo.com/en/reference/get_campaigns

This PR takes advantage of ascending timestamps to support fault
tolerant incremental syncs that picks up at the last left-off place.
  • Loading branch information
JichaoS authored Dec 15, 2023
1 parent d02a42a commit b723520
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 0 deletions.
3 changes: 3 additions & 0 deletions tap_klaviyo/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,9 @@ def get_url_params(
else:
filter_timestamp = DEFAULT_START_DATE

if self.is_sorted:
params["sort"] = self.replication_key

params["filter"] = f"greater-than({self.replication_key},{filter_timestamp})"

return params
16 changes: 16 additions & 0 deletions tap_klaviyo/streams.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ def post_process(
row["datetime"] = row["attributes"]["datetime"]
return row

@property
def is_sorted(self) -> bool:
return True


class CampaignsStream(KlaviyoStream):
"""Define custom stream."""
Expand Down Expand Up @@ -73,6 +77,10 @@ def post_process(
row["updated_at"] = row["attributes"]["updated_at"]
return row

@property
def is_sorted(self) -> bool:
return True


class ProfilesStream(KlaviyoStream):
"""Define custom stream."""
Expand All @@ -91,6 +99,10 @@ def post_process(
row["updated"] = row["attributes"]["updated"]
return row

@property
def is_sorted(self) -> bool:
return True


class MetricsStream(KlaviyoStream):
"""Define custom stream."""
Expand Down Expand Up @@ -175,3 +187,7 @@ def post_process(
) -> dict | None:
row["updated"] = row["attributes"]["updated"]
return row

@property
def is_sorted(self) -> bool:
return True

0 comments on commit b723520

Please sign in to comment.