Skip to content

Commit

Permalink
Feature/10 merge state (#12)
Browse files Browse the repository at this point in the history
* Added state merging

* Bumped version
  • Loading branch information
JulesHuisman authored Aug 17, 2023
1 parent b6226dc commit b1efe8b
Show file tree
Hide file tree
Showing 5 changed files with 36 additions and 2 deletions.
7 changes: 6 additions & 1 deletion elx/state.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,9 +140,14 @@ def save(self, state_file_name: str, state: dict = {}) -> None:
Args:
state_file_name (str): The name of the state file to save.
"""
# We first merge with any existing state to ensure that we don't overwrite
existing_state = self.load(state_file_name)
merged_state = {**existing_state, **state}

# Then we write the merged state to the state file
with open(
f"{self.base_path}/{state_file_name}",
"wb",
transport_params=self.state_client.params,
) as state_file:
state_file.write(json.dumps(state).encode("utf-8"))
state_file.write(json.dumps(merged_state).encode("utf-8"))
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "elx"
version = "0.1.0a4"
version = "0.1.0a5"
description = "A lightweight Python interface for extracting and loading using the Singer.io spec."
authors = ["Jules Huisman <[email protected]>"]
readme = "README.md"
Expand Down
1 change: 1 addition & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@
from fixtures.tap import tap
from fixtures.target import target
from fixtures.runner import runner
from fixtures.state import state_manager
13 changes: 13 additions & 0 deletions tests/fixtures/state.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
from typing import Generator
import pytest
from elx import StateManager


@pytest.fixture
def state_manager(tmp_path) -> Generator[StateManager, None, None]:
"""
Return a StateManager instance.
"""
yield StateManager(
base_path=str(tmp_path),
)
15 changes: 15 additions & 0 deletions tests/test_elx/test_state.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from anyio import Path
from click import File
import pytest
from pytest import MonkeyPatch
Expand Down Expand Up @@ -39,3 +40,17 @@ def test_transport_parameters_gcs_token(monkeypatch: MonkeyPatch):
gcs_state_client = state_client_factory("gs://bucket/path")
assert "client" in gcs_state_client.params
assert type(gcs_state_client.params["client"]) == Client


def test_state_save(state_manager: StateManager):
"""
Test that state is saved to the correct path.
"""
state_manager.save("test.json", {"foo": "bar"})
assert Path(state_manager.base_path, "test.json").exists()


def test_state_merge(state_manager: StateManager):
state_manager.save("test.json", {"foo": "bar"})
state_manager.save("test.json", {"bar": "foo"})
assert state_manager.load("test.json") == {"foo": "bar", "bar": "foo"}

0 comments on commit b1efe8b

Please sign in to comment.