Skip to content

Commit

Permalink
Refactor test for clickhouse gcs_s3 compatibility
Browse files Browse the repository at this point in the history
Signed-off-by: Marcel Coetzee <[email protected]>
  • Loading branch information
Pipboyguy committed May 28, 2024
1 parent abd87f8 commit 4c3186b
Showing 1 changed file with 7 additions and 9 deletions.
16 changes: 7 additions & 9 deletions tests/load/clickhouse/test_clickhouse_gcs_s3_compatibility.py
Original file line number Diff line number Diff line change
@@ -1,25 +1,23 @@
from typing import Generator, Dict

import pytest

import dlt
from dlt.destinations import filesystem
from tests.load.utils import GCS_BUCKET
from tests.pipeline.utils import assert_load_info


@pytest.mark.essential
def test_clickhouse_gcs_s3_compatibility() -> None:
@dlt.resource
def dummy_data() -> Generator[Dict[str, int], None, None]:
yield {"field1": 1, "field2": 2}

gcp_bucket = filesystem(
GCS_BUCKET.replace("gs://", "s3://"), destination_name="filesystem_s3_gcs_comp"
)

pipe = dlt.pipeline(
pipeline_name="gcs_s3_compatibility",
destination="clickhouse",
staging=gcp_bucket,
full_refresh=True,
)
gcp_bucket = filesystem(GCS_BUCKET.replace("gs://", "s3://"), destination_name="filesystem_s3_gcs_comp")

pipe = dlt.pipeline(pipeline_name="gcs_s3_compatibility", destination="clickhouse", staging=gcp_bucket,
full_refresh=True, )
pack = pipe.run([dummy_data])
assert_load_info(pack)

0 comments on commit 4c3186b

Please sign in to comment.