-
Notifications
You must be signed in to change notification settings - Fork 215
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Refactor test for clickhouse gcs_s3 compatibility
Signed-off-by: Marcel Coetzee <[email protected]>
- Loading branch information
Showing
1 changed file
with
7 additions
and
9 deletions.
There are no files selected for viewing
16 changes: 7 additions & 9 deletions
16
tests/load/clickhouse/test_clickhouse_gcs_s3_compatibility.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,25 +1,23 @@ | ||
from typing import Generator, Dict | ||
|
||
import pytest | ||
|
||
import dlt | ||
from dlt.destinations import filesystem | ||
from tests.load.utils import GCS_BUCKET | ||
from tests.pipeline.utils import assert_load_info | ||
|
||
|
||
@pytest.mark.essential | ||
def test_clickhouse_gcs_s3_compatibility() -> None: | ||
@dlt.resource | ||
def dummy_data() -> Generator[Dict[str, int], None, None]: | ||
yield {"field1": 1, "field2": 2} | ||
|
||
gcp_bucket = filesystem( | ||
GCS_BUCKET.replace("gs://", "s3://"), destination_name="filesystem_s3_gcs_comp" | ||
) | ||
|
||
pipe = dlt.pipeline( | ||
pipeline_name="gcs_s3_compatibility", | ||
destination="clickhouse", | ||
staging=gcp_bucket, | ||
full_refresh=True, | ||
) | ||
gcp_bucket = filesystem(GCS_BUCKET.replace("gs://", "s3://"), destination_name="filesystem_s3_gcs_comp") | ||
|
||
pipe = dlt.pipeline(pipeline_name="gcs_s3_compatibility", destination="clickhouse", staging=gcp_bucket, | ||
full_refresh=True, ) | ||
pack = pipe.run([dummy_data]) | ||
assert_load_info(pack) |