dlt patch update (fixing MotherDuck version bump). Patch updates of d…

…ependencies.

.gitignore

@@ -1,6 +1,8 @@
 # Custom
 *.duckdb
 gas_storage.duckdb
+r2.py
+src/mimosa/r2.py
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

CHANGELOG.md

@@ -7,7 +7,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 <!-- insertion marker -->
 ## [Unreleased]
 
-## [0.2.0] 2023-09-26
+## [0.2.1] 2023-10-17
+
+### Changed
+- Version/patch bump for dlt, fixing MotherDuck/DuckDB destination (v0.9.1)
+
+## [0.2.0] 2023-10-16
 
 ### Changed
 - Use duckdb version >=0.9.1

pyproject.toml

@@ -18,7 +18,7 @@ disallow_incomplete_defs = true
 
 [tool.poetry]
 name = "ternyxmimosa"
-version = "0.2.0"
+version = "0.2.1"
 description = "A minimal modern data stack with working data pipelines in a single Docker container."
 authors = ["Erik Oosterop <[email protected]>"]
 license = "MIT"
@@ -31,7 +31,7 @@ packages = [ { include = "mimosa", from = "src" } ]
 
 [tool.poetry.dependencies]
 python = "^3.11"
-dlt = {extras = ["motherduck"], version = "^0.3.17"}
+dlt = {extras = ["cli", "filesystem", "motherduck"], version = "^0.3.21"}
 python-dotenv = "^1.0.0"
 loguru = "^0.7.2"
 pandas = "2.0.3"

src/mimosa/pipelines.py

@@ -41,6 +41,14 @@ def __init__(self, destination="motherduck"):
                 msg = "MotherDuck credentials missing. Set in environment variable DESTINATION__MOTHERDUCK__CREDENTIALS."
                 raise MissingEnvironmentVariableError(msg) from e
 
+        if self.destination == "filesystem___XXX":
+            # TODO: Potentiall remove. Is here to try Cloudlfare R2, but that is not working yet.
+            try:
+                _ = os.environ["DESTINATION__FILESYSTEM__BUCKET_URL"]
+            except KeyError as e:
+                msg = "Filesystem bucket url missing. Set in environment variable DESTINATION__FILESYSTEM__BUCKET_URL."
+                raise MissingEnvironmentVariableError(msg) from e
+
     @dlt.resource(
         primary_key=("gasDayStart", "code"),
         table_name="storage",

src/mimosa/wip.py

@@ -15,9 +15,14 @@
 if run_this:
     pipeline.run_landing_pipeline(
         gas_date=date(2023, 9, 27),  # still from 2019-01-01 to 2019-09-01
-        to_gas_date=date(2023, 10, 1),
+        to_gas_date=date(2023, 9, 28),
         reporting_update=reporting_update,
     )
+    """
+    pipeline.run_landing_pipeline(
+        reporting_update=reporting_update,
+    )
+    """
 else:
     pipeline.run_landing_pipeline()