Merge pull request #11 from EJOOSTEROP/updt_dickdb_versionb

Ready for DuckDB v1 (and MotherDuck)

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 <!-- insertion marker -->
 ## [Unreleased]
 
+## [0.X] 2024-08-02
+### Changed
+- POSSIBLE BREAKING CHANGE: Update dbt to accommodate DuckDB v1.0. Rewquires newer versions of DuckDB,
+- Create new pipeline version/name
+
 ## [0.4.6] 2024-03-03
 ### Added
 - Release version for sentry.io

pyproject.toml

@@ -18,7 +18,7 @@ disallow_incomplete_defs = true
 
 [tool.poetry]
 name = "ternyxmimosa"
-version = "0.4.6"
+version = "0.5.0"
 description = "A minimal modern data stack with working data pipelines in a single Docker container."
 authors = ["Erik Oosterop <[email protected]>"]
 license = "MIT"
@@ -33,7 +33,7 @@ packages = [ { include = "mimosa", from = "src" } ]
 python = ">=3.11,<3.13"
 python-dotenv = "^1.0.0"
 loguru = "^0.7.2"
-pandas = "2.0.3"
+pandas = "^2.2.2"
 streamlit = "^1.26.0"
 dlt = {extras = ["cli", "filesystem", "motherduck", "parquet"], version = "^0.4.4"}
 sentry-sdk = "^1.40.6"

src/mimosa/dbt/dbt_project.yml

@@ -18,6 +18,11 @@ clean-targets:
     - "target"
     - "dbt_modules"
 
+# The following causes an error. Not certain how these yml files are updated in dbt venv.
+#flags:
+#  send_anonymous_usage_stats: False
+#  use_colors: True
+
 # Configuring models
 # Full documentation: https://docs.getdbt.com/docs/configuring-models
 

src/mimosa/dbt/profiles.yml

@@ -1,7 +1,4 @@
 # https://docs.getdbt.com/docs/core/connect-data-platform/connection-profiles
-config:
-  send_anonymous_usage_stats: False
-  use_colors: True
 gei:
   #target: "{{ env_var('MELTANO_ENVIRONMENT', 'dev') }}"
   target: "dev"

src/mimosa/pipelines.py

@@ -43,7 +43,7 @@ def __init__(self, destination="motherduck"):
         self.timing_key = "gasDayStart"
         self.primary_key = ("gasDayStart", "code")
 
-        self.pipeline_name = "gas_storage"
+        self.pipeline_name = "gas_storage_v2"
         # credentials for the destination may be required
         self.destination = destination  # "duckdb" "motherduck"
         logger.info(f"Using destination: {self.destination}")
@@ -168,7 +168,7 @@ def run_reporting_pipeline(self):
         logger.debug("Starting to obtain a dbt venv.")
         venv = dlt.dbt.get_venv(pipeline)
         venv.run_module(
-            "pip", "install", "duckdb==0.9.2"
+            "pip", "install", "dbt-duckdb"
         )  # TODO: this does not always need to be a fixed version (20230926)
 
         # get runner, optionally pass the venv