Merge pull request #205 from henriblancke/support-versioned-models

feat: support dbt model versions

demo_duckdb/models/customers.sql

@@ -1,6 +1,6 @@
 with customers as (
 
-    select * from {{ ref('stg_customers') }}
+    select * from {{ ref('stg_customers', v=1) }}
 
 ),
 

demo_duckdb/models/staging/jaffle_shop/main/stg_customers.yml

@@ -2,14 +2,29 @@
 version: 2
 models:
   - name: stg_customers
-    columns:
-      - name: customer_id
-        tests:
-          - unique
-          - not_null
-
-        data_type: INTEGER
-      - name: first_name
-        data_type: VARCHAR
-      - name: last_name
-        data_type: VARCHAR
+    latest_version: 1
+    versions:
+      - v: 1
+        config:
+          alias: stg_customers
+        columns:
+          - name: customer_id
+            tests:
+              - unique
+              - not_null
+            data_type: INTEGER
+          - name: first_name
+            data_type: VARCHAR
+          - name: last_name
+            data_type: VARCHAR
+      - v: 2
+        columns:
+          - name: id
+            tests:
+              - unique
+              - not_null
+            data_type: INTEGER
+          - name: first_name
+            data_type: VARCHAR
+          - name: last_name
+            data_type: VARCHAR

demo_duckdb/models/staging/stg_customers_v2.sql

@@ -0,0 +1,22 @@
+with source as (
+
+    {#-
+    Normally we would select from the table here, but we are using seeds to load
+    our data in this project
+    #}
+    select * from {{ ref('raw_customers') }}
+
+),
+
+renamed as (
+
+    select
+        id,
+        first_name,
+        last_name
+
+    from source
+
+)
+
+select * from renamed

src/dbt_osmosis/core/osmosis.py

@@ -1515,8 +1515,17 @@ def sync_node_to_yaml(
         doc_obj: dict[str, t.Any] | None = None
         for item in doc_list:
             if item.get("name") == node.name:
-                doc_obj = item
-                break
+                # check if model is versioned
+                # versions have to be defined in yaml meaning
+                # there'll always be an existing properties yaml file
+                if isinstance(node, ModelNode) and node.version is not None:
+                    for version in item.get("versions", []):
+                        if version.get("v") == node.version:
+                            doc_obj = version
+                            break
+                else:
+                    doc_obj = item
+                    break
         if not doc_obj:
             doc_obj = {
                 "name": node.name,

tests/test_core.py

@@ -235,6 +235,11 @@ def test_sync_node_to_yaml(yaml_context: YamlRefactorContext, fresh_caches):
     sync_node_to_yaml(yaml_context, node, commit=False)
 
 
+def test_sync_node_to_yaml_versioned(yaml_context: YamlRefactorContext, fresh_caches):
+    node = yaml_context.project.manifest.nodes["model.jaffle_shop_duckdb.stg_customers.v2"]
+    sync_node_to_yaml(yaml_context, node, commit=False)
+
+
 def test_commit_yamls_no_write(yaml_context: YamlRefactorContext):
     """
     Since dry_run=True, commit_yamls should not actually write anything to disk.

tests/test_yaml_inheritance.py

@@ -54,7 +54,7 @@ def load_manifest() -> Manifest:
             {
                 "generation_0": ["model.jaffle_shop_duckdb.customers"],
                 "generation_1": [
-                    "model.jaffle_shop_duckdb.stg_customers",
+                    "model.jaffle_shop_duckdb.stg_customers.v1",
                     "model.jaffle_shop_duckdb.stg_orders",
                     "model.jaffle_shop_duckdb.stg_payments",
                 ],
@@ -85,7 +85,7 @@ def test_build_node_ancestor_tree(node_id: str, expected_tree: dict[str, list[st
         (
             {"force_inherit_descriptions": False, "add_progenitor_to_meta": True},
             {
-                "stg_customers.customer_id": {
+                "stg_customers.v1.customer_id": {
                     "description": "I will be inherited, forcibly so :)",
                     "meta": {"a": 1, "b": 2},
                     "tags": ["foo", "bar"],
@@ -97,7 +97,7 @@ def test_build_node_ancestor_tree(node_id: str, expected_tree: dict[str, list[st
                     "a": 1,
                     "b": 2,
                     "c": 3,
-                    "osmosis_progenitor": "model.jaffle_shop_duckdb.stg_customers",
+                    "osmosis_progenitor": "model.jaffle_shop_duckdb.stg_customers.v1",
                 },
                 "tags": ["foo", "bar", "baz"],
             },
@@ -106,7 +106,7 @@ def test_build_node_ancestor_tree(node_id: str, expected_tree: dict[str, list[st
         (
             {"force_inherit_descriptions": True, "add_progenitor_to_meta": True},
             {
-                "stg_customers.customer_id": {
+                "stg_customers.v1.customer_id": {
                     "description": "I will be inherited, forcibly so :)",
                     "meta": {"a": 1, "b": 2},
                     "tags": ["foo", "bar"],
@@ -118,7 +118,7 @@ def test_build_node_ancestor_tree(node_id: str, expected_tree: dict[str, list[st
                     "a": 1,
                     "b": 2,
                     "c": 3,
-                    "osmosis_progenitor": "model.jaffle_shop_duckdb.stg_customers",
+                    "osmosis_progenitor": "model.jaffle_shop_duckdb.stg_customers.v1",
                 },
                 "tags": ["foo", "bar", "baz"],
             },
@@ -127,7 +127,7 @@ def test_build_node_ancestor_tree(node_id: str, expected_tree: dict[str, list[st
         (
             {"skip_add_tags": True, "skip_merge_meta": True},
             {
-                "stg_customers.customer_id": {
+                "stg_customers.v1.customer_id": {
                     "description": "I will not be inherited, since the customer table documents me",
                     "meta": {"a": 1},
                     "tags": ["foo", "bar"],
@@ -143,7 +143,7 @@ def test_build_node_ancestor_tree(node_id: str, expected_tree: dict[str, list[st
         (
             {"use_unrendered_descriptions": True, "force_inherit_descriptions": True},
             {
-                "stg_customers.customer_id": {
+                "stg_customers.v1.customer_id": {
                     "description": "{{ doc('stg_customer_description') }}",
                     "meta": {"d": 4},
                     "tags": ["rendered", "unrendered"],
@@ -159,7 +159,7 @@ def test_build_node_ancestor_tree(node_id: str, expected_tree: dict[str, list[st
         (
             {"skip_add_data_types": True, "add_inheritance_for_specified_keys": ["quote"]},
             {
-                "stg_customers.customer_id": {
+                "stg_customers.v1.customer_id": {
                     "description": "Keep on, keeping on",
                     "meta": {"e": 5},
                     "tags": ["constrainted"],
@@ -177,7 +177,7 @@ def test_build_node_ancestor_tree(node_id: str, expected_tree: dict[str, list[st
         (
             {"output_to_lower": True},
             {
-                "stg_customers.customer_id": {
+                "stg_customers.v1.customer_id": {
                     "name": "WTF",
                 }
             },
@@ -197,7 +197,7 @@ def test_build_node_ancestor_tree(node_id: str, expected_tree: dict[str, list[st
                 "force_inherit_descriptions": True,
             },
             {
-                "stg_customers.customer_id": {
+                "stg_customers.v1.customer_id": {
                     "description": "I will prevail",
                     "meta": {"a": 1},
                     "tags": ["foo", "bar"],
@@ -233,8 +233,16 @@ def test_inherit_upstream_column_knowledge_with_various_settings(
 
     # Modify upstream column data
     for column_path, mods in upstream_mutations.items():
-        node_id, column_name = column_path.split(".")
-        upstream_col = manifest.nodes[f"model.jaffle_shop_duckdb.{node_id}"].columns[column_name]
+        components = column_path.split(".")
+
+        if len(components) > 2:
+            node_id, version, column_name = components
+            node = f"model.jaffle_shop_duckdb.{node_id}.{version}"
+        else:
+            node_id, column_name = components
+            node = f"model.jaffle_shop_duckdb.{node_id}"
+
+        upstream_col = manifest.nodes[node].columns[column_name]
         for attr, attr_value in mods.items():
             setattr(upstream_col, attr, attr_value)
 
@@ -289,15 +297,15 @@ def test_use_unrendered_descriptions(
 
 def test_inherit_upstream_column_knowledge(yaml_context: YamlRefactorContext):
     manifest = yaml_context.project.manifest
-    manifest.nodes["model.jaffle_shop_duckdb.stg_customers"].columns[
+    manifest.nodes["model.jaffle_shop_duckdb.stg_customers.v1"].columns[
         "customer_id"
     ].description = "THIS COLUMN IS UPDATED FOR TESTING"
 
     expect: dict[str, t.Any] = {
         "customer_id": {
             "name": "customer_id",
             "description": "THIS COLUMN IS UPDATED FOR TESTING",
-            "meta": {"osmosis_progenitor": "model.jaffle_shop_duckdb.stg_customers"},
+            "meta": {"osmosis_progenitor": "model.jaffle_shop_duckdb.stg_customers.v1"},
             "data_type": "INTEGER",
             "constraints": [],
             "quote": None,

tests/test_yaml_knowledge_graph.py

@@ -28,7 +28,7 @@ def test_get_prior_knowledge(self, yaml_context: YamlRefactorContext):
         # Progenitor gives us an idea of where the inherited traits will come from
         knowledge: dict[str, t.Any] = {
             "customer_id": {
-                "meta": {"osmosis_progenitor": "model.jaffle_shop_duckdb.stg_customers"},
+                "meta": {"osmosis_progenitor": "model.jaffle_shop_duckdb.stg_customers.v1"},
                 "name": "customer_id",
                 "data_type": "INTEGER",
                 "constraints": [],