Merge pull request #66 from dagster-io/christian-use-dbt-package-and-…

…reload-manifest

Use dbt package for tests and reload manifest

.github/workflows/deploy-dagster-cloud.yml

@@ -79,7 +79,8 @@ jobs:
         run: |
           pip install pyopenssl --upgrade;
           pip install click --upgrade;
-          pip install dbt-core dbt-duckdb dbt-snowflake;
+          pip install dbt-core dbt-duckdb dbt-snowflake; 
+          make deps
           make manifest
 
       # Copy production manifest.json to S3 on merge

.gitignore

@@ -6,6 +6,7 @@ pyvenv.cfg
 *.egg-info
 .ipynb_checkpoints/
 .bashrc
+dbt_project/dbt_packages*
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

Makefile

@@ -8,6 +8,9 @@ clean:
 manifest:
 	dbt parse --project-dir=dbt_project --profiles-dir=dbt_project/config --target BRANCH
 
+deps:
+	dbt deps --project-dir=dbt_project --profiles-dir=dbt_project/config
+
 stateful_dev: clean manifest
 	export DAGSTER_HOME="~/.dagster_home"; dagster dev
 

dbt_project/dbt_project.yml

@@ -4,6 +4,9 @@ config-version: 2
 
 profile: "dbt_project"
 
+vars:
+  'dbt_date:time_zone': 'America/Los_Angeles'
+
 analysis-paths: ["analyses"]
 test-paths: ["tests"]
 macro-paths: ["macros"]
@@ -16,6 +19,8 @@ clean-targets:
 
 models:
   +materialized: table
+  +post-hook:
+    - "{{ dagster.log_columns_in_relation() }}"
 
 query-comment:
   comment: "snowflake_dagster_dbt_v1_opaque_id[[[{{ node.unique_id }}:{{ invocation_id }}]]]"

dbt_project/models/CLEANED/schema.yml

@@ -2,6 +2,8 @@ version: 2
 
 models:
   - name: orders_cleaned
+    config:
+      alias: orders_something_else
     description: "Cleaned version of raw orders data"
     columns:
       - name: user_id
@@ -27,8 +29,10 @@ models:
         description: "The total purchase price for this order"
         data_type: "float"
         tests:
-          - greater_than_zero
-
+          - greater_than_zero:
+              config:
+                alias: testing_alias
+
   - name: users_cleaned
     description: "Raw users data with test accounts removed"
     columns:
@@ -38,6 +42,11 @@ models:
       - name: "company"
         description: "The name of the company that this user belongs to."
         data_type: "str"
+        tests:
+          - dbt_expectations.expect_column_values_to_match_like_pattern_list:
+              like_pattern_list: ["%Sport%","%Co%","%Ltd%", "%Shop%"]
+              match_on: any # (Optional. Default is 'any', which applies an 'OR' for each pattern. If 'all', it applies an 'AND' for each regex.)
+              alias: example_alias
       - name: "created_at"
         description: "When the user account was crated"
         data_type: "timestamp"

dbt_project/package-lock.yml

@@ -0,0 +1,9 @@
+packages:
+- package: calogica/dbt_expectations
+  version: 0.10.3
+- git: https://github.com/dagster-io/dagster.git
+  revision: 3d9a8c7d4276159e7026b183644193ddc7bac576
+  subdirectory: python_modules/libraries/dagster-dbt/dbt_packages/dagster
+- package: calogica/dbt_date
+  version: 0.10.0
+sha1_hash: 480611b79bfc348cb41eff504f7e9a529e329461

dbt_project/packages.yml

@@ -0,0 +1,6 @@
+packages:
+  - package: calogica/dbt_expectations
+    version: 0.10.3
+  - git: "https://github.com/dagster-io/dagster.git"
+    subdirectory: "python_modules/libraries/dagster-dbt/dbt_packages/dagster"
+    revision: 1.6.9 # replace with the version of `dagster` you are using.

hooli_data_eng/assets/dbt_assets.py

@@ -1,8 +1,8 @@
 import json
 import textwrap
+import os
 from pathlib import Path
 from typing import Any, Mapping
-
 from dagster import (
     AutoMaterializePolicy,
     AutoMaterializeRule,
@@ -20,7 +20,8 @@
     DagsterDbtTranslator,
     load_assets_from_dbt_project,
     default_metadata_from_dbt_resource_props,
-    DagsterDbtTranslatorSettings
+    DagsterDbtTranslatorSettings,
+    DbtArtifacts,
 )
 from dagster_dbt.asset_decorator import dbt_assets
 from dagster._utils import file_relative_path
@@ -36,11 +37,19 @@
 DBT_PROJECT_DIR = file_relative_path(__file__, "../../dbt_project")
 DBT_PROFILES_DIR = file_relative_path(__file__, "../../dbt_project/config")
 
-# this manifest is created at build/deploy time, see the Makefile & .github/workflows/deploy-dagster-cloud.yml#70
-# see also: https://docs.dagster.io/integrations/dbt/reference#deploying-a-dagster-project-with-a-dbt-project
-DBT_MANIFEST = Path(
-    file_relative_path(__file__, "../../dbt_project/target/manifest.json")
+# new in 1.6.9, DbtArtifacts is an experimental class that creates a manifest on load 
+# if DAGSTER_DBT_PARSE_PROJECT_ON_LOAD is present, 
+# otherwise it points to the already-built manifest
+dbt_artifacts = DbtArtifacts(
+    project_dir=DBT_PROJECT_DIR,
+    prepare_command=["--quiet",
+                     "parse",
+                     "--target",
+                     "BRANCH",
+                     "--profiles-dir",
+                     DBT_PROFILES_DIR],
 )
+DBT_MANIFEST = dbt_artifacts.manifest_path
 
 # this manifest represents the last successful dbt deployment and will be compared against the current deployment
 SLIM_CI_MANIFEST =  Path(