Demo/bigquery (#16)

* update configs

* update profile

* draft PR

---------

Co-authored-by: Sung Won Chung <[email protected]>

dbt_project.yml

@@ -1,13 +1,13 @@
 name: "jaffle_shop"
 # Specify a requirement on an _exact_ version of dbt
-require-dbt-version: 0.19.0
+# require-dbt-version: 0.19.0
 version: 1.0.0
 config-version: 2
 profile: "jaffle_shop"
-source-paths: ["models"]
+model-paths: ["models"]
 analysis-paths: ["analysis"]
 test-paths: ["tests"]
-data-paths: ["data"]
+seed-paths: ["data"]
 macro-paths: ["macros"]
 snapshot-paths: ["snapshots"] # https://docs.getdbt.com/docs/snapshots#section-what-are-snapshots-
 

.github/workflows/dbt_operations.yml → dbt_pull_request_job.yml

@@ -8,9 +8,9 @@ on:
 
 env:
   #TODO(developer): update PROJECT_ID
-  PROJECT_ID: "dvc-demo-297316"
+  PROJECT_ID: "dbt-demo-386220"
   DBT_PROFILES_DIR: ./
-  DBT_GOOGLE_BIGQUERY_KEYFILE: ./service_account.json
+  GOOGLE_APPLICATION_CREDENTIALS: ./service_account.json
 
 jobs:
   dbt:
@@ -28,7 +28,7 @@ jobs:
       - run: 'echo "$KEYFILE" > ./service_account.json' # persist file in root dir
         shell: bash
         env:
-          KEYFILE: ${{secrets.DBT_GOOGLE_BIGQUERY_KEYFILE}} # add a github secret
+          KEYFILE: ${{secrets.GOOGLE_APPLICATION_CREDENTIALS}} # add a github secret
       # run actual dbt commands
       - run: dbt debug --target service_account_runs
       - run: dbt seed --full-refresh --show --target service_account_runs

models/sources/sources.yml

@@ -2,7 +2,7 @@ version: 2
 
 sources:
   - name: dbt_bq_example
-    database: "{{ env_var('PROJECT_ID', 'dvc-demo-297316') }}" 
+    database: "{{ env_var('PROJECT_ID', 'dbt-demo-386220') }}" 
     schema: dbt_bq_example
 
     freshness:

profiles.yml

@@ -6,25 +6,12 @@ jaffle_shop:
   target: dev
   outputs:
     dev:
-      type: bigquery
-      method: oauth # for production runs: https://docs.getdbt.com/docs/profile-bigquery#section-service-account-file-authentication
-      project: "{{ env_var('PROJECT_ID', 'wam-bam-258119') }}" # parameterize project for dbt runs based on environment variable with default project
-      dataset: dbt_bq_example # You can also use "schema" here, can add env_var here if desired
-      threads: 8 # https://docs.getdbt.com/docs/configure-your-profile#section-understanding-threads
-      timeout_seconds: 300
-      location: US # Optional, one of US or EU
-      priority: interactive # interactive or batch
-
-    # use the below for service account authenticated dbt runs
-    service_account_runs:
       type: bigquery
       method: service-account # for production runs: https://docs.getdbt.com/docs/profile-bigquery#section-service-account-file-authentication
-      project: "{{ env_var('PROJECT_ID', 'wam-bam-258119') }}" # parameterize project for dbt runs based on environment variable with default project
-      dataset: dbt_bq_example # You can also use "schema" here
-      threads: 8 # https://docs.getdbt.com/docs/configure-your-profile#section-understanding-threads
-      keyfile: "{{ env_var('DBT_GOOGLE_BIGQUERY_KEYFILE', 'service_account.json') }}"
+      project: "{{ env_var('PROJECT_ID', 'dbt-demo-386220') }}" # parameterize project for dbt runs based on environment variable with default project
+      dataset: dbt_sung # You can also use "schema" here
+      threads: 24 # https://docs.getdbt.com/docs/configure-your-profile#section-understanding-threads
+      keyfile: "{{ env_var('GOOGLE_APPLICATION_CREDENTIALS', 'service_account.json') }}"
       timeout_seconds: 300
       location: US # Optional, one of US or EU
       priority: interactive # interactive or batch
-  config:
-    send_anonymous_usage_stats: False

requirements.txt

@@ -1 +1 @@
-dbt==0.19.0
+dbt-bigquery==1.6.5