diff --git a/.github/workflows/cloud_run.yml b/.github/workflows/cloud_run.yml deleted file mode 100644 index d9d6af4..0000000 --- a/.github/workflows/cloud_run.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: Cloud Run Website - -on: - push: - branches: [master] - pull_request: - branches: [master] - -env: - #TODO(developer): update PROJECT_ID, REGION, IMAGE - PROJECT_ID: "dvc-demo-297316" - REGION: "us-central1" - IMAGE: "dbt-docs-cloud-run" - DBT_PROFILES_DIR: ./ - DBT_GOOGLE_BIGQUERY_KEYFILE: ./service_account.json - -jobs: - setup-build-deploy: - name: Setup, Build, and Deploy - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v2 - - # Setup gcloud CLI - - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master - with: - version: "296.0.0" - service_account_key: ${{secrets.DBT_GOOGLE_BIGQUERY_KEYFILE}} - - # Setup dbt configs - - uses: actions/setup-python@v1 - with: - python-version: "3.8.x" - - run: pip3 install -r ./requirements.txt - - run: dbt --version - - run: 'echo "$KEYFILE" > ./service_account.json' # persist file in root dir - shell: bash - env: - KEYFILE: ${{secrets.DBT_GOOGLE_BIGQUERY_KEYFILE}} # add a github secret - - run: dbt debug --target service_account_runs - - # Build and push image to Google Container Registry - # gcloud builds submit CLI is NOT used as it does not persist dbt docs generation run beforehand - - name: Build and push to Google Container Registry - run: |- - dbt docs generate --target service_account_runs && - docker build . --tag gcr.io/$PROJECT_ID/$IMAGE:$GITHUB_SHA && - gcloud auth configure-docker --quiet && - docker push gcr.io/$PROJECT_ID/$IMAGE:$GITHUB_SHA - - # Deploy image to Cloud Run based on branch name - - name: Deploy-Live URL can be found HERE - run: |- - gcloud run deploy "$IMAGE-${GITHUB_REF##*/}" \ - --quiet \ - --project "$PROJECT_ID" \ - --region "$REGION" \ - --image "gcr.io/$PROJECT_ID/$IMAGE:$GITHUB_SHA" \ - --platform "managed" \ - --allow-unauthenticated diff --git a/dbt_project.yml b/dbt_project.yml index af3a07b..316286e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,13 +1,13 @@ name: "jaffle_shop" # Specify a requirement on an _exact_ version of dbt -require-dbt-version: 0.19.0 +# require-dbt-version: 0.19.0 version: 1.0.0 config-version: 2 profile: "jaffle_shop" -source-paths: ["models"] +model-paths: ["models"] analysis-paths: ["analysis"] test-paths: ["tests"] -data-paths: ["data"] +seed-paths: ["data"] macro-paths: ["macros"] snapshot-paths: ["snapshots"] # https://docs.getdbt.com/docs/snapshots#section-what-are-snapshots- diff --git a/.github/workflows/dbt_operations.yml b/dbt_pull_request_job.yml similarity index 84% rename from .github/workflows/dbt_operations.yml rename to dbt_pull_request_job.yml index 75eec27..c112477 100644 --- a/.github/workflows/dbt_operations.yml +++ b/dbt_pull_request_job.yml @@ -8,9 +8,9 @@ on: env: #TODO(developer): update PROJECT_ID - PROJECT_ID: "dvc-demo-297316" + PROJECT_ID: "dbt-demo-386220" DBT_PROFILES_DIR: ./ - DBT_GOOGLE_BIGQUERY_KEYFILE: ./service_account.json + GOOGLE_APPLICATION_CREDENTIALS: ./service_account.json jobs: dbt: @@ -28,7 +28,7 @@ jobs: - run: 'echo "$KEYFILE" > ./service_account.json' # persist file in root dir shell: bash env: - KEYFILE: ${{secrets.DBT_GOOGLE_BIGQUERY_KEYFILE}} # add a github secret + KEYFILE: ${{secrets.GOOGLE_APPLICATION_CREDENTIALS}} # add a github secret # run actual dbt commands - run: dbt debug --target service_account_runs - run: dbt seed --full-refresh --show --target service_account_runs diff --git a/models/sources/sources.yml b/models/sources/sources.yml index 2a0bbf8..5eb7dc3 100644 --- a/models/sources/sources.yml +++ b/models/sources/sources.yml @@ -2,7 +2,7 @@ version: 2 sources: - name: dbt_bq_example - database: "{{ env_var('PROJECT_ID', 'dvc-demo-297316') }}" + database: "{{ env_var('PROJECT_ID', 'dbt-demo-386220') }}" schema: dbt_bq_example freshness: diff --git a/profiles.yml b/profiles.yml index 07df366..7a09028 100644 --- a/profiles.yml +++ b/profiles.yml @@ -6,25 +6,12 @@ jaffle_shop: target: dev outputs: dev: - type: bigquery - method: oauth # for production runs: https://docs.getdbt.com/docs/profile-bigquery#section-service-account-file-authentication - project: "{{ env_var('PROJECT_ID', 'wam-bam-258119') }}" # parameterize project for dbt runs based on environment variable with default project - dataset: dbt_bq_example # You can also use "schema" here, can add env_var here if desired - threads: 8 # https://docs.getdbt.com/docs/configure-your-profile#section-understanding-threads - timeout_seconds: 300 - location: US # Optional, one of US or EU - priority: interactive # interactive or batch - - # use the below for service account authenticated dbt runs - service_account_runs: type: bigquery method: service-account # for production runs: https://docs.getdbt.com/docs/profile-bigquery#section-service-account-file-authentication - project: "{{ env_var('PROJECT_ID', 'wam-bam-258119') }}" # parameterize project for dbt runs based on environment variable with default project - dataset: dbt_bq_example # You can also use "schema" here - threads: 8 # https://docs.getdbt.com/docs/configure-your-profile#section-understanding-threads - keyfile: "{{ env_var('DBT_GOOGLE_BIGQUERY_KEYFILE', 'service_account.json') }}" + project: "{{ env_var('PROJECT_ID', 'dbt-demo-386220') }}" # parameterize project for dbt runs based on environment variable with default project + dataset: dbt_sung # You can also use "schema" here + threads: 24 # https://docs.getdbt.com/docs/configure-your-profile#section-understanding-threads + keyfile: "{{ env_var('GOOGLE_APPLICATION_CREDENTIALS', 'service_account.json') }}" timeout_seconds: 300 location: US # Optional, one of US or EU priority: interactive # interactive or batch - config: - send_anonymous_usage_stats: False diff --git a/requirements.txt b/requirements.txt index 38e8ae6..856690a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -dbt==0.19.0 \ No newline at end of file +dbt-bigquery==1.6.5 \ No newline at end of file diff --git a/snapshots/orders_snapshot.sql b/snapshots/orders_snapshot.sql deleted file mode 100644 index e4a4bae..0000000 --- a/snapshots/orders_snapshot.sql +++ /dev/null @@ -1,27 +0,0 @@ -/* - This snapshot table will live in: - analytics.snapshots.orders_snapshot - Run with below command: - dbt snapshot - - https://github.com/fishtown-analytics/dbt/issues/1599#issuecomment-510528614 -*/ - - -{% snapshot orders_snapshot %} - - {{ - config -( - target_schema='dbt_bq_example', - unique_key='id', - - strategy='check', - check_cols=['status'], - ) - }} - --- Pro-Tip: Use sources in snapshots! -select * from {{ source('dbt_bq_example','raw_orders') }} - -{% endsnapshot %}