diff --git a/pipelines/dbt/pipeline.yaml b/pipelines/dbt/pipeline.yaml new file mode 100644 index 0000000..9fdec7e --- /dev/null +++ b/pipelines/dbt/pipeline.yaml @@ -0,0 +1,58 @@ +env: + - PYTHON_VERSION=latest + +steps: + - label: ":package: Install dependencies" + commands: + - echo "+++ :package: Installing dependencies" + - pip install -r requirements.txt + - dbt deps + key: "deps" + plugins: &plugins + - docker#v5.9.0: + image: "python3-slim-$PYTHON_VERSION" + + - label: ":dbt: sql lint" + depends_on: "deps" + key: "dbt-sql-lint" + commands: + - echo "+++ :dbt: Running dbt sql lint" + - pip install sqlfluff sqlfluff-templater-dbt + - sqlfluff lint --templater dbt + plugins: + <<: *plugins + + - label: ":dbt: dbt source freshness tests" + depends_on: "deps" + command: | + echo "+++ :dbt: Running dbt source freshness" + dbt source freshness --target prod + plugins: + <<: *plugins + + - label: ":dbt: dbt seed" + depends_on: "deps" + command: | + echo "+++ :dbt: Running dbt seed" + dbt seed --target prod + plugins: + <<: *plugins + + - wait + + - label: ":dbt: dbt run against production db" + command: | + echo "+++ :dbt: Running dbt job" + dbt run --target prod + plugins: + <<: *plugins + + - wait + + - label: ":dbt: dbt model tests" + depends_on: "deps" + command: | + echo "+++ :dbt: Running dbt model tests" + dbt test --target prod + plugins: + <<: *plugins \ No newline at end of file diff --git a/pipelines/dbt/readme.md b/pipelines/dbt/readme.md new file mode 100644 index 0000000..bfec87d --- /dev/null +++ b/pipelines/dbt/readme.md @@ -0,0 +1,37 @@ +--- +title: DBT +description: Lint, build and run DBT models +tags: ["CI", "Python", "DBT", "Data"] +author: Buildkite +--- + +# CI for dbt with Buildkite + +This template gives you a continuous integration pipeline that automates your dbt workflow of linting, testing and running your dbt models. + +At a glance: + +- For DBT projects to lint, seed, run and test in CI +- Uses Docker on a python image + +## How it works + +This template: + +1. Installs dependencies using pip and dbt deps on a python docker image, saving into a `plugins` variable +2. Lints SQL using [sqlfluff](https://sqlfluff.com/) and [sqlfluff-templater-dbt](https://pypi.org/project/sqlfluff-templater-dbt/) +3. Runs DBT source freshness tests +4. Seeds the DBT project +5. Runs DBT +6. Runs DBT model tests + +## Next steps + +After you select Use template, you’ll: + +1. Connect the Git repository with your DBT project. +2. Ensure the dbt commands match your dbt profiles. +3. Configure the compute—run locally, on-premises, or in the cloud. +4. Run the pipeline. + +You can then play around with the pipeline settings. For example, run the pipeline locally while you iterate on the definition or set a schedule to trigger a nightly build.