From 6f1d2601eb457e082c51394cdc59a7ec6f10e237 Mon Sep 17 00:00:00 2001 From: Reuben Frankel Date: Wed, 2 Aug 2023 11:16:33 +0100 Subject: [PATCH] Update lineage pipeline definition --- bundle/pipelines/Create dbt Model Datasets.yml | 8 -------- bundle/pipelines/Model lineage.yml | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 8 deletions(-) delete mode 100644 bundle/pipelines/Create dbt Model Datasets.yml create mode 100644 bundle/pipelines/Model lineage.yml diff --git a/bundle/pipelines/Create dbt Model Datasets.yml b/bundle/pipelines/Create dbt Model Datasets.yml deleted file mode 100644 index 65f180f..0000000 --- a/bundle/pipelines/Create dbt Model Datasets.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: pipelines/v0.1 -name: Create dbt Model Datasets -timeout: 0 -script: "meltano install transformer dbt\nmeltano install utility matatika\nmeltano invoke dbt docs generate" -data_components: -- dbt -- matatika - diff --git a/bundle/pipelines/Model lineage.yml b/bundle/pipelines/Model lineage.yml new file mode 100644 index 0000000..64f87d0 --- /dev/null +++ b/bundle/pipelines/Model lineage.yml @@ -0,0 +1,17 @@ +version: pipelines/v0.1 +name: Model lineage +data_components: +- dbt +- utilities/dbt-artifacts--matatika +- utilities/matatika--matatika +script: |- + meltano install transformer dbt + meltano install utilities dbt-artifacts matatika + + meltano invoke dbt deps + meltano invoke dbt docs generate + meltano invoke dbt-artifacts convert --format matatika + + for dataset in output/*.yml; do + meltano invoke matatika publish $dataset + done