From 7a1be6efc7145c42ead7f0fc87c8b87092f423e7 Mon Sep 17 00:00:00 2001 From: ErikO <49541561+EJOOSTEROP@users.noreply.github.com> Date: Sat, 26 Aug 2023 18:30:04 +0200 Subject: [PATCH] Dashboard update (#30) * Initial changes to incorporate elementary-data. * Initial * Elementary WIP * Exlude UA, GB, Serbia from dashboard/dataset. --- Mimodast.code-workspace | 8 ++++++++ meltano.yml | 8 +++++++- meltano_transform/dbt_project.yml | 7 ++++++- meltano_transform/models/overview.md | 3 +-- meltano_transform/packages.yml | 4 +++- .../profiles/duckdb/profiles.yml | 14 ++++++++++++++ standup/dashboards.zip | Bin 25375 -> 25443 bytes 7 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 Mimodast.code-workspace diff --git a/Mimodast.code-workspace b/Mimodast.code-workspace new file mode 100644 index 0000000..876a149 --- /dev/null +++ b/Mimodast.code-workspace @@ -0,0 +1,8 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": {} +} \ No newline at end of file diff --git a/meltano.yml b/meltano.yml index 966450f..1e526b6 100644 --- a/meltano.yml +++ b/meltano.yml @@ -194,7 +194,13 @@ plugins: variant: great-expectations pip_url: great_expectations[sqlalchemy] duckdb-engine==0.7.0 -# sqlalchemy==1.4.47 +# sqlalchemy==1.4.47 + - name: elementary + variant: elementary + pip_url: elementary-data[sqlalchemy]==0.8.0 git+https://github.com/potloc/elementary-ext.git + duckdb-engine==0.7.0 +#commands: +# dbt run --select elementary jobs: - name: usgs-to-duckdb-rpt tasks: diff --git a/meltano_transform/dbt_project.yml b/meltano_transform/dbt_project.yml index d9c18fb..11eeea4 100644 --- a/meltano_transform/dbt_project.yml +++ b/meltano_transform/dbt_project.yml @@ -33,4 +33,9 @@ models: usgs_rpt: +materialized: table +tags: - - usgs \ No newline at end of file + - usgs + elementary: + ## elementary models will be created in the schema '_elementary' + +schema: "elementary" + ## To disable elementary for dev, uncomment this: + # enabled: "{{ target.name in ['prod','analytics'] }}" \ No newline at end of file diff --git a/meltano_transform/models/overview.md b/meltano_transform/models/overview.md index 31c53e8..55e60b9 100644 --- a/meltano_transform/models/overview.md +++ b/meltano_transform/models/overview.md @@ -17,5 +17,4 @@ This **dbt project** defines the transformation and tests related to the two dat [GIEAPI-url]: https://agsi.gie.eu/ [GIEAccount-url]: https://agsi.gie.eu/account -{% enddocs %} - +{% enddocs %} \ No newline at end of file diff --git a/meltano_transform/packages.yml b/meltano_transform/packages.yml index d7c5292..2e58d04 100644 --- a/meltano_transform/packages.yml +++ b/meltano_transform/packages.yml @@ -1,3 +1,5 @@ packages: - package: dbt-labs/dbt_utils - version: 1.0.0 \ No newline at end of file + version: 1.0.0 + - package: elementary-data/elementary + version: 0.8.0 \ No newline at end of file diff --git a/meltano_transform/profiles/duckdb/profiles.yml b/meltano_transform/profiles/duckdb/profiles.yml index 156b435..5554f80 100644 --- a/meltano_transform/profiles/duckdb/profiles.yml +++ b/meltano_transform/profiles/duckdb/profiles.yml @@ -21,3 +21,17 @@ meltano: #path: "{{ env_var('DBT_DUCKDB_PATH') }}" path: "/project/data/prod/data.duckdb" threads: 4 +elementary: + outputs: + default: + type: duckdb + #path: "{{ env_var('DBT_DUCKDB_PATH') }}" + path: "/project/data/dev/data.duckdb" + database: data + threads: 1 + dev: + type: duckdb + #path: "{{ env_var('DBT_DUCKDB_PATH') }}" + path: "/project/data/dev/data.duckdb" + database: data + threads: 1 diff --git a/standup/dashboards.zip b/standup/dashboards.zip index b07795c5b85b2b6958db2ae8cace2a7b87581ad1..62295c33da7cfc6dc02e0fd401023517ec994885 100644 GIT binary patch delta 966 zcmbP#jPda?M(F@=W)=|!2vB5DTzzGkFOUPmdXw#)B)Ke%%t8#!49tzqC)(MISm=Ve zRtDx)#^!pK<_1P4MiVEBv3zge5IAwQ5|Zl4mT{^R|6F4OY(~n(^D17Gg6CE6_WEy^Gb>;74q^+6f*M^G}PUlwA6JJ)I%LXWKgiW z=HwdJg3Z%~rVArkvDrfHC?hhTU1K}5EW+RV$r*`7CB^!ld09XgWaj6^8*jd?y^Bo_ zo09loh>PM)Hg7iKVrS8c+*CCApe0JkytRBY`HQv6prbCbF_Maps)x=$iZ{P8(TLG2Wla zPHnP-lLWHplz0awUJZ27AMwsi=d?kh_(OZLW4!3(_5>-WrA8pd1i~GpS_TyCk{k>S zAZ!Os-5{!g@v|kmGu~RtP4-IEV#;trmjOk<>?E zPX`I`z^rLtWDG`E3yvPJKTd`stNb4dQfWmX%JeD|b8`Z`S=m5V@B(2KD+9yjSP%~Y DCw>?& delta 892 zcmaESjB)-kM(F@=W)=|!2vB6ud3eIm7svr&y~ziRB)LqCOhXKeObm@oC)(MInCODJ zRwjm4My7feMyAFlrV}TMu>}0g`af~C5|Zl4wQ;Hw|6H4FVWTqn8{>OMi^+Q15( zqsiuzPUcL=q6LDYo8?`__>jfygQX^)a1z>_9ejdG9@BlkiRpQnC8a5;@#gVnn}fn8 zva^7qYVz|~17uSaXS_UN!plK9DH88HX zM0bXvmE7dML@g#2Cv+KbB!F$dFk&Xly9!J$Pts!A8-y+kj3P$U m$q$nBnRbVQ