From 41fef839ce25d90609c385717e80692c99cc0c74 Mon Sep 17 00:00:00 2001 From: EJOOSTEROP <49541561+EJOOSTEROP@users.noreply.github.com> Date: Thu, 30 Mar 2023 20:20:09 -0400 Subject: [PATCH 1/9] dbt-duckdb:run instead of build. Build ran into a testing error. Test is currently not very meaningful hence suppress. --- dags/gie_dag.py | 4 +- meltano_transform/tests/stage_timing.sql | 2 +- standup/datasources.yml | 5 - superset/dashboards/20221125_035050.json | 307 ------------------ superset/dashboards/20230320_120551.json | 269 --------------- .../dashboard_export_20230324T211710.zip | Bin 10556 -> 0 bytes 6 files changed, 3 insertions(+), 584 deletions(-) delete mode 100644 standup/datasources.yml delete mode 100644 superset/dashboards/20221125_035050.json delete mode 100644 superset/dashboards/20230320_120551.json delete mode 100644 superset/dashboards/dashboard_export_20230324T211710.zip diff --git a/dags/gie_dag.py b/dags/gie_dag.py index 30e0725..b9628b7 100644 --- a/dags/gie_dag.py +++ b/dags/gie_dag.py @@ -56,7 +56,7 @@ def stage_gie_dag(my_date : str = '-'): run_this_2nd = BashOperator( task_id='Update_reporting_table', - bash_command = "cd " + PROJECT_ROOT + "; ENV_DATE_GIE={{ dag_run.conf.get('my_date', '-') if dag_run else '-' }} " + MELTANO_BIN + " invoke " + "dbt-duckdb:build --select tag:gie", + bash_command = "cd " + PROJECT_ROOT + "; ENV_DATE_GIE={{ dag_run.conf.get('my_date', '-') if dag_run else '-' }} " + MELTANO_BIN + " invoke " + "dbt-duckdb:run --select tag:gie", ) run_this_2nd @@ -96,7 +96,7 @@ def stage_gie_backfill_dag(): run_this_2nd = BashOperator( task_id='Update_reporting_table', - bash_command = "cd " + PROJECT_ROOT + "; " + MELTANO_BIN + " invoke " + "dbt-duckdb:build --select tag:gie", + bash_command = "cd " + PROJECT_ROOT + "; " + MELTANO_BIN + " invoke " + "dbt-duckdb:run --select tag:gie", ) run_this_2nd diff --git a/meltano_transform/tests/stage_timing.sql b/meltano_transform/tests/stage_timing.sql index f4481c2..5883c06 100644 --- a/meltano_transform/tests/stage_timing.sql +++ b/meltano_transform/tests/stage_timing.sql @@ -1,3 +1,3 @@ SELECT * FROM {{ ref('rpt_gie_storage')}} -WHERE _sdc_batched_at < _sdc_extracted_at \ No newline at end of file +WHERE _sdc_batched_at <= _sdc_extracted_at \ No newline at end of file diff --git a/standup/datasources.yml b/standup/datasources.yml deleted file mode 100644 index 7ce5f1b..0000000 --- a/standup/datasources.yml +++ /dev/null @@ -1,5 +0,0 @@ -databases: -- database_name: DuckDB - sqlalchemy_uri: duckdb:////project/data/dev/data.duckdb - cache_timeout: null - expose_in_sqllab: true \ No newline at end of file diff --git a/superset/dashboards/20221125_035050.json b/superset/dashboards/20221125_035050.json deleted file mode 100644 index 503db89..0000000 --- a/superset/dashboards/20221125_035050.json +++ /dev/null @@ -1,307 +0,0 @@ -{ - "dashboards": [ - { - "__Dashboard__": { - "css": "", - "dashboard_title": "Staging GEI - European Gas Storage", - "description": null, - "json_metadata": "{\"show_native_filters\": true, \"shared_label_colors\": {}, \"color_scheme\": \"\", \"refresh_frequency\": 0, \"expanded_slices\": {}, \"label_colors\": {}, \"timed_refresh_immune_slices\": [], \"default_filters\": \"{}\", \"chart_configuration\": {}, \"remote_id\": 1}", - "position_json": "{\"CHART-_0I8kiR-xr\":{\"children\":[],\"id\":\"CHART-_0I8kiR-xr\",\"meta\":{\"chartId\":1,\"height\":52,\"sliceName\":\"Fill Ratio\",\"sliceNameOverride\":\"Fill Ratio\",\"uuid\":\"563d16f8-808a-4509-9b51-e3bbcc5ee1c8\",\"width\":3},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-Zm9Q2PUZMz\"],\"type\":\"CHART\"},\"CHART-p3i4Bkt9CF\":{\"children\":[],\"id\":\"CHART-p3i4Bkt9CF\",\"meta\":{\"chartId\":3,\"height\":52,\"sliceName\":\"Injection and Withdrawal\",\"uuid\":\"ca9f9177-255c-4149-8727-985b44f7dfe7\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-Zm9Q2PUZMz\"],\"type\":\"CHART\"},\"CHART-pw59cYzQyD\":{\"children\":[],\"id\":\"CHART-pw59cYzQyD\",\"meta\":{\"chartId\":2,\"height\":52,\"sliceName\":\"Calendar View\",\"uuid\":\"e490cbe6-ba7c-45d8-afbb-ec7d7e9a04c3\",\"width\":3},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-Zm9Q2PUZMz\"],\"type\":\"CHART\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"HEADER-KRT7hPQf5s\",\"ROW-Zm9Q2PUZMz\"],\"id\":\"GRID_ID\",\"parents\":[\"ROOT_ID\"],\"type\":\"GRID\"},\"HEADER-KRT7hPQf5s\":{\"children\":[],\"id\":\"HEADER-KRT7hPQf5s\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\",\"headerSize\":\"MEDIUM_HEADER\",\"text\":\"Total Gas Stored as Percentage of Capacity\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"HEADER\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"Staging GEI - European Gas Storage\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-Zm9Q2PUZMz\":{\"children\":[\"CHART-_0I8kiR-xr\",\"CHART-pw59cYzQyD\",\"CHART-p3i4Bkt9CF\"],\"id\":\"ROW-Zm9Q2PUZMz\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"}}", - "slices": [ - { - "__Slice__": { - "cache_timeout": null, - "datasource_name": "main.STG_STORED", - "datasource_type": "table", - "id": 1, - "params": "{\"adhoc_filters\": [], \"color_picker\": {\"a\": 1, \"b\": 135, \"g\": 122, \"r\": 0}, \"datasource\": \"1__table\", \"extra_form_data\": {}, \"granularity_sqla\": \"REPORTING_DATE\", \"header_font_size\": 0.4, \"metric\": {\"aggregate\": \"MIN\", \"column\": {\"certification_details\": null, \"certified_by\": null, \"column_name\": \"FILL_RATIO\", \"description\": null, \"expression\": null, \"filterable\": true, \"groupby\": false, \"id\": 65, \"is_certified\": false, \"is_dttm\": false, \"python_date_format\": null, \"type\": \"NUMBER\", \"type_generic\": null, \"verbose_name\": null, \"warning_markdown\": null}, \"expressionType\": \"SIMPLE\", \"hasCustomLabel\": false, \"isNew\": false, \"label\": \"MIN(FILL_RATIO)\", \"optionName\": \"metric_v4b50ld4vuc_gev7u7z7ud7\", \"sqlExpression\": null}, \"rolling_type\": \"None\", \"show_timestamp\": true, \"show_trend_line\": true, \"slice_id\": 1, \"start_y_axis_at_zero\": false, \"subheader_font_size\": 0.15, \"time_format\": \"smart_date\", \"time_grain_sqla\": \"P1D\", \"time_range\": \"No filter\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"viz_type\": \"big_number\", \"y_axis_format\": \"SMART_NUMBER\", \"remote_id\": 1, \"datasource_name\": \"STG_STORED\", \"schema\": \"main\", \"database_name\": \"DuckDB\"}", - "query_context": "{\"datasource\":{\"id\":1,\"type\":\"table\"},\"force\":false,\"queries\":[{\"time_range\":\"No filter\",\"granularity\":\"REPORTING_DATE\",\"filters\":[],\"extras\":{\"time_grain_sqla\":\"P1D\",\"time_range_endpoints\":[\"inclusive\",\"exclusive\"],\"having\":\"\",\"having_druid\":[],\"where\":\"\"},\"applied_time_extras\":{},\"columns\":[],\"metrics\":[{\"expressionType\":\"SIMPLE\",\"column\":{\"id\":65,\"column_name\":\"FILL_RATIO\",\"verbose_name\":null,\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":false,\"is_dttm\":false,\"type\":\"NUMBER\",\"type_generic\":null,\"python_date_format\":null,\"is_certified\":false,\"certified_by\":null,\"certification_details\":null,\"warning_markdown\":null},\"aggregate\":\"MIN\",\"sqlExpression\":null,\"isNew\":false,\"hasCustomLabel\":false,\"label\":\"MIN(FILL_RATIO)\",\"optionName\":\"metric_v4b50ld4vuc_gev7u7z7ud7\"}],\"annotation_layers\":[],\"timeseries_limit\":0,\"order_desc\":true,\"url_params\":{},\"custom_params\":{},\"custom_form_data\":{},\"is_timeseries\":true,\"post_processing\":[{\"operation\":\"pivot\",\"options\":{\"index\":[\"__timestamp\"],\"columns\":[],\"aggregates\":{\"MIN(FILL_RATIO)\":{\"operator\":\"mean\"}},\"drop_missing_columns\":false,\"flatten_columns\":false,\"reset_index\":false}},null,null,{\"operation\":\"flatten\"}]}],\"form_data\":{\"viz_type\":\"big_number\",\"datasource\":\"1__table\",\"slice_id\":1,\"time_range_endpoints\":[\"inclusive\",\"exclusive\"],\"granularity_sqla\":\"REPORTING_DATE\",\"time_grain_sqla\":\"P1D\",\"time_range\":\"No filter\",\"metric\":{\"expressionType\":\"SIMPLE\",\"column\":{\"id\":65,\"column_name\":\"FILL_RATIO\",\"verbose_name\":null,\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":false,\"is_dttm\":false,\"type\":\"NUMBER\",\"type_generic\":null,\"python_date_format\":null,\"is_certified\":false,\"certified_by\":null,\"certification_details\":null,\"warning_markdown\":null},\"aggregate\":\"MIN\",\"sqlExpression\":null,\"isNew\":false,\"hasCustomLabel\":false,\"label\":\"MIN(FILL_RATIO)\",\"optionName\":\"metric_v4b50ld4vuc_gev7u7z7ud7\"},\"adhoc_filters\":[],\"show_timestamp\":true,\"show_trend_line\":true,\"start_y_axis_at_zero\":false,\"color_picker\":{\"a\":1,\"b\":135,\"g\":122,\"r\":0},\"header_font_size\":0.4,\"subheader_font_size\":0.15,\"y_axis_format\":\"SMART_NUMBER\",\"time_format\":\"smart_date\",\"rolling_type\":\"None\",\"extra_form_data\":{},\"force\":false,\"result_format\":\"json\",\"result_type\":\"full\"},\"result_format\":\"json\",\"result_type\":\"full\"}", - "slice_name": "Fill Ratio", - "viz_type": "big_number" - } - }, - { - "__Slice__": { - "cache_timeout": null, - "datasource_name": "main.STG_STORED", - "datasource_type": "table", - "id": 2, - "params": "{\"adhoc_filters\": [], \"cell_padding\": 3, \"cell_radius\": 6, \"cell_size\": 20, \"datasource\": \"1__table\", \"domain_granularity\": \"month\", \"extra_form_data\": {}, \"granularity_sqla\": \"REPORTING_DATE\", \"linear_color_scheme\": \"schemeRdBu\", \"metrics\": [{\"aggregate\": \"MIN\", \"column\": {\"certification_details\": null, \"certified_by\": null, \"column_name\": \"FILL_RATIO\", \"description\": null, \"expression\": null, \"filterable\": true, \"groupby\": false, \"id\": 65, \"is_certified\": false, \"is_dttm\": false, \"python_date_format\": null, \"type\": \"NUMBER\", \"type_generic\": null, \"verbose_name\": null, \"warning_markdown\": null}, \"expressionType\": \"SIMPLE\", \"hasCustomLabel\": false, \"isNew\": false, \"label\": \"MIN(FILL_RATIO)\", \"optionName\": \"metric_9m38fcu6hl7_7x1hendj3a6\", \"sqlExpression\": null}], \"show_legend\": true, \"show_metric_name\": false, \"show_values\": false, \"slice_id\": 2, \"steps\": 10, \"subdomain_granularity\": \"day\", \"time_range\": \"Last month\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"viz_type\": \"cal_heatmap\", \"x_axis_time_format\": \"smart_date\", \"y_axis_format\": \"SMART_NUMBER\", \"remote_id\": 2, \"datasource_name\": \"STG_STORED\", \"schema\": \"main\", \"database_name\": \"DuckDB\"}", - "query_context": "{\"datasource\":{\"id\":1,\"type\":\"table\"},\"force\":false,\"queries\":[{\"time_range\":\"Last month\",\"granularity\":\"REPORTING_DATE\",\"filters\":[],\"extras\":{\"time_range_endpoints\":[\"inclusive\",\"exclusive\"],\"having\":\"\",\"having_druid\":[],\"where\":\"\"},\"applied_time_extras\":{},\"columns\":[],\"metrics\":[{\"expressionType\":\"SIMPLE\",\"column\":{\"id\":65,\"column_name\":\"FILL_RATIO\",\"verbose_name\":null,\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":false,\"is_dttm\":false,\"type\":\"NUMBER\",\"type_generic\":null,\"python_date_format\":null,\"is_certified\":false,\"certified_by\":null,\"certification_details\":null,\"warning_markdown\":null},\"aggregate\":\"MIN\",\"sqlExpression\":null,\"isNew\":false,\"hasCustomLabel\":false,\"label\":\"MIN(FILL_RATIO)\",\"optionName\":\"metric_9m38fcu6hl7_7x1hendj3a6\"}],\"annotation_layers\":[],\"timeseries_limit\":0,\"order_desc\":true,\"url_params\":{},\"custom_params\":{},\"custom_form_data\":{}}],\"form_data\":{\"viz_type\":\"cal_heatmap\",\"datasource\":\"1__table\",\"slice_id\":2,\"time_range_endpoints\":[\"inclusive\",\"exclusive\"],\"granularity_sqla\":\"REPORTING_DATE\",\"time_range\":\"Last month\",\"domain_granularity\":\"month\",\"subdomain_granularity\":\"day\",\"metrics\":[{\"expressionType\":\"SIMPLE\",\"column\":{\"id\":65,\"column_name\":\"FILL_RATIO\",\"verbose_name\":null,\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":false,\"is_dttm\":false,\"type\":\"NUMBER\",\"type_generic\":null,\"python_date_format\":null,\"is_certified\":false,\"certified_by\":null,\"certification_details\":null,\"warning_markdown\":null},\"aggregate\":\"MIN\",\"sqlExpression\":null,\"isNew\":false,\"hasCustomLabel\":false,\"label\":\"MIN(FILL_RATIO)\",\"optionName\":\"metric_9m38fcu6hl7_7x1hendj3a6\"}],\"adhoc_filters\":[],\"linear_color_scheme\":\"schemeRdBu\",\"cell_size\":20,\"cell_padding\":3,\"cell_radius\":6,\"steps\":10,\"y_axis_format\":\"SMART_NUMBER\",\"x_axis_time_format\":\"smart_date\",\"show_legend\":true,\"show_values\":false,\"show_metric_name\":false,\"extra_form_data\":{},\"force\":false,\"result_format\":\"json\",\"result_type\":\"full\"},\"result_format\":\"json\",\"result_type\":\"full\"}", - "slice_name": "Calendar View", - "viz_type": "cal_heatmap" - } - }, - { - "__Slice__": { - "cache_timeout": null, - "datasource_name": "main.STG_STORED", - "datasource_type": "table", - "id": 3, - "params": "{\"adhoc_filters\": [], \"annotation_layers\": [], \"color_scheme\": \"d3Category10\", \"comparison_type\": \"values\", \"datasource\": \"1__table\", \"extra_form_data\": {}, \"forecastEnabled\": false, \"forecastInterval\": 0.8, \"forecastPeriods\": \"2\", \"forecastSeasonalityDaily\": false, \"forecastSeasonalityWeekly\": false, \"forecastSeasonalityYearly\": false, \"granularity_sqla\": \"REPORTING_DATE\", \"groupby\": [], \"legendOrientation\": \"top\", \"legendType\": \"scroll\", \"logAxis\": false, \"markerEnabled\": false, \"markerSize\": 6, \"metrics\": [{\"aggregate\": \"SUM\", \"column\": {\"certification_details\": null, \"certified_by\": null, \"column_name\": \"GAS_INJECTION\", \"description\": null, \"expression\": null, \"filterable\": true, \"groupby\": false, \"id\": 63, \"is_certified\": false, \"is_dttm\": false, \"python_date_format\": null, \"type\": \"NUMBER\", \"type_generic\": null, \"verbose_name\": null, \"warning_markdown\": null}, \"expressionType\": \"SIMPLE\", \"hasCustomLabel\": true, \"isNew\": false, \"label\": \"Injection\", \"optionName\": \"metric_kvv4pul1p8_zlhrk6ugdf\", \"sqlExpression\": null}, {\"aggregate\": \"SUM\", \"column\": {\"certification_details\": null, \"certified_by\": null, \"column_name\": \"GAS_WITHDRAWAL\", \"description\": null, \"expression\": null, \"filterable\": true, \"groupby\": false, \"id\": 64, \"is_certified\": false, \"is_dttm\": false, \"python_date_format\": null, \"type\": \"NUMBER\", \"type_generic\": null, \"verbose_name\": null, \"warning_markdown\": null}, \"expressionType\": \"SIMPLE\", \"hasCustomLabel\": true, \"isNew\": false, \"label\": \"Withdrawal\", \"optionName\": \"metric_rc8n3jgl3w_zr0b31evfme\", \"sqlExpression\": null}], \"minorSplitLine\": true, \"only_total\": true, \"opacity\": 0.4, \"order_desc\": true, \"rich_tooltip\": true, \"row_limit\": 10000, \"seriesType\": \"line\", \"show_legend\": true, \"show_value\": false, \"slice_id\": 3, \"stack\": false, \"time_grain_sqla\": \"P1D\", \"time_range\": \"No filter\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"tooltipTimeFormat\": \"smart_date\", \"truncateYAxis\": false, \"viz_type\": \"echarts_area\", \"xAxisLabelRotation\": 45, \"x_axis_time_format\": \"%d/%m/%Y\", \"x_axis_title_margin\": 15, \"y_axis_bounds\": [null, null], \"y_axis_format\": \".3s\", \"y_axis_title\": \"Volume\", \"y_axis_title_margin\": 15, \"y_axis_title_position\": \"Left\", \"zoomable\": false, \"remote_id\": 3, \"datasource_name\": \"STG_STORED\", \"schema\": \"main\", \"database_name\": \"DuckDB\"}", - "query_context": "{\"datasource\":{\"id\":1,\"type\":\"table\"},\"force\":false,\"queries\":[{\"time_range\":\"No filter\",\"granularity\":\"REPORTING_DATE\",\"filters\":[],\"extras\":{\"time_grain_sqla\":\"P1D\",\"time_range_endpoints\":[\"inclusive\",\"exclusive\"],\"having\":\"\",\"having_druid\":[],\"where\":\"\"},\"applied_time_extras\":{},\"columns\":[],\"metrics\":[{\"aggregate\":\"SUM\",\"column\":{\"certification_details\":null,\"certified_by\":null,\"column_name\":\"GAS_INJECTION\",\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":false,\"id\":63,\"is_certified\":false,\"is_dttm\":false,\"python_date_format\":null,\"type\":\"NUMBER\",\"type_generic\":null,\"verbose_name\":null,\"warning_markdown\":null},\"expressionType\":\"SIMPLE\",\"hasCustomLabel\":true,\"isNew\":false,\"label\":\"Injection\",\"optionName\":\"metric_kvv4pul1p8_zlhrk6ugdf\",\"sqlExpression\":null},{\"aggregate\":\"SUM\",\"column\":{\"certification_details\":null,\"certified_by\":null,\"column_name\":\"GAS_WITHDRAWAL\",\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":false,\"id\":64,\"is_certified\":false,\"is_dttm\":false,\"python_date_format\":null,\"type\":\"NUMBER\",\"type_generic\":null,\"verbose_name\":null,\"warning_markdown\":null},\"expressionType\":\"SIMPLE\",\"hasCustomLabel\":true,\"isNew\":false,\"label\":\"Withdrawal\",\"optionName\":\"metric_rc8n3jgl3w_zr0b31evfme\",\"sqlExpression\":null}],\"orderby\":[[{\"aggregate\":\"SUM\",\"column\":{\"certification_details\":null,\"certified_by\":null,\"column_name\":\"GAS_INJECTION\",\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":false,\"id\":63,\"is_certified\":false,\"is_dttm\":false,\"python_date_format\":null,\"type\":\"NUMBER\",\"type_generic\":null,\"verbose_name\":null,\"warning_markdown\":null},\"expressionType\":\"SIMPLE\",\"hasCustomLabel\":true,\"isNew\":false,\"label\":\"Injection\",\"optionName\":\"metric_kvv4pul1p8_zlhrk6ugdf\",\"sqlExpression\":null},false]],\"annotation_layers\":[],\"row_limit\":10000,\"series_columns\":[],\"timeseries_limit\":0,\"order_desc\":true,\"url_params\":{},\"custom_params\":{},\"custom_form_data\":{},\"is_timeseries\":true,\"time_offsets\":[],\"post_processing\":[{\"operation\":\"pivot\",\"options\":{\"index\":[\"__timestamp\"],\"columns\":[],\"aggregates\":{\"Injection\":{\"operator\":\"mean\"},\"Withdrawal\":{\"operator\":\"mean\"}},\"drop_missing_columns\":false,\"flatten_columns\":false,\"reset_index\":false}},null,null,null,null,{\"operation\":\"flatten\"},null,null]}],\"form_data\":{\"viz_type\":\"echarts_area\",\"datasource\":\"1__table\",\"slice_id\":3,\"time_range_endpoints\":[\"inclusive\",\"exclusive\"],\"granularity_sqla\":\"REPORTING_DATE\",\"time_grain_sqla\":\"P1D\",\"time_range\":\"No filter\",\"metrics\":[{\"aggregate\":\"SUM\",\"column\":{\"certification_details\":null,\"certified_by\":null,\"column_name\":\"GAS_INJECTION\",\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":false,\"id\":63,\"is_certified\":false,\"is_dttm\":false,\"python_date_format\":null,\"type\":\"NUMBER\",\"type_generic\":null,\"verbose_name\":null,\"warning_markdown\":null},\"expressionType\":\"SIMPLE\",\"hasCustomLabel\":true,\"isNew\":false,\"label\":\"Injection\",\"optionName\":\"metric_kvv4pul1p8_zlhrk6ugdf\",\"sqlExpression\":null},{\"aggregate\":\"SUM\",\"column\":{\"certification_details\":null,\"certified_by\":null,\"column_name\":\"GAS_WITHDRAWAL\",\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":false,\"id\":64,\"is_certified\":false,\"is_dttm\":false,\"python_date_format\":null,\"type\":\"NUMBER\",\"type_generic\":null,\"verbose_name\":null,\"warning_markdown\":null},\"expressionType\":\"SIMPLE\",\"hasCustomLabel\":true,\"isNew\":false,\"label\":\"Withdrawal\",\"optionName\":\"metric_rc8n3jgl3w_zr0b31evfme\",\"sqlExpression\":null}],\"groupby\":[],\"adhoc_filters\":[],\"order_desc\":true,\"row_limit\":10000,\"comparison_type\":\"values\",\"annotation_layers\":[],\"forecastEnabled\":false,\"forecastPeriods\":\"2\",\"forecastInterval\":0.8,\"forecastSeasonalityYearly\":false,\"forecastSeasonalityWeekly\":false,\"forecastSeasonalityDaily\":false,\"x_axis_title_margin\":15,\"y_axis_title\":\"Volume\",\"y_axis_title_margin\":15,\"y_axis_title_position\":\"Left\",\"color_scheme\":\"d3Category10\",\"seriesType\":\"line\",\"opacity\":0.4,\"show_value\":false,\"stack\":false,\"only_total\":true,\"markerEnabled\":false,\"markerSize\":6,\"zoomable\":false,\"show_legend\":true,\"legendType\":\"scroll\",\"legendOrientation\":\"top\",\"x_axis_time_format\":\"%d/%m/%Y\",\"xAxisLabelRotation\":45,\"rich_tooltip\":true,\"tooltipTimeFormat\":\"smart_date\",\"y_axis_format\":\".3s\",\"logAxis\":false,\"minorSplitLine\":true,\"truncateYAxis\":false,\"y_axis_bounds\":[null,null],\"extra_form_data\":{},\"force\":false,\"result_format\":\"json\",\"result_type\":\"full\"},\"result_format\":\"json\",\"result_type\":\"full\"}", - "slice_name": "Injection and Withdrawal", - "viz_type": "echarts_area" - } - } - ], - "slug": null - } - } - ], - "datasources": [ - { - "__SqlaTable__": { - "cache_timeout": null, - "columns": [ - { - "__TableColumn__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-11-25T00:32:13" - }, - "column_name": "REPORTING_DATE", - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-11-24T01:31:17" - }, - "description": null, - "expression": "", - "extra": "{}", - "filterable": true, - "groupby": true, - "id": 31, - "is_active": null, - "is_dttm": true, - "python_date_format": null, - "table_id": 1, - "type": "DATE", - "uuid": "948348e2-88ce-4f42-8628-8cf0a73b1e0a", - "verbose_name": null - } - }, - { - "__TableColumn__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-11-25T00:32:13" - }, - "column_name": "day(_sdc_batched_at)", - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-11-24T01:31:17" - }, - "description": null, - "expression": "", - "extra": "{}", - "filterable": true, - "groupby": true, - "id": 32, - "is_active": null, - "is_dttm": false, - "python_date_format": null, - "table_id": 1, - "type": "NUMBER", - "uuid": "fcb97929-cc87-4f64-ae64-67ca355a5ad5", - "verbose_name": null - } - }, - { - "__TableColumn__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-11-25T00:35:39" - }, - "column_name": "count_star()", - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-11-24T01:31:17" - }, - "description": null, - "expression": "", - "extra": "{}", - "filterable": true, - "groupby": false, - "id": 33, - "is_active": null, - "is_dttm": false, - "python_date_format": null, - "table_id": 1, - "type": "NUMBER", - "uuid": "7feda35d-e44a-4d41-8791-e7500e25ae48", - "verbose_name": null - } - }, - { - "__TableColumn__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-11-25T00:32:13" - }, - "column_name": "GAS_STORED", - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-11-24T01:31:17" - }, - "description": null, - "expression": "", - "extra": "{}", - "filterable": true, - "groupby": false, - "id": 34, - "is_active": null, - "is_dttm": false, - "python_date_format": null, - "table_id": 1, - "type": "NUMBER", - "uuid": "299afe25-0400-4bd4-ae82-09a3f80aced6", - "verbose_name": null - } - }, - { - "__TableColumn__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-11-25T00:32:13" - }, - "column_name": "CAPACITY", - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-11-24T01:31:17" - }, - "description": null, - "expression": "", - "extra": "{}", - "filterable": true, - "groupby": false, - "id": 35, - "is_active": null, - "is_dttm": false, - "python_date_format": null, - "table_id": 1, - "type": "NUMBER", - "uuid": "3a79b399-262c-4397-8aa9-859029e0a249", - "verbose_name": null - } - }, - { - "__TableColumn__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-11-25T00:35:39" - }, - "column_name": "GAS_INJECTION", - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-11-25T00:35:39" - }, - "description": null, - "expression": null, - "extra": "{}", - "filterable": true, - "groupby": false, - "id": 63, - "is_active": true, - "is_dttm": false, - "python_date_format": null, - "table_id": 1, - "type": "NUMBER", - "uuid": "66dc16a0-91ca-4bf5-9685-65295e9ec094", - "verbose_name": null - } - }, - { - "__TableColumn__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-11-25T00:35:39" - }, - "column_name": "GAS_WITHDRAWAL", - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-11-25T00:35:39" - }, - "description": null, - "expression": null, - "extra": "{}", - "filterable": true, - "groupby": false, - "id": 64, - "is_active": true, - "is_dttm": false, - "python_date_format": null, - "table_id": 1, - "type": "NUMBER", - "uuid": "144e188d-ab7d-44ae-93d4-bde4115e2b0d", - "verbose_name": null - } - }, - { - "__TableColumn__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-11-25T00:35:39" - }, - "column_name": "FILL_RATIO", - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-11-25T00:35:39" - }, - "description": null, - "expression": null, - "extra": "{}", - "filterable": true, - "groupby": false, - "id": 65, - "is_active": true, - "is_dttm": false, - "python_date_format": null, - "table_id": 1, - "type": "NUMBER", - "uuid": "4abead9d-bda5-46a7-bd7c-617f7b0de11b", - "verbose_name": null - } - } - ], - "database_id": 1, - "default_endpoint": null, - "description": null, - "extra": null, - "fetch_values_predicate": null, - "filter_select_enabled": false, - "main_dttm_col": "REPORTING_DATE", - "metrics": [ - { - "__SqlMetric__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-11-25T00:32:13" - }, - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-11-24T01:06:17" - }, - "d3format": null, - "description": null, - "expression": "COUNT(*)", - "extra": "{\"warning_markdown\":\"\"}", - "id": 1, - "metric_name": "count", - "metric_type": null, - "table_id": 1, - "uuid": "70980057-3b66-4e6f-8894-453486d00207", - "verbose_name": "COUNT(*)", - "warning_text": null - } - } - ], - "offset": 0, - "params": "{\"remote_id\": 1, \"database_name\": \"DuckDB\"}", - "schema": "main", - "sql": "-- Note: Unless you save your query, these tabs will NOT persist if you clear your cookies or change browsers.\n\nSELECT \n gasdaystart::DATE as REPORTING_DATE, \n day(_sdc_batched_at), \n count(*) , \n sum(TRY_CAST(injection as DOUBLE )) as GAS_INJECTION,\n sum(TRY_CAST(withdrawal as DOUBLE )) as GAS_WITHDRAWAL,\n sum(TRY_CAST(gasinstorage as DOUBLE )) as GAS_STORED,\n sum(TRY_CAST(workinggasvolume as DOUBLE )) as CAPACITY, \n GAS_STORED/CAPACITY as FILL_RATIO\nfrom stg_gie_storage \ngroup by gasdaystart, day(_sdc_batched_at)\norder by gasdaystart desc;", - "table_name": "STG_STORED", - "template_params": null - } - } - ] -} \ No newline at end of file diff --git a/superset/dashboards/20230320_120551.json b/superset/dashboards/20230320_120551.json deleted file mode 100644 index e426eba..0000000 --- a/superset/dashboards/20230320_120551.json +++ /dev/null @@ -1,269 +0,0 @@ -{ - "dashboards": [ - { - "__Dashboard__": { - "css": "", - "dashboard_title": "Stage - Gas Storage", - "description": null, - "json_metadata": "{\"show_native_filters\": true, \"shared_label_colors\": {}, \"color_scheme\": \"\", \"refresh_frequency\": 0, \"expanded_slices\": {}, \"label_colors\": {}, \"timed_refresh_immune_slices\": [], \"default_filters\": \"{}\", \"chart_configuration\": {}, \"remote_id\": 1}", - "position_json": "{\"CHART-QC3WsFVbOJ\":{\"children\":[],\"id\":\"CHART-QC3WsFVbOJ\",\"meta\":{\"chartId\":2,\"height\":50,\"sliceName\":\"Injection and Withdrawal\",\"uuid\":\"ff3cad73-fd29-4ad8-a610-3f6bb3a6168a\",\"width\":8},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-iIeeQGGSXo\"],\"type\":\"CHART\"},\"CHART-rYfX7seBa_\":{\"children\":[],\"id\":\"CHART-rYfX7seBa_\",\"meta\":{\"chartId\":1,\"height\":50,\"sliceName\":\"Fill Ratio\",\"uuid\":\"b26934ad-2ab6-4a14-ad8d-de6a8aa3ffa7\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-iIeeQGGSXo\"],\"type\":\"CHART\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-iIeeQGGSXo\"],\"id\":\"GRID_ID\",\"parents\":[\"ROOT_ID\"],\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"Stage - Gas Storage\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-iIeeQGGSXo\":{\"children\":[\"CHART-rYfX7seBa_\",\"CHART-QC3WsFVbOJ\"],\"id\":\"ROW-iIeeQGGSXo\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"}}", - "slices": [ - { - "__Slice__": { - "cache_timeout": null, - "datasource_name": "main.STG_STORAGE", - "datasource_type": "table", - "id": 1, - "params": "{\"adhoc_filters\": [], \"color_picker\": {\"a\": 1, \"b\": 135, \"g\": 122, \"r\": 0}, \"compare_lag\": 7, \"compare_suffix\": \" compared to last week\", \"datasource\": \"1__table\", \"extra_form_data\": {}, \"force_timestamp_formatting\": false, \"granularity_sqla\": \"RECORD_DATE\", \"header_font_size\": 0.4, \"metric\": {\"aggregate\": \"MIN\", \"column\": {\"certification_details\": null, \"certified_by\": null, \"column_name\": \"FILL_RATIO\", \"description\": null, \"expression\": null, \"filterable\": true, \"groupby\": true, \"id\": 5, \"is_certified\": false, \"is_dttm\": false, \"python_date_format\": null, \"type\": \"NUMBER\", \"type_generic\": null, \"verbose_name\": null, \"warning_markdown\": null}, \"expressionType\": \"SIMPLE\", \"hasCustomLabel\": false, \"isNew\": false, \"label\": \"MIN(FILL_RATIO)\", \"optionName\": \"metric_2n3me2g8jw1_emmu08tdw65\", \"sqlExpression\": null}, \"rolling_type\": \"None\", \"show_timestamp\": true, \"show_trend_line\": true, \"start_y_axis_at_zero\": false, \"subheader_font_size\": 0.15, \"time_format\": \"smart_date\", \"time_grain_sqla\": \"P1D\", \"time_range\": \"No filter\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"viz_type\": \"big_number\", \"y_axis_format\": \".2%\", \"remote_id\": 1, \"datasource_name\": \"STG_STORAGE\", \"schema\": \"main\", \"database_name\": \"DuckDB\"}", - "query_context": "{\"datasource\":{\"id\":1,\"type\":\"table\"},\"force\":false,\"queries\":[{\"time_range\":\"No filter\",\"granularity\":\"RECORD_DATE\",\"filters\":[],\"extras\":{\"time_grain_sqla\":\"P1D\",\"time_range_endpoints\":[\"inclusive\",\"exclusive\"],\"having\":\"\",\"having_druid\":[],\"where\":\"\"},\"applied_time_extras\":{},\"columns\":[],\"metrics\":[{\"expressionType\":\"SIMPLE\",\"column\":{\"id\":5,\"column_name\":\"FILL_RATIO\",\"verbose_name\":null,\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":true,\"is_dttm\":false,\"type\":\"NUMBER\",\"type_generic\":null,\"python_date_format\":null,\"is_certified\":false,\"certified_by\":null,\"certification_details\":null,\"warning_markdown\":null},\"aggregate\":\"MIN\",\"sqlExpression\":null,\"isNew\":false,\"hasCustomLabel\":false,\"label\":\"MIN(FILL_RATIO)\",\"optionName\":\"metric_2n3me2g8jw1_emmu08tdw65\"}],\"annotation_layers\":[],\"timeseries_limit\":0,\"order_desc\":true,\"url_params\":{},\"custom_params\":{},\"custom_form_data\":{},\"is_timeseries\":true,\"post_processing\":[{\"operation\":\"pivot\",\"options\":{\"index\":[\"__timestamp\"],\"columns\":[],\"aggregates\":{\"MIN(FILL_RATIO)\":{\"operator\":\"mean\"}},\"drop_missing_columns\":false,\"flatten_columns\":false,\"reset_index\":false}},null,null,{\"operation\":\"flatten\"}]}],\"form_data\":{\"viz_type\":\"big_number\",\"datasource\":\"1__table\",\"time_range_endpoints\":[\"inclusive\",\"exclusive\"],\"granularity_sqla\":\"RECORD_DATE\",\"time_grain_sqla\":\"P1D\",\"time_range\":\"No filter\",\"metric\":{\"expressionType\":\"SIMPLE\",\"column\":{\"id\":5,\"column_name\":\"FILL_RATIO\",\"verbose_name\":null,\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":true,\"is_dttm\":false,\"type\":\"NUMBER\",\"type_generic\":null,\"python_date_format\":null,\"is_certified\":false,\"certified_by\":null,\"certification_details\":null,\"warning_markdown\":null},\"aggregate\":\"MIN\",\"sqlExpression\":null,\"isNew\":false,\"hasCustomLabel\":false,\"label\":\"MIN(FILL_RATIO)\",\"optionName\":\"metric_2n3me2g8jw1_emmu08tdw65\"},\"adhoc_filters\":[],\"compare_lag\":7,\"compare_suffix\":\" compared to last week\",\"show_timestamp\":true,\"show_trend_line\":true,\"start_y_axis_at_zero\":false,\"color_picker\":{\"r\":0,\"g\":122,\"b\":135,\"a\":1},\"header_font_size\":0.4,\"subheader_font_size\":0.15,\"y_axis_format\":\".2%\",\"time_format\":\"smart_date\",\"force_timestamp_formatting\":false,\"rolling_type\":\"None\",\"extra_form_data\":{},\"force\":false,\"result_format\":\"json\",\"result_type\":\"full\"},\"result_format\":\"json\",\"result_type\":\"full\"}", - "slice_name": "Fill Ratio", - "viz_type": "big_number" - } - }, - { - "__Slice__": { - "cache_timeout": null, - "datasource_name": "main.STG_STORAGE", - "datasource_type": "table", - "id": 2, - "params": "{\"adhoc_filters\": [], \"annotation_layers\": [], \"color_scheme\": \"echarts4Colors\", \"comparison_type\": \"values\", \"datasource\": \"1__table\", \"extra_form_data\": {}, \"forecastInterval\": 0.8, \"forecastPeriods\": 10, \"granularity_sqla\": \"RECORD_DATE\", \"groupby\": [], \"legendOrientation\": \"top\", \"legendType\": \"scroll\", \"markerSize\": 6, \"metrics\": [{\"aggregate\": \"MIN\", \"column\": {\"certification_details\": null, \"certified_by\": null, \"column_name\": \"GAS_INJECTION\", \"description\": null, \"expression\": null, \"filterable\": true, \"groupby\": true, \"id\": 6, \"is_certified\": false, \"is_dttm\": false, \"python_date_format\": null, \"type\": \"NUMBER\", \"type_generic\": null, \"verbose_name\": null, \"warning_markdown\": null}, \"expressionType\": \"SIMPLE\", \"hasCustomLabel\": true, \"isNew\": false, \"label\": \"Injection\", \"optionName\": \"metric_zgzkbap4cgc_9g1ov62gdt5\", \"sqlExpression\": null}, {\"aggregate\": \"MAX\", \"column\": {\"certification_details\": null, \"certified_by\": null, \"column_name\": \"GAS_WITHDRAWAL\", \"description\": null, \"expression\": null, \"filterable\": true, \"groupby\": true, \"id\": 7, \"is_certified\": false, \"is_dttm\": false, \"python_date_format\": null, \"type\": \"NUMBER\", \"type_generic\": null, \"verbose_name\": null, \"warning_markdown\": null}, \"expressionType\": \"SIMPLE\", \"hasCustomLabel\": true, \"isNew\": false, \"label\": \"Withdrawal\", \"optionName\": \"metric_jraiqofdn6_yrjg0joxl5d\", \"sqlExpression\": null}], \"only_total\": true, \"opacity\": 0.2, \"order_desc\": true, \"rich_tooltip\": true, \"row_limit\": 10000, \"seriesType\": \"line\", \"show_legend\": true, \"slice_id\": 2, \"time_grain_sqla\": \"P1D\", \"time_range\": \"No filter\", \"time_range_endpoints\": [\"inclusive\", \"exclusive\"], \"tooltipTimeFormat\": \"smart_date\", \"viz_type\": \"echarts_area\", \"x_axis_time_format\": \"smart_date\", \"x_axis_title_margin\": 15, \"y_axis_bounds\": [null, null], \"y_axis_format\": \"SMART_NUMBER\", \"y_axis_title_margin\": 15, \"y_axis_title_position\": \"Left\", \"remote_id\": 2, \"datasource_name\": \"STG_STORAGE\", \"schema\": \"main\", \"database_name\": \"DuckDB\"}", - "query_context": "{\"datasource\":{\"id\":1,\"type\":\"table\"},\"force\":false,\"queries\":[{\"time_range\":\"No filter\",\"granularity\":\"RECORD_DATE\",\"filters\":[],\"extras\":{\"time_grain_sqla\":\"P1D\",\"time_range_endpoints\":[\"inclusive\",\"exclusive\"],\"having\":\"\",\"having_druid\":[],\"where\":\"\"},\"applied_time_extras\":{},\"columns\":[],\"metrics\":[{\"expressionType\":\"SIMPLE\",\"column\":{\"certification_details\":null,\"certified_by\":null,\"column_name\":\"GAS_INJECTION\",\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":true,\"id\":6,\"is_certified\":false,\"is_dttm\":false,\"python_date_format\":null,\"type\":\"NUMBER\",\"type_generic\":null,\"verbose_name\":null,\"warning_markdown\":null},\"aggregate\":\"MIN\",\"sqlExpression\":null,\"isNew\":false,\"hasCustomLabel\":true,\"label\":\"Injection\",\"optionName\":\"metric_zgzkbap4cgc_9g1ov62gdt5\"},{\"expressionType\":\"SIMPLE\",\"column\":{\"certification_details\":null,\"certified_by\":null,\"column_name\":\"GAS_WITHDRAWAL\",\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":true,\"id\":7,\"is_certified\":false,\"is_dttm\":false,\"python_date_format\":null,\"type\":\"NUMBER\",\"type_generic\":null,\"verbose_name\":null,\"warning_markdown\":null},\"aggregate\":\"MAX\",\"sqlExpression\":null,\"isNew\":false,\"hasCustomLabel\":true,\"label\":\"Withdrawal\",\"optionName\":\"metric_jraiqofdn6_yrjg0joxl5d\"}],\"orderby\":[[{\"expressionType\":\"SIMPLE\",\"column\":{\"certification_details\":null,\"certified_by\":null,\"column_name\":\"GAS_INJECTION\",\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":true,\"id\":6,\"is_certified\":false,\"is_dttm\":false,\"python_date_format\":null,\"type\":\"NUMBER\",\"type_generic\":null,\"verbose_name\":null,\"warning_markdown\":null},\"aggregate\":\"MIN\",\"sqlExpression\":null,\"isNew\":false,\"hasCustomLabel\":true,\"label\":\"Injection\",\"optionName\":\"metric_zgzkbap4cgc_9g1ov62gdt5\"},false]],\"annotation_layers\":[],\"row_limit\":10000,\"series_columns\":[],\"timeseries_limit\":0,\"order_desc\":true,\"url_params\":{},\"custom_params\":{},\"custom_form_data\":{},\"is_timeseries\":true,\"time_offsets\":[],\"post_processing\":[{\"operation\":\"pivot\",\"options\":{\"index\":[\"__timestamp\"],\"columns\":[],\"aggregates\":{\"Injection\":{\"operator\":\"mean\"},\"Withdrawal\":{\"operator\":\"mean\"}},\"drop_missing_columns\":false,\"flatten_columns\":false,\"reset_index\":false}},null,null,null,null,{\"operation\":\"flatten\"},null,null]}],\"form_data\":{\"viz_type\":\"echarts_area\",\"datasource\":\"1__table\",\"slice_id\":2,\"time_range_endpoints\":[\"inclusive\",\"exclusive\"],\"granularity_sqla\":\"RECORD_DATE\",\"time_grain_sqla\":\"P1D\",\"time_range\":\"No filter\",\"metrics\":[{\"expressionType\":\"SIMPLE\",\"column\":{\"certification_details\":null,\"certified_by\":null,\"column_name\":\"GAS_INJECTION\",\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":true,\"id\":6,\"is_certified\":false,\"is_dttm\":false,\"python_date_format\":null,\"type\":\"NUMBER\",\"type_generic\":null,\"verbose_name\":null,\"warning_markdown\":null},\"aggregate\":\"MIN\",\"sqlExpression\":null,\"isNew\":false,\"hasCustomLabel\":true,\"label\":\"Injection\",\"optionName\":\"metric_zgzkbap4cgc_9g1ov62gdt5\"},{\"expressionType\":\"SIMPLE\",\"column\":{\"certification_details\":null,\"certified_by\":null,\"column_name\":\"GAS_WITHDRAWAL\",\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":true,\"id\":7,\"is_certified\":false,\"is_dttm\":false,\"python_date_format\":null,\"type\":\"NUMBER\",\"type_generic\":null,\"verbose_name\":null,\"warning_markdown\":null},\"aggregate\":\"MAX\",\"sqlExpression\":null,\"isNew\":false,\"hasCustomLabel\":true,\"label\":\"Withdrawal\",\"optionName\":\"metric_jraiqofdn6_yrjg0joxl5d\"}],\"groupby\":[],\"adhoc_filters\":[],\"order_desc\":true,\"row_limit\":10000,\"comparison_type\":\"values\",\"annotation_layers\":[],\"forecastPeriods\":10,\"forecastInterval\":0.8,\"x_axis_title_margin\":15,\"y_axis_title_margin\":15,\"y_axis_title_position\":\"Left\",\"color_scheme\":\"echarts4Colors\",\"seriesType\":\"line\",\"opacity\":0.2,\"only_total\":true,\"markerSize\":6,\"show_legend\":true,\"legendType\":\"scroll\",\"legendOrientation\":\"top\",\"x_axis_time_format\":\"smart_date\",\"rich_tooltip\":true,\"tooltipTimeFormat\":\"smart_date\",\"y_axis_format\":\"SMART_NUMBER\",\"y_axis_bounds\":[null,null],\"extra_form_data\":{},\"force\":false,\"result_format\":\"json\",\"result_type\":\"full\"},\"result_format\":\"json\",\"result_type\":\"full\"}", - "slice_name": "Injection and Withdrawal", - "viz_type": "echarts_area" - } - } - ], - "slug": null - } - } - ], - "datasources": [ - { - "__SqlaTable__": { - "cache_timeout": null, - "columns": [ - { - "__TableColumn__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-12-06T20:56:03" - }, - "column_name": "day(_sdc_batched_at)", - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-12-06T20:56:03" - }, - "description": null, - "expression": null, - "extra": null, - "filterable": true, - "groupby": true, - "id": 1, - "is_active": true, - "is_dttm": false, - "python_date_format": null, - "table_id": 1, - "type": "NUMBER", - "uuid": "f2caded3-79cb-444a-8b02-509d2079fef9", - "verbose_name": null - } - }, - { - "__TableColumn__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-12-06T20:56:03" - }, - "column_name": "RECORD_DATE", - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-12-06T20:56:03" - }, - "description": null, - "expression": null, - "extra": null, - "filterable": true, - "groupby": true, - "id": 2, - "is_active": true, - "is_dttm": true, - "python_date_format": null, - "table_id": 1, - "type": "DATE", - "uuid": "b97fb7c5-8d4d-4df6-8126-8ed3268d16a2", - "verbose_name": null - } - }, - { - "__TableColumn__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-12-06T20:56:03" - }, - "column_name": "GAS_IN_STORAGE", - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-12-06T20:56:03" - }, - "description": null, - "expression": null, - "extra": null, - "filterable": true, - "groupby": true, - "id": 3, - "is_active": true, - "is_dttm": false, - "python_date_format": null, - "table_id": 1, - "type": "NUMBER", - "uuid": "9c311544-1f1d-49f7-a111-7455529bba1f", - "verbose_name": null - } - }, - { - "__TableColumn__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-12-06T20:56:03" - }, - "column_name": "GAS_STORAGE_CAPACITY", - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-12-06T20:56:03" - }, - "description": null, - "expression": null, - "extra": null, - "filterable": true, - "groupby": true, - "id": 4, - "is_active": true, - "is_dttm": false, - "python_date_format": null, - "table_id": 1, - "type": "NUMBER", - "uuid": "0b61b24d-f2c5-4644-8eb8-1c180a97fb04", - "verbose_name": null - } - }, - { - "__TableColumn__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-12-06T20:56:03" - }, - "column_name": "FILL_RATIO", - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-12-06T20:56:03" - }, - "description": null, - "expression": null, - "extra": null, - "filterable": true, - "groupby": true, - "id": 5, - "is_active": true, - "is_dttm": false, - "python_date_format": null, - "table_id": 1, - "type": "NUMBER", - "uuid": "5ee19b4f-b3a7-41da-8e41-51ba330af442", - "verbose_name": null - } - }, - { - "__TableColumn__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-12-06T20:56:03" - }, - "column_name": "GAS_INJECTION", - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-12-06T20:56:03" - }, - "description": null, - "expression": null, - "extra": null, - "filterable": true, - "groupby": true, - "id": 6, - "is_active": true, - "is_dttm": false, - "python_date_format": null, - "table_id": 1, - "type": "NUMBER", - "uuid": "4576296d-9e63-4d9f-8b6f-10c4c95cc018", - "verbose_name": null - } - }, - { - "__TableColumn__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-12-06T20:56:03" - }, - "column_name": "GAS_WITHDRAWAL", - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-12-06T20:56:03" - }, - "description": null, - "expression": null, - "extra": null, - "filterable": true, - "groupby": true, - "id": 7, - "is_active": true, - "is_dttm": false, - "python_date_format": null, - "table_id": 1, - "type": "NUMBER", - "uuid": "67cebf29-1984-4072-9af6-0c676f71ede9", - "verbose_name": null - } - } - ], - "database_id": 1, - "default_endpoint": null, - "description": null, - "extra": null, - "fetch_values_predicate": null, - "filter_select_enabled": false, - "main_dttm_col": null, - "metrics": [ - { - "__SqlMetric__": { - "changed_by_fk": 1, - "changed_on": { - "__datetime__": "2022-12-06T20:56:03" - }, - "created_by_fk": 1, - "created_on": { - "__datetime__": "2022-12-06T20:56:03" - }, - "d3format": null, - "description": null, - "expression": "count(*)", - "extra": null, - "id": 1, - "metric_name": "count", - "metric_type": null, - "table_id": 1, - "uuid": "0d2daff5-3676-480f-9295-94046ba48b56", - "verbose_name": null, - "warning_text": null - } - } - ], - "offset": 0, - "params": "{\"remote_id\": 1, \"database_name\": \"DuckDB-dev\"}", - "schema": "main", - "sql": "-- Note: Unless you save your query, these tabs will NOT persist if you clear your cookies or change browsers.\n\nSELECT \n DAY(_sdc_batched_at),\n gasdaystart::DATE RECORD_DATE,\n --sum(gasinstorage::DOUBLE ) as GAS_STORED,\n sum((case when gasinstorage='-' then 0 else gasinstorage end)::DOUBLE ) GAS_IN_STORAGE,\n sum((case when workinggasvolume ='-' then 0 else workinggasvolume end)::DOUBLE ) GAS_STORAGE_CAPACITY,\n GAS_IN_STORAGE/GAS_STORAGE_CAPACITY FILL_RATIO,\n sum((case when injection ='-' then 0 else injection end)::DOUBLE ) GAS_INJECTION,\n sum((case when withdrawal ='-' then 0 else withdrawal end)::DOUBLE ) GAS_WITHDRAWAL, \n --sum(injection::DOUBLE ) as GAS_INJECTED\nFROM \n stg_gie_storage \ngroup BY\n DAY(_sdc_batched_at), \n gasdaystart::DATE \norder by 2 desc", - "table_name": "STG_STORAGE", - "template_params": null - } - } - ] -} \ No newline at end of file diff --git a/superset/dashboards/dashboard_export_20230324T211710.zip b/superset/dashboards/dashboard_export_20230324T211710.zip deleted file mode 100644 index 54682f49da09798a95cad83a99019d30567819d4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10556 zcmd^FOK%)m74A&PAbG(8Wh9n2BZ6muvb*}>cK6PYPGTi~(C$n$Aar%B>Q;Bf)m80! z*zQCljaaea2Y|!^i7nzEuw=!*V1;JEh9AIp?!DEIa>r>4G@!Vnid^?`&OPr_=iHO$ z_rLS7qJNJn-@N(PpAPXt&jub(q(dhbiO1#DI8HKNtJdn(daco~*>=mWZjNLoJdufw zi5P_!GD(9t>M(nwx=~%rCS%!Q`vr)#EEvf&6Qgm5trHxpTDKIWj@|0m*wJjX>f1l5 zRy)<|`r66D!9Mxs!C(LIpi-fyyUM|)Cr>xKc@mGMi1<^Ha*IDb=yF?~ce;jWK^Eey z#Y)DolvxZ**F2fJNifb7VNo82Yi^o?g7vjD%m*E&hVne{I?Qj>x9h%aTFtg6twvk4 zET`o(E$M7|Ro`iEISsKkj?+K^TVrhR*-o!-Ig@zw!hhA1R~?Ko<_?3os6 zYHI3c9z6-dko5#)Wfp6$2)pjrn$4EgZZ;eUxlyyW8&1QrMWboAovQGKZ#EdNG}P;V zlPmi>r_XkecY6E$i*E1q;P{9??|#`~7etm}aEVXq2qcSZ{3hj^7s?Ii)S{uq0QaoB zv)}DOlo}I(nGPvz^Ni2r6*O^~d6B|vxOz9X8iiR-!TeDP=e%}lI5eW5%#*qt*ixgi zt}C6QM`N1@dW}p+xMzbT&LbRbcW3YU)86sRqkZ1*?Hrw+?DV=vbTGzlVDX}X6a>9m zEBq|ckq+Za9ziiLB=>_bgV*h_EXk#sgYkJh6pjqJ8^dZa_x3Bys285PLrD$*?u)TR z`iV@39RHu^GIA#!Mm7iwf=d7t)8sPB$svUacI4I-B*_kMy zs02T6^otk48?L1xHB=5h!inS`$jCcR z0=Ri2yDT1KnV#sYxR0nABUmyv63LlNPJ=f%_7=U%ED2o2G>eJBAdv&mZvrWq2iTL3 zpbsjt(f||^dP=E<27UUo7k~rrEz-R_%@93F-KRFdSdbpcOT=RlrofiGpar zNfW*J@8$R)@m2@k)mOR-uq)Ndfru5NSI&{m|yd0`~C$k0nMRYeh*aSg$%Rz<_Mo7Q&I@sZ=CGZ$J3W_bU~8R{Ru0+>#jraqgb&?`|gJj1K~23R#?p zfmEMjDwr9|EzPfuNZrVRP(rg9nXdb`BI7`S8plCY@GJIx9D#hrGH$A@2tki9t-%&m zeq>_1)`3ip#vy6y!l+hsp2`rBfXj%aHuZhJ%wUN+dUFIYk9>O(=Ty)(4(}F{n8r@a zgi*sX7s^XDotPi;IMnz-#I*C zuMYapj$if}jj*2|ACYIjAr4Q|9D`ss{d z4!Qa;xTe)*WhgR@cWJ|t9AXs}D8PQZIzZV!NA4X&#yj3Y@-O4$4CxxUa6ySJWhRS$ z?Ss~Uh5Y>v(S3K7{%-60^fUTynYk+SL=&9=QOW*4GxaZadQ{8=oK<4m|0KPcr}!?> zD+B71eDLC>tc{AOcZJrwB34v*bE-an@CvNHij&D*Aa$|0f3sxY^R2zV)Bpeas0KCj zmk_Aj<)Hjz>RqjRt?IV6(UW(*rqvLxZ8@^-TFvdPTHCL=ZLjT@>Rq4w^VffQd#_TV zXKR(VQ-QK5c$vEf<{YNcq)K5_YZ4)~^YrH{KiR4{e?|}w&}?W^~Z)@ zu(XJU$^=!vb!iq24^vGQA^Q3pg;p-oNrXDF{;<`e@x3_s?{hXR99!EGBq|`S*pLONnL+KHHXfM(g$iyQ~{WwMcc_#JXeulNd*?W zh~wWq8oj<>snFA2MVzLK_=GAF>N)_gDT&Va4QbNc6`0koonXMDeB{XFJ$*uy#X#^l zaM35zjcU{)^)(Ji(XLaK8XXw?S)-yq8n?RXp<9I#%|*;n8>SdYbKeiHIt*n5SEBiq zX6#bRGptb6-&K2v@9PQWVj21emv}9 z+TBo1Rc?9;BI9pl63-@cXBk`V8Y#g-dt z?!uI;QEA|9`w#a%dU*fcx71+8>i0D~^lk5~ojhNJUA}0hAy+D2-}~(fBX@>{Tm! Date: Thu, 30 Mar 2023 20:25:10 -0400 Subject: [PATCH 2/9] Changed test case. Back to dbt_duckdb:build (instead of just run). --- dags/gie_dag.py | 4 ++-- meltano_transform/tests/{stage_timing.sql => storage_max.sql} | 2 +- meltano_transform/tests/{stage_timing.yml => storage_max.yml} | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename meltano_transform/tests/{stage_timing.sql => storage_max.sql} (50%) rename meltano_transform/tests/{stage_timing.yml => storage_max.yml} (88%) diff --git a/dags/gie_dag.py b/dags/gie_dag.py index b9628b7..30e0725 100644 --- a/dags/gie_dag.py +++ b/dags/gie_dag.py @@ -56,7 +56,7 @@ def stage_gie_dag(my_date : str = '-'): run_this_2nd = BashOperator( task_id='Update_reporting_table', - bash_command = "cd " + PROJECT_ROOT + "; ENV_DATE_GIE={{ dag_run.conf.get('my_date', '-') if dag_run else '-' }} " + MELTANO_BIN + " invoke " + "dbt-duckdb:run --select tag:gie", + bash_command = "cd " + PROJECT_ROOT + "; ENV_DATE_GIE={{ dag_run.conf.get('my_date', '-') if dag_run else '-' }} " + MELTANO_BIN + " invoke " + "dbt-duckdb:build --select tag:gie", ) run_this_2nd @@ -96,7 +96,7 @@ def stage_gie_backfill_dag(): run_this_2nd = BashOperator( task_id='Update_reporting_table', - bash_command = "cd " + PROJECT_ROOT + "; " + MELTANO_BIN + " invoke " + "dbt-duckdb:run --select tag:gie", + bash_command = "cd " + PROJECT_ROOT + "; " + MELTANO_BIN + " invoke " + "dbt-duckdb:build --select tag:gie", ) run_this_2nd diff --git a/meltano_transform/tests/stage_timing.sql b/meltano_transform/tests/storage_max.sql similarity index 50% rename from meltano_transform/tests/stage_timing.sql rename to meltano_transform/tests/storage_max.sql index 5883c06..08a1e8f 100644 --- a/meltano_transform/tests/stage_timing.sql +++ b/meltano_transform/tests/storage_max.sql @@ -1,3 +1,3 @@ SELECT * FROM {{ ref('rpt_gie_storage')}} -WHERE _sdc_batched_at <= _sdc_extracted_at \ No newline at end of file +WHERE gasinstorage > workinggasvolume \ No newline at end of file diff --git a/meltano_transform/tests/stage_timing.yml b/meltano_transform/tests/storage_max.yml similarity index 88% rename from meltano_transform/tests/stage_timing.yml rename to meltano_transform/tests/storage_max.yml index d10fd97..d5c0b30 100644 --- a/meltano_transform/tests/stage_timing.yml +++ b/meltano_transform/tests/storage_max.yml @@ -2,5 +2,5 @@ version: 2 # NOTE: THIS DOES NOT WORK. UNCLEAR IF YOU CAN DOCUMENT A TEST AT THIS STAGE. tests: - - name: stage_timing + - name: storage_max description: Data needs to be extracted before it can be stored in the database. \ No newline at end of file From 35a7960ee4f46446391b8e432872b2ed661eeb5f Mon Sep 17 00:00:00 2001 From: EJOOSTEROP <49541561+EJOOSTEROP@users.noreply.github.com> Date: Thu, 30 Mar 2023 20:28:49 -0400 Subject: [PATCH 3/9] Add margin to test case. --- meltano_transform/tests/storage_max.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meltano_transform/tests/storage_max.sql b/meltano_transform/tests/storage_max.sql index 08a1e8f..ad8bcb5 100644 --- a/meltano_transform/tests/storage_max.sql +++ b/meltano_transform/tests/storage_max.sql @@ -1,3 +1,3 @@ SELECT * FROM {{ ref('rpt_gie_storage')}} -WHERE gasinstorage > workinggasvolume \ No newline at end of file +WHERE gasinstorage - workinggasvolume > 1 \ No newline at end of file From 7e1433ec11cd04ad87c0df1f0708f2a68d46e589 Mon Sep 17 00:00:00 2001 From: EJOOSTEROP <49541561+EJOOSTEROP@users.noreply.github.com> Date: Sun, 2 Apr 2023 20:23:48 -0400 Subject: [PATCH 4/9] Add mapping (key hash field) to GIE tap. --- meltano.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/meltano.yml b/meltano.yml index 7eab47e..cde5b25 100644 --- a/meltano.yml +++ b/meltano.yml @@ -60,6 +60,18 @@ plugins: params: date: $ENV_DATE_GIE #date: '2023-02-15' + stream_maps: + stg_gie_storage: + key_hash: md5(config['hash_seed'] + (gasDayStart + code)) + #__alias__: stg_gie_storage_vX + stg_gie_company: + key_hash: md5(config['hash_seed'] + (gasDayStart + code)) + stg_gie_country: + key_hash: md5(config['hash_seed'] + (gasDayStart + code)) + stg_gie_region: + key_hash: md5(config['hash_seed'] + (gasDayStart + code)) + stream_map_config: + hash_seed: 01AWZh7A6DzGm6iJZZ2T streams: - name: stg_gie_storage path: /api @@ -132,7 +144,6 @@ plugins: pip_url: target-duckdb~=0.4 config: add_metadata_columns: true - #default_target_schema: gie_stage default_target_schema: main filepath: $DB_LOCATION data_flattening_max_level: 10 @@ -140,7 +151,6 @@ plugins: inherit_from: target-duckdb config: add_metadata_columns: true - #default_target_schema: usgs_stage default_target_schema: main filepath: $DB_LOCATION data_flattening_max_level: 10 From f230d7e09e5d9b3030d3ee08c0472b3d6ca7cd29 Mon Sep 17 00:00:00 2001 From: EJOOSTEROP <49541561+EJOOSTEROP@users.noreply.github.com> Date: Sun, 2 Apr 2023 21:53:05 -0400 Subject: [PATCH 5/9] Expanded dbt tests (incl using dbt utils), update dashboard, added hash keys using Meltano inline mappings. --- dockerfile | 3 ++- meltano.yml | 13 ------------- .../models/gie_rpt/rpt_gie_storage.sql | 2 ++ .../models/gie_rpt/rpt_gie_storage.yml | 12 ++++++++++++ meltano_transform/packages.yml | 3 +++ standup/dashboards prior.zip | Bin 0 -> 20295 bytes standup/dashboards.zip | Bin 20295 -> 22806 bytes 7 files changed, 19 insertions(+), 14 deletions(-) create mode 100644 meltano_transform/packages.yml create mode 100644 standup/dashboards prior.zip diff --git a/dockerfile b/dockerfile index 8fee113..11586c8 100644 --- a/dockerfile +++ b/dockerfile @@ -51,7 +51,8 @@ RUN mkdir -p /${MELTANO_PROJ_ROOT}/data/dev/ \ && mkdir -p /${MELTANO_PROJ_ROOT}/data/prod/ \ && /${MELTANO_PROJ_ROOT}/${DUCKDB_CLI_FOLDER}/duckdb /${MELTANO_PROJ_ROOT}/data/dev/data.duckdb "select * from pg_tables;" \ && /${MELTANO_PROJ_ROOT}/${DUCKDB_CLI_FOLDER}/duckdb /${MELTANO_PROJ_ROOT}/data/test/data.duckdb "select * from pg_tables;" \ -&& /${MELTANO_PROJ_ROOT}/${DUCKDB_CLI_FOLDER}/duckdb /${MELTANO_PROJ_ROOT}/data/prod/data.duckdb "select * from pg_tables;" +&& /${MELTANO_PROJ_ROOT}/${DUCKDB_CLI_FOLDER}/duckdb /${MELTANO_PROJ_ROOT}/data/prod/data.duckdb "select * from pg_tables;" \ +&& meltano invoke dbt-duckdb:deps ###RUN chmod -R u+x /project/data/ diff --git a/meltano.yml b/meltano.yml index cde5b25..eaab872 100644 --- a/meltano.yml +++ b/meltano.yml @@ -159,7 +159,6 @@ plugins: inherit_from: target-duckdb config: add_metadata_columns: true - #default_target_schema: gie default_target_schema: main filepath: $DB_LOCATION data_flattening_max_level: 10 @@ -168,10 +167,6 @@ plugins: - name: airflow variant: apache pip_url: apache-airflow==2.1.2 --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.1.2/constraints-${MELTANO__PYTHON_VERSION}.txt -# transformers: -# - name: dbt-duckdb -# variant: jwills -# pip_url: dbt-core~=1.2.0 dbt-duckdb~=1.2.0 files: - name: files-airflow variant: meltano @@ -179,13 +174,11 @@ plugins: utilities: - name: superset variant: apache - #pip_url: apache-superset==1.5.0 markupsafe==2.0.1 duckdb-engine==0.6.4 pip_url: apache-superset==2.0.0 flask==2.0.3 werkzeug==2.0.3 jinja2==3.0.1 wtforms==2.3.3 git+https://github.com/meltano/superset-ext.git@main cryptography==3.4.7 markupsafe==2.0.1 duckdb-engine==0.7.0 - name: dbt-duckdb variant: jwills - #pip_url: dbt-core~=1.3.0 dbt-duckdb~=1.3.0 git+https://github.com/meltano/dbt-ext.git@main pip_url: dbt-core~=1.4.0 dbt-duckdb~=1.4.0 git+https://github.com/meltano/dbt-ext.git@main commands: usgs: @@ -198,12 +191,6 @@ jobs: tasks: - stg_usgs target-duckdb-usgs dbt-duckdb:usgs schedules: -#- name: USGS-Earthquake -# interval: 35 */1 * * * -# extractor: stg_usgs -# loader: target-duckdb-usgs -# transform: skip -# start_date: 2023-01-01 15:40:21.295936 - name: USGS-Earthquake interval: 35 */1 * * * job: usgs-to-duckdb-rpt diff --git a/meltano_transform/models/gie_rpt/rpt_gie_storage.sql b/meltano_transform/models/gie_rpt/rpt_gie_storage.sql index 04ab3d8..bb272c8 100644 --- a/meltano_transform/models/gie_rpt/rpt_gie_storage.sql +++ b/meltano_transform/models/gie_rpt/rpt_gie_storage.sql @@ -6,6 +6,7 @@ SSO AS gasdaystart::DATE gasdaystart, split_part(url, '/', 2) as country, split_part(url, '/', 3) as company_eic, + key_hash, code as sso_eic, name as sso_name, status, @@ -36,6 +37,7 @@ FROM select _sdc_batched_at, _sdc_extracted_at, + key_hash, sso.gasdaystart, country, SSO.company_eic, diff --git a/meltano_transform/models/gie_rpt/rpt_gie_storage.yml b/meltano_transform/models/gie_rpt/rpt_gie_storage.yml index f7fe67e..56ad3e9 100644 --- a/meltano_transform/models/gie_rpt/rpt_gie_storage.yml +++ b/meltano_transform/models/gie_rpt/rpt_gie_storage.yml @@ -6,6 +6,12 @@ models: +tags: - gie + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - sso_eic + - gasdaystart + columns: - name: _sdc_batched_at description: Timestamp when the data was captured in the database. @@ -13,6 +19,12 @@ models: - name: _sdc_extracted_at description: Timestamp when the data was retrieved from the REST API. + - name: key_hash + description: Has of sso_eic and gasdaystart. + tests: + - unique + - not_null + - name: gasdaystart description: Date of the observation. Ex. the injection field refers to the injection on this date. gasinstorage as per end of the gasdaystart. tests: diff --git a/meltano_transform/packages.yml b/meltano_transform/packages.yml new file mode 100644 index 0000000..d7c5292 --- /dev/null +++ b/meltano_transform/packages.yml @@ -0,0 +1,3 @@ +packages: + - package: dbt-labs/dbt_utils + version: 1.0.0 \ No newline at end of file diff --git a/standup/dashboards prior.zip b/standup/dashboards prior.zip new file mode 100644 index 0000000000000000000000000000000000000000..bfe4b19fed1f7a414ca6ea050928ae00a57d9e76 GIT binary patch literal 20295 zcmeGkO^+i-)k_jUgMtJ~B)D}}B$GgT#_exg$->Nzv+L~6$9lZGSr95!cURlJ)9&{4 z$IN&`q#QUPA+8{hkU-*!K>PrXoH)Txh&UjGkhp_*ud4dP?Y76mZZ@zJuX;7Es;;W{ zQSZHaA3nJC-19G$#OIaLPsZ>4c^AImvjLAugOTo0-z4m8;`t$|lq=P8wbB|?DwS5H zxjAMbHEBpUrgZF_Fh8(8x1(qq<&E-MIGwPLvJ+3S7TRMLgmgUVDC;mrcR~R zsWdlQje5IX{YJUmDVNvR4rVs?^50(g!!KVbmGJ4Vvaw|6!Dct|y$Pc(xj_S>k{i7) z(fEFoF(kA@2lg6|R1_Ghs0s{SGg)Bx_9Wzhxsl_n89@LLtgi)5H0&t6DT-{fqgb_S zt7@^jT5p?7t+i=W)th=?PAF_X48+D%rN6&FAiW*l1t{(JcG7SC{rl?OptZe!rnwKqJ_XlsaN!jJHZY&# z`An^-+PNsojO9xf;;6#AmgDFh+i{dW1zO3$0xMLjT9taeskZAi9f(}3sI8h_Q#D$v zYi+$uEo#XZBvxwt>yyOF&eq}0?ftF(4!PIuANKb5$gS>!j&g!vNd)Kc#J7O7n8PL*I*B*x2^g^;2 z?|%S2{0j+;5kmn0;!9C{X89}_5%~W&Vy-dmD0l|p1c69^S-8N6-!GUF1-eP%S?uvR za#`{cmt7{aXykw38Ii(HY5@)<;B%>5Jh2}UK@BEmC4grT=G3QSF6pQ_@{ANt4n1{U zFXX@hL8hEcynU7wX7xG^+0gT+8e-q@#xScLc&=RVggOxmpa=DA;6=Ux^wS6lsqP@j z;b{4kSe`#7I6{&ezB7XcVb2BrfGL2q8!Z?ew;wRy_DmoyUv%hG&||P{I3>Zc1CtDC zVA3h5Yd?gykf|p+@CV<&!-mW?_kA10yrf;|O<84Zx7Vr)XM;sSoemhIv4G)8Kj(grGf< z-IFt*D0Z;NPC*}2CqN9r#>4=@1H8rK8eM2Zu_Hn&N8UrTQnpV#a-=gi7}YJmiAaMp zckFbNU38SgySuO3?n7oE=e_|WW*~uH{*nZQ|E(z0JfW@u2L`DmX`6ruP%}TYE!zhzv~PugTTXvkXZ;42oK#0n8ZV7u2br}wmT%qiKchDl(l-h2e-R?nG17Q)`lu#FU_CW z!N@LGA~Fpu=lJ-TR$ZO1m18ftch3d66@my!dE1*%0}d&kW>of`Z!(`Cg%e@{N=ERK z=Y%#G+EOj}pfMbKYy*z9GJN3G0FcK5aiARA70LsL@xz_GDWoGOG`xfopMbEiT`49G zG_(bHGl-BO@XVg42o}P2v5f(|^^PQQ07M8m)v@1?Twr)c#0g9mT_9~0X0D@fP5uf=ZUoAJOVLN3Hx}nl93q2) z(&nIfQG7@)V4>%9D%T<`Bw_}$T&6WmtE;WLZh?(&Qk7bDOVt}y+OC_LZq@bNRjhgC zk6-!GmrEslR=kRV-eMsr;>bAK+1~UgAsO0WDTJO+hm2pvxH6L`H`l*5Ms`C1f(Fg3 zW}266vBd!jYU0^$j4#i!U<)WKim=J8B2ap`(i)saZXZeAE_fhhlVLVFWXU?$4;SkG~#Ed|~MD3Rz+RjN^mG6R?w_4KR=ybjjq;PtPg zev&FqNY!|BRs&?l&Lb6+NHu1Dc{T(9Fteh}i&t)4#?8X^=dTHc2!~9SN?vZC1g1fB z3itx$M&|*MYr+8+dPGV!_)b9spQM^fz&Ggab`J+zy9dhs-r(l`-GPEF%D4CTP_i%4 z!-<(A7bwt$S)9pj72#`%c9WzBYqtmsyw?aXGaiD#Lqpmj{9-W2)p=%!EsaIoP+&u> z!UFNMPnQSK`X^v}+pd(3mk|6@&p!fl4X|*6hAmnqvvTd3=71Ud{Sv`_d71ul^LuiQ zzFfzgyLsHHFo4iy|9D6J-c}!-nJ~{Py6qD&Z?X_yCU}{RQIcn#ylAzd6ZNvddRfQ{ z9^MGO&(Ayq^INe$y$qnvI`@}T_ME%+&eq_6>rxHSWRnok-KB&3N&35*)k@iDwjd^N zn02*A4NcWq+feJRMx||4jJDafa{jKD|M7$0zkj7v!l$uH*ePBzc6iBL1Gx_wXyUFg z^fgfjLvm9R`F2M!q1n_soA8y0oakfp9o$x<4#DGF47I-Gm`z>6v5b%)RRO&!n zpMay4&|vCKYL~T@VQb6pY|%`lu^WY z&?CXq0Z4^YG#fX>PMIr^`mS|*NZe?wGyj}8A-Kf=;E8QOoJ<6&!51mgIDm>;6}{9D zfq_3O==6uctq6LEP$7fhBJ^PxCT<8K%d*cp3b+vr2Enf&R8AQ?f)U*NyJ`&ayr|%a zQ4Bv=o*R;Y$M?z`HCRMA8zm%9HNcml1LCP53Nsz*)d>UNtVQU!(u{^)`{amntmc9+ z7~{)cO5ad6dE765T|@UOhB`6AB>aziUK&mbAr}VX@o*lJ=mrm}5*bsVA$i1nFYS!< zBD881WP+LS#%v_zVu6`LCQU*iv8#9e%?<6<^+I7IPUA|sFuBvnIq-4f57A$)IA8sI zsf5q9RSs~H<^`!>5PTXPNA!pV@)ReXNe);#8u+YPg()-3pon(d}qXUx>w8r7>tTfQLEn=~P*ATV-~ ztgjpI=&XOI=eT?_ac*XlzJ?F(-}2n<+3~$|QIihkvlXI72+`%JNl>C(i=*0-(QA68 zT5Gq}Mjg!tqtRBWp;NWmHVnhm45L{Qc>)j(d5l!JzIo?ZyL0rOb7LV^j<>sJc-SB8 z+&LF3>CnYksq(JnSlO`y$k@;$%ym%o=>oBgQ=}n4H(DNKua{+ z^=boSqWF27ytt((T9y~L)4SJ$gf&&$8Fx?Ky}exvmmqL{aEd~aPIcidE;%L4b8d!S z4n~o-H+x`IWx*CPQlHI{TXHPLp?i#UzKoO1%!xYB*YD&dr+s+-*7@C|!i=j#9e2U=2raOXhS#u93U;3f>Z@2*Bb8{7= zBhEd{$WBPxB8i3})wxT=VmX_dYa^J?1G{{D&XodgQkX3Pd2|8cY2}bGkkR+z#t|oj z2GImA%`Cjm^F@KmMl^{M8Pm19N9r(Vojz$pIDP2 z(R@n7|7~8)N`w$MR-Tw5{`@qr=F_|y%xHa<@{2O(`#g94IZogI(*!GlfXH{yAA{a7 zkl>k&;koW4-z-)xix}c?(C_Wxoo$uiYdje^I3YDZo9LO%bjK9L^|z=pg7S*DuO?f& zdhIrB{jE)U4Z2~$c|i>qlf{|U3vH>SClZn#G7rW56ig+^q_&}^h8J2q z9s({FiZ9{@iHnah)lifdK+FM)1fG3x6FTD+^-@5b2#iYsEp?!H=2AckN$AGY=HO36 zG~4Jbe43K!=|Bk(J=y5=zh3*u`o(8TC49bh0mp=l0%RN#wK<2xe54?~MT_b~Sq%ZC zDy)A@oe=0dQk9r{RYk}rysYz%toZ?@bn-aI!-SfWQBo+|k)-O_`y&dtbQp9qg#1<@#v1dXg5bE>b)r?Q0!mIS5CJPt@ zg6TsD>Pm+Qug~OeH-iB_b(*dA#fpA{&Or?Ph#DHiDgTv)qz>xI1PTSg7rRVW3LF)% zKlq(wZpKPC_1R$lKVSTN`+HChj8EqR+`k+9!bp?U=lDM9#V}CH@^R%a$1ty+LuxE^ z#Bs(p92x~c@cTEr{Vue`F?tHMPhjFVk%O{k|A5!d9;Q`@;+C+Vsa6#&q&W;)C$7)D zGcq3Pr^jbb!xfP^JLqCbaGckH3(OV7!Vh9Vbp_wUwYSA4;-(&tAao(Gr%whqxAw?> zpLFls-4Zpoxgfc`9e`sv(lK2HYpBw&;%WuZ^X|u$N}$hmLP8k(u}Yw4WK_;&$%`a+ zWf`GF!UfBs7nDKsLM+KCybhZGz4U5{6n;|W!&&m-yhRsGY%jQ+U~~3TE5$T6C)mrh zq1dX#@kh8>Khwp-@injDD9YM&QGl1AAG1md6IhNy`beuy%=(x-DnEJ{1*4!YexaoJ zXpU`gryounI|R=`nuJnRTRdM(>}6~GL3oEy!-!YLm5YpvT=Z#4agx(ATT=W0R#@c6 ze{v-@ zHnYNI`8}(oJv<4We&sVSJ-7H*s6esv&xa6vyXbFH9o(9oJO4vcV&+om!z({sW$q+h zXvxn?<(JI=Zb`c2&p*2YIB_I#`LCGd7tL2`r;C2=iz_TDv$WzA*20?Y{F3=Z*>uU5 zURiNT%+!t7 h+wIj?ORfURMd9Gq3vg;nr7yt0Uq4?eeg9qj_8&bYJf#2t literal 0 HcmV?d00001 diff --git a/standup/dashboards.zip b/standup/dashboards.zip index bfe4b19fed1f7a414ca6ea050928ae00a57d9e76..10c00a7649dd62aae97aaae11cf7f9bc91733bf8 100644 GIT binary patch delta 1429 zcma)6-*4Mg6po!WX|pX2w2S%+a7epl?VLEN+r}P{b=^iXH*MENR4^gy#&%p2J8o>J zTPiWyNeK3s9RYh{8V|ex5?h|8y=>AD;$a$p!z9E@HC2M~fcAi$RjO-&#I>%h^PO|g z{mysqxj%o6eR&rfUgG=*`h71)%=gXq8=FPjaAp#6;lU71O_r!M&1ABzcXItS*)ds~ zVKdp})J%GYVp^?esQl#VnQK9x?^>{RXT-GzlHAyx7x!?ug$F_v@q-#&H&%5Se#B#L z^GRX|o4zp~KUt^|7cJdDvJ@gZqtQ@AmM9BMSwezHN>kHmlFrg;5@iL3WT*9B3O{A%&st6KaLz=`AF74Kp4c=<5*`TPH7Q8+8 z8eEMB`pUXx=^C#gLsn`mff|+$f286eU2SMJlO-;F2;1?c0ol+SYvm?umrW8fGDXxh zEqi~^yN99F^D$hH_TmHquSBBoR}Y8Hz~;H%;fK)}v&XL^nZ9NzdTpV@PLnJ{5qM3d zl=aISXfh-5ikYS5Dyo}T4A?q2(rbv4!Hbe9utWu^COke!0-x`PU#Wo!OVK1#G8D<= zRYg-Q{$)OgBS9cpHY6EYcD=>I0<83GL|n8xA-iNxf_|37WD#mSVd(??Hp!lWfgY9M<9|8jQ+0%Ku0gTZ`sqM)ozy! z4~mKQC#ekX!rJ%K6JfW#llya@|Jmtpg9n^P0dAhB`dk;%mLG}8LVy*IxVj(yK0oft zXSe?%ezhA;g$LZ|ZSDfz74f3c`9=Ksn_l$#{3-m=s27dJ2O&_5jO=Tg-B--{ZDOA$ zoHehzDE~_j+3ZvJA>Qo}C;4zu2CEd9UZ$ X>7nQn*A;M>9h?>oUg8-R3Ne zlY2PSkQ8j*!%@qQ#GhOjs=7Ht;Ght)*vC+@&7R8Z8Ii;`GpSBwo-Ad6VwRUdHV3kr z{T5M7Nc_pH5vrR#tl9XG#5PBGenD2VIm~AnI}0cnCWROvsheySB8TM5$w?tvNU{^J zt4`h>qQx|qZL*?(1d_z$Jsj$j4MMe;RQS+k>O%dQL=@3QKZZImJyt>&4Gq%-`c;4j z8cz%jj7!we)p!}mO}-i?FgY+>ib+x*UA8|wh^f{PU6eJ#pJ|glx@bj&7t;nObkS!K TN_^Z5+ze}37#Lc-KrR6QcYl#S From f05ec776f32d2971597ec3ce84ee5278cc1a5ee1 Mon Sep 17 00:00:00 2001 From: EJOOSTEROP <49541561+EJOOSTEROP@users.noreply.github.com> Date: Thu, 13 Apr 2023 23:39:15 -0400 Subject: [PATCH 6/9] Edits to readme. --- readme.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/readme.md b/readme.md index 1671695..b70c0f1 100644 --- a/readme.md +++ b/readme.md @@ -100,7 +100,7 @@ A minimal modern data stack with working data pipelines in a single Docker conta Mimodast can be used to explore the functionality of the tools by using the examples as-is; and to modify and expand on the exmples for further exploration. -It is a starting point for exploration. The project is not a showcase of all or even the best functionality that each tool has to offer. The tools have more functionality than is accounted for in Mimodast. +It is a starting point for exploration. The project is not a showcase of all or even the best functionality that each tool has to offer.

(back to top)

- - ## Usage -The pipelines include the injection of data from source into a database, transformation, data testing, documentation and reporting. Various tools are leveraged. +Two basic examples are used to illustrate an extensive date pipeline encompassing the following components: -This section highlights some of the key items for the tools being used. For full documentation refer to the respective websites of each tool. +- obtaining the data from source +- capturing the data in a database +- transformation +- scheduling +- testing of data +- reporting +- integrated documentation of the process -

(back to top)

+The two pipelines involve [USGS Earthquake][USGSEarthquakeAPI-url] data and [European Gas Inventory][GIEAPI-url] data. +Below we highlight the core configuration for these components. For (much) more additional information refer to the respective documentation of the tools. + +

(back to top)

### Definition and Configuration @@ -207,78 +204,193 @@ The data pipelines are fully defined in a set of files. This includes the source These files are all found in the `/project/mimodast/` folder in the Docker container. It is best practice to capture this folder in a version control tool. Git is included in the Docker image. -The core files include: +Some of the core files include: - `/project/mimodast/meltano.yml` - this contains items like the source specification, destination database and schedule. - `/project/mimodast/orhestration/dags/gie_dag.py` - python code defining how to orchestrate a data pipeline in Airflow. Note that the GIE data uses this manually created file, whereas the USGS data orhestration relies purely on logic defined in `meltano.yml`. - `/project/mimodast/tranformation/` - this folder contains transformation logic (under `models/`) and also tests and documentation. +

(back to top)

+ +### USGS Earthquake Data Pipeline + +#### Obtaining the data from source -### Meltano +The `meltano.yml` file specifies how to obtain the data from source. Specifically, configuring the connection to the data source is centered around this `tap` section in `meltano.yml` (To be found at `/project/mimodast/meltano.yml` in the Docker container.): -Meltano's functionality is largely driven by the meltano.yml file. It is used to install most of the other tools contained in the image, to define the ELT pipeline and to schedule the [USGS Earthquake][USGSEarthquakeAPI-url] pipeline. The file can be found at: `/project/mimodast/meltano.yml`. +```yaml + - name: stg_usgs + inherit_from: tap-rest-api-msdk + config: + api_url: https://earthquake.usgs.gov/fdsnws +``` -This file can for example be edited to change the period or minimum earthquake magnitude selected when getting data from USGS. Search the file for `name: stg_usgs` and review the fews lines following, especially the settings for `starttime` and `minmagnitude`. +In the example, this tap is inherited from a publicly available tap for a REST API: -Meltano can be used to invoke many of the tools. For example the following command preforms the tests defined in dbt: `meltano invoke dbt-duckdb:test`. +```yaml +plugins: + extractors: + - name: tap-rest-api-msdk + variant: widen + pip_url: tap-rest-api-msdk + ``` + +`pip_url` specifies the location of the source code for the tap plug-in. In this case the actual location is derived by Meltano based on the tap name. Note that the tap for the other data pipeline (Gas Inventory data) also inherits from the same tap, but is clearly configured differently. + +`meltano.yml` contains the ful configuration of the tap. For example the following element specifies to select only earthquakes with a minimum magnitute: + +```yaml + minmagnitude: 6 +``` + +Many more items are configured for this tap.

(back to top)

+#### Capturing data in a database + +The following section in `meltano.yml` configures the database where the data will be stored: -### Airflow +```yaml + - name: target-duckdb + pip_url: target-duckdb~=0.4 + config: + add_metadata_columns: true + default_target_schema: main + filepath: $DB_LOCATION + data_flattening_max_level: 10 +``` -Airflow is used to trigger the ELT pipelines based on a schedule. The Airflow UI can be access from `localhost:8085`, with `admin/admin` as user/password. There are three pipelines: one for USGS and two for GIE. +A [duckdb][DuckDB-url] database is used in this example. This database contains the data captured from source and is accessed for reporting. The database file is located at `/project/data/dev/data.duckdb`. This database location is specified in `meltano.yml`. -The USGS pipeline and schedule are fully managed by meltano.yml. Note that one may need to wait (up to a minute) for changes in meltano.yml to be recognized by Airflow. +The image contains the DuckDB command line interface. Use the command: -The GIE pipeline and schedule are defined as DAGs in a python file located at `/project/mimodast/orchestrate/dags/gie_dag.py`. The GIE pipelines are initially paused as they will not function without the API key referenced above. Once the API key has been provided the GIE pipelines can be activated using the Airflow UI. +```sh +/project/duckdb_cli/duckdb /project/data/dev/data.duckdb +``` -The GIE backfill pipeline uses Airflow variables to select the period for which to capture historic data. These variables can be changed using the Airflow UI. Again one may need to wait (up to a minute) for any changes to be recognized by Airflow. Note that this is not a [best practice][AirflowBestPractices-url] design but is used for simplicity. +to start the CLI and browse the database using SQL (the CLI has a help function and is documented online). + +Note that the database maybe unavailable if another process (pipeline, reporting) is already accessing it, resulting in an error message.

(back to top)

+#### Transformation -### Superset +Transformation is performed by [dbt][dbt-url] and specified by a set of `SQL` files. In the container these are located at: `/project/mimodast/transform/models/usgs_rpt/`. -Superset is used to visualize/dashboard the data. The Superset UI can be accessed from `localhost:8093`, with `admin/admin` as user/password. Two dashboards exist. +For this example pipeline the transformation is fully captured in `rpt_usgs_events.sql`. In this case the 'transformation' simply copies some source attributes into a reporting table. -At the start the database in mimodast is empty so make sure to run the datapipeline(s) before reviewing the dashboards. +

(back to top)

-Due to an incorrect setup in mimodast individual charts in a dashboard frequently do not show. For now: +#### Scheduling -- Refresh each chart by selecting this option in the right top corner of each chart. -- Possibly wait for other processes accessing the database (say the data pipelines) to complete. +The following section of `meltano.yml` configures the pipeline's run schedule: -

(back to top)

+```yaml +schedules: +- name: USGS-Earthquake + interval: 35 */1 * * * + job: usgs-to-duckdb-rpt + start_date: 2023-01-01 15:40:21.295936 +``` +The scheduling `interval` uses the [Cron job][Cron-url] format. In this case the pipeline is schedulted to run 35 minutes after every hour. -### DuckDB +This section refers to a `job`, wich is configured in the same file. This job consists of a `tap` to obtain data from source, a loader specifying where the data needs to be stored and a transformation component: -The data for the pipelines is captured in a DuckDB database. The database is located at `/project/data/dev/data.duckdb`. +```yaml +jobs: +- name: usgs-to-duckdb-rpt + tasks: + - stg_usgs target-duckdb-usgs dbt-duckdb:usgs +``` -The image contains the DuckDB command line interface which can be used to query the database. The command `/project/duckdb_cli/duckdb /project/data/dev/data.duckdb` will start the CLI and open the database (the CLI has a help function and is documented online). +In this docker container [Airflow][Airflow-url] is used as the scheduler/orchestrator. Meltano ensures that the specified schedule is set in Airflow. Note that the schedule can also be set directly in Airflow, as is the case for our [GIE Gas Inventory][GIEAPI-url] pipeline. -Note that the database maybe unavailable if another process (pipeline, reporting) is already accessing it. +To monitor job progress access the Airflow UI from `localhost:8085`, with `admin/admin` as user/password. There are three pipelines: one for USGS and two for GIE.

(back to top)

+#### Testing -### dbt -dbt defines data transformations. Mimodast contains transformations from staged data to data used for reporting. dbt is configured by using the files in `/project/mimodast/transformers/`. +Data tests are specified in two locations: -New transformations can be triggered by running the pipelines in Airflow; or manually triggered using `meltano invoke dbt-duckdb:run` from within the `/project/mimodast/` folder. +- SQL files in the `/project/mimodast/transform/models/tests/` folder. (In this specific case just a basic test that the magnitude of an earthquake is not unrealistically high.) Tests are performed by dbt. +- yml files in the `/project/mimodast/transform/models/usgs_rpt/` folder. For example the following tests that the `id` field is not null: -Data tests and documentation have been setup in mimodast. +```yaml + - name: id + description: Unique ID of earthquake or related event assigned by USGS. + tests: + - not_null +``` -The documentation can be viewed at `localhost:8094`. The tests can be triggered by invoking `meltano invoked dbt-duckdb:test` from within the `/project/mimodast/` folder. +Tests can be initiated manually from the command line in the Docker container: +```sh +cd /project/mimodast/ +meltano invoke dbt-duckdb:test +``` +

(back to top)

+#### Reporting + +[Superset][Superset-url] is used to visualize/dashboard the data. The Superset UI can be accessed from `localhost:8093`, with `admin/admin` as user/password. + +Initially the database in mimodast is empty so make sure to run the datapipeline(s) before reviewing the dashboards. + +Due to an outstanding issue in mimodast, individual charts in a dashboard frequently do not show. For now: + +- Refresh each chart by selecting this option in the right top corner of each chart. +- Possibly wait for other processes accessing the database (say the data pipelines) to complete

(back to top)

+#### Documentation +dbt is used for documentation of data and processes. Documentation is maitained in .yml files and `overview.md` in the `/project/mimodast/transform/models/` folder. +This documentation can be consulted at `localhost:8094`. +

(back to top)

+ +### Gas Inventory Pipeline + +The [Gas][GIEAPI-url] pipeline is setup in a similar way as the USGS earthquake pipeline. + +The following differences are noteworthy: + +1. The pipeline requires an account to be setup. This is free, quick and avaiable to everyone. Refer here for instructions. + - Note that for this reasong the jobs/DAGs for this pipeline are initially paused in Airflow. Once the API key is obtained an setup in the docker container use the Airflow UI to unpause the jobs. +2. The `meltano.yml` file is configured to obtain this key from an environment variable as follows: + +```yaml + headers: + x-key: $ENV_GIE_XKEY +``` + +3. Schedule/orhestration is not configured using `meltano.yml` but instead with two manually coded Airflow DAGs. The Python file containing the code for these can be found at `/project/mimodast/orchestrate/dags/gie_dag.py`. + - The backfill dag captures historic data from source. To specify the date range, two Airflow variables are used. These values can be changed using the Airflow UI. + - It takes some time (<1 minute) for the new date range to be reflected in the DAG. + - Note that using Airflow variables in a DAG in this way is not a [best practice][AirflowBestPractices-url] design but is used for simplicity. +4. Meltano's inline data mapping functionality is used to create hashed ID fields when obtaining data from source using the following configuration in `meltano.yml`: + +```yaml + stream_maps: + stg_gie_storage: + key_hash: md5(config['hash_seed'] + (gasDayStart + code)) +``` +5. Additional test types are included. As for example (from `rpt_gie_storage.yml`): +```yaml + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - sso_eic + - gasdaystart +``` + +

(back to top)

## Roadmap @@ -400,6 +512,7 @@ Project Link: [https://github.com/EJOOSTEROP/mimodast](https://github.com/EJOOST [GreatExpectations-url]: https://greatexpectations.io/ [PRQL-url]: https://prql-lang.org/ +[Cron-url]: https://en.wikipedia.org/wiki/Cron [DockerDesktop-url]: https://www.docker.com/products/docker-desktop/ [USGSEarthquakeAPI-url]: https://earthquake.usgs.gov/fdsnws/event/1/ [GIEAPI-url]: https://agsi.gie.eu/ From 5d645920b08aebc2ec499be88ac276c65b9c2de0 Mon Sep 17 00:00:00 2001 From: EJOOSTEROP <49541561+EJOOSTEROP@users.noreply.github.com> Date: Tue, 18 Apr 2023 00:15:34 -0400 Subject: [PATCH 8/9] Documentation update and license file renamed. --- MIT_LICENSE.txt => LICENSE.txt | 0 readme.md | 17 +++-------------- 2 files changed, 3 insertions(+), 14 deletions(-) rename MIT_LICENSE.txt => LICENSE.txt (100%) diff --git a/MIT_LICENSE.txt b/LICENSE.txt similarity index 100% rename from MIT_LICENSE.txt rename to LICENSE.txt diff --git a/readme.md b/readme.md index 186ad0c..3a08ad9 100644 --- a/readme.md +++ b/readme.md @@ -381,7 +381,7 @@ The following differences are noteworthy: stg_gie_storage: key_hash: md5(config['hash_seed'] + (gasDayStart + code)) ``` -5. Additional test types are included. As for example (from `rpt_gie_storage.yml`): +5. Additional test types are included. For example (from `rpt_gie_storage.yml`): ```yaml tests: - dbt_utils.unique_combination_of_columns: @@ -404,19 +404,10 @@ The following differences are noteworthy: - [ ] Nested Feature --> -See the [open issues](https://github.com/EJOOSTEROP/mimodast/issues) for a full list of proposed features (and known issues). - -Observed issues: -[ ] Clashing DuckDB database access when viewing data in Superset while ELT pipeline is running (or vice versa). Workaround: wait for ELT to complete. This should be a fixable issue (though I don't know how at the moment.) -[ ] Superset dashboard cannot currently refresh all elements in a dashboard at once (an error shows for the unsuccessful grahps). Workaround: manually refresh each graph using Superset UI. This should be fixable in the same way as the prior issue. -[ ] Logging into Superset and Airflow simutaneously seems to encounter problems. Workaround: close the browser tab with the 'other' app, open a new clean tab to log into the desired app. - - +See the [open issues](https://github.com/EJOOSTEROP/mimodast/issues) for a full list of proposed features and known issues.

(back to top)

- - ## Contributing @@ -434,12 +425,10 @@ Don't forget to give the project a star! Thanks again!

(back to top)

- - ## License -Distributed under the MIT License. The tools and the sample data are subject to their own respective licenses. +Distributed under the MIT License. See `LICENSE.txt` for more information. The tools and the sample data are subject to their own respective licenses.

(back to top)

From fa9433c29480213cada06227b14642c8fbbeed5b Mon Sep 17 00:00:00 2001 From: EJOOSTEROP <49541561+EJOOSTEROP@users.noreply.github.com> Date: Tue, 18 Apr 2023 14:26:14 -0400 Subject: [PATCH 9/9] Issue with logo in dbt docs. --- meltano_transform/models/overview.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/meltano_transform/models/overview.md b/meltano_transform/models/overview.md index 31c53e8..e3c6844 100644 --- a/meltano_transform/models/overview.md +++ b/meltano_transform/models/overview.md @@ -2,6 +2,10 @@ ![mimodast Logo](https://github.com/EJOOSTEROP/mimodast/blob/master/assets/hatchful/logo_transparent.png) + + Logo + + # Mimodast dbt Project Mimodast is a minimal modern data stack with working data pipelines in a single Docker container.