From c3936018fbd6076f7e8bb41be183e70cbc87ae60 Mon Sep 17 00:00:00 2001 From: mutantsan Date: Thu, 13 Jun 2024 07:45:14 +0300 Subject: [PATCH] DATASHADES-338 / implement filter, wip 1 --- ckanext/charts/assets/css/charts.css | 2 +- ckanext/charts/assets/js/charts-filters.js | 293 ++++++++++++++++++ ckanext/charts/assets/js/charts-global.js | 20 +- .../charts/assets/js/charts-render-chartjs.js | 2 +- .../assets/js/charts-render-observable.js | 1 - ckanext/charts/assets/js/charts-select.js | 46 ++- ckanext/charts/assets/webassets.yml | 1 + ckanext/charts/chart_builders/base.py | 82 ++++- ckanext/charts/chart_builders/chartjs.py | 5 + ckanext/charts/helpers.py | 16 +- .../scheming/form_snippets/chart_filter.html | 18 ++ .../scheming/form_snippets/chart_select.html | 4 +- ckanext/charts/theme/charts.scss | 47 +++ ckanext/charts/views.py | 24 +- 14 files changed, 519 insertions(+), 42 deletions(-) create mode 100644 ckanext/charts/assets/js/charts-filters.js create mode 100644 ckanext/charts/templates/scheming/form_snippets/chart_filter.html diff --git a/ckanext/charts/assets/css/charts.css b/ckanext/charts/assets/css/charts.css index 2e3416e..f944c83 100644 --- a/ckanext/charts/assets/css/charts.css +++ b/ckanext/charts/assets/css/charts.css @@ -1 +1 @@ -.row.wrapper.charts-view .dataset-resource-form{position:relative;padding:0}.row.wrapper.charts-view label.no-dots:after{display:none}.row.wrapper.charts-view #chart-container{max-height:600px} \ No newline at end of file +.row.wrapper.charts-view .dataset-resource-form{position:relative;padding:0}.row.wrapper.charts-view label.no-dots:after{display:none}.row.wrapper.charts-view #chart-container{max-height:600px}.charts-filters .filter-container{margin-bottom:1rem}.charts-filters .filter-container .filter-pair{display:flex;gap:.5rem;margin-bottom:1rem;padding-bottom:1rem;border-bottom:1px solid beige}.charts-filters .filter-container .filter-pair:last-of-type{margin-bottom:0;padding-bottom:0;border-bottom:unset}.charts-filters .filter-container .filter-pair .remove-pair{top:0;flex-basis:10%;height:fit-content}.charts-filters .filter-container .filter-pair .ts-wrapper{flex-basis:45%}.charts-view--form .ts-wrapper .ts-control .clear-button{right:1rem}.charts-view--form .ts-wrapper .ts-dropdown{top:unset;margin-top:1px} \ No newline at end of file diff --git a/ckanext/charts/assets/js/charts-filters.js b/ckanext/charts/assets/js/charts-filters.js new file mode 100644 index 0000000..82676bb --- /dev/null +++ b/ckanext/charts/assets/js/charts-filters.js @@ -0,0 +1,293 @@ +ckan.module("charts-filters", function ($, _) { + "use strict"; + + return { + const: { + cls: { + FILTER_PAIR: 'filter-pair', + REMOVE_PAIR: 'remove-pair', + ADD_FILTER_BTN: 'add-filter-btn', + FILTER_CONTAINER: 'filter-container', + SELECT_COLUMN: 'column-selector', + SELECT_VALUE: 'value-selector', + FILTER_INPUT: 'filters-input' + }, + PAIR_DIVIDER: '|', + KEY_VALUE_DIVIDER: ':', + + }, + options: { + resourceId: null, + columns: null + }, + initialize: function () { + $.proxyAll(this, /_/); + + if (!this.options.resourceId || !this.options.columns) { + console.error("Resource ID and columns are required"); + return; + } + + this.filterInput = $(`.${this.const.cls.FILTER_INPUT}`); + + // Add event listeners + $(`.${this.const.cls.ADD_FILTER_BTN}`).on("click", this._addFilterPair); + + // Add event listeners on dynamic elements + $('body').on('click', `.${this.const.cls.REMOVE_PAIR}`, this._removeFilterPair); + + // On init + this._recreateFilterPairs(); + }, + + /** + * Recreate the filter pairs based on the input value. + * When we're initializing the chart form, we need to recreate the + * filter pairs based on the field that stores the filter params. + */ + _recreateFilterPairs: function () { + const parsedData = this._decodeFilterParams(); + + if (!parsedData) { + this._addFilterPair(); + } + + for (const [key, value] of Object.entries(parsedData)) { + this._addFilterPair(key, value); + }; + }, + + /** + * Add a new filter pair + */ + _addFilterPair: function (column, value) { + var filterPair = $('
').attr({ + class: this.const.cls.FILTER_PAIR, + }); + + var selectIdx = this._countFilterPairs() + 1; + + var fieldColumnSelect = $('').attr({ + id: `chart-column-${selectIdx}`, + name: `chart-column-${selectIdx}`, + class: this.const.cls.SELECT_COLUMN, + filterIndex: selectIdx, + }); + + var fieldValueSelect = $('').attr({ + id: `chart-value-${selectIdx}`, + name: `chart-value-${selectIdx}`, + class: this.const.cls.SELECT_VALUE, + multiple: true, + disabled: true, + filterIndex: selectIdx, + }); + + var removeButton = $('').attr({ + class: 'btn btn-primary remove-pair', + href: '#' + }).text('Remove'); + + filterPair.append(fieldColumnSelect); + filterPair.append(fieldValueSelect); + filterPair.append(removeButton); + + $(`.${this.const.cls.FILTER_CONTAINER}`).append(filterPair); + + this._initColumnSelector(fieldColumnSelect, column); + this._initValueSelector(fieldValueSelect, value); + }, + + /** + * Initialize the column selector for a given select element + * + * @param {JQuery} selectEl + * @param {String} value + */ + _initColumnSelector: function (selectEl, value) { + let control = new TomSelect(selectEl, { + valueField: 'id', + labelField: 'title', + searchField: 'title', + options: this.options.columns, + maxOptions: null, + plugins: { + 'input_autogrow': {}, + 'clear_button': { + 'title': 'Remove all selected options', + } + }, + onChange: (value) => { + this._changeValueSelectorState(selectEl.attr("filterindex"), value); + } + }); + + if (value) { + control.setValue(value, true); + }; + }, + + _initValueSelector: function (selectEl, value) { + let control = new TomSelect(selectEl, { + valueField: 'id', + labelField: 'title', + searchField: 'title', + maxOptions: null, + placeholder: "Add more...", + plugins: { + 'remove_button': {}, + 'input_autogrow': {} + }, + onChange: (_) => { + this._encodeFilterParams(); + } + }); + + let colSelector = selectEl.parent().find("select.column-selector"); + + if (colSelector.val()) { + this._initValueSelectorOptions(control, colSelector.val(), value); + } + }, + + /** + * Remove a filter pair + * + * @param {Event} e + */ + _removeFilterPair: function (e) { + if (this._countFilterPairs() == 1) { + alert('At least one filter pair is required'); + return; + } + + e.target.parentElement.remove() + + this._recalculateColumnSelectorIndexes(); + + // we want to recalculate the filter params after removing a pair + this._encodeFilterParams(); + + // trigger a change event on the tab content to update the chart + htmx.trigger($(".tab-content").get(0), "change") + }, + + /** + * Count the number of filter pairs + * + * @returns {Number} The number of filter pairs + */ + _countFilterPairs: function () { + return $(`.${this.const.cls.FILTER_PAIR}`).length; + }, + + /** + * Change the state of the value selector based on the column selector value + * + * @param {Number} index + * @param {String} column_value + */ + _changeValueSelectorState: function (index, column_value) { + let element = $(`select.value-selector[filterindex=${index}]`).get(0); + + element.tomselect.clear(); + element.tomselect.clearOptions(); + + if (!!column_value) { + element.tomselect.enable(); + this._initValueSelectorOptions(element.tomselect, column_value); + } else { + element.tomselect.disable(); + } + }, + + _initValueSelectorOptions: function (control, column, value) { + control.enable() + + $.ajax({ + url: this.sandbox.url(`/api/utils/charts/get-values`), + data: { "resource_id": this.options.resourceId, "column": column }, + success: (options) => { + console.log(options); + let selectValue = value; + + for (let index = 0; index < options.length; index++) { + control.addOption({ + id: options[index], + title: options[index], + }); + } + + if (selectValue) { + control.setValue(selectValue); + } + } + }); + }, + + /** + * Encode the filter params to be stored in the input value + */ + _encodeFilterParams: function () { + let pairs = []; + + $(`.${this.const.cls.FILTER_PAIR}`).each((_, element) => { + let columnSelector = element.querySelector(`select.${this.const.cls.SELECT_COLUMN}`); + let valueSelector = element.querySelector(`select.${this.const.cls.SELECT_VALUE}`); + + valueSelector.tomselect.getValue().forEach(value => { + pairs.push(`${columnSelector.tomselect.getValue()}${this.const.KEY_VALUE_DIVIDER}${value}`); + }); + }); + + this._updateFilterInput(pairs.join(this.const.PAIR_DIVIDER)); + }, + + /** + * Decode the filter params from the input value + * + * @returns {Object} The parsed filter params + */ + _decodeFilterParams: function () { + let filterVal = this.filterInput.val(); + + if (!filterVal) { + return {}; + } + + const keyValuePairs = filterVal.split(this.const.PAIR_DIVIDER); + + const parsedData = {}; + + keyValuePairs.forEach(pair => { + const [key, value] = pair.split(this.const.KEY_VALUE_DIVIDER); + + if (parsedData[key]) { + parsedData[key].push(value); + } else { + parsedData[key] = [value]; + } + }); + return parsedData; + }, + + /** + * Update the hidden input value + * + * @param {String} value + */ + _updateFilterInput: function (value) { + this.filterInput.val(value); + }, + + _recalculateColumnSelectorIndexes: function () { + $(`select.${this.const.cls.SELECT_COLUMN}`).each((index, element) => { + element.setAttribute("filterindex", index + 1); + }); + + $(`select.${this.const.cls.SELECT_VALUE}`).each((index, element) => { + element.setAttribute("filterindex", index + 1); + }); + } + }; +}); diff --git a/ckanext/charts/assets/js/charts-global.js b/ckanext/charts/assets/js/charts-global.js index 090838f..cecf2e7 100644 --- a/ckanext/charts/assets/js/charts-global.js +++ b/ckanext/charts/assets/js/charts-global.js @@ -5,20 +5,14 @@ ckan.module("charts-global", function ($, _) { initialize: function () { $.proxyAll(this, /_/); - // this hack is to dispatch a change event when a select2 element is selected - // so the HTMX could update the chart - // this.el.find('select').on('change', function (e) { - // htmx.trigger($(this).closest('.tab-content').get(0), "change") - // }); - - // initialize CKAN modules for HTMX loaded pages - htmx.on("htmx:afterSettle", function (event) { - var elements = event.target.querySelectorAll("[data-module]"); + // // initialize CKAN modules for HTMX loaded pages + // htmx.on("htmx:afterSettle", function (event) { + // var elements = event.target.querySelectorAll("[data-module]"); - for (let node of elements) { - ckan.module.initializeElement(node); - } - }); + // for (let node of elements) { + // ckan.module.initializeElement(node); + // } + // }); } }; }); diff --git a/ckanext/charts/assets/js/charts-render-chartjs.js b/ckanext/charts/assets/js/charts-render-chartjs.js index ac13448..faff9f5 100644 --- a/ckanext/charts/assets/js/charts-render-chartjs.js +++ b/ckanext/charts/assets/js/charts-render-chartjs.js @@ -13,7 +13,7 @@ ckan.module("charts-render-chartjs", function ($, _) { window.charts_chartjs.destroy(); } - console.log(this.options.config); + console.debug(this.options.config); if (!this.options.config) { console.error("No configuration provided"); diff --git a/ckanext/charts/assets/js/charts-render-observable.js b/ckanext/charts/assets/js/charts-render-observable.js index fba1681..ae05286 100644 --- a/ckanext/charts/assets/js/charts-render-observable.js +++ b/ckanext/charts/assets/js/charts-render-observable.js @@ -68,7 +68,6 @@ function PieChart(data, { fontSize = 12 // font size of labels } = {}) { // Compute values. - console.log(strokeWidth); const N = d3.map(data, (data) => data[names]); const V = d3.map(data, (data) => data[values]); diff --git a/ckanext/charts/assets/js/charts-select.js b/ckanext/charts/assets/js/charts-select.js index 7267b3c..00b3f37 100644 --- a/ckanext/charts/assets/js/charts-select.js +++ b/ckanext/charts/assets/js/charts-select.js @@ -5,28 +5,44 @@ ckan.module("charts-select", function ($, _) { "use strict"; return { + options: { + multiple: false, + clearButton: false, + maxOptions: null + }, initialize: function () { $.proxyAll(this, /_/); - let selectEl = this.el.find("select")[0]; + let selectEl = this.el[0]; - if (selectEl.tomselect) { + if (this.el[0].tomselect) { selectEl.tomselect.destroy(); } - new TomSelect(selectEl, { - plugins: { - 'checkbox_options': { - 'checkedClassNames': ['ts-checked'], - 'uncheckedClassNames': ['ts-unchecked'], - }, - 'remove_button': {}, - 'clear_button': { - 'title': 'Remove all selected options', - } - }, - maxItems: selectEl.getAttribute("maxitems") || null, - }); + var config = { + plugins: {}, + maxOptions: this.options.maxOptions, + placeholder: "Add more..." + } + + if (this.options.clearButton) { + config.plugins.clear_button = { + title: this.options.clearButton + }; + } + + if (this.options.multiple) { + config.plugins.checkbox_options = { + checkedClassNames: ['ts-checked'], + uncheckedClassNames: ['ts-unchecked'], + }; + + config.plugins.remove_button = {}; + + config.maxItems = selectEl.getAttribute("maxitems") || null; + } + console.log(config); + new TomSelect(selectEl, config); } }; }); diff --git a/ckanext/charts/assets/webassets.yml b/ckanext/charts/assets/webassets.yml index 1b79b96..3f954aa 100644 --- a/ckanext/charts/assets/webassets.yml +++ b/ckanext/charts/assets/webassets.yml @@ -35,6 +35,7 @@ charts-js: contents: - js/charts-global.js - js/charts-select.js + - js/charts-filters.js extra: preload: - base/main diff --git a/ckanext/charts/chart_builders/base.py b/ckanext/charts/chart_builders/base.py index 1b952a3..50cd76a 100644 --- a/ckanext/charts/chart_builders/base.py +++ b/ckanext/charts/chart_builders/base.py @@ -13,6 +13,33 @@ from ckanext.charts import fetchers +class FilterDecoder: + def __init__( + self, filter_input: str, pair_divider: str = "|", key_value_divider: str = ":" + ): + self.filter_input = filter_input + self.pair_divider = pair_divider + self.key_value_divider = key_value_divider + + def decode_filter_params(self) -> dict[str, list[str | int | float]]: + if not self.filter_input: + return {} + + key_value_pairs = self.filter_input.split(self.pair_divider) + + parsed_data: dict[str, list[str | int | float]] = {} + + for pair in key_value_pairs: + key, value = pair.split(self.key_value_divider) + + if key in parsed_data: + parsed_data[key].append(value) + else: + parsed_data[key] = [value] + + return parsed_data + + class BaseChartBuilder(ABC): def __init__( self, @@ -22,6 +49,26 @@ def __init__( self.df = dataframe self.settings = settings + if filter_input := self.settings.pop("filter", None): + filter_decoder = FilterDecoder(filter_input) + filter_params = filter_decoder.decode_filter_params() + + filtered_df = self.df.copy() + + for column, values in filter_params.items(): + column_type = filtered_df[column].convert_dtypes().dtype.type + + # TODO: requires more work here... + # I'm not sure about other types, that column can have + if column_type == np.int64: + values = [int(value) for value in values] + elif column_type == np.float64: + values = [float(value) for value in values] + + filtered_df = filtered_df[filtered_df[column].isin(values)] + + self.df = filtered_df + if self.settings.pop("sort_x", False): self.df.sort_values(by=self.settings["x"], inplace=True) @@ -136,7 +183,18 @@ def get_form_fields(self) -> list[dict[str, Any]]: dataset schema fields.""" def get_form_tabs(self) -> list[str]: - return ["General", "Structure", "Data", "Styles"] + result = [] + + for field in self.get_form_fields(): + if "group" not in field: + continue + + if field["group"] in result: + continue + + result.append(field["group"]) + + return result def get_expanded_form_fields(self): """Expands the presets.""" @@ -243,7 +301,7 @@ def engine_field(self) -> dict[str, Any]: return { "field_name": "engine", "label": "Engine", - "preset": "select", + "form_snippet": "chart_select.html", "required": True, "choices": tk.h.get_available_chart_engines_options(), "group": "Structure", @@ -263,7 +321,7 @@ def type_field(self, choices: list[dict[str, str]]) -> dict[str, Any]: return { "field_name": "type", "label": "Type", - "preset": "select", + "form_snippet": "chart_select.html", "required": True, "choices": choices, "group": "Structure", @@ -276,6 +334,7 @@ def type_field(self, choices: list[dict[str, str]]) -> dict[str, Any]: "hx-trigger": "change", "hx-include": "closest form", "hx-target": ".charts-view--form", + "data-module-clear-button": True, }, } @@ -283,7 +342,7 @@ def x_axis_field(self, choices: list[dict[str, str]]) -> dict[str, Any]: return { "field_name": "x", "label": "X Axis", - "preset": "select", + "form_snippet": "chart_select.html", "required": True, "choices": choices, "group": "Data", @@ -329,8 +388,8 @@ def y_multi_axis_field( ], "output_validators": [self.get_validator("not_empty")], "form_attrs": { - "multiple ": "1", "class": "tom-select", + "data-module-multiple": "true", }, "help_text": help_text, } @@ -534,3 +593,16 @@ def size_field(self, choices: list[dict[str, str]]) -> dict[str, Any]: field.update({"field_name": "size", "label": "Size", "group": "Structure"}) return field + + def filter_field(self, choices: list[dict[str, str]]) -> dict[str, Any]: + return { + "field_name": "filter", + "label": "Filter", + "form_snippet": "chart_filter.html", + "choices": choices, + "validators": [ + self.get_validator("ignore_empty"), + self.get_validator("unicode_safe"), + ], + "group": "Filter", + } diff --git a/ckanext/charts/chart_builders/chartjs.py b/ckanext/charts/chart_builders/chartjs.py index 7f9e687..22490eb 100644 --- a/ckanext/charts/chart_builders/chartjs.py +++ b/ckanext/charts/chart_builders/chartjs.py @@ -85,6 +85,7 @@ def get_form_fields(self): self.sort_x_field(), self.sort_y_field(), self.limit_field(), + self.filter_field(columns) ] @@ -144,6 +145,7 @@ def get_form_fields(self): self.sort_x_field(), self.sort_y_field(), self.limit_field(), + self.filter_field(columns) ] @@ -197,6 +199,7 @@ def get_form_fields(self): self.values_field(columns), self.names_field(columns), self.limit_field(), + self.filter_field(columns) ] @@ -257,6 +260,7 @@ def get_form_fields(self): self.sort_x_field(), self.sort_y_field(), self.limit_field(), + self.filter_field(columns) ] @@ -376,4 +380,5 @@ def get_form_fields(self): help_text="Select 3 or more different categorical variables (dimensions)", ), self.limit_field(), + self.filter_field(columns) ] diff --git a/ckanext/charts/helpers.py b/ckanext/charts/helpers.py index d80243b..9df2aee 100644 --- a/ckanext/charts/helpers.py +++ b/ckanext/charts/helpers.py @@ -1,8 +1,12 @@ from __future__ import annotations +import json + from ckanext.charts import utils from ckanext.charts.cache import count_file_cache_size, count_redis_cache_size from ckanext.charts import config +from ckanext.charts.fetchers import DatastoreDataFetcher +from ckanext.charts.chart_builders import get_chart_engines def get_redis_cache_size(): @@ -17,10 +21,18 @@ def get_file_cache_size(): def get_available_chart_engines_options(): """Get the available chart engines.""" - from ckanext.charts.chart_builders import get_chart_engines - return [{"value": engine, "text": engine} for engine in get_chart_engines()] + def charts_include_htmx_asset(): """Include HTMX asset if enabled.""" return config.include_htmx_asset() + + +def charts_get_resource_columns(resource_id: str) -> str: + """Get the columns of the given resource.""" + fetcher = DatastoreDataFetcher(resource_id) + + return json.dumps( + [{"id": col, "title": col} for col in fetcher.fetch_data().columns] + ) diff --git a/ckanext/charts/templates/scheming/form_snippets/chart_filter.html b/ckanext/charts/templates/scheming/form_snippets/chart_filter.html new file mode 100644 index 0000000..e70b54e --- /dev/null +++ b/ckanext/charts/templates/scheming/form_snippets/chart_filter.html @@ -0,0 +1,18 @@ +{#} HACK, pass resource ID more explicitly #} +{% set resource_id = data.__extras.resource_id %} + +
+ +
+
+ + + +
+
diff --git a/ckanext/charts/templates/scheming/form_snippets/chart_select.html b/ckanext/charts/templates/scheming/form_snippets/chart_select.html index dec8f54..951394d 100644 --- a/ckanext/charts/templates/scheming/form_snippets/chart_select.html +++ b/ckanext/charts/templates/scheming/form_snippets/chart_select.html @@ -39,8 +39,8 @@ {{ field_label or _('Custom') }} -
- {% for option in options %} {% endfor %} diff --git a/ckanext/charts/theme/charts.scss b/ckanext/charts/theme/charts.scss index 5ccf553..d1fd952 100644 --- a/ckanext/charts/theme/charts.scss +++ b/ckanext/charts/theme/charts.scss @@ -18,3 +18,50 @@ max-height: 600px; } } + + +.charts-filters { + .filter-container { + margin-bottom: 1rem; + + .filter-pair { + display: flex; + gap: 0.5rem; + margin-bottom: 1rem; + padding-bottom: 1rem; + border-bottom: 1px solid beige; + + &:last-of-type { + margin-bottom: 0; + padding-bottom: 0; + border-bottom: unset; + } + + .remove-pair { + top: 0; + flex-basis: 10%; + height: fit-content; + } + + .ts-wrapper { + flex-basis: 45%; + } + } + } +} + +.charts-view--form { + .ts-wrapper { + .ts-control { + // position select clear button + .clear-button { + right: 1rem; + } + } + + .ts-dropdown { + top: unset; + margin-top: 1px; + } + } +} diff --git a/ckanext/charts/views.py b/ckanext/charts/views.py index 7addde4..2d1ef9a 100644 --- a/ckanext/charts/views.py +++ b/ckanext/charts/views.py @@ -1,13 +1,13 @@ from __future__ import annotations -from flask import Blueprint +from flask import Blueprint, jsonify from flask.views import MethodView import ckan.plugins.toolkit as tk from ckan.logic import parse_params from ckan.plugins import plugin_loaded -from ckanext.charts import cache, exception, utils +from ckanext.charts import cache, exception, utils, fetchers charts = Blueprint("charts_view", __name__) @@ -101,6 +101,26 @@ def _get_form_builder(data: dict): return builder(data["resource_id"]) +@charts.route("/api/utils/charts/get-values") +def get_chart_column_values(): + data = parse_params(tk.request.args) + + resource_id = tk.get_or_bust(data, "resource_id") + column = tk.get_or_bust(data, "column") + + fetcher = fetchers.DatastoreDataFetcher(resource_id) + + result = [] + + for val in fetcher.fetch_data()[column].tolist(): + if val in result: + continue + + result.append(val) + + return jsonify(sorted(result)) + + if plugin_loaded("admin_panel"): from ckanext.ap_main.utils import ap_before_request from ckanext.ap_main.views.generics import ApConfigurationPageView