diff --git a/README.md b/README.md index d94b9a60..fd13a372 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,11 @@ Imported dashboards can be found in Configuration > Data Sources > select your C ### Using Ad-Hoc Filters -A second helper variable must be created in addition to the standard ad-hoc filter type variable of any name. It should be a `constant` type named `clickhouse_adhoc_query` and contain a valid ClickHouse query. The query results will be used to populate your ad-hoc filter's selectable filters. You may choose to hide this variable from view as it serves no further purpose. +By default, Ad-Hoc filters will be populated with all Tables and Columns. If you have a default database defined in the Datasource settings, all Tables from that database will be used to populate the filters. As this could be slow/expensive, you can introduce a second variable to allow limiting the Ad-Hoc filters. It should be a `constant` type named `clickhouse_adhoc_query` and can contain: a comma delimited list of databases, just one database, or a database.table combination to show only columns for a single table. + +#### Using a query for Ad-Hoc filters + +The second `clickhouse_adhoc_query` also allows any valid Clickhouse query. The query results will be used to populate your ad-hoc filter's selectable filters. You may choose to hide this variable from view as it serves no further purpose. If `clickhouse_adhoc_query` is set to `SELECT DISTINCT machine_name FROM mgbench.logs1` you would be able to select which machine names are filtered for in the dashboard. diff --git a/package.json b/package.json index 47901626..e6ed45d9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clickhouse-datasource", - "version": "0.9.13", + "version": "0.10.0", "description": "Clickhouse Datasource", "scripts": { "build": "grafana-toolkit plugin:build", diff --git a/src/data/CHDatasource.test.ts b/src/data/CHDatasource.test.ts index 63bbb846..40feab48 100644 --- a/src/data/CHDatasource.test.ts +++ b/src/data/CHDatasource.test.ts @@ -1,20 +1,21 @@ -import { toDataFrame } from '@grafana/data'; +import { ArrayDataFrame, toDataFrame } from '@grafana/data'; import { of } from 'rxjs'; import { mockDatasource } from '__mocks__/datasource'; import { CHQuery, QueryType } from 'types'; +import { cloneDeep } from 'lodash'; interface InstanceConfig { queryResponse: {} | []; } -const templateSrvMock = { replace: jest.fn(), getVariables: jest.fn() }; +const templateSrvMock = { replace: jest.fn(), getVariables: jest.fn(), getAdhocFilters: jest.fn() }; jest.mock('@grafana/runtime', () => ({ ...(jest.requireActual('@grafana/runtime') as unknown as object), getTemplateSrv: () => templateSrvMock, })); const createInstance = ({ queryResponse }: Partial = {}) => { - const instance = mockDatasource; + const instance = cloneDeep(mockDatasource); jest.spyOn(instance, 'query').mockImplementation((request) => of({ data: [toDataFrame(queryResponse ?? [])] })); return instance; }; @@ -56,4 +57,95 @@ describe('ClickHouseDatasource', () => { expect(val).toEqual({ rawSql, queryType: QueryType.SQL }); }); }); + + describe('Tag Keys', () => { + it('should Fetch Default Tags When No Second AdHoc Variable', async () => { + const spyOnReplace = jest.spyOn(templateSrvMock, 'replace').mockImplementation(() => '$clickhouse_adhoc_query'); + const ds = cloneDeep(mockDatasource); + ds.settings.jsonData.defaultDatabase = undefined; + const frame = new ArrayDataFrame([{ name: 'foo', type: 'string', table: 'table' }]); + const spyOnQuery = jest.spyOn(ds, 'query').mockImplementation((request) => of({ data: [frame] })); + + const keys = await ds.getTagKeys(); + expect(spyOnReplace).toHaveBeenCalled(); + const expected = { rawSql: 'SELECT name, type, table FROM system.columns' }; + + expect(spyOnQuery).toHaveBeenCalledWith( + expect.objectContaining({ targets: expect.arrayContaining([expect.objectContaining(expected)]) }) + ); + + expect(keys).toEqual([{ text: 'table.foo' }]); + }); + + it('should Fetch Tags With Default Database', async () => { + const spyOnReplace = jest.spyOn(templateSrvMock, 'replace').mockImplementation(() => '$clickhouse_adhoc_query'); + const frame = new ArrayDataFrame([{ name: 'foo', type: 'string', table: 'table' }]); + const ds = cloneDeep(mockDatasource); + const spyOnQuery = jest.spyOn(ds, 'query').mockImplementation((request) => of({ data: [frame] })); + + const keys = await ds.getTagKeys(); + expect(spyOnReplace).toHaveBeenCalled(); + const expected = { rawSql: "SELECT name, type, table FROM system.columns WHERE database IN ('foo')" }; + + expect(spyOnQuery).toHaveBeenCalledWith( + expect.objectContaining({ targets: expect.arrayContaining([expect.objectContaining(expected)]) }) + ); + + expect(keys).toEqual([{ text: 'table.foo' }]); + }); + + it('should Fetch Tags From Query', async () => { + const spyOnReplace = jest.spyOn(templateSrvMock, 'replace').mockImplementation(() => 'select name from foo'); + const frame = new ArrayDataFrame([{ name: 'foo' }]); + const ds = cloneDeep(mockDatasource); + const spyOnQuery = jest.spyOn(ds, 'query').mockImplementation((request) => of({ data: [frame] })); + + const keys = await ds.getTagKeys(); + expect(spyOnReplace).toHaveBeenCalled(); + const expected = { rawSql: 'select name from foo' }; + + expect(spyOnQuery).toHaveBeenCalledWith( + expect.objectContaining({ targets: expect.arrayContaining([expect.objectContaining(expected)]) }) + ); + + expect(keys).toEqual([{ text: 'name' }]); + }); + }); + + describe('Tag Values', () => { + it('should Fetch Tag Values from Schema', async () => { + const spyOnReplace = jest.spyOn(templateSrvMock, 'replace').mockImplementation(() => '$clickhouse_adhoc_query'); + const ds = cloneDeep(mockDatasource); + ds.settings.jsonData.defaultDatabase = undefined; + const frame = new ArrayDataFrame([{ bar: 'foo' }]); + const spyOnQuery = jest.spyOn(ds, 'query').mockImplementation((request) => of({ data: [frame] })); + + const values = await ds.getTagValues({ key: 'foo.bar' }); + expect(spyOnReplace).toHaveBeenCalled(); + const expected = { rawSql: 'select distinct bar from foo limit 1000' }; + + expect(spyOnQuery).toHaveBeenCalledWith( + expect.objectContaining({ targets: expect.arrayContaining([expect.objectContaining(expected)]) }) + ); + + expect(values).toEqual([{ text: 'foo' }]); + }); + + it('should Fetch Tag Values from Query', async () => { + const spyOnReplace = jest.spyOn(templateSrvMock, 'replace').mockImplementation(() => 'select name from bar'); + const ds = cloneDeep(mockDatasource); + const frame = new ArrayDataFrame([{ name: 'foo' }]); + const spyOnQuery = jest.spyOn(ds, 'query').mockImplementation((request) => of({ data: [frame] })); + + const values = await ds.getTagValues({ key: 'name' }); + expect(spyOnReplace).toHaveBeenCalled(); + const expected = { rawSql: 'select name from bar' }; + + expect(spyOnQuery).toHaveBeenCalledWith( + expect.objectContaining({ targets: expect.arrayContaining([expect.objectContaining(expected)]) }) + ); + + expect(values).toEqual([{ text: 'foo' }]); + }); + }); }); diff --git a/src/data/CHDatasource.ts b/src/data/CHDatasource.ts index d6b01d11..1bf7dda0 100644 --- a/src/data/CHDatasource.ts +++ b/src/data/CHDatasource.ts @@ -1,5 +1,4 @@ import { - ArrayDataFrame, DataFrame, DataFrameView, DataQueryRequest, @@ -9,24 +8,22 @@ import { ScopedVars, vectorator, } from '@grafana/data'; -import { DataSourceWithBackend, getTemplateSrv, TemplateSrv } from '@grafana/runtime'; +import { DataSourceWithBackend, getTemplateSrv } from '@grafana/runtime'; import { CHConfig, CHQuery, FullField, QueryType } from '../types'; import { AdHocFilter } from './adHocFilter'; -import { isString } from 'lodash'; +import { isString, isEmpty } from 'lodash'; import { removeConditionalAlls } from './removeConditionalAlls'; export class Datasource extends DataSourceWithBackend { // This enables default annotation support for 7.2+ annotations = {}; settings: DataSourceInstanceSettings; - templateSrv: TemplateSrv; adHocFilter: AdHocFilter; - skipAdHocFilter = false; + skipAdHocFilter = false; // don't apply adhoc filters to the query constructor(instanceSettings: DataSourceInstanceSettings) { super(instanceSettings); this.settings = instanceSettings; - this.templateSrv = getTemplateSrv(); this.adHocFilter = new AdHocFilter(); } @@ -55,25 +52,27 @@ export class Datasource extends DataSourceWithBackend { applyTemplateVariables(query: CHQuery, scoped: ScopedVars): CHQuery { let rawQuery = query.rawSql || ''; // we want to skip applying ad hoc filters when we are getting values for ad hoc filters + const templateSrv = getTemplateSrv(); if (!this.skipAdHocFilter) { - rawQuery = this.adHocFilter.apply(rawQuery, (this.templateSrv as any)?.getAdhocFilters(this.name)); + const adHocFilters = (templateSrv as any)?.getAdhocFilters(this.name); + rawQuery = this.adHocFilter.apply(rawQuery, adHocFilters); } this.skipAdHocFilter = false; - rawQuery = removeConditionalAlls(rawQuery, getTemplateSrv().getVariables()); + rawQuery = removeConditionalAlls(rawQuery, templateSrv.getVariables()); return { ...query, rawSql: this.replace(rawQuery, scoped) || '', }; } - replace(value?: string, scopedVars?: ScopedVars) { + private replace(value?: string, scopedVars?: ScopedVars) { if (value !== undefined) { return getTemplateSrv().replace(value, scopedVars, this.format); } return value; } - format(value: any) { + private format(value: any) { if (Array.isArray(value)) { return `'${value.join("','")}'`; } @@ -117,12 +116,12 @@ export class Datasource extends DataSourceWithBackend { })); } - async fetchData(rawSql: string) { + private async fetchData(rawSql: string) { const frame = await this.runQuery({ rawSql }); return this.values(frame); } - runQuery(request: Partial): Promise { + private runQuery(request: Partial): Promise { return new Promise((resolve) => { const req = { targets: [{ ...request, refId: String(Math.random()) }], @@ -133,7 +132,7 @@ export class Datasource extends DataSourceWithBackend { }); } - values(frame: DataFrame) { + private values(frame: DataFrame) { if (frame.fields?.length === 0) { return []; } @@ -141,12 +140,45 @@ export class Datasource extends DataSourceWithBackend { } async getTagKeys(): Promise { - const frame = await this.fetchTags(); - return frame.fields.map((f) => ({ text: f.name })); + const { type, frame } = await this.fetchTags(); + if (type === TagType.query) { + return frame.fields.map((f) => ({ text: f.name })); + } + const view = new DataFrameView(frame); + return view.map((item) => ({ + text: `${item[2]}.${item[0]}`, + })); } async getTagValues({ key }: any): Promise { - const frame = await this.fetchTags(); + const { type } = this.getTagSource(); + this.skipAdHocFilter = true; + if (type === TagType.query) { + return this.fetchTagValuesFromQuery(key); + } + return this.fetchTagValuesFromSchema(key); + } + + private async fetchTagValuesFromSchema(key: string): Promise { + const { from } = this.getTagSource(); + const [table, col] = key.split('.'); + const source = from?.includes('.') ? `${from.split('.')[0]}.${table}` : table; + const rawSql = `select distinct ${col} from ${source} limit 1000`; + const frame = await this.runQuery({ rawSql }); + if (frame.fields?.length === 0) { + return []; + } + const field = frame.fields[0]; + // Convert to string to avoid https://github.com/grafana/grafana/issues/12209 + return vectorator(field.values) + .filter((value) => value !== null) + .map((value) => { + return { text: String(value) }; + }); + } + + private async fetchTagValuesFromQuery(key: string): Promise { + const { frame } = await this.fetchTags(); const field = frame.fields.find((f) => f.name === key); if (field) { // Convert to string to avoid https://github.com/grafana/grafana/issues/12209 @@ -159,15 +191,59 @@ export class Datasource extends DataSourceWithBackend { return []; } - async fetchTags(): Promise { - // @todo https://github.com/grafana/grafana/issues/13109 - const rawSql = this.templateSrv.replace('$clickhouse_adhoc_query'); - if (rawSql === '$clickhouse_adhoc_query') { - return new ArrayDataFrame([]); + private async fetchTags(): Promise { + const tagSource = this.getTagSource(); + this.skipAdHocFilter = true; + + if (tagSource.source === undefined) { + this.adHocFilter.setTargetTable('default'); + const rawSql = 'SELECT name, type, table FROM system.columns'; + const results = await this.runQuery({ rawSql }); + return { type: TagType.schema, frame: results }; + } + + if (tagSource.type === TagType.query) { + this.adHocFilter.setTargetTableFromQuery(tagSource.source); } else { - this.skipAdHocFilter = true; - this.adHocFilter.setTargetTable(rawSql); - return await this.runQuery({ rawSql }); + let table = tagSource.from; + if (table?.includes('.')) { + table = table.split('.')[1]; + } + this.adHocFilter.setTargetTable(table || ''); + } + + const results = await this.runQuery({ rawSql: tagSource.source }); + return { type: tagSource.type, frame: results }; + } + + private getTagSource() { + // @todo https://github.com/grafana/grafana/issues/13109 + const ADHOC_VAR = '$clickhouse_adhoc_query'; + const defaultDatabase = this.getDefaultDatabase(); + let source = getTemplateSrv().replace(ADHOC_VAR); + if (source === ADHOC_VAR && isEmpty(defaultDatabase)) { + return { type: TagType.schema, source: undefined }; + } + source = source === ADHOC_VAR ? defaultDatabase! : source; + if (source.toLowerCase().startsWith('select')) { + return { type: TagType.query, source }; } + if (!source.includes('.')) { + const sql = `SELECT name, type, table FROM system.columns WHERE database IN ('${source}')`; + return { type: TagType.schema, source: sql, from: source }; + } + const [db, table] = source.split('.'); + const sql = `SELECT name, type, table FROM system.columns WHERE database IN ('${db}') AND table = '${table}'`; + return { type: TagType.schema, source: sql, from: source }; } } + +enum TagType { + query, + schema, +} + +interface Tags { + type?: TagType; + frame: DataFrame; +} diff --git a/src/data/adHocFilter.test.ts b/src/data/adHocFilter.test.ts index 50366922..aa140a96 100644 --- a/src/data/adHocFilter.test.ts +++ b/src/data/adHocFilter.test.ts @@ -3,7 +3,7 @@ import { AdHocFilter, AdHocVariableFilter } from './adHocFilter'; describe('AdHocManager', () => { it('apply ad hoc filter with no inner query and existing WHERE', () => { const ahm = new AdHocFilter(); - ahm.setTargetTable('SELECT * FROM table'); + ahm.setTargetTableFromQuery('SELECT * FROM table'); const val = ahm.apply('SELECT stuff FROM table WHERE col = test', [ { key: 'key', operator: '=', value: 'val' }, { key: 'keyNum', operator: '=', value: '123' }, @@ -12,7 +12,7 @@ describe('AdHocManager', () => { }); it('apply ad hoc filter with no inner query and no existing WHERE', () => { const ahm = new AdHocFilter(); - ahm.setTargetTable('SELECT * FROM table'); + ahm.setTargetTableFromQuery('SELECT * FROM table'); const val = ahm.apply('SELECT stuff FROM table', [ { key: 'key', operator: '=', value: 'val' }, { key: 'keyNum', operator: '=', value: '123' }, @@ -21,7 +21,7 @@ describe('AdHocManager', () => { }); it('apply ad hoc filter with an inner query without existing WHERE', () => { const ahm = new AdHocFilter(); - ahm.setTargetTable('SELECT * FROM table'); + ahm.setTargetTableFromQuery('SELECT * FROM table'); const val = ahm.apply(`SELECT stuff FROM (SELECT * FROM table) as r, table2 GROUP BY s ORDER BY s`, [ { key: 'key', operator: '=', value: 'val' }, { key: 'keyNum', operator: '=', value: '123' }, @@ -32,7 +32,7 @@ describe('AdHocManager', () => { }); it('apply ad hoc filter with an inner from query with existing WHERE', () => { const ahm = new AdHocFilter(); - ahm.setTargetTable('SELECT * FROM table'); + ahm.setTargetTableFromQuery('SELECT * FROM table'); const val = ahm.apply(`SELECT stuff FROM (SELECT * FROM table WHERE col = test) as r GROUP BY s ORDER BY s`, [ { key: 'key', operator: '=', value: 'val' }, { key: 'keyNum', operator: '=', value: '123' }, @@ -43,7 +43,7 @@ describe('AdHocManager', () => { }); it('apply ad hoc filter with an inner where query with existing WHERE', () => { const ahm = new AdHocFilter(); - ahm.setTargetTable('SELECT * FROM table'); + ahm.setTargetTableFromQuery('SELECT * FROM table'); const val = ahm.apply( `SELECT * FROM table WHERE (name = stuff) AND (name IN ( SELECT * FROM table WHERE (field = 'hello') GROUP BY name ORDER BY count() DESC LIMIT 10 )) GROUP BY name, time ORDER BY time`, [{ key: 'key', operator: '=', value: 'val' }] as AdHocVariableFilter[] @@ -54,7 +54,7 @@ describe('AdHocManager', () => { }); it('does not apply ad hoc filter when the target table is not in the query', () => { const ahm = new AdHocFilter(); - ahm.setTargetTable('SELECT * FROM table2'); + ahm.setTargetTableFromQuery('SELECT * FROM table2'); const val = ahm.apply('select stuff from table', [ { key: 'key', operator: '=', value: 'val' }, ] as AdHocVariableFilter[]); @@ -62,7 +62,7 @@ describe('AdHocManager', () => { }); it('apply ad hoc filter when the ad hoc options are from a query with a from inline query', () => { const ahm = new AdHocFilter(); - ahm.setTargetTable('SELECT * FROM (select * from table)'); + ahm.setTargetTableFromQuery('SELECT * FROM (select * from table)'); const val = ahm.apply('select stuff from table', [ { key: 'key', operator: '=', value: 'val' }, ] as AdHocVariableFilter[]); @@ -70,7 +70,9 @@ describe('AdHocManager', () => { }); it('apply ad hoc filter when the ad hoc options are from a query with a where inline query', () => { const ahm = new AdHocFilter(); - ahm.setTargetTable('SELECT * FROM table where stuff = stuff and (repo in (select * from table)) order by stuff'); + ahm.setTargetTableFromQuery( + 'SELECT * FROM table where stuff = stuff and (repo in (select * from table)) order by stuff' + ); const val = ahm.apply('select stuff from table', [ { key: 'key', operator: '=', value: 'val' }, ] as AdHocVariableFilter[]); diff --git a/src/data/adHocFilter.ts b/src/data/adHocFilter.ts index 4c4953d6..ad6e66bd 100644 --- a/src/data/adHocFilter.ts +++ b/src/data/adHocFilter.ts @@ -4,12 +4,16 @@ import sqlToAST, { astToSql, AST, applyFiltersToAST } from './ast'; export class AdHocFilter { private _targetTable = ''; - setTargetTable(query: string) { + setTargetTable(table: string) { + this._targetTable = table; + } + + setTargetTableFromQuery(query: string) { const ast = sqlToAST(query); - this.setTargetTableFroAST(ast); + this.setTargetTableFromAST(ast); } - private setTargetTableFroAST(ast: AST) { + private setTargetTableFromAST(ast: AST) { if (!ast.get('FROM')) { return; } @@ -18,11 +22,20 @@ export class AdHocFilter { this._targetTable = from.trim().replace(/(\(|\)|,)/gi, ''); return; } - this.setTargetTableFroAST(from!); + if (from) { + this.setTargetTableFromAST(from); + } } apply(sql: string, adHocFilters: AdHocVariableFilter[]): string { - if (this._targetTable === '' || sql === '' || !adHocFilters || adHocFilters.length === 0) { + if (sql === '' || !adHocFilters || adHocFilters.length === 0) { + return sql; + } + const filter = adHocFilters[0]; + if (filter.key.includes('.')) { + this._targetTable = filter.key.split('.')[0]; + } + if (this._targetTable === '') { return sql; } let whereClause = '';