ad hoc from database or table (#67)

* ad hoc from database or table
grafana · Feb 10, 2022 · 59dcbaa · 59dcbaa
1 parent 39844fc
commit 59dcbaa
Show file tree

Hide file tree

Showing 6 changed files with 229 additions and 42 deletions.
diff --git a/README.md b/README.md
@@ -123,7 +123,11 @@ Imported dashboards can be found in Configuration > Data Sources > select your C
 
 ### Using Ad-Hoc Filters
 
-A second helper variable must be created in addition to the standard ad-hoc filter type variable of any name. It should be a `constant` type named `clickhouse_adhoc_query` and contain a valid ClickHouse query. The query results will be used to populate your ad-hoc filter's selectable filters. You may choose to hide this variable from view as it serves no further purpose.
+By default, Ad-Hoc filters will be populated with all Tables and Columns.  If you have a default database defined in the Datasource settings, all Tables from that database will be used to populate the filters. As this could be slow/expensive, you can introduce a second variable to allow limiting the Ad-Hoc filters. It should be a `constant` type named `clickhouse_adhoc_query` and can contain: a comma delimited list of databases, just one database, or a database.table combination to show only columns for a single table.
+
+#### Using a query for Ad-Hoc filters
+
+The second `clickhouse_adhoc_query` also allows any valid Clickhouse query. The query results will be used to populate your ad-hoc filter's selectable filters. You may choose to hide this variable from view as it serves no further purpose.
 
 If `clickhouse_adhoc_query` is set to `SELECT DISTINCT machine_name FROM mgbench.logs1` you would be able to select which machine names are filtered for in the dashboard.
 

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "clickhouse-datasource",
-  "version": "0.9.13",
+  "version": "0.10.0",
   "description": "Clickhouse Datasource",
   "scripts": {
     "build": "grafana-toolkit plugin:build",

diff --git a/src/data/CHDatasource.test.ts b/src/data/CHDatasource.test.ts
@@ -1,20 +1,21 @@
-import { toDataFrame } from '@grafana/data';
+import { ArrayDataFrame, toDataFrame } from '@grafana/data';
 import { of } from 'rxjs';
 import { mockDatasource } from '__mocks__/datasource';
 import { CHQuery, QueryType } from 'types';
+import { cloneDeep } from 'lodash';
 
 interface InstanceConfig {
   queryResponse: {} | [];
 }
 
-const templateSrvMock = { replace: jest.fn(), getVariables: jest.fn() };
+const templateSrvMock = { replace: jest.fn(), getVariables: jest.fn(), getAdhocFilters: jest.fn() };
 jest.mock('@grafana/runtime', () => ({
   ...(jest.requireActual('@grafana/runtime') as unknown as object),
   getTemplateSrv: () => templateSrvMock,
 }));
 
 const createInstance = ({ queryResponse }: Partial<InstanceConfig> = {}) => {
-  const instance = mockDatasource;
+  const instance = cloneDeep(mockDatasource);
   jest.spyOn(instance, 'query').mockImplementation((request) => of({ data: [toDataFrame(queryResponse ?? [])] }));
   return instance;
 };
@@ -56,4 +57,95 @@ describe('ClickHouseDatasource', () => {
       expect(val).toEqual({ rawSql, queryType: QueryType.SQL });
     });
   });
+
+  describe('Tag Keys', () => {
+    it('should Fetch Default Tags When No Second AdHoc Variable', async () => {
+      const spyOnReplace = jest.spyOn(templateSrvMock, 'replace').mockImplementation(() => '$clickhouse_adhoc_query');
+      const ds = cloneDeep(mockDatasource);
+      ds.settings.jsonData.defaultDatabase = undefined;
+      const frame = new ArrayDataFrame([{ name: 'foo', type: 'string', table: 'table' }]);
+      const spyOnQuery = jest.spyOn(ds, 'query').mockImplementation((request) => of({ data: [frame] }));
+
+      const keys = await ds.getTagKeys();
+      expect(spyOnReplace).toHaveBeenCalled();
+      const expected = { rawSql: 'SELECT name, type, table FROM system.columns' };
+
+      expect(spyOnQuery).toHaveBeenCalledWith(
+        expect.objectContaining({ targets: expect.arrayContaining([expect.objectContaining(expected)]) })
+      );
+
+      expect(keys).toEqual([{ text: 'table.foo' }]);
+    });
+
+    it('should Fetch Tags With Default Database', async () => {
+      const spyOnReplace = jest.spyOn(templateSrvMock, 'replace').mockImplementation(() => '$clickhouse_adhoc_query');
+      const frame = new ArrayDataFrame([{ name: 'foo', type: 'string', table: 'table' }]);
+      const ds = cloneDeep(mockDatasource);
+      const spyOnQuery = jest.spyOn(ds, 'query').mockImplementation((request) => of({ data: [frame] }));
+
+      const keys = await ds.getTagKeys();
+      expect(spyOnReplace).toHaveBeenCalled();
+      const expected = { rawSql: "SELECT name, type, table FROM system.columns WHERE database IN ('foo')" };
+
+      expect(spyOnQuery).toHaveBeenCalledWith(
+        expect.objectContaining({ targets: expect.arrayContaining([expect.objectContaining(expected)]) })
+      );
+
+      expect(keys).toEqual([{ text: 'table.foo' }]);
+    });
+
+    it('should Fetch Tags From Query', async () => {
+      const spyOnReplace = jest.spyOn(templateSrvMock, 'replace').mockImplementation(() => 'select name from foo');
+      const frame = new ArrayDataFrame([{ name: 'foo' }]);
+      const ds = cloneDeep(mockDatasource);
+      const spyOnQuery = jest.spyOn(ds, 'query').mockImplementation((request) => of({ data: [frame] }));
+
+      const keys = await ds.getTagKeys();
+      expect(spyOnReplace).toHaveBeenCalled();
+      const expected = { rawSql: 'select name from foo' };
+
+      expect(spyOnQuery).toHaveBeenCalledWith(
+        expect.objectContaining({ targets: expect.arrayContaining([expect.objectContaining(expected)]) })
+      );
+
+      expect(keys).toEqual([{ text: 'name' }]);
+    });
+  });
+
+  describe('Tag Values', () => {
+    it('should Fetch Tag Values from Schema', async () => {
+      const spyOnReplace = jest.spyOn(templateSrvMock, 'replace').mockImplementation(() => '$clickhouse_adhoc_query');
+      const ds = cloneDeep(mockDatasource);
+      ds.settings.jsonData.defaultDatabase = undefined;
+      const frame = new ArrayDataFrame([{ bar: 'foo' }]);
+      const spyOnQuery = jest.spyOn(ds, 'query').mockImplementation((request) => of({ data: [frame] }));
+
+      const values = await ds.getTagValues({ key: 'foo.bar' });
+      expect(spyOnReplace).toHaveBeenCalled();
+      const expected = { rawSql: 'select distinct bar from foo limit 1000' };
+
+      expect(spyOnQuery).toHaveBeenCalledWith(
+        expect.objectContaining({ targets: expect.arrayContaining([expect.objectContaining(expected)]) })
+      );
+
+      expect(values).toEqual([{ text: 'foo' }]);
+    });
+
+    it('should Fetch Tag Values from Query', async () => {
+      const spyOnReplace = jest.spyOn(templateSrvMock, 'replace').mockImplementation(() => 'select name from bar');
+      const ds = cloneDeep(mockDatasource);
+      const frame = new ArrayDataFrame([{ name: 'foo' }]);
+      const spyOnQuery = jest.spyOn(ds, 'query').mockImplementation((request) => of({ data: [frame] }));
+
+      const values = await ds.getTagValues({ key: 'name' });
+      expect(spyOnReplace).toHaveBeenCalled();
+      const expected = { rawSql: 'select name from bar' };
+
+      expect(spyOnQuery).toHaveBeenCalledWith(
+        expect.objectContaining({ targets: expect.arrayContaining([expect.objectContaining(expected)]) })
+      );
+
+      expect(values).toEqual([{ text: 'foo' }]);
+    });
+  });
 });
diff --git a/src/data/CHDatasource.ts b/src/data/CHDatasource.ts
@@ -1,5 +1,4 @@
 import {
-  ArrayDataFrame,
   DataFrame,
   DataFrameView,
   DataQueryRequest,
@@ -9,24 +8,22 @@ import {
   ScopedVars,
   vectorator,
 } from '@grafana/data';
-import { DataSourceWithBackend, getTemplateSrv, TemplateSrv } from '@grafana/runtime';
+import { DataSourceWithBackend, getTemplateSrv } from '@grafana/runtime';
 import { CHConfig, CHQuery, FullField, QueryType } from '../types';
 import { AdHocFilter } from './adHocFilter';
-import { isString } from 'lodash';
+import { isString, isEmpty } from 'lodash';
 import { removeConditionalAlls } from './removeConditionalAlls';
 
 export class Datasource extends DataSourceWithBackend<CHQuery, CHConfig> {
   // This enables default annotation support for 7.2+
   annotations = {};
   settings: DataSourceInstanceSettings<CHConfig>;
-  templateSrv: TemplateSrv;
   adHocFilter: AdHocFilter;
-  skipAdHocFilter = false;
+  skipAdHocFilter = false; // don't apply adhoc filters to the query
 
   constructor(instanceSettings: DataSourceInstanceSettings<CHConfig>) {
     super(instanceSettings);
     this.settings = instanceSettings;
-    this.templateSrv = getTemplateSrv();
     this.adHocFilter = new AdHocFilter();
   }
 
@@ -55,25 +52,27 @@ export class Datasource extends DataSourceWithBackend<CHQuery, CHConfig> {
   applyTemplateVariables(query: CHQuery, scoped: ScopedVars): CHQuery {
     let rawQuery = query.rawSql || '';
     // we want to skip applying ad hoc filters when we are getting values for ad hoc filters
+    const templateSrv = getTemplateSrv();
     if (!this.skipAdHocFilter) {
-      rawQuery = this.adHocFilter.apply(rawQuery, (this.templateSrv as any)?.getAdhocFilters(this.name));
+      const adHocFilters = (templateSrv as any)?.getAdhocFilters(this.name);
+      rawQuery = this.adHocFilter.apply(rawQuery, adHocFilters);
     }
     this.skipAdHocFilter = false;
-    rawQuery = removeConditionalAlls(rawQuery, getTemplateSrv().getVariables());
+    rawQuery = removeConditionalAlls(rawQuery, templateSrv.getVariables());
     return {
       ...query,
       rawSql: this.replace(rawQuery, scoped) || '',
     };
   }
 
-  replace(value?: string, scopedVars?: ScopedVars) {
+  private replace(value?: string, scopedVars?: ScopedVars) {
     if (value !== undefined) {
       return getTemplateSrv().replace(value, scopedVars, this.format);
     }
     return value;
   }
 
-  format(value: any) {
+  private format(value: any) {
     if (Array.isArray(value)) {
       return `'${value.join("','")}'`;
     }
@@ -117,12 +116,12 @@ export class Datasource extends DataSourceWithBackend<CHQuery, CHConfig> {
     }));
   }
 
-  async fetchData(rawSql: string) {
+  private async fetchData(rawSql: string) {
     const frame = await this.runQuery({ rawSql });
     return this.values(frame);
   }
 
-  runQuery(request: Partial<CHQuery>): Promise<DataFrame> {
+  private runQuery(request: Partial<CHQuery>): Promise<DataFrame> {
     return new Promise((resolve) => {
       const req = {
         targets: [{ ...request, refId: String(Math.random()) }],
@@ -133,20 +132,53 @@ export class Datasource extends DataSourceWithBackend<CHQuery, CHConfig> {
     });
   }
 
-  values(frame: DataFrame) {
+  private values(frame: DataFrame) {
     if (frame.fields?.length === 0) {
       return [];
     }
     return vectorator(frame?.fields[0]?.values).map((text) => text);
   }
 
   async getTagKeys(): Promise<MetricFindValue[]> {
-    const frame = await this.fetchTags();
-    return frame.fields.map((f) => ({ text: f.name }));
+    const { type, frame } = await this.fetchTags();
+    if (type === TagType.query) {
+      return frame.fields.map((f) => ({ text: f.name }));
+    }
+    const view = new DataFrameView(frame);
+    return view.map((item) => ({
+      text: `${item[2]}.${item[0]}`,
+    }));
   }
 
   async getTagValues({ key }: any): Promise<MetricFindValue[]> {
-    const frame = await this.fetchTags();
+    const { type } = this.getTagSource();
+    this.skipAdHocFilter = true;
+    if (type === TagType.query) {
+      return this.fetchTagValuesFromQuery(key);
+    }
+    return this.fetchTagValuesFromSchema(key);
+  }
+
+  private async fetchTagValuesFromSchema(key: string): Promise<MetricFindValue[]> {
+    const { from } = this.getTagSource();
+    const [table, col] = key.split('.');
+    const source = from?.includes('.') ? `${from.split('.')[0]}.${table}` : table;
+    const rawSql = `select distinct ${col} from ${source} limit 1000`;
+    const frame = await this.runQuery({ rawSql });
+    if (frame.fields?.length === 0) {
+      return [];
+    }
+    const field = frame.fields[0];
+    // Convert to string to avoid https://github.com/grafana/grafana/issues/12209
+    return vectorator(field.values)
+      .filter((value) => value !== null)
+      .map((value) => {
+        return { text: String(value) };
+      });
+  }
+
+  private async fetchTagValuesFromQuery(key: string): Promise<MetricFindValue[]> {
+    const { frame } = await this.fetchTags();
     const field = frame.fields.find((f) => f.name === key);
     if (field) {
       // Convert to string to avoid https://github.com/grafana/grafana/issues/12209
@@ -159,15 +191,59 @@ export class Datasource extends DataSourceWithBackend<CHQuery, CHConfig> {
     return [];
   }
 
-  async fetchTags(): Promise<DataFrame> {
-    // @todo https://github.com/grafana/grafana/issues/13109
-    const rawSql = this.templateSrv.replace('$clickhouse_adhoc_query');
-    if (rawSql === '$clickhouse_adhoc_query') {
-      return new ArrayDataFrame([]);
+  private async fetchTags(): Promise<Tags> {
+    const tagSource = this.getTagSource();
+    this.skipAdHocFilter = true;
+
+    if (tagSource.source === undefined) {
+      this.adHocFilter.setTargetTable('default');
+      const rawSql = 'SELECT name, type, table FROM system.columns';
+      const results = await this.runQuery({ rawSql });
+      return { type: TagType.schema, frame: results };
+    }
+
+    if (tagSource.type === TagType.query) {
+      this.adHocFilter.setTargetTableFromQuery(tagSource.source);
     } else {
-      this.skipAdHocFilter = true;
-      this.adHocFilter.setTargetTable(rawSql);
-      return await this.runQuery({ rawSql });
+      let table = tagSource.from;
+      if (table?.includes('.')) {
+        table = table.split('.')[1];
+      }
+      this.adHocFilter.setTargetTable(table || '');
+    }
+
+    const results = await this.runQuery({ rawSql: tagSource.source });
+    return { type: tagSource.type, frame: results };
+  }
+
+  private getTagSource() {
+    // @todo https://github.com/grafana/grafana/issues/13109
+    const ADHOC_VAR = '$clickhouse_adhoc_query';
+    const defaultDatabase = this.getDefaultDatabase();
+    let source = getTemplateSrv().replace(ADHOC_VAR);
+    if (source === ADHOC_VAR && isEmpty(defaultDatabase)) {
+      return { type: TagType.schema, source: undefined };
+    }
+    source = source === ADHOC_VAR ? defaultDatabase! : source;
+    if (source.toLowerCase().startsWith('select')) {
+      return { type: TagType.query, source };
     }
+    if (!source.includes('.')) {
+      const sql = `SELECT name, type, table FROM system.columns WHERE database IN ('${source}')`;
+      return { type: TagType.schema, source: sql, from: source };
+    }
+    const [db, table] = source.split('.');
+    const sql = `SELECT name, type, table FROM system.columns WHERE database IN ('${db}') AND table = '${table}'`;
+    return { type: TagType.schema, source: sql, from: source };
   }
 }
+
+enum TagType {
+  query,
+  schema,
+}
+
+interface Tags {
+  type?: TagType;
+  frame: DataFrame;
+}