Performance improvements: use multiSearchAny instead of IN to sup…

…port Clickhouse projections for logs volumes (#1052)
grafana · Dec 6, 2024 · 7027569 · 7027569
1 parent 02f9b52
commit 7027569
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 17 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### Fixes
+
+- Optimized performance for log volumes processing using ClickHouse `multiSearchAny`
+
 ## 4.5.1
 
 ### Fixes

diff --git a/src/data/CHDatasource.test.ts b/src/data/CHDatasource.test.ts
@@ -477,13 +477,13 @@ describe('ClickHouseDatasource', () => {
         const result = datasource.getSupplementaryLogsVolumeQuery(request, query);
         expect(result?.rawSql).toEqual(
           `SELECT toStartOfInterval("created_at", INTERVAL 1 DAY) as "time", ` +
-            `sum(toString("level") IN ('critical','fatal','crit','alert','emerg','CRITICAL','FATAL','CRIT','ALERT','EMERG','Critical','Fatal','Crit','Alert','Emerg')) as critical, ` +
-            `sum(toString("level") IN ('error','err','eror','ERROR','ERR','EROR','Error','Err','Eror')) as error, ` +
-            `sum(toString("level") IN ('warn','warning','WARN','WARNING','Warn','Warning')) as warn, ` +
-            `sum(toString("level") IN ('info','information','informational','INFO','INFORMATION','INFORMATIONAL','Info','Information','Informational')) as info, ` +
-            `sum(toString("level") IN ('debug','dbug','DEBUG','DBUG','Debug','Dbug')) as debug, ` +
-            `sum(toString("level") IN ('trace','TRACE','Trace')) as trace, ` +
-            `sum(toString("level") IN ('unknown','UNKNOWN','Unknown')) as unknown ` +
+            `sum(multiSearchAny(toString("level"), ['critical','fatal','crit','alert','emerg','CRITICAL','FATAL','CRIT','ALERT','EMERG','Critical','Fatal','Crit','Alert','Emerg'])) as critical, ` +
+            `sum(multiSearchAny(toString("level"), ['error','err','eror','ERROR','ERR','EROR','Error','Err','Eror'])) as error, ` +
+            `sum(multiSearchAny(toString("level"), ['warn','warning','WARN','WARNING','Warn','Warning'])) as warn, ` +
+            `sum(multiSearchAny(toString("level"), ['info','information','informational','INFO','INFORMATION','INFORMATIONAL','Info','Information','Informational'])) as info, ` +
+            `sum(multiSearchAny(toString("level"), ['debug','dbug','DEBUG','DBUG','Debug','Dbug'])) as debug, ` +
+            `sum(multiSearchAny(toString("level"), ['trace','TRACE','Trace'])) as trace, ` +
+            `sum(multiSearchAny(toString("level"), ['unknown','UNKNOWN','Unknown'])) as unknown ` +
             `FROM "default"."logs" ` +
             `GROUP BY time ` +
             `ORDER BY time ASC`

diff --git a/src/data/CHDatasource.ts b/src/data/CHDatasource.ts
@@ -157,7 +157,7 @@ export class Datasource
       const llf = `toString("${logLevelColumn.name}")`;
       let level: keyof typeof LOG_LEVEL_TO_IN_CLAUSE;
       for (level in LOG_LEVEL_TO_IN_CLAUSE) {
-        aggregates.push({ aggregateType: AggregateType.Sum, column: `${llf} ${LOG_LEVEL_TO_IN_CLAUSE[level]}`, alias: level });
+        aggregates.push({ aggregateType: AggregateType.Sum, column: `multiSearchAny(${llf}, [${LOG_LEVEL_TO_IN_CLAUSE[level]}])`, alias: level });
       }
     } else {
       // Count all logs if level column isn't selected

diff --git a/src/data/logs.test.ts b/src/data/logs.test.ts
@@ -226,13 +226,13 @@ describe('logs', () => {
     it('should generate correct IN clauses', async () => {
       expect(LOG_LEVEL_TO_IN_CLAUSE).toEqual({
         critical:
-          "IN ('critical','fatal','crit','alert','emerg','CRITICAL','FATAL','CRIT','ALERT','EMERG','Critical','Fatal','Crit','Alert','Emerg')",
-        debug: "IN ('debug','dbug','DEBUG','DBUG','Debug','Dbug')",
-        error: "IN ('error','err','eror','ERROR','ERR','EROR','Error','Err','Eror')",
-        info: "IN ('info','information','informational','INFO','INFORMATION','INFORMATIONAL','Info','Information','Informational')",
-        trace: "IN ('trace','TRACE','Trace')",
-        unknown: "IN ('unknown','UNKNOWN','Unknown')",
-        warn: "IN ('warn','warning','WARN','WARNING','Warn','Warning')",
+          "'critical','fatal','crit','alert','emerg','CRITICAL','FATAL','CRIT','ALERT','EMERG','Critical','Fatal','Crit','Alert','Emerg'",
+        debug: "'debug','dbug','DEBUG','DBUG','Debug','Dbug'",
+        error: "'error','err','eror','ERROR','ERR','EROR','Error','Err','Eror'",
+        info: "'info','information','informational','INFO','INFORMATION','INFORMATIONAL','Info','Information','Informational'",
+        trace: "'trace','TRACE','Trace'",
+        unknown: "'unknown','UNKNOWN','Unknown'",
+        warn: "'warn','warning','WARN','WARNING','Warn','Warning'",
       });
     });
   });

diff --git a/src/data/logs.ts b/src/data/logs.ts
@@ -245,11 +245,11 @@ export const LOG_LEVEL_TO_IN_CLAUSE: LogLevelToInClause = (() => {
     unknown: ['unknown'],
   };
   return (Object.keys(levels) as Array<keyof typeof levels>).reduce((allLevels, level) => {
-    allLevels[level] = `IN (${[
+    allLevels[level] = `${[
       ...levels[level].map((l) => `'${l}'`),
       ...levels[level].map((l) => `'${l.toUpperCase()}'`),
       ...levels[level].map((l) => `'${l.charAt(0).toUpperCase() + l.slice(1)}'`),
-    ].join(',')})`;
+    ].join(',')}`;
     return allLevels;
   }, {} as LogLevelToInClause);
 })();