Merge pull request #2200 from umami-software/feat/um-311-relational-i…

…ndexing Feat/um 311 relational indexing
umami-software · Aug 17, 2023 · bd87858 · bd87858
2 parents 2e2b141 + 6dba68c
commit bd87858
Show file tree

Hide file tree

Showing 7 changed files with 155 additions and 9 deletions.
diff --git a/db/mysql/migrations/03_metric_performance_index/migration.sql b/db/mysql/migrations/03_metric_performance_index/migration.sql
@@ -0,0 +1,50 @@
+-- CreateIndex
+CREATE INDEX `event_data_website_id_created_at_idx` ON `event_data`(`website_id`, `created_at`);
+
+-- CreateIndex
+CREATE INDEX `event_data_website_id_created_at_event_key_idx` ON `event_data`(`website_id`, `created_at`, `event_key`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_idx` ON `session`(`website_id`, `created_at`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_hostname_idx` ON `session`(`website_id`, `created_at`, `hostname`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_browser_idx` ON `session`(`website_id`, `created_at`, `browser`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_os_idx` ON `session`(`website_id`, `created_at`, `os`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_device_idx` ON `session`(`website_id`, `created_at`, `device`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_screen_idx` ON `session`(`website_id`, `created_at`, `screen`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_language_idx` ON `session`(`website_id`, `created_at`, `language`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_country_idx` ON `session`(`website_id`, `created_at`, `country`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_subdivision1_idx` ON `session`(`website_id`, `created_at`, `subdivision1`);
+
+-- CreateIndex
+CREATE INDEX `session_website_id_created_at_city_idx` ON `session`(`website_id`, `created_at`, `city`);
+
+-- CreateIndex
+CREATE INDEX `website_event_website_id_created_at_url_path_idx` ON `website_event`(`website_id`, `created_at`, `url_path`);
+
+-- CreateIndex
+CREATE INDEX `website_event_website_id_created_at_url_query_idx` ON `website_event`(`website_id`, `created_at`, `url_query`);
+
+-- CreateIndex
+CREATE INDEX `website_event_website_id_created_at_referrer_domain_idx` ON `website_event`(`website_id`, `created_at`, `referrer_domain`);
+
+-- CreateIndex
+CREATE INDEX `website_event_website_id_created_at_page_title_idx` ON `website_event`(`website_id`, `created_at`, `page_title`);
+
+-- CreateIndex
+CREATE INDEX `website_event_website_id_created_at_event_name_idx` ON `website_event`(`website_id`, `created_at`, `event_name`);
diff --git a/db/mysql/schema.prisma b/db/mysql/schema.prisma
@@ -44,6 +44,16 @@ model Session {
 
   @@index([createdAt])
   @@index([websiteId])
+  @@index([websiteId, createdAt])
+  @@index([websiteId, createdAt, hostname])
+  @@index([websiteId, createdAt, browser])
+  @@index([websiteId, createdAt, os])
+  @@index([websiteId, createdAt, device])
+  @@index([websiteId, createdAt, screen])
+  @@index([websiteId, createdAt, language])
+  @@index([websiteId, createdAt, country])
+  @@index([websiteId, createdAt, subdivision1])
+  @@index([websiteId, createdAt, city])
   @@map("session")
 }
 
@@ -91,6 +101,11 @@ model WebsiteEvent {
   @@index([sessionId])
   @@index([websiteId])
   @@index([websiteId, createdAt])
+  @@index([websiteId, createdAt, urlPath])
+  @@index([websiteId, createdAt, urlQuery])
+  @@index([websiteId, createdAt, referrerDomain])
+  @@index([websiteId, createdAt, pageTitle])
+  @@index([websiteId, createdAt, eventName])
   @@index([websiteId, sessionId, createdAt])
   @@map("website_event")
 }
@@ -113,6 +128,8 @@ model EventData {
   @@index([websiteId])
   @@index([websiteEventId])
   @@index([websiteId, websiteEventId, createdAt])
+  @@index([websiteId, createdAt])
+  @@index([websiteId, createdAt, eventKey])
   @@map("event_data")
 }
 

diff --git a/db/postgresql/migrations/03_metric_performance_index/migration.sql b/db/postgresql/migrations/03_metric_performance_index/migration.sql
@@ -0,0 +1,50 @@
+-- CreateIndex
+CREATE INDEX "event_data_website_id_created_at_idx" ON "event_data"("website_id", "created_at");
+
+-- CreateIndex
+CREATE INDEX "event_data_website_id_created_at_event_key_idx" ON "event_data"("website_id", "created_at", "event_key");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_idx" ON "session"("website_id", "created_at");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_hostname_idx" ON "session"("website_id", "created_at", "hostname");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_browser_idx" ON "session"("website_id", "created_at", "browser");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_os_idx" ON "session"("website_id", "created_at", "os");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_device_idx" ON "session"("website_id", "created_at", "device");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_screen_idx" ON "session"("website_id", "created_at", "screen");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_language_idx" ON "session"("website_id", "created_at", "language");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_country_idx" ON "session"("website_id", "created_at", "country");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_subdivision1_idx" ON "session"("website_id", "created_at", "subdivision1");
+
+-- CreateIndex
+CREATE INDEX "session_website_id_created_at_city_idx" ON "session"("website_id", "created_at", "city");
+
+-- CreateIndex
+CREATE INDEX "website_event_website_id_created_at_url_path_idx" ON "website_event"("website_id", "created_at", "url_path");
+
+-- CreateIndex
+CREATE INDEX "website_event_website_id_created_at_url_query_idx" ON "website_event"("website_id", "created_at", "url_query");
+
+-- CreateIndex
+CREATE INDEX "website_event_website_id_created_at_referrer_domain_idx" ON "website_event"("website_id", "created_at", "referrer_domain");
+
+-- CreateIndex
+CREATE INDEX "website_event_website_id_created_at_page_title_idx" ON "website_event"("website_id", "created_at", "page_title");
+
+-- CreateIndex
+CREATE INDEX "website_event_website_id_created_at_event_name_idx" ON "website_event"("website_id", "created_at", "event_name");
diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma
@@ -44,6 +44,16 @@ model Session {
 
   @@index([createdAt])
   @@index([websiteId])
+  @@index([websiteId, createdAt])
+  @@index([websiteId, createdAt, hostname])
+  @@index([websiteId, createdAt, browser])
+  @@index([websiteId, createdAt, os])
+  @@index([websiteId, createdAt, device])
+  @@index([websiteId, createdAt, screen])
+  @@index([websiteId, createdAt, language])
+  @@index([websiteId, createdAt, country])
+  @@index([websiteId, createdAt, subdivision1])
+  @@index([websiteId, createdAt, city])
   @@map("session")
 }
 
@@ -91,6 +101,11 @@ model WebsiteEvent {
   @@index([sessionId])
   @@index([websiteId])
   @@index([websiteId, createdAt])
+  @@index([websiteId, createdAt, urlPath])
+  @@index([websiteId, createdAt, urlQuery])
+  @@index([websiteId, createdAt, referrerDomain])
+  @@index([websiteId, createdAt, pageTitle])
+  @@index([websiteId, createdAt, eventName])
   @@index([websiteId, sessionId, createdAt])
   @@map("website_event")
 }
@@ -112,6 +127,8 @@ model EventData {
   @@index([createdAt])
   @@index([websiteId])
   @@index([websiteEventId])
+  @@index([websiteId, createdAt])
+  @@index([websiteId, createdAt, eventKey])
   @@map("event_data")
 }
 

diff --git a/lib/clickhouse.ts b/lib/clickhouse.ts
@@ -66,7 +66,7 @@ function getDateFormat(date) {
 function mapFilter(column, operator, name, type = 'String') {
   switch (operator) {
     case OPERATORS.equals:
-      return `${column} = {${name}:${type}`;
+      return `${column} = {${name}:${type}}`;
     case OPERATORS.notEquals:
       return `${column} != {${name}:${type}}`;
     default:
@@ -94,13 +94,23 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {})
   return query.join('\n');
 }
 
+function normalizeFilters(filters = {}) {
+  return Object.keys(filters).reduce((obj, key) => {
+    const value = filters[key];
+
+    obj[key] = value?.value ?? value;
+
+    return obj;
+  }, {});
+}
+
 async function parseFilters(websiteId: string, filters: QueryFilters = {}, options?: QueryOptions) {
   const website = await loadWebsite(websiteId);
 
   return {
     filterQuery: getFilterQuery(filters, options),
     params: {
-      ...filters,
+      ...normalizeFilters(filters),
       websiteId,
       startDate: maxDate(filters.startDate, website.resetAt),
       websiteDomain: website.domain,

diff --git a/lib/prisma.ts b/lib/prisma.ts
@@ -92,12 +92,12 @@ function getTimestampIntervalQuery(field: string): string {
   }
 }
 
-function mapFilter(column, operator, name, type = 'String') {
+function mapFilter(column, operator, name, type = 'varchar') {
   switch (operator) {
     case OPERATORS.equals:
-      return `${column} = {${name}:${type}`;
+      return `${column} = {{${name}::${type}}}`;
     case OPERATORS.notEquals:
-      return `${column} != {${name}:${type}}`;
+      return `${column} != {{${name}::${type}}}`;
     default:
       return '';
   }
@@ -161,7 +161,7 @@ async function rawQuery(sql: string, data: object): Promise<any> {
     return Promise.reject(new Error('Unknown database.'));
   }
 
-  const query = sql?.replaceAll(/\{\{\s*(\w+)(::\w+)?\s*}}/g, (...args) => {
+  const query = sql?.replaceAll(/\{\{\s*(\w+)(::\w+)?\s*\}\}/g, (...args) => {
     const [, name, type] = args;
     params.push(data[name]);
 

diff --git a/queries/analytics/reports/getFunnel.ts b/queries/analytics/reports/getFunnel.ts
@@ -57,12 +57,14 @@ async function relationalQuery(
             from level${i} l
             join website_event we
                 on l.session_id = we.session_id
-            where we.created_at between l.created_at 
-                and ${getAddMinutesQuery(`l.created_at `, windowMinutes)}
+            where we.website_id = {{websiteId::uuid}}
+                and we.created_at between l.created_at and ${getAddMinutesQuery(
+                  `l.created_at `,
+                  windowMinutes,
+                )}
                 and we.referrer_path = {{${i - 1}}}
                 and we.url_path = {{${i}}}
                 and we.created_at <= {{endDate}}
-                and we.website_id = {{websiteId::uuid}}
           )`;
         }