From 7df142b02b31afee1576ce55ae887ea6aa9b8695 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Wed, 16 Aug 2023 12:12:45 -0700 Subject: [PATCH 1/4] reorder getfunnel where to match index --- queries/analytics/reports/getFunnel.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/queries/analytics/reports/getFunnel.ts b/queries/analytics/reports/getFunnel.ts index 3c5c65e09c..1bbbc8782b 100644 --- a/queries/analytics/reports/getFunnel.ts +++ b/queries/analytics/reports/getFunnel.ts @@ -57,12 +57,14 @@ async function relationalQuery( from level${i} l join website_event we on l.session_id = we.session_id - where we.created_at between l.created_at - and ${getAddMinutesQuery(`l.created_at `, windowMinutes)} + where we.website_id = {{websiteId::uuid}} + and we.created_at between l.created_at and ${getAddMinutesQuery( + `l.created_at `, + windowMinutes, + )} and we.referrer_path = {{${i - 1}}} and we.url_path = {{${i}}} and we.created_at <= {{endDate}} - and we.website_id = {{websiteId::uuid}} )`; } From d6c8f3aa18bdcfe6a46c2397819ddb45424cb03c Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Wed, 16 Aug 2023 13:47:43 -0700 Subject: [PATCH 2/4] fix mapfilter / rawquery for relational --- lib/clickhouse.ts | 2 +- lib/prisma.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/clickhouse.ts b/lib/clickhouse.ts index aa2f21ed3b..8ce7bc980c 100644 --- a/lib/clickhouse.ts +++ b/lib/clickhouse.ts @@ -66,7 +66,7 @@ function getDateFormat(date) { function mapFilter(column, operator, name, type = 'String') { switch (operator) { case OPERATORS.equals: - return `${column} = {${name}:${type}`; + return `${column} = {${name}:${type}}`; case OPERATORS.notEquals: return `${column} != {${name}:${type}}`; default: diff --git a/lib/prisma.ts b/lib/prisma.ts index cebd71932d..12bafa5136 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -92,12 +92,12 @@ function getTimestampIntervalQuery(field: string): string { } } -function mapFilter(column, operator, name, type = 'String') { +function mapFilter(column, operator, name, type = 'varchar') { switch (operator) { case OPERATORS.equals: - return `${column} = {${name}:${type}`; + return `${column} = {{${name}::${type}}}`; case OPERATORS.notEquals: - return `${column} != {${name}:${type}}`; + return `${column} != {{${name}::${type}}}`; default: return ''; } @@ -161,7 +161,7 @@ async function rawQuery(sql: string, data: object): Promise { return Promise.reject(new Error('Unknown database.')); } - const query = sql?.replaceAll(/\{\{\s*(\w+)(::\w+)?\s*}}/g, (...args) => { + const query = sql?.replaceAll(/\{\{\s*(\w+)(::\w+)?\s*\}\}/g, (...args) => { const [, name, type] = args; params.push(data[name]); From 146650586b3cbd4d484e321c15d45c7c4baa0109 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Wed, 16 Aug 2023 13:56:41 -0700 Subject: [PATCH 3/4] add normalize filters to clickhouse --- lib/clickhouse.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/clickhouse.ts b/lib/clickhouse.ts index 8ce7bc980c..bc10a6d4e7 100644 --- a/lib/clickhouse.ts +++ b/lib/clickhouse.ts @@ -94,13 +94,23 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) return query.join('\n'); } +function normalizeFilters(filters = {}) { + return Object.keys(filters).reduce((obj, key) => { + const value = filters[key]; + + obj[key] = value?.value ?? value; + + return obj; + }, {}); +} + async function parseFilters(websiteId: string, filters: QueryFilters = {}, options?: QueryOptions) { const website = await loadWebsite(websiteId); return { filterQuery: getFilterQuery(filters, options), params: { - ...filters, + ...normalizeFilters(filters), websiteId, startDate: maxDate(filters.startDate, website.resetAt), websiteDomain: website.domain, From 6dba68c823df80302fc261b3a5d854a660d7cade Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Wed, 16 Aug 2023 14:33:55 -0700 Subject: [PATCH 4/4] add metric performance indexes --- .../03_metric_performance_index/migration.sql | 50 +++++++++++++++++++ db/mysql/schema.prisma | 17 +++++++ .../03_metric_performance_index/migration.sql | 50 +++++++++++++++++++ db/postgresql/schema.prisma | 17 +++++++ 4 files changed, 134 insertions(+) create mode 100644 db/mysql/migrations/03_metric_performance_index/migration.sql create mode 100644 db/postgresql/migrations/03_metric_performance_index/migration.sql diff --git a/db/mysql/migrations/03_metric_performance_index/migration.sql b/db/mysql/migrations/03_metric_performance_index/migration.sql new file mode 100644 index 0000000000..6468136479 --- /dev/null +++ b/db/mysql/migrations/03_metric_performance_index/migration.sql @@ -0,0 +1,50 @@ +-- CreateIndex +CREATE INDEX `event_data_website_id_created_at_idx` ON `event_data`(`website_id`, `created_at`); + +-- CreateIndex +CREATE INDEX `event_data_website_id_created_at_event_key_idx` ON `event_data`(`website_id`, `created_at`, `event_key`); + +-- CreateIndex +CREATE INDEX `session_website_id_created_at_idx` ON `session`(`website_id`, `created_at`); + +-- CreateIndex +CREATE INDEX `session_website_id_created_at_hostname_idx` ON `session`(`website_id`, `created_at`, `hostname`); + +-- CreateIndex +CREATE INDEX `session_website_id_created_at_browser_idx` ON `session`(`website_id`, `created_at`, `browser`); + +-- CreateIndex +CREATE INDEX `session_website_id_created_at_os_idx` ON `session`(`website_id`, `created_at`, `os`); + +-- CreateIndex +CREATE INDEX `session_website_id_created_at_device_idx` ON `session`(`website_id`, `created_at`, `device`); + +-- CreateIndex +CREATE INDEX `session_website_id_created_at_screen_idx` ON `session`(`website_id`, `created_at`, `screen`); + +-- CreateIndex +CREATE INDEX `session_website_id_created_at_language_idx` ON `session`(`website_id`, `created_at`, `language`); + +-- CreateIndex +CREATE INDEX `session_website_id_created_at_country_idx` ON `session`(`website_id`, `created_at`, `country`); + +-- CreateIndex +CREATE INDEX `session_website_id_created_at_subdivision1_idx` ON `session`(`website_id`, `created_at`, `subdivision1`); + +-- CreateIndex +CREATE INDEX `session_website_id_created_at_city_idx` ON `session`(`website_id`, `created_at`, `city`); + +-- CreateIndex +CREATE INDEX `website_event_website_id_created_at_url_path_idx` ON `website_event`(`website_id`, `created_at`, `url_path`); + +-- CreateIndex +CREATE INDEX `website_event_website_id_created_at_url_query_idx` ON `website_event`(`website_id`, `created_at`, `url_query`); + +-- CreateIndex +CREATE INDEX `website_event_website_id_created_at_referrer_domain_idx` ON `website_event`(`website_id`, `created_at`, `referrer_domain`); + +-- CreateIndex +CREATE INDEX `website_event_website_id_created_at_page_title_idx` ON `website_event`(`website_id`, `created_at`, `page_title`); + +-- CreateIndex +CREATE INDEX `website_event_website_id_created_at_event_name_idx` ON `website_event`(`website_id`, `created_at`, `event_name`); diff --git a/db/mysql/schema.prisma b/db/mysql/schema.prisma index a25405dffa..38bb91f4be 100644 --- a/db/mysql/schema.prisma +++ b/db/mysql/schema.prisma @@ -44,6 +44,16 @@ model Session { @@index([createdAt]) @@index([websiteId]) + @@index([websiteId, createdAt]) + @@index([websiteId, createdAt, hostname]) + @@index([websiteId, createdAt, browser]) + @@index([websiteId, createdAt, os]) + @@index([websiteId, createdAt, device]) + @@index([websiteId, createdAt, screen]) + @@index([websiteId, createdAt, language]) + @@index([websiteId, createdAt, country]) + @@index([websiteId, createdAt, subdivision1]) + @@index([websiteId, createdAt, city]) @@map("session") } @@ -91,6 +101,11 @@ model WebsiteEvent { @@index([sessionId]) @@index([websiteId]) @@index([websiteId, createdAt]) + @@index([websiteId, createdAt, urlPath]) + @@index([websiteId, createdAt, urlQuery]) + @@index([websiteId, createdAt, referrerDomain]) + @@index([websiteId, createdAt, pageTitle]) + @@index([websiteId, createdAt, eventName]) @@index([websiteId, sessionId, createdAt]) @@map("website_event") } @@ -113,6 +128,8 @@ model EventData { @@index([websiteId]) @@index([websiteEventId]) @@index([websiteId, websiteEventId, createdAt]) + @@index([websiteId, createdAt]) + @@index([websiteId, createdAt, eventKey]) @@map("event_data") } diff --git a/db/postgresql/migrations/03_metric_performance_index/migration.sql b/db/postgresql/migrations/03_metric_performance_index/migration.sql new file mode 100644 index 0000000000..5db7aa50c8 --- /dev/null +++ b/db/postgresql/migrations/03_metric_performance_index/migration.sql @@ -0,0 +1,50 @@ +-- CreateIndex +CREATE INDEX "event_data_website_id_created_at_idx" ON "event_data"("website_id", "created_at"); + +-- CreateIndex +CREATE INDEX "event_data_website_id_created_at_event_key_idx" ON "event_data"("website_id", "created_at", "event_key"); + +-- CreateIndex +CREATE INDEX "session_website_id_created_at_idx" ON "session"("website_id", "created_at"); + +-- CreateIndex +CREATE INDEX "session_website_id_created_at_hostname_idx" ON "session"("website_id", "created_at", "hostname"); + +-- CreateIndex +CREATE INDEX "session_website_id_created_at_browser_idx" ON "session"("website_id", "created_at", "browser"); + +-- CreateIndex +CREATE INDEX "session_website_id_created_at_os_idx" ON "session"("website_id", "created_at", "os"); + +-- CreateIndex +CREATE INDEX "session_website_id_created_at_device_idx" ON "session"("website_id", "created_at", "device"); + +-- CreateIndex +CREATE INDEX "session_website_id_created_at_screen_idx" ON "session"("website_id", "created_at", "screen"); + +-- CreateIndex +CREATE INDEX "session_website_id_created_at_language_idx" ON "session"("website_id", "created_at", "language"); + +-- CreateIndex +CREATE INDEX "session_website_id_created_at_country_idx" ON "session"("website_id", "created_at", "country"); + +-- CreateIndex +CREATE INDEX "session_website_id_created_at_subdivision1_idx" ON "session"("website_id", "created_at", "subdivision1"); + +-- CreateIndex +CREATE INDEX "session_website_id_created_at_city_idx" ON "session"("website_id", "created_at", "city"); + +-- CreateIndex +CREATE INDEX "website_event_website_id_created_at_url_path_idx" ON "website_event"("website_id", "created_at", "url_path"); + +-- CreateIndex +CREATE INDEX "website_event_website_id_created_at_url_query_idx" ON "website_event"("website_id", "created_at", "url_query"); + +-- CreateIndex +CREATE INDEX "website_event_website_id_created_at_referrer_domain_idx" ON "website_event"("website_id", "created_at", "referrer_domain"); + +-- CreateIndex +CREATE INDEX "website_event_website_id_created_at_page_title_idx" ON "website_event"("website_id", "created_at", "page_title"); + +-- CreateIndex +CREATE INDEX "website_event_website_id_created_at_event_name_idx" ON "website_event"("website_id", "created_at", "event_name"); diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma index 5753c6ef7e..d7a70ab055 100644 --- a/db/postgresql/schema.prisma +++ b/db/postgresql/schema.prisma @@ -44,6 +44,16 @@ model Session { @@index([createdAt]) @@index([websiteId]) + @@index([websiteId, createdAt]) + @@index([websiteId, createdAt, hostname]) + @@index([websiteId, createdAt, browser]) + @@index([websiteId, createdAt, os]) + @@index([websiteId, createdAt, device]) + @@index([websiteId, createdAt, screen]) + @@index([websiteId, createdAt, language]) + @@index([websiteId, createdAt, country]) + @@index([websiteId, createdAt, subdivision1]) + @@index([websiteId, createdAt, city]) @@map("session") } @@ -91,6 +101,11 @@ model WebsiteEvent { @@index([sessionId]) @@index([websiteId]) @@index([websiteId, createdAt]) + @@index([websiteId, createdAt, urlPath]) + @@index([websiteId, createdAt, urlQuery]) + @@index([websiteId, createdAt, referrerDomain]) + @@index([websiteId, createdAt, pageTitle]) + @@index([websiteId, createdAt, eventName]) @@index([websiteId, sessionId, createdAt]) @@map("website_event") } @@ -112,6 +127,8 @@ model EventData { @@index([createdAt]) @@index([websiteId]) @@index([websiteEventId]) + @@index([websiteId, createdAt]) + @@index([websiteId, createdAt, eventKey]) @@map("event_data") }