diff --git a/src/__tests__/join-plan.test.ts b/src/__tests__/join-plan.test.ts new file mode 100644 index 0000000..681c413 --- /dev/null +++ b/src/__tests__/join-plan.test.ts @@ -0,0 +1,293 @@ +import * as semanticLayer from "../index.js"; + +import { assert, describe, it } from "vitest"; + +const aModel = semanticLayer + .model() + .withName("a") + .fromTable("a") + .withDimension("a_id", { + type: "number", + primaryKey: true, + sql: ({ model, sql }) => sql`${model.column("a_id")}`, + }); + +const bModel = semanticLayer + .model() + .withName("b") + .fromTable("b") + .withDimension("b_id", { + type: "number", + primaryKey: true, + sql: ({ model, sql }) => sql`${model.column("b_id")}`, + }) + .withDimension("a_id", { + type: "number", + sql: ({ model }) => model.column("a_id"), + }); + +const cModel = semanticLayer + .model() + .withName("c") + .fromTable("c") + .withDimension("c_id", { + type: "number", + primaryKey: true, + sql: ({ model, sql }) => sql`${model.column("c_id")}`, + }) + .withDimension("a_id", { + type: "number", + sql: ({ model }) => model.column("a_id"), + }); + +const dModel = semanticLayer + .model() + .withName("d") + .fromTable("d") + .withDimension("d_id", { + type: "number", + primaryKey: true, + sql: ({ model, sql }) => sql`${model.column("d_id")}`, + }) + .withDimension("b_id", { + type: "number", + sql: ({ model }) => model.column("b_id"), + }) + .withDimension("c_id", { + type: "number", + sql: ({ model }) => model.column("c_id"), + }); + +describe("join plan", () => { + it("can generate a join plan when no joins have priority set", () => { + const repository = semanticLayer + .repository() + .withModel(aModel) + .withModel(bModel) + .withModel(cModel) + .withModel(dModel) + .joinOneToOne( + "a", + "b", + ({ sql, models }) => + sql`${models.a.dimension("a_id")} = ${models.b.dimension("a_id")}`, + ) + .joinOneToMany( + "a", + "c", + ({ sql, models }) => + sql`${models.a.dimension("a_id")} = ${models.c.dimension("a_id")}`, + ) + .joinOneToOne( + "b", + "d", + ({ sql, models }) => + sql`${models.b.dimension("b_id")} = ${models.d.dimension("b_id")}`, + ) + .joinOneToMany( + "c", + "d", + ({ sql, models }) => + sql`${models.c.dimension("c_id")} = ${models.d.dimension("c_id")}`, + ); + + const queryBuilder = repository.build("postgresql"); + + const queryContext = new semanticLayer.QueryContext( + queryBuilder.repository, + queryBuilder.dialect, + undefined, + ); + const query: semanticLayer.AnyInputQuery = { + members: ["a.a_id", "b.b_id", "c.c_id", "d.d_id"], + }; + const queryPlan = queryBuilder.getQueryPlan(queryContext, undefined, query); + assert.deepEqual(queryPlan.segments[0]!.joinPlan, { + hasRowMultiplication: true, + initialModel: "a", + joins: [ + { + leftModel: "a", + rightModel: "c", + joinType: "left", + }, + { + leftModel: "a", + rightModel: "b", + joinType: "left", + }, + { + leftModel: "b", + rightModel: "d", + joinType: "left", + }, + ], + }); + }); + + it("can generate a join plan when joins have priority set", () => { + const repository = semanticLayer + .repository() + .withModel(aModel) + .withModel(bModel) + .withModel(cModel) + .withModel(dModel) + .joinOneToOne( + "a", + "b", + ({ sql, models }) => + sql`${models.a.dimension("a_id")} = ${models.b.dimension("a_id")}`, + ) + .joinOneToMany( + "a", + "c", + ({ sql, models }) => + sql`${models.a.dimension("a_id")} = ${models.c.dimension("a_id")}`, + ) + .joinOneToOne( + "b", + "d", + ({ sql, models }) => + sql`${models.b.dimension("b_id")} = ${models.d.dimension("b_id")}`, + ) + .joinOneToMany( + "c", + "d", + ({ sql, models }) => + sql`${models.c.dimension("c_id")} = ${models.d.dimension("c_id")}`, + { priority: "high" }, + ); + + const queryBuilder = repository.build("postgresql"); + + const queryContext = new semanticLayer.QueryContext( + queryBuilder.repository, + queryBuilder.dialect, + undefined, + ); + const query1: semanticLayer.AnyInputQuery = { + members: ["a.a_id", "b.b_id", "c.c_id", "d.d_id"], + }; + const queryPlan1 = queryBuilder.getQueryPlan( + queryContext, + undefined, + query1, + ); + + assert.deepEqual(queryPlan1.segments[0]!.joinPlan, { + hasRowMultiplication: false, + initialModel: "a", + joins: [ + { + leftModel: "a", + rightModel: "b", + joinType: "left", + }, + { + leftModel: "b", + rightModel: "d", + joinType: "left", + }, + { + leftModel: "d", + rightModel: "c", + joinType: "right", + }, + ], + }); + const query2: semanticLayer.AnyInputQuery = { + members: ["a.a_id", "b.b_id", "d.d_id"], + }; + const queryPlan2 = queryBuilder.getQueryPlan( + queryContext, + undefined, + query2, + ); + + assert.deepEqual(queryPlan2.segments[0]!.joinPlan, { + hasRowMultiplication: false, + initialModel: "a", + joins: [ + { leftModel: "a", rightModel: "b", joinType: "left" }, + { leftModel: "b", rightModel: "d", joinType: "left" }, + ], + }); + }); + + it("can generate a join plan with explicit join type set", () => { + const repository = semanticLayer + .repository() + .withModel(aModel) + .withModel(bModel) + .withModel(cModel) + .withModel(dModel) + .joinOneToOne( + "a", + "b", + ({ sql, models }) => + sql`${models.a.dimension("a_id")} = ${models.b.dimension("a_id")}`, + { type: "inner" }, + ) + .joinOneToMany( + "a", + "c", + ({ sql, models }) => + sql`${models.a.dimension("a_id")} = ${models.c.dimension("a_id")}`, + ) + .joinOneToOne( + "b", + "d", + ({ sql, models }) => + sql`${models.b.dimension("b_id")} = ${models.d.dimension("b_id")}`, + { type: "full" }, + ) + .joinOneToMany( + "c", + "d", + ({ sql, models }) => + sql`${models.c.dimension("c_id")} = ${models.d.dimension("c_id")}`, + { priority: "high" }, + ); + + const queryBuilder = repository.build("postgresql"); + + const queryContext = new semanticLayer.QueryContext( + queryBuilder.repository, + queryBuilder.dialect, + undefined, + ); + const query: semanticLayer.AnyInputQuery = { + members: ["a.a_id", "b.b_id", "c.c_id", "d.d_id"], + }; + const queryPlan = queryBuilder.getQueryPlan(queryContext, undefined, query); + + assert.deepEqual(queryPlan.segments[0]!.joinPlan, { + hasRowMultiplication: false, + initialModel: "a", + joins: [ + { + leftModel: "a", + rightModel: "b", + joinType: "inner", + }, + { + leftModel: "b", + rightModel: "d", + joinType: "full", + }, + { + leftModel: "d", + rightModel: "c", + joinType: "right", + }, + ], + }); + + const { sql } = queryBuilder.unsafeBuildQuery(query, undefined); + + assert.equal( + sql, + 'select "q0"."a___a_id" as "a___a_id", "q0"."b___b_id" as "b___b_id", "q0"."c___c_id" as "c___c_id", "q0"."d___d_id" as "d___d_id" from (select "a"."a_id" as "a___a_id", "b"."b_id" as "b___b_id", "c"."c_id" as "c___c_id", "d"."d_id" as "d___d_id" from "a" inner join "b" on "a"."a_id" = "b"."a_id" full join "d" on "b"."b_id" = "d"."b_id" right join "c" on "c"."c_id" = "d"."c_id") as "q0" group by "q0"."a___a_id", "q0"."b___b_id", "q0"."c___c_id", "q0"."d___d_id" order by "a___a_id" asc', + ); + }); +}); diff --git a/src/__tests__/query-builder/query-plan.test.ts b/src/__tests__/query-builder/query-plan.test.ts index 23d8eac..3348ff4 100644 --- a/src/__tests__/query-builder/query-plan.test.ts +++ b/src/__tests__/query-builder/query-plan.test.ts @@ -85,14 +85,14 @@ it("can crate a query plan", () => { { leftModel: "tracks", rightModel: "albums", - joinType: "rightJoin", + joinType: "right", }, { leftModel: "albums", rightModel: "artists", - joinType: "leftJoin", + joinType: "left", }, - { leftModel: "tracks", rightModel: "genres", joinType: "leftJoin" }, + { leftModel: "tracks", rightModel: "genres", joinType: "left" }, ], }, filters: [ @@ -150,19 +150,19 @@ it("can crate a query plan", () => { { leftModel: "invoice_lines", rightModel: "tracks", - joinType: "leftJoin", + joinType: "left", }, { leftModel: "tracks", rightModel: "albums", - joinType: "rightJoin", + joinType: "right", }, { leftModel: "albums", rightModel: "artists", - joinType: "leftJoin", + joinType: "left", }, - { leftModel: "tracks", rightModel: "genres", joinType: "leftJoin" }, + { leftModel: "tracks", rightModel: "genres", joinType: "left" }, ], }, filters: [ @@ -213,24 +213,24 @@ it("can crate a query plan", () => { { leftModel: "invoices", rightModel: "invoice_lines", - joinType: "leftJoin", + joinType: "left", }, { leftModel: "invoice_lines", rightModel: "tracks", - joinType: "leftJoin", + joinType: "left", }, { leftModel: "tracks", rightModel: "albums", - joinType: "rightJoin", + joinType: "right", }, { leftModel: "albums", rightModel: "artists", - joinType: "leftJoin", + joinType: "left", }, - { leftModel: "tracks", rightModel: "genres", joinType: "leftJoin" }, + { leftModel: "tracks", rightModel: "genres", joinType: "left" }, ], }, filters: [ diff --git a/src/lib/join.ts b/src/lib/join.ts index b6ae71e..cdeb6cc 100644 --- a/src/lib/join.ts +++ b/src/lib/join.ts @@ -3,12 +3,15 @@ import { ColumnRef, DimensionRef, IdentifierRef, SqlFn } from "./sql-fn.js"; import { AnyModel } from "./model.js"; import { ModelMemberWithoutModelPrefix } from "./types.js"; +export type ExplicitJoinType = "inner" | "full"; + export interface Join { left: string; right: string; joinOnDef: (context: C) => SqlFn; reversed: boolean; type: "oneToOne" | "oneToMany" | "manyToOne" | "manyToMany"; + joinType?: ExplicitJoinType; } export type AnyJoin = Join; @@ -47,11 +50,30 @@ export function makeModelJoinPayload(model: AnyModel, context: unknown) { }; } -export const JOIN_WEIGHTS: Record = { - oneToOne: 1, - oneToMany: 3, - manyToOne: 2, - manyToMany: 4, +export const JOIN_PRIORITIES = ["low", "normal", "high"] as const; + +export const JOIN_WEIGHTS: Record< + (typeof JOIN_PRIORITIES)[number], + Record +> = { + low: { + oneToOne: 100, + oneToMany: 300, + manyToOne: 200, + manyToMany: 400, + }, + normal: { + oneToOne: 10, + oneToMany: 30, + manyToOne: 20, + manyToMany: 40, + }, + high: { + oneToOne: 1, + oneToMany: 3, + manyToOne: 2, + manyToMany: 4, + }, }; export const REVERSED_JOIN: Record = { @@ -60,3 +82,8 @@ export const REVERSED_JOIN: Record = { manyToOne: "oneToMany", manyToMany: "manyToMany", }; + +export type JoinOptions = { + priority?: (typeof JOIN_PRIORITIES)[number]; + type?: ExplicitJoinType; +}; diff --git a/src/lib/query-builder.ts b/src/lib/query-builder.ts index d6b1d87..5a260e8 100644 --- a/src/lib/query-builder.ts +++ b/src/lib/query-builder.ts @@ -135,12 +135,20 @@ export class QueryBuilder< this.dialect, context, ); - const queryPlan = getQueryPlan(this, queryContext, context, parsedQuery); + const queryPlan = this.getQueryPlan(queryContext, context, parsedQuery); const sqlQuery = buildQuery(this, queryContext, context, queryPlan); return sqlQuery.toSQL(); } + getQueryPlan( + queryContext: QueryContext, + context: unknown, + query: AnyInputQuery, + ) { + return getQueryPlan(this, queryContext, context, query); + } + unsafeBuildQuery(payload: unknown, context: unknown) { const parsedQuery: AnyInputQuery = this.querySchema.parse(payload); const { sql, bindings } = this.unsafeBuildGenericQueryWithoutSchemaParse( diff --git a/src/lib/query-builder/build-query.ts b/src/lib/query-builder/build-query.ts index 817071d..0b2f45e 100644 --- a/src/lib/query-builder/build-query.ts +++ b/src/lib/query-builder/build-query.ts @@ -54,6 +54,7 @@ function joinModelQueryModels( joinType, } of segment.joinPlan.joins) { const join = queryBuilder.repository.getJoin(leftModelName, rightModelName); + const joinFn: `${typeof joinType}Join` = `${joinType}Join`; invariant( join, @@ -72,7 +73,7 @@ function joinModelQueryModels( context, ); - sqlQuery[joinType]( + sqlQuery[joinFn]( joinSubject, queryBuilder.dialect.fragment(joinOn.sql, joinOn.bindings), ); @@ -237,7 +238,7 @@ function buildSegmentQuery( } // We always GROUP BY the dimensions, if there are no metrics, it will behave as DISTINCT - // For metrics, this is currently NOOP because Metric returns an empty array + // Metrics can also participate in the GROUP BY, if aggregated or groupBy is called on a referenced column/dimension/metric const segmentQueryGroupBy = queryMember.getSegmentQueryGroupBy(modelQueryAlias); diff --git a/src/lib/query-builder/optimal-join-graph.ts b/src/lib/query-builder/optimal-join-graph.ts index ab76e4d..2ba280d 100644 --- a/src/lib/query-builder/optimal-join-graph.ts +++ b/src/lib/query-builder/optimal-join-graph.ts @@ -48,7 +48,9 @@ export function findOptimalJoinGraph( return makeSingleModelGraph(requestedModels[0]!); } - const paths = graphlib.alg.dijkstraAll(originalGraph); + const paths = graphlib.alg.dijkstraAll(originalGraph, (e) => { + return originalGraph.edge(e); + }); const completeGraph = buildCompleteGraph(paths, requestedModels); const mst = graphlib.alg.prim(completeGraph, (e) => { return completeGraph.edge(e); diff --git a/src/lib/query-builder/query-plan.ts b/src/lib/query-builder/query-plan.ts index 9e28fcd..2108dc9 100644 --- a/src/lib/query-builder/query-plan.ts +++ b/src/lib/query-builder/query-plan.ts @@ -318,6 +318,7 @@ function getSegmentQueryMetricsRefsSubQueryPlan( } } +// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Essential complexity for join planning function getSegmentQueryJoins( queryBuilder: AnyQueryBuilder, models: string[], @@ -327,13 +328,13 @@ function getSegmentQueryJoins( const visitedModels = new Set(); const modelsToProcess: { modelName: string; - join?: { join: AnyJoin; left: string; right: string }; + join?: { config: AnyJoin; leftModel: string; rightModel: string }; }[] = [{ modelName: initialModel }]; const joins: { leftModel: string; rightModel: string; - joinType: "leftJoin" | "rightJoin"; + joinType: "left" | "right" | "inner" | "full"; }[] = []; let hasRowMultiplication = false; @@ -350,14 +351,21 @@ function getSegmentQueryJoins( ); if (join) { - if (join.join.type === "manyToMany" || join.join.type === "oneToMany") { + if ( + join.config.type === "manyToMany" || + join.config.type === "oneToMany" + ) { hasRowMultiplication = true; } - const joinType = join.join.reversed ? "rightJoin" : "leftJoin"; + const joinType = join.config.joinType + ? join.config.joinType + : join.config.reversed + ? "right" + : "left"; joins.push({ - leftModel: join.left, - rightModel: join.right, + leftModel: join.leftModel, + rightModel: join.rightModel, joinType, }); } @@ -371,7 +379,11 @@ function getSegmentQueryJoins( return { modelName: unvisitedModelName, join: join - ? { left: modelName, right: unvisitedModelName, join } + ? { + leftModel: modelName, + rightModel: unvisitedModelName, + config: join, + } : undefined, }; }), diff --git a/src/lib/repository.ts b/src/lib/repository.ts index d989735..15cb296 100644 --- a/src/lib/repository.ts +++ b/src/lib/repository.ts @@ -12,6 +12,7 @@ import { JOIN_WEIGHTS, JoinFn, JoinFnModels, + JoinOptions, REVERSED_JOIN, makeModelJoinPayload, } from "./join.js"; @@ -246,6 +247,7 @@ export class Repository< modelName1: N1, modelName2: N2, joinSqlDefFn: JoinFn, + opts?: JoinOptions, ) { const model1 = this.models[modelName1]; const model2 = this.models[modelName2]; @@ -275,9 +277,14 @@ export class Repository< }; const reversedType = REVERSED_JOIN[type]; + const priority = opts?.priority ?? "normal"; - this.graph.setEdge(model1.name, model2.name, JOIN_WEIGHTS[type]); - this.graph.setEdge(model2.name, model1.name, JOIN_WEIGHTS[reversedType]); + this.graph.setEdge(model1.name, model2.name, JOIN_WEIGHTS[priority][type]); + this.graph.setEdge( + model2.name, + model1.name, + JOIN_WEIGHTS[priority][reversedType], + ); this.joins[model1.name] ||= {}; this.joins[model1.name]![model2.name] = { @@ -286,6 +293,7 @@ export class Repository< joinOnDef: joinSqlDef, type: type, reversed: false, + joinType: opts?.type, }; this.joins[model2.name] ||= {}; this.joins[model2.name]![model1.name] = { @@ -294,6 +302,7 @@ export class Repository< joinOnDef: joinSqlDef, type: reversedType, reversed: true, + joinType: opts?.type, }; return this; } @@ -305,8 +314,9 @@ export class Repository< model1: N1, model2: N2, joinSqlDefFn: JoinFn, + opts?: JoinOptions, ) { - return this.join("oneToOne", model1, model2, joinSqlDefFn); + return this.join("oneToOne", model1, model2, joinSqlDefFn, opts); } joinOneToMany< @@ -316,8 +326,9 @@ export class Repository< model1: N1, model2: N2, joinSqlDefFn: JoinFn, + opts?: JoinOptions, ) { - return this.join("oneToMany", model1, model2, joinSqlDefFn); + return this.join("oneToMany", model1, model2, joinSqlDefFn, opts); } joinManyToOne< @@ -327,8 +338,9 @@ export class Repository< model1: N1, model2: N2, joinSqlDefFn: JoinFn, + opts?: JoinOptions, ) { - return this.join("manyToOne", model1, model2, joinSqlDefFn); + return this.join("manyToOne", model1, model2, joinSqlDefFn, opts); } joinManyToMany< @@ -338,8 +350,9 @@ export class Repository< model1: N1, model2: N2, joinSqlDefFn: JoinFn, + opts?: JoinOptions, ) { - return this.join("manyToMany", model1, model2, joinSqlDefFn); + return this.join("manyToMany", model1, model2, joinSqlDefFn, opts); } getDimension(dimensionName: string): Dimension { diff --git a/src/lib/semantic-layer.ts b/src/lib/semantic-layer.ts index 50710ba..b5f53a1 100644 --- a/src/lib/semantic-layer.ts +++ b/src/lib/semantic-layer.ts @@ -13,7 +13,7 @@ export type * from "./dialect.js"; export type * from "./hierarchy.js"; export type * from "./member.js"; export type * from "./model/member.js"; -export type * from "./query-builder/query-plan/query-context.js"; +export * from "./query-builder/query-plan/query-context.js"; export type * from "./query-builder/query-plan/query-member.js"; export type * from "./repository/member.js"; diff --git a/src/lib/sql-builder.ts b/src/lib/sql-builder.ts index eb4f457..f459f96 100644 --- a/src/lib/sql-builder.ts +++ b/src/lib/sql-builder.ts @@ -5,7 +5,7 @@ import { AnyBaseDialect } from "./dialect/base.js"; export interface QueryJoin { table: string | SqlFragment | SqlQueryBuilder; on: string | SqlFragment; - type: "inner" | "left" | "right"; + type: "inner" | "left" | "right" | "full"; } export interface SqlQueryStructure { @@ -82,7 +82,7 @@ export class SqlQueryBuilder { join( table: string | SqlFragment | SqlQueryBuilder, on: string | SqlFragment, - type: "inner" | "left" | "right", + type: "inner" | "left" | "right" | "full", ) { this.query = { ...this.query, @@ -112,6 +112,13 @@ export class SqlQueryBuilder { return this.join(table, on, "inner"); } + fullJoin( + table: string | SqlFragment | SqlQueryBuilder, + on: string | SqlFragment, + ) { + return this.join(table, on, "full"); + } + toSQL(): SqlQuery { return toSQL(this); }