From 4d5f96bb8ecd72731560d4dba3012da4edbb1bf5 Mon Sep 17 00:00:00 2001 From: Chuck Driesler Date: Sun, 12 Jan 2025 00:03:14 +0000 Subject: [PATCH 01/28] feat(regions): repo functions for copying project branches and commits --- .../typedefs/workspaces.graphql | 1 + .../modules/core/repositories/objects.ts | 824 +++++++++--------- .../server/modules/shared/helpers/dbHelper.ts | 4 +- .../modules/workspaces/domain/operations.ts | 8 + .../workspaces/repositories/regions.ts | 176 +++- .../modules/workspaces/services/projects.ts | 269 +++--- 6 files changed, 719 insertions(+), 563 deletions(-) diff --git a/packages/server/assets/workspacesCore/typedefs/workspaces.graphql b/packages/server/assets/workspacesCore/typedefs/workspaces.graphql index 226b28a18e..8c5d699975 100644 --- a/packages/server/assets/workspacesCore/typedefs/workspaces.graphql +++ b/packages/server/assets/workspacesCore/typedefs/workspaces.graphql @@ -170,6 +170,7 @@ type WorkspaceProjectMutations { updateRole(input: ProjectUpdateRoleInput!): Project! @hasStreamRole(role: STREAM_OWNER) @hasWorkspaceRole(role: MEMBER) + moveToRegion(projectId: String!, regionKey: String!): Project! moveToWorkspace(projectId: String!, workspaceId: String!): Project! create(input: WorkspaceProjectCreateInput!): Project! } diff --git a/packages/server/modules/core/repositories/objects.ts b/packages/server/modules/core/repositories/objects.ts index 851bf78fa3..bdf8a82ef8 100644 --- a/packages/server/modules/core/repositories/objects.ts +++ b/packages/server/modules/core/repositories/objects.ts @@ -40,255 +40,255 @@ const tables = { export const getStreamObjectsFactory = (deps: { db: Knex }): GetStreamObjects => - async (streamId: string, objectIds: string[]): Promise => { - if (!objectIds?.length) return [] + async (streamId: string, objectIds: string[]): Promise => { + if (!objectIds?.length) return [] - const q = tables - .objects(deps.db) - .where(Objects.col.streamId, streamId) - .whereIn(Objects.col.id, objectIds) + const q = tables + .objects(deps.db) + .where(Objects.col.streamId, streamId) + .whereIn(Objects.col.id, objectIds) - return await q - } + return await q + } export const getObjectFactory = (deps: { db: Knex }): GetObject => - async (objectId: string, streamId: string): Promise> => { - return await tables - .objects(deps.db) - .where(Objects.col.id, objectId) - .andWhere(Objects.col.streamId, streamId) - .first() - } + async (objectId: string, streamId: string): Promise> => { + return await tables + .objects(deps.db) + .where(Objects.col.id, objectId) + .andWhere(Objects.col.streamId, streamId) + .first() + } export const getFormattedObjectFactory = (deps: { db: Knex }): GetFormattedObject => - async ({ streamId, objectId }) => { - const res = await tables - .objects(deps.db) - .where({ streamId, id: objectId }) - .select('*') - .first() - if (!res) return null - - // TODO: Why tho? A lot if not most of places already just use getObjectFactory, - const finalRes: SetOptional = res - if (finalRes.data) finalRes.data.totalChildrenCount = res.totalChildrenCount // move this back - delete finalRes.streamId // backwards compatibility - - return finalRes - } + async ({ streamId, objectId }) => { + const res = await tables + .objects(deps.db) + .where({ streamId, id: objectId }) + .select('*') + .first() + if (!res) return null + + // TODO: Why tho? A lot if not most of places already just use getObjectFactory, + const finalRes: SetOptional = res + if (finalRes.data) finalRes.data.totalChildrenCount = res.totalChildrenCount // move this back + delete finalRes.streamId // backwards compatibility + + return finalRes + } export const getBatchedStreamObjectsFactory = (deps: { db: Knex }): GetBatchedStreamObjects => - (streamId: string, options?: Partial) => { - const baseQuery = tables - .objects(deps.db) - .select('*') - .where(Objects.col.streamId, streamId) - .orderBy(Objects.col.id) - - return executeBatchedSelect(baseQuery, options) - } + (streamId: string, options?: Partial) => { + const baseQuery = tables + .objects(deps.db) + .select('*') + .where(Objects.col.streamId, streamId) + .orderBy(Objects.col.id) + + return executeBatchedSelect(baseQuery, options) + } export const insertObjectsFactory = (deps: { db: Knex }): StoreObjects => - async (objects: ObjectRecord[], options?: Partial<{ trx: Knex.Transaction }>) => { - const q = tables.objects(deps.db).insert(objects) - if (options?.trx) q.transacting(options.trx) - return await q - } + async (objects: ObjectRecord[], options?: Partial<{ trx: Knex.Transaction }>) => { + const q = tables.objects(deps.db).insert(objects) + if (options?.trx) q.transacting(options.trx) + return await q + } export const storeSingleObjectIfNotFoundFactory = (deps: { db: Knex }): StoreSingleObjectIfNotFound => - async (insertionObject) => { - await tables - .objects(deps.db) - .insert( - // knex is bothered by string being inserted into jsonb, which is actually fine - insertionObject as SpeckleObject - ) - .onConflict() - .ignore() - } + async (insertionObject) => { + await tables + .objects(deps.db) + .insert( + // knex is bothered by string being inserted into jsonb, which is actually fine + insertionObject as SpeckleObject + ) + .onConflict() + .ignore() + } export const storeObjectsIfNotFoundFactory = (deps: { db: Knex }): StoreObjectsIfNotFound => - async (batch) => { - await tables - .objects(deps.db) - .insert( - // knex is bothered by string being inserted into jsonb, which is actually fine - batch as SpeckleObject[] - ) - .onConflict() - .ignore() - } + async (batch) => { + await tables + .objects(deps.db) + .insert( + // knex is bothered by string being inserted into jsonb, which is actually fine + batch as SpeckleObject[] + ) + .onConflict() + .ignore() + } export const storeClosuresIfNotFoundFactory = (deps: { db: Knex }): StoreClosuresIfNotFound => - async (closuresBatch) => { - await tables - .objectChildrenClosure(deps.db) - .insert(closuresBatch) - .onConflict() - .ignore() - } + async (closuresBatch) => { + await tables + .objectChildrenClosure(deps.db) + .insert(closuresBatch) + .onConflict() + .ignore() + } export const getObjectChildrenStreamFactory = (deps: { db: Knex }): GetObjectChildrenStream => - async ({ streamId, objectId }) => { - const q = deps.db.with( - 'object_children_closure', - knex.raw( - `SELECT objects.id as parent, d.key as child, d.value as mindepth, ? as "streamId" + async ({ streamId, objectId }) => { + const q = deps.db.with( + 'object_children_closure', + knex.raw( + `SELECT objects.id as parent, d.key as child, d.value as mindepth, ? as "streamId" FROM objects JOIN jsonb_each_text(objects.data->'__closure') d ON true where objects.id = ?`, - [streamId, objectId] - ) - ) - q.select('id') - q.select(knex.raw('data::text as "dataText"')) - q.from('object_children_closure') - - q.rightJoin('objects', function () { - this.on('objects.streamId', '=', 'object_children_closure.streamId').andOn( - 'objects.id', - '=', - 'object_children_closure.child' - ) - }) - .where( - knex.raw('object_children_closure."streamId" = ? AND parent = ?', [ - streamId, - objectId - ]) + [streamId, objectId] + ) ) - .orderBy('objects.id') - return q.stream({ highWaterMark: 500 }) - } + q.select('id') + q.select(knex.raw('data::text as "dataText"')) + q.from('object_children_closure') + + q.rightJoin('objects', function () { + this.on('objects.streamId', '=', 'object_children_closure.streamId').andOn( + 'objects.id', + '=', + 'object_children_closure.child' + ) + }) + .where( + knex.raw('object_children_closure."streamId" = ? AND parent = ?', [ + streamId, + objectId + ]) + ) + .orderBy('objects.id') + return q.stream({ highWaterMark: 500 }) + } export const getObjectsStreamFactory = (deps: { db: Knex }): GetObjectsStream => - async ({ streamId, objectIds }) => { - const res = tables - .objects(deps.db) - .whereIn('id', objectIds) - .andWhere('streamId', streamId) - .orderBy('id') - .select( - knex.raw( - '"id", "speckleType", "totalChildrenCount", "totalChildrenCountByDepth", "createdAt", data::text as "dataText"' + async ({ streamId, objectIds }) => { + const res = tables + .objects(deps.db) + .whereIn('id', objectIds) + .andWhere('streamId', streamId) + .orderBy('id') + .select( + knex.raw( + '"id", "speckleType", "totalChildrenCount", "totalChildrenCountByDepth", "createdAt", data::text as "dataText"' + ) ) - ) - return res.stream({ highWaterMark: 500 }) - } + return res.stream({ highWaterMark: 500 }) + } export const hasObjectsFactory = (deps: { db: Knex }): HasObjects => - async ({ streamId, objectIds }) => { - const dbRes = await tables - .objects(deps.db) - .whereIn('id', objectIds) - .andWhere('streamId', streamId) - .select('id') - - const res: Record = {} - // eslint-disable-next-line @typescript-eslint/no-for-in-array - for (const i in objectIds) { - res[objectIds[i]] = false - } - // eslint-disable-next-line @typescript-eslint/no-for-in-array - for (const i in dbRes) { - res[dbRes[i].id] = true + async ({ streamId, objectIds }) => { + const dbRes = await tables + .objects(deps.db) + .whereIn('id', objectIds) + .andWhere('streamId', streamId) + .select('id') + + const res: Record = {} + // eslint-disable-next-line @typescript-eslint/no-for-in-array + for (const i in objectIds) { + res[objectIds[i]] = false + } + // eslint-disable-next-line @typescript-eslint/no-for-in-array + for (const i in dbRes) { + res[dbRes[i].id] = true + } + return res } - return res - } export const getObjectChildrenFactory = (deps: { db: Knex }): GetObjectChildren => - async ({ streamId, objectId, limit, depth, select, cursor }) => { - limit = toNumber(limit || 0) || 50 - depth = toNumber(depth || 0) || 1000 + async ({ streamId, objectId, limit, depth, select, cursor }) => { + limit = toNumber(limit || 0) || 50 + depth = toNumber(depth || 0) || 1000 - let fullObjectSelect = false + let fullObjectSelect = false - const q = deps.db.with( - 'object_children_closure', - knex.raw( - `SELECT objects.id as parent, d.key as child, d.value as mindepth, ? as "streamId" + const q = deps.db.with( + 'object_children_closure', + knex.raw( + `SELECT objects.id as parent, d.key as child, d.value as mindepth, ? as "streamId" FROM objects JOIN jsonb_each_text(objects.data->'__closure') d ON true where objects.id = ?`, - [streamId, objectId] - ) - ) - - if (Array.isArray(select)) { - select.forEach((field, index) => { - q.select( - knex.raw('jsonb_path_query(data, :path) as :name:', { - path: '$.' + field, - name: '' + index - }) + [streamId, objectId] ) - }) - } else { - fullObjectSelect = true - q.select('data') - } + ) + + if (Array.isArray(select)) { + select.forEach((field, index) => { + q.select( + knex.raw('jsonb_path_query(data, :path) as :name:', { + path: '$.' + field, + name: '' + index + }) + ) + }) + } else { + fullObjectSelect = true + q.select('data') + } - q.select('id') - q.select('createdAt') - q.select('speckleType') - q.select('totalChildrenCount') + q.select('id') + q.select('createdAt') + q.select('speckleType') + q.select('totalChildrenCount') - q.from('object_children_closure') + q.from('object_children_closure') - q.rightJoin('objects', function () { - this.on('objects.streamId', '=', 'object_children_closure.streamId').andOn( - 'objects.id', - '=', - 'object_children_closure.child' - ) - }) - .where( - knex.raw('object_children_closure."streamId" = ? AND parent = ?', [ - streamId, - objectId - ]) - ) - .andWhere(knex.raw('object_children_closure.mindepth < ?', [depth])) - .andWhere(knex.raw('id > ?', [cursor ? cursor : '0'])) - .orderBy('objects.id') - .limit(limit) + q.rightJoin('objects', function () { + this.on('objects.streamId', '=', 'object_children_closure.streamId').andOn( + 'objects.id', + '=', + 'object_children_closure.child' + ) + }) + .where( + knex.raw('object_children_closure."streamId" = ? AND parent = ?', [ + streamId, + objectId + ]) + ) + .andWhere(knex.raw('object_children_closure.mindepth < ?', [depth])) + .andWhere(knex.raw('id > ?', [cursor ? cursor : '0'])) + .orderBy('objects.id') + .limit(limit) - const rows = await q + const rows = await q - if (rows.length === 0) { - return { objects: rows, cursor: null } - } + if (rows.length === 0) { + return { objects: rows, cursor: null } + } - if (!fullObjectSelect) - rows.forEach((o, i, arr) => { - const no = { - id: o.id, - createdAt: o.createdAt, - speckleType: o.speckleType, - totalChildrenCount: o.totalChildrenCount, - data: {} - } - let k = 0 - for (const field of select || []) { - set(no.data, field, o[k++]) - } - arr[i] = no - }) + if (!fullObjectSelect) + rows.forEach((o, i, arr) => { + const no = { + id: o.id, + createdAt: o.createdAt, + speckleType: o.speckleType, + totalChildrenCount: o.totalChildrenCount, + data: {} + } + let k = 0 + for (const field of select || []) { + set(no.data, field, o[k++]) + } + arr[i] = no + }) - const lastId = rows[rows.length - 1].id - return { objects: rows, cursor: lastId } - } + const lastId = rows[rows.length - 1].id + return { objects: rows, cursor: lastId } + } /** * This query is inefficient on larger sets (n * 10k objects) as we need to return the total count on an arbitrarily (user) defined selection of objects. @@ -296,246 +296,246 @@ export const getObjectChildrenFactory = */ export const getObjectChildrenQueryFactory = (deps: { db: Knex }): GetObjectChildrenQuery => - async (params) => { - const { streamId, objectId, select, query } = params - - const limit = toNumber(params.limit || 0) || 50 - const depth = toNumber(params.depth || 0) || 1000 - const orderBy = params.orderBy || { field: 'id', direction: 'asc' } - - // Cursors received by this service should be base64 encoded. They are generated on first entry query by this service; They should never be client-side generated. - const cursor: Optional<{ - value: unknown - operator: string - field: string - lastSeenId?: string - }> = params.cursor - ? JSON.parse(Buffer.from(params.cursor, 'base64').toString('binary')) - : undefined - - // Flag that keeps track of whether we select the whole "data" part of an object or not - let fullObjectSelect = false - if (Array.isArray(select)) { - // if we order by a field that we do not select, select it! - if (orderBy && select.indexOf(orderBy.field) === -1) { - select.push(orderBy.field) + async (params) => { + const { streamId, objectId, select, query } = params + + const limit = toNumber(params.limit || 0) || 50 + const depth = toNumber(params.depth || 0) || 1000 + const orderBy = params.orderBy || { field: 'id', direction: 'asc' } + + // Cursors received by this service should be base64 encoded. They are generated on first entry query by this service; They should never be client-side generated. + const cursor: Optional<{ + value: unknown + operator: string + field: string + lastSeenId?: string + }> = params.cursor + ? JSON.parse(Buffer.from(params.cursor, 'base64').toString('binary')) + : undefined + + // Flag that keeps track of whether we select the whole "data" part of an object or not + let fullObjectSelect = false + if (Array.isArray(select)) { + // if we order by a field that we do not select, select it! + if (orderBy && select.indexOf(orderBy.field) === -1) { + select.push(orderBy.field) + } + // // always add the id! + // if ( select.indexOf( 'id' ) === -1 ) select.unshift( 'id' ) + } else { + fullObjectSelect = true } - // // always add the id! - // if ( select.indexOf( 'id' ) === -1 ) select.unshift( 'id' ) - } else { - fullObjectSelect = true - } - const additionalIdOrderBy = orderBy.field !== 'id' + const additionalIdOrderBy = orderBy.field !== 'id' - const operatorsWhitelist = ['=', '>', '>=', '<', '<=', '!='] + const operatorsWhitelist = ['=', '>', '>=', '<', '<=', '!='] - const mainQuery = deps.db - .with( - 'object_children_closure', - knex.raw( - `SELECT objects.id as parent, d.key as child, d.value as mindepth, ? as "streamId" + const mainQuery = deps.db + .with( + 'object_children_closure', + knex.raw( + `SELECT objects.id as parent, d.key as child, d.value as mindepth, ? as "streamId" FROM objects JOIN jsonb_each_text(objects.data->'__closure') d ON true where objects.id = ?`, - [streamId, objectId] + [streamId, objectId] + ) ) - ) - .with('objs', (cteInnerQuery) => { - // always select the id - cteInnerQuery.select('id').from('object_children_closure') - cteInnerQuery.select('createdAt') - cteInnerQuery.select('speckleType') - cteInnerQuery.select('totalChildrenCount') - - // if there are any select fields, add them - if (Array.isArray(select)) { - select.forEach((field, index) => { - cteInnerQuery.select( - knex.raw('jsonb_path_query(data, :path) as :name:', { - path: '$.' + field, - name: '' + index + .with('objs', (cteInnerQuery) => { + // always select the id + cteInnerQuery.select('id').from('object_children_closure') + cteInnerQuery.select('createdAt') + cteInnerQuery.select('speckleType') + cteInnerQuery.select('totalChildrenCount') + + // if there are any select fields, add them + if (Array.isArray(select)) { + select.forEach((field, index) => { + cteInnerQuery.select( + knex.raw('jsonb_path_query(data, :path) as :name:', { + path: '$.' + field, + name: '' + index + }) + ) + }) + // otherwise, get the whole object, as stored in the jsonb column + } else { + cteInnerQuery.select('data') + } + + // join on objects table + cteInnerQuery + .join('objects', function () { + this.on('objects.streamId', '=', 'object_children_closure.streamId').andOn( + 'objects.id', + '=', + 'object_children_closure.child' + ) + }) + .where('object_children_closure.streamId', streamId) + .andWhere('parent', objectId) + .andWhere('mindepth', '<', depth) + + // Add user provided filters/queries. + if (Array.isArray(query) && query.length > 0) { + cteInnerQuery.andWhere((nestedWhereQuery) => { + query.forEach((statement, index) => { + let castType = 'text' + if (typeof statement.value === 'string') castType = 'text' + if (typeof statement.value === 'boolean') castType = 'boolean' + if (typeof statement.value === 'number') castType = 'numeric' + + if (operatorsWhitelist.indexOf(statement.operator) === -1) + throw new Error('Invalid operator for query') + + // Determine the correct where clause (where, and where, or where) + let whereClause: keyof typeof nestedWhereQuery + if (index === 0) whereClause = 'where' + else if (statement.verb && statement.verb.toLowerCase() === 'or') + whereClause = 'orWhere' + else whereClause = 'andWhere' + + // Note: castType is generated from the statement's value and operators are matched against a whitelist. + // If comparing with strings, the jsonb_path_query(_first) func returns json encoded strings (ie, `bar` is actually `"bar"`), hence we need to add the quotes manually to the raw provided comparison value. + nestedWhereQuery[whereClause]( + knex.raw( + `jsonb_path_query_first( data, ? )::${castType} ${statement.operator} ? `, + [ + '$.' + statement.field, + castType === 'text' ? `"${statement.value}"` : statement.value + ] + ) + ) }) + }) + } + + // Order by clause; validate direction! + const direction = + orderBy.direction && orderBy.direction.toLowerCase() === 'desc' + ? 'desc' + : 'asc' + if (orderBy.field === 'id') { + cteInnerQuery.orderBy('id', direction) + } else { + cteInnerQuery.orderByRaw( + knex.raw(`jsonb_path_query_first( data, ? ) ${direction}, id asc`, [ + '$.' + orderBy.field + ]) ) - }) - // otherwise, get the whole object, as stored in the jsonb column + } + }) + .select('*') + .from('objs') + .joinRaw('RIGHT JOIN ( SELECT count(*) FROM "objs" ) c(total_count) ON TRUE') + + // Set cursor clause, if present. If it's not present, it's an entry query; this method will return a cursor based on its given query. + // We have implemented keyset pagination for more efficient searches on larger sets. This approach depends on an order by value provided by the user and a (hidden) primary key. + // logger.debug( cursor ) + if (cursor) { + let castType = 'text' + if (typeof cursor.value === 'string') castType = 'text' + if (typeof cursor.value === 'boolean') castType = 'boolean' + if (typeof cursor.value === 'number') castType = 'numeric' + + // When strings are used inside an order clause, as mentioned above, we need to add quotes around the comparison value, as the jsonb_path_query funcs return json encoded strings (`{"test":"foo"}` => test is returned as `"foo"`) + if (castType === 'text') cursor.value = `"${cursor.value}"` + + if (operatorsWhitelist.indexOf(cursor.operator) === -1) + throw new Error('Invalid operator for cursor') + + // Unwrapping the tuple comparison of ( userOrderByField, id ) > ( lastValueOfUserOrderBy, lastSeenId ) + if (fullObjectSelect) { + if (cursor.field === 'id') { + mainQuery.where(knex.raw(`id ${cursor.operator} ? `, [cursor.value])) + } else { + mainQuery.where( + knex.raw( + `jsonb_path_query_first( data, ? )::${castType} ${cursor.operator}= ? `, + ['$.' + cursor.field, cursor.value] + ) + ) + } } else { - cteInnerQuery.select('data') + mainQuery.where( + knex.raw(`??::${castType} ${cursor.operator}= ? `, [ + (select || []).indexOf(cursor.field).toString(), + cursor.value + ]) + ) } - // join on objects table - cteInnerQuery - .join('objects', function () { - this.on('objects.streamId', '=', 'object_children_closure.streamId').andOn( - 'objects.id', - '=', - 'object_children_closure.child' - ) - }) - .where('object_children_closure.streamId', streamId) - .andWhere('parent', objectId) - .andWhere('mindepth', '<', depth) - - // Add user provided filters/queries. - if (Array.isArray(query) && query.length > 0) { - cteInnerQuery.andWhere((nestedWhereQuery) => { - query.forEach((statement, index) => { - let castType = 'text' - if (typeof statement.value === 'string') castType = 'text' - if (typeof statement.value === 'boolean') castType = 'boolean' - if (typeof statement.value === 'number') castType = 'numeric' - - if (operatorsWhitelist.indexOf(statement.operator) === -1) - throw new Error('Invalid operator for query') - - // Determine the correct where clause (where, and where, or where) - let whereClause: keyof typeof nestedWhereQuery - if (index === 0) whereClause = 'where' - else if (statement.verb && statement.verb.toLowerCase() === 'or') - whereClause = 'orWhere' - else whereClause = 'andWhere' - - // Note: castType is generated from the statement's value and operators are matched against a whitelist. - // If comparing with strings, the jsonb_path_query(_first) func returns json encoded strings (ie, `bar` is actually `"bar"`), hence we need to add the quotes manually to the raw provided comparison value. - nestedWhereQuery[whereClause]( + if (cursor.lastSeenId) { + mainQuery.andWhere((qb) => { + // Dunno what the TS issue here is, the JS code itself seemed to work fine as JS + // eslint-disable-next-line @typescript-eslint/no-explicit-any + qb.where('id' as any, '>', cursor!.lastSeenId as any) + if (fullObjectSelect) + qb.orWhere( knex.raw( - `jsonb_path_query_first( data, ? )::${castType} ${statement.operator} ? `, - [ - '$.' + statement.field, - castType === 'text' ? `"${statement.value}"` : statement.value - ] + `jsonb_path_query_first( data, ? )::${castType} ${cursor!.operator} ? `, + ['$.' + cursor!.field, cursor!.value] ) ) - }) + else + qb.orWhere( + knex.raw(`??::${castType} ${cursor!.operator} ? `, [ + (select || []).indexOf(cursor!.field).toString(), + cursor!.value + ]) + ) }) } - - // Order by clause; validate direction! - const direction = - orderBy.direction && orderBy.direction.toLowerCase() === 'desc' - ? 'desc' - : 'asc' - if (orderBy.field === 'id') { - cteInnerQuery.orderBy('id', direction) - } else { - cteInnerQuery.orderByRaw( - knex.raw(`jsonb_path_query_first( data, ? ) ${direction}, id asc`, [ - '$.' + orderBy.field - ]) - ) - } - }) - .select('*') - .from('objs') - .joinRaw('RIGHT JOIN ( SELECT count(*) FROM "objs" ) c(total_count) ON TRUE') - - // Set cursor clause, if present. If it's not present, it's an entry query; this method will return a cursor based on its given query. - // We have implemented keyset pagination for more efficient searches on larger sets. This approach depends on an order by value provided by the user and a (hidden) primary key. - // logger.debug( cursor ) - if (cursor) { - let castType = 'text' - if (typeof cursor.value === 'string') castType = 'text' - if (typeof cursor.value === 'boolean') castType = 'boolean' - if (typeof cursor.value === 'number') castType = 'numeric' - - // When strings are used inside an order clause, as mentioned above, we need to add quotes around the comparison value, as the jsonb_path_query funcs return json encoded strings (`{"test":"foo"}` => test is returned as `"foo"`) - if (castType === 'text') cursor.value = `"${cursor.value}"` - - if (operatorsWhitelist.indexOf(cursor.operator) === -1) - throw new Error('Invalid operator for cursor') - - // Unwrapping the tuple comparison of ( userOrderByField, id ) > ( lastValueOfUserOrderBy, lastSeenId ) - if (fullObjectSelect) { - if (cursor.field === 'id') { - mainQuery.where(knex.raw(`id ${cursor.operator} ? `, [cursor.value])) - } else { - mainQuery.where( - knex.raw( - `jsonb_path_query_first( data, ? )::${castType} ${cursor.operator}= ? `, - ['$.' + cursor.field, cursor.value] - ) - ) - } - } else { - mainQuery.where( - knex.raw(`??::${castType} ${cursor.operator}= ? `, [ - (select || []).indexOf(cursor.field).toString(), - cursor.value - ]) - ) } - if (cursor.lastSeenId) { - mainQuery.andWhere((qb) => { - // Dunno what the TS issue here is, the JS code itself seemed to work fine as JS - // eslint-disable-next-line @typescript-eslint/no-explicit-any - qb.where('id' as any, '>', cursor!.lastSeenId as any) - if (fullObjectSelect) - qb.orWhere( - knex.raw( - `jsonb_path_query_first( data, ? )::${castType} ${cursor!.operator} ? `, - ['$.' + cursor!.field, cursor!.value] - ) - ) - else - qb.orWhere( - knex.raw(`??::${castType} ${cursor!.operator} ? `, [ - (select || []).indexOf(cursor!.field).toString(), - cursor!.value - ]) - ) + mainQuery.limit(limit) + // logger.debug( mainQuery.toString() ) + // Finally, execute the query + const rows = await mainQuery + const totalCount = rows && rows.length > 0 ? parseInt(rows[0].total_count) : 0 + + // Return early + if (totalCount === 0) return { totalCount, objects: [], cursor: null } + + // Reconstruct the object based on the provided select paths. + if (!fullObjectSelect) { + rows.forEach((o, i, arr) => { + const no = { + id: o.id, + createdAt: o.createdAt, + speckleType: o.speckleType, + totalChildrenCount: o.totalChildrenCount, + data: {} + } + let k = 0 + for (const field of select || []) { + set(no.data, field, o[k++]) + } + arr[i] = no }) } - } - - mainQuery.limit(limit) - // logger.debug( mainQuery.toString() ) - // Finally, execute the query - const rows = await mainQuery - const totalCount = rows && rows.length > 0 ? parseInt(rows[0].total_count) : 0 - - // Return early - if (totalCount === 0) return { totalCount, objects: [], cursor: null } - - // Reconstruct the object based on the provided select paths. - if (!fullObjectSelect) { - rows.forEach((o, i, arr) => { - const no = { - id: o.id, - createdAt: o.createdAt, - speckleType: o.speckleType, - totalChildrenCount: o.totalChildrenCount, - data: {} - } - let k = 0 - for (const field of select || []) { - set(no.data, field, o[k++]) - } - arr[i] = no - }) - } - // Assemble the cursor for an eventual next call - const cursorObj: typeof cursor = { - field: cursor?.field || orderBy.field, - operator: - cursor?.operator || - (orderBy.direction && orderBy.direction.toLowerCase() === 'desc' ? '<' : '>'), - value: get(rows[rows.length - 1], `data.${orderBy.field}`) - } + // Assemble the cursor for an eventual next call + const cursorObj: typeof cursor = { + field: cursor?.field || orderBy.field, + operator: + cursor?.operator || + (orderBy.direction && orderBy.direction.toLowerCase() === 'desc' ? '<' : '>'), + value: get(rows[rows.length - 1], `data.${orderBy.field}`) + } - // If we're not ordering by id (default case, where no order by argument is provided), we need to add the last seen id of this query in order to enable keyset pagination. - if (additionalIdOrderBy) { - cursorObj.lastSeenId = rows[rows.length - 1].id - } + // If we're not ordering by id (default case, where no order by argument is provided), we need to add the last seen id of this query in order to enable keyset pagination. + if (additionalIdOrderBy) { + cursorObj.lastSeenId = rows[rows.length - 1].id + } - // Cursor objects should be client-side opaque, hence we encode them to base64. - const cursorEncoded = Buffer.from(JSON.stringify(cursorObj), 'binary').toString( - 'base64' - ) - return { - totalCount, - objects: rows, - cursor: rows.length === limit ? cursorEncoded : null + // Cursor objects should be client-side opaque, hence we encode them to base64. + const cursorEncoded = Buffer.from(JSON.stringify(cursorObj), 'binary').toString( + 'base64' + ) + return { + totalCount, + objects: rows, + cursor: rows.length === limit ? cursorEncoded : null + } } - } diff --git a/packages/server/modules/shared/helpers/dbHelper.ts b/packages/server/modules/shared/helpers/dbHelper.ts index 5fa9e8659c..d8fac4a0b2 100644 --- a/packages/server/modules/shared/helpers/dbHelper.ts +++ b/packages/server/modules/shared/helpers/dbHelper.ts @@ -22,7 +22,7 @@ export async function* executeBatchedSelect< >( selectQuery: Knex.QueryBuilder, options?: Partial -): AsyncGenerator { +): AsyncGenerator, void, unknown> { const { batchSize = 100, trx } = options || {} if (trx) selectQuery.transacting(trx) @@ -33,7 +33,7 @@ export async function* executeBatchedSelect< let currentOffset = 0 while (hasMorePages) { const q = selectQuery.clone().offset(currentOffset) - const results = (await q) as TResult + const results = (await q) as Awaited<(typeof selectQuery)> if (!results.length) { hasMorePages = false diff --git a/packages/server/modules/workspaces/domain/operations.ts b/packages/server/modules/workspaces/domain/operations.ts index 66e39b30fa..eb54b5a106 100644 --- a/packages/server/modules/workspaces/domain/operations.ts +++ b/packages/server/modules/workspaces/domain/operations.ts @@ -308,3 +308,11 @@ export type GetWorkspaceCreationState = (params: { export type UpsertWorkspaceCreationState = (params: { workspaceCreationState: WorkspaceCreationState }) => Promise + +/** + * Project regions + */ + +export type CopyProjects = (params: { projectIds: string[] }) => Promise +export type CopyProjectModels = (params: { projectIds: string[] }) => Promise> +export type CopyProjectVersions = (params: { projectIds: string[] }) => Promise> diff --git a/packages/server/modules/workspaces/repositories/regions.ts b/packages/server/modules/workspaces/repositories/regions.ts index 20c267c622..c6688386da 100644 --- a/packages/server/modules/workspaces/repositories/regions.ts +++ b/packages/server/modules/workspaces/repositories/regions.ts @@ -1,7 +1,15 @@ -import { buildTableHelper } from '@/modules/core/dbSchema' +import { BranchCommits, Branches, buildTableHelper, Commits, StreamCommits, StreamFavorites, Streams, StreamsMeta } from '@/modules/core/dbSchema' +import { Branch } from '@/modules/core/domain/branches/types' +import { Commit } from '@/modules/core/domain/commits/types' +import { Stream } from '@/modules/core/domain/streams/types' +import { BranchCommitRecord, StreamCommitRecord, StreamFavoriteRecord } from '@/modules/core/helpers/types' import { RegionRecord } from '@/modules/multiregion/helpers/types' import { Regions } from '@/modules/multiregion/repositories' +import { executeBatchedSelect } from '@/modules/shared/helpers/dbHelper' import { + CopyProjectModels, + CopyProjects, + CopyProjectVersions, GetDefaultRegion, UpsertRegionAssignment } from '@/modules/workspaces/domain/operations' @@ -15,32 +23,156 @@ export const WorkspaceRegions = buildTableHelper('workspace_regions', [ const tables = { workspaceRegions: (db: Knex) => db(WorkspaceRegions.name), - regions: (db: Knex) => db(Regions.name) + regions: (db: Knex) => db(Regions.name), + projects: (db: Knex) => db(Streams.name), + models: (db: Knex) => db(Branches.name), + versions: (db: Knex) => db(Commits.name), + branchCommits: (db: Knex) => db(BranchCommits.name), + streamCommits: (db: Knex) => db(StreamCommits.name), + streamFavorites: (db: Knex) => db(StreamFavorites.name), + streamsMeta: (db: Knex) => db(StreamsMeta.name) } export const upsertRegionAssignmentFactory = (deps: { db: Knex }): UpsertRegionAssignment => - async (params) => { - const { workspaceId, regionKey } = params - const [row] = await tables - .workspaceRegions(deps.db) - .insert({ workspaceId, regionKey }, '*') - .onConflict(['workspaceId', 'regionKey']) - .merge() + async (params) => { + const { workspaceId, regionKey } = params + const [row] = await tables + .workspaceRegions(deps.db) + .insert({ workspaceId, regionKey }, '*') + .onConflict(['workspaceId', 'regionKey']) + .merge() - return row - } + return row + } export const getDefaultRegionFactory = (deps: { db: Knex }): GetDefaultRegion => - async (params) => { - const { workspaceId } = params - const row = await tables - .regions(deps.db) - .select(Regions.cols) - .join(WorkspaceRegions.name, WorkspaceRegions.col.regionKey, Regions.col.key) - .where({ [WorkspaceRegions.col.workspaceId]: workspaceId }) - .first() - - return row - } + async (params) => { + const { workspaceId } = params + const row = await tables + .regions(deps.db) + .select(Regions.cols) + .join(WorkspaceRegions.name, WorkspaceRegions.col.regionKey, Regions.col.key) + .where({ [WorkspaceRegions.col.workspaceId]: workspaceId }) + .first() + + return row + } + + +export const copyProjects = + (deps: { sourceDb: Knex, targetDb: Knex }): CopyProjects => + async ({ projectIds }) => { + const selectProjects = tables.projects(deps.sourceDb).select('*').whereIn(Streams.col.id, projectIds) + const copiedProjectIds: string[] = [] + + // Copy project record + for await (const projects of executeBatchedSelect(selectProjects)) { + for (const project of projects) { + // Store copied project id + copiedProjectIds.push(project.id) + + // Copy `streams` row to target db + await tables.projects(deps.targetDb) + .insert(project) + .onConflict() + .ignore() + } + + const projectIds = projects.map((project) => project.id) + + // Fetch `stream_favorites` rows for projects in batch + const selectStreamFavorites = tables.streamFavorites(deps.sourceDb).select('*').whereIn(StreamFavorites.col.streamId, projectIds) + + for await (const streamFavorites of executeBatchedSelect(selectStreamFavorites)) { + for (const streamFavorite of streamFavorites) { + // Copy `stream_favorites` row to target db + await tables.streamFavorites(deps.targetDb).insert(streamFavorite).onConflict().ignore() + } + } + + // Fetch `streams_meta` rows for projects in batch + const selectStreamsMetadata = tables.streamsMeta(deps.sourceDb).select('*').whereIn(StreamsMeta.col.streamId, projectIds) + + for await (const streamsMetadataBatch of executeBatchedSelect(selectStreamsMetadata)) { + for (const streamMetadata of streamsMetadataBatch) { + // Copy `streams_meta` row to target db + await tables.streamsMeta(deps.targetDb).insert(streamMetadata).onConflict().ignore() + } + } + } + + return copiedProjectIds + } + +export const copyProjectModels = + (deps: { sourceDb: Knex, targetDb: Knex }): CopyProjectModels => + async ({ projectIds }) => { + const copiedModelIds: Record = projectIds.reduce((result, id) => ({ ...result, [id]: [] }), {}) + + for (const projectId of projectIds) { + const selectModels = tables.models(deps.sourceDb).select('*').where({ streamId: projectId }) + + for await (const models of executeBatchedSelect(selectModels)) { + for (const model of models) { + // Store copied model ids + copiedModelIds[projectId].push(model.id) + + // Copy `branches` row to target db + await tables.models(deps.targetDb).insert(model).onConflict().ignore() + } + } + } + + return copiedModelIds + } + +export const copyProjectVersions = + (deps: { sourceDb: Knex, targetDb: Knex }): CopyProjectVersions => + async ({ projectIds }) => { + const copiedVersionIds: Record = projectIds.reduce((result, id) => ({ ...result, [id]: [] }), {}) + + for (const projectId of projectIds) { + const selectVersions = tables.streamCommits(deps.sourceDb).select('*') + .join(Commits.name, Commits.col.id, StreamCommits.col.commitId) + .where({ streamId: projectId }) + + for await (const versions of executeBatchedSelect(selectVersions)) { + for (const version of versions) { + const { commitId, ...commit } = version + + // Store copied version id + copiedVersionIds[projectId].push(commitId) + + // Copy `commits` row to target db + await tables.versions(deps.targetDb).insert(commit).onConflict().ignore() + } + + const commitIds = versions.map((version) => version.commitId) + + // Fetch `branch_commits` rows for versions in batch + const selectBranchCommits = tables.branchCommits(deps.sourceDb).select('*').whereIn(BranchCommits.col.commitId, commitIds) + + for await (const branchCommits of executeBatchedSelect(selectBranchCommits)) { + for (const branchCommit of branchCommits) { + // Copy `branch_commits` row to target db + await tables.branchCommits(deps.targetDb).insert(branchCommit).onConflict().ignore() + } + } + + // Fetch `stream_commits` rows for versions in batch + const selectStreamCommits = tables.streamCommits(deps.sourceDb).select('*').whereIn(StreamCommits.col.commitId, commitIds) + + for await (const streamCommits of executeBatchedSelect(selectStreamCommits)) { + for (const streamCommit of streamCommits) { + // Copy `stream_commits` row to target db + await tables.streamCommits(deps.targetDb).insert(streamCommit).onConflict().ignore() + } + } + } + } + + return copiedVersionIds + } + diff --git a/packages/server/modules/workspaces/services/projects.ts b/packages/server/modules/workspaces/services/projects.ts index 886a28c99c..f8d81f0581 100644 --- a/packages/server/modules/workspaces/services/projects.ts +++ b/packages/server/modules/workspaces/services/projects.ts @@ -1,5 +1,8 @@ import { StreamRecord } from '@/modules/core/helpers/types' import { + CopyProjectModels, + CopyProjects, + CopyProjectVersions, GetDefaultRegion, GetWorkspace, GetWorkspaceRoleForUser, @@ -97,23 +100,23 @@ type GetWorkspaceProjectsReturnValue = { export const getWorkspaceProjectsFactory = ({ getStreams }: { getStreams: GetUserStreamsPage }) => - async ( - args: GetWorkspaceProjectsArgs, - opts: GetWorkspaceProjectsOptions - ): Promise => { - const { streams, cursor } = await getStreams({ - cursor: opts.cursor, - limit: opts.limit || 25, - searchQuery: opts.filter?.search || undefined, - workspaceId: args.workspaceId, - userId: opts.filter.userId - }) + async ( + args: GetWorkspaceProjectsArgs, + opts: GetWorkspaceProjectsOptions + ): Promise => { + const { streams, cursor } = await getStreams({ + cursor: opts.cursor, + limit: opts.limit || 25, + searchQuery: opts.filter?.search || undefined, + workspaceId: args.workspaceId, + userId: opts.filter.userId + }) - return { - items: streams, - cursor + return { + items: streams, + cursor + } } - } type MoveProjectToWorkspaceArgs = { projectId: string @@ -138,66 +141,78 @@ export const moveProjectToWorkspaceFactory = getWorkspaceRoleToDefaultProjectRoleMapping: GetWorkspaceRoleToDefaultProjectRoleMapping updateWorkspaceRole: UpdateWorkspaceRole }) => - async ({ - projectId, - workspaceId - }: MoveProjectToWorkspaceArgs): Promise => { - const project = await getProject({ projectId }) + async ({ + projectId, + workspaceId + }: MoveProjectToWorkspaceArgs): Promise => { + const project = await getProject({ projectId }) - if (!project) throw new ProjectNotFoundError() - if (project.workspaceId?.length) { - // We do not currently support moving projects between workspaces - throw new WorkspaceInvalidProjectError( - 'Specified project already belongs to a workspace. Moving between workspaces is not yet supported.' + if (!project) throw new ProjectNotFoundError() + if (project.workspaceId?.length) { + // We do not currently support moving projects between workspaces + throw new WorkspaceInvalidProjectError( + 'Specified project already belongs to a workspace. Moving between workspaces is not yet supported.' + ) + } + + // Update roles for current project members + const projectTeam = await getProjectCollaborators({ projectId }) + const workspaceTeam = await getWorkspaceRoles({ workspaceId }) + const defaultProjectRoleMapping = await getWorkspaceRoleToDefaultProjectRoleMapping( + { workspaceId } ) - } - // Update roles for current project members - const projectTeam = await getProjectCollaborators({ projectId }) - const workspaceTeam = await getWorkspaceRoles({ workspaceId }) - const defaultProjectRoleMapping = await getWorkspaceRoleToDefaultProjectRoleMapping( - { workspaceId } - ) + for (const projectMembers of chunk(projectTeam, 5)) { + await Promise.all( + projectMembers.map( + async ({ id: userId, role: serverRole, streamRole: currentProjectRole }) => { + // Update workspace role. Prefer existing workspace role if there is one. + const currentWorkspaceRole = workspaceTeam.find( + (role) => role.userId === userId + ) + const nextWorkspaceRole = currentWorkspaceRole ?? { + userId, + workspaceId, + role: + serverRole === Roles.Server.Guest + ? Roles.Workspace.Guest + : Roles.Workspace.Member, + createdAt: new Date() + } + await updateWorkspaceRole(nextWorkspaceRole) - for (const projectMembers of chunk(projectTeam, 5)) { - await Promise.all( - projectMembers.map( - async ({ id: userId, role: serverRole, streamRole: currentProjectRole }) => { - // Update workspace role. Prefer existing workspace role if there is one. - const currentWorkspaceRole = workspaceTeam.find( - (role) => role.userId === userId - ) - const nextWorkspaceRole = currentWorkspaceRole ?? { - userId, - workspaceId, - role: - serverRole === Roles.Server.Guest - ? Roles.Workspace.Guest - : Roles.Workspace.Member, - createdAt: new Date() + // Update project role. Prefer default workspace project role if more permissive. + const defaultProjectRole = + defaultProjectRoleMapping[nextWorkspaceRole.role] ?? Roles.Stream.Reviewer + const nextProjectRole = orderByWeight( + [currentProjectRole, defaultProjectRole], + coreUserRoles + )[0] + await upsertProjectRole({ + userId, + projectId, + role: nextProjectRole.name as StreamRoles + }) } - await updateWorkspaceRole(nextWorkspaceRole) - - // Update project role. Prefer default workspace project role if more permissive. - const defaultProjectRole = - defaultProjectRoleMapping[nextWorkspaceRole.role] ?? Roles.Stream.Reviewer - const nextProjectRole = orderByWeight( - [currentProjectRole, defaultProjectRole], - coreUserRoles - )[0] - await upsertProjectRole({ - userId, - projectId, - role: nextProjectRole.name as StreamRoles - }) - } + ) ) - ) + } + + // Assign project to workspace + return await updateProject({ projectUpdate: { id: projectId, workspaceId } }) } - // Assign project to workspace - return await updateProject({ projectUpdate: { id: projectId, workspaceId } }) - } +export const moveProjectToRegionFactory = + (deps: { + copyProjects: CopyProjects, + copyProjectModels: CopyProjectModels, + copyProjectVersions: CopyProjectVersions + }) => + async (args: { projectId: string }): Promise => { + const projectIds = await deps.copyProjects({ projectIds: [args.projectId] }) + const modelIdsByProjectId = await deps.copyProjectModels({ projectIds }) + const versionIdsByProjectId = await deps.copyProjectVersions({ projectIds }) + } export const getWorkspaceRoleToDefaultProjectRoleMappingFactory = ({ @@ -205,19 +220,19 @@ export const getWorkspaceRoleToDefaultProjectRoleMappingFactory = }: { getWorkspace: GetWorkspace }): GetWorkspaceRoleToDefaultProjectRoleMapping => - async ({ workspaceId }) => { - const workspace = await getWorkspace({ workspaceId }) + async ({ workspaceId }) => { + const workspace = await getWorkspace({ workspaceId }) - if (!workspace) { - throw new WorkspaceNotFoundError() - } + if (!workspace) { + throw new WorkspaceNotFoundError() + } - return { - [Roles.Workspace.Guest]: null, - [Roles.Workspace.Member]: workspace.defaultProjectRole, - [Roles.Workspace.Admin]: Roles.Stream.Owner + return { + [Roles.Workspace.Guest]: null, + [Roles.Workspace.Member]: workspace.defaultProjectRole, + [Roles.Workspace.Admin]: Roles.Stream.Owner + } } - } export const updateWorkspaceProjectRoleFactory = ({ @@ -229,63 +244,63 @@ export const updateWorkspaceProjectRoleFactory = getWorkspaceRoleForUser: GetWorkspaceRoleForUser updateStreamRoleAndNotify: UpdateStreamRole }): UpdateWorkspaceProjectRole => - async ({ role, updater }) => { - const { workspaceId } = (await getStream({ streamId: role.projectId })) ?? {} - if (!workspaceId) throw new WorkspaceInvalidProjectError() + async ({ role, updater }) => { + const { workspaceId } = (await getStream({ streamId: role.projectId })) ?? {} + if (!workspaceId) throw new WorkspaceInvalidProjectError() - const currentWorkspaceRole = await getWorkspaceRoleForUser({ - workspaceId, - userId: role.userId - }) + const currentWorkspaceRole = await getWorkspaceRoleForUser({ + workspaceId, + userId: role.userId + }) - if (currentWorkspaceRole?.role === Roles.Workspace.Admin) { - // User is workspace admin and cannot have their project roles changed - throw new WorkspaceAdminError() - } + if (currentWorkspaceRole?.role === Roles.Workspace.Admin) { + // User is workspace admin and cannot have their project roles changed + throw new WorkspaceAdminError() + } - if ( - currentWorkspaceRole?.role === Roles.Workspace.Guest && - role.role === Roles.Stream.Owner - ) { - // Workspace guests cannot be project owners - throw new WorkspaceInvalidRoleError('Workspace guests cannot be project owners.') - } + if ( + currentWorkspaceRole?.role === Roles.Workspace.Guest && + role.role === Roles.Stream.Owner + ) { + // Workspace guests cannot be project owners + throw new WorkspaceInvalidRoleError('Workspace guests cannot be project owners.') + } - return await updateStreamRoleAndNotify( - role, - updater.userId!, - updater.resourceAccessRules - ) - } + return await updateStreamRoleAndNotify( + role, + updater.userId!, + updater.resourceAccessRules + ) + } export const createWorkspaceProjectFactory = (deps: { getDefaultRegion: GetDefaultRegion }) => - async (params: { input: WorkspaceProjectCreateInput; ownerId: string }) => { - const { input, ownerId } = params - const workspaceDefaultRegion = await deps.getDefaultRegion({ - workspaceId: input.workspaceId - }) - const regionKey = workspaceDefaultRegion?.key - const projectDb = await getDb({ regionKey }) - const db = mainDb + async (params: { input: WorkspaceProjectCreateInput; ownerId: string }) => { + const { input, ownerId } = params + const workspaceDefaultRegion = await deps.getDefaultRegion({ + workspaceId: input.workspaceId + }) + const regionKey = workspaceDefaultRegion?.key + const projectDb = await getDb({ regionKey }) + const db = mainDb - // todo, use the command factory here, but for that, we need to migrate to the event bus - // deps not injected to ensure proper DB injection - const createNewProject = createNewProjectFactory({ - storeProject: storeProjectFactory({ db: projectDb }), - getProject: getProjectFactory({ db }), - deleteProject: deleteProjectFactory({ db: projectDb }), - storeModel: storeModelFactory({ db: projectDb }), - // THIS MUST GO TO THE MAIN DB - storeProjectRole: storeProjectRoleFactory({ db }), - projectsEventsEmitter: ProjectsEmitter.emit - }) + // todo, use the command factory here, but for that, we need to migrate to the event bus + // deps not injected to ensure proper DB injection + const createNewProject = createNewProjectFactory({ + storeProject: storeProjectFactory({ db: projectDb }), + getProject: getProjectFactory({ db }), + deleteProject: deleteProjectFactory({ db: projectDb }), + storeModel: storeModelFactory({ db: projectDb }), + // THIS MUST GO TO THE MAIN DB + storeProjectRole: storeProjectRoleFactory({ db }), + projectsEventsEmitter: ProjectsEmitter.emit + }) - const project = await createNewProject({ - ...input, - regionKey, - ownerId - }) + const project = await createNewProject({ + ...input, + regionKey, + ownerId + }) - return project - } + return project + } From e5d04e5a3c090b76f746d903b0960e5867914613 Mon Sep 17 00:00:00 2001 From: Chuck Driesler Date: Mon, 13 Jan 2025 16:19:33 +0000 Subject: [PATCH 02/28] chore(regions): wire up move to resolver --- .../workspacesCore/typedefs/regions.graphql | 8 + .../typedefs/workspaces.graphql | 1 - .../modules/core/graph/generated/graphql.ts | 8 + .../modules/core/repositories/objects.ts | 824 +++++++++--------- .../graph/generated/graphql.ts | 7 + .../server/modules/shared/helpers/dbHelper.ts | 4 +- .../modules/workspaces/domain/operations.ts | 18 +- .../modules/workspaces/errors/regions.ts | 6 + .../workspaces/graph/resolvers/regions.ts | 45 +- .../workspaces/repositories/regions.ts | 290 +++--- .../modules/workspaces/services/projects.ts | 270 +++--- .../modules/workspaces/services/regions.ts | 88 +- .../workspaces/tests/helpers/creation.ts | 4 +- .../server/test/graphql/generated/graphql.ts | 7 + 14 files changed, 900 insertions(+), 680 deletions(-) diff --git a/packages/server/assets/workspacesCore/typedefs/regions.graphql b/packages/server/assets/workspacesCore/typedefs/regions.graphql index b394c86555..e61b93102a 100644 --- a/packages/server/assets/workspacesCore/typedefs/regions.graphql +++ b/packages/server/assets/workspacesCore/typedefs/regions.graphql @@ -12,3 +12,11 @@ extend type WorkspaceMutations { """ setDefaultRegion(workspaceId: String!, regionKey: String!): Workspace! } + +extend type WorkspaceProjectMutations { + """ + Update project region and move all regional data to new db. + TODO: Currently performs all operations synchronously in request, should probably be scheduled. + """ + moveToRegion(projectId: String!, regionKey: String!): Project! +} diff --git a/packages/server/assets/workspacesCore/typedefs/workspaces.graphql b/packages/server/assets/workspacesCore/typedefs/workspaces.graphql index 8c5d699975..226b28a18e 100644 --- a/packages/server/assets/workspacesCore/typedefs/workspaces.graphql +++ b/packages/server/assets/workspacesCore/typedefs/workspaces.graphql @@ -170,7 +170,6 @@ type WorkspaceProjectMutations { updateRole(input: ProjectUpdateRoleInput!): Project! @hasStreamRole(role: STREAM_OWNER) @hasWorkspaceRole(role: MEMBER) - moveToRegion(projectId: String!, regionKey: String!): Project! moveToWorkspace(projectId: String!, workspaceId: String!): Project! create(input: WorkspaceProjectCreateInput!): Project! } diff --git a/packages/server/modules/core/graph/generated/graphql.ts b/packages/server/modules/core/graph/generated/graphql.ts index c6b5b27351..9f57629824 100644 --- a/packages/server/modules/core/graph/generated/graphql.ts +++ b/packages/server/modules/core/graph/generated/graphql.ts @@ -4514,6 +4514,7 @@ export type WorkspaceProjectInviteCreateInput = { export type WorkspaceProjectMutations = { __typename?: 'WorkspaceProjectMutations'; create: Project; + moveToRegion: Project; moveToWorkspace: Project; updateRole: Project; }; @@ -4524,6 +4525,12 @@ export type WorkspaceProjectMutationsCreateArgs = { }; +export type WorkspaceProjectMutationsMoveToRegionArgs = { + projectId: Scalars['String']['input']; + regionKey: Scalars['String']['input']; +}; + + export type WorkspaceProjectMutationsMoveToWorkspaceArgs = { projectId: Scalars['String']['input']; workspaceId: Scalars['String']['input']; @@ -6799,6 +6806,7 @@ export type WorkspacePlanResolvers = { create?: Resolver>; + moveToRegion?: Resolver>; moveToWorkspace?: Resolver>; updateRole?: Resolver>; __isTypeOf?: IsTypeOfResolverFn; diff --git a/packages/server/modules/core/repositories/objects.ts b/packages/server/modules/core/repositories/objects.ts index bdf8a82ef8..851bf78fa3 100644 --- a/packages/server/modules/core/repositories/objects.ts +++ b/packages/server/modules/core/repositories/objects.ts @@ -40,502 +40,502 @@ const tables = { export const getStreamObjectsFactory = (deps: { db: Knex }): GetStreamObjects => - async (streamId: string, objectIds: string[]): Promise => { - if (!objectIds?.length) return [] + async (streamId: string, objectIds: string[]): Promise => { + if (!objectIds?.length) return [] - const q = tables - .objects(deps.db) - .where(Objects.col.streamId, streamId) - .whereIn(Objects.col.id, objectIds) + const q = tables + .objects(deps.db) + .where(Objects.col.streamId, streamId) + .whereIn(Objects.col.id, objectIds) - return await q - } + return await q + } export const getObjectFactory = (deps: { db: Knex }): GetObject => - async (objectId: string, streamId: string): Promise> => { - return await tables - .objects(deps.db) - .where(Objects.col.id, objectId) - .andWhere(Objects.col.streamId, streamId) - .first() - } + async (objectId: string, streamId: string): Promise> => { + return await tables + .objects(deps.db) + .where(Objects.col.id, objectId) + .andWhere(Objects.col.streamId, streamId) + .first() + } export const getFormattedObjectFactory = (deps: { db: Knex }): GetFormattedObject => - async ({ streamId, objectId }) => { - const res = await tables - .objects(deps.db) - .where({ streamId, id: objectId }) - .select('*') - .first() - if (!res) return null - - // TODO: Why tho? A lot if not most of places already just use getObjectFactory, - const finalRes: SetOptional = res - if (finalRes.data) finalRes.data.totalChildrenCount = res.totalChildrenCount // move this back - delete finalRes.streamId // backwards compatibility - - return finalRes - } + async ({ streamId, objectId }) => { + const res = await tables + .objects(deps.db) + .where({ streamId, id: objectId }) + .select('*') + .first() + if (!res) return null + + // TODO: Why tho? A lot if not most of places already just use getObjectFactory, + const finalRes: SetOptional = res + if (finalRes.data) finalRes.data.totalChildrenCount = res.totalChildrenCount // move this back + delete finalRes.streamId // backwards compatibility + + return finalRes + } export const getBatchedStreamObjectsFactory = (deps: { db: Knex }): GetBatchedStreamObjects => - (streamId: string, options?: Partial) => { - const baseQuery = tables - .objects(deps.db) - .select('*') - .where(Objects.col.streamId, streamId) - .orderBy(Objects.col.id) - - return executeBatchedSelect(baseQuery, options) - } + (streamId: string, options?: Partial) => { + const baseQuery = tables + .objects(deps.db) + .select('*') + .where(Objects.col.streamId, streamId) + .orderBy(Objects.col.id) + + return executeBatchedSelect(baseQuery, options) + } export const insertObjectsFactory = (deps: { db: Knex }): StoreObjects => - async (objects: ObjectRecord[], options?: Partial<{ trx: Knex.Transaction }>) => { - const q = tables.objects(deps.db).insert(objects) - if (options?.trx) q.transacting(options.trx) - return await q - } + async (objects: ObjectRecord[], options?: Partial<{ trx: Knex.Transaction }>) => { + const q = tables.objects(deps.db).insert(objects) + if (options?.trx) q.transacting(options.trx) + return await q + } export const storeSingleObjectIfNotFoundFactory = (deps: { db: Knex }): StoreSingleObjectIfNotFound => - async (insertionObject) => { - await tables - .objects(deps.db) - .insert( - // knex is bothered by string being inserted into jsonb, which is actually fine - insertionObject as SpeckleObject - ) - .onConflict() - .ignore() - } + async (insertionObject) => { + await tables + .objects(deps.db) + .insert( + // knex is bothered by string being inserted into jsonb, which is actually fine + insertionObject as SpeckleObject + ) + .onConflict() + .ignore() + } export const storeObjectsIfNotFoundFactory = (deps: { db: Knex }): StoreObjectsIfNotFound => - async (batch) => { - await tables - .objects(deps.db) - .insert( - // knex is bothered by string being inserted into jsonb, which is actually fine - batch as SpeckleObject[] - ) - .onConflict() - .ignore() - } + async (batch) => { + await tables + .objects(deps.db) + .insert( + // knex is bothered by string being inserted into jsonb, which is actually fine + batch as SpeckleObject[] + ) + .onConflict() + .ignore() + } export const storeClosuresIfNotFoundFactory = (deps: { db: Knex }): StoreClosuresIfNotFound => - async (closuresBatch) => { - await tables - .objectChildrenClosure(deps.db) - .insert(closuresBatch) - .onConflict() - .ignore() - } + async (closuresBatch) => { + await tables + .objectChildrenClosure(deps.db) + .insert(closuresBatch) + .onConflict() + .ignore() + } export const getObjectChildrenStreamFactory = (deps: { db: Knex }): GetObjectChildrenStream => - async ({ streamId, objectId }) => { - const q = deps.db.with( - 'object_children_closure', - knex.raw( - `SELECT objects.id as parent, d.key as child, d.value as mindepth, ? as "streamId" + async ({ streamId, objectId }) => { + const q = deps.db.with( + 'object_children_closure', + knex.raw( + `SELECT objects.id as parent, d.key as child, d.value as mindepth, ? as "streamId" FROM objects JOIN jsonb_each_text(objects.data->'__closure') d ON true where objects.id = ?`, - [streamId, objectId] - ) + [streamId, objectId] ) - q.select('id') - q.select(knex.raw('data::text as "dataText"')) - q.from('object_children_closure') - - q.rightJoin('objects', function () { - this.on('objects.streamId', '=', 'object_children_closure.streamId').andOn( - 'objects.id', - '=', - 'object_children_closure.child' - ) - }) - .where( - knex.raw('object_children_closure."streamId" = ? AND parent = ?', [ - streamId, - objectId - ]) - ) - .orderBy('objects.id') - return q.stream({ highWaterMark: 500 }) - } + ) + q.select('id') + q.select(knex.raw('data::text as "dataText"')) + q.from('object_children_closure') + + q.rightJoin('objects', function () { + this.on('objects.streamId', '=', 'object_children_closure.streamId').andOn( + 'objects.id', + '=', + 'object_children_closure.child' + ) + }) + .where( + knex.raw('object_children_closure."streamId" = ? AND parent = ?', [ + streamId, + objectId + ]) + ) + .orderBy('objects.id') + return q.stream({ highWaterMark: 500 }) + } export const getObjectsStreamFactory = (deps: { db: Knex }): GetObjectsStream => - async ({ streamId, objectIds }) => { - const res = tables - .objects(deps.db) - .whereIn('id', objectIds) - .andWhere('streamId', streamId) - .orderBy('id') - .select( - knex.raw( - '"id", "speckleType", "totalChildrenCount", "totalChildrenCountByDepth", "createdAt", data::text as "dataText"' - ) + async ({ streamId, objectIds }) => { + const res = tables + .objects(deps.db) + .whereIn('id', objectIds) + .andWhere('streamId', streamId) + .orderBy('id') + .select( + knex.raw( + '"id", "speckleType", "totalChildrenCount", "totalChildrenCountByDepth", "createdAt", data::text as "dataText"' ) - return res.stream({ highWaterMark: 500 }) - } + ) + return res.stream({ highWaterMark: 500 }) + } export const hasObjectsFactory = (deps: { db: Knex }): HasObjects => - async ({ streamId, objectIds }) => { - const dbRes = await tables - .objects(deps.db) - .whereIn('id', objectIds) - .andWhere('streamId', streamId) - .select('id') - - const res: Record = {} - // eslint-disable-next-line @typescript-eslint/no-for-in-array - for (const i in objectIds) { - res[objectIds[i]] = false - } - // eslint-disable-next-line @typescript-eslint/no-for-in-array - for (const i in dbRes) { - res[dbRes[i].id] = true - } - return res + async ({ streamId, objectIds }) => { + const dbRes = await tables + .objects(deps.db) + .whereIn('id', objectIds) + .andWhere('streamId', streamId) + .select('id') + + const res: Record = {} + // eslint-disable-next-line @typescript-eslint/no-for-in-array + for (const i in objectIds) { + res[objectIds[i]] = false + } + // eslint-disable-next-line @typescript-eslint/no-for-in-array + for (const i in dbRes) { + res[dbRes[i].id] = true } + return res + } export const getObjectChildrenFactory = (deps: { db: Knex }): GetObjectChildren => - async ({ streamId, objectId, limit, depth, select, cursor }) => { - limit = toNumber(limit || 0) || 50 - depth = toNumber(depth || 0) || 1000 + async ({ streamId, objectId, limit, depth, select, cursor }) => { + limit = toNumber(limit || 0) || 50 + depth = toNumber(depth || 0) || 1000 - let fullObjectSelect = false + let fullObjectSelect = false - const q = deps.db.with( - 'object_children_closure', - knex.raw( - `SELECT objects.id as parent, d.key as child, d.value as mindepth, ? as "streamId" + const q = deps.db.with( + 'object_children_closure', + knex.raw( + `SELECT objects.id as parent, d.key as child, d.value as mindepth, ? as "streamId" FROM objects JOIN jsonb_each_text(objects.data->'__closure') d ON true where objects.id = ?`, - [streamId, objectId] - ) + [streamId, objectId] ) - - if (Array.isArray(select)) { - select.forEach((field, index) => { - q.select( - knex.raw('jsonb_path_query(data, :path) as :name:', { - path: '$.' + field, - name: '' + index - }) - ) - }) - } else { - fullObjectSelect = true - q.select('data') - } - - q.select('id') - q.select('createdAt') - q.select('speckleType') - q.select('totalChildrenCount') - - q.from('object_children_closure') - - q.rightJoin('objects', function () { - this.on('objects.streamId', '=', 'object_children_closure.streamId').andOn( - 'objects.id', - '=', - 'object_children_closure.child' + ) + + if (Array.isArray(select)) { + select.forEach((field, index) => { + q.select( + knex.raw('jsonb_path_query(data, :path) as :name:', { + path: '$.' + field, + name: '' + index + }) ) }) - .where( - knex.raw('object_children_closure."streamId" = ? AND parent = ?', [ - streamId, - objectId - ]) - ) - .andWhere(knex.raw('object_children_closure.mindepth < ?', [depth])) - .andWhere(knex.raw('id > ?', [cursor ? cursor : '0'])) - .orderBy('objects.id') - .limit(limit) + } else { + fullObjectSelect = true + q.select('data') + } - const rows = await q + q.select('id') + q.select('createdAt') + q.select('speckleType') + q.select('totalChildrenCount') - if (rows.length === 0) { - return { objects: rows, cursor: null } - } + q.from('object_children_closure') - if (!fullObjectSelect) - rows.forEach((o, i, arr) => { - const no = { - id: o.id, - createdAt: o.createdAt, - speckleType: o.speckleType, - totalChildrenCount: o.totalChildrenCount, - data: {} - } - let k = 0 - for (const field of select || []) { - set(no.data, field, o[k++]) - } - arr[i] = no - }) + q.rightJoin('objects', function () { + this.on('objects.streamId', '=', 'object_children_closure.streamId').andOn( + 'objects.id', + '=', + 'object_children_closure.child' + ) + }) + .where( + knex.raw('object_children_closure."streamId" = ? AND parent = ?', [ + streamId, + objectId + ]) + ) + .andWhere(knex.raw('object_children_closure.mindepth < ?', [depth])) + .andWhere(knex.raw('id > ?', [cursor ? cursor : '0'])) + .orderBy('objects.id') + .limit(limit) + + const rows = await q - const lastId = rows[rows.length - 1].id - return { objects: rows, cursor: lastId } + if (rows.length === 0) { + return { objects: rows, cursor: null } } + if (!fullObjectSelect) + rows.forEach((o, i, arr) => { + const no = { + id: o.id, + createdAt: o.createdAt, + speckleType: o.speckleType, + totalChildrenCount: o.totalChildrenCount, + data: {} + } + let k = 0 + for (const field of select || []) { + set(no.data, field, o[k++]) + } + arr[i] = no + }) + + const lastId = rows[rows.length - 1].id + return { objects: rows, cursor: lastId } + } + /** * This query is inefficient on larger sets (n * 10k objects) as we need to return the total count on an arbitrarily (user) defined selection of objects. * A possible future optimisation route would be to cache the total count of a query (as objects are immutable, it will not change) on a first run, and, if found on a subsequent round, do a simpler query and merge the total count result. */ export const getObjectChildrenQueryFactory = (deps: { db: Knex }): GetObjectChildrenQuery => - async (params) => { - const { streamId, objectId, select, query } = params - - const limit = toNumber(params.limit || 0) || 50 - const depth = toNumber(params.depth || 0) || 1000 - const orderBy = params.orderBy || { field: 'id', direction: 'asc' } - - // Cursors received by this service should be base64 encoded. They are generated on first entry query by this service; They should never be client-side generated. - const cursor: Optional<{ - value: unknown - operator: string - field: string - lastSeenId?: string - }> = params.cursor - ? JSON.parse(Buffer.from(params.cursor, 'base64').toString('binary')) - : undefined - - // Flag that keeps track of whether we select the whole "data" part of an object or not - let fullObjectSelect = false - if (Array.isArray(select)) { - // if we order by a field that we do not select, select it! - if (orderBy && select.indexOf(orderBy.field) === -1) { - select.push(orderBy.field) - } - // // always add the id! - // if ( select.indexOf( 'id' ) === -1 ) select.unshift( 'id' ) - } else { - fullObjectSelect = true + async (params) => { + const { streamId, objectId, select, query } = params + + const limit = toNumber(params.limit || 0) || 50 + const depth = toNumber(params.depth || 0) || 1000 + const orderBy = params.orderBy || { field: 'id', direction: 'asc' } + + // Cursors received by this service should be base64 encoded. They are generated on first entry query by this service; They should never be client-side generated. + const cursor: Optional<{ + value: unknown + operator: string + field: string + lastSeenId?: string + }> = params.cursor + ? JSON.parse(Buffer.from(params.cursor, 'base64').toString('binary')) + : undefined + + // Flag that keeps track of whether we select the whole "data" part of an object or not + let fullObjectSelect = false + if (Array.isArray(select)) { + // if we order by a field that we do not select, select it! + if (orderBy && select.indexOf(orderBy.field) === -1) { + select.push(orderBy.field) } + // // always add the id! + // if ( select.indexOf( 'id' ) === -1 ) select.unshift( 'id' ) + } else { + fullObjectSelect = true + } - const additionalIdOrderBy = orderBy.field !== 'id' + const additionalIdOrderBy = orderBy.field !== 'id' - const operatorsWhitelist = ['=', '>', '>=', '<', '<=', '!='] + const operatorsWhitelist = ['=', '>', '>=', '<', '<=', '!='] - const mainQuery = deps.db - .with( - 'object_children_closure', - knex.raw( - `SELECT objects.id as parent, d.key as child, d.value as mindepth, ? as "streamId" + const mainQuery = deps.db + .with( + 'object_children_closure', + knex.raw( + `SELECT objects.id as parent, d.key as child, d.value as mindepth, ? as "streamId" FROM objects JOIN jsonb_each_text(objects.data->'__closure') d ON true where objects.id = ?`, - [streamId, objectId] - ) + [streamId, objectId] ) - .with('objs', (cteInnerQuery) => { - // always select the id - cteInnerQuery.select('id').from('object_children_closure') - cteInnerQuery.select('createdAt') - cteInnerQuery.select('speckleType') - cteInnerQuery.select('totalChildrenCount') - - // if there are any select fields, add them - if (Array.isArray(select)) { - select.forEach((field, index) => { - cteInnerQuery.select( - knex.raw('jsonb_path_query(data, :path) as :name:', { - path: '$.' + field, - name: '' + index - }) - ) - }) - // otherwise, get the whole object, as stored in the jsonb column - } else { - cteInnerQuery.select('data') - } - - // join on objects table - cteInnerQuery - .join('objects', function () { - this.on('objects.streamId', '=', 'object_children_closure.streamId').andOn( - 'objects.id', - '=', - 'object_children_closure.child' - ) - }) - .where('object_children_closure.streamId', streamId) - .andWhere('parent', objectId) - .andWhere('mindepth', '<', depth) - - // Add user provided filters/queries. - if (Array.isArray(query) && query.length > 0) { - cteInnerQuery.andWhere((nestedWhereQuery) => { - query.forEach((statement, index) => { - let castType = 'text' - if (typeof statement.value === 'string') castType = 'text' - if (typeof statement.value === 'boolean') castType = 'boolean' - if (typeof statement.value === 'number') castType = 'numeric' - - if (operatorsWhitelist.indexOf(statement.operator) === -1) - throw new Error('Invalid operator for query') - - // Determine the correct where clause (where, and where, or where) - let whereClause: keyof typeof nestedWhereQuery - if (index === 0) whereClause = 'where' - else if (statement.verb && statement.verb.toLowerCase() === 'or') - whereClause = 'orWhere' - else whereClause = 'andWhere' - - // Note: castType is generated from the statement's value and operators are matched against a whitelist. - // If comparing with strings, the jsonb_path_query(_first) func returns json encoded strings (ie, `bar` is actually `"bar"`), hence we need to add the quotes manually to the raw provided comparison value. - nestedWhereQuery[whereClause]( - knex.raw( - `jsonb_path_query_first( data, ? )::${castType} ${statement.operator} ? `, - [ - '$.' + statement.field, - castType === 'text' ? `"${statement.value}"` : statement.value - ] - ) - ) + ) + .with('objs', (cteInnerQuery) => { + // always select the id + cteInnerQuery.select('id').from('object_children_closure') + cteInnerQuery.select('createdAt') + cteInnerQuery.select('speckleType') + cteInnerQuery.select('totalChildrenCount') + + // if there are any select fields, add them + if (Array.isArray(select)) { + select.forEach((field, index) => { + cteInnerQuery.select( + knex.raw('jsonb_path_query(data, :path) as :name:', { + path: '$.' + field, + name: '' + index }) - }) - } - - // Order by clause; validate direction! - const direction = - orderBy.direction && orderBy.direction.toLowerCase() === 'desc' - ? 'desc' - : 'asc' - if (orderBy.field === 'id') { - cteInnerQuery.orderBy('id', direction) - } else { - cteInnerQuery.orderByRaw( - knex.raw(`jsonb_path_query_first( data, ? ) ${direction}, id asc`, [ - '$.' + orderBy.field - ]) - ) - } - }) - .select('*') - .from('objs') - .joinRaw('RIGHT JOIN ( SELECT count(*) FROM "objs" ) c(total_count) ON TRUE') - - // Set cursor clause, if present. If it's not present, it's an entry query; this method will return a cursor based on its given query. - // We have implemented keyset pagination for more efficient searches on larger sets. This approach depends on an order by value provided by the user and a (hidden) primary key. - // logger.debug( cursor ) - if (cursor) { - let castType = 'text' - if (typeof cursor.value === 'string') castType = 'text' - if (typeof cursor.value === 'boolean') castType = 'boolean' - if (typeof cursor.value === 'number') castType = 'numeric' - - // When strings are used inside an order clause, as mentioned above, we need to add quotes around the comparison value, as the jsonb_path_query funcs return json encoded strings (`{"test":"foo"}` => test is returned as `"foo"`) - if (castType === 'text') cursor.value = `"${cursor.value}"` - - if (operatorsWhitelist.indexOf(cursor.operator) === -1) - throw new Error('Invalid operator for cursor') - - // Unwrapping the tuple comparison of ( userOrderByField, id ) > ( lastValueOfUserOrderBy, lastSeenId ) - if (fullObjectSelect) { - if (cursor.field === 'id') { - mainQuery.where(knex.raw(`id ${cursor.operator} ? `, [cursor.value])) - } else { - mainQuery.where( - knex.raw( - `jsonb_path_query_first( data, ? )::${castType} ${cursor.operator}= ? `, - ['$.' + cursor.field, cursor.value] - ) ) - } + }) + // otherwise, get the whole object, as stored in the jsonb column } else { - mainQuery.where( - knex.raw(`??::${castType} ${cursor.operator}= ? `, [ - (select || []).indexOf(cursor.field).toString(), - cursor.value - ]) - ) + cteInnerQuery.select('data') } - if (cursor.lastSeenId) { - mainQuery.andWhere((qb) => { - // Dunno what the TS issue here is, the JS code itself seemed to work fine as JS - // eslint-disable-next-line @typescript-eslint/no-explicit-any - qb.where('id' as any, '>', cursor!.lastSeenId as any) - if (fullObjectSelect) - qb.orWhere( + // join on objects table + cteInnerQuery + .join('objects', function () { + this.on('objects.streamId', '=', 'object_children_closure.streamId').andOn( + 'objects.id', + '=', + 'object_children_closure.child' + ) + }) + .where('object_children_closure.streamId', streamId) + .andWhere('parent', objectId) + .andWhere('mindepth', '<', depth) + + // Add user provided filters/queries. + if (Array.isArray(query) && query.length > 0) { + cteInnerQuery.andWhere((nestedWhereQuery) => { + query.forEach((statement, index) => { + let castType = 'text' + if (typeof statement.value === 'string') castType = 'text' + if (typeof statement.value === 'boolean') castType = 'boolean' + if (typeof statement.value === 'number') castType = 'numeric' + + if (operatorsWhitelist.indexOf(statement.operator) === -1) + throw new Error('Invalid operator for query') + + // Determine the correct where clause (where, and where, or where) + let whereClause: keyof typeof nestedWhereQuery + if (index === 0) whereClause = 'where' + else if (statement.verb && statement.verb.toLowerCase() === 'or') + whereClause = 'orWhere' + else whereClause = 'andWhere' + + // Note: castType is generated from the statement's value and operators are matched against a whitelist. + // If comparing with strings, the jsonb_path_query(_first) func returns json encoded strings (ie, `bar` is actually `"bar"`), hence we need to add the quotes manually to the raw provided comparison value. + nestedWhereQuery[whereClause]( knex.raw( - `jsonb_path_query_first( data, ? )::${castType} ${cursor!.operator} ? `, - ['$.' + cursor!.field, cursor!.value] + `jsonb_path_query_first( data, ? )::${castType} ${statement.operator} ? `, + [ + '$.' + statement.field, + castType === 'text' ? `"${statement.value}"` : statement.value + ] ) ) - else - qb.orWhere( - knex.raw(`??::${castType} ${cursor!.operator} ? `, [ - (select || []).indexOf(cursor!.field).toString(), - cursor!.value - ]) - ) + }) }) } + + // Order by clause; validate direction! + const direction = + orderBy.direction && orderBy.direction.toLowerCase() === 'desc' + ? 'desc' + : 'asc' + if (orderBy.field === 'id') { + cteInnerQuery.orderBy('id', direction) + } else { + cteInnerQuery.orderByRaw( + knex.raw(`jsonb_path_query_first( data, ? ) ${direction}, id asc`, [ + '$.' + orderBy.field + ]) + ) + } + }) + .select('*') + .from('objs') + .joinRaw('RIGHT JOIN ( SELECT count(*) FROM "objs" ) c(total_count) ON TRUE') + + // Set cursor clause, if present. If it's not present, it's an entry query; this method will return a cursor based on its given query. + // We have implemented keyset pagination for more efficient searches on larger sets. This approach depends on an order by value provided by the user and a (hidden) primary key. + // logger.debug( cursor ) + if (cursor) { + let castType = 'text' + if (typeof cursor.value === 'string') castType = 'text' + if (typeof cursor.value === 'boolean') castType = 'boolean' + if (typeof cursor.value === 'number') castType = 'numeric' + + // When strings are used inside an order clause, as mentioned above, we need to add quotes around the comparison value, as the jsonb_path_query funcs return json encoded strings (`{"test":"foo"}` => test is returned as `"foo"`) + if (castType === 'text') cursor.value = `"${cursor.value}"` + + if (operatorsWhitelist.indexOf(cursor.operator) === -1) + throw new Error('Invalid operator for cursor') + + // Unwrapping the tuple comparison of ( userOrderByField, id ) > ( lastValueOfUserOrderBy, lastSeenId ) + if (fullObjectSelect) { + if (cursor.field === 'id') { + mainQuery.where(knex.raw(`id ${cursor.operator} ? `, [cursor.value])) + } else { + mainQuery.where( + knex.raw( + `jsonb_path_query_first( data, ? )::${castType} ${cursor.operator}= ? `, + ['$.' + cursor.field, cursor.value] + ) + ) + } + } else { + mainQuery.where( + knex.raw(`??::${castType} ${cursor.operator}= ? `, [ + (select || []).indexOf(cursor.field).toString(), + cursor.value + ]) + ) } - mainQuery.limit(limit) - // logger.debug( mainQuery.toString() ) - // Finally, execute the query - const rows = await mainQuery - const totalCount = rows && rows.length > 0 ? parseInt(rows[0].total_count) : 0 - - // Return early - if (totalCount === 0) return { totalCount, objects: [], cursor: null } - - // Reconstruct the object based on the provided select paths. - if (!fullObjectSelect) { - rows.forEach((o, i, arr) => { - const no = { - id: o.id, - createdAt: o.createdAt, - speckleType: o.speckleType, - totalChildrenCount: o.totalChildrenCount, - data: {} - } - let k = 0 - for (const field of select || []) { - set(no.data, field, o[k++]) - } - arr[i] = no + if (cursor.lastSeenId) { + mainQuery.andWhere((qb) => { + // Dunno what the TS issue here is, the JS code itself seemed to work fine as JS + // eslint-disable-next-line @typescript-eslint/no-explicit-any + qb.where('id' as any, '>', cursor!.lastSeenId as any) + if (fullObjectSelect) + qb.orWhere( + knex.raw( + `jsonb_path_query_first( data, ? )::${castType} ${cursor!.operator} ? `, + ['$.' + cursor!.field, cursor!.value] + ) + ) + else + qb.orWhere( + knex.raw(`??::${castType} ${cursor!.operator} ? `, [ + (select || []).indexOf(cursor!.field).toString(), + cursor!.value + ]) + ) }) } + } - // Assemble the cursor for an eventual next call - const cursorObj: typeof cursor = { - field: cursor?.field || orderBy.field, - operator: - cursor?.operator || - (orderBy.direction && orderBy.direction.toLowerCase() === 'desc' ? '<' : '>'), - value: get(rows[rows.length - 1], `data.${orderBy.field}`) - } + mainQuery.limit(limit) + // logger.debug( mainQuery.toString() ) + // Finally, execute the query + const rows = await mainQuery + const totalCount = rows && rows.length > 0 ? parseInt(rows[0].total_count) : 0 + + // Return early + if (totalCount === 0) return { totalCount, objects: [], cursor: null } + + // Reconstruct the object based on the provided select paths. + if (!fullObjectSelect) { + rows.forEach((o, i, arr) => { + const no = { + id: o.id, + createdAt: o.createdAt, + speckleType: o.speckleType, + totalChildrenCount: o.totalChildrenCount, + data: {} + } + let k = 0 + for (const field of select || []) { + set(no.data, field, o[k++]) + } + arr[i] = no + }) + } - // If we're not ordering by id (default case, where no order by argument is provided), we need to add the last seen id of this query in order to enable keyset pagination. - if (additionalIdOrderBy) { - cursorObj.lastSeenId = rows[rows.length - 1].id - } + // Assemble the cursor for an eventual next call + const cursorObj: typeof cursor = { + field: cursor?.field || orderBy.field, + operator: + cursor?.operator || + (orderBy.direction && orderBy.direction.toLowerCase() === 'desc' ? '<' : '>'), + value: get(rows[rows.length - 1], `data.${orderBy.field}`) + } - // Cursor objects should be client-side opaque, hence we encode them to base64. - const cursorEncoded = Buffer.from(JSON.stringify(cursorObj), 'binary').toString( - 'base64' - ) - return { - totalCount, - objects: rows, - cursor: rows.length === limit ? cursorEncoded : null - } + // If we're not ordering by id (default case, where no order by argument is provided), we need to add the last seen id of this query in order to enable keyset pagination. + if (additionalIdOrderBy) { + cursorObj.lastSeenId = rows[rows.length - 1].id + } + + // Cursor objects should be client-side opaque, hence we encode them to base64. + const cursorEncoded = Buffer.from(JSON.stringify(cursorObj), 'binary').toString( + 'base64' + ) + return { + totalCount, + objects: rows, + cursor: rows.length === limit ? cursorEncoded : null } + } diff --git a/packages/server/modules/cross-server-sync/graph/generated/graphql.ts b/packages/server/modules/cross-server-sync/graph/generated/graphql.ts index b2c96a2a62..860a5ee9dc 100644 --- a/packages/server/modules/cross-server-sync/graph/generated/graphql.ts +++ b/packages/server/modules/cross-server-sync/graph/generated/graphql.ts @@ -4495,6 +4495,7 @@ export type WorkspaceProjectInviteCreateInput = { export type WorkspaceProjectMutations = { __typename?: 'WorkspaceProjectMutations'; create: Project; + moveToRegion: Project; moveToWorkspace: Project; updateRole: Project; }; @@ -4505,6 +4506,12 @@ export type WorkspaceProjectMutationsCreateArgs = { }; +export type WorkspaceProjectMutationsMoveToRegionArgs = { + projectId: Scalars['String']['input']; + regionKey: Scalars['String']['input']; +}; + + export type WorkspaceProjectMutationsMoveToWorkspaceArgs = { projectId: Scalars['String']['input']; workspaceId: Scalars['String']['input']; diff --git a/packages/server/modules/shared/helpers/dbHelper.ts b/packages/server/modules/shared/helpers/dbHelper.ts index d8fac4a0b2..b8aa593445 100644 --- a/packages/server/modules/shared/helpers/dbHelper.ts +++ b/packages/server/modules/shared/helpers/dbHelper.ts @@ -22,7 +22,7 @@ export async function* executeBatchedSelect< >( selectQuery: Knex.QueryBuilder, options?: Partial -): AsyncGenerator, void, unknown> { +): AsyncGenerator, void, unknown> { const { batchSize = 100, trx } = options || {} if (trx) selectQuery.transacting(trx) @@ -33,7 +33,7 @@ export async function* executeBatchedSelect< let currentOffset = 0 while (hasMorePages) { const q = selectQuery.clone().offset(currentOffset) - const results = (await q) as Awaited<(typeof selectQuery)> + const results = (await q) as Awaited if (!results.length) { hasMorePages = false diff --git a/packages/server/modules/workspaces/domain/operations.ts b/packages/server/modules/workspaces/domain/operations.ts index eb54b5a106..7b646bfa34 100644 --- a/packages/server/modules/workspaces/domain/operations.ts +++ b/packages/server/modules/workspaces/domain/operations.ts @@ -287,7 +287,7 @@ export type GetAvailableRegions = (params: { workspaceId: string }) => Promise -export type AssignRegion = (params: { +export type AssignWorkspaceRegion = (params: { workspaceId: string regionKey: string }) => Promise @@ -313,6 +313,18 @@ export type UpsertWorkspaceCreationState = (params: { * Project regions */ +/** + * Updates project region and moves all regional data to target regional db + */ +export type UpdateProjectRegion = (params: { + projectId: string + regionKey: string +}) => Promise + export type CopyProjects = (params: { projectIds: string[] }) => Promise -export type CopyProjectModels = (params: { projectIds: string[] }) => Promise> -export type CopyProjectVersions = (params: { projectIds: string[] }) => Promise> +export type CopyProjectModels = (params: { + projectIds: string[] +}) => Promise> +export type CopyProjectVersions = (params: { + projectIds: string[] +}) => Promise> diff --git a/packages/server/modules/workspaces/errors/regions.ts b/packages/server/modules/workspaces/errors/regions.ts index 60ce6774e2..5a73b52bcf 100644 --- a/packages/server/modules/workspaces/errors/regions.ts +++ b/packages/server/modules/workspaces/errors/regions.ts @@ -5,3 +5,9 @@ export class WorkspaceRegionAssignmentError extends BaseError { static code = 'WORKSPACE_REGION_ASSIGNMENT_ERROR' static statusCode = 400 } + +export class ProjectRegionAssignmentError extends BaseError { + static defaultMessage = 'Failed to assign region to project' + static code = 'PROJECT_REGION_ASSIGNMENT_ERROR' + static statusCode = 400 +} diff --git a/packages/server/modules/workspaces/graph/resolvers/regions.ts b/packages/server/modules/workspaces/graph/resolvers/regions.ts index a80df9e942..80ae195c4d 100644 --- a/packages/server/modules/workspaces/graph/resolvers/regions.ts +++ b/packages/server/modules/workspaces/graph/resolvers/regions.ts @@ -2,10 +2,13 @@ import { db } from '@/db/knex' import { Resolvers } from '@/modules/core/graph/generated/graphql' import { getWorkspacePlanFactory } from '@/modules/gatekeeper/repositories/billing' import { canWorkspaceUseRegionsFactory } from '@/modules/gatekeeper/services/featureAuthorization' -import { getDb } from '@/modules/multiregion/utils/dbSelector' +import { getDb, getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { getRegionsFactory } from '@/modules/multiregion/repositories' import { authorizeResolver } from '@/modules/shared' import { + copyProjectModelsFactory, + copyProjectsFactory, + copyProjectVersionsFactory, getDefaultRegionFactory, upsertRegionAssignmentFactory } from '@/modules/workspaces/repositories/regions' @@ -14,10 +17,14 @@ import { upsertWorkspaceFactory } from '@/modules/workspaces/repositories/workspaces' import { - assignRegionFactory, - getAvailableRegionsFactory + assignWorkspaceRegionFactory, + getAvailableRegionsFactory, + updateProjectRegionFactory } from '@/modules/workspaces/services/regions' import { Roles } from '@speckle/shared' +import { getProjectFactory } from '@/modules/core/repositories/streams' +import { getStreamBranchCountFactory } from '@/modules/core/repositories/branches' +import { getStreamCommitCountFactory } from '@/modules/core/repositories/commits' export default { Workspace: { @@ -37,7 +44,7 @@ export default { const regionDb = await getDb({ regionKey: args.regionKey }) - const assignRegion = assignRegionFactory({ + const assignRegion = assignWorkspaceRegionFactory({ getAvailableRegions: getAvailableRegionsFactory({ getRegions: getRegionsFactory({ db }), canWorkspaceUseRegions: canWorkspaceUseRegionsFactory({ @@ -53,5 +60,35 @@ export default { return await ctx.loaders.workspaces!.getWorkspace.load(args.workspaceId) } + }, + WorkspaceProjectMutations: { + moveToRegion: async (_parent, args, context) => { + await authorizeResolver( + context.userId, + args.projectId, + Roles.Stream.Owner, + context.resourceAccessRules + ) + + const sourceDb = await getProjectDbClient({ projectId: args.projectId }) + const targetDb = await getDb({ regionKey: args.regionKey }) + + const updateProjectRegion = updateProjectRegionFactory({ + getProject: getProjectFactory({ db: sourceDb }), + countProjectModels: getStreamBranchCountFactory({ db: sourceDb }), + countProjectVersions: getStreamCommitCountFactory({ db: sourceDb }), + getAvailableRegions: getAvailableRegionsFactory({ + getRegions: getRegionsFactory({ db }), + canWorkspaceUseRegions: canWorkspaceUseRegionsFactory({ + getWorkspacePlan: getWorkspacePlanFactory({ db }) + }) + }), + copyProjects: copyProjectsFactory({ sourceDb, targetDb }), + copyProjectModels: copyProjectModelsFactory({ sourceDb, targetDb }), + copyProjectVersions: copyProjectVersionsFactory({ sourceDb, targetDb }) + }) + + return await updateProjectRegion(args) + } } } as Resolvers diff --git a/packages/server/modules/workspaces/repositories/regions.ts b/packages/server/modules/workspaces/repositories/regions.ts index c6688386da..962330ba4a 100644 --- a/packages/server/modules/workspaces/repositories/regions.ts +++ b/packages/server/modules/workspaces/repositories/regions.ts @@ -1,8 +1,21 @@ -import { BranchCommits, Branches, buildTableHelper, Commits, StreamCommits, StreamFavorites, Streams, StreamsMeta } from '@/modules/core/dbSchema' +import { + BranchCommits, + Branches, + buildTableHelper, + Commits, + StreamCommits, + StreamFavorites, + Streams, + StreamsMeta +} from '@/modules/core/dbSchema' import { Branch } from '@/modules/core/domain/branches/types' import { Commit } from '@/modules/core/domain/commits/types' import { Stream } from '@/modules/core/domain/streams/types' -import { BranchCommitRecord, StreamCommitRecord, StreamFavoriteRecord } from '@/modules/core/helpers/types' +import { + BranchCommitRecord, + StreamCommitRecord, + StreamFavoriteRecord +} from '@/modules/core/helpers/types' import { RegionRecord } from '@/modules/multiregion/helpers/types' import { Regions } from '@/modules/multiregion/repositories' import { executeBatchedSelect } from '@/modules/shared/helpers/dbHelper' @@ -35,144 +48,187 @@ const tables = { export const upsertRegionAssignmentFactory = (deps: { db: Knex }): UpsertRegionAssignment => - async (params) => { - const { workspaceId, regionKey } = params - const [row] = await tables - .workspaceRegions(deps.db) - .insert({ workspaceId, regionKey }, '*') - .onConflict(['workspaceId', 'regionKey']) - .merge() - - return row - } + async (params) => { + const { workspaceId, regionKey } = params + const [row] = await tables + .workspaceRegions(deps.db) + .insert({ workspaceId, regionKey }, '*') + .onConflict(['workspaceId', 'regionKey']) + .merge() + + return row + } export const getDefaultRegionFactory = (deps: { db: Knex }): GetDefaultRegion => - async (params) => { - const { workspaceId } = params - const row = await tables - .regions(deps.db) - .select(Regions.cols) - .join(WorkspaceRegions.name, WorkspaceRegions.col.regionKey, Regions.col.key) - .where({ [WorkspaceRegions.col.workspaceId]: workspaceId }) - .first() - - return row - } - + async (params) => { + const { workspaceId } = params + const row = await tables + .regions(deps.db) + .select(Regions.cols) + .join(WorkspaceRegions.name, WorkspaceRegions.col.regionKey, Regions.col.key) + .where({ [WorkspaceRegions.col.workspaceId]: workspaceId }) + .first() + + return row + } + +export const copyProjectsFactory = + (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjects => + async ({ projectIds }) => { + const selectProjects = tables + .projects(deps.sourceDb) + .select('*') + .whereIn(Streams.col.id, projectIds) + const copiedProjectIds: string[] = [] + + // Copy project record + for await (const projects of executeBatchedSelect(selectProjects)) { + for (const project of projects) { + // Store copied project id + copiedProjectIds.push(project.id) + + // Copy `streams` row to target db + await tables.projects(deps.targetDb).insert(project).onConflict().ignore() + } -export const copyProjects = - (deps: { sourceDb: Knex, targetDb: Knex }): CopyProjects => - async ({ projectIds }) => { - const selectProjects = tables.projects(deps.sourceDb).select('*').whereIn(Streams.col.id, projectIds) - const copiedProjectIds: string[] = [] + const projectIds = projects.map((project) => project.id) - // Copy project record - for await (const projects of executeBatchedSelect(selectProjects)) { - for (const project of projects) { - // Store copied project id - copiedProjectIds.push(project.id) + // Fetch `stream_favorites` rows for projects in batch + const selectStreamFavorites = tables + .streamFavorites(deps.sourceDb) + .select('*') + .whereIn(StreamFavorites.col.streamId, projectIds) - // Copy `streams` row to target db - await tables.projects(deps.targetDb) - .insert(project) + for await (const streamFavorites of executeBatchedSelect(selectStreamFavorites)) { + for (const streamFavorite of streamFavorites) { + // Copy `stream_favorites` row to target db + await tables + .streamFavorites(deps.targetDb) + .insert(streamFavorite) .onConflict() .ignore() } + } - const projectIds = projects.map((project) => project.id) - - // Fetch `stream_favorites` rows for projects in batch - const selectStreamFavorites = tables.streamFavorites(deps.sourceDb).select('*').whereIn(StreamFavorites.col.streamId, projectIds) - - for await (const streamFavorites of executeBatchedSelect(selectStreamFavorites)) { - for (const streamFavorite of streamFavorites) { - // Copy `stream_favorites` row to target db - await tables.streamFavorites(deps.targetDb).insert(streamFavorite).onConflict().ignore() - } - } - - // Fetch `streams_meta` rows for projects in batch - const selectStreamsMetadata = tables.streamsMeta(deps.sourceDb).select('*').whereIn(StreamsMeta.col.streamId, projectIds) - - for await (const streamsMetadataBatch of executeBatchedSelect(selectStreamsMetadata)) { - for (const streamMetadata of streamsMetadataBatch) { - // Copy `streams_meta` row to target db - await tables.streamsMeta(deps.targetDb).insert(streamMetadata).onConflict().ignore() - } + // Fetch `streams_meta` rows for projects in batch + const selectStreamsMetadata = tables + .streamsMeta(deps.sourceDb) + .select('*') + .whereIn(StreamsMeta.col.streamId, projectIds) + + for await (const streamsMetadataBatch of executeBatchedSelect( + selectStreamsMetadata + )) { + for (const streamMetadata of streamsMetadataBatch) { + // Copy `streams_meta` row to target db + await tables + .streamsMeta(deps.targetDb) + .insert(streamMetadata) + .onConflict() + .ignore() } } - - return copiedProjectIds } -export const copyProjectModels = - (deps: { sourceDb: Knex, targetDb: Knex }): CopyProjectModels => - async ({ projectIds }) => { - const copiedModelIds: Record = projectIds.reduce((result, id) => ({ ...result, [id]: [] }), {}) - - for (const projectId of projectIds) { - const selectModels = tables.models(deps.sourceDb).select('*').where({ streamId: projectId }) - - for await (const models of executeBatchedSelect(selectModels)) { - for (const model of models) { - // Store copied model ids - copiedModelIds[projectId].push(model.id) - - // Copy `branches` row to target db - await tables.models(deps.targetDb).insert(model).onConflict().ignore() - } + return copiedProjectIds + } + +export const copyProjectModelsFactory = + (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectModels => + async ({ projectIds }) => { + const copiedModelIds: Record = projectIds.reduce( + (result, id) => ({ ...result, [id]: [] }), + {} + ) + + for (const projectId of projectIds) { + const selectModels = tables + .models(deps.sourceDb) + .select('*') + .where({ streamId: projectId }) + + for await (const models of executeBatchedSelect(selectModels)) { + for (const model of models) { + // Store copied model ids + copiedModelIds[projectId].push(model.id) + + // Copy `branches` row to target db + await tables.models(deps.targetDb).insert(model).onConflict().ignore() } } - - return copiedModelIds } -export const copyProjectVersions = - (deps: { sourceDb: Knex, targetDb: Knex }): CopyProjectVersions => - async ({ projectIds }) => { - const copiedVersionIds: Record = projectIds.reduce((result, id) => ({ ...result, [id]: [] }), {}) - - for (const projectId of projectIds) { - const selectVersions = tables.streamCommits(deps.sourceDb).select('*') - .join(Commits.name, Commits.col.id, StreamCommits.col.commitId) - .where({ streamId: projectId }) - - for await (const versions of executeBatchedSelect(selectVersions)) { - for (const version of versions) { - const { commitId, ...commit } = version - - // Store copied version id - copiedVersionIds[projectId].push(commitId) - - // Copy `commits` row to target db - await tables.versions(deps.targetDb).insert(commit).onConflict().ignore() - } - - const commitIds = versions.map((version) => version.commitId) - - // Fetch `branch_commits` rows for versions in batch - const selectBranchCommits = tables.branchCommits(deps.sourceDb).select('*').whereIn(BranchCommits.col.commitId, commitIds) + return copiedModelIds + } + +export const copyProjectVersionsFactory = + (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectVersions => + async ({ projectIds }) => { + const copiedVersionIds: Record = projectIds.reduce( + (result, id) => ({ ...result, [id]: [] }), + {} + ) + + for (const projectId of projectIds) { + const selectVersions = tables + .streamCommits(deps.sourceDb) + .select('*') + .join( + Commits.name, + Commits.col.id, + StreamCommits.col.commitId + ) + .where({ streamId: projectId }) + + for await (const versions of executeBatchedSelect(selectVersions)) { + for (const version of versions) { + const { commitId, ...commit } = version + + // Store copied version id + copiedVersionIds[projectId].push(commitId) + + // Copy `commits` row to target db + await tables.versions(deps.targetDb).insert(commit).onConflict().ignore() + } - for await (const branchCommits of executeBatchedSelect(selectBranchCommits)) { - for (const branchCommit of branchCommits) { - // Copy `branch_commits` row to target db - await tables.branchCommits(deps.targetDb).insert(branchCommit).onConflict().ignore() - } + const commitIds = versions.map((version) => version.commitId) + + // Fetch `branch_commits` rows for versions in batch + const selectBranchCommits = tables + .branchCommits(deps.sourceDb) + .select('*') + .whereIn(BranchCommits.col.commitId, commitIds) + + for await (const branchCommits of executeBatchedSelect(selectBranchCommits)) { + for (const branchCommit of branchCommits) { + // Copy `branch_commits` row to target db + await tables + .branchCommits(deps.targetDb) + .insert(branchCommit) + .onConflict() + .ignore() } + } - // Fetch `stream_commits` rows for versions in batch - const selectStreamCommits = tables.streamCommits(deps.sourceDb).select('*').whereIn(StreamCommits.col.commitId, commitIds) - - for await (const streamCommits of executeBatchedSelect(selectStreamCommits)) { - for (const streamCommit of streamCommits) { - // Copy `stream_commits` row to target db - await tables.streamCommits(deps.targetDb).insert(streamCommit).onConflict().ignore() - } + // Fetch `stream_commits` rows for versions in batch + const selectStreamCommits = tables + .streamCommits(deps.sourceDb) + .select('*') + .whereIn(StreamCommits.col.commitId, commitIds) + + for await (const streamCommits of executeBatchedSelect(selectStreamCommits)) { + for (const streamCommit of streamCommits) { + // Copy `stream_commits` row to target db + await tables + .streamCommits(deps.targetDb) + .insert(streamCommit) + .onConflict() + .ignore() } } } - - return copiedVersionIds } + return copiedVersionIds + } diff --git a/packages/server/modules/workspaces/services/projects.ts b/packages/server/modules/workspaces/services/projects.ts index f8d81f0581..6ce8483aea 100644 --- a/packages/server/modules/workspaces/services/projects.ts +++ b/packages/server/modules/workspaces/services/projects.ts @@ -100,23 +100,23 @@ type GetWorkspaceProjectsReturnValue = { export const getWorkspaceProjectsFactory = ({ getStreams }: { getStreams: GetUserStreamsPage }) => - async ( - args: GetWorkspaceProjectsArgs, - opts: GetWorkspaceProjectsOptions - ): Promise => { - const { streams, cursor } = await getStreams({ - cursor: opts.cursor, - limit: opts.limit || 25, - searchQuery: opts.filter?.search || undefined, - workspaceId: args.workspaceId, - userId: opts.filter.userId - }) + async ( + args: GetWorkspaceProjectsArgs, + opts: GetWorkspaceProjectsOptions + ): Promise => { + const { streams, cursor } = await getStreams({ + cursor: opts.cursor, + limit: opts.limit || 25, + searchQuery: opts.filter?.search || undefined, + workspaceId: args.workspaceId, + userId: opts.filter.userId + }) - return { - items: streams, - cursor - } + return { + items: streams, + cursor } + } type MoveProjectToWorkspaceArgs = { projectId: string @@ -141,78 +141,78 @@ export const moveProjectToWorkspaceFactory = getWorkspaceRoleToDefaultProjectRoleMapping: GetWorkspaceRoleToDefaultProjectRoleMapping updateWorkspaceRole: UpdateWorkspaceRole }) => - async ({ - projectId, - workspaceId - }: MoveProjectToWorkspaceArgs): Promise => { - const project = await getProject({ projectId }) - - if (!project) throw new ProjectNotFoundError() - if (project.workspaceId?.length) { - // We do not currently support moving projects between workspaces - throw new WorkspaceInvalidProjectError( - 'Specified project already belongs to a workspace. Moving between workspaces is not yet supported.' - ) - } + async ({ + projectId, + workspaceId + }: MoveProjectToWorkspaceArgs): Promise => { + const project = await getProject({ projectId }) - // Update roles for current project members - const projectTeam = await getProjectCollaborators({ projectId }) - const workspaceTeam = await getWorkspaceRoles({ workspaceId }) - const defaultProjectRoleMapping = await getWorkspaceRoleToDefaultProjectRoleMapping( - { workspaceId } + if (!project) throw new ProjectNotFoundError() + if (project.workspaceId?.length) { + // We do not currently support moving projects between workspaces + throw new WorkspaceInvalidProjectError( + 'Specified project already belongs to a workspace. Moving between workspaces is not yet supported.' ) + } - for (const projectMembers of chunk(projectTeam, 5)) { - await Promise.all( - projectMembers.map( - async ({ id: userId, role: serverRole, streamRole: currentProjectRole }) => { - // Update workspace role. Prefer existing workspace role if there is one. - const currentWorkspaceRole = workspaceTeam.find( - (role) => role.userId === userId - ) - const nextWorkspaceRole = currentWorkspaceRole ?? { - userId, - workspaceId, - role: - serverRole === Roles.Server.Guest - ? Roles.Workspace.Guest - : Roles.Workspace.Member, - createdAt: new Date() - } - await updateWorkspaceRole(nextWorkspaceRole) + // Update roles for current project members + const projectTeam = await getProjectCollaborators({ projectId }) + const workspaceTeam = await getWorkspaceRoles({ workspaceId }) + const defaultProjectRoleMapping = await getWorkspaceRoleToDefaultProjectRoleMapping( + { workspaceId } + ) - // Update project role. Prefer default workspace project role if more permissive. - const defaultProjectRole = - defaultProjectRoleMapping[nextWorkspaceRole.role] ?? Roles.Stream.Reviewer - const nextProjectRole = orderByWeight( - [currentProjectRole, defaultProjectRole], - coreUserRoles - )[0] - await upsertProjectRole({ - userId, - projectId, - role: nextProjectRole.name as StreamRoles - }) + for (const projectMembers of chunk(projectTeam, 5)) { + await Promise.all( + projectMembers.map( + async ({ id: userId, role: serverRole, streamRole: currentProjectRole }) => { + // Update workspace role. Prefer existing workspace role if there is one. + const currentWorkspaceRole = workspaceTeam.find( + (role) => role.userId === userId + ) + const nextWorkspaceRole = currentWorkspaceRole ?? { + userId, + workspaceId, + role: + serverRole === Roles.Server.Guest + ? Roles.Workspace.Guest + : Roles.Workspace.Member, + createdAt: new Date() } - ) - ) - } + await updateWorkspaceRole(nextWorkspaceRole) - // Assign project to workspace - return await updateProject({ projectUpdate: { id: projectId, workspaceId } }) + // Update project role. Prefer default workspace project role if more permissive. + const defaultProjectRole = + defaultProjectRoleMapping[nextWorkspaceRole.role] ?? Roles.Stream.Reviewer + const nextProjectRole = orderByWeight( + [currentProjectRole, defaultProjectRole], + coreUserRoles + )[0] + await upsertProjectRole({ + userId, + projectId, + role: nextProjectRole.name as StreamRoles + }) + } + ) + ) } + // Assign project to workspace + return await updateProject({ projectUpdate: { id: projectId, workspaceId } }) + } + export const moveProjectToRegionFactory = (deps: { - copyProjects: CopyProjects, - copyProjectModels: CopyProjectModels, + copyProjects: CopyProjects + copyProjectModels: CopyProjectModels copyProjectVersions: CopyProjectVersions }) => - async (args: { projectId: string }): Promise => { - const projectIds = await deps.copyProjects({ projectIds: [args.projectId] }) - const modelIdsByProjectId = await deps.copyProjectModels({ projectIds }) - const versionIdsByProjectId = await deps.copyProjectVersions({ projectIds }) - } + async (args: { projectId: string }): Promise => { + const projectIds = await deps.copyProjects({ projectIds: [args.projectId] }) + const modelIdsByProjectId = await deps.copyProjectModels({ projectIds }) + const versionIdsByProjectId = await deps.copyProjectVersions({ projectIds }) + } export const getWorkspaceRoleToDefaultProjectRoleMappingFactory = ({ @@ -220,19 +220,19 @@ export const getWorkspaceRoleToDefaultProjectRoleMappingFactory = }: { getWorkspace: GetWorkspace }): GetWorkspaceRoleToDefaultProjectRoleMapping => - async ({ workspaceId }) => { - const workspace = await getWorkspace({ workspaceId }) + async ({ workspaceId }) => { + const workspace = await getWorkspace({ workspaceId }) - if (!workspace) { - throw new WorkspaceNotFoundError() - } + if (!workspace) { + throw new WorkspaceNotFoundError() + } - return { - [Roles.Workspace.Guest]: null, - [Roles.Workspace.Member]: workspace.defaultProjectRole, - [Roles.Workspace.Admin]: Roles.Stream.Owner - } + return { + [Roles.Workspace.Guest]: null, + [Roles.Workspace.Member]: workspace.defaultProjectRole, + [Roles.Workspace.Admin]: Roles.Stream.Owner } + } export const updateWorkspaceProjectRoleFactory = ({ @@ -244,63 +244,63 @@ export const updateWorkspaceProjectRoleFactory = getWorkspaceRoleForUser: GetWorkspaceRoleForUser updateStreamRoleAndNotify: UpdateStreamRole }): UpdateWorkspaceProjectRole => - async ({ role, updater }) => { - const { workspaceId } = (await getStream({ streamId: role.projectId })) ?? {} - if (!workspaceId) throw new WorkspaceInvalidProjectError() - - const currentWorkspaceRole = await getWorkspaceRoleForUser({ - workspaceId, - userId: role.userId - }) + async ({ role, updater }) => { + const { workspaceId } = (await getStream({ streamId: role.projectId })) ?? {} + if (!workspaceId) throw new WorkspaceInvalidProjectError() - if (currentWorkspaceRole?.role === Roles.Workspace.Admin) { - // User is workspace admin and cannot have their project roles changed - throw new WorkspaceAdminError() - } + const currentWorkspaceRole = await getWorkspaceRoleForUser({ + workspaceId, + userId: role.userId + }) - if ( - currentWorkspaceRole?.role === Roles.Workspace.Guest && - role.role === Roles.Stream.Owner - ) { - // Workspace guests cannot be project owners - throw new WorkspaceInvalidRoleError('Workspace guests cannot be project owners.') - } + if (currentWorkspaceRole?.role === Roles.Workspace.Admin) { + // User is workspace admin and cannot have their project roles changed + throw new WorkspaceAdminError() + } - return await updateStreamRoleAndNotify( - role, - updater.userId!, - updater.resourceAccessRules - ) + if ( + currentWorkspaceRole?.role === Roles.Workspace.Guest && + role.role === Roles.Stream.Owner + ) { + // Workspace guests cannot be project owners + throw new WorkspaceInvalidRoleError('Workspace guests cannot be project owners.') } + return await updateStreamRoleAndNotify( + role, + updater.userId!, + updater.resourceAccessRules + ) + } + export const createWorkspaceProjectFactory = (deps: { getDefaultRegion: GetDefaultRegion }) => - async (params: { input: WorkspaceProjectCreateInput; ownerId: string }) => { - const { input, ownerId } = params - const workspaceDefaultRegion = await deps.getDefaultRegion({ - workspaceId: input.workspaceId - }) - const regionKey = workspaceDefaultRegion?.key - const projectDb = await getDb({ regionKey }) - const db = mainDb + async (params: { input: WorkspaceProjectCreateInput; ownerId: string }) => { + const { input, ownerId } = params + const workspaceDefaultRegion = await deps.getDefaultRegion({ + workspaceId: input.workspaceId + }) + const regionKey = workspaceDefaultRegion?.key + const projectDb = await getDb({ regionKey }) + const db = mainDb - // todo, use the command factory here, but for that, we need to migrate to the event bus - // deps not injected to ensure proper DB injection - const createNewProject = createNewProjectFactory({ - storeProject: storeProjectFactory({ db: projectDb }), - getProject: getProjectFactory({ db }), - deleteProject: deleteProjectFactory({ db: projectDb }), - storeModel: storeModelFactory({ db: projectDb }), - // THIS MUST GO TO THE MAIN DB - storeProjectRole: storeProjectRoleFactory({ db }), - projectsEventsEmitter: ProjectsEmitter.emit - }) + // todo, use the command factory here, but for that, we need to migrate to the event bus + // deps not injected to ensure proper DB injection + const createNewProject = createNewProjectFactory({ + storeProject: storeProjectFactory({ db: projectDb }), + getProject: getProjectFactory({ db }), + deleteProject: deleteProjectFactory({ db: projectDb }), + storeModel: storeModelFactory({ db: projectDb }), + // THIS MUST GO TO THE MAIN DB + storeProjectRole: storeProjectRoleFactory({ db }), + projectsEventsEmitter: ProjectsEmitter.emit + }) - const project = await createNewProject({ - ...input, - regionKey, - ownerId - }) + const project = await createNewProject({ + ...input, + regionKey, + ownerId + }) - return project - } + return project + } diff --git a/packages/server/modules/workspaces/services/regions.ts b/packages/server/modules/workspaces/services/regions.ts index 6d5c9cb229..e673fc5220 100644 --- a/packages/server/modules/workspaces/services/regions.ts +++ b/packages/server/modules/workspaces/services/regions.ts @@ -1,14 +1,24 @@ +import { GetStreamBranchCount } from '@/modules/core/domain/branches/operations' +import { GetStreamCommitCount } from '@/modules/core/domain/commits/operations' +import { GetProject } from '@/modules/core/domain/projects/operations' import { WorkspaceFeatureAccessFunction } from '@/modules/gatekeeper/domain/operations' import { GetRegions } from '@/modules/multiregion/domain/operations' import { - AssignRegion, + AssignWorkspaceRegion, + CopyProjectModels, + CopyProjects, + CopyProjectVersions, GetAvailableRegions, GetDefaultRegion, GetWorkspace, + UpdateProjectRegion, UpsertRegionAssignment, UpsertWorkspace } from '@/modules/workspaces/domain/operations' -import { WorkspaceRegionAssignmentError } from '@/modules/workspaces/errors/regions' +import { + ProjectRegionAssignmentError, + WorkspaceRegionAssignmentError +} from '@/modules/workspaces/errors/regions' export const getAvailableRegionsFactory = (deps: { @@ -25,14 +35,14 @@ export const getAvailableRegionsFactory = return await deps.getRegions() } -export const assignRegionFactory = +export const assignWorkspaceRegionFactory = (deps: { getAvailableRegions: GetAvailableRegions upsertRegionAssignment: UpsertRegionAssignment getDefaultRegion: GetDefaultRegion getWorkspace: GetWorkspace insertRegionWorkspace: UpsertWorkspace - }): AssignRegion => + }): AssignWorkspaceRegion => async (params) => { const { workspaceId, regionKey } = params @@ -69,3 +79,73 @@ export const assignRegionFactory = // Copy workspace into region db await deps.insertRegionWorkspace({ workspace }) } + +export const updateProjectRegionFactory = + (deps: { + getProject: GetProject + countProjectModels: GetStreamBranchCount + countProjectVersions: GetStreamCommitCount + getAvailableRegions: GetAvailableRegions + copyProjects: CopyProjects + copyProjectModels: CopyProjectModels + copyProjectVersions: CopyProjectVersions + }): UpdateProjectRegion => + async (params) => { + const { projectId, regionKey } = params + + const project = await deps.getProject({ projectId }) + if (!project) { + throw new ProjectRegionAssignmentError('Project not found', { + info: { params } + }) + } + if (!project.workspaceId) { + throw new ProjectRegionAssignmentError('Project not a part of a workspace', { + info: { params } + }) + } + + const availableRegions = await deps.getAvailableRegions({ + workspaceId: project.workspaceId + }) + if (!availableRegions.find((region) => region.key === regionKey)) { + throw new ProjectRegionAssignmentError( + 'Specified region not available for workspace', + { + info: { + params, + workspaceId: project.workspaceId + } + } + ) + } + + // Move commits + const projectIds = await deps.copyProjects({ projectIds: [projectId] }) + const modelIds = await deps.copyProjectModels({ projectIds }) + const versionIds = await deps.copyProjectVersions({ projectIds }) + + // TODO: Move objects + // TODO: Move automations + // TODO: Move comments + // TODO: Move file blobs + // TODO: Move webhooks + + // TODO: Validate state after move captures latest state of project + const sourceProjectModelCount = await deps.countProjectModels(projectId) + const sourceProjectVersionCount = await deps.countProjectVersions(projectId) + + const isReconciled = + modelIds[projectId].length === sourceProjectModelCount && + versionIds[projectId].length === sourceProjectVersionCount + + if (!isReconciled) { + // TODO: Move failed or source project added data while changing regions. Retry move. + throw new ProjectRegionAssignmentError( + 'Missing data from source project in target region copy after move.' + ) + } + + // TODO: Update project region in db + return { ...project, regionKey } + } diff --git a/packages/server/modules/workspaces/tests/helpers/creation.ts b/packages/server/modules/workspaces/tests/helpers/creation.ts index bb35888bdc..500fe1d35e 100644 --- a/packages/server/modules/workspaces/tests/helpers/creation.ts +++ b/packages/server/modules/workspaces/tests/helpers/creation.ts @@ -64,7 +64,7 @@ import { import { SetOptional } from 'type-fest' import { isMultiRegionTestMode } from '@/test/speckle-helpers/regions' import { - assignRegionFactory, + assignWorkspaceRegionFactory, getAvailableRegionsFactory } from '@/modules/workspaces/services/regions' import { getRegionsFactory } from '@/modules/multiregion/repositories' @@ -185,7 +185,7 @@ export const createTestWorkspace = async ( if (useRegion) { const regionDb = await getDb({ regionKey }) - const assignRegion = assignRegionFactory({ + const assignRegion = assignWorkspaceRegionFactory({ getAvailableRegions: getAvailableRegionsFactory({ getRegions: getRegionsFactory({ db }), canWorkspaceUseRegions: canWorkspaceUseRegionsFactory({ diff --git a/packages/server/test/graphql/generated/graphql.ts b/packages/server/test/graphql/generated/graphql.ts index cabbd586e6..19789627f6 100644 --- a/packages/server/test/graphql/generated/graphql.ts +++ b/packages/server/test/graphql/generated/graphql.ts @@ -4496,6 +4496,7 @@ export type WorkspaceProjectInviteCreateInput = { export type WorkspaceProjectMutations = { __typename?: 'WorkspaceProjectMutations'; create: Project; + moveToRegion: Project; moveToWorkspace: Project; updateRole: Project; }; @@ -4506,6 +4507,12 @@ export type WorkspaceProjectMutationsCreateArgs = { }; +export type WorkspaceProjectMutationsMoveToRegionArgs = { + projectId: Scalars['String']['input']; + regionKey: Scalars['String']['input']; +}; + + export type WorkspaceProjectMutationsMoveToWorkspaceArgs = { projectId: Scalars['String']['input']; workspaceId: Scalars['String']['input']; From f64a8bbabaa753842b3a387a9f467eccd211bba5 Mon Sep 17 00:00:00 2001 From: Chuck Driesler Date: Tue, 14 Jan 2025 15:13:32 +0000 Subject: [PATCH 03/28] chore(regions): successful basic test of project region change --- .../lib/common/generated/gql/graphql.ts | 15 +++- .../modules/core/graph/generated/graphql.ts | 4 + .../server/modules/core/services/projects.ts | 2 +- .../graph/generated/graphql.ts | 4 + .../modules/workspaces/services/projects.ts | 15 ---- .../tests/integration/projects.graph.spec.ts | 79 ++++++++++++++++++- .../server/test/graphql/generated/graphql.ts | 13 +++ packages/server/test/graphql/multiRegion.ts | 12 +++ 8 files changed, 126 insertions(+), 18 deletions(-) diff --git a/packages/frontend-2/lib/common/generated/gql/graphql.ts b/packages/frontend-2/lib/common/generated/gql/graphql.ts index 3468bbbbb6..c27fb8c719 100644 --- a/packages/frontend-2/lib/common/generated/gql/graphql.ts +++ b/packages/frontend-2/lib/common/generated/gql/graphql.ts @@ -2595,7 +2595,7 @@ export type Query = { /** Look up server users */ users: UserSearchResultCollection; /** Look up server users with a collection of emails */ - usersByEmail: UserSearchResultCollection; + usersByEmail: Array>; /** Validates the slug, to make sure it contains only valid characters and its not taken. */ validateWorkspaceSlug: Scalars['Boolean']['output']; workspace: Workspace; @@ -3872,6 +3872,7 @@ export type UserProjectsFilter = { onlyWithRoles?: InputMaybe>; /** Filter out projects by name */ search?: InputMaybe; + workspaceId?: InputMaybe; }; export type UserProjectsUpdatedMessage = { @@ -4491,6 +4492,11 @@ export type WorkspaceProjectInviteCreateInput = { export type WorkspaceProjectMutations = { __typename?: 'WorkspaceProjectMutations'; create: Project; + /** + * Update project region and move all regional data to new db. + * TODO: Currently performs all operations synchronously in request, should probably be scheduled. + */ + moveToRegion: Project; moveToWorkspace: Project; updateRole: Project; }; @@ -4501,6 +4507,12 @@ export type WorkspaceProjectMutationsCreateArgs = { }; +export type WorkspaceProjectMutationsMoveToRegionArgs = { + projectId: Scalars['String']['input']; + regionKey: Scalars['String']['input']; +}; + + export type WorkspaceProjectMutationsMoveToWorkspaceArgs = { projectId: Scalars['String']['input']; workspaceId: Scalars['String']['input']; @@ -8132,6 +8144,7 @@ export type WorkspacePlanFieldArgs = { } export type WorkspaceProjectMutationsFieldArgs = { create: WorkspaceProjectMutationsCreateArgs, + moveToRegion: WorkspaceProjectMutationsMoveToRegionArgs, moveToWorkspace: WorkspaceProjectMutationsMoveToWorkspaceArgs, updateRole: WorkspaceProjectMutationsUpdateRoleArgs, } diff --git a/packages/server/modules/core/graph/generated/graphql.ts b/packages/server/modules/core/graph/generated/graphql.ts index 9f57629824..ac5f14b7c0 100644 --- a/packages/server/modules/core/graph/generated/graphql.ts +++ b/packages/server/modules/core/graph/generated/graphql.ts @@ -4514,6 +4514,10 @@ export type WorkspaceProjectInviteCreateInput = { export type WorkspaceProjectMutations = { __typename?: 'WorkspaceProjectMutations'; create: Project; + /** + * Update project region and move all regional data to new db. + * TODO: Currently performs all operations synchronously in request, should probably be scheduled. + */ moveToRegion: Project; moveToWorkspace: Project; updateRole: Project; diff --git a/packages/server/modules/core/services/projects.ts b/packages/server/modules/core/services/projects.ts index f5f12abea9..3e87bd3916 100644 --- a/packages/server/modules/core/services/projects.ts +++ b/packages/server/modules/core/services/projects.ts @@ -62,7 +62,7 @@ export const createNewProjectFactory = async () => { await getProject({ projectId }) }, - { maxAttempts: 10 } + { maxAttempts: 100 } ) } catch (err) { if (err instanceof StreamNotFoundError) { diff --git a/packages/server/modules/cross-server-sync/graph/generated/graphql.ts b/packages/server/modules/cross-server-sync/graph/generated/graphql.ts index 860a5ee9dc..406faf17f7 100644 --- a/packages/server/modules/cross-server-sync/graph/generated/graphql.ts +++ b/packages/server/modules/cross-server-sync/graph/generated/graphql.ts @@ -4495,6 +4495,10 @@ export type WorkspaceProjectInviteCreateInput = { export type WorkspaceProjectMutations = { __typename?: 'WorkspaceProjectMutations'; create: Project; + /** + * Update project region and move all regional data to new db. + * TODO: Currently performs all operations synchronously in request, should probably be scheduled. + */ moveToRegion: Project; moveToWorkspace: Project; updateRole: Project; diff --git a/packages/server/modules/workspaces/services/projects.ts b/packages/server/modules/workspaces/services/projects.ts index 6ce8483aea..886a28c99c 100644 --- a/packages/server/modules/workspaces/services/projects.ts +++ b/packages/server/modules/workspaces/services/projects.ts @@ -1,8 +1,5 @@ import { StreamRecord } from '@/modules/core/helpers/types' import { - CopyProjectModels, - CopyProjects, - CopyProjectVersions, GetDefaultRegion, GetWorkspace, GetWorkspaceRoleForUser, @@ -202,18 +199,6 @@ export const moveProjectToWorkspaceFactory = return await updateProject({ projectUpdate: { id: projectId, workspaceId } }) } -export const moveProjectToRegionFactory = - (deps: { - copyProjects: CopyProjects - copyProjectModels: CopyProjectModels - copyProjectVersions: CopyProjectVersions - }) => - async (args: { projectId: string }): Promise => { - const projectIds = await deps.copyProjects({ projectIds: [args.projectId] }) - const modelIdsByProjectId = await deps.copyProjectModels({ projectIds }) - const versionIdsByProjectId = await deps.copyProjectVersions({ projectIds }) - } - export const getWorkspaceRoleToDefaultProjectRoleMappingFactory = ({ getWorkspace diff --git a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts index d9c66f8590..ad844e2de6 100644 --- a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts @@ -1,5 +1,7 @@ import { db } from '@/db/knex' import { AllScopes } from '@/modules/core/helpers/mainConstants' +import { createRandomEmail } from '@/modules/core/helpers/testHelpers' +import { StreamRecord } from '@/modules/core/helpers/types' import { grantStreamPermissionsFactory } from '@/modules/core/repositories/streams' import { BasicTestWorkspace, @@ -8,6 +10,7 @@ import { import { BasicTestUser, createAuthTokenForUser, + createTestUser, createTestUsers } from '@/test/authHelper' import { @@ -15,7 +18,8 @@ import { CreateWorkspaceProjectDocument, GetWorkspaceProjectsDocument, GetWorkspaceTeamDocument, - MoveProjectToWorkspaceDocument + MoveProjectToWorkspaceDocument, + UpdateProjectRegionDocument } from '@/test/graphql/generated/graphql' import { createTestContext, @@ -23,10 +27,16 @@ import { TestApolloServer } from '@/test/graphqlHelper' import { beforeEachContext } from '@/test/hooks' +import { + getMainTestRegionClient, + isMultiRegionTestMode, + waitForRegionUser +} from '@/test/speckle-helpers/regions' import { BasicTestStream, createTestStream } from '@/test/speckle-helpers/streamHelper' import { Roles } from '@speckle/shared' import { expect } from 'chai' import cryptoRandomString from 'crypto-random-string' +import { Knex } from 'knex' const grantStreamPermissions = grantStreamPermissionsFactory({ db }) @@ -272,3 +282,70 @@ describe('Workspace project GQL CRUD', () => { }) }) }) + +isMultiRegionTestMode() + ? describe('Workspace project region changes', () => { + const testRegionKey = 'region1' + + const adminUser: BasicTestUser = { + id: '', + name: 'John Speckle', + email: createRandomEmail() + } + + const testWorkspace: BasicTestWorkspace = { + id: '', + ownerId: '', + slug: '', + name: 'Unlimited Workspace' + } + + const testProject: BasicTestStream = { + id: '', + ownerId: '', + name: 'Regional Project', + isPublic: true + } + + let apollo: TestApolloServer + let regionDb: Knex + + before(async () => { + await createTestUser(adminUser) + await waitForRegionUser(adminUser) + await createTestWorkspace(testWorkspace, adminUser, { + regionKey: testRegionKey, + addPlan: { + name: 'unlimited', + status: 'valid' + } + }) + + testProject.workspaceId = testWorkspace.id + + apollo = await testApolloServer({ authUserId: adminUser.id }) + regionDb = getMainTestRegionClient() + }) + + beforeEach(async () => { + await createTestStream(testProject, adminUser) + }) + + it('moves project record to target regional db', async () => { + const res = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: testRegionKey + }) + + expect(res).to.not.haveGraphQLErrors() + + const project = await regionDb + .table('streams') + .select('*') + .where({ id: testProject.id }) + .first() + + expect(project).to.not.be.undefined + }) + }) + : void 0 diff --git a/packages/server/test/graphql/generated/graphql.ts b/packages/server/test/graphql/generated/graphql.ts index 19789627f6..17565adea6 100644 --- a/packages/server/test/graphql/generated/graphql.ts +++ b/packages/server/test/graphql/generated/graphql.ts @@ -4496,6 +4496,10 @@ export type WorkspaceProjectInviteCreateInput = { export type WorkspaceProjectMutations = { __typename?: 'WorkspaceProjectMutations'; create: Project; + /** + * Update project region and move all regional data to new db. + * TODO: Currently performs all operations synchronously in request, should probably be scheduled. + */ moveToRegion: Project; moveToWorkspace: Project; updateRole: Project; @@ -5087,6 +5091,14 @@ export type UpdateRegionMutationVariables = Exact<{ export type UpdateRegionMutation = { __typename?: 'Mutation', serverInfoMutations: { __typename?: 'ServerInfoMutations', multiRegion: { __typename?: 'ServerRegionMutations', update: { __typename?: 'ServerRegionItem', id: string, key: string, name: string, description?: string | null } } } }; +export type UpdateProjectRegionMutationVariables = Exact<{ + projectId: Scalars['String']['input']; + regionKey: Scalars['String']['input']; +}>; + + +export type UpdateProjectRegionMutation = { __typename?: 'Mutation', workspaceMutations: { __typename?: 'WorkspaceMutations', projects: { __typename?: 'WorkspaceProjectMutations', moveToRegion: { __typename?: 'Project', id: string } } } }; + export type BasicProjectAccessRequestFieldsFragment = { __typename?: 'ProjectAccessRequest', id: string, requesterId: string, projectId: string, createdAt: string, requester: { __typename?: 'LimitedUser', id: string, name: string } }; export type CreateProjectAccessRequestMutationVariables = Exact<{ @@ -5608,6 +5620,7 @@ export const GetAvailableRegionKeysDocument = {"kind":"Document","definitions":[ export const CreateNewRegionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CreateNewRegion"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"CreateServerRegionInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"serverInfoMutations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"multiRegion"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"create"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"MainRegionMetadata"}}]}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"MainRegionMetadata"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerRegionItem"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"key"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}}]}}]} as unknown as DocumentNode; export const GetRegionsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetRegions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"serverInfo"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"multiRegion"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"regions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"MainRegionMetadata"}}]}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"MainRegionMetadata"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerRegionItem"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"key"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}}]}}]} as unknown as DocumentNode; export const UpdateRegionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"UpdateRegion"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"UpdateServerRegionInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"serverInfoMutations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"multiRegion"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"update"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"MainRegionMetadata"}}]}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"MainRegionMetadata"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerRegionItem"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"key"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}}]}}]} as unknown as DocumentNode; +export const UpdateProjectRegionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"UpdateProjectRegion"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"regionKey"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"workspaceMutations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"projects"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"moveToRegion"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"projectId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}}},{"kind":"Argument","name":{"kind":"Name","value":"regionKey"},"value":{"kind":"Variable","name":{"kind":"Name","value":"regionKey"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}}]}}]}}]}}]}}]} as unknown as DocumentNode; export const CreateProjectAccessRequestDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CreateProjectAccessRequest"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"projectMutations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"accessRequestMutations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"create"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"projectId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"BasicProjectAccessRequestFields"}}]}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"BasicProjectAccessRequestFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ProjectAccessRequest"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"requester"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}},{"kind":"Field","name":{"kind":"Name","value":"requesterId"}},{"kind":"Field","name":{"kind":"Name","value":"projectId"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}}]}}]} as unknown as DocumentNode; export const GetActiveUserProjectAccessRequestDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetActiveUserProjectAccessRequest"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"activeUser"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"projectAccessRequest"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"projectId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"BasicProjectAccessRequestFields"}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"BasicProjectAccessRequestFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ProjectAccessRequest"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"requester"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}},{"kind":"Field","name":{"kind":"Name","value":"requesterId"}},{"kind":"Field","name":{"kind":"Name","value":"projectId"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}}]}}]} as unknown as DocumentNode; export const GetActiveUserFullProjectAccessRequestDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetActiveUserFullProjectAccessRequest"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"activeUser"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"projectAccessRequest"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"projectId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"BasicProjectAccessRequestFields"}},{"kind":"Field","name":{"kind":"Name","value":"project"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"BasicProjectAccessRequestFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ProjectAccessRequest"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"requester"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}},{"kind":"Field","name":{"kind":"Name","value":"requesterId"}},{"kind":"Field","name":{"kind":"Name","value":"projectId"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}}]}}]} as unknown as DocumentNode; diff --git a/packages/server/test/graphql/multiRegion.ts b/packages/server/test/graphql/multiRegion.ts index ea44f957fd..71a73e57b5 100644 --- a/packages/server/test/graphql/multiRegion.ts +++ b/packages/server/test/graphql/multiRegion.ts @@ -60,3 +60,15 @@ export const updateRegionMutation = gql` ${mainRegionMetadataFragment} ` + +export const updateProjectRegionMutation = gql` + mutation UpdateProjectRegion($projectId: String!, $regionKey: String!) { + workspaceMutations { + projects { + moveToRegion(projectId: $projectId, regionKey: $regionKey) { + id + } + } + } + } +` From f1a82e63112e39507878b26c16b6a4f6db0147dd Mon Sep 17 00:00:00 2001 From: Chuck Driesler Date: Thu, 16 Jan 2025 19:46:02 +0000 Subject: [PATCH 04/28] fix(regions): sabrina carpenter please please please --- docker-compose-deps.yml | 28 +++++++++++++++++++ .../tests/integration/projects.graph.spec.ts | 11 ++++---- packages/server/multiregion.test.example.json | 14 ++++++++++ 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/docker-compose-deps.yml b/docker-compose-deps.yml index d5068bf4e4..e6e59ea026 100644 --- a/docker-compose-deps.yml +++ b/docker-compose-deps.yml @@ -34,6 +34,22 @@ services: ports: - '127.0.0.1:5401:5432' + postgres-region2: + build: + context: . + dockerfile: utils/postgres/Dockerfile + restart: always + environment: + POSTGRES_DB: speckle + POSTGRES_USER: speckle + POSTGRES_PASSWORD: speckle + volumes: + - postgres-region2-data:/var/lib/postgresql/data/ + - ./setup/db/10-docker_postgres_init.sql:/docker-entrypoint-initdb.d/10-docker_postgres_init.sql + - ./setup/db/11-docker_postgres_keycloack_init.sql:/docker-entrypoint-initdb.d/11-docker_postgres_keycloack_init.sql + ports: + - '127.0.0.1:5402:5432' + redis: image: 'redis:7-alpine' restart: always @@ -62,6 +78,16 @@ services: - '127.0.0.1:9020:9000' - '127.0.0.1:9021:9001' + minio-region2: + image: 'minio/minio' + command: server /data --console-address ":9001" + restart: always + volumes: + - minio-region2-data:/data + ports: + - '127.0.0.1:9040:9000' + - '127.0.0.1:9041:9001' + # Local OIDC provider for testing keycloak: image: quay.io/keycloak/keycloak:25.0 @@ -133,8 +159,10 @@ services: volumes: postgres-data: postgres-region1-data: + postgres-region2-data: redis-data: pgadmin-data: redis_insight-data: minio-data: minio-region1-data: + minio-region2-data: diff --git a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts index ad844e2de6..60c749375e 100644 --- a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts @@ -3,6 +3,7 @@ import { AllScopes } from '@/modules/core/helpers/mainConstants' import { createRandomEmail } from '@/modules/core/helpers/testHelpers' import { StreamRecord } from '@/modules/core/helpers/types' import { grantStreamPermissionsFactory } from '@/modules/core/repositories/streams' +import { getDb } from '@/modules/multiregion/utils/dbSelector' import { BasicTestWorkspace, createTestWorkspace @@ -28,7 +29,6 @@ import { } from '@/test/graphqlHelper' import { beforeEachContext } from '@/test/hooks' import { - getMainTestRegionClient, isMultiRegionTestMode, waitForRegionUser } from '@/test/speckle-helpers/regions' @@ -285,7 +285,8 @@ describe('Workspace project GQL CRUD', () => { isMultiRegionTestMode() ? describe('Workspace project region changes', () => { - const testRegionKey = 'region1' + const regionKey1 = 'region1' + const regionKey2 = 'region2' const adminUser: BasicTestUser = { id: '', @@ -314,7 +315,7 @@ isMultiRegionTestMode() await createTestUser(adminUser) await waitForRegionUser(adminUser) await createTestWorkspace(testWorkspace, adminUser, { - regionKey: testRegionKey, + regionKey: regionKey1, addPlan: { name: 'unlimited', status: 'valid' @@ -324,7 +325,7 @@ isMultiRegionTestMode() testProject.workspaceId = testWorkspace.id apollo = await testApolloServer({ authUserId: adminUser.id }) - regionDb = getMainTestRegionClient() + regionDb = await getDb({ regionKey: regionKey2 }) }) beforeEach(async () => { @@ -334,7 +335,7 @@ isMultiRegionTestMode() it('moves project record to target regional db', async () => { const res = await apollo.execute(UpdateProjectRegionDocument, { projectId: testProject.id, - regionKey: testRegionKey + regionKey: regionKey2 }) expect(res).to.not.haveGraphQLErrors() diff --git a/packages/server/multiregion.test.example.json b/packages/server/multiregion.test.example.json index 0eff189562..6fc82a924c 100644 --- a/packages/server/multiregion.test.example.json +++ b/packages/server/multiregion.test.example.json @@ -27,6 +27,20 @@ "endpoint": "http://127.0.0.1:9020", "s3Region": "us-east-1" } + }, + "region2": { + "postgres": { + "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5402/speckle2_test", + "privateConnectionUri": "postgresql://speckle:speckle@postgres-region2:5432/speckle2_test" + }, + "blobStorage": { + "accessKey": "minioadmin", + "secretKey": "minioadmin", + "bucket": "test-speckle-server", + "createBucketIfNotExists": true, + "endpoint": "http://127.0.0.1:9040", + "s3Region": "us-east-1" + } } } } From 88bafb233ad33b562366409df94bb9b9df99ede3 Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Wed, 22 Jan 2025 17:12:41 +0000 Subject: [PATCH 05/28] fix(regions): repair multiregion test setup --- .../modules/multiregion/utils/dbSelector.ts | 7 ++++- .../modules/workspaces/domain/operations.ts | 1 + .../workspaces/graph/resolvers/regions.ts | 2 ++ .../workspaces/repositories/regions.ts | 26 ++++++++++++++++++- .../modules/workspaces/services/regions.ts | 5 ++++ 5 files changed, 39 insertions(+), 2 deletions(-) diff --git a/packages/server/modules/multiregion/utils/dbSelector.ts b/packages/server/modules/multiregion/utils/dbSelector.ts index ae6b210e8f..a01c5af9d1 100644 --- a/packages/server/modules/multiregion/utils/dbSelector.ts +++ b/packages/server/modules/multiregion/utils/dbSelector.ts @@ -215,7 +215,12 @@ const setUpUserReplication = async ({ info: { pubName, regionName } } ) - if (!err.message.includes('already exists')) throw err + if ( + !['already exists', 'violates unique constraint'].some((message) => + err.message.includes(message) + ) + ) + throw err } const fromUrl = new URL( diff --git a/packages/server/modules/workspaces/domain/operations.ts b/packages/server/modules/workspaces/domain/operations.ts index 223bb6789c..2dfb5ab379 100644 --- a/packages/server/modules/workspaces/domain/operations.ts +++ b/packages/server/modules/workspaces/domain/operations.ts @@ -352,6 +352,7 @@ export type UpdateProjectRegion = (params: { regionKey: string }) => Promise +export type CopyWorkspace = (params: { workspaceId: string }) => Promise export type CopyProjects = (params: { projectIds: string[] }) => Promise export type CopyProjectModels = (params: { projectIds: string[] diff --git a/packages/server/modules/workspaces/graph/resolvers/regions.ts b/packages/server/modules/workspaces/graph/resolvers/regions.ts index 80ae195c4d..453738ced8 100644 --- a/packages/server/modules/workspaces/graph/resolvers/regions.ts +++ b/packages/server/modules/workspaces/graph/resolvers/regions.ts @@ -9,6 +9,7 @@ import { copyProjectModelsFactory, copyProjectsFactory, copyProjectVersionsFactory, + copyWorkspaceFactory, getDefaultRegionFactory, upsertRegionAssignmentFactory } from '@/modules/workspaces/repositories/regions' @@ -83,6 +84,7 @@ export default { getWorkspacePlan: getWorkspacePlanFactory({ db }) }) }), + copyWorkspace: copyWorkspaceFactory({ sourceDb, targetDb }), copyProjects: copyProjectsFactory({ sourceDb, targetDb }), copyProjectModels: copyProjectModelsFactory({ sourceDb, targetDb }), copyProjectVersions: copyProjectVersionsFactory({ sourceDb, targetDb }) diff --git a/packages/server/modules/workspaces/repositories/regions.ts b/packages/server/modules/workspaces/repositories/regions.ts index 962330ba4a..57b375bba4 100644 --- a/packages/server/modules/workspaces/repositories/regions.ts +++ b/packages/server/modules/workspaces/repositories/regions.ts @@ -23,10 +23,16 @@ import { CopyProjectModels, CopyProjects, CopyProjectVersions, + CopyWorkspace, GetDefaultRegion, UpsertRegionAssignment } from '@/modules/workspaces/domain/operations' -import { WorkspaceRegionAssignment } from '@/modules/workspacesCore/domain/types' +import { WorkspaceNotFoundError } from '@/modules/workspaces/errors/workspace' +import { + Workspace, + WorkspaceRegionAssignment +} from '@/modules/workspacesCore/domain/types' +import { Workspaces } from '@/modules/workspacesCore/helpers/db' import { Knex } from 'knex' export const WorkspaceRegions = buildTableHelper('workspace_regions', [ @@ -35,6 +41,7 @@ export const WorkspaceRegions = buildTableHelper('workspace_regions', [ ]) const tables = { + workspaces: (db: Knex) => db(Workspaces.name), workspaceRegions: (db: Knex) => db(WorkspaceRegions.name), regions: (db: Knex) => db(Regions.name), projects: (db: Knex) => db(Streams.name), @@ -73,6 +80,23 @@ export const getDefaultRegionFactory = return row } +export const copyWorkspaceFactory = + (deps: { sourceDb: Knex; targetDb: Knex }): CopyWorkspace => + async ({ workspaceId }) => { + const workspace = await tables + .workspaces(deps.sourceDb) + .select('*') + .where({ id: workspaceId }) + + if (!workspace) { + throw new WorkspaceNotFoundError() + } + + await tables.workspaces(deps.targetDb).insert(workspace) + + return workspaceId + } + export const copyProjectsFactory = (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjects => async ({ projectIds }) => { diff --git a/packages/server/modules/workspaces/services/regions.ts b/packages/server/modules/workspaces/services/regions.ts index e673fc5220..0fc1323930 100644 --- a/packages/server/modules/workspaces/services/regions.ts +++ b/packages/server/modules/workspaces/services/regions.ts @@ -8,6 +8,7 @@ import { CopyProjectModels, CopyProjects, CopyProjectVersions, + CopyWorkspace, GetAvailableRegions, GetDefaultRegion, GetWorkspace, @@ -86,6 +87,7 @@ export const updateProjectRegionFactory = countProjectModels: GetStreamBranchCount countProjectVersions: GetStreamCommitCount getAvailableRegions: GetAvailableRegions + copyWorkspace: CopyWorkspace copyProjects: CopyProjects copyProjectModels: CopyProjectModels copyProjectVersions: CopyProjectVersions @@ -120,6 +122,9 @@ export const updateProjectRegionFactory = ) } + // Move workspace + await deps.copyWorkspace({ workspaceId: project.workspaceId }) + // Move commits const projectIds = await deps.copyProjects({ projectIds: [projectId] }) const modelIds = await deps.copyProjectModels({ projectIds }) From 1fb3b43ffa7369775cf202586bee385ac466f2e4 Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Wed, 22 Jan 2025 17:31:45 +0000 Subject: [PATCH 06/28] chore(regions): appease ts --- packages/server/modules/multiregion/utils/dbSelector.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/server/modules/multiregion/utils/dbSelector.ts b/packages/server/modules/multiregion/utils/dbSelector.ts index a01c5af9d1..3746e91b69 100644 --- a/packages/server/modules/multiregion/utils/dbSelector.ts +++ b/packages/server/modules/multiregion/utils/dbSelector.ts @@ -206,7 +206,7 @@ const setUpUserReplication = async ({ try { await from.public.raw(`CREATE PUBLICATION ${pubName} FOR TABLE users;`) } catch (err) { - if (!(err instanceof Error)) + if (!(err instanceof Error)) { throw new DatabaseError( 'Could not create publication {pubName} when setting up user replication for region {regionName}', from.public, @@ -215,9 +215,13 @@ const setUpUserReplication = async ({ info: { pubName, regionName } } ) + } + + const errorMessage = err.message + if ( !['already exists', 'violates unique constraint'].some((message) => - err.message.includes(message) + errorMessage.includes(message) ) ) throw err From 54b61bfec1bc020d1f3632c1eb6244b1665a23db Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Wed, 22 Jan 2025 22:23:28 +0000 Subject: [PATCH 07/28] chore(multiregion): update test multiregion config --- .circleci/multiregion.test-ci.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.circleci/multiregion.test-ci.json b/.circleci/multiregion.test-ci.json index 3d5a9ec1c3..78619c2af2 100644 --- a/.circleci/multiregion.test-ci.json +++ b/.circleci/multiregion.test-ci.json @@ -25,6 +25,19 @@ "endpoint": "http://127.0.0.1:9020", "s3Region": "us-east-1" } + }, + "region2": { + "postgres": { + "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5434/speckle2_test" + }, + "blobStorage": { + "accessKey": "minioadmin", + "secretKey": "minioadmin", + "bucket": "speckle-server", + "createBucketIfNotExists": true, + "endpoint": "http://127.0.0.1:9040", + "s3Region": "us-east-1" + } } } } From de8d78a594cf87d3f3cc15256e6e4e3d1c2cf02a Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Wed, 22 Jan 2025 23:05:17 +0000 Subject: [PATCH 08/28] chore(multiregion): fix test docker config and test --- .circleci/config.yml | 8 ++ .../workspaces/repositories/regions.ts | 4 +- .../tests/integration/projects.graph.spec.ts | 90 ++++++++++++++++--- 3 files changed, 90 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 47030bceeb..8ffa834c25 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -598,10 +598,18 @@ jobs: POSTGRES_PASSWORD: speckle POSTGRES_USER: speckle command: -c 'max_connections=1000' -c 'port=5433' -c 'wal_level=logical' + - image: 'speckle/speckle-postgres' + environment: + POSTGRES_DB: speckle2_test + POSTGRES_PASSWORD: speckle + POSTGRES_USER: speckle + command: -c 'max_connections=1000' -c 'port=5434' -c 'wal_level=logical' - image: 'minio/minio' command: server /data --console-address ":9001" --address "0.0.0.0:9000" - image: 'minio/minio' command: server /data --console-address ":9021" --address "0.0.0.0:9020" + - image: 'minio/minio' + command: server /data --console-address ":9041" --address "0.0.0.0:9040" environment: # Same as test-server: NODE_ENV: test diff --git a/packages/server/modules/workspaces/repositories/regions.ts b/packages/server/modules/workspaces/repositories/regions.ts index 57b375bba4..1fdca98480 100644 --- a/packages/server/modules/workspaces/repositories/regions.ts +++ b/packages/server/modules/workspaces/repositories/regions.ts @@ -207,10 +207,10 @@ export const copyProjectVersionsFactory = for await (const versions of executeBatchedSelect(selectVersions)) { for (const version of versions) { - const { commitId, ...commit } = version + const { commitId, streamId, ...commit } = version // Store copied version id - copiedVersionIds[projectId].push(commitId) + copiedVersionIds[streamId].push(commitId) // Copy `commits` row to target db await tables.versions(deps.targetDb).insert(commit).onConflict().ignore() diff --git a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts index 60c749375e..6d219beb9d 100644 --- a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts @@ -1,7 +1,7 @@ import { db } from '@/db/knex' import { AllScopes } from '@/modules/core/helpers/mainConstants' import { createRandomEmail } from '@/modules/core/helpers/testHelpers' -import { StreamRecord } from '@/modules/core/helpers/types' +import { BranchRecord, CommitRecord, StreamRecord } from '@/modules/core/helpers/types' import { grantStreamPermissionsFactory } from '@/modules/core/repositories/streams' import { getDb } from '@/modules/multiregion/utils/dbSelector' import { @@ -28,6 +28,12 @@ import { TestApolloServer } from '@/test/graphqlHelper' import { beforeEachContext } from '@/test/hooks' +import { BasicTestBranch, createTestBranch } from '@/test/speckle-helpers/branchHelper' +import { + BasicTestCommit, + createTestCommit, + createTestObject +} from '@/test/speckle-helpers/commitHelper' import { isMultiRegionTestMode, waitForRegionUser @@ -37,6 +43,7 @@ import { Roles } from '@speckle/shared' import { expect } from 'chai' import cryptoRandomString from 'crypto-random-string' import { Knex } from 'knex' +import { SetOptional } from 'type-fest' const grantStreamPermissions = grantStreamPermissionsFactory({ db }) @@ -294,10 +301,9 @@ isMultiRegionTestMode() email: createRandomEmail() } - const testWorkspace: BasicTestWorkspace = { + const testWorkspace: SetOptional = { id: '', ownerId: '', - slug: '', name: 'Unlimited Workspace' } @@ -308,12 +314,34 @@ isMultiRegionTestMode() isPublic: true } + const testModel: BasicTestBranch = { + id: '', + name: cryptoRandomString({ length: 8 }), + streamId: '', + authorId: '' + } + + const testVersion: BasicTestCommit = { + id: '', + objectId: '', + streamId: '', + authorId: '' + } + let apollo: TestApolloServer - let regionDb: Knex + let targetRegionDb: Knex before(async () => { await createTestUser(adminUser) await waitForRegionUser(adminUser) + + apollo = await testApolloServer({ authUserId: adminUser.id }) + targetRegionDb = await getDb({ regionKey: regionKey2 }) + }) + + beforeEach(async () => { + delete testWorkspace.slug + await createTestWorkspace(testWorkspace, adminUser, { regionKey: regionKey1, addPlan: { @@ -324,12 +352,20 @@ isMultiRegionTestMode() testProject.workspaceId = testWorkspace.id - apollo = await testApolloServer({ authUserId: adminUser.id }) - regionDb = await getDb({ regionKey: regionKey2 }) - }) - - beforeEach(async () => { await createTestStream(testProject, adminUser) + await createTestBranch({ + stream: testProject, + branch: testModel, + owner: adminUser + }) + + testVersion.branchName = testModel.name + testVersion.objectId = await createTestObject({ projectId: testProject.id }) + + await createTestCommit(testVersion, { + owner: adminUser, + stream: testProject + }) }) it('moves project record to target regional db', async () => { @@ -340,7 +376,7 @@ isMultiRegionTestMode() expect(res).to.not.haveGraphQLErrors() - const project = await regionDb + const project = await targetRegionDb .table('streams') .select('*') .where({ id: testProject.id }) @@ -348,5 +384,39 @@ isMultiRegionTestMode() expect(project).to.not.be.undefined }) + + it('moves project models to target regional db', async () => { + const res = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + + expect(res).to.not.haveGraphQLErrors() + + const branch = await targetRegionDb + .table('branches') + .select('*') + .where({ id: testModel.id }) + .first() + + expect(branch).to.not.be.undefined + }) + + it('moves project model versions to target regional db', async () => { + const res = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + + expect(res).to.not.haveGraphQLErrors() + + const version = await targetRegionDb + .table('commits') + .select('*') + .where({ id: testVersion.id }) + .first() + + expect(version).to.not.be.undefined + }) }) : void 0 From ca850399b70ed08fbf32f15252770bb7d4f0cef1 Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Thu, 23 Jan 2025 11:52:35 +0000 Subject: [PATCH 09/28] chore(multiregion): use transaction --- .circleci/config.yml | 2 +- .../server/modules/workspaces/graph/resolvers/regions.ts | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8ffa834c25..1ea6bcbd1b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -615,7 +615,7 @@ jobs: NODE_ENV: test DATABASE_URL: 'postgres://speckle:speckle@127.0.0.1:5432/speckle2_test' PGDATABASE: speckle2_test - POSTGRES_MAX_CONNECTIONS_SERVER: 20 + POSTGRES_MAX_CONNECTIONS_SERVER: 50 PGUSER: speckle SESSION_SECRET: 'keyboard cat' STRATEGY_LOCAL: 'true' diff --git a/packages/server/modules/workspaces/graph/resolvers/regions.ts b/packages/server/modules/workspaces/graph/resolvers/regions.ts index 453738ced8..b58c27270c 100644 --- a/packages/server/modules/workspaces/graph/resolvers/regions.ts +++ b/packages/server/modules/workspaces/graph/resolvers/regions.ts @@ -26,6 +26,7 @@ import { Roles } from '@speckle/shared' import { getProjectFactory } from '@/modules/core/repositories/streams' import { getStreamBranchCountFactory } from '@/modules/core/repositories/branches' import { getStreamCommitCountFactory } from '@/modules/core/repositories/commits' +import { withTransaction } from '@/modules/shared/helpers/dbHelper' export default { Workspace: { @@ -72,7 +73,7 @@ export default { ) const sourceDb = await getProjectDbClient({ projectId: args.projectId }) - const targetDb = await getDb({ regionKey: args.regionKey }) + const targetDb = await (await getDb({ regionKey: args.regionKey })).transaction() const updateProjectRegion = updateProjectRegionFactory({ getProject: getProjectFactory({ db: sourceDb }), @@ -90,7 +91,7 @@ export default { copyProjectVersions: copyProjectVersionsFactory({ sourceDb, targetDb }) }) - return await updateProjectRegion(args) + return await withTransaction(updateProjectRegion(args), targetDb) } } } as Resolvers From 201fe2e3ec7617f1fdb64d3cb1fc01a2f90d9290 Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Thu, 23 Jan 2025 16:57:43 +0000 Subject: [PATCH 10/28] chore(multiregion): maybe this will work --- .../modules/auth/tests/apps.graphql.spec.js | 6 ------ .../server/modules/auth/tests/auth.spec.js | 6 ------ .../server/modules/stats/tests/stats.spec.ts | 9 +-------- .../modules/webhooks/tests/webhooks.spec.js | 20 ++----------------- 4 files changed, 3 insertions(+), 38 deletions(-) diff --git a/packages/server/modules/auth/tests/apps.graphql.spec.js b/packages/server/modules/auth/tests/apps.graphql.spec.js index 6f651fde0c..4e90f06f86 100644 --- a/packages/server/modules/auth/tests/apps.graphql.spec.js +++ b/packages/server/modules/auth/tests/apps.graphql.spec.js @@ -63,7 +63,6 @@ const { getServerInfoFactory } = require('@/modules/core/repositories/server') const { getEventBus } = require('@/modules/shared/services/eventBus') let sendRequest -let server const createAppToken = createAppTokenFactory({ storeApiToken: storeApiTokenFactory({ db }), @@ -128,7 +127,6 @@ describe('GraphQL @apps-api', () => { before(async () => { const ctx = await beforeEachContext() - server = ctx.server ;({ sendRequest } = await initializeTestServer(ctx)) testUser = { name: 'Dimitrie Stefanescu', @@ -157,10 +155,6 @@ describe('GraphQL @apps-api', () => { ])}` }) - after(async () => { - await server.close() - }) - let testAppId let testApp diff --git a/packages/server/modules/auth/tests/auth.spec.js b/packages/server/modules/auth/tests/auth.spec.js index 45f75cd951..67888198b2 100644 --- a/packages/server/modules/auth/tests/auth.spec.js +++ b/packages/server/modules/auth/tests/auth.spec.js @@ -138,7 +138,6 @@ const expect = chai.expect let app let sendRequest -let server describe('Auth @auth', () => { describe('Local authN & authZ (token endpoints)', () => { @@ -160,7 +159,6 @@ describe('Auth @auth', () => { before(async () => { const ctx = await beforeEachContext() - server = ctx.server app = ctx.app ;({ sendRequest } = await initializeTestServer(ctx)) @@ -173,10 +171,6 @@ describe('Auth @auth', () => { ) }) - after(async () => { - await server.close() - }) - it('Should register a new user (speckle frontend)', async () => { await request(app) .post('/auth/local/register?challenge=test') diff --git a/packages/server/modules/stats/tests/stats.spec.ts b/packages/server/modules/stats/tests/stats.spec.ts index f0fd0c17b6..757f8df636 100644 --- a/packages/server/modules/stats/tests/stats.spec.ts +++ b/packages/server/modules/stats/tests/stats.spec.ts @@ -8,7 +8,6 @@ import { getTotalUserCountFactory } from '@/modules/stats/repositories/index' import { Scopes } from '@speckle/shared' -import { Server } from 'node:http' import { db } from '@/db/knex' import { createCommitByBranchIdFactory, @@ -196,8 +195,7 @@ describe('Server stats services @stats-services', function () { }) describe('Server stats api @stats-api', function () { - let server: Server, - sendRequest: Awaited>['sendRequest'] + let sendRequest: Awaited>['sendRequest'] const adminUser = { name: 'Dimitrie', @@ -235,7 +233,6 @@ describe('Server stats api @stats-api', function () { before(async function () { this.timeout(15000) const ctx = await beforeEachContext() - server = ctx.server ;({ sendRequest } = await initializeTestServer(ctx)) adminUser.id = await createUser(adminUser) @@ -265,10 +262,6 @@ describe('Server stats api @stats-api', function () { await seedDb(params) }) - after(async function () { - await server.close() - }) - it('Should not get stats if user is not admin', async () => { const res = await sendRequest(adminUser.badToken, { query: fullQuery }) expect(res.body.errors).to.exist diff --git a/packages/server/modules/webhooks/tests/webhooks.spec.js b/packages/server/modules/webhooks/tests/webhooks.spec.js index 0c8001a165..0befc5da0c 100644 --- a/packages/server/modules/webhooks/tests/webhooks.spec.js +++ b/packages/server/modules/webhooks/tests/webhooks.spec.js @@ -2,11 +2,7 @@ const expect = require('chai').expect const assert = require('assert') -const { - beforeEachContext, - initializeTestServer, - truncateTables -} = require('@/test/hooks') +const { beforeEachContext, initializeTestServer } = require('@/test/hooks') const { noErrors } = require('@/test/helpers') const { Scopes, Roles } = require('@speckle/shared') const { @@ -26,7 +22,6 @@ const { deleteWebhookFactory, dispatchStreamEventFactory } = require('@/modules/webhooks/services/webhooks') -const { Users, Streams } = require('@/modules/core/dbSchema') const { getStreamFactory, createStreamFactory, @@ -166,7 +161,7 @@ const createPersonalAccessToken = createPersonalAccessTokenFactory({ describe('Webhooks @webhooks', () => { const getWebhook = getWebhookByIdFactory({ db }) - let server, sendRequest + let sendRequest const userOne = { name: 'User', @@ -191,7 +186,6 @@ describe('Webhooks @webhooks', () => { before(async () => { const ctx = await beforeEachContext() - server = ctx.server ;({ sendRequest } = await initializeTestServer(ctx)) userOne.id = await createUser(userOne) @@ -201,16 +195,6 @@ describe('Webhooks @webhooks', () => { webhookOne.streamId = streamOne.id }) - after(async () => { - await truncateTables([ - Users.name, - Streams.name, - 'webhooks_config', - 'webhooks_events' - ]) - await server.close() - }) - describe('Create, Read, Update, Delete Webhooks', () => { it('Should create a webhook', async () => { webhookOne.id = await createWebhookFactory({ From 08b53ccba305bde10789a753bc11d4170f9a98fe Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Fri, 24 Jan 2025 12:26:52 +0000 Subject: [PATCH 11/28] fix(multiregion): drop subs synchronously --- packages/server/test/hooks.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/server/test/hooks.ts b/packages/server/test/hooks.ts index 8bc17fac2f..c86fd4cf87 100644 --- a/packages/server/test/hooks.ts +++ b/packages/server/test/hooks.ts @@ -210,8 +210,10 @@ export const resetPubSubFactory = (deps: { db: Knex }) => async () => { } // Drop all subs - // (concurrently, cause it seems possible and we have those delays there) - await Promise.all(subscriptions.rows.map(dropSubs)) + for (const subscription of subscriptions.rows) { + await dropSubs(subscription) + await wait(1000) + } // Drop all pubs for (const pub of publications.rows) { From 8edde647dfad341d34908e3c858568a9d1a3a392 Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Fri, 24 Jan 2025 13:23:00 +0000 Subject: [PATCH 12/28] chore(multiregion): desperate test logs --- packages/server/test/hooks.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/server/test/hooks.ts b/packages/server/test/hooks.ts index c86fd4cf87..ed299ef474 100644 --- a/packages/server/test/hooks.ts +++ b/packages/server/test/hooks.ts @@ -209,8 +209,12 @@ export const resetPubSubFactory = (deps: { db: Knex }) => async () => { ) } + console.log(subscriptions.rows.length) + console.log(subscriptions.rows) + // Drop all subs for (const subscription of subscriptions.rows) { + console.log(JSON.stringify(subscription, null, 2)) await dropSubs(subscription) await wait(1000) } From afc06d78d749265033b7b4274c282c90c63a146d Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Fri, 24 Jan 2025 14:01:01 +0000 Subject: [PATCH 13/28] chore(multiregion): somehow that worked? --- packages/server/test/hooks.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/server/test/hooks.ts b/packages/server/test/hooks.ts index ed299ef474..fe6ac34c93 100644 --- a/packages/server/test/hooks.ts +++ b/packages/server/test/hooks.ts @@ -209,14 +209,10 @@ export const resetPubSubFactory = (deps: { db: Knex }) => async () => { ) } - console.log(subscriptions.rows.length) - console.log(subscriptions.rows) - // Drop all subs for (const subscription of subscriptions.rows) { - console.log(JSON.stringify(subscription, null, 2)) await dropSubs(subscription) - await wait(1000) + await wait(500) } // Drop all pubs From 6d6f800fbf82ac25a3d690aa349a75abdf2eddb2 Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Fri, 24 Jan 2025 14:34:12 +0000 Subject: [PATCH 14/28] chore(multiregion): add load-bearing log statement --- packages/server/test/hooks.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/server/test/hooks.ts b/packages/server/test/hooks.ts index fe6ac34c93..0368148274 100644 --- a/packages/server/test/hooks.ts +++ b/packages/server/test/hooks.ts @@ -211,6 +211,8 @@ export const resetPubSubFactory = (deps: { db: Knex }) => async () => { // Drop all subs for (const subscription of subscriptions.rows) { + // If we do not log something here, CircleCI may kill the job while we wait all `dropSubs` calls to finish. + console.log(`Dropping subscription ${subscription.subname}`) await dropSubs(subscription) await wait(500) } From a27b97ab13a4e850be2e97b0cf724843847ad8dc Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Fri, 24 Jan 2025 14:49:41 +0000 Subject: [PATCH 15/28] chore(multiregion): move services --- .../workspaces/graph/resolvers/regions.ts | 14 +- .../workspaces/repositories/projectRegions.ts | 216 +++++++++++++++++ .../workspaces/repositories/regions.ts | 218 +----------------- .../workspaces/services/projectRegions.ts | 86 +++++++ .../modules/workspaces/services/regions.ts | 87 +------ 5 files changed, 314 insertions(+), 307 deletions(-) create mode 100644 packages/server/modules/workspaces/repositories/projectRegions.ts create mode 100644 packages/server/modules/workspaces/services/projectRegions.ts diff --git a/packages/server/modules/workspaces/graph/resolvers/regions.ts b/packages/server/modules/workspaces/graph/resolvers/regions.ts index b58c27270c..57a45aa214 100644 --- a/packages/server/modules/workspaces/graph/resolvers/regions.ts +++ b/packages/server/modules/workspaces/graph/resolvers/regions.ts @@ -6,22 +6,24 @@ import { getDb, getProjectDbClient } from '@/modules/multiregion/utils/dbSelecto import { getRegionsFactory } from '@/modules/multiregion/repositories' import { authorizeResolver } from '@/modules/shared' import { - copyProjectModelsFactory, - copyProjectsFactory, - copyProjectVersionsFactory, - copyWorkspaceFactory, getDefaultRegionFactory, upsertRegionAssignmentFactory } from '@/modules/workspaces/repositories/regions' +import { + copyProjectModelsFactory, + copyProjectsFactory, + copyProjectVersionsFactory, + copyWorkspaceFactory +} from '@/modules/workspaces/repositories/projectRegions' import { getWorkspaceFactory, upsertWorkspaceFactory } from '@/modules/workspaces/repositories/workspaces' import { assignWorkspaceRegionFactory, - getAvailableRegionsFactory, - updateProjectRegionFactory + getAvailableRegionsFactory } from '@/modules/workspaces/services/regions' +import { updateProjectRegionFactory } from '@/modules/workspaces/services/projectRegions' import { Roles } from '@speckle/shared' import { getProjectFactory } from '@/modules/core/repositories/streams' import { getStreamBranchCountFactory } from '@/modules/core/repositories/branches' diff --git a/packages/server/modules/workspaces/repositories/projectRegions.ts b/packages/server/modules/workspaces/repositories/projectRegions.ts new file mode 100644 index 0000000000..94da98d14d --- /dev/null +++ b/packages/server/modules/workspaces/repositories/projectRegions.ts @@ -0,0 +1,216 @@ +import { + BranchCommits, + Branches, + Commits, + StreamCommits, + StreamFavorites, + Streams, + StreamsMeta +} from '@/modules/core/dbSchema' +import { Branch } from '@/modules/core/domain/branches/types' +import { Commit } from '@/modules/core/domain/commits/types' +import { Stream } from '@/modules/core/domain/streams/types' +import { + BranchCommitRecord, + StreamCommitRecord, + StreamFavoriteRecord +} from '@/modules/core/helpers/types' +import { executeBatchedSelect } from '@/modules/shared/helpers/dbHelper' +import { + CopyProjectModels, + CopyProjects, + CopyProjectVersions, + CopyWorkspace +} from '@/modules/workspaces/domain/operations' +import { WorkspaceNotFoundError } from '@/modules/workspaces/errors/workspace' +import { Knex } from 'knex' +import { Workspace } from '@/modules/workspacesCore/domain/types' +import { Workspaces } from '@/modules/workspacesCore/helpers/db' + +const tables = { + workspaces: (db: Knex) => db(Workspaces.name), + projects: (db: Knex) => db(Streams.name), + models: (db: Knex) => db(Branches.name), + versions: (db: Knex) => db(Commits.name), + branchCommits: (db: Knex) => db(BranchCommits.name), + streamCommits: (db: Knex) => db(StreamCommits.name), + streamFavorites: (db: Knex) => db(StreamFavorites.name), + streamsMeta: (db: Knex) => db(StreamsMeta.name) +} + +export const copyWorkspaceFactory = + (deps: { sourceDb: Knex; targetDb: Knex }): CopyWorkspace => + async ({ workspaceId }) => { + const workspace = await tables + .workspaces(deps.sourceDb) + .select('*') + .where({ id: workspaceId }) + + if (!workspace) { + throw new WorkspaceNotFoundError() + } + + await tables.workspaces(deps.targetDb).insert(workspace) + + return workspaceId + } + +export const copyProjectsFactory = + (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjects => + async ({ projectIds }) => { + const selectProjects = tables + .projects(deps.sourceDb) + .select('*') + .whereIn(Streams.col.id, projectIds) + const copiedProjectIds: string[] = [] + + // Copy project record + for await (const projects of executeBatchedSelect(selectProjects)) { + for (const project of projects) { + // Store copied project id + copiedProjectIds.push(project.id) + + // Copy `streams` row to target db + await tables.projects(deps.targetDb).insert(project).onConflict().ignore() + } + + const projectIds = projects.map((project) => project.id) + + // Fetch `stream_favorites` rows for projects in batch + const selectStreamFavorites = tables + .streamFavorites(deps.sourceDb) + .select('*') + .whereIn(StreamFavorites.col.streamId, projectIds) + + for await (const streamFavorites of executeBatchedSelect(selectStreamFavorites)) { + for (const streamFavorite of streamFavorites) { + // Copy `stream_favorites` row to target db + await tables + .streamFavorites(deps.targetDb) + .insert(streamFavorite) + .onConflict() + .ignore() + } + } + + // Fetch `streams_meta` rows for projects in batch + const selectStreamsMetadata = tables + .streamsMeta(deps.sourceDb) + .select('*') + .whereIn(StreamsMeta.col.streamId, projectIds) + + for await (const streamsMetadataBatch of executeBatchedSelect( + selectStreamsMetadata + )) { + for (const streamMetadata of streamsMetadataBatch) { + // Copy `streams_meta` row to target db + await tables + .streamsMeta(deps.targetDb) + .insert(streamMetadata) + .onConflict() + .ignore() + } + } + } + + return copiedProjectIds + } + +export const copyProjectModelsFactory = + (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectModels => + async ({ projectIds }) => { + const copiedModelIds: Record = projectIds.reduce( + (result, id) => ({ ...result, [id]: [] }), + {} + ) + + for (const projectId of projectIds) { + const selectModels = tables + .models(deps.sourceDb) + .select('*') + .where({ streamId: projectId }) + + for await (const models of executeBatchedSelect(selectModels)) { + for (const model of models) { + // Store copied model ids + copiedModelIds[projectId].push(model.id) + + // Copy `branches` row to target db + await tables.models(deps.targetDb).insert(model).onConflict().ignore() + } + } + } + + return copiedModelIds + } + +export const copyProjectVersionsFactory = + (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectVersions => + async ({ projectIds }) => { + const copiedVersionIds: Record = projectIds.reduce( + (result, id) => ({ ...result, [id]: [] }), + {} + ) + + for (const projectId of projectIds) { + const selectVersions = tables + .streamCommits(deps.sourceDb) + .select('*') + .join( + Commits.name, + Commits.col.id, + StreamCommits.col.commitId + ) + .where({ streamId: projectId }) + + for await (const versions of executeBatchedSelect(selectVersions)) { + for (const version of versions) { + const { commitId, streamId, ...commit } = version + + // Store copied version id + copiedVersionIds[streamId].push(commitId) + + // Copy `commits` row to target db + await tables.versions(deps.targetDb).insert(commit).onConflict().ignore() + } + + const commitIds = versions.map((version) => version.commitId) + + // Fetch `branch_commits` rows for versions in batch + const selectBranchCommits = tables + .branchCommits(deps.sourceDb) + .select('*') + .whereIn(BranchCommits.col.commitId, commitIds) + + for await (const branchCommits of executeBatchedSelect(selectBranchCommits)) { + for (const branchCommit of branchCommits) { + // Copy `branch_commits` row to target db + await tables + .branchCommits(deps.targetDb) + .insert(branchCommit) + .onConflict() + .ignore() + } + } + + // Fetch `stream_commits` rows for versions in batch + const selectStreamCommits = tables + .streamCommits(deps.sourceDb) + .select('*') + .whereIn(StreamCommits.col.commitId, commitIds) + + for await (const streamCommits of executeBatchedSelect(selectStreamCommits)) { + for (const streamCommit of streamCommits) { + // Copy `stream_commits` row to target db + await tables + .streamCommits(deps.targetDb) + .insert(streamCommit) + .onConflict() + .ignore() + } + } + } + } + + return copiedVersionIds + } diff --git a/packages/server/modules/workspaces/repositories/regions.ts b/packages/server/modules/workspaces/repositories/regions.ts index 1fdca98480..dc95638296 100644 --- a/packages/server/modules/workspaces/repositories/regions.ts +++ b/packages/server/modules/workspaces/repositories/regions.ts @@ -1,38 +1,11 @@ -import { - BranchCommits, - Branches, - buildTableHelper, - Commits, - StreamCommits, - StreamFavorites, - Streams, - StreamsMeta -} from '@/modules/core/dbSchema' -import { Branch } from '@/modules/core/domain/branches/types' -import { Commit } from '@/modules/core/domain/commits/types' -import { Stream } from '@/modules/core/domain/streams/types' -import { - BranchCommitRecord, - StreamCommitRecord, - StreamFavoriteRecord -} from '@/modules/core/helpers/types' +import { buildTableHelper } from '@/modules/core/dbSchema' import { RegionRecord } from '@/modules/multiregion/helpers/types' import { Regions } from '@/modules/multiregion/repositories' -import { executeBatchedSelect } from '@/modules/shared/helpers/dbHelper' import { - CopyProjectModels, - CopyProjects, - CopyProjectVersions, - CopyWorkspace, GetDefaultRegion, UpsertRegionAssignment } from '@/modules/workspaces/domain/operations' -import { WorkspaceNotFoundError } from '@/modules/workspaces/errors/workspace' -import { - Workspace, - WorkspaceRegionAssignment -} from '@/modules/workspacesCore/domain/types' -import { Workspaces } from '@/modules/workspacesCore/helpers/db' +import { WorkspaceRegionAssignment } from '@/modules/workspacesCore/domain/types' import { Knex } from 'knex' export const WorkspaceRegions = buildTableHelper('workspace_regions', [ @@ -41,16 +14,8 @@ export const WorkspaceRegions = buildTableHelper('workspace_regions', [ ]) const tables = { - workspaces: (db: Knex) => db(Workspaces.name), - workspaceRegions: (db: Knex) => db(WorkspaceRegions.name), regions: (db: Knex) => db(Regions.name), - projects: (db: Knex) => db(Streams.name), - models: (db: Knex) => db(Branches.name), - versions: (db: Knex) => db(Commits.name), - branchCommits: (db: Knex) => db(BranchCommits.name), - streamCommits: (db: Knex) => db(StreamCommits.name), - streamFavorites: (db: Knex) => db(StreamFavorites.name), - streamsMeta: (db: Knex) => db(StreamsMeta.name) + workspaceRegions: (db: Knex) => db(WorkspaceRegions.name) } export const upsertRegionAssignmentFactory = @@ -79,180 +44,3 @@ export const getDefaultRegionFactory = return row } - -export const copyWorkspaceFactory = - (deps: { sourceDb: Knex; targetDb: Knex }): CopyWorkspace => - async ({ workspaceId }) => { - const workspace = await tables - .workspaces(deps.sourceDb) - .select('*') - .where({ id: workspaceId }) - - if (!workspace) { - throw new WorkspaceNotFoundError() - } - - await tables.workspaces(deps.targetDb).insert(workspace) - - return workspaceId - } - -export const copyProjectsFactory = - (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjects => - async ({ projectIds }) => { - const selectProjects = tables - .projects(deps.sourceDb) - .select('*') - .whereIn(Streams.col.id, projectIds) - const copiedProjectIds: string[] = [] - - // Copy project record - for await (const projects of executeBatchedSelect(selectProjects)) { - for (const project of projects) { - // Store copied project id - copiedProjectIds.push(project.id) - - // Copy `streams` row to target db - await tables.projects(deps.targetDb).insert(project).onConflict().ignore() - } - - const projectIds = projects.map((project) => project.id) - - // Fetch `stream_favorites` rows for projects in batch - const selectStreamFavorites = tables - .streamFavorites(deps.sourceDb) - .select('*') - .whereIn(StreamFavorites.col.streamId, projectIds) - - for await (const streamFavorites of executeBatchedSelect(selectStreamFavorites)) { - for (const streamFavorite of streamFavorites) { - // Copy `stream_favorites` row to target db - await tables - .streamFavorites(deps.targetDb) - .insert(streamFavorite) - .onConflict() - .ignore() - } - } - - // Fetch `streams_meta` rows for projects in batch - const selectStreamsMetadata = tables - .streamsMeta(deps.sourceDb) - .select('*') - .whereIn(StreamsMeta.col.streamId, projectIds) - - for await (const streamsMetadataBatch of executeBatchedSelect( - selectStreamsMetadata - )) { - for (const streamMetadata of streamsMetadataBatch) { - // Copy `streams_meta` row to target db - await tables - .streamsMeta(deps.targetDb) - .insert(streamMetadata) - .onConflict() - .ignore() - } - } - } - - return copiedProjectIds - } - -export const copyProjectModelsFactory = - (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectModels => - async ({ projectIds }) => { - const copiedModelIds: Record = projectIds.reduce( - (result, id) => ({ ...result, [id]: [] }), - {} - ) - - for (const projectId of projectIds) { - const selectModels = tables - .models(deps.sourceDb) - .select('*') - .where({ streamId: projectId }) - - for await (const models of executeBatchedSelect(selectModels)) { - for (const model of models) { - // Store copied model ids - copiedModelIds[projectId].push(model.id) - - // Copy `branches` row to target db - await tables.models(deps.targetDb).insert(model).onConflict().ignore() - } - } - } - - return copiedModelIds - } - -export const copyProjectVersionsFactory = - (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectVersions => - async ({ projectIds }) => { - const copiedVersionIds: Record = projectIds.reduce( - (result, id) => ({ ...result, [id]: [] }), - {} - ) - - for (const projectId of projectIds) { - const selectVersions = tables - .streamCommits(deps.sourceDb) - .select('*') - .join( - Commits.name, - Commits.col.id, - StreamCommits.col.commitId - ) - .where({ streamId: projectId }) - - for await (const versions of executeBatchedSelect(selectVersions)) { - for (const version of versions) { - const { commitId, streamId, ...commit } = version - - // Store copied version id - copiedVersionIds[streamId].push(commitId) - - // Copy `commits` row to target db - await tables.versions(deps.targetDb).insert(commit).onConflict().ignore() - } - - const commitIds = versions.map((version) => version.commitId) - - // Fetch `branch_commits` rows for versions in batch - const selectBranchCommits = tables - .branchCommits(deps.sourceDb) - .select('*') - .whereIn(BranchCommits.col.commitId, commitIds) - - for await (const branchCommits of executeBatchedSelect(selectBranchCommits)) { - for (const branchCommit of branchCommits) { - // Copy `branch_commits` row to target db - await tables - .branchCommits(deps.targetDb) - .insert(branchCommit) - .onConflict() - .ignore() - } - } - - // Fetch `stream_commits` rows for versions in batch - const selectStreamCommits = tables - .streamCommits(deps.sourceDb) - .select('*') - .whereIn(StreamCommits.col.commitId, commitIds) - - for await (const streamCommits of executeBatchedSelect(selectStreamCommits)) { - for (const streamCommit of streamCommits) { - // Copy `stream_commits` row to target db - await tables - .streamCommits(deps.targetDb) - .insert(streamCommit) - .onConflict() - .ignore() - } - } - } - } - - return copiedVersionIds - } diff --git a/packages/server/modules/workspaces/services/projectRegions.ts b/packages/server/modules/workspaces/services/projectRegions.ts new file mode 100644 index 0000000000..561aee792f --- /dev/null +++ b/packages/server/modules/workspaces/services/projectRegions.ts @@ -0,0 +1,86 @@ +import { GetStreamBranchCount } from '@/modules/core/domain/branches/operations' +import { GetStreamCommitCount } from '@/modules/core/domain/commits/operations' +import { GetProject } from '@/modules/core/domain/projects/operations' +import { + CopyProjectModels, + CopyProjects, + CopyProjectVersions, + CopyWorkspace, + GetAvailableRegions, + UpdateProjectRegion +} from '@/modules/workspaces/domain/operations' +import { ProjectRegionAssignmentError } from '@/modules/workspaces/errors/regions' + +export const updateProjectRegionFactory = + (deps: { + getProject: GetProject + countProjectModels: GetStreamBranchCount + countProjectVersions: GetStreamCommitCount + getAvailableRegions: GetAvailableRegions + copyWorkspace: CopyWorkspace + copyProjects: CopyProjects + copyProjectModels: CopyProjectModels + copyProjectVersions: CopyProjectVersions + }): UpdateProjectRegion => + async (params) => { + const { projectId, regionKey } = params + + const project = await deps.getProject({ projectId }) + if (!project) { + throw new ProjectRegionAssignmentError('Project not found', { + info: { params } + }) + } + if (!project.workspaceId) { + throw new ProjectRegionAssignmentError('Project not a part of a workspace', { + info: { params } + }) + } + + const availableRegions = await deps.getAvailableRegions({ + workspaceId: project.workspaceId + }) + if (!availableRegions.find((region) => region.key === regionKey)) { + throw new ProjectRegionAssignmentError( + 'Specified region not available for workspace', + { + info: { + params, + workspaceId: project.workspaceId + } + } + ) + } + + // Move workspace + await deps.copyWorkspace({ workspaceId: project.workspaceId }) + + // Move commits + const projectIds = await deps.copyProjects({ projectIds: [projectId] }) + const modelIds = await deps.copyProjectModels({ projectIds }) + const versionIds = await deps.copyProjectVersions({ projectIds }) + + // TODO: Move objects + // TODO: Move automations + // TODO: Move comments + // TODO: Move file blobs + // TODO: Move webhooks + + // TODO: Validate state after move captures latest state of project + const sourceProjectModelCount = await deps.countProjectModels(projectId) + const sourceProjectVersionCount = await deps.countProjectVersions(projectId) + + const isReconciled = + modelIds[projectId].length === sourceProjectModelCount && + versionIds[projectId].length === sourceProjectVersionCount + + if (!isReconciled) { + // TODO: Move failed or source project added data while changing regions. Retry move. + throw new ProjectRegionAssignmentError( + 'Missing data from source project in target region copy after move.' + ) + } + + // TODO: Update project region in db + return { ...project, regionKey } + } diff --git a/packages/server/modules/workspaces/services/regions.ts b/packages/server/modules/workspaces/services/regions.ts index 0fc1323930..88623c067a 100644 --- a/packages/server/modules/workspaces/services/regions.ts +++ b/packages/server/modules/workspaces/services/regions.ts @@ -1,25 +1,14 @@ -import { GetStreamBranchCount } from '@/modules/core/domain/branches/operations' -import { GetStreamCommitCount } from '@/modules/core/domain/commits/operations' -import { GetProject } from '@/modules/core/domain/projects/operations' import { WorkspaceFeatureAccessFunction } from '@/modules/gatekeeper/domain/operations' import { GetRegions } from '@/modules/multiregion/domain/operations' import { AssignWorkspaceRegion, - CopyProjectModels, - CopyProjects, - CopyProjectVersions, - CopyWorkspace, GetAvailableRegions, GetDefaultRegion, GetWorkspace, - UpdateProjectRegion, UpsertRegionAssignment, UpsertWorkspace } from '@/modules/workspaces/domain/operations' -import { - ProjectRegionAssignmentError, - WorkspaceRegionAssignmentError -} from '@/modules/workspaces/errors/regions' +import { WorkspaceRegionAssignmentError } from '@/modules/workspaces/errors/regions' export const getAvailableRegionsFactory = (deps: { @@ -80,77 +69,3 @@ export const assignWorkspaceRegionFactory = // Copy workspace into region db await deps.insertRegionWorkspace({ workspace }) } - -export const updateProjectRegionFactory = - (deps: { - getProject: GetProject - countProjectModels: GetStreamBranchCount - countProjectVersions: GetStreamCommitCount - getAvailableRegions: GetAvailableRegions - copyWorkspace: CopyWorkspace - copyProjects: CopyProjects - copyProjectModels: CopyProjectModels - copyProjectVersions: CopyProjectVersions - }): UpdateProjectRegion => - async (params) => { - const { projectId, regionKey } = params - - const project = await deps.getProject({ projectId }) - if (!project) { - throw new ProjectRegionAssignmentError('Project not found', { - info: { params } - }) - } - if (!project.workspaceId) { - throw new ProjectRegionAssignmentError('Project not a part of a workspace', { - info: { params } - }) - } - - const availableRegions = await deps.getAvailableRegions({ - workspaceId: project.workspaceId - }) - if (!availableRegions.find((region) => region.key === regionKey)) { - throw new ProjectRegionAssignmentError( - 'Specified region not available for workspace', - { - info: { - params, - workspaceId: project.workspaceId - } - } - ) - } - - // Move workspace - await deps.copyWorkspace({ workspaceId: project.workspaceId }) - - // Move commits - const projectIds = await deps.copyProjects({ projectIds: [projectId] }) - const modelIds = await deps.copyProjectModels({ projectIds }) - const versionIds = await deps.copyProjectVersions({ projectIds }) - - // TODO: Move objects - // TODO: Move automations - // TODO: Move comments - // TODO: Move file blobs - // TODO: Move webhooks - - // TODO: Validate state after move captures latest state of project - const sourceProjectModelCount = await deps.countProjectModels(projectId) - const sourceProjectVersionCount = await deps.countProjectVersions(projectId) - - const isReconciled = - modelIds[projectId].length === sourceProjectModelCount && - versionIds[projectId].length === sourceProjectVersionCount - - if (!isReconciled) { - // TODO: Move failed or source project added data while changing regions. Retry move. - throw new ProjectRegionAssignmentError( - 'Missing data from source project in target region copy after move.' - ) - } - - // TODO: Update project region in db - return { ...project, regionKey } - } From 6a0fadcc82acbfd7567e3173415f0b25fcf13e8b Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Fri, 24 Jan 2025 15:03:52 +0000 Subject: [PATCH 16/28] fix(multiregion): test drop waits --- packages/server/test/hooks.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/packages/server/test/hooks.ts b/packages/server/test/hooks.ts index 0368148274..99a3cf402c 100644 --- a/packages/server/test/hooks.ts +++ b/packages/server/test/hooks.ts @@ -22,8 +22,7 @@ import { MaybeAsync, MaybeNullOrUndefined, Nullable, - Optional, - wait + Optional } from '@speckle/shared' import * as mocha from 'mocha' import { @@ -199,22 +198,17 @@ export const resetPubSubFactory = (deps: { db: Knex }) => async () => { await deps.db.raw( `SELECT * FROM aiven_extras.pg_alter_subscription_disable('${info.subname}');` ) - await wait(500) await deps.db.raw( `SELECT * FROM aiven_extras.pg_drop_subscription('${info.subname}');` ) - await wait(1000) await deps.db.raw( `SELECT * FROM aiven_extras.dblink_slot_create_or_drop('${info.subconninfo}', '${info.subslotname}', 'drop');` ) } // Drop all subs - for (const subscription of subscriptions.rows) { - // If we do not log something here, CircleCI may kill the job while we wait all `dropSubs` calls to finish. - console.log(`Dropping subscription ${subscription.subname}`) - await dropSubs(subscription) - await wait(500) + for (const sub of subscriptions.rows) { + await dropSubs(sub) } // Drop all pubs From 0811916ef38c4d5e098590f7f5d76148f0819440 Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Mon, 3 Feb 2025 14:03:34 +0000 Subject: [PATCH 17/28] chore(regions): fix import --- packages/server/modules/workspaces/graph/resolvers/regions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/modules/workspaces/graph/resolvers/regions.ts b/packages/server/modules/workspaces/graph/resolvers/regions.ts index 57a45aa214..73770ec14a 100644 --- a/packages/server/modules/workspaces/graph/resolvers/regions.ts +++ b/packages/server/modules/workspaces/graph/resolvers/regions.ts @@ -25,7 +25,7 @@ import { } from '@/modules/workspaces/services/regions' import { updateProjectRegionFactory } from '@/modules/workspaces/services/projectRegions' import { Roles } from '@speckle/shared' -import { getProjectFactory } from '@/modules/core/repositories/streams' +import { getProjectFactory } from '@/modules/core/repositories/projects' import { getStreamBranchCountFactory } from '@/modules/core/repositories/branches' import { getStreamCommitCountFactory } from '@/modules/core/repositories/commits' import { withTransaction } from '@/modules/shared/helpers/dbHelper' From b48721e85ab55d828bf70cc7e73e27d5c4118451 Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Mon, 3 Feb 2025 14:29:32 +0000 Subject: [PATCH 18/28] chore(regions): make test a bit more thorough for good measure --- .../tests/integration/projects.graph.spec.ts | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts index 6d219beb9d..9b33dc0adc 100644 --- a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts @@ -1,7 +1,13 @@ import { db } from '@/db/knex' import { AllScopes } from '@/modules/core/helpers/mainConstants' import { createRandomEmail } from '@/modules/core/helpers/testHelpers' -import { BranchRecord, CommitRecord, StreamRecord } from '@/modules/core/helpers/types' +import { + BranchCommitRecord, + BranchRecord, + CommitRecord, + StreamCommitRecord, + StreamRecord +} from '@/modules/core/helpers/types' import { grantStreamPermissionsFactory } from '@/modules/core/repositories/streams' import { getDb } from '@/modules/multiregion/utils/dbSelector' import { @@ -415,8 +421,21 @@ isMultiRegionTestMode() .select('*') .where({ id: testVersion.id }) .first() - expect(version).to.not.be.undefined + + const streamCommitsRecord = await targetRegionDb + .table('stream_commits') + .select('*') + .where({ commitId: testVersion.id }) + .first() + expect(streamCommitsRecord).to.not.be.undefined + + const branchCommitsRecord = await targetRegionDb + .table('branch_commits') + .select('*') + .where({ commitId: testVersion.id }) + .first() + expect(branchCommitsRecord).to.not.be.undefined }) }) : void 0 From 2b8f6af4cd991409e5989a5319072202dcea51d6 Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Mon, 3 Feb 2025 15:31:45 +0000 Subject: [PATCH 19/28] fix(regions): move project objects --- .../modules/core/domain/objects/operations.ts | 2 + .../modules/core/repositories/objects.ts | 12 ++ .../modules/workspaces/domain/operations.ts | 3 + .../workspaces/graph/resolvers/regions.ts | 6 +- .../workspaces/repositories/projectRegions.ts | 125 ++++++++++++++++-- .../workspaces/services/projectRegions.ts | 21 ++- 6 files changed, 149 insertions(+), 20 deletions(-) diff --git a/packages/server/modules/core/domain/objects/operations.ts b/packages/server/modules/core/domain/objects/operations.ts index 7e9de7115e..cb9d10747e 100644 --- a/packages/server/modules/core/domain/objects/operations.ts +++ b/packages/server/modules/core/domain/objects/operations.ts @@ -15,6 +15,8 @@ export type GetStreamObjects = ( objectIds: string[] ) => Promise +export type GetStreamObjectCount = (params: { streamId: string }) => Promise + export type GetObject = ( objectId: string, streamId: string diff --git a/packages/server/modules/core/repositories/objects.ts b/packages/server/modules/core/repositories/objects.ts index 120a91e725..3f8ffab446 100644 --- a/packages/server/modules/core/repositories/objects.ts +++ b/packages/server/modules/core/repositories/objects.ts @@ -14,6 +14,7 @@ import { GetObjectChildrenQuery, GetObjectChildrenStream, GetObjectsStream, + GetStreamObjectCount, GetStreamObjects, HasObjects, StoreClosuresIfNotFound, @@ -92,6 +93,17 @@ export const getBatchedStreamObjectsFactory = return executeBatchedSelect(baseQuery, options) } +export const getStreamObjectCountFactory = + (deps: { db: Knex }): GetStreamObjectCount => + async ({ streamId }) => { + const [res] = await tables + .objects(deps.db) + .where(Objects.col.streamId, streamId) + .count() + + return parseInt(res.count as string) + } + export const insertObjectsFactory = (deps: { db: Knex }): StoreObjects => async (objects: ObjectRecord[], options?: Partial<{ trx: Knex.Transaction }>) => { diff --git a/packages/server/modules/workspaces/domain/operations.ts b/packages/server/modules/workspaces/domain/operations.ts index 83ee87e856..d88acb9784 100644 --- a/packages/server/modules/workspaces/domain/operations.ts +++ b/packages/server/modules/workspaces/domain/operations.ts @@ -363,3 +363,6 @@ export type CopyProjectModels = (params: { export type CopyProjectVersions = (params: { projectIds: string[] }) => Promise> +export type CopyProjectObjects = (params: { + projectIds: string[] +}) => Promise> diff --git a/packages/server/modules/workspaces/graph/resolvers/regions.ts b/packages/server/modules/workspaces/graph/resolvers/regions.ts index 73770ec14a..92b685f6a3 100644 --- a/packages/server/modules/workspaces/graph/resolvers/regions.ts +++ b/packages/server/modules/workspaces/graph/resolvers/regions.ts @@ -11,6 +11,7 @@ import { } from '@/modules/workspaces/repositories/regions' import { copyProjectModelsFactory, + copyProjectObjectsFactory, copyProjectsFactory, copyProjectVersionsFactory, copyWorkspaceFactory @@ -29,6 +30,7 @@ import { getProjectFactory } from '@/modules/core/repositories/projects' import { getStreamBranchCountFactory } from '@/modules/core/repositories/branches' import { getStreamCommitCountFactory } from '@/modules/core/repositories/commits' import { withTransaction } from '@/modules/shared/helpers/dbHelper' +import { getStreamObjectCountFactory } from '@/modules/core/repositories/objects' export default { Workspace: { @@ -81,6 +83,7 @@ export default { getProject: getProjectFactory({ db: sourceDb }), countProjectModels: getStreamBranchCountFactory({ db: sourceDb }), countProjectVersions: getStreamCommitCountFactory({ db: sourceDb }), + countProjectObjects: getStreamObjectCountFactory({ db: sourceDb }), getAvailableRegions: getAvailableRegionsFactory({ getRegions: getRegionsFactory({ db }), canWorkspaceUseRegions: canWorkspaceUseRegionsFactory({ @@ -90,7 +93,8 @@ export default { copyWorkspace: copyWorkspaceFactory({ sourceDb, targetDb }), copyProjects: copyProjectsFactory({ sourceDb, targetDb }), copyProjectModels: copyProjectModelsFactory({ sourceDb, targetDb }), - copyProjectVersions: copyProjectVersionsFactory({ sourceDb, targetDb }) + copyProjectVersions: copyProjectVersionsFactory({ sourceDb, targetDb }), + copyProjectObjects: copyProjectObjectsFactory({ sourceDb, targetDb }) }) return await withTransaction(updateProjectRegion(args), targetDb) diff --git a/packages/server/modules/workspaces/repositories/projectRegions.ts b/packages/server/modules/workspaces/repositories/projectRegions.ts index 94da98d14d..b18c5aac7c 100644 --- a/packages/server/modules/workspaces/repositories/projectRegions.ts +++ b/packages/server/modules/workspaces/repositories/projectRegions.ts @@ -2,6 +2,7 @@ import { BranchCommits, Branches, Commits, + Objects, StreamCommits, StreamFavorites, Streams, @@ -12,12 +13,15 @@ import { Commit } from '@/modules/core/domain/commits/types' import { Stream } from '@/modules/core/domain/streams/types' import { BranchCommitRecord, + ObjectChildrenClosureRecord, + ObjectRecord, StreamCommitRecord, StreamFavoriteRecord } from '@/modules/core/helpers/types' import { executeBatchedSelect } from '@/modules/shared/helpers/dbHelper' import { CopyProjectModels, + CopyProjectObjects, CopyProjects, CopyProjectVersions, CopyWorkspace @@ -26,6 +30,7 @@ import { WorkspaceNotFoundError } from '@/modules/workspaces/errors/workspace' import { Knex } from 'knex' import { Workspace } from '@/modules/workspacesCore/domain/types' import { Workspaces } from '@/modules/workspacesCore/helpers/db' +import { ObjectPreview } from '@/modules/previews/domain/types' const tables = { workspaces: (db: Knex) => db(Workspaces.name), @@ -35,9 +40,17 @@ const tables = { branchCommits: (db: Knex) => db(BranchCommits.name), streamCommits: (db: Knex) => db(StreamCommits.name), streamFavorites: (db: Knex) => db(StreamFavorites.name), - streamsMeta: (db: Knex) => db(StreamsMeta.name) + streamsMeta: (db: Knex) => db(StreamsMeta.name), + objects: (db: Knex) => db(Objects.name), + objectClosures: (db: Knex) => + db('object_children_closure'), + objectPreviews: (db: Knex) => db('object_preview') } +/** + * Copies rows from the following tables: + * - workspaces + */ export const copyWorkspaceFactory = (deps: { sourceDb: Knex; targetDb: Knex }): CopyWorkspace => async ({ workspaceId }) => { @@ -55,6 +68,12 @@ export const copyWorkspaceFactory = return workspaceId } +/** + * Copies rows from the following tables: + * - streams + * - streams_meta + * - stream_favorites + */ export const copyProjectsFactory = (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjects => async ({ projectIds }) => { @@ -116,15 +135,18 @@ export const copyProjectsFactory = return copiedProjectIds } +/** + * Copies rows from the following tables: + * - branches + */ export const copyProjectModelsFactory = (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectModels => async ({ projectIds }) => { - const copiedModelIds: Record = projectIds.reduce( - (result, id) => ({ ...result, [id]: [] }), - {} - ) + const copiedModelIds: Record = {} for (const projectId of projectIds) { + copiedModelIds[projectId] = [] + const selectModels = tables .models(deps.sourceDb) .select('*') @@ -144,15 +166,21 @@ export const copyProjectModelsFactory = return copiedModelIds } +/** + * Copies rows from the following tables: + * - commits + * - branch_commits + * - stream_commits + */ export const copyProjectVersionsFactory = (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectVersions => async ({ projectIds }) => { - const copiedVersionIds: Record = projectIds.reduce( - (result, id) => ({ ...result, [id]: [] }), - {} - ) + const copiedVersionIds: Record = {} for (const projectId of projectIds) { + copiedVersionIds[projectId] = [] + + // Copy `commits` table rows in batches const selectVersions = tables .streamCommits(deps.sourceDb) .select('*') @@ -170,13 +198,13 @@ export const copyProjectVersionsFactory = // Store copied version id copiedVersionIds[streamId].push(commitId) - // Copy `commits` row to target db + // Write `commits` row to target db await tables.versions(deps.targetDb).insert(commit).onConflict().ignore() } const commitIds = versions.map((version) => version.commitId) - // Fetch `branch_commits` rows for versions in batch + // Copy `branch_commits` table rows for current batch of versions const selectBranchCommits = tables .branchCommits(deps.sourceDb) .select('*') @@ -184,7 +212,7 @@ export const copyProjectVersionsFactory = for await (const branchCommits of executeBatchedSelect(selectBranchCommits)) { for (const branchCommit of branchCommits) { - // Copy `branch_commits` row to target db + // Write `branch_commits` row to target db await tables .branchCommits(deps.targetDb) .insert(branchCommit) @@ -193,7 +221,7 @@ export const copyProjectVersionsFactory = } } - // Fetch `stream_commits` rows for versions in batch + // Copy `stream_commits` table rows for current batch of versions const selectStreamCommits = tables .streamCommits(deps.sourceDb) .select('*') @@ -201,7 +229,7 @@ export const copyProjectVersionsFactory = for await (const streamCommits of executeBatchedSelect(selectStreamCommits)) { for (const streamCommit of streamCommits) { - // Copy `stream_commits` row to target db + // Write `stream_commits` row to target db await tables .streamCommits(deps.targetDb) .insert(streamCommit) @@ -214,3 +242,72 @@ export const copyProjectVersionsFactory = return copiedVersionIds } + +/** + * Copies rows from the following tables: + * - objects + * - object_children_closure + * - object_preview + */ +export const copyProjectObjectsFactory = + (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectObjects => + async ({ projectIds }) => { + const copiedObjectIds: Record = {} + + for (const projectId of projectIds) { + copiedObjectIds[projectId] = [] + + // Copy `objects` table rows in batches + const selectObjects = tables + .objects(deps.sourceDb) + .select('*') + .where(Objects.col.streamId, projectId) + .orderBy(Objects.col.id) + + for await (const objects of executeBatchedSelect(selectObjects)) { + for (const object of objects) { + // Store copied object ids by source project + copiedObjectIds[projectId].push(object.id) + + // Write `objects` table row to target db + await tables.objects(deps.targetDb).insert(object).onConflict().ignore() + } + } + + // Copy `object_children_closure` rows in batches + const selectObjectClosures = tables + .objectClosures(deps.sourceDb) + .select('*') + .where('streamId', projectId) + + for await (const closures of executeBatchedSelect(selectObjectClosures)) { + for (const closure of closures) { + // Write `object_children_closure` row to target db + await tables + .objectClosures(deps.targetDb) + .insert(closure) + .onConflict() + .ignore() + } + } + + // Copy `object_preview` rows in batches + const selectObjectPreviews = tables + .objectPreviews(deps.sourceDb) + .select('*') + .where('streamId', projectId) + + for await (const previews of executeBatchedSelect(selectObjectPreviews)) { + for (const preview of previews) { + // Write `object_preview` row to target db + await tables + .objectPreviews(deps.targetDb) + .insert(preview) + .onConflict() + .ignore() + } + } + } + + return copiedObjectIds + } diff --git a/packages/server/modules/workspaces/services/projectRegions.ts b/packages/server/modules/workspaces/services/projectRegions.ts index 561aee792f..f09328e932 100644 --- a/packages/server/modules/workspaces/services/projectRegions.ts +++ b/packages/server/modules/workspaces/services/projectRegions.ts @@ -1,8 +1,10 @@ import { GetStreamBranchCount } from '@/modules/core/domain/branches/operations' import { GetStreamCommitCount } from '@/modules/core/domain/commits/operations' +import { GetStreamObjectCount } from '@/modules/core/domain/objects/operations' import { GetProject } from '@/modules/core/domain/projects/operations' import { CopyProjectModels, + CopyProjectObjects, CopyProjects, CopyProjectVersions, CopyWorkspace, @@ -16,11 +18,13 @@ export const updateProjectRegionFactory = getProject: GetProject countProjectModels: GetStreamBranchCount countProjectVersions: GetStreamCommitCount + countProjectObjects: GetStreamObjectCount getAvailableRegions: GetAvailableRegions copyWorkspace: CopyWorkspace copyProjects: CopyProjects copyProjectModels: CopyProjectModels copyProjectVersions: CopyProjectVersions + copyProjectObjects: CopyProjectObjects }): UpdateProjectRegion => async (params) => { const { projectId, regionKey } = params @@ -60,7 +64,9 @@ export const updateProjectRegionFactory = const modelIds = await deps.copyProjectModels({ projectIds }) const versionIds = await deps.copyProjectVersions({ projectIds }) - // TODO: Move objects + // Move objects + const objectIds = await deps.copyProjectObjects({ projectIds }) + // TODO: Move automations // TODO: Move comments // TODO: Move file blobs @@ -69,12 +75,17 @@ export const updateProjectRegionFactory = // TODO: Validate state after move captures latest state of project const sourceProjectModelCount = await deps.countProjectModels(projectId) const sourceProjectVersionCount = await deps.countProjectVersions(projectId) + const sourceProjectObjectCount = await deps.countProjectObjects({ + streamId: projectId + }) - const isReconciled = - modelIds[projectId].length === sourceProjectModelCount && - versionIds[projectId].length === sourceProjectVersionCount + const tests = [ + modelIds[projectId].length === sourceProjectModelCount, + versionIds[projectId].length === sourceProjectVersionCount, + objectIds[projectId].length === sourceProjectObjectCount + ] - if (!isReconciled) { + if (!tests.every((test) => !!test)) { // TODO: Move failed or source project added data while changing regions. Retry move. throw new ProjectRegionAssignmentError( 'Missing data from source project in target region copy after move.' From 62466f5cb968ac1f3652699c0461612f5252a44f Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Mon, 3 Feb 2025 15:52:10 +0000 Subject: [PATCH 20/28] chore(regions): add tests for object move --- .../core/tests/integration/objects.spec.ts | 79 +++++++++++++++++++ .../tests/integration/projects.graph.spec.ts | 43 ++++++++-- 2 files changed, 114 insertions(+), 8 deletions(-) create mode 100644 packages/server/modules/core/tests/integration/objects.spec.ts diff --git a/packages/server/modules/core/tests/integration/objects.spec.ts b/packages/server/modules/core/tests/integration/objects.spec.ts new file mode 100644 index 0000000000..a2436f38a6 --- /dev/null +++ b/packages/server/modules/core/tests/integration/objects.spec.ts @@ -0,0 +1,79 @@ +import { createRandomEmail } from '@/modules/core/helpers/testHelpers' +import { getStreamObjectCountFactory } from '@/modules/core/repositories/objects' +import { BasicTestUser, createTestUser } from '@/test/authHelper' +import { BasicTestBranch, createTestBranch } from '@/test/speckle-helpers/branchHelper' +import { + BasicTestCommit, + createTestCommit, + createTestObject +} from '@/test/speckle-helpers/commitHelper' +import { BasicTestStream, createTestStream } from '@/test/speckle-helpers/streamHelper' +import cryptoRandomString from 'crypto-random-string' +import { db } from '@/db/knex' +import { expect } from 'chai' + +const getStreamObjectCount = getStreamObjectCountFactory({ db }) + +describe('Object repository functions', () => { + const adminUser: BasicTestUser = { + id: '', + name: 'John Speckle', + email: createRandomEmail() + } + + const testProject: BasicTestStream = { + id: '', + ownerId: '', + name: 'Test Project', + isPublic: true + } + + const testModel: BasicTestBranch = { + id: '', + name: cryptoRandomString({ length: 8 }), + streamId: '', + authorId: '' + } + + const testVersion: BasicTestCommit = { + id: '', + objectId: '', + streamId: '', + authorId: '' + } + + before(async () => { + await createTestUser(adminUser) + }) + + beforeEach(async () => { + await createTestStream(testProject, adminUser) + await createTestBranch({ + stream: testProject, + branch: testModel, + owner: adminUser + }) + + testVersion.branchName = testModel.name + testVersion.objectId = await createTestObject({ projectId: testProject.id }) + + await createTestCommit(testVersion, { + owner: adminUser, + stream: testProject + }) + }) + + describe('getStreamObjectCountFactory creates a function, that', () => { + it('correctly counts the number of objects in a project', async () => { + const count = await getStreamObjectCount({ streamId: testProject.id }) + expect(count).to.equal(1) + }) + + it('returns 0 if the project does not exist', async () => { + const count = await getStreamObjectCount({ + streamId: cryptoRandomString({ length: 9 }) + }) + expect(count).to.equal(0) + }) + }) +}) diff --git a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts index 9b33dc0adc..7813b8fc23 100644 --- a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts @@ -5,6 +5,7 @@ import { BranchCommitRecord, BranchRecord, CommitRecord, + ObjectRecord, StreamCommitRecord, StreamRecord } from '@/modules/core/helpers/types' @@ -51,6 +52,15 @@ import cryptoRandomString from 'crypto-random-string' import { Knex } from 'knex' import { SetOptional } from 'type-fest' +const tables = { + projects: (db: Knex) => db.table('streams'), + models: (db: Knex) => db.table('branches'), + versions: (db: Knex) => db.table('commits'), + streamCommits: (db: Knex) => db.table('stream_commits'), + branchCommits: (db: Knex) => db.table('branch_commits'), + objects: (db: Knex) => db.table('objects') +} + const grantStreamPermissions = grantStreamPermissionsFactory({ db }) describe('Workspace project GQL CRUD', () => { @@ -382,8 +392,8 @@ isMultiRegionTestMode() expect(res).to.not.haveGraphQLErrors() - const project = await targetRegionDb - .table('streams') + const project = await tables + .projects(targetRegionDb) .select('*') .where({ id: testProject.id }) .first() @@ -416,26 +426,43 @@ isMultiRegionTestMode() expect(res).to.not.haveGraphQLErrors() - const version = await targetRegionDb - .table('commits') + const version = await tables + .versions(targetRegionDb) .select('*') .where({ id: testVersion.id }) .first() expect(version).to.not.be.undefined - const streamCommitsRecord = await targetRegionDb - .table('stream_commits') + const streamCommitsRecord = await tables + .streamCommits(targetRegionDb) .select('*') .where({ commitId: testVersion.id }) .first() expect(streamCommitsRecord).to.not.be.undefined - const branchCommitsRecord = await targetRegionDb - .table('branch_commits') + const branchCommitsRecord = await tables + .branchCommits(targetRegionDb) .select('*') .where({ commitId: testVersion.id }) .first() expect(branchCommitsRecord).to.not.be.undefined }) + + it('moves project version objects to target regional db', async () => { + const res = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + + expect(res).to.not.haveGraphQLErrors() + + const object = await tables + .objects(targetRegionDb) + .select('*') + .where({ id: testVersion.objectId }) + .first() + + expect(object).to.not.be.undefined + }) }) : void 0 From 51ba709e66669cf4023ba44b7e5b30211a7b02d0 Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Mon, 3 Feb 2025 23:55:11 +0000 Subject: [PATCH 21/28] feat(regions): move project automations --- .../modules/automate/domain/operations.ts | 8 +- .../modules/automate/helpers/graphTypes.ts | 4 +- .../server/modules/automate/helpers/types.ts | 4 +- .../automate/repositories/automations.ts | 19 +- .../modules/automate/services/encryption.ts | 4 +- .../modules/core/graph/dataloaders/index.ts | 4 +- .../modules/workspaces/domain/operations.ts | 3 + .../workspaces/graph/resolvers/regions.ts | 8 +- .../workspaces/repositories/projectRegions.ts | 194 +++++++++++++++++- .../workspaces/services/projectRegions.ts | 14 +- 10 files changed, 240 insertions(+), 22 deletions(-) diff --git a/packages/server/modules/automate/domain/operations.ts b/packages/server/modules/automate/domain/operations.ts index b9fe28a2b1..4e795d1957 100644 --- a/packages/server/modules/automate/domain/operations.ts +++ b/packages/server/modules/automate/domain/operations.ts @@ -1,6 +1,6 @@ import { InsertableAutomationFunctionRun } from '@/modules/automate/domain/types' import { - AutomateRevisionFunctionRecord, + AutomationRevisionFunctionRecord, AutomationFunctionRunRecord, AutomationRecord, AutomationRevisionRecord, @@ -175,7 +175,7 @@ export type GetRevisionsTriggerDefinitions = (params: { export type GetRevisionsFunctions = (params: { automationRevisionIds: string[] -}) => Promise<{ [automationRevisionId: string]: AutomateRevisionFunctionRecord[] }> +}) => Promise<{ [automationRevisionId: string]: AutomationRevisionFunctionRecord[] }> export type CreateStoredAuthCode = ( params: Omit @@ -204,3 +204,7 @@ export type TriggerAutomationRevisionRun = < manifest: M source?: RunTriggerSource }) => Promise<{ automationRunId: string }> + +export type GetProjectAutomationCount = (params: { + projectId: string +}) => Promise diff --git a/packages/server/modules/automate/helpers/graphTypes.ts b/packages/server/modules/automate/helpers/graphTypes.ts index 40be15930a..366a99f5c1 100644 --- a/packages/server/modules/automate/helpers/graphTypes.ts +++ b/packages/server/modules/automate/helpers/graphTypes.ts @@ -1,5 +1,5 @@ import { - AutomateRevisionFunctionRecord, + AutomationRevisionFunctionRecord, AutomationFunctionRunRecord, AutomationRecord, AutomationRevisionRecord, @@ -56,7 +56,7 @@ export type AutomationRunTriggerGraphQLReturn = AutomationRunTriggerRecord & { } export type AutomationRevisionFunctionGraphQLReturn = Merge< - AutomateRevisionFunctionRecord, + AutomationRevisionFunctionRecord, { functionInputs: Nullable> release: AutomateFunctionReleaseGraphQLReturn diff --git a/packages/server/modules/automate/helpers/types.ts b/packages/server/modules/automate/helpers/types.ts index 3c8b82d121..7881385d6e 100644 --- a/packages/server/modules/automate/helpers/types.ts +++ b/packages/server/modules/automate/helpers/types.ts @@ -66,7 +66,7 @@ export type AutomationRunRecord = { executionEngineRunId: string | null } -export type AutomateRevisionFunctionRecord = { +export type AutomationRevisionFunctionRecord = { functionReleaseId: string functionId: string functionInputs: string | null @@ -118,7 +118,7 @@ export type AutomationTokenRecord = { } export type AutomationRevisionWithTriggersFunctions = AutomationRevisionRecord & { - functions: AutomateRevisionFunctionRecord[] + functions: AutomationRevisionFunctionRecord[] triggers: AutomationTriggerDefinitionRecord[] } diff --git a/packages/server/modules/automate/repositories/automations.ts b/packages/server/modules/automate/repositories/automations.ts index c424b3a8ea..c07a21b40a 100644 --- a/packages/server/modules/automate/repositories/automations.ts +++ b/packages/server/modules/automate/repositories/automations.ts @@ -16,6 +16,7 @@ import { GetLatestAutomationRevision, GetLatestAutomationRevisions, GetLatestVersionAutomationRuns, + GetProjectAutomationCount, GetRevisionsFunctions, GetRevisionsTriggerDefinitions, StoreAutomation, @@ -37,7 +38,7 @@ import { AutomationRunRecord, AutomationTokenRecord, AutomationTriggerRecordBase, - AutomateRevisionFunctionRecord, + AutomationRevisionFunctionRecord, AutomationRunWithTriggersFunctionRuns, AutomationRunTriggerRecord, AutomationFunctionRunRecord, @@ -86,7 +87,7 @@ const tables = { automationRevisions: (db: Knex) => db(AutomationRevisions.name), automationRevisionFunctions: (db: Knex) => - db(AutomationRevisionFunctions.name), + db(AutomationRevisionFunctions.name), automationTriggers: (db: Knex) => db(AutomationTriggers.name), automationRuns: (db: Knex) => db(AutomationRuns.name), @@ -321,7 +322,7 @@ export const storeAutomationTokenFactory = } export type InsertableAutomationRevisionFunction = Omit< - AutomateRevisionFunctionRecord, + AutomationRevisionFunctionRecord, 'automationRevisionId' > @@ -369,7 +370,7 @@ export const storeAutomationRevisionFactory = .automationRevisionFunctions(deps.db) .insert( revision.functions.map( - (f): AutomateRevisionFunctionRecord => ({ + (f): AutomationRevisionFunctionRecord => ({ ...f, automationRevisionId: id }) @@ -742,10 +743,12 @@ const getProjectAutomationsBaseQueryFactory = } export const getProjectAutomationsTotalCountFactory = - (deps: { db: Knex }) => async (params: GetProjectAutomationsParams) => { - const q = getProjectAutomationsBaseQueryFactory(deps)(params).count< - [{ count: string }] - >(Automations.col.id) + (deps: { db: Knex }): GetProjectAutomationCount => + async ({ projectId }) => { + const q = getProjectAutomationsBaseQueryFactory(deps)({ + projectId, + args: {} + }).count<[{ count: string }]>(Automations.col.id) const [ret] = await q diff --git a/packages/server/modules/automate/services/encryption.ts b/packages/server/modules/automate/services/encryption.ts index 3b8228c8bf..8cfc98d66c 100644 --- a/packages/server/modules/automate/services/encryption.ts +++ b/packages/server/modules/automate/services/encryption.ts @@ -7,7 +7,7 @@ import { Nullable, Optional } from '@speckle/shared' import { MisconfiguredEnvironmentError } from '@/modules/shared/errors' import { AutomationFunctionInputEncryptionError } from '@/modules/automate/errors/management' import { KeyPair, buildDecryptor } from '@/modules/shared/utils/libsodium' -import { AutomateRevisionFunctionRecord } from '@/modules/automate/helpers/types' +import { AutomationRevisionFunctionRecord } from '@/modules/automate/helpers/types' import { AutomationRevisionFunctionGraphQLReturn } from '@/modules/automate/helpers/graphTypes' import { FunctionReleaseSchemaType } from '@/modules/automate/helpers/executionEngine' import { LibsodiumEncryptionError } from '@/modules/shared/errors/encryption' @@ -118,7 +118,7 @@ export type GetFunctionInputsForFrontendDeps = { } & GetFunctionInputDecryptorDeps export type AutomationRevisionFunctionForInputRedaction = Merge< - AutomateRevisionFunctionRecord, + AutomationRevisionFunctionRecord, { release: FunctionReleaseSchemaType } > diff --git a/packages/server/modules/core/graph/dataloaders/index.ts b/packages/server/modules/core/graph/dataloaders/index.ts index 958019035d..f6ed23fd11 100644 --- a/packages/server/modules/core/graph/dataloaders/index.ts +++ b/packages/server/modules/core/graph/dataloaders/index.ts @@ -56,7 +56,7 @@ import { Users } from '@/modules/core/dbSchema' import { getStreamPendingModelsFactory } from '@/modules/fileuploads/repositories/fileUploads' import { FileUploadRecord } from '@/modules/fileuploads/helpers/types' import { - AutomateRevisionFunctionRecord, + AutomationRevisionFunctionRecord, AutomationRecord, AutomationRevisionRecord, AutomationRunTriggerRecord, @@ -595,7 +595,7 @@ const dataLoadersDefinition = defineRequestDataloaders( }) return ids.map((i) => results[i] || []) }), - getRevisionFunctions: createLoader( + getRevisionFunctions: createLoader( async (ids) => { const results = await getRevisionsFunctions({ automationRevisionIds: ids.slice() diff --git a/packages/server/modules/workspaces/domain/operations.ts b/packages/server/modules/workspaces/domain/operations.ts index d88acb9784..51dbdf23bc 100644 --- a/packages/server/modules/workspaces/domain/operations.ts +++ b/packages/server/modules/workspaces/domain/operations.ts @@ -366,3 +366,6 @@ export type CopyProjectVersions = (params: { export type CopyProjectObjects = (params: { projectIds: string[] }) => Promise> +export type CopyProjectAutomations = (params: { + projectIds: string[] +}) => Promise> diff --git a/packages/server/modules/workspaces/graph/resolvers/regions.ts b/packages/server/modules/workspaces/graph/resolvers/regions.ts index 92b685f6a3..f43fe0ab32 100644 --- a/packages/server/modules/workspaces/graph/resolvers/regions.ts +++ b/packages/server/modules/workspaces/graph/resolvers/regions.ts @@ -10,6 +10,7 @@ import { upsertRegionAssignmentFactory } from '@/modules/workspaces/repositories/regions' import { + copyProjectAutomationsFactory, copyProjectModelsFactory, copyProjectObjectsFactory, copyProjectsFactory, @@ -31,6 +32,7 @@ import { getStreamBranchCountFactory } from '@/modules/core/repositories/branche import { getStreamCommitCountFactory } from '@/modules/core/repositories/commits' import { withTransaction } from '@/modules/shared/helpers/dbHelper' import { getStreamObjectCountFactory } from '@/modules/core/repositories/objects' +import { getProjectAutomationsTotalCountFactory } from '@/modules/automate/repositories/automations' export default { Workspace: { @@ -84,6 +86,9 @@ export default { countProjectModels: getStreamBranchCountFactory({ db: sourceDb }), countProjectVersions: getStreamCommitCountFactory({ db: sourceDb }), countProjectObjects: getStreamObjectCountFactory({ db: sourceDb }), + countProjectAutomations: getProjectAutomationsTotalCountFactory({ + db: sourceDb + }), getAvailableRegions: getAvailableRegionsFactory({ getRegions: getRegionsFactory({ db }), canWorkspaceUseRegions: canWorkspaceUseRegionsFactory({ @@ -94,7 +99,8 @@ export default { copyProjects: copyProjectsFactory({ sourceDb, targetDb }), copyProjectModels: copyProjectModelsFactory({ sourceDb, targetDb }), copyProjectVersions: copyProjectVersionsFactory({ sourceDb, targetDb }), - copyProjectObjects: copyProjectObjectsFactory({ sourceDb, targetDb }) + copyProjectObjects: copyProjectObjectsFactory({ sourceDb, targetDb }), + copyProjectAutomations: copyProjectAutomationsFactory({ sourceDb, targetDb }) }) return await withTransaction(updateProjectRegion(args), targetDb) diff --git a/packages/server/modules/workspaces/repositories/projectRegions.ts b/packages/server/modules/workspaces/repositories/projectRegions.ts index b18c5aac7c..4c7926b3ef 100644 --- a/packages/server/modules/workspaces/repositories/projectRegions.ts +++ b/packages/server/modules/workspaces/repositories/projectRegions.ts @@ -1,4 +1,12 @@ import { + AutomationFunctionRuns, + AutomationRevisionFunctions, + AutomationRevisions, + AutomationRuns, + AutomationRunTriggers, + Automations, + AutomationTokens, + AutomationTriggers, BranchCommits, Branches, Commits, @@ -20,6 +28,7 @@ import { } from '@/modules/core/helpers/types' import { executeBatchedSelect } from '@/modules/shared/helpers/dbHelper' import { + CopyProjectAutomations, CopyProjectModels, CopyProjectObjects, CopyProjects, @@ -31,6 +40,16 @@ import { Knex } from 'knex' import { Workspace } from '@/modules/workspacesCore/domain/types' import { Workspaces } from '@/modules/workspacesCore/helpers/db' import { ObjectPreview } from '@/modules/previews/domain/types' +import { + AutomationFunctionRunRecord, + AutomationRecord, + AutomationRevisionFunctionRecord, + AutomationRevisionRecord, + AutomationRunRecord, + AutomationRunTriggerRecord, + AutomationTokenRecord, + AutomationTriggerDefinitionRecord +} from '@/modules/automate/helpers/types' const tables = { workspaces: (db: Knex) => db(Workspaces.name), @@ -44,7 +63,20 @@ const tables = { objects: (db: Knex) => db(Objects.name), objectClosures: (db: Knex) => db('object_children_closure'), - objectPreviews: (db: Knex) => db('object_preview') + objectPreviews: (db: Knex) => db('object_preview'), + automations: (db: Knex) => db(Automations.name), + automationTokens: (db: Knex) => db(AutomationTokens.name), + automationRevisions: (db: Knex) => + db(AutomationRevisions.name), + automationTriggers: (db: Knex) => + db(AutomationTriggers.name), + automationRevisionFunctions: (db: Knex) => + db(AutomationRevisionFunctions.name), + automationRuns: (db: Knex) => db(AutomationRuns.name), + automationRunTriggers: (db: Knex) => + db(AutomationRunTriggers.name), + automationFunctionRuns: (db: Knex) => + db(AutomationFunctionRuns.name) } /** @@ -311,3 +343,163 @@ export const copyProjectObjectsFactory = return copiedObjectIds } + +/** + * Copies rows from the following tables: + * - automations + * - automation_tokens + * - automation_revisions + * - automation_triggers + * - automation_revision_functions + * - automation_runs + * - automation_run_triggers + * - automation_function_runs + */ +export const copyProjectAutomationsFactory = + (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectAutomations => + async ({ projectIds }) => { + const copiedAutomationIds: Record = {} + + for (const projectId of projectIds) { + copiedAutomationIds[projectId] = [] + + // Copy `automations` table rows in batches + const selectAutomations = tables + .automations(deps.sourceDb) + .select('*') + .where(Automations.col.projectId, projectId) + + for await (const automations of executeBatchedSelect(selectAutomations)) { + for (const automation of automations) { + // Store copied automation id + copiedAutomationIds[projectId].push(automation.id) + + // Write `automations` table row to target db + await tables + .automations(deps.targetDb) + .insert(automation) + .onConflict() + .ignore() + + // Copy `automation_tokens` rows for automation + const selectAutomationTokens = tables + .automationTokens(deps.sourceDb) + .select('*') + .where(AutomationTokens.col.automationId, automation.id) + + for await (const tokens of executeBatchedSelect(selectAutomationTokens)) { + for (const token of tokens) { + // Write `automation_tokens` row to target db + await tables + .automationTokens(deps.targetDb) + .insert(token) + .onConflict() + .ignore() + } + } + + // Copy `automation_revisions` rows for automation + const selectAutomationRevisions = tables + .automationRevisions(deps.sourceDb) + .select('*') + .where(AutomationRevisions.col.automationId, automation.id) + + for await (const automationRevisions of executeBatchedSelect( + selectAutomationRevisions + )) { + for (const automationRevision of automationRevisions) { + // Write `automation_revisions` row to target db + await tables + .automationRevisions(deps.targetDb) + .insert(automationRevision) + .onConflict() + .ignore() + + // Copy `automation_triggers` rows for automation revision + const automationTriggers = await tables + .automationTriggers(deps.sourceDb) + .select('*') + .where( + AutomationTriggers.col.automationRevisionId, + automationRevision.id + ) + + for (const automationTrigger of automationTriggers) { + await tables + .automationTriggers(deps.targetDb) + .insert(automationTrigger) + .onConflict() + .ignore() + } + + // Copy `automation_revision_functions` rows for automation revision + const automationRevisionFunctions = await tables + .automationRevisionFunctions(deps.sourceDb) + .select('*') + .where( + AutomationRevisionFunctions.col.automationRevisionId, + automationRevision.id + ) + + for (const automationRevisionFunction of automationRevisionFunctions) { + await tables + .automationRevisionFunctions(deps.targetDb) + .insert(automationRevisionFunction) + .onConflict() + .ignore() + } + + // Copy `automation_runs` rows for automation revision + const selectAutomationRuns = tables + .automationRuns(deps.sourceDb) + .select('*') + .where(AutomationRuns.col.automationRevisionId, automationRevision.id) + + for await (const automationRuns of executeBatchedSelect( + selectAutomationRuns + )) { + for (const automationRun of automationRuns) { + // Write `automation_runs` row to target db + await tables + .automationRuns(deps.targetDb) + .insert(automationRun) + .onConflict() + .ignore() + + // Copy `automation_run_triggers` rows for automation run + const automationRunTriggers = await tables + .automationRunTriggers(deps.sourceDb) + .select('*') + .where(AutomationRunTriggers.col.automationRunId, automationRun.id) + + for (const automationRunTrigger of automationRunTriggers) { + await tables + .automationRunTriggers(deps.targetDb) + .insert(automationRunTrigger) + .onConflict() + .ignore() + } + + // Copy `automation_function_runs` rows for automation run + const automationFunctionRuns = await tables + .automationFunctionRuns(deps.sourceDb) + .select('*') + .where(AutomationFunctionRuns.col.runId, automationRun.id) + + for (const automationFunctionRun of automationFunctionRuns) { + await tables + .automationFunctionRuns(deps.targetDb) + .insert(automationFunctionRun) + .onConflict() + .ignore() + } + } + } + } + } + } + } + } + + return copiedAutomationIds + } diff --git a/packages/server/modules/workspaces/services/projectRegions.ts b/packages/server/modules/workspaces/services/projectRegions.ts index f09328e932..dce91ed773 100644 --- a/packages/server/modules/workspaces/services/projectRegions.ts +++ b/packages/server/modules/workspaces/services/projectRegions.ts @@ -1,8 +1,10 @@ +import { GetProjectAutomationCount } from '@/modules/automate/domain/operations' import { GetStreamBranchCount } from '@/modules/core/domain/branches/operations' import { GetStreamCommitCount } from '@/modules/core/domain/commits/operations' import { GetStreamObjectCount } from '@/modules/core/domain/objects/operations' import { GetProject } from '@/modules/core/domain/projects/operations' import { + CopyProjectAutomations, CopyProjectModels, CopyProjectObjects, CopyProjects, @@ -19,12 +21,14 @@ export const updateProjectRegionFactory = countProjectModels: GetStreamBranchCount countProjectVersions: GetStreamCommitCount countProjectObjects: GetStreamObjectCount + countProjectAutomations: GetProjectAutomationCount getAvailableRegions: GetAvailableRegions copyWorkspace: CopyWorkspace copyProjects: CopyProjects copyProjectModels: CopyProjectModels copyProjectVersions: CopyProjectVersions copyProjectObjects: CopyProjectObjects + copyProjectAutomations: CopyProjectAutomations }): UpdateProjectRegion => async (params) => { const { projectId, regionKey } = params @@ -67,7 +71,9 @@ export const updateProjectRegionFactory = // Move objects const objectIds = await deps.copyProjectObjects({ projectIds }) - // TODO: Move automations + // Move automations + const automationIds = await deps.copyProjectAutomations({ projectIds }) + // TODO: Move comments // TODO: Move file blobs // TODO: Move webhooks @@ -78,11 +84,15 @@ export const updateProjectRegionFactory = const sourceProjectObjectCount = await deps.countProjectObjects({ streamId: projectId }) + const sourceProjectAutomationCount = await deps.countProjectAutomations({ + projectId + }) const tests = [ modelIds[projectId].length === sourceProjectModelCount, versionIds[projectId].length === sourceProjectVersionCount, - objectIds[projectId].length === sourceProjectObjectCount + objectIds[projectId].length === sourceProjectObjectCount, + automationIds[projectId].length === sourceProjectAutomationCount ] if (!tests.every((test) => !!test)) { From de3173e87e8e6b25b057e1701588f0b0ec1d1160 Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Tue, 4 Feb 2025 13:51:27 +0000 Subject: [PATCH 22/28] chore(regions): add tests for moving automations --- .../tests/integration/projects.graph.spec.ts | 90 ++++++++++++++++++- .../test/speckle-helpers/automationHelper.ts | 63 +++++++++---- 2 files changed, 135 insertions(+), 18 deletions(-) diff --git a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts index 7813b8fc23..6a9b3eafae 100644 --- a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts @@ -1,4 +1,23 @@ import { db } from '@/db/knex' +import { + AutomationFunctionRunRecord, + AutomationRecord, + AutomationRevisionFunctionRecord, + AutomationRevisionRecord, + AutomationRunRecord, + AutomationRunTriggerRecord, + AutomationTokenRecord, + AutomationTriggerDefinitionRecord +} from '@/modules/automate/helpers/types' +import { + AutomationFunctionRuns, + AutomationRevisionFunctions, + AutomationRevisions, + AutomationRuns, + AutomationRunTriggers, + AutomationTokens, + AutomationTriggers +} from '@/modules/core/dbSchema' import { AllScopes } from '@/modules/core/helpers/mainConstants' import { createRandomEmail } from '@/modules/core/helpers/testHelpers' import { @@ -35,6 +54,7 @@ import { TestApolloServer } from '@/test/graphqlHelper' import { beforeEachContext } from '@/test/hooks' +import { createTestAutomation } from '@/test/speckle-helpers/automationHelper' import { BasicTestBranch, createTestBranch } from '@/test/speckle-helpers/branchHelper' import { BasicTestCommit, @@ -58,7 +78,20 @@ const tables = { versions: (db: Knex) => db.table('commits'), streamCommits: (db: Knex) => db.table('stream_commits'), branchCommits: (db: Knex) => db.table('branch_commits'), - objects: (db: Knex) => db.table('objects') + objects: (db: Knex) => db.table('objects'), + automations: (db: Knex) => db.table('automations'), + automationTokens: (db: Knex) => db(AutomationTokens.name), + automationRevisions: (db: Knex) => + db(AutomationRevisions.name), + automationTriggers: (db: Knex) => + db(AutomationTriggers.name), + automationRevisionFunctions: (db: Knex) => + db(AutomationRevisionFunctions.name), + automationRuns: (db: Knex) => db(AutomationRuns.name), + automationRunTriggers: (db: Knex) => + db(AutomationRunTriggers.name), + automationFunctionRuns: (db: Knex) => + db(AutomationFunctionRuns.name) } const grantStreamPermissions = grantStreamPermissionsFactory({ db }) @@ -344,6 +377,10 @@ isMultiRegionTestMode() authorId: '' } + let testAutomation: AutomationRecord + let testAutomationToken: AutomationTokenRecord + let testAutomationRevision: AutomationRevisionRecord + let apollo: TestApolloServer let targetRegionDb: Knex @@ -382,6 +419,23 @@ isMultiRegionTestMode() owner: adminUser, stream: testProject }) + + const { automation, revision } = await createTestAutomation({ + userId: adminUser.id, + projectId: testProject.id, + revision: { + functionId: cryptoRandomString({ length: 9 }), + functionReleaseId: cryptoRandomString({ length: 9 }) + } + }) + + if (!revision) { + throw new Error('Failed to create automation revision.') + } + + testAutomation = automation.automation + testAutomationToken = automation.token + testAutomationRevision = revision }) it('moves project record to target regional db', async () => { @@ -464,5 +518,39 @@ isMultiRegionTestMode() expect(object).to.not.be.undefined }) + + it('moves project automation data to target regional db', async () => { + const res = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + + expect(res).to.not.haveGraphQLErrors() + + const automation = await tables + .automations(targetRegionDb) + .select('*') + .where({ id: testAutomation.id }) + .first() + expect(automation).to.not.be.undefined + + const automationToken = await tables + .automationTokens(targetRegionDb) + .select('*') + .where({ automationId: testAutomation.id }) + .first() + expect(automationToken).to.not.be.undefined + expect(automationToken?.automateToken).to.equal( + testAutomationToken.automateToken + ) + + const automationRevision = await tables + .automationRevisions(targetRegionDb) + .select('*') + .where({ automationId: testAutomation.id }) + .first() + expect(automationRevision).to.not.be.undefined + expect(automationRevision?.id).to.equal(testAutomationRevision.id) + }) }) : void 0 diff --git a/packages/server/test/speckle-helpers/automationHelper.ts b/packages/server/test/speckle-helpers/automationHelper.ts index 0e94fb315d..bf921a6e76 100644 --- a/packages/server/test/speckle-helpers/automationHelper.ts +++ b/packages/server/test/speckle-helpers/automationHelper.ts @@ -42,22 +42,29 @@ import { db } from '@/db/knex' import { validateStreamAccessFactory } from '@/modules/core/services/streams/access' import { authorizeResolver } from '@/modules/shared' import { getEventBus } from '@/modules/shared/services/eventBus' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' +import { Knex } from 'knex' -const storeAutomation = storeAutomationFactory({ db }) -const storeAutomationToken = storeAutomationTokenFactory({ db }) -const storeAutomationRevision = storeAutomationRevisionFactory({ db }) -const getAutomation = getAutomationFactory({ db }) -const getLatestStreamBranch = getLatestStreamBranchFactory({ db }) const validateStreamAccess = validateStreamAccessFactory({ authorizeResolver }) export const generateFunctionId = () => cryptoRandomString({ length: 10 }) export const generateFunctionReleaseId = () => cryptoRandomString({ length: 10 }) +/** + * @param overrides By default, we mock requests to the execution engine. You can replace those mocks here. + */ export const buildAutomationCreate = ( - overrides?: Partial<{ - createDbAutomation: typeof clientCreateAutomation - }> + params: { + dbClient: Knex + overrides?: Partial<{ + createDbAutomation: typeof clientCreateAutomation + }> + } = { + dbClient: db + } ) => { + const { dbClient, overrides } = params + const create = createAutomationFactory({ createAuthCode: createStoredAuthCodeFactory({ redis: createInmemoryRedisClient() }), automateCreateAutomation: @@ -66,8 +73,8 @@ export const buildAutomationCreate = ( automationId: cryptoRandomString({ length: 10 }), token: cryptoRandomString({ length: 10 }) })), - storeAutomation, - storeAutomationToken, + storeAutomation: storeAutomationFactory({ db: dbClient }), + storeAutomationToken: storeAutomationTokenFactory({ db: dbClient }), validateStreamAccess, eventEmit: getEventBus().emit }) @@ -75,9 +82,19 @@ export const buildAutomationCreate = ( return create } +/** + * @param overrides By default, we mock requests to the execution engine. You can replace those mocks here. + */ export const buildAutomationRevisionCreate = ( - overrides?: Partial + params: { + dbClient: Knex + overrides?: Partial + } = { + dbClient: db + } ) => { + const { dbClient, overrides } = params + const fakeGetRelease = (params: { functionReleaseId: string functionId: string @@ -91,9 +108,9 @@ export const buildAutomationRevisionCreate = ( }) const create = createAutomationRevisionFactory({ - getAutomation, - storeAutomationRevision, - getBranchesByIds: getBranchesByIdsFactory({ db }), + getAutomation: getAutomationFactory({ db: dbClient }), + storeAutomationRevision: storeAutomationRevisionFactory({ db: dbClient }), + getBranchesByIds: getBranchesByIdsFactory({ db: dbClient }), getFunctionRelease: async (params) => fakeGetRelease(params), getFunctionReleases: async (params) => params.ids.map(fakeGetRelease), getEncryptionKeyPair, @@ -128,8 +145,10 @@ export const createTestAutomation = async (params: { revision: { input: revisionInput, functionReleaseId, functionId } = {} } = params - const createAutomation = buildAutomationCreate() - const createRevision = buildAutomationRevisionCreate() + const projectDb = await getProjectDbClient({ projectId }) + + const createAutomation = buildAutomationCreate({ dbClient: projectDb }) + const createRevision = buildAutomationRevisionCreate({ dbClient: projectDb }) const automationRet = await createAutomation({ input: { @@ -143,7 +162,7 @@ export const createTestAutomation = async (params: { let revisionRet: Awaited> | null = null if (functionReleaseId?.length && functionId?.length) { - const firstModel = await getLatestStreamBranch(projectId) + const firstModel = await getLatestStreamBranchFactory({ db: projectDb })(projectId) if (!firstModel) throw new Error( @@ -186,6 +205,16 @@ export type TestAutomationWithRevision = Awaited< ReturnType > +// export const createTestAutomationRun = async (params: { +// automationId: string +// automationRunData?: Partial +// automationFunctionRunData?: Partial[] +// }) => { +// const { automationId, automationRunData = {}, automationFunctionRunData = {} } = params + +// const latestRevision = getLatestAutomationRevisionFactory({ db }) +// } + export const truncateAutomations = async () => { await truncateTables([ AutomationRunTriggers.name, From e371633702711944081217f67c49dfa6104abeab Mon Sep 17 00:00:00 2001 From: Charles Driesler Date: Thu, 6 Feb 2025 00:47:29 +0000 Subject: [PATCH 23/28] chore(regions): more tests for moving automate data --- .../modules/automate/services/trigger.ts | 7 +- .../tests/integration/projects.graph.spec.ts | 58 ++++++++++++++++- .../test/speckle-helpers/automationHelper.ts | 64 ++++++++++++++++--- 3 files changed, 116 insertions(+), 13 deletions(-) diff --git a/packages/server/modules/automate/services/trigger.ts b/packages/server/modules/automate/services/trigger.ts index 855e0dfe35..289ee9aa69 100644 --- a/packages/server/modules/automate/services/trigger.ts +++ b/packages/server/modules/automate/services/trigger.ts @@ -46,6 +46,7 @@ import { ValidateStreamAccess } from '@/modules/core/domain/streams/operations' import { CreateAndStoreAppToken } from '@/modules/core/domain/tokens/operations' import { EventBusEmit } from '@/modules/shared/services/eventBus' import { AutomationRunEvents } from '@/modules/automate/domain/events' +import { isTestEnv } from '@/modules/shared/helpers/envHelper' export type OnModelVersionCreateDeps = { getAutomation: GetAutomation @@ -130,7 +131,7 @@ type CreateAutomationRunDataDeps = { getFunctionInputDecryptor: FunctionInputDecryptor } -const createAutomationRunDataFactory = +export const createAutomationRunDataFactory = (deps: CreateAutomationRunDataDeps) => async (params: { manifests: BaseTriggerManifest[] @@ -413,7 +414,7 @@ type ComposeTriggerDataDeps = { getBranchLatestCommits: GetBranchLatestCommits } -const composeTriggerDataFactory = +export const composeTriggerDataFactory = (deps: ComposeTriggerDataDeps) => async (params: { projectId: string @@ -572,7 +573,7 @@ export const createTestAutomationRunFactory = throw new TriggerAutomationError('Automation not found') } - if (!automationRecord.isTestAutomation) { + if (!isTestEnv() && !automationRecord.isTestAutomation) { throw new TriggerAutomationError( 'Automation is not a test automation and cannot create test function runs' ) diff --git a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts index 6a9b3eafae..8481f53931 100644 --- a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts @@ -54,7 +54,10 @@ import { TestApolloServer } from '@/test/graphqlHelper' import { beforeEachContext } from '@/test/hooks' -import { createTestAutomation } from '@/test/speckle-helpers/automationHelper' +import { + createTestAutomation, + createTestAutomationRun +} from '@/test/speckle-helpers/automationHelper' import { BasicTestBranch, createTestBranch } from '@/test/speckle-helpers/branchHelper' import { BasicTestCommit, @@ -380,6 +383,8 @@ isMultiRegionTestMode() let testAutomation: AutomationRecord let testAutomationToken: AutomationTokenRecord let testAutomationRevision: AutomationRevisionRecord + let testAutomationRun: AutomationRunRecord + let testAutomationFunctionRuns: AutomationFunctionRunRecord[] let apollo: TestApolloServer let targetRegionDb: Knex @@ -436,6 +441,15 @@ isMultiRegionTestMode() testAutomation = automation.automation testAutomationToken = automation.token testAutomationRevision = revision + + const { automationRun, functionRuns } = await createTestAutomationRun({ + userId: adminUser.id, + projectId: testProject.id, + automationId: testAutomation.id + }) + + testAutomationRun = automationRun + testAutomationFunctionRuns = functionRuns }) it('moves project record to target regional db', async () => { @@ -551,6 +565,48 @@ isMultiRegionTestMode() .first() expect(automationRevision).to.not.be.undefined expect(automationRevision?.id).to.equal(testAutomationRevision.id) + + const automationTrigger = await tables + .automationTriggers(targetRegionDb) + .select('*') + .where({ automationRevisionId: testAutomationRevision.id }) + .first() + expect(automationTrigger).to.not.be.undefined + }) + + it('moves project automation runs to target regional db', async () => { + const res = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + + expect(res).to.not.haveGraphQLErrors() + + const automationRun = await tables + .automationRuns(targetRegionDb) + .select('*') + .where({ id: testAutomationRun.id }) + .first() + expect(automationRun).to.not.be.undefined + + const automationRunTriggers = await tables + .automationRunTriggers(targetRegionDb) + .select('*') + .where({ automationRunId: testAutomationRun.id }) + expect(automationRunTriggers.length).to.not.equal(0) + + const automationFunctionRuns = await tables + .automationFunctionRuns(targetRegionDb) + .select('*') + .where({ runId: testAutomationRun.id }) + expect(automationFunctionRuns.length).to.equal( + testAutomationFunctionRuns.length + ) + expect( + automationFunctionRuns.every((run) => + testAutomationFunctionRuns.some((testRun) => testRun.id === run.id) + ) + ) }) }) : void 0 diff --git a/packages/server/test/speckle-helpers/automationHelper.ts b/packages/server/test/speckle-helpers/automationHelper.ts index bf921a6e76..809a9bc5dd 100644 --- a/packages/server/test/speckle-helpers/automationHelper.ts +++ b/packages/server/test/speckle-helpers/automationHelper.ts @@ -1,8 +1,12 @@ import { getAutomationFactory, + getFullAutomationRevisionMetadataFactory, + getFullAutomationRunByIdFactory, + getLatestAutomationRevisionFactory, storeAutomationFactory, storeAutomationRevisionFactory, - storeAutomationTokenFactory + storeAutomationTokenFactory, + upsertAutomationRunFactory } from '@/modules/automate/repositories/automations' import { CreateAutomationRevisionDeps, @@ -15,6 +19,7 @@ import cryptoRandomString from 'crypto-random-string' import { createAutomation as clientCreateAutomation } from '@/modules/automate/clients/executionEngine' import { getBranchesByIdsFactory, + getBranchLatestCommitsFactory, getLatestStreamBranchFactory } from '@/modules/core/repositories/branches' @@ -35,6 +40,7 @@ import { import { faker } from '@faker-js/faker' import { getEncryptionKeyPair, + getEncryptionKeyPairFor, getFunctionInputDecryptorFactory } from '@/modules/automate/services/encryption' import { buildDecryptor } from '@/modules/shared/utils/libsodium' @@ -44,6 +50,7 @@ import { authorizeResolver } from '@/modules/shared' import { getEventBus } from '@/modules/shared/services/eventBus' import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { Knex } from 'knex' +import { createTestAutomationRunFactory } from '@/modules/automate/services/trigger' const validateStreamAccess = validateStreamAccessFactory({ authorizeResolver }) @@ -205,15 +212,54 @@ export type TestAutomationWithRevision = Awaited< ReturnType > -// export const createTestAutomationRun = async (params: { -// automationId: string -// automationRunData?: Partial -// automationFunctionRunData?: Partial[] -// }) => { -// const { automationId, automationRunData = {}, automationFunctionRunData = {} } = params +export const createTestAutomationRun = async (params: { + userId: string + projectId: string + automationId: string +}) => { + const { userId, projectId, automationId } = params -// const latestRevision = getLatestAutomationRevisionFactory({ db }) -// } + const projectDb = await getProjectDbClient({ projectId }) + + const { automationRunId } = await createTestAutomationRunFactory({ + getEncryptionKeyPairFor, + getFunctionInputDecryptor: getFunctionInputDecryptorFactory({ + buildDecryptor + }), + getAutomation: getAutomationFactory({ + db: projectDb + }), + getLatestAutomationRevision: getLatestAutomationRevisionFactory({ + db: projectDb + }), + getFullAutomationRevisionMetadata: getFullAutomationRevisionMetadataFactory({ + db: projectDb + }), + upsertAutomationRun: upsertAutomationRunFactory({ + db: projectDb + }), + getBranchLatestCommits: getBranchLatestCommitsFactory({ + db: projectDb + }), + validateStreamAccess + })({ projectId, automationId, userId }) + + const automationRunData = await getFullAutomationRunByIdFactory({ db: projectDb })( + automationRunId + ) + + if (!automationRunData) { + throw new Error('Failed to create test automation run!') + } + + const { triggers, functionRuns, ...automationRun } = automationRunData + + return { + automationRun, + functionRuns, + triggers + } +} export const truncateAutomations = async () => { await truncateTables([ From 04edd9c6af29aa3fd8dddf7ff751ccd8c744b52c Mon Sep 17 00:00:00 2001 From: Chuck Driesler Date: Thu, 6 Feb 2025 16:59:44 +0000 Subject: [PATCH 24/28] fix(regions): speed up inserts --- .../modules/workspaces/domain/operations.ts | 4 +- .../workspaces/graph/resolvers/regions.ts | 8 + .../workspaces/repositories/projectRegions.ts | 202 +++++++++--------- .../workspaces/services/projectRegions.ts | 9 +- .../tests/integration/projects.graph.spec.ts | 5 + packages/shared/src/environment/index.ts | 6 + 6 files changed, 129 insertions(+), 105 deletions(-) diff --git a/packages/server/modules/workspaces/domain/operations.ts b/packages/server/modules/workspaces/domain/operations.ts index 83ee87e856..66f0983c30 100644 --- a/packages/server/modules/workspaces/domain/operations.ts +++ b/packages/server/modules/workspaces/domain/operations.ts @@ -359,7 +359,7 @@ export type CopyWorkspace = (params: { workspaceId: string }) => Promise export type CopyProjects = (params: { projectIds: string[] }) => Promise export type CopyProjectModels = (params: { projectIds: string[] -}) => Promise> +}) => Promise> export type CopyProjectVersions = (params: { projectIds: string[] -}) => Promise> +}) => Promise> diff --git a/packages/server/modules/workspaces/graph/resolvers/regions.ts b/packages/server/modules/workspaces/graph/resolvers/regions.ts index 73770ec14a..bb01c81951 100644 --- a/packages/server/modules/workspaces/graph/resolvers/regions.ts +++ b/packages/server/modules/workspaces/graph/resolvers/regions.ts @@ -29,6 +29,10 @@ import { getProjectFactory } from '@/modules/core/repositories/projects' import { getStreamBranchCountFactory } from '@/modules/core/repositories/branches' import { getStreamCommitCountFactory } from '@/modules/core/repositories/commits' import { withTransaction } from '@/modules/shared/helpers/dbHelper' +import { getFeatureFlags, isTestEnv } from '@/modules/shared/helpers/envHelper' +import { WorkspacesNotYetImplementedError } from '@/modules/workspaces/errors/workspace' + +const { FF_MOVE_PROJECT_REGION_ENABLED } = getFeatureFlags() export default { Workspace: { @@ -67,6 +71,10 @@ export default { }, WorkspaceProjectMutations: { moveToRegion: async (_parent, args, context) => { + if (!FF_MOVE_PROJECT_REGION_ENABLED && !isTestEnv()) { + throw new WorkspacesNotYetImplementedError() + } + await authorizeResolver( context.userId, args.projectId, diff --git a/packages/server/modules/workspaces/repositories/projectRegions.ts b/packages/server/modules/workspaces/repositories/projectRegions.ts index 94da98d14d..4c7e5eee79 100644 --- a/packages/server/modules/workspaces/repositories/projectRegions.ts +++ b/packages/server/modules/workspaces/repositories/projectRegions.ts @@ -12,8 +12,10 @@ import { Commit } from '@/modules/core/domain/commits/types' import { Stream } from '@/modules/core/domain/streams/types' import { BranchCommitRecord, + CommitRecord, StreamCommitRecord, - StreamFavoriteRecord + StreamFavoriteRecord, + StreamRecord } from '@/modules/core/helpers/types' import { executeBatchedSelect } from '@/modules/shared/helpers/dbHelper' import { @@ -50,7 +52,11 @@ export const copyWorkspaceFactory = throw new WorkspaceNotFoundError() } - await tables.workspaces(deps.targetDb).insert(workspace) + await tables + .workspaces(deps.targetDb) + .insert(workspace) + .onConflict(Workspaces.withoutTablePrefix.col.id) + .merge(Workspaces.withoutTablePrefix.cols as (keyof Workspace)[]) return workspaceId } @@ -66,15 +72,15 @@ export const copyProjectsFactory = // Copy project record for await (const projects of executeBatchedSelect(selectProjects)) { - for (const project of projects) { - // Store copied project id - copiedProjectIds.push(project.id) - - // Copy `streams` row to target db - await tables.projects(deps.targetDb).insert(project).onConflict().ignore() - } - const projectIds = projects.map((project) => project.id) + copiedProjectIds.push(...projectIds) + + // Copy `streams` rows to target db + await tables + .projects(deps.targetDb) + .insert(projects) + .onConflict(Streams.withoutTablePrefix.col.id) + .merge(Streams.withoutTablePrefix.cols as (keyof StreamRecord)[]) // Fetch `stream_favorites` rows for projects in batch const selectStreamFavorites = tables @@ -83,14 +89,12 @@ export const copyProjectsFactory = .whereIn(StreamFavorites.col.streamId, projectIds) for await (const streamFavorites of executeBatchedSelect(selectStreamFavorites)) { - for (const streamFavorite of streamFavorites) { - // Copy `stream_favorites` row to target db - await tables - .streamFavorites(deps.targetDb) - .insert(streamFavorite) - .onConflict() - .ignore() - } + // Copy `stream_favorites` rows to target db + await tables + .streamFavorites(deps.targetDb) + .insert(streamFavorites) + .onConflict() + .ignore() } // Fetch `streams_meta` rows for projects in batch @@ -102,14 +106,12 @@ export const copyProjectsFactory = for await (const streamsMetadataBatch of executeBatchedSelect( selectStreamsMetadata )) { - for (const streamMetadata of streamsMetadataBatch) { - // Copy `streams_meta` row to target db - await tables - .streamsMeta(deps.targetDb) - .insert(streamMetadata) - .onConflict() - .ignore() - } + // Copy `streams_meta` rows to target db + await tables + .streamsMeta(deps.targetDb) + .insert(streamsMetadataBatch) + .onConflict() + .ignore() } } @@ -119,98 +121,98 @@ export const copyProjectsFactory = export const copyProjectModelsFactory = (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectModels => async ({ projectIds }) => { - const copiedModelIds: Record = projectIds.reduce( - (result, id) => ({ ...result, [id]: [] }), - {} - ) - - for (const projectId of projectIds) { - const selectModels = tables - .models(deps.sourceDb) - .select('*') - .where({ streamId: projectId }) + const copiedModelCountByProjectId: Record = {} - for await (const models of executeBatchedSelect(selectModels)) { - for (const model of models) { - // Store copied model ids - copiedModelIds[projectId].push(model.id) + // Fetch `branches` rows for projects in batch + const selectModels = tables + .models(deps.sourceDb) + .select('*') + .whereIn(Branches.col.streamId, projectIds) - // Copy `branches` row to target db - await tables.models(deps.targetDb).insert(model).onConflict().ignore() - } + for await (const models of executeBatchedSelect(selectModels)) { + // Copy `branches` rows to target db + await tables.models(deps.targetDb).insert(models).onConflict().ignore() + + for (const model of models) { + copiedModelCountByProjectId[model.streamId] ??= 0 + copiedModelCountByProjectId[model.streamId]++ } } - return copiedModelIds + return copiedModelCountByProjectId } export const copyProjectVersionsFactory = (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectVersions => async ({ projectIds }) => { - const copiedVersionIds: Record = projectIds.reduce( - (result, id) => ({ ...result, [id]: [] }), - {} - ) + const copiedVersionCountByProjectId: Record = {} - for (const projectId of projectIds) { - const selectVersions = tables - .streamCommits(deps.sourceDb) - .select('*') - .join( - Commits.name, - Commits.col.id, - StreamCommits.col.commitId - ) - .where({ streamId: projectId }) - - for await (const versions of executeBatchedSelect(selectVersions)) { - for (const version of versions) { + const selectVersions = tables + .streamCommits(deps.sourceDb) + .select('*') + .join( + Commits.name, + Commits.col.id, + StreamCommits.col.commitId + ) + .whereIn(StreamCommits.col.streamId, projectIds) + + for await (const versions of executeBatchedSelect(selectVersions)) { + const { commitIds, commits } = versions.reduce( + (all, version) => { const { commitId, streamId, ...commit } = version - // Store copied version id - copiedVersionIds[streamId].push(commitId) + all.commitIds.push(commitId) + all.streamIds.push(streamId) + all.commits.push(commit) - // Copy `commits` row to target db - await tables.versions(deps.targetDb).insert(commit).onConflict().ignore() + return all + }, + { commitIds: [], streamIds: [], commits: [] } as { + commitIds: string[] + streamIds: string[] + commits: CommitRecord[] } + ) - const commitIds = versions.map((version) => version.commitId) - - // Fetch `branch_commits` rows for versions in batch - const selectBranchCommits = tables - .branchCommits(deps.sourceDb) - .select('*') - .whereIn(BranchCommits.col.commitId, commitIds) - - for await (const branchCommits of executeBatchedSelect(selectBranchCommits)) { - for (const branchCommit of branchCommits) { - // Copy `branch_commits` row to target db - await tables - .branchCommits(deps.targetDb) - .insert(branchCommit) - .onConflict() - .ignore() - } - } + // Copy `commits` rows to target db + await tables.versions(deps.targetDb).insert(commits).onConflict().ignore() - // Fetch `stream_commits` rows for versions in batch - const selectStreamCommits = tables - .streamCommits(deps.sourceDb) - .select('*') - .whereIn(StreamCommits.col.commitId, commitIds) - - for await (const streamCommits of executeBatchedSelect(selectStreamCommits)) { - for (const streamCommit of streamCommits) { - // Copy `stream_commits` row to target db - await tables - .streamCommits(deps.targetDb) - .insert(streamCommit) - .onConflict() - .ignore() - } - } + for (const version of versions) { + copiedVersionCountByProjectId[version.streamId] ??= 0 + copiedVersionCountByProjectId[version.streamId]++ + } + + // Fetch `branch_commits` rows for versions in batch + const selectBranchCommits = tables + .branchCommits(deps.sourceDb) + .select('*') + .whereIn(BranchCommits.col.commitId, commitIds) + + for await (const branchCommits of executeBatchedSelect(selectBranchCommits)) { + // Copy `branch_commits` row to target db + await tables + .branchCommits(deps.targetDb) + .insert(branchCommits) + .onConflict() + .ignore() + } + + // Fetch `stream_commits` rows for versions in batch + const selectStreamCommits = tables + .streamCommits(deps.sourceDb) + .select('*') + .whereIn(StreamCommits.col.commitId, commitIds) + + for await (const streamCommits of executeBatchedSelect(selectStreamCommits)) { + // Copy `stream_commits` row to target db + await tables + .streamCommits(deps.targetDb) + .insert(streamCommits) + .onConflict() + .ignore() } } - return copiedVersionIds + return copiedVersionCountByProjectId } diff --git a/packages/server/modules/workspaces/services/projectRegions.ts b/packages/server/modules/workspaces/services/projectRegions.ts index 561aee792f..2bc60cd66d 100644 --- a/packages/server/modules/workspaces/services/projectRegions.ts +++ b/packages/server/modules/workspaces/services/projectRegions.ts @@ -70,9 +70,12 @@ export const updateProjectRegionFactory = const sourceProjectModelCount = await deps.countProjectModels(projectId) const sourceProjectVersionCount = await deps.countProjectVersions(projectId) - const isReconciled = - modelIds[projectId].length === sourceProjectModelCount && - versionIds[projectId].length === sourceProjectVersionCount + const tests = [ + modelIds[projectId] === sourceProjectModelCount, + versionIds[projectId] === sourceProjectVersionCount + ] + + const isReconciled = tests.every((test) => !!test) if (!isReconciled) { // TODO: Move failed or source project added data while changing regions. Retry move. diff --git a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts index 9b33dc0adc..985e1c6ee2 100644 --- a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts @@ -382,6 +382,7 @@ isMultiRegionTestMode() expect(res).to.not.haveGraphQLErrors() + // TODO: Replace with gql query when possible const project = await targetRegionDb .table('streams') .select('*') @@ -399,6 +400,7 @@ isMultiRegionTestMode() expect(res).to.not.haveGraphQLErrors() + // TODO: Replace with gql query when possible const branch = await targetRegionDb .table('branches') .select('*') @@ -416,6 +418,7 @@ isMultiRegionTestMode() expect(res).to.not.haveGraphQLErrors() + // TODO: Replace with gql query when possible const version = await targetRegionDb .table('commits') .select('*') @@ -423,6 +426,7 @@ isMultiRegionTestMode() .first() expect(version).to.not.be.undefined + // TODO: Replace with gql query when possible const streamCommitsRecord = await targetRegionDb .table('stream_commits') .select('*') @@ -430,6 +434,7 @@ isMultiRegionTestMode() .first() expect(streamCommitsRecord).to.not.be.undefined + // TODO: Replace with gql query when possible const branchCommitsRecord = await targetRegionDb .table('branch_commits') .select('*') diff --git a/packages/shared/src/environment/index.ts b/packages/shared/src/environment/index.ts index 0d35eacdb1..d81d9fc273 100644 --- a/packages/shared/src/environment/index.ts +++ b/packages/shared/src/environment/index.ts @@ -70,6 +70,11 @@ const parseFeatureFlags = () => { FF_OBJECTS_STREAMING_FIX: { schema: z.boolean(), defaults: { production: false, _: false } + }, + // Enables endpoint(s) for updating a project's region + FF_MOVE_PROJECT_REGION_ENABLED: { + schema: z.boolean(), + defaults: { production: false, _: true } } }) @@ -98,6 +103,7 @@ export function getFeatureFlags(): { FF_FORCE_EMAIL_VERIFICATION: boolean FF_FORCE_ONBOARDING: boolean FF_OBJECTS_STREAMING_FIX: boolean + FF_MOVE_PROJECT_REGION_ENABLED: boolean } { if (!parsedFlags) parsedFlags = parseFeatureFlags() return parsedFlags From 72167e6194c7823729835ef1fb24c23c8841daf8 Mon Sep 17 00:00:00 2001 From: Chuck Driesler Date: Fri, 7 Feb 2025 14:55:33 +0000 Subject: [PATCH 25/28] fix(regions): simplify postgres usage --- .../modules/workspaces/domain/operations.ts | 2 +- .../workspaces/repositories/projectRegions.ts | 80 ++++++------------- .../workspaces/services/projectRegions.ts | 12 +-- 3 files changed, 33 insertions(+), 61 deletions(-) diff --git a/packages/server/modules/workspaces/domain/operations.ts b/packages/server/modules/workspaces/domain/operations.ts index 1f57d9f66c..7039c2463b 100644 --- a/packages/server/modules/workspaces/domain/operations.ts +++ b/packages/server/modules/workspaces/domain/operations.ts @@ -365,4 +365,4 @@ export type CopyProjectVersions = (params: { }) => Promise> export type CopyProjectObjects = (params: { projectIds: string[] -}) => Promise> +}) => Promise> diff --git a/packages/server/modules/workspaces/repositories/projectRegions.ts b/packages/server/modules/workspaces/repositories/projectRegions.ts index a091636192..07a92d631a 100644 --- a/packages/server/modules/workspaces/repositories/projectRegions.ts +++ b/packages/server/modules/workspaces/repositories/projectRegions.ts @@ -249,68 +249,40 @@ export const copyProjectVersionsFactory = /** * Copies rows from the following tables: * - objects - * - object_children_closure * - object_preview */ export const copyProjectObjectsFactory = (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectObjects => async ({ projectIds }) => { - const copiedObjectIds: Record = {} - - for (const projectId of projectIds) { - copiedObjectIds[projectId] = [] - - // Copy `objects` table rows in batches - const selectObjects = tables - .objects(deps.sourceDb) - .select('*') - .where(Objects.col.streamId, projectId) - .orderBy(Objects.col.id) - - for await (const objects of executeBatchedSelect(selectObjects)) { - for (const object of objects) { - // Store copied object ids by source project - copiedObjectIds[projectId].push(object.id) - - // Write `objects` table row to target db - await tables.objects(deps.targetDb).insert(object).onConflict().ignore() - } + const copiedObjectCountByProjectId: Record = {} + + // Copy `objects` table rows in batches + const selectObjects = tables + .objects(deps.sourceDb) + .select('*') + .whereIn(Objects.col.streamId, projectIds) + .orderBy(Objects.col.id) + + for await (const objects of executeBatchedSelect(selectObjects)) { + // Write `objects` table rows to target db + await tables.objects(deps.targetDb).insert(objects).onConflict().ignore() + + for (const object of objects) { + copiedObjectCountByProjectId[object.streamId] ??= 0 + copiedObjectCountByProjectId[object.streamId]++ } + } - // Copy `object_children_closure` rows in batches - const selectObjectClosures = tables - .objectClosures(deps.sourceDb) - .select('*') - .where('streamId', projectId) - - for await (const closures of executeBatchedSelect(selectObjectClosures)) { - for (const closure of closures) { - // Write `object_children_closure` row to target db - await tables - .objectClosures(deps.targetDb) - .insert(closure) - .onConflict() - .ignore() - } - } + // Copy `object_preview` rows in batches + const selectObjectPreviews = tables + .objectPreviews(deps.sourceDb) + .select('*') + .whereIn('streamId', projectIds) - // Copy `object_preview` rows in batches - const selectObjectPreviews = tables - .objectPreviews(deps.sourceDb) - .select('*') - .where('streamId', projectId) - - for await (const previews of executeBatchedSelect(selectObjectPreviews)) { - for (const preview of previews) { - // Write `object_preview` row to target db - await tables - .objectPreviews(deps.targetDb) - .insert(preview) - .onConflict() - .ignore() - } - } + for await (const previews of executeBatchedSelect(selectObjectPreviews)) { + // Write `object_preview` rows to target db + await tables.objectPreviews(deps.targetDb).insert(previews).onConflict().ignore() } - return copiedObjectIds + return copiedObjectCountByProjectId } diff --git a/packages/server/modules/workspaces/services/projectRegions.ts b/packages/server/modules/workspaces/services/projectRegions.ts index e318c829e7..84f5688cae 100644 --- a/packages/server/modules/workspaces/services/projectRegions.ts +++ b/packages/server/modules/workspaces/services/projectRegions.ts @@ -61,11 +61,11 @@ export const updateProjectRegionFactory = // Move commits const projectIds = await deps.copyProjects({ projectIds: [projectId] }) - const modelIds = await deps.copyProjectModels({ projectIds }) - const versionIds = await deps.copyProjectVersions({ projectIds }) + const copiedModelCount = await deps.copyProjectModels({ projectIds }) + const copiedVersionCount = await deps.copyProjectVersions({ projectIds }) // Move objects - const objectIds = await deps.copyProjectObjects({ projectIds }) + const copiedObjectCount = await deps.copyProjectObjects({ projectIds }) // TODO: Move automations // TODO: Move comments @@ -80,9 +80,9 @@ export const updateProjectRegionFactory = }) const tests = [ - modelIds[projectId] === sourceProjectModelCount, - versionIds[projectId] === sourceProjectVersionCount, - objectIds[projectId].length === sourceProjectObjectCount + copiedModelCount[projectId] === sourceProjectModelCount, + copiedVersionCount[projectId] === sourceProjectVersionCount, + copiedObjectCount[projectId] === sourceProjectObjectCount ] if (!tests.every((test) => !!test)) { From 25eacc30529a6b2f963d6a7037b75e581053735b Mon Sep 17 00:00:00 2001 From: Chuck Driesler Date: Fri, 7 Feb 2025 16:05:07 +0000 Subject: [PATCH 26/28] chore(regions): repair build --- .../modules/automate/tests/automations.spec.ts | 8 +++++--- .../workspaces/repositories/projectRegions.ts | 3 --- .../test/speckle-helpers/automationHelper.ts | 16 ++++++---------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/packages/server/modules/automate/tests/automations.spec.ts b/packages/server/modules/automate/tests/automations.spec.ts index cb916dcd0b..a0cf32e0e9 100644 --- a/packages/server/modules/automate/tests/automations.spec.ts +++ b/packages/server/modules/automate/tests/automations.spec.ts @@ -516,9 +516,11 @@ const buildAutomationUpdate = () => { it('fails when refering to nonexistent function releases', async () => { const create = buildAutomationRevisionCreate({ - getFunctionRelease: async () => { - // TODO: Update once we know how exec engine should respond - throw new Error('Function release with ID XXX not found') + overrides: { + getFunctionRelease: async () => { + // TODO: Update once we know how exec engine should respond + throw new Error('Function release with ID XXX not found') + } } }) diff --git a/packages/server/modules/workspaces/repositories/projectRegions.ts b/packages/server/modules/workspaces/repositories/projectRegions.ts index 341102921c..31c11e5dcd 100644 --- a/packages/server/modules/workspaces/repositories/projectRegions.ts +++ b/packages/server/modules/workspaces/repositories/projectRegions.ts @@ -21,7 +21,6 @@ import { Commit } from '@/modules/core/domain/commits/types' import { Stream } from '@/modules/core/domain/streams/types' import { BranchCommitRecord, - ObjectChildrenClosureRecord, ObjectRecord, CommitRecord, StreamCommitRecord, @@ -63,8 +62,6 @@ const tables = { streamFavorites: (db: Knex) => db(StreamFavorites.name), streamsMeta: (db: Knex) => db(StreamsMeta.name), objects: (db: Knex) => db(Objects.name), - objectClosures: (db: Knex) => - db('object_children_closure'), objectPreviews: (db: Knex) => db('object_preview'), automations: (db: Knex) => db(Automations.name), automationTokens: (db: Knex) => db(AutomationTokens.name), diff --git a/packages/server/test/speckle-helpers/automationHelper.ts b/packages/server/test/speckle-helpers/automationHelper.ts index 809a9bc5dd..a9b99b3d34 100644 --- a/packages/server/test/speckle-helpers/automationHelper.ts +++ b/packages/server/test/speckle-helpers/automationHelper.ts @@ -62,15 +62,13 @@ export const generateFunctionReleaseId = () => cryptoRandomString({ length: 10 } */ export const buildAutomationCreate = ( params: { - dbClient: Knex + dbClient?: Knex overrides?: Partial<{ createDbAutomation: typeof clientCreateAutomation }> - } = { - dbClient: db - } + } = {} ) => { - const { dbClient, overrides } = params + const { dbClient = db, overrides } = params const create = createAutomationFactory({ createAuthCode: createStoredAuthCodeFactory({ redis: createInmemoryRedisClient() }), @@ -94,13 +92,11 @@ export const buildAutomationCreate = ( */ export const buildAutomationRevisionCreate = ( params: { - dbClient: Knex + dbClient?: Knex overrides?: Partial - } = { - dbClient: db - } + } = {} ) => { - const { dbClient, overrides } = params + const { dbClient = db, overrides } = params const fakeGetRelease = (params: { functionReleaseId: string From 78e8ff8c32eee9f19a749d184d1b7dfae0e0463e Mon Sep 17 00:00:00 2001 From: Chuck Driesler Date: Fri, 7 Feb 2025 16:25:20 +0000 Subject: [PATCH 27/28] fix(regions): improve queries --- .../modules/workspaces/domain/operations.ts | 2 +- .../workspaces/repositories/projectRegions.ts | 242 ++++++++---------- .../workspaces/services/projectRegions.ts | 4 +- 3 files changed, 116 insertions(+), 132 deletions(-) diff --git a/packages/server/modules/workspaces/domain/operations.ts b/packages/server/modules/workspaces/domain/operations.ts index a8b57f9110..6252c389e1 100644 --- a/packages/server/modules/workspaces/domain/operations.ts +++ b/packages/server/modules/workspaces/domain/operations.ts @@ -368,4 +368,4 @@ export type CopyProjectObjects = (params: { }) => Promise> export type CopyProjectAutomations = (params: { projectIds: string[] -}) => Promise> +}) => Promise> diff --git a/packages/server/modules/workspaces/repositories/projectRegions.ts b/packages/server/modules/workspaces/repositories/projectRegions.ts index 31c11e5dcd..9423b4a4ec 100644 --- a/packages/server/modules/workspaces/repositories/projectRegions.ts +++ b/packages/server/modules/workspaces/repositories/projectRegions.ts @@ -330,148 +330,132 @@ export const copyProjectObjectsFactory = export const copyProjectAutomationsFactory = (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectAutomations => async ({ projectIds }) => { - const copiedAutomationIds: Record = {} + const copiedAutomationCountByProjectId: Record = {} - for (const projectId of projectIds) { - copiedAutomationIds[projectId] = [] + // Copy `automations` table rows in batches + const selectAutomations = tables + .automations(deps.sourceDb) + .select('*') + .whereIn(Automations.col.projectId, projectIds) + + for await (const automations of executeBatchedSelect(selectAutomations)) { + const automationIds = automations.map((automation) => automation.id) + + // Write `automations` table rows to target db + await tables + .automations(deps.targetDb) + // Cast ignores unexpected behavior in how knex handles object union types + .insert(automations as unknown as AutomationRecord) + .onConflict() + .ignore() + + for (const automation of automations) { + copiedAutomationCountByProjectId[automation.projectId] ??= 0 + copiedAutomationCountByProjectId[automation.projectId]++ + } - // Copy `automations` table rows in batches - const selectAutomations = tables - .automations(deps.sourceDb) + // Copy `automation_tokens` rows for automation + const selectAutomationTokens = tables + .automationTokens(deps.sourceDb) .select('*') - .where(Automations.col.projectId, projectId) + .whereIn(AutomationTokens.col.automationId, automationIds) + + for await (const tokens of executeBatchedSelect(selectAutomationTokens)) { + // Write `automation_tokens` row to target db + await tables + .automationTokens(deps.targetDb) + .insert(tokens) + .onConflict() + .ignore() + } + + // Copy `automation_revisions` rows for automation + const selectAutomationRevisions = tables + .automationRevisions(deps.sourceDb) + .select('*') + .whereIn(AutomationRevisions.col.automationId, automationIds) + + for await (const automationRevisions of executeBatchedSelect( + selectAutomationRevisions + )) { + const automationRevisionIds = automationRevisions.map((revision) => revision.id) + + // Write `automation_revisions` rows to target db + await tables + .automationRevisions(deps.targetDb) + .insert(automationRevisions) + .onConflict() + .ignore() + + // Copy `automation_triggers` rows for automation revisions + const automationTriggers = await tables + .automationTriggers(deps.sourceDb) + .select('*') + .whereIn(AutomationTriggers.col.automationRevisionId, automationRevisionIds) - for await (const automations of executeBatchedSelect(selectAutomations)) { - for (const automation of automations) { - // Store copied automation id - copiedAutomationIds[projectId].push(automation.id) + await tables + .automationTriggers(deps.targetDb) + .insert(automationTriggers) + .onConflict() + .ignore() + + // Copy `automation_revision_functions` rows for automation revisions + const automationRevisionFunctions = await tables + .automationRevisionFunctions(deps.sourceDb) + .select('*') + .whereIn( + AutomationRevisionFunctions.col.automationRevisionId, + automationRevisionIds + ) + + await tables + .automationRevisionFunctions(deps.targetDb) + .insert(automationRevisionFunctions) + .onConflict() + .ignore() + + // Copy `automation_runs` rows for automation revision + const selectAutomationRuns = tables + .automationRuns(deps.sourceDb) + .select('*') + .whereIn(AutomationRuns.col.automationRevisionId, automationRevisionIds) + + for await (const automationRuns of executeBatchedSelect(selectAutomationRuns)) { + const automationRunIds = automationRuns.map((run) => run.id) - // Write `automations` table row to target db + // Write `automation_runs` row to target db await tables - .automations(deps.targetDb) - .insert(automation) + .automationRuns(deps.targetDb) + .insert(automationRuns) .onConflict() .ignore() - // Copy `automation_tokens` rows for automation - const selectAutomationTokens = tables - .automationTokens(deps.sourceDb) + // Copy `automation_run_triggers` rows for automation run + const automationRunTriggers = await tables + .automationRunTriggers(deps.sourceDb) .select('*') - .where(AutomationTokens.col.automationId, automation.id) - - for await (const tokens of executeBatchedSelect(selectAutomationTokens)) { - for (const token of tokens) { - // Write `automation_tokens` row to target db - await tables - .automationTokens(deps.targetDb) - .insert(token) - .onConflict() - .ignore() - } - } - - // Copy `automation_revisions` rows for automation - const selectAutomationRevisions = tables - .automationRevisions(deps.sourceDb) + .whereIn(AutomationRunTriggers.col.automationRunId, automationRunIds) + + await tables + .automationRunTriggers(deps.targetDb) + .insert(automationRunTriggers) + .onConflict() + .ignore() + + // Copy `automation_function_runs` rows for automation run + const automationFunctionRuns = await tables + .automationFunctionRuns(deps.sourceDb) .select('*') - .where(AutomationRevisions.col.automationId, automation.id) - - for await (const automationRevisions of executeBatchedSelect( - selectAutomationRevisions - )) { - for (const automationRevision of automationRevisions) { - // Write `automation_revisions` row to target db - await tables - .automationRevisions(deps.targetDb) - .insert(automationRevision) - .onConflict() - .ignore() - - // Copy `automation_triggers` rows for automation revision - const automationTriggers = await tables - .automationTriggers(deps.sourceDb) - .select('*') - .where( - AutomationTriggers.col.automationRevisionId, - automationRevision.id - ) - - for (const automationTrigger of automationTriggers) { - await tables - .automationTriggers(deps.targetDb) - .insert(automationTrigger) - .onConflict() - .ignore() - } - - // Copy `automation_revision_functions` rows for automation revision - const automationRevisionFunctions = await tables - .automationRevisionFunctions(deps.sourceDb) - .select('*') - .where( - AutomationRevisionFunctions.col.automationRevisionId, - automationRevision.id - ) - - for (const automationRevisionFunction of automationRevisionFunctions) { - await tables - .automationRevisionFunctions(deps.targetDb) - .insert(automationRevisionFunction) - .onConflict() - .ignore() - } - - // Copy `automation_runs` rows for automation revision - const selectAutomationRuns = tables - .automationRuns(deps.sourceDb) - .select('*') - .where(AutomationRuns.col.automationRevisionId, automationRevision.id) - - for await (const automationRuns of executeBatchedSelect( - selectAutomationRuns - )) { - for (const automationRun of automationRuns) { - // Write `automation_runs` row to target db - await tables - .automationRuns(deps.targetDb) - .insert(automationRun) - .onConflict() - .ignore() - - // Copy `automation_run_triggers` rows for automation run - const automationRunTriggers = await tables - .automationRunTriggers(deps.sourceDb) - .select('*') - .where(AutomationRunTriggers.col.automationRunId, automationRun.id) - - for (const automationRunTrigger of automationRunTriggers) { - await tables - .automationRunTriggers(deps.targetDb) - .insert(automationRunTrigger) - .onConflict() - .ignore() - } - - // Copy `automation_function_runs` rows for automation run - const automationFunctionRuns = await tables - .automationFunctionRuns(deps.sourceDb) - .select('*') - .where(AutomationFunctionRuns.col.runId, automationRun.id) - - for (const automationFunctionRun of automationFunctionRuns) { - await tables - .automationFunctionRuns(deps.targetDb) - .insert(automationFunctionRun) - .onConflict() - .ignore() - } - } - } - } - } + .whereIn(AutomationFunctionRuns.col.runId, automationRunIds) + + await tables + .automationFunctionRuns(deps.targetDb) + .insert(automationFunctionRuns) + .onConflict() + .ignore() } } } - return copiedAutomationIds + return copiedAutomationCountByProjectId } diff --git a/packages/server/modules/workspaces/services/projectRegions.ts b/packages/server/modules/workspaces/services/projectRegions.ts index 869bf5c67f..78cc9b7cff 100644 --- a/packages/server/modules/workspaces/services/projectRegions.ts +++ b/packages/server/modules/workspaces/services/projectRegions.ts @@ -72,7 +72,7 @@ export const updateProjectRegionFactory = const copiedObjectCount = await deps.copyProjectObjects({ projectIds }) // Move automations - const automationIds = await deps.copyProjectAutomations({ projectIds }) + const copiedAutomationCount = await deps.copyProjectAutomations({ projectIds }) // TODO: Move comments // TODO: Move file blobs @@ -92,7 +92,7 @@ export const updateProjectRegionFactory = copiedModelCount[projectId] === sourceProjectModelCount, copiedVersionCount[projectId] === sourceProjectVersionCount, copiedObjectCount[projectId] === sourceProjectObjectCount, - automationIds[projectId].length === sourceProjectAutomationCount + copiedAutomationCount[projectId] === sourceProjectAutomationCount ] if (!tests.every((test) => !!test)) { From e391f887291197e0e558d1e28b95bf1930de9c67 Mon Sep 17 00:00:00 2001 From: Chuck Driesler Date: Tue, 18 Feb 2025 10:36:43 +0000 Subject: [PATCH 28/28] chore(regions): again --- packages/server/modules/core/services/projects.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/modules/core/services/projects.ts b/packages/server/modules/core/services/projects.ts index 7a9196146f..8c88dca4a4 100644 --- a/packages/server/modules/core/services/projects.ts +++ b/packages/server/modules/core/services/projects.ts @@ -61,7 +61,7 @@ export const createNewProjectFactory = const replicatedProject = await getProject({ projectId }) if (!replicatedProject) throw new StreamNotFoundError() }, - { maxAttempts: 100 } + { maxAttempts: 10 } ) } catch (err) { if (err instanceof StreamNotFoundError) {