fix(server): optimized commit download (specklesystems#1913)

* fix(server): optimized commit download

* added test

packages/frontend-2/plugins/001-logger.ts

@@ -52,8 +52,8 @@ export default defineNuxtPlugin(async (nuxtApp) => {
           speckleServerVersion,
           serverName,
           frontendType: 'frontend-2',
-          route: route.path,
-          routeDefinition: route.matched[route.matched.length - 1].path,
+          route: route?.path,
+          routeDefinition: route.matched?.[route.matched.length - 1]?.path,
           ...collectBrowserInfo()
         }
       }

packages/server/modules/core/services/objects.js

@@ -50,7 +50,11 @@ module.exports = {
     await Objects().insert(insertionObject).onConflict().ignore()
 
     if (closures.length > 0) {
-      await Closures().insert(closures).onConflict().ignore()
+      const batchSize = 10000
+      while (closures.length > 0) {
+        const closuresBatch = closures.splice(0, batchSize) // splice so that we don't take up more memory
+        await Closures().insert(closuresBatch).onConflict().ignore()
+      }
     }
 
     return insertionObject.id

packages/server/modules/core/tests/objects.spec.js

@@ -2,6 +2,7 @@
 /* eslint-disable camelcase */
 const expect = require('chai').expect
 const assert = require('assert')
+const { cloneDeep, times, random, padStart } = require('lodash')
 
 const { beforeEachContext } = require('@/test/hooks')
 const { getAnIdForThisOnePlease } = require('@/test/helpers')
@@ -115,6 +116,21 @@ describe('Objects @core-objects', () => {
     expect(myIds).to.have.lengthOf(objCount_2)
   }).timeout(30000)
 
+  it(`Should create a single object w/ a ton of closures`, async () => {
+    const obj = {
+      ...cloneDeep(sampleObject),
+      __closure: times(200000, (i) => [
+        'testa078f8b935d3e329e9080b' + padStart(i.toString(), 6, '0'),
+        random(1, 10)
+      ]).reduce((obj, [key, value]) => {
+        obj[key] = value
+        return obj
+      }, {})
+    }
+    const id = await createObject(stream.id, obj)
+    expect(id).to.be.ok
+  })
+
   it('Should get a single object', async () => {
     const obj = await getObject({ streamId: stream.id, objectId: sampleCommit.id })
     expect(obj).to.not.be.null

packages/server/modules/cross-server-sync/services/commit.ts

@@ -4,7 +4,7 @@ import { getFrontendOrigin } from '@/modules/shared/helpers/envHelper'
 import { CreateCommentInput } from '@/test/graphql/generated/graphql'
 import { getStreamBranchByName } from '@/modules/core/repositories/branches'
 import { getStream, getStreamCollaborators } from '@/modules/core/repositories/streams'
-import { Roles } from '@speckle/shared'
+import { Roles, timeoutAt } from '@speckle/shared'
 import { addCommitCreatedActivity } from '@/modules/activitystream/services/commitActivity'
 import { createObject } from '@/modules/core/services/objects'
 import { getObject } from '@/modules/core/repositories/objects'
@@ -518,11 +518,27 @@ const loadAllObjectsFromParent = async (
 
   // Iterate over all objects and download them into the DB
   const totalObjectCount = (sourceCommit.totalChildrenCount || 0) + 1
+  const batchSize = 50
+  let batchPromises: Promise<unknown>[] = []
   let processedObjectCount = 1
+
   for await (const obj of objectLoader.getObjectIterator()) {
     const typedObj = obj as ObjectLoaderObject
-    logger.debug(`Processing ${obj.id} - ${processedObjectCount++}/${totalObjectCount}`)
-    await createNewObject(typedObj, targetStreamId, { logger })
+    const work = async () => {
+      const id = `${obj.id} - ${processedObjectCount++}/${totalObjectCount}`
+      logger.debug(`Processing ${id}...`)
+      await Promise.race([
+        createNewObject(typedObj, targetStreamId, { logger }),
+        timeoutAt(60 * 1000, `Object create timed out! - ${id}`)
+      ])
+      logger.debug(`Processed! ${id}`)
+    }
+
+    batchPromises.push(work())
+    if (batchPromises.length >= batchSize) {
+      await Promise.all(batchPromises)
+      batchPromises = []
+    }
   }
 }
 

packages/server/package.json

@@ -27,6 +27,7 @@
     "lint:tsc": "tsc --noEmit",
     "lint:eslint": "eslint . --ext .js,.ts",
     "cli": "cross-env LOG_PRETTY=true NODE_ENV=development ts-node ./modules/cli/index.js",
+    "cli:download:commit": "cross-env LOG_PRETTY=true LOG_LEVEL=debug yarn cli download commit",
     "migrate": "yarn cli db migrate",
     "migrate:test": "cross-env NODE_ENV=test ts-node ./modules/cli/index.js db migrate",
     "gqlgen": "graphql-codegen --config codegen.yml",

workspace.code-workspace

@@ -77,7 +77,7 @@
     "editor.snippetSuggestions": "top",
     "editor.formatOnSave": true,
     "editor.codeActionsOnSave": {
-      "source.fixAll.eslint": true
+      "source.fixAll.eslint": "explicit"
     },
     "editor.defaultFormatter": "esbenp.prettier-vscode",
     "search.useParentIgnoreFiles": true,
@@ -107,6 +107,9 @@
     },
     "[dockercompose]": {
       "editor.defaultFormatter": "esbenp.prettier-vscode"
+    },
+    "[json]": {
+      "editor.defaultFormatter": "esbenp.prettier-vscode"
     }
   },
   "extensions": {