From 6d912e61951fe6292de2c723d51e1cf1d6a606a6 Mon Sep 17 00:00:00 2001 From: Michael Belousov Date: Thu, 21 Sep 2023 13:50:50 -0400 Subject: [PATCH 1/4] do not care about internals in tests, we need several of them --- packages/transformer/.eslintrc.js | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/transformer/.eslintrc.js b/packages/transformer/.eslintrc.js index b6a7a7d3..40bf39c8 100644 --- a/packages/transformer/.eslintrc.js +++ b/packages/transformer/.eslintrc.js @@ -1,26 +1,35 @@ const itwinjsRecommended = require("@itwin/eslint-plugin/dist/configs/itwinjs-recommended"); +/** @type {import("eslint").ESLint.ConfigData} */ module.exports = { rules: { "@itwin/no-internal": [ "warn", { tag: [ - "internal" - ] - } + "internal", + ], + }, ], "@typescript-eslint/naming-convention": [ ...itwinjsRecommended.rules["@typescript-eslint/naming-convention"], { selector: "objectLiteralProperty", format: null, - leadingUnderscore: "allowSingleOrDouble" + leadingUnderscore: "allowSingleOrDouble", }, - ] + ], }, "parserOptions": { "project": "./tsconfig.json" - } + }, + overrides: [ + { + files: "", + rules: { + "@itwin/no-internal": ["off"], + }, + }, + ], }; From 6d41076e6fa0e1e3867ad0f276a7e4573d937495 Mon Sep 17 00:00:00 2001 From: Michael Belousov Date: Fri, 22 Sep 2023 10:38:37 -0400 Subject: [PATCH 2/4] working on new local processChanges of remote imodel test --- .../standalone/IModelTransformerHub.test.ts | 115 +++++++++++++++++- 1 file changed, 112 insertions(+), 3 deletions(-) diff --git a/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts b/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts index 12fd5627..a8b2eb0a 100644 --- a/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts +++ b/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts @@ -4,17 +4,18 @@ *--------------------------------------------------------------------------------------------*/ import { assert, expect } from "chai"; +import * as fs from "fs"; import * as path from "path"; import * as semver from "semver"; import { BisCoreSchema, BriefcaseDb, BriefcaseManager, CategorySelector, DefinitionContainer, DefinitionModel, DefinitionPartition, deleteElementTree, DisplayStyle3d, Element, ElementOwnsChildElements, ElementOwnsExternalSourceAspects, ElementRefersToElements, ExternalSourceAspect, GenericSchema, HubMock, IModelDb, IModelHost, IModelJsFs, IModelJsNative, ModelSelector, NativeLoggerCategory, PhysicalModel, - PhysicalObject, SnapshotDb, SpatialCategory, SpatialViewDefinition, Subject, SubjectOwnsPartitionElements, + PhysicalObject, SnapshotDb, SpatialCategory, SpatialViewDefinition, SQLiteDb, Subject, SubjectOwnsPartitionElements, } from "@itwin/core-backend"; import * as TestUtils from "../TestUtils"; -import { AccessToken, Guid, GuidString, Id64, Id64String, Logger, LogLevel } from "@itwin/core-bentley"; -import { Code, ColorDef, DefinitionElementProps, ElementProps, ExternalSourceAspectProps, IModel, IModelVersion, InformationPartitionElementProps, ModelProps, SpatialViewDefinitionProps, SubCategoryAppearance } from "@itwin/core-common"; +import { AccessToken, DbResult, Guid, GuidString, Id64, Id64String, Logger, LogLevel } from "@itwin/core-bentley"; +import { BriefcaseIdValue, ChangesetFileProps, Code, ColorDef, DefinitionElementProps, ElementProps, ExternalSourceAspectProps, IModel, IModelVersion, InformationPartitionElementProps, ModelProps, SpatialViewDefinitionProps, SubCategoryAppearance } from "@itwin/core-common"; import { Point3d, YawPitchRollAngles } from "@itwin/core-geometry"; import { IModelExporter, IModelImporter, IModelTransformer, TransformerLoggerCategory } from "../../transformer"; import { @@ -1040,5 +1041,113 @@ describe("IModelTransformerHub", () => { await tearDown(); sinon.restore(); }); + + // unskip this and set the env vars "REMOTE_IMODEL_DIR" to a path and "FORK_CHANGSET_ID" to a + // changeset id, to locally test synchronizing an remote iModel to a fork + // You must have used some kind of imodel-downloader tool to have all changesets stored locally + it.only("should test transforming a remote iModel to local", async () => { + const forkChangesetId = process.env.FORK_CHANGESET_ID; + assert(forkChangesetId, "FORK_CHANGSET_ID env var not set"); + + const remoteIModelDir = process.env.REMOTE_IMODEL_DIR; + assert(remoteIModelDir, "This test requires the REMOTE_IMODEL_DIR env variable to be set"); + assert(fs.existsSync(remoteIModelDir), "REMOTE_IMODEL_DIR points to a non existent path"); + const dirContents = await fs.promises.readdir(remoteIModelDir); + assert(dirContents); + + const seedFileName = dirContents.find((f) => f.endsWith(".bim")); + assert(seedFileName, "no seed file (.bim) in REMOTE_IMODEL_DIR"); + const seedPath = path.join(remoteIModelDir, seedFileName); + + const changesetsPath = path.join(remoteIModelDir, "changeSets"); + assert(fs.existsSync(remoteIModelDir), "no changeSets directory in REMOTE_IMODEL_DIR"); + + const changesetsDataPath = path.join(remoteIModelDir, "changeSets.json"); + const changesetsDataText = await fs.promises.readFile(changesetsDataPath, { encoding: "utf8" }); + const changesetsData = JSON.parse(changesetsDataText) as ChangesetFileProps[]; + + let remoteITwinId!: string; + + const seedRawDb = new SQLiteDb(); + seedRawDb.withOpenDb({ dbName: seedPath }, () => { + remoteITwinId = seedRawDb.withSqliteStatement( + "SELECT Data FROM be_Prop WHERE Namespace='be_Db' AND Name='ProjectGuid'", + (stmt) => { + assert(stmt.step() === DbResult.BE_SQLITE_ROW, seedRawDb.nativeDb.getLastError()); + return stmt.getValueGuid(0); + }, + ); + }); + seedRawDb.closeDb(); + + let sourceIModelId: string | undefined; + let targetIModelId: string | undefined; + try { + sourceIModelId = await HubMock.createNewIModel({ + iTwinId: remoteITwinId, + version0: seedPath, + iModelName: "remoteIModelMaster", + }); + + const modelHub = HubMock.findLocalHub(sourceIModelId); + + changesetsData.forEach((changeset) => { + const pathname = path.resolve(changesetsPath, `${changeset.id}.cs`); + const briefcases = modelHub.getBriefcases(); + const briefcaseExists = briefcases.some((b) => b.id === changeset.briefcaseId); + if (!briefcaseExists) { + const MAX_BRIEFCASES = Number(process.env.MAX_REMOTE_IMODEL_BRIEFCASES) || 1000; + for (let i = 0; i < MAX_BRIEFCASES; ++i) { + const newBriefcase = modelHub.acquireNewBriefcaseId("test"); + if (newBriefcase >= changeset.briefcaseId) + break; + } + } + // NOTE: local download seems to be missing the "size" property + modelHub.addChangeset({ ...changeset, pathname }); + }); + + // FIXME: technically don't need to open it at all, just download it + const forkPointDb = await HubWrappers.downloadAndOpenBriefcase({ + accessToken, + iTwinId: remoteITwinId, + iModelId: sourceIModelId, + // FIXME: I want _exact_ + asOf: { afterChangeSetId: forkChangesetId }, + briefcaseId: BriefcaseIdValue.Unassigned, + }); + + targetIModelId = await IModelHost.hubAccess.createNewIModel({ + iTwinId: remoteITwinId, + version0: forkPointDb.pathName, + iModelName: "amalgam-model", + }); + + forkPointDb.close(); + + const branchDb = await HubWrappers.downloadAndOpenBriefcase({ accessToken, iTwinId, iModelId: targetIModelId }); + + const forkInitializer = new IModelTransformer(forkPointDb, branchDb, { wasSourceIModelCopiedToTarget: true }); + await forkInitializer.processAll(); + forkInitializer.dispose(); + + const masterDb = await HubWrappers.downloadAndOpenBriefcase({ + accessToken, + iTwinId: remoteITwinId, + iModelId: sourceIModelId, + briefcaseId: BriefcaseIdValue.Unassigned, + }); + + const forwardSyncer = new IModelTransformer(masterDb, branchDb); + forwardSyncer.processChanges({ accessToken, startChangeset: forkChangesetId }); + + + masterDb.close(); + branchDb.close(); + } finally { + sourceIModelId && await IModelHost.hubAccess.deleteIModel({ iTwinId: remoteITwinId, iModelId: sourceIModelId }); + targetIModelId && await IModelHost.hubAccess.deleteIModel({ iTwinId: remoteITwinId, iModelId: targetIModelId }); + } + }); }); From 2a9edb5848b4e7849a61c35118653726ebb6af9b Mon Sep 17 00:00:00 2001 From: Michael Belousov Date: Fri, 22 Sep 2023 11:25:34 -0400 Subject: [PATCH 3/4] fix db locking --- .../standalone/IModelTransformerHub.test.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts b/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts index a8b2eb0a..f4b10b36 100644 --- a/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts +++ b/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts @@ -1042,12 +1042,12 @@ describe("IModelTransformerHub", () => { sinon.restore(); }); - // unskip this and set the env vars "REMOTE_IMODEL_DIR" to a path and "FORK_CHANGSET_ID" to a + // unskip this and set the env vars "REMOTE_IMODEL_DIR" to a path and "FORK_CHANGESET_ID" to a // changeset id, to locally test synchronizing an remote iModel to a fork // You must have used some kind of imodel-downloader tool to have all changesets stored locally it.only("should test transforming a remote iModel to local", async () => { const forkChangesetId = process.env.FORK_CHANGESET_ID; - assert(forkChangesetId, "FORK_CHANGSET_ID env var not set"); + assert(forkChangesetId, "FORK_CHANGESET_ID env var not set"); const remoteIModelDir = process.env.REMOTE_IMODEL_DIR; assert(remoteIModelDir, "This test requires the REMOTE_IMODEL_DIR env variable to be set"); @@ -1087,6 +1087,7 @@ describe("IModelTransformerHub", () => { iTwinId: remoteITwinId, version0: seedPath, iModelName: "remoteIModelMaster", + noLocks: true, }); const modelHub = HubMock.findLocalHub(sourceIModelId); @@ -1121,11 +1122,10 @@ describe("IModelTransformerHub", () => { iTwinId: remoteITwinId, version0: forkPointDb.pathName, iModelName: "amalgam-model", + noLocks: true, }); - forkPointDb.close(); - - const branchDb = await HubWrappers.downloadAndOpenBriefcase({ accessToken, iTwinId, iModelId: targetIModelId }); + const branchDb = await HubWrappers.downloadAndOpenBriefcase({ accessToken, iTwinId: remoteITwinId, iModelId: targetIModelId }); const forkInitializer = new IModelTransformer(forkPointDb, branchDb, { wasSourceIModelCopiedToTarget: true }); await forkInitializer.processAll(); @@ -1135,13 +1135,13 @@ describe("IModelTransformerHub", () => { accessToken, iTwinId: remoteITwinId, iModelId: sourceIModelId, - briefcaseId: BriefcaseIdValue.Unassigned, + //briefcaseId: BriefcaseIdValue.Unassigned, }); - const forwardSyncer = new IModelTransformer(masterDb, branchDb); - forwardSyncer.processChanges({ accessToken, startChangeset: forkChangesetId }); - + const forwardSyncer = new IModelTransformer(masterDb, branchDb,); + await forwardSyncer.processChanges({ accessToken, startChangeset: { id: forkChangesetId } }); + forkPointDb.close(); masterDb.close(); branchDb.close(); } finally { From 9648022eecea8a81e2b8d9c5e72d667fb6f911ab Mon Sep 17 00:00:00 2001 From: Michael Belousov Date: Fri, 22 Sep 2023 12:29:11 -0400 Subject: [PATCH 4/4] restrict no-internal override to src/test --- packages/transformer/.eslintrc.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/transformer/.eslintrc.js b/packages/transformer/.eslintrc.js index 40bf39c8..a3411f7a 100644 --- a/packages/transformer/.eslintrc.js +++ b/packages/transformer/.eslintrc.js @@ -25,7 +25,7 @@ module.exports = { }, overrides: [ { - files: "", + files: "./src/test", rules: { "@itwin/no-internal": ["off"], },