From a4118e9eb87c1bfa82a1d79c75c3c5b07003507a Mon Sep 17 00:00:00 2001 From: beyondkmp Date: Thu, 31 Oct 2024 03:50:12 +0800 Subject: [PATCH 01/14] refactor smartUnpack --- packages/app-builder-lib/src/asar/asarUtil.ts | 2 +- .../src/asar/unpackDetector.ts | 71 +++---------------- .../src/util/NodeModuleCopyHelper.ts | 3 +- packages/app-builder-lib/src/util/filter.ts | 2 +- packages/builder-util/src/fs.ts | 3 +- 5 files changed, 16 insertions(+), 65 deletions(-) diff --git a/packages/app-builder-lib/src/asar/asarUtil.ts b/packages/app-builder-lib/src/asar/asarUtil.ts index 386dc993907..244d1dc4b42 100644 --- a/packages/app-builder-lib/src/asar/asarUtil.ts +++ b/packages/app-builder-lib/src/asar/asarUtil.ts @@ -119,7 +119,7 @@ export class AsarPackager { for await (const fileSet of fileSets) { if (this.config.options.smartUnpack !== false) { - detectUnpackedDirs(fileSet, unpackedPaths, this.config.defaultDestination) + detectUnpackedDirs(fileSet, unpackedPaths) } for (let i = 0; i < fileSet.files.length; i++) { const file = fileSet.files[i] diff --git a/packages/app-builder-lib/src/asar/unpackDetector.ts b/packages/app-builder-lib/src/asar/unpackDetector.ts index 177f6281bea..c4a6d0d9f83 100644 --- a/packages/app-builder-lib/src/asar/unpackDetector.ts +++ b/packages/app-builder-lib/src/asar/unpackDetector.ts @@ -1,18 +1,7 @@ -import { log } from "builder-util" +import { log, FilterStats } from "builder-util" import { isBinaryFileSync } from "isbinaryfile" import * as path from "path" -import { NODE_MODULES_PATTERN } from "../fileTransformer" -import { getDestinationPath, ResolvedFileSet } from "../util/appFileCopier" - -function addValue(map: Map>, key: string, value: string) { - let list = map.get(key) - if (list == null) { - list = [value] - map.set(key, list) - } else { - list.push(value) - } -} +import { ResolvedFileSet } from "../util/appFileCopier" export function isLibOrExe(file: string): boolean { // https://github.com/electron-userland/electron-builder/issues/3038 @@ -20,64 +9,24 @@ export function isLibOrExe(file: string): boolean { } /** @internal */ -export function detectUnpackedDirs(fileSet: ResolvedFileSet, autoUnpackDirs: Set, defaultDestination: string) { - const dirToCreate = new Map>() +export function detectUnpackedDirs(fileSet: ResolvedFileSet, autoUnpackDirs: Set) { const metadata = fileSet.metadata - function addParents(child: string, root: string) { - child = path.dirname(child) - if (autoUnpackDirs.has(child)) { - return - } - - do { - autoUnpackDirs.add(child) - const p = path.dirname(child) - // create parent dir to be able to copy file later without directory existence check - addValue(dirToCreate, p, path.basename(child)) - - if (child === root || p === root || autoUnpackDirs.has(p)) { - break - } - child = p - } while (true) - - autoUnpackDirs.add(root) - } - for (let i = 0, n = fileSet.files.length; i < n; i++) { const file = fileSet.files[i] - const index = file.lastIndexOf(NODE_MODULES_PATTERN) - if (index < 0) { + const stat: FilterStats = metadata.get(file)! + if (stat.destNodeModulesPath) { continue } - let nextSlashIndex = file.indexOf(path.sep, index + NODE_MODULES_PATTERN.length + 1) - if (nextSlashIndex < 0) { - continue - } - - if (file[index + NODE_MODULES_PATTERN.length] === "@") { - nextSlashIndex = file.indexOf(path.sep, nextSlashIndex + 1) - } - - if (!metadata.get(file)!.isFile()) { - continue - } - - const packageDir = file.substring(0, nextSlashIndex) - const packageDirPathInArchive = path.relative(defaultDestination, getDestinationPath(packageDir, fileSet)) - const pathInArchive = path.relative(defaultDestination, getDestinationPath(file, fileSet)) - if (autoUnpackDirs.has(packageDirPathInArchive)) { - // if package dir is unpacked, any file also unpacked - addParents(pathInArchive, packageDirPathInArchive) + if (!stat.isFile()) { continue } // https://github.com/electron-userland/electron-builder/issues/2679 let shouldUnpack = false // ffprobe-static and ffmpeg-static are known packages to always unpack - const moduleName = path.basename(packageDir) + const moduleName = stat.moduleName const fileBaseName = path.basename(file) const hasExtension = path.extname(fileBaseName) if (moduleName === "ffprobe-static" || moduleName === "ffmpeg-static" || isLibOrExe(file)) { @@ -91,9 +40,9 @@ export function detectUnpackedDirs(fileSet: ResolvedFileSet, autoUnpackDirs: Set } if (log.isDebugEnabled) { - log.debug({ file: pathInArchive, reason: "contains executable code" }, "not packed into asar archive") + log.debug({ file: stat.destNodeModulesPath, reason: "contains executable code" }, "not packed into asar archive") } - - addParents(pathInArchive, packageDirPathInArchive) + autoUnpackDirs.add(fileSet.destination) + break } } diff --git a/packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts b/packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts index 827c0e8d097..031e9f08fb1 100644 --- a/packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts +++ b/packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts @@ -114,7 +114,8 @@ export class NodeModuleCopyHelper extends FileCopyHelper { } return lstat(filePath).then((stat: FilterStats) => { - stat.relativeNodeModulesPath = path.join("node_modules", moduleName, path.relative(depPath, filePath)) + stat.destNodeModulesPath = path.join("node_modules", moduleName, path.relative(depPath, filePath)) + stat.moduleName = moduleName if (filter != null && !filter(filePath, stat)) { return null } diff --git a/packages/app-builder-lib/src/util/filter.ts b/packages/app-builder-lib/src/util/filter.ts index c831d35a79b..990ab162292 100644 --- a/packages/app-builder-lib/src/util/filter.ts +++ b/packages/app-builder-lib/src/util/filter.ts @@ -24,7 +24,7 @@ function ensureEndSlash(s: string) { } function getRelativePath(file: string, srcWithEndSlash: string, stat: FilterStats) { - let relative = stat.relativeNodeModulesPath || file.substring(srcWithEndSlash.length) + let relative = stat.destNodeModulesPath || file.substring(srcWithEndSlash.length) if (path.sep === "\\") { if (relative.startsWith("\\")) { // windows problem: double backslash, the above substring call removes root path with a single slash, so here can me some leftovers diff --git a/packages/builder-util/src/fs.ts b/packages/builder-util/src/fs.ts index d53bf812483..6834f866c75 100644 --- a/packages/builder-util/src/fs.ts +++ b/packages/builder-util/src/fs.ts @@ -22,7 +22,8 @@ export type FileTransformer = (file: string) => Promise Date: Thu, 31 Oct 2024 09:48:10 +0800 Subject: [PATCH 02/14] add destNodeModulesDirPath --- packages/app-builder-lib/src/asar/unpackDetector.ts | 6 +++--- packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts | 5 +++-- packages/app-builder-lib/src/util/appFileCopier.ts | 2 +- packages/app-builder-lib/src/util/filter.ts | 2 +- packages/builder-util/src/fs.ts | 3 ++- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/app-builder-lib/src/asar/unpackDetector.ts b/packages/app-builder-lib/src/asar/unpackDetector.ts index c4a6d0d9f83..d6087e2a975 100644 --- a/packages/app-builder-lib/src/asar/unpackDetector.ts +++ b/packages/app-builder-lib/src/asar/unpackDetector.ts @@ -15,7 +15,7 @@ export function detectUnpackedDirs(fileSet: ResolvedFileSet, autoUnpackDirs: Set for (let i = 0, n = fileSet.files.length; i < n; i++) { const file = fileSet.files[i] const stat: FilterStats = metadata.get(file)! - if (stat.destNodeModulesPath) { + if (!stat.destNodeModulesFilePath) { continue } @@ -40,9 +40,9 @@ export function detectUnpackedDirs(fileSet: ResolvedFileSet, autoUnpackDirs: Set } if (log.isDebugEnabled) { - log.debug({ file: stat.destNodeModulesPath, reason: "contains executable code" }, "not packed into asar archive") + log.debug({ file: stat.destNodeModulesFilePath, reason: "contains executable code" }, "not packed into asar archive") } - autoUnpackDirs.add(fileSet.destination) + autoUnpackDirs.add(stat.destNodeModulesDirPath!) break } } diff --git a/packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts b/packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts index 031e9f08fb1..f4da419990d 100644 --- a/packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts +++ b/packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts @@ -49,7 +49,7 @@ export class NodeModuleCopyHelper extends FileCopyHelper { super(matcher, matcher.isEmpty() ? null : matcher.createFilter(), packager) } - async collectNodeModules(moduleInfo: NodeModuleInfo, nodeModuleExcludedExts: Array): Promise> { + async collectNodeModules(moduleInfo: NodeModuleInfo, nodeModuleExcludedExts: Array, destination:string): Promise> { const filter = this.filter const metadata = this.metadata @@ -114,8 +114,9 @@ export class NodeModuleCopyHelper extends FileCopyHelper { } return lstat(filePath).then((stat: FilterStats) => { - stat.destNodeModulesPath = path.join("node_modules", moduleName, path.relative(depPath, filePath)) stat.moduleName = moduleName + stat.destNodeModulesDirPath = destination + stat.destNodeModulesFilePath = path.join(destination, path.relative(depPath, filePath)) if (filter != null && !filter(filePath, stat)) { return null } diff --git a/packages/app-builder-lib/src/util/appFileCopier.ts b/packages/app-builder-lib/src/util/appFileCopier.ts index ec21b9faf71..85fb851210b 100644 --- a/packages/app-builder-lib/src/util/appFileCopier.ts +++ b/packages/app-builder-lib/src/util/appFileCopier.ts @@ -192,7 +192,7 @@ export async function computeNodeModuleFileSets(platformPackager: PlatformPackag const source = dep.dir const matcher = new FileMatcher(source, destination, mainMatcher.macroExpander, mainMatcher.patterns) const copier = new NodeModuleCopyHelper(matcher, platformPackager.info) - const files = await copier.collectNodeModules(dep, nodeModuleExcludedExts) + const files = await copier.collectNodeModules(dep, nodeModuleExcludedExts, destination) result[index++] = validateFileSet({ src: source, destination, files, metadata: copier.metadata }) if (dep.conflictDependency) { diff --git a/packages/app-builder-lib/src/util/filter.ts b/packages/app-builder-lib/src/util/filter.ts index 990ab162292..37bbe1e937f 100644 --- a/packages/app-builder-lib/src/util/filter.ts +++ b/packages/app-builder-lib/src/util/filter.ts @@ -24,7 +24,7 @@ function ensureEndSlash(s: string) { } function getRelativePath(file: string, srcWithEndSlash: string, stat: FilterStats) { - let relative = stat.destNodeModulesPath || file.substring(srcWithEndSlash.length) + let relative = stat.destNodeModulesFilePath || file.substring(srcWithEndSlash.length) if (path.sep === "\\") { if (relative.startsWith("\\")) { // windows problem: double backslash, the above substring call removes root path with a single slash, so here can me some leftovers diff --git a/packages/builder-util/src/fs.ts b/packages/builder-util/src/fs.ts index 6834f866c75..0a77d325d8c 100644 --- a/packages/builder-util/src/fs.ts +++ b/packages/builder-util/src/fs.ts @@ -22,7 +22,8 @@ export type FileTransformer = (file: string) => Promise Date: Thu, 31 Oct 2024 09:51:38 +0800 Subject: [PATCH 03/14] format code --- packages/app-builder-lib/src/asar/unpackDetector.ts | 5 ++--- packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/app-builder-lib/src/asar/unpackDetector.ts b/packages/app-builder-lib/src/asar/unpackDetector.ts index d6087e2a975..b7a48d0eb6c 100644 --- a/packages/app-builder-lib/src/asar/unpackDetector.ts +++ b/packages/app-builder-lib/src/asar/unpackDetector.ts @@ -15,7 +15,7 @@ export function detectUnpackedDirs(fileSet: ResolvedFileSet, autoUnpackDirs: Set for (let i = 0, n = fileSet.files.length; i < n; i++) { const file = fileSet.files[i] const stat: FilterStats = metadata.get(file)! - if (!stat.destNodeModulesFilePath) { + if (!stat.destNodeModulesDirPath) { continue } @@ -42,7 +42,6 @@ export function detectUnpackedDirs(fileSet: ResolvedFileSet, autoUnpackDirs: Set if (log.isDebugEnabled) { log.debug({ file: stat.destNodeModulesFilePath, reason: "contains executable code" }, "not packed into asar archive") } - autoUnpackDirs.add(stat.destNodeModulesDirPath!) - break + autoUnpackDirs.add(stat.destNodeModulesDirPath) } } diff --git a/packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts b/packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts index f4da419990d..a76230fb3a1 100644 --- a/packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts +++ b/packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts @@ -49,7 +49,7 @@ export class NodeModuleCopyHelper extends FileCopyHelper { super(matcher, matcher.isEmpty() ? null : matcher.createFilter(), packager) } - async collectNodeModules(moduleInfo: NodeModuleInfo, nodeModuleExcludedExts: Array, destination:string): Promise> { + async collectNodeModules(moduleInfo: NodeModuleInfo, nodeModuleExcludedExts: Array, destination: string): Promise> { const filter = this.filter const metadata = this.metadata From 6591c425b6507b977bf24ebb17b7e08b46bafb41 Mon Sep 17 00:00:00 2001 From: beyondkmp Date: Thu, 31 Oct 2024 10:57:30 +0800 Subject: [PATCH 04/14] change to relative destination --- packages/app-builder-lib/src/util/appFileCopier.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/app-builder-lib/src/util/appFileCopier.ts b/packages/app-builder-lib/src/util/appFileCopier.ts index 85fb851210b..c31adda3e91 100644 --- a/packages/app-builder-lib/src/util/appFileCopier.ts +++ b/packages/app-builder-lib/src/util/appFileCopier.ts @@ -192,7 +192,7 @@ export async function computeNodeModuleFileSets(platformPackager: PlatformPackag const source = dep.dir const matcher = new FileMatcher(source, destination, mainMatcher.macroExpander, mainMatcher.patterns) const copier = new NodeModuleCopyHelper(matcher, platformPackager.info) - const files = await copier.collectNodeModules(dep, nodeModuleExcludedExts, destination) + const files = await copier.collectNodeModules(dep, nodeModuleExcludedExts, path.relative(mainMatcher.to, destination)) result[index++] = validateFileSet({ src: source, destination, files, metadata: copier.metadata }) if (dep.conflictDependency) { From 8a55d4abf91ddc04bb80c0829e4153a86ab4d859 Mon Sep 17 00:00:00 2001 From: beyondkmp Date: Thu, 31 Oct 2024 11:12:55 +0800 Subject: [PATCH 05/14] fix ut --- packages/app-builder-lib/src/asar/unpackDetector.ts | 2 +- test/src/BuildTest.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/app-builder-lib/src/asar/unpackDetector.ts b/packages/app-builder-lib/src/asar/unpackDetector.ts index b7a48d0eb6c..940d82a3283 100644 --- a/packages/app-builder-lib/src/asar/unpackDetector.ts +++ b/packages/app-builder-lib/src/asar/unpackDetector.ts @@ -15,7 +15,7 @@ export function detectUnpackedDirs(fileSet: ResolvedFileSet, autoUnpackDirs: Set for (let i = 0, n = fileSet.files.length; i < n; i++) { const file = fileSet.files[i] const stat: FilterStats = metadata.get(file)! - if (!stat.destNodeModulesDirPath) { + if (!stat.destNodeModulesDirPath || autoUnpackDirs.has(stat.destNodeModulesDirPath)) { continue } diff --git a/test/src/BuildTest.ts b/test/src/BuildTest.ts index 6684bc96148..2209f52fc7b 100644 --- a/test/src/BuildTest.ts +++ b/test/src/BuildTest.ts @@ -283,8 +283,7 @@ test.ifLinuxOrDevMac("beforeBuild", () => { ) }) -// https://github.com/electron-userland/electron-builder/issues/1738 -test.ifDevOrLinuxCi("win smart unpack", () => { +test("win smart unpack", () => { // test onNodeModuleFile hook const nodeModuleFiles: Array = [] let p = "" From 4450dc996637790b8a5440179948d467e981f08f Mon Sep 17 00:00:00 2001 From: beyondkmp Date: Thu, 31 Oct 2024 11:30:22 +0800 Subject: [PATCH 06/14] add replace --- packages/app-builder-lib/src/asar/unpackDetector.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/app-builder-lib/src/asar/unpackDetector.ts b/packages/app-builder-lib/src/asar/unpackDetector.ts index 940d82a3283..a1da6185428 100644 --- a/packages/app-builder-lib/src/asar/unpackDetector.ts +++ b/packages/app-builder-lib/src/asar/unpackDetector.ts @@ -15,7 +15,7 @@ export function detectUnpackedDirs(fileSet: ResolvedFileSet, autoUnpackDirs: Set for (let i = 0, n = fileSet.files.length; i < n; i++) { const file = fileSet.files[i] const stat: FilterStats = metadata.get(file)! - if (!stat.destNodeModulesDirPath || autoUnpackDirs.has(stat.destNodeModulesDirPath)) { + if (!stat.destNodeModulesDirPath || autoUnpackDirs.has(stat.destNodeModulesDirPath.replace(/\\/g, "/"))) { continue } @@ -42,6 +42,6 @@ export function detectUnpackedDirs(fileSet: ResolvedFileSet, autoUnpackDirs: Set if (log.isDebugEnabled) { log.debug({ file: stat.destNodeModulesFilePath, reason: "contains executable code" }, "not packed into asar archive") } - autoUnpackDirs.add(stat.destNodeModulesDirPath) + autoUnpackDirs.add(stat.destNodeModulesDirPath.replace(/\\/g, "/")) } } From 1de53eabb862d1c2da812c1f549744de427f323b Mon Sep 17 00:00:00 2001 From: beyondkmp Date: Thu, 31 Oct 2024 11:46:59 +0800 Subject: [PATCH 07/14] add debug info --- test/src/BuildTest.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/src/BuildTest.ts b/test/src/BuildTest.ts index 2209f52fc7b..50e04be2b3d 100644 --- a/test/src/BuildTest.ts +++ b/test/src/BuildTest.ts @@ -293,6 +293,7 @@ test("win smart unpack", () => { config: { npmRebuild: true, onNodeModuleFile: file => { + console.log("test is her,", p, file, nodeModuleFiles) const name = toSystemIndependentPath(path.relative(p, file)) if (!name.startsWith(".") && !name.endsWith(".dll") && name.includes(".")) { nodeModuleFiles.push(name) From b9fcfedc789d3f5434d320b566f6e2954a752a04 Mon Sep 17 00:00:00 2001 From: beyondkmp Date: Thu, 31 Oct 2024 11:53:26 +0800 Subject: [PATCH 08/14] add more debug info --- test/src/BuildTest.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/BuildTest.ts b/test/src/BuildTest.ts index 50e04be2b3d..14e95d9d501 100644 --- a/test/src/BuildTest.ts +++ b/test/src/BuildTest.ts @@ -293,8 +293,8 @@ test("win smart unpack", () => { config: { npmRebuild: true, onNodeModuleFile: file => { - console.log("test is her,", p, file, nodeModuleFiles) const name = toSystemIndependentPath(path.relative(p, file)) + console.log("test is her,", p, file, nodeModuleFiles, name) if (!name.startsWith(".") && !name.endsWith(".dll") && name.includes(".")) { nodeModuleFiles.push(name) } From 71fd764b5c437830db1e9fe1a053bfb7d137b5b7 Mon Sep 17 00:00:00 2001 From: beyondkmp Date: Thu, 31 Oct 2024 14:25:40 +0800 Subject: [PATCH 09/14] fix ut --- packages/app-builder-lib/src/asar/unpackDetector.ts | 4 ++-- test/src/BuildTest.ts | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/app-builder-lib/src/asar/unpackDetector.ts b/packages/app-builder-lib/src/asar/unpackDetector.ts index a1da6185428..940d82a3283 100644 --- a/packages/app-builder-lib/src/asar/unpackDetector.ts +++ b/packages/app-builder-lib/src/asar/unpackDetector.ts @@ -15,7 +15,7 @@ export function detectUnpackedDirs(fileSet: ResolvedFileSet, autoUnpackDirs: Set for (let i = 0, n = fileSet.files.length; i < n; i++) { const file = fileSet.files[i] const stat: FilterStats = metadata.get(file)! - if (!stat.destNodeModulesDirPath || autoUnpackDirs.has(stat.destNodeModulesDirPath.replace(/\\/g, "/"))) { + if (!stat.destNodeModulesDirPath || autoUnpackDirs.has(stat.destNodeModulesDirPath)) { continue } @@ -42,6 +42,6 @@ export function detectUnpackedDirs(fileSet: ResolvedFileSet, autoUnpackDirs: Set if (log.isDebugEnabled) { log.debug({ file: stat.destNodeModulesFilePath, reason: "contains executable code" }, "not packed into asar archive") } - autoUnpackDirs.add(stat.destNodeModulesDirPath.replace(/\\/g, "/")) + autoUnpackDirs.add(stat.destNodeModulesDirPath) } } diff --git a/test/src/BuildTest.ts b/test/src/BuildTest.ts index 14e95d9d501..bbfcd872739 100644 --- a/test/src/BuildTest.ts +++ b/test/src/BuildTest.ts @@ -283,18 +283,17 @@ test.ifLinuxOrDevMac("beforeBuild", () => { ) }) -test("win smart unpack", () => { +test.ifDevOrWinCi("win smart unpack", () => { // test onNodeModuleFile hook const nodeModuleFiles: Array = [] - let p = "" return app( { targets: Platform.WINDOWS.createTarget(DIR_TARGET), config: { npmRebuild: true, onNodeModuleFile: file => { - const name = toSystemIndependentPath(path.relative(p, file)) - console.log("test is her,", p, file, nodeModuleFiles, name) + const index = file.indexOf(path.sep + "node_modules"+ path.sep) + const name = toSystemIndependentPath(file.slice(index + 1)) if (!name.startsWith(".") && !name.endsWith(".dll") && name.includes(".")) { nodeModuleFiles.push(name) } @@ -305,8 +304,8 @@ test("win smart unpack", () => { }, }, { + isInstallDepsBefore:true, projectDirCreated: projectDir => { - p = projectDir return packageJson(it => { it.dependencies = { debug: "3.1.0", From b6d5aa3b7eddde15e90c6a96b996ff33236c7136 Mon Sep 17 00:00:00 2001 From: beyondkmp Date: Thu, 31 Oct 2024 14:41:24 +0800 Subject: [PATCH 10/14] add local module ut --- test/snapshots/BuildTest.js.snap | 249 +++++++++++++++++++++++++++++++ test/src/BuildTest.ts | 30 +++- 2 files changed, 278 insertions(+), 1 deletion(-) diff --git a/test/snapshots/BuildTest.js.snap b/test/snapshots/BuildTest.js.snap index 414198dd780..7c0f810249f 100644 --- a/test/snapshots/BuildTest.js.snap +++ b/test/snapshots/BuildTest.js.snap @@ -4920,6 +4920,255 @@ exports[`relative index 1`] = ` } `; +exports[`smart unpack local module with dll file 1`] = ` +{ + "win": [], +} +`; + +exports[`smart unpack local module with dll file 2`] = ` +{ + "files": { + "app": { + "files": { + "package.json": { + "files": { + "readme.md": { + "size": "", + }, + }, + }, + "readme.md": { + "size": "", + }, + }, + }, + "index.html": { + "size": "", + }, + "index.js": { + "size": "", + }, + "node_modules": { + "files": { + "debug": { + "files": { + "LICENSE": { + "size": "", + }, + "Makefile": { + "size": "", + }, + "node.js": { + "size": "", + }, + "package.json": { + "size": "", + }, + "src": { + "files": { + "browser.js": { + "size": "", + }, + "debug.js": { + "size": "", + }, + "index.js": { + "size": "", + }, + "node.js": { + "size": "", + }, + }, + }, + }, + }, + "edge-cs": { + "files": { + "LICENSE.txt": { + "size": "", + "unpacked": true, + }, + "lib": { + "files": { + "bootstrap": { + "files": { + "Dummy.cs": { + "size": "", + "unpacked": true, + }, + "project.json": { + "size": "", + "unpacked": true, + }, + }, + "unpacked": true, + }, + "edge-cs.dll": { + "size": "", + "unpacked": true, + }, + "edge-cs.js": { + "size": "", + "unpacked": true, + }, + }, + "unpacked": true, + }, + "package.json": { + "size": "", + "unpacked": true, + }, + "src": { + "files": { + "Edge.js.CSharp": { + "files": { + "EdgeCompiler.cs": { + "size": "", + "unpacked": true, + }, + "gulpfile.js": { + "size": "", + "unpacked": true, + }, + "package.json": { + "size": "", + "unpacked": true, + }, + "project.json": { + "size": "", + "unpacked": true, + }, + }, + "unpacked": true, + }, + "edge-cs": { + "files": { + "EdgeCompiler.cs": { + "size": "", + "unpacked": true, + }, + "Properties": { + "files": { + "AssemblyInfo.cs": { + "size": "", + "unpacked": true, + }, + }, + "unpacked": true, + }, + }, + "unpacked": true, + }, + }, + "unpacked": true, + }, + "tools": { + "files": { + "install.js": { + "size": "", + "unpacked": true, + }, + }, + "unpacked": true, + }, + }, + "unpacked": true, + }, + "foo": { + "files": { + "package.json": { + "size": "", + "unpacked": true, + }, + "test.dll": { + "size": "", + "unpacked": true, + }, + }, + "unpacked": true, + }, + "ms": { + "files": { + "index.js": { + "size": "", + }, + "license.md": { + "size": "", + }, + "package.json": { + "size": "", + }, + }, + }, + }, + }, + "package.json": { + "size": "", + }, + }, +} +`; + +exports[`smart unpack local module with dll file 3`] = ` +[ + "app.asar", + { + "content": "{"name":"foo","version":"9.0.0","main":"index.js","license":"MIT","dependencies":{"ms":"2.0.0"}}", + "name": "app.asar.unpacked/node_modules/foo/package.json", + }, + "app.asar.unpacked/node_modules/foo/test.dll", + "app.asar.unpacked/node_modules/edge-cs/LICENSE.txt", + { + "content": "{ + "name": "edge-cs", + "author": { + "name": "Tomasz Janczuk ", + "url": "http://tomasz.janczuk.org", + "twitter": "tjanczuk" + }, + "version": "1.2.1", + "description": "Edge-cs: run C# and node.js code in-process with edge.js", + "main": "./lib/edge-cs.js", + "engines": { + "node": ">= 0.8" + }, + "licenses": [ + { + "type": "Apache", + "url": "http://www.apache.org/licenses/LICENSE-2.0" + } + ], + "dependencies": {}, + "homepage": "http://tjanczuk.github.com/edge", + "repository": { + "type": "git", + "url": "git@github.com:tjanczuk/edge-cs.git" + } +}", + "name": "app.asar.unpacked/node_modules/edge-cs/package.json", + }, + "app.asar.unpacked/node_modules/edge-cs/tools/install.js", + "app.asar.unpacked/node_modules/edge-cs/src/edge-cs/EdgeCompiler.cs", + "app.asar.unpacked/node_modules/edge-cs/src/edge-cs/Properties/AssemblyInfo.cs", + "app.asar.unpacked/node_modules/edge-cs/src/Edge.js.CSharp/EdgeCompiler.cs", + "app.asar.unpacked/node_modules/edge-cs/src/Edge.js.CSharp/gulpfile.js", + { + "content": "{ + "devDependencies": { + "gulp": "3.9.0" + } +}", + "name": "app.asar.unpacked/node_modules/edge-cs/src/Edge.js.CSharp/package.json", + }, + "app.asar.unpacked/node_modules/edge-cs/src/Edge.js.CSharp/project.json", + "app.asar.unpacked/node_modules/edge-cs/lib/edge-cs.dll", + "app.asar.unpacked/node_modules/edge-cs/lib/edge-cs.js", + "app.asar.unpacked/node_modules/edge-cs/lib/bootstrap/Dummy.cs", + "app.asar.unpacked/node_modules/edge-cs/lib/bootstrap/project.json", +] +`; + exports[`win smart unpack 1`] = ` { "win": [], diff --git a/test/src/BuildTest.ts b/test/src/BuildTest.ts index bbfcd872739..6d894824222 100644 --- a/test/src/BuildTest.ts +++ b/test/src/BuildTest.ts @@ -2,7 +2,7 @@ import { checkBuildRequestOptions } from "app-builder-lib" import { doMergeConfigs } from "app-builder-lib/out/util/config/config" import { Arch, createTargets, DIR_TARGET, Platform } from "electron-builder" import { promises as fs } from "fs" -import { outputJson } from "fs-extra" +import { outputJson ,outputFile} from "fs-extra" import * as path from "path" import { createYargs } from "electron-builder/out/builder" import { app, appTwo, appTwoThrows, assertPack, linuxDirTarget, modifyPackageJson, packageJson, toSystemIndependentPath } from "./helpers/packTester" @@ -323,6 +323,34 @@ test.ifDevOrWinCi("win smart unpack", () => { )() }) +test.ifDevOrWinCi("smart unpack local module with dll file", () => { + return app( + { + targets: Platform.WINDOWS.createTarget(DIR_TARGET), + }, + { + isInstallDepsBefore:true, + projectDirCreated: async (projectDir, tmpDir) => { + const tempDir = await tmpDir.getTempDir() + let localPath = path.join(tempDir, "foo") + await outputFile(path.join(localPath, "package.json"), `{"name":"foo","version":"9.0.0","main":"index.js","license":"MIT","dependencies":{"ms":"2.0.0"}}`) + await outputFile(path.join(localPath, "test.dll"), `test`) + await modifyPackageJson(projectDir, data => { + data.dependencies = { + debug: "3.1.0", + "edge-cs": "1.2.1", + foo: `file:${localPath}`, + } + }) + }, + packed: async context => { + await verifySmartUnpack(context.getResources(Platform.WINDOWS)) + }, + } + )() +}) + + // https://github.com/electron-userland/electron-builder/issues/1738 test.ifDevOrLinuxCi( "posix smart unpack", From 4251e08d3c37b584907d885ae2b9a1cae3768198 Mon Sep 17 00:00:00 2001 From: beyondkmp Date: Thu, 31 Oct 2024 15:18:07 +0800 Subject: [PATCH 11/14] add comments --- .../app-builder-lib/src/asar/unpackDetector.ts | 6 +++--- .../src/util/NodeModuleCopyHelper.ts | 4 ++-- packages/app-builder-lib/src/util/filter.ts | 2 +- packages/builder-util/src/fs.ts | 14 ++++++++++---- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/app-builder-lib/src/asar/unpackDetector.ts b/packages/app-builder-lib/src/asar/unpackDetector.ts index 940d82a3283..649b48a741f 100644 --- a/packages/app-builder-lib/src/asar/unpackDetector.ts +++ b/packages/app-builder-lib/src/asar/unpackDetector.ts @@ -15,7 +15,7 @@ export function detectUnpackedDirs(fileSet: ResolvedFileSet, autoUnpackDirs: Set for (let i = 0, n = fileSet.files.length; i < n; i++) { const file = fileSet.files[i] const stat: FilterStats = metadata.get(file)! - if (!stat.destNodeModulesDirPath || autoUnpackDirs.has(stat.destNodeModulesDirPath)) { + if (!stat.moduleRootPath || autoUnpackDirs.has(stat.moduleRootPath)) { continue } @@ -40,8 +40,8 @@ export function detectUnpackedDirs(fileSet: ResolvedFileSet, autoUnpackDirs: Set } if (log.isDebugEnabled) { - log.debug({ file: stat.destNodeModulesFilePath, reason: "contains executable code" }, "not packed into asar archive") + log.debug({ file: stat.moduleFullFilePath, reason: "contains executable code" }, "not packed into asar archive") } - autoUnpackDirs.add(stat.destNodeModulesDirPath) + autoUnpackDirs.add(stat.moduleRootPath) } } diff --git a/packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts b/packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts index a76230fb3a1..5a71774742d 100644 --- a/packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts +++ b/packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts @@ -115,8 +115,8 @@ export class NodeModuleCopyHelper extends FileCopyHelper { return lstat(filePath).then((stat: FilterStats) => { stat.moduleName = moduleName - stat.destNodeModulesDirPath = destination - stat.destNodeModulesFilePath = path.join(destination, path.relative(depPath, filePath)) + stat.moduleRootPath = destination + stat.moduleFullFilePath = path.join(destination, path.relative(depPath, filePath)) if (filter != null && !filter(filePath, stat)) { return null } diff --git a/packages/app-builder-lib/src/util/filter.ts b/packages/app-builder-lib/src/util/filter.ts index 37bbe1e937f..fa8a1d6e2de 100644 --- a/packages/app-builder-lib/src/util/filter.ts +++ b/packages/app-builder-lib/src/util/filter.ts @@ -24,7 +24,7 @@ function ensureEndSlash(s: string) { } function getRelativePath(file: string, srcWithEndSlash: string, stat: FilterStats) { - let relative = stat.destNodeModulesFilePath || file.substring(srcWithEndSlash.length) + let relative = stat.moduleFullFilePath || file.substring(srcWithEndSlash.length) if (path.sep === "\\") { if (relative.startsWith("\\")) { // windows problem: double backslash, the above substring call removes root path with a single slash, so here can me some leftovers diff --git a/packages/builder-util/src/fs.ts b/packages/builder-util/src/fs.ts index 0a77d325d8c..307bddced65 100644 --- a/packages/builder-util/src/fs.ts +++ b/packages/builder-util/src/fs.ts @@ -20,11 +20,17 @@ export class CopyFileTransformer { export type FileTransformer = (file: string) => Promise | null | string | Buffer | CopyFileTransformer export interface FilterStats extends Stats { - // relative path of the dependency(node_modules + moduleName + file) - // Mainly used for filter, such as files filtering and asarUnpack filtering - destNodeModulesFilePath?: string - destNodeModulesDirPath?: string + // These module name and paths are mainly used for: + // 1. File filtering + // 2. Asar unpacking rules + // 3. Dependency resolution + + // The name of the node module (e.g. 'express') moduleName?: string + // The root path of the node module (e.g. 'node_modules/express') + moduleRootPath?: string + // The full file path within the node module (e.g. 'node_modules/express/lib/application.js') + moduleFullFilePath?: string // deal with asar unpack sysmlink relativeLink?: string linkRelativeToFile?: string From 179b690ad52ac918d1d8498836f5123abf5b9191 Mon Sep 17 00:00:00 2001 From: beyondkmp Date: Thu, 31 Oct 2024 15:18:56 +0800 Subject: [PATCH 12/14] add changeset --- .changeset/brave-snakes-unite.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/brave-snakes-unite.md diff --git a/.changeset/brave-snakes-unite.md b/.changeset/brave-snakes-unite.md new file mode 100644 index 00000000000..38e77dc8318 --- /dev/null +++ b/.changeset/brave-snakes-unite.md @@ -0,0 +1,6 @@ +--- +"app-builder-lib": patch +"builder-util": patch +--- + +fix: smart unpack for local module with dll From ead15e139bca0690d37609c9ff8433a07bb27a9f Mon Sep 17 00:00:00 2001 From: beyondkmp Date: Thu, 31 Oct 2024 15:30:31 +0800 Subject: [PATCH 13/14] fix ut --- test/src/BuildTest.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/src/BuildTest.ts b/test/src/BuildTest.ts index 6d894824222..697f38b0df4 100644 --- a/test/src/BuildTest.ts +++ b/test/src/BuildTest.ts @@ -283,17 +283,18 @@ test.ifLinuxOrDevMac("beforeBuild", () => { ) }) -test.ifDevOrWinCi("win smart unpack", () => { +// https://github.com/electron-userland/electron-builder/issues/1738 +test.ifDevOrLinuxCi("win smart unpack", () => { // test onNodeModuleFile hook const nodeModuleFiles: Array = [] + let p = "" return app( { targets: Platform.WINDOWS.createTarget(DIR_TARGET), config: { npmRebuild: true, onNodeModuleFile: file => { - const index = file.indexOf(path.sep + "node_modules"+ path.sep) - const name = toSystemIndependentPath(file.slice(index + 1)) + const name = toSystemIndependentPath(path.relative(p, file)) if (!name.startsWith(".") && !name.endsWith(".dll") && name.includes(".")) { nodeModuleFiles.push(name) } @@ -304,8 +305,8 @@ test.ifDevOrWinCi("win smart unpack", () => { }, }, { - isInstallDepsBefore:true, projectDirCreated: projectDir => { + p = projectDir return packageJson(it => { it.dependencies = { debug: "3.1.0", From f009f00bdb673cc826db742b37c25978a2fe539d Mon Sep 17 00:00:00 2001 From: beyondkmp Date: Thu, 7 Nov 2024 04:25:02 -0500 Subject: [PATCH 14/14] update ut --- test/snapshots/globTest.js.snap | 62 ++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/test/snapshots/globTest.js.snap b/test/snapshots/globTest.js.snap index 897d306549f..13afce25840 100644 --- a/test/snapshots/globTest.js.snap +++ b/test/snapshots/globTest.js.snap @@ -9896,6 +9896,7 @@ exports[`symlinks everywhere w/ static framework 3`] = ` "files": { "hello-world-framework.sh": { "size": "", + "unpacked": true, }, "lib": { "files": { @@ -9922,84 +9923,94 @@ exports[`symlinks everywhere w/ static framework 3`] = ` "Headers": { "files": { "HelloFramework.h": { - "executable": true, "size": "", + "unpacked": true, }, }, + "unpacked": true, }, "Hello": { - "executable": true, "size": "", + "unpacked": true, }, "Info.plist": { - "executable": true, "size": "", + "unpacked": true, }, "Modules": { "files": { "Hello.abi.json": { - "executable": true, "size": "", + "unpacked": true, }, "Hello.swiftdoc": { - "executable": true, "size": "", + "unpacked": true, }, "Hello.swiftmodule": { "files": { "arm64.private.swiftinterface": { - "executable": true, "size": "", + "unpacked": true, }, "arm64.swiftinterface": { - "executable": true, "size": "", + "unpacked": true, }, "arm64.swiftmodule": { - "executable": true, "size": "", + "unpacked": true, }, "x86_64.private.swiftinterface": { - "executable": true, "size": "", + "unpacked": true, }, "x86_64.swiftinterface": { - "executable": true, "size": "", + "unpacked": true, }, "x86_64.swiftmodule": { - "executable": true, "size": "", + "unpacked": true, }, }, + "unpacked": true, }, "Hello.swiftsourceinfo": { - "executable": true, "size": "", + "unpacked": true, }, "module.modulemap": { - "executable": true, "size": "", + "unpacked": true, }, }, + "unpacked": true, }, }, + "unpacked": true, }, "Current": { "link": "node_modules/hello-world/lib/Release/Hello.framework/Versions/A", }, }, + "unpacked": true, }, }, + "unpacked": true, }, }, + "unpacked": true, }, }, + "unpacked": true, }, "package.json": { "size": "", + "unpacked": true, }, }, + "unpacked": true, }, "ms": { "files": { @@ -10026,6 +10037,31 @@ exports[`symlinks everywhere w/ static framework 3`] = ` exports[`symlinks everywhere w/ static framework 4`] = ` [ "app.asar", + "app.asar.unpacked/node_modules/hello-world/hello-world-framework.sh", + { + "content": "{ + "name": "stripped-native-dep", + "version": "1.0.0", + "description": "", + "author": "", + "license": "ISC" + } + ", + "name": "app.asar.unpacked/node_modules/hello-world/package.json", + }, + "app.asar.unpacked/node_modules/hello-world/lib/Release/Hello.framework/Versions/A/Hello", + "app.asar.unpacked/node_modules/hello-world/lib/Release/Hello.framework/Versions/A/Info.plist", + "app.asar.unpacked/node_modules/hello-world/lib/Release/Hello.framework/Versions/A/Modules/Hello.abi.json", + "app.asar.unpacked/node_modules/hello-world/lib/Release/Hello.framework/Versions/A/Modules/Hello.swiftdoc", + "app.asar.unpacked/node_modules/hello-world/lib/Release/Hello.framework/Versions/A/Modules/Hello.swiftsourceinfo", + "app.asar.unpacked/node_modules/hello-world/lib/Release/Hello.framework/Versions/A/Modules/module.modulemap", + "app.asar.unpacked/node_modules/hello-world/lib/Release/Hello.framework/Versions/A/Modules/Hello.swiftmodule/arm64.private.swiftinterface", + "app.asar.unpacked/node_modules/hello-world/lib/Release/Hello.framework/Versions/A/Modules/Hello.swiftmodule/arm64.swiftinterface", + "app.asar.unpacked/node_modules/hello-world/lib/Release/Hello.framework/Versions/A/Modules/Hello.swiftmodule/arm64.swiftmodule", + "app.asar.unpacked/node_modules/hello-world/lib/Release/Hello.framework/Versions/A/Modules/Hello.swiftmodule/x86_64.private.swiftinterface", + "app.asar.unpacked/node_modules/hello-world/lib/Release/Hello.framework/Versions/A/Modules/Hello.swiftmodule/x86_64.swiftinterface", + "app.asar.unpacked/node_modules/hello-world/lib/Release/Hello.framework/Versions/A/Modules/Hello.swiftmodule/x86_64.swiftmodule", + "app.asar.unpacked/node_modules/hello-world/lib/Release/Hello.framework/Versions/A/Headers/HelloFramework.h", ] `;