diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index 68fe659f5da5c..2e18ff1e86eba 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -33,6 +33,7 @@ graph LR; libnpmexec-->npmcli-arborist["@npmcli/arborist"]; libnpmexec-->npmcli-eslint-config["@npmcli/eslint-config"]; libnpmexec-->npmcli-mock-registry["@npmcli/mock-registry"]; + libnpmexec-->npmcli-package-json["@npmcli/package-json"]; libnpmexec-->npmcli-run-script["@npmcli/run-script"]; libnpmexec-->npmcli-template-oss["@npmcli/template-oss"]; libnpmexec-->pacote; @@ -342,6 +343,7 @@ graph LR; libnpmexec-->npmcli-arborist["@npmcli/arborist"]; libnpmexec-->npmcli-eslint-config["@npmcli/eslint-config"]; libnpmexec-->npmcli-mock-registry["@npmcli/mock-registry"]; + libnpmexec-->npmcli-package-json["@npmcli/package-json"]; libnpmexec-->npmcli-run-script["@npmcli/run-script"]; libnpmexec-->npmcli-template-oss["@npmcli/template-oss"]; libnpmexec-->pacote; diff --git a/docs/lib/content/commands/npm-cache.md b/docs/lib/content/commands/npm-cache.md index 20836b512a12f..f41282969d09e 100644 --- a/docs/lib/content/commands/npm-cache.md +++ b/docs/lib/content/commands/npm-cache.md @@ -10,53 +10,47 @@ description: Manipulates packages cache ### Description -Used to add, list, or clean the npm cache folder. +Used to add, list, or clean the `npm cache` folder. +Also used to view info about entries in the `npm exec` (aka `npx`) cache folder. + +#### `npm cache` * add: - Add the specified packages to the local cache. This command is primarily - intended to be used internally by npm, but it can provide a way to - add data to the local installation cache explicitly. + Add the specified packages to the local cache. This command is primarily intended to be used internally by npm, but it can provide a way to add data to the local installation cache explicitly. * clean: - Delete all data out of the cache folder. Note that this is typically - unnecessary, as npm's cache is self-healing and resistant to data - corruption issues. + Delete a single entry or all entries out of the cache folder. Note that this is typically unnecessary, as npm's cache is self-healing and resistant to data corruption issues. + +* ls: + List given entries or all entries in the local cache. * verify: - Verify the contents of the cache folder, garbage collecting any unneeded - data, and verifying the integrity of the cache index and all cached data. + Verify the contents of the cache folder, garbage collecting any unneeded data, and verifying the integrity of the cache index and all cached data. + +#### `npm cache npx` + +* ls: + List all entries in the npx cache. + +* rm: + Remove given entries or all entries from the npx cache. + +* info: + Get detailed information about given entries in the npx cache. ### Details -npm stores cache data in an opaque directory within the configured `cache`, -named `_cacache`. This directory is a -[`cacache`](http://npm.im/cacache)-based content-addressable cache that -stores all http request data as well as other package-related data. This -directory is primarily accessed through `pacote`, the library responsible -for all package fetching as of npm@5. +npm stores cache data in an opaque directory within the configured `cache`, named `_cacache`. This directory is a [`cacache`](http://npm.im/cacache)-based content-addressable cache that stores all http request data as well as other package-related data. This directory is primarily accessed through `pacote`, the library responsible for all package fetching as of npm@5. -All data that passes through the cache is fully verified for integrity on -both insertion and extraction. Cache corruption will either trigger an -error, or signal to `pacote` that the data must be refetched, which it will -do automatically. For this reason, it should never be necessary to clear -the cache for any reason other than reclaiming disk space, thus why `clean` -now requires `--force` to run. +All data that passes through the cache is fully verified for integrity on both insertion and extraction. Cache corruption will either trigger an error, or signal to `pacote` that the data must be refetched, which it will do automatically. For this reason, it should never be necessary to clear the cache for any reason other than reclaiming disk space, thus why `clean` now requires `--force` to run. -There is currently no method exposed through npm to inspect or directly -manage the contents of this cache. In order to access it, `cacache` must be -used directly. +There is currently no method exposed through npm to inspect or directly manage the contents of this cache. In order to access it, `cacache` must be used directly. -npm will not remove data by itself: the cache will grow as new packages are -installed. +npm will not remove data by itself: the cache will grow as new packages are installed. ### A note about the cache's design -The npm cache is strictly a cache: it should not be relied upon as a -persistent and reliable data store for package data. npm makes no guarantee -that a previously-cached piece of data will be available later, and will -automatically delete corrupted contents. The primary guarantee that the -cache makes is that, if it does return data, that data will be exactly the -data that was inserted. +The npm cache is strictly a cache: it should not be relied upon as a persistent and reliable data store for package data. npm makes no guarantee that a previously-cached piece of data will be available later, and will automatically delete corrupted contents. The primary guarantee that the cache makes is that, if it does return data, that data will be exactly the data that was inserted. To run an offline verification of existing cache contents, use `npm cache verify`. @@ -74,6 +68,7 @@ verify`. * [npm install](/commands/npm-install) * [npm publish](/commands/npm-publish) * [npm pack](/commands/npm-pack) +* [npm exec](/commands/npm-exec) * https://npm.im/cacache * https://npm.im/pacote * https://npm.im/@npmcli/arborist diff --git a/lib/commands/cache.js b/lib/commands/cache.js index ddfeeb818a817..59831a193fe8b 100644 --- a/lib/commands/cache.js +++ b/lib/commands/cache.js @@ -1,13 +1,15 @@ -const cacache = require('cacache') -const pacote = require('pacote') const fs = require('node:fs/promises') const { join } = require('node:path') +const cacache = require('cacache') +const pacote = require('pacote') const semver = require('semver') -const BaseCommand = require('../base-cmd.js') const npa = require('npm-package-arg') const jsonParse = require('json-parse-even-better-errors') const localeCompare = require('@isaacs/string-locale-compare')('en') const { log, output } = require('proc-log') +const PkgJson = require('@npmcli/package-json') +const BaseCommand = require('../base-cmd.js') +const abbrev = require('abbrev') const searchCachePackage = async (path, parsed, cacheKeys) => { const searchMFH = new RegExp(`^make-fetch-happen:request-cache:.*(? { } class Cache extends BaseCommand { - static description = 'Manipulates packages cache' + static description = 'Manipulates packages and npx cache' static name = 'cache' static params = ['cache'] static usage = [ @@ -70,12 +72,15 @@ class Cache extends BaseCommand { 'clean []', 'ls [@]', 'verify', + 'npx ls', + 'npx rm [...]', + 'npx info ...', ] static async completion (opts) { const argv = opts.conf.argv.remain if (argv.length === 2) { - return ['add', 'clean', 'verify', 'ls'] + return ['add', 'clean', 'verify', 'ls', 'npx'] } // TODO - eventually... @@ -99,14 +104,31 @@ class Cache extends BaseCommand { return await this.verify() case 'ls': return await this.ls(args) + case 'npx': + return await this.npx(args) + default: + throw this.usageError() + } + } + + // npm cache npx + async npx ([cmd, ...keys]) { + switch (cmd) { + case 'ls': + return await this.npxLs(keys) + case 'rm': + return await this.npxRm(keys) + case 'info': + return await this.npxInfo(keys) default: throw this.usageError() } } - // npm cache clean [pkg]* + // npm cache clean [spec]* async clean (args) { - const cachePath = join(this.npm.cache, '_cacache') + // this is a derived value + const cachePath = this.npm.flatOptions.cache if (args.length === 0) { if (!this.npm.config.get('force')) { throw new Error(`As of npm@5, the npm cache self-heals from corruption issues @@ -169,11 +191,12 @@ class Cache extends BaseCommand { } async verify () { - const cache = join(this.npm.cache, '_cacache') - const prefix = cache.indexOf(process.env.HOME) === 0 - ? `~${cache.slice(process.env.HOME.length)}` - : cache - const stats = await cacache.verify(cache) + // this is a derived value + const cachePath = this.npm.flatOptions.cache + const prefix = cachePath.indexOf(process.env.HOME) === 0 + ? `~${cachePath.slice(process.env.HOME.length)}` + : cachePath + const stats = await cacache.verify(cachePath) output.standard(`Cache verified and compressed (${prefix})`) output.standard(`Content verified: ${stats.verifiedContent} (${stats.keptSize} bytes)`) if (stats.badContentCount) { @@ -189,9 +212,10 @@ class Cache extends BaseCommand { output.standard(`Finished in ${stats.runTime.total / 1000}s`) } - // npm cache ls [--package ...] + // npm cache ls [ ...] async ls (specs) { - const cachePath = join(this.npm.cache, '_cacache') + // This is a derived value + const { cache: cachePath } = this.npm.flatOptions const cacheKeys = Object.keys(await cacache.ls(cachePath)) if (specs.length > 0) { // get results for each package spec specified @@ -211,6 +235,136 @@ class Cache extends BaseCommand { } cacheKeys.sort(localeCompare).forEach(key => output.standard(key)) } + + async #npxCache (keys = []) { + // This is a derived value + const { npxCache } = this.npm.flatOptions + let dirs + try { + dirs = await fs.readdir(npxCache, { encoding: 'utf-8' }) + } catch { + output.standard('npx cache does not exist') + return + } + const cache = {} + const { default: pMap } = await import('p-map') + await pMap(dirs, async e => { + const pkgPath = join(npxCache, e) + cache[e] = { + hash: e, + path: pkgPath, + valid: false, + } + try { + const pkgJson = await PkgJson.load(pkgPath) + cache[e].package = pkgJson.content + cache[e].valid = true + } catch { + // Defaults to not valid already + } + }, { concurrency: 20 }) + if (!keys.length) { + return cache + } + const result = {} + const abbrevs = abbrev(Object.keys(cache)) + for (const key of keys) { + if (!abbrevs[key]) { + throw this.usageError(`Invalid npx key ${key}`) + } + result[abbrevs[key]] = cache[abbrevs[key]] + } + return result + } + + async npxLs () { + const cache = await this.#npxCache() + for (const key in cache) { + const { hash, valid, package: pkg } = cache[key] + let result = `${hash}:` + if (!valid) { + result = `${result} (empty/invalid)` + } else if (pkg?._npx) { + result = `${result} ${pkg._npx.packages.join(', ')}` + } else { + result = `${result} (unknown)` + } + output.standard(result) + } + } + + async npxRm (keys) { + if (!keys.length) { + if (!this.npm.config.get('force')) { + throw this.usageError('Please use --force to remove entire npx cache') + } + const { npxCache } = this.npm.flatOptions + if (!this.npm.config.get('dry-run')) { + return fs.rm(npxCache, { recursive: true, force: true }) + } + } + + const cache = await this.#npxCache(keys) + for (const key in cache) { + const { path: cachePath } = cache[key] + output.standard(`Removing npx key at ${cachePath}`) + if (!this.npm.config.get('dry-run')) { + await fs.rm(cachePath, { recursive: true }) + } + } + } + + async npxInfo (keys) { + const chalk = this.npm.chalk + if (!keys.length) { + throw this.usageError() + } + const cache = await this.#npxCache(keys) + const Arborist = require('@npmcli/arborist') + for (const key in cache) { + const { hash, path, package: pkg } = cache[key] + let valid = cache[key].valid + const results = [] + try { + if (valid) { + const arb = new Arborist({ path }) + const tree = await arb.loadVirtual() + if (pkg._npx) { + results.push('packages:') + for (const p of pkg._npx.packages) { + const parsed = npa(p) + if (parsed.type === 'directory') { + // in the tree the spec is relative, even if the dependency spec is absolute, so we can't find it by name or spec. + results.push(`- ${chalk.cyan(p)}`) + } else { + results.push(`- ${chalk.cyan(p)} (${chalk.blue(tree.children.get(parsed.name).pkgid)})`) + } + } + } else { + results.push('packages: (unknown)') + results.push(`dependencies:`) + for (const dep in pkg.dependencies) { + const child = tree.children.get(dep) + if (child.isLink) { + results.push(`- ${chalk.cyan(child.realpath)}`) + } else { + results.push(`- ${chalk.cyan(child.pkgid)}`) + } + } + } + } + } catch (ex) { + valid = false + } + const v = valid ? chalk.green('valid') : chalk.red('invalid') + output.standard(`${v} npx cache entry with key ${chalk.blue(hash)}`) + output.standard(`location: ${chalk.blue(path)}`) + if (valid) { + output.standard(results.join('\n')) + } + output.standard('') + } + } } module.exports = Cache diff --git a/package-lock.json b/package-lock.json index df0fd3270d0ef..130de74935dc6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18876,6 +18876,7 @@ "license": "ISC", "dependencies": { "@npmcli/arborist": "^9.0.0", + "@npmcli/package-json": "^6.1.1", "@npmcli/run-script": "^9.0.1", "ci-info": "^4.0.0", "npm-package-arg": "^12.0.0", diff --git a/tap-snapshots/test/lib/commands/cache.js.test.cjs b/tap-snapshots/test/lib/commands/cache.js.test.cjs index 1cd699453478e..d5ce43922f024 100644 --- a/tap-snapshots/test/lib/commands/cache.js.test.cjs +++ b/tap-snapshots/test/lib/commands/cache.js.test.cjs @@ -41,6 +41,82 @@ make-fetch-happen:request-cache:https://registry.npmjs.org/foo make-fetch-happen:request-cache:https://registry.npmjs.org/foo/-/foo-1.2.3-beta.tgz ` +exports[`test/lib/commands/cache.js TAP cache npx info: valid and invalid entry > shows invalid package info 1`] = ` +invalid npx cache entry with key deadbeef +location: {CWD}/cache/_npx/deadbeef + +invalid npx cache entry with key badc0de +location: {CWD}/cache/_npx/badc0de +` + +exports[`test/lib/commands/cache.js TAP cache npx info: valid and invalid entry > shows valid package info 1`] = ` +invalid npx cache entry with key deadbeef +location: {CWD}/cache/_npx/deadbeef +` + +exports[`test/lib/commands/cache.js TAP cache npx info: valid entry with _npx directory package > shows valid package info with _npx directory package 1`] = ` +valid npx cache entry with key valid123 +location: {CWD}/cache/_npx/valid123 +packages: +- /path/to/valid-package +` + +exports[`test/lib/commands/cache.js TAP cache npx info: valid entry with _npx packages > shows valid package info with _npx packages 1`] = ` +valid npx cache entry with key valid123 +location: {CWD}/cache/_npx/valid123 +packages: +- valid-package@1.0.0 (valid-package@1.0.0) +` + +exports[`test/lib/commands/cache.js TAP cache npx info: valid entry with a link dependency > shows link dependency realpath (child.isLink branch) 1`] = ` +valid npx cache entry with key link123 +location: {CWD}/cache/_npx/link123 +packages: (unknown) +dependencies: +- {CWD}/cache/_npx/some-other-loc +` + +exports[`test/lib/commands/cache.js TAP cache npx info: valid entry with dependencies > shows valid package info with dependencies 1`] = ` +valid npx cache entry with key valid456 +location: {CWD}/cache/_npx/valid456 +packages: (unknown) +dependencies: +- dep-package@1.0.0 +` + +exports[`test/lib/commands/cache.js TAP cache npx ls: empty cache > logs message for empty npx cache 1`] = ` +npx cache does not exist +` + +exports[`test/lib/commands/cache.js TAP cache npx ls: entry with unknown package > lists entry with unknown package 1`] = ` +unknown123: (unknown) +` + +exports[`test/lib/commands/cache.js TAP cache npx ls: some entries > lists one valid and one invalid entry 1`] = ` +abc123: fake-npx-package@1.0.0 +z9y8x7: (empty/invalid) +` + +exports[`test/lib/commands/cache.js TAP cache npx rm: remove single entry > logs removing single npx cache entry 1`] = ` +Removing npx key at {CWD}/cache/_npx/123removeme +Removing npx key at {CWD}/cache/_npx/123removeme +` + +exports[`test/lib/commands/cache.js TAP cache npx rm: removing all with --force works > logs removing everything 1`] = ` +Removing npx key at {CWD}/cache/_npx/remove-all-yes-force +` + +exports[`test/lib/commands/cache.js TAP cache npx rm: removing all without --force fails > logs usage error when removing all without --force 1`] = ` + +` + +exports[`test/lib/commands/cache.js TAP cache npx rm: removing more than 1, less than all entries > logs removing 2 of 3 entries 1`] = ` +Removing npx key at {CWD}/cache/_npx/123removeme +Removing npx key at {CWD}/cache/_npx/456removeme +Removing npx key at {CWD}/cache/_npx/123removeme +Removing npx key at {CWD}/cache/_npx/456removeme +` + exports[`test/lib/commands/cache.js TAP cache rm > logs deleting single entry 1`] = ` Deleted: make-fetch-happen:request-cache:https://registry.npmjs.org/test-package/-/test-package-1.0.0.tgz ` diff --git a/tap-snapshots/test/lib/docs.js.test.cjs b/tap-snapshots/test/lib/docs.js.test.cjs index 28e150390dc1e..f43b8cd116b92 100644 --- a/tap-snapshots/test/lib/docs.js.test.cjs +++ b/tap-snapshots/test/lib/docs.js.test.cjs @@ -2689,13 +2689,16 @@ alias: issues ` exports[`test/lib/docs.js TAP usage cache > must match snapshot 1`] = ` -Manipulates packages cache +Manipulates packages and npx cache Usage: npm cache add npm cache clean [] npm cache ls [@] npm cache verify +npm cache npx ls +npm cache npx rm [...] +npm cache npx info ... Options: [--cache ] @@ -2707,6 +2710,9 @@ npm cache add npm cache clean [] npm cache ls [@] npm cache verify +npm cache npx ls +npm cache npx rm [...] +npm cache npx info ... \`\`\` Note: This command is unaware of workspaces. diff --git a/test/lib/commands/cache.js b/test/lib/commands/cache.js index a1f2a8fbfda02..7b79e111cd246 100644 --- a/test/lib/commands/cache.js +++ b/test/lib/commands/cache.js @@ -5,11 +5,26 @@ const MockRegistry = require('@npmcli/mock-registry') const cacache = require('cacache') const fs = require('node:fs') const path = require('node:path') +const { cleanCwd } = require('../../fixtures/clean-snapshot.js') const pkg = 'test-package' +const createNpxCacheEntry = (npxCacheDir, hash, pkgJson, shrinkwrapJson) => { + fs.mkdirSync(path.join(npxCacheDir, hash)) + fs.writeFileSync( + path.join(npxCacheDir, hash, 'package.json'), + JSON.stringify(pkgJson) + ) + if (shrinkwrapJson) { + fs.writeFileSync( + path.join(npxCacheDir, hash, 'npm-shrinkwrap.json'), + JSON.stringify(shrinkwrapJson) + ) + } +} + t.cleanSnapshot = str => { - return str + return cleanCwd(str) .replace(/Finished in [0-9.s]+/g, 'Finished in xxxs') .replace(/Cache verified and compressed (.*)/, 'Cache verified and compressed ({PATH})') } @@ -310,3 +325,393 @@ t.test('cache completion', async t => { testComp(['npm', 'cache', 'verify'], []), ]) }) + +t.test('cache npx ls: empty cache', async t => { + const { npm, joinedOutput } = await loadMockNpm(t) + await npm.exec('cache', ['npx', 'ls']) + t.matchSnapshot(joinedOutput(), 'logs message for empty npx cache') +}) + +t.test('cache npx ls: some entries', async t => { + const { npm, joinedOutput } = await loadMockNpm(t) + const npxCacheDir = path.join(npm.flatOptions.npxCache || path.join(npm.cache, '..', '_npx')) + fs.mkdirSync(npxCacheDir, { recursive: true }) + + // Make two fake entries: one valid, one invalid + const hash1 = 'abc123' + const hash2 = 'z9y8x7' + fs.mkdirSync(path.join(npxCacheDir, hash1)) + fs.writeFileSync( + path.join(npxCacheDir, hash1, 'package.json'), + JSON.stringify({ + name: 'fake-npx-package', + version: '1.0.0', + _npx: { packages: ['fake-npx-package@1.0.0'] }, + }) + ) + // invalid (missing or broken package.json) directory + fs.mkdirSync(path.join(npxCacheDir, hash2)) + + await npm.exec('cache', ['npx', 'ls']) + t.matchSnapshot(joinedOutput(), 'lists one valid and one invalid entry') +}) + +t.test('cache npx info: valid and invalid entry', async t => { + const { npm, joinedOutput } = await loadMockNpm(t) + const npxCacheDir = path.join(npm.flatOptions.npxCache || path.join(npm.cache, '..', '_npx')) + fs.mkdirSync(npxCacheDir, { recursive: true }) + + const goodHash = 'deadbeef' + fs.mkdirSync(path.join(npxCacheDir, goodHash)) + fs.writeFileSync( + path.join(npxCacheDir, goodHash, 'package.json'), + JSON.stringify({ + name: 'good-npx-package', + version: '2.0.0', + dependencies: { + rimraf: '^3.0.0', + }, + _npx: { packages: ['good-npx-package@2.0.0'] }, + }) + ) + + const badHash = 'badc0de' + fs.mkdirSync(path.join(npxCacheDir, badHash)) + + await npm.exec('cache', ['npx', 'info', goodHash]) + t.matchSnapshot(joinedOutput(), 'shows valid package info') + + await npm.exec('cache', ['npx', 'info', badHash]) + t.matchSnapshot(joinedOutput(), 'shows invalid package info') +}) + +t.test('cache npx rm: remove single entry', async t => { + const { npm, joinedOutput } = await loadMockNpm(t) + const npxCacheDir = path.join(npm.flatOptions.npxCache || path.join(npm.cache, '..', '_npx')) + fs.mkdirSync(npxCacheDir, { recursive: true }) + + const removableHash = '123removeme' + fs.mkdirSync(path.join(npxCacheDir, removableHash)) + fs.writeFileSync( + path.join(npxCacheDir, removableHash, 'package.json'), + JSON.stringify({ name: 'removable-package', _npx: { packages: ['removable-package@1.0.0'] } }) + ) + + npm.config.set('dry-run', true) + await npm.exec('cache', ['npx', 'rm', removableHash]) + t.ok(fs.existsSync(path.join(npxCacheDir, removableHash)), 'entry folder remains') + npm.config.set('dry-run', false) + + await npm.exec('cache', ['npx', 'rm', removableHash]) + t.matchSnapshot(joinedOutput(), 'logs removing single npx cache entry') + t.notOk(fs.existsSync(path.join(npxCacheDir, removableHash)), 'entry folder removed') +}) + +t.test('cache npx rm: removing all without --force fails', async t => { + const { npm, joinedOutput } = await loadMockNpm(t) + const npxCacheDir = path.join(npm.flatOptions.npxCache || path.join(npm.cache, '..', '_npx')) + fs.mkdirSync(npxCacheDir, { recursive: true }) + + const testHash = 'remove-all-no-force' + fs.mkdirSync(path.join(npxCacheDir, testHash)) + fs.writeFileSync( + path.join(npxCacheDir, testHash, 'package.json'), + JSON.stringify({ name: 'no-force-pkg', _npx: { packages: ['no-force-pkg@1.0.0'] } }) + ) + + await t.rejects( + npm.exec('cache', ['npx', 'rm']), + /Please use --force to remove entire npx cache/, + 'fails without --force' + ) + t.matchSnapshot(joinedOutput(), 'logs usage error when removing all without --force') + + t.ok(fs.existsSync(path.join(npxCacheDir, testHash)), 'folder still exists') +}) + +t.test('cache npx rm: removing all with --force works', async t => { + const { npm, joinedOutput } = await loadMockNpm(t, { + config: { force: true }, + }) + const npxCacheDir = path.join(npm.flatOptions.npxCache || path.join(npm.cache, '..', '_npx')) + fs.mkdirSync(npxCacheDir, { recursive: true }) + + const testHash = 'remove-all-yes-force' + fs.mkdirSync(path.join(npxCacheDir, testHash)) + fs.writeFileSync( + path.join(npxCacheDir, testHash, 'package.json'), + JSON.stringify({ name: 'yes-force-pkg', _npx: { packages: ['yes-force-pkg@1.0.0'] } }) + ) + + npm.config.set('dry-run', true) + await npm.exec('cache', ['npx', 'rm']) + t.ok(fs.existsSync(npxCacheDir), 'npx cache directory remains') + npm.config.set('dry-run', false) + + await npm.exec('cache', ['npx', 'rm']) + + t.matchSnapshot(joinedOutput(), 'logs removing everything') + t.notOk(fs.existsSync(npxCacheDir), 'npx cache directory removed') +}) + +t.test('cache npx rm: removing more than 1, less than all entries', async t => { + const { npm, joinedOutput } = await loadMockNpm(t) + const npxCacheDir = path.join(npm.flatOptions.npxCache || path.join(npm.cache, '..', '_npx')) + fs.mkdirSync(npxCacheDir, { recursive: true }) + + // Removable folder + const removableHash = '123removeme' + fs.mkdirSync(path.join(npxCacheDir, removableHash)) + fs.writeFileSync( + path.join(npxCacheDir, removableHash, 'package.json'), + JSON.stringify({ name: 'removable-package', _npx: { packages: ['removable-package@1.0.0'] } }) + ) + + // Another Removable folder + const anotherRemovableHash = '456removeme' + fs.mkdirSync(path.join(npxCacheDir, anotherRemovableHash)) + fs.writeFileSync( + path.join(npxCacheDir, anotherRemovableHash, 'package.json'), + JSON.stringify({ name: 'another-removable-package', _npx: { packages: ['another-removable-package@1.0.0'] } }) + ) + + // Another folder that should remain + const keepHash = '999keep' + fs.mkdirSync(path.join(npxCacheDir, keepHash)) + fs.writeFileSync( + path.join(npxCacheDir, keepHash, 'package.json'), + JSON.stringify({ name: 'keep-package', _npx: { packages: ['keep-package@1.0.0'] } }) + ) + + npm.config.set('dry-run', true) + await npm.exec('cache', ['npx', 'rm', removableHash, anotherRemovableHash]) + t.ok(fs.existsSync(path.join(npxCacheDir, removableHash)), 'entry folder remains') + t.ok(fs.existsSync(path.join(npxCacheDir, anotherRemovableHash)), 'entry folder remains') + t.ok(fs.existsSync(path.join(npxCacheDir, keepHash)), 'entry folder remains') + npm.config.set('dry-run', false) + + await npm.exec('cache', ['npx', 'rm', removableHash, anotherRemovableHash]) + t.matchSnapshot(joinedOutput(), 'logs removing 2 of 3 entries') + + t.notOk(fs.existsSync(path.join(npxCacheDir, removableHash)), 'removed folder no longer exists') + t.notOk(fs.existsSync(path.join(npxCacheDir, anotherRemovableHash)), 'the other folder no longer exists') + t.ok(fs.existsSync(path.join(npxCacheDir, keepHash)), 'the other folder remains') +}) + +t.test('cache npx should throw usage error', async t => { + const { npm } = await loadMockNpm(t) + await t.rejects( + npm.exec('cache', ['npx', 'badcommand']), + { code: 'EUSAGE' }, + 'should throw usage error' + ) +}) + +t.test('cache npx should throw usage error for invalid key', async t => { + const { npm } = await loadMockNpm(t) + const npxCacheDir = path.join(npm.flatOptions.npxCache || path.join(npm.cache, '..', '_npx')) + fs.mkdirSync(npxCacheDir, { recursive: true }) + + const key = 'badkey' + await t.rejects( + npm.exec('cache', ['npx', 'rm', key]), + { code: 'EUSAGE' }, + `Invalid npx key ${key}` + ) +}) + +t.test('cache npx ls: entry with unknown package', async t => { + const { npm, joinedOutput } = await loadMockNpm(t) + const npxCacheDir = path.join(npm.flatOptions.npxCache || path.join(npm.cache, '..', '_npx')) + fs.mkdirSync(npxCacheDir, { recursive: true }) + + // Create an entry without the _npx property + const unknownHash = 'unknown123' + fs.mkdirSync(path.join(npxCacheDir, unknownHash)) + fs.writeFileSync( + path.join(npxCacheDir, unknownHash, 'package.json'), + JSON.stringify({ + name: 'unknown-package', + version: '1.0.0', + }) + ) + + await npm.exec('cache', ['npx', 'ls']) + t.matchSnapshot(joinedOutput(), 'lists entry with unknown package') +}) + +t.test('cache npx info: should throw usage error when no keys are provided', async t => { + const { npm } = await loadMockNpm(t) + await t.rejects( + npm.exec('cache', ['npx', 'info']), + { code: 'EUSAGE' }, + 'should throw usage error when no keys are provided' + ) +}) + +t.test('cache npx info: valid entry with _npx packages', async t => { + const { npm, joinedOutput } = await loadMockNpm(t) + const npxCacheDir = path.join(npm.flatOptions.npxCache || path.join(npm.cache, '..', '_npx')) + fs.mkdirSync(npxCacheDir, { recursive: true }) + + const validHash = 'valid123' + createNpxCacheEntry(npxCacheDir, validHash, { + name: 'valid-package', + version: '1.0.0', + _npx: { packages: ['valid-package@1.0.0'] }, + }, { + name: 'valid-package', + version: '1.0.0', + dependencies: { + 'valid-package': { + version: '1.0.0', + resolved: 'https://registry.npmjs.org/valid-package/-/valid-package-1.0.0.tgz', + integrity: 'sha512-...', + }, + }, + }) + + const nodeModulesDir = path.join(npxCacheDir, validHash, 'node_modules') + fs.mkdirSync(nodeModulesDir, { recursive: true }) + fs.mkdirSync(path.join(nodeModulesDir, 'valid-package')) + fs.writeFileSync( + path.join(nodeModulesDir, 'valid-package', 'package.json'), + JSON.stringify({ + name: 'valid-package', + version: '1.0.0', + }) + ) + + await npm.exec('cache', ['npx', 'info', validHash]) + t.matchSnapshot(joinedOutput(), 'shows valid package info with _npx packages') +}) + +t.test('cache npx info: valid entry with dependencies', async t => { + const { npm, joinedOutput } = await loadMockNpm(t) + const npxCacheDir = path.join(npm.flatOptions.npxCache || path.join(npm.cache, '..', '_npx')) + fs.mkdirSync(npxCacheDir, { recursive: true }) + + const validHash = 'valid456' + createNpxCacheEntry(npxCacheDir, validHash, { + name: 'valid-package', + version: '1.0.0', + dependencies: { + 'dep-package': '1.0.0', + }, + }, { + name: 'valid-package', + version: '1.0.0', + dependencies: { + 'dep-package': { + version: '1.0.0', + resolved: 'https://registry.npmjs.org/dep-package/-/dep-package-1.0.0.tgz', + integrity: 'sha512-...', + }, + }, + }) + + const nodeModulesDir = path.join(npxCacheDir, validHash, 'node_modules') + fs.mkdirSync(nodeModulesDir, { recursive: true }) + fs.mkdirSync(path.join(nodeModulesDir, 'dep-package')) + fs.writeFileSync( + path.join(nodeModulesDir, 'dep-package', 'package.json'), + JSON.stringify({ + name: 'dep-package', + version: '1.0.0', + }) + ) + + await npm.exec('cache', ['npx', 'info', validHash]) + t.matchSnapshot(joinedOutput(), 'shows valid package info with dependencies') +}) + +t.test('cache npx info: valid entry with _npx directory package', async t => { + const { npm, joinedOutput } = await loadMockNpm(t) + const npxCacheDir = path.join(npm.flatOptions.npxCache || path.join(npm.cache, '..', '_npx')) + fs.mkdirSync(npxCacheDir, { recursive: true }) + + const validHash = 'valid123' + createNpxCacheEntry(npxCacheDir, validHash, { + name: 'valid-package', + version: '1.0.0', + _npx: { packages: ['/path/to/valid-package'] }, + }, { + name: 'valid-package', + version: '1.0.0', + dependencies: { + 'valid-package': { + version: '1.0.0', + resolved: 'https://registry.npmjs.org/valid-package/-/valid-package-1.0.0.tgz', + integrity: 'sha512-...', + }, + }, + }) + + const nodeModulesDir = path.join(npxCacheDir, validHash, 'node_modules') + fs.mkdirSync(nodeModulesDir, { recursive: true }) + fs.mkdirSync(path.join(nodeModulesDir, 'valid-package')) + fs.writeFileSync( + path.join(nodeModulesDir, 'valid-package', 'package.json'), + JSON.stringify({ + name: 'valid-package', + version: '1.0.0', + }) + ) + + await npm.exec('cache', ['npx', 'info', validHash]) + t.matchSnapshot(joinedOutput(), 'shows valid package info with _npx directory package') +}) + +t.test('cache npx info: valid entry with a link dependency', async t => { + const { npm, joinedOutput } = await loadMockNpm(t) + const npxCacheDir = path.join( + npm.flatOptions.npxCache || path.join(npm.cache, '..', '_npx') + ) + fs.mkdirSync(npxCacheDir, { recursive: true }) + + const validHash = 'link123' + const pkgDir = path.join(npxCacheDir, validHash) + fs.mkdirSync(pkgDir) + + fs.writeFileSync( + path.join(pkgDir, 'package.json'), + JSON.stringify({ + name: 'link-package', + version: '1.0.0', + dependencies: { + 'linked-dep': 'file:./some-other-loc', + }, + }) + ) + + fs.writeFileSync( + path.join(pkgDir, 'npm-shrinkwrap.json'), + JSON.stringify({ + name: 'link-package', + version: '1.0.0', + dependencies: { + 'linked-dep': { + version: 'file:../some-other-loc', + }, + }, + }) + ) + + const nodeModulesDir = path.join(pkgDir, 'node_modules') + fs.mkdirSync(nodeModulesDir, { recursive: true }) + + const linkTarget = path.join(pkgDir, 'some-other-loc') + fs.mkdirSync(linkTarget) + fs.writeFileSync( + path.join(linkTarget, 'package.json'), + JSON.stringify({ name: 'linked-dep', version: '1.0.0' }) + ) + + fs.symlinkSync('../some-other-loc', path.join(nodeModulesDir, 'linked-dep')) + await npm.exec('cache', ['npx', 'info', validHash]) + + t.matchSnapshot( + joinedOutput(), + 'shows link dependency realpath (child.isLink branch)' + ) +}) diff --git a/workspaces/libnpmexec/lib/index.js b/workspaces/libnpmexec/lib/index.js index 78633a8cadb3c..2ffaae2290a67 100644 --- a/workspaces/libnpmexec/lib/index.js +++ b/workspaces/libnpmexec/lib/index.js @@ -1,20 +1,21 @@ 'use strict' +const { dirname, resolve } = require('node:path') +const crypto = require('node:crypto') const { mkdir } = require('node:fs/promises') const Arborist = require('@npmcli/arborist') const ciInfo = require('ci-info') -const crypto = require('node:crypto') const { log, input } = require('proc-log') const npa = require('npm-package-arg') const pacote = require('pacote') const { read } = require('read') const semver = require('semver') +const PackageJson = require('@npmcli/package-json') const { fileExists, localFileExists } = require('./file-exists.js') const getBinFromManifest = require('./get-bin-from-manifest.js') const noTTY = require('./no-tty.js') const runScript = require('./run-script.js') const isWindows = require('./is-windows.js') -const { dirname, resolve } = require('node:path') const binPaths = [] @@ -37,6 +38,7 @@ const missingFromTree = async ({ spec, tree, flatOptions, isNpxTree, shallow }) // - In local or global mode go with anything in the tree that matches // - If looking in the npx cache check if a newer version is available const npxByNameOnly = isNpxTree && spec.name === spec.raw + // If they gave a range and not a tag we still need to check if it's outdated. if (spec.registry && spec.type !== 'tag' && !npxByNameOnly) { // registry spec that is not a specific tag. const nodesBySpec = tree.inventory.query('packageName', spec.name) @@ -53,7 +55,8 @@ const missingFromTree = async ({ spec, tree, flatOptions, isNpxTree, shallow }) return { node } } // package requested by version range, only remaining registry type - if (semver.satisfies(node.package.version, spec.rawSpec)) { + // the npx tree shouldn't be ok w/ an outdated version + if (!isNpxTree && semver.satisfies(node.package.version, spec.rawSpec)) { return { node } } } @@ -293,6 +296,9 @@ const exec = async (opts) => { }) } binPaths.push(resolve(installDir, 'node_modules/.bin')) + const pkgJson = await PackageJson.load(installDir) + pkgJson.update({ _npx: { packages } }) + await pkgJson.save() } return await run() diff --git a/workspaces/libnpmexec/package.json b/workspaces/libnpmexec/package.json index 5009d76d12fe5..843ff7b5e3cbb 100644 --- a/workspaces/libnpmexec/package.json +++ b/workspaces/libnpmexec/package.json @@ -61,6 +61,7 @@ }, "dependencies": { "@npmcli/arborist": "^9.0.0", + "@npmcli/package-json": "^6.1.1", "@npmcli/run-script": "^9.0.1", "ci-info": "^4.0.0", "npm-package-arg": "^12.0.0",