add npm cache npx command and fix stale-version reuse in npx

docs/lib/content/commands/npm-cache.md

@@ -10,53 +10,47 @@ description: Manipulates packages cache
 
 ### Description
 
-Used to add, list, or clean the npm cache folder.
+Used to add, list, or clean the `npm cache` folder.
+Also used to view info about entries in the `npm exec` (aka `npx`) cache folder.
+
+#### `npm cache`
 
 * add:
-  Add the specified packages to the local cache.  This command is primarily
-  intended to be used internally by npm, but it can provide a way to
-  add data to the local installation cache explicitly.
+  Add the specified packages to the local cache.  This command is primarily intended to be used internally by npm, but it can provide a way to add data to the local installation cache explicitly.
 
 * clean:
-  Delete all data out of the cache folder.  Note that this is typically
-  unnecessary, as npm's cache is self-healing and resistant to data
-  corruption issues.
+  Delete a single entry or all entries out of the cache folder.  Note that this is typically unnecessary, as npm's cache is self-healing and resistant to data corruption issues.
+
+* ls:
+  List given entries or all entries in the local cache.
 
 * verify:
-  Verify the contents of the cache folder, garbage collecting any unneeded
-  data, and verifying the integrity of the cache index and all cached data.
+  Verify the contents of the cache folder, garbage collecting any unneeded data, and verifying the integrity of the cache index and all cached data.
+
+#### `npm cache npx`
+
+* ls:
+  List all entries in the npx cache.
+
+* rm:
+  Remove given entries or all entries from the npx cache.
+
+* info:
+  Get detailed information about given entries in the npx cache.
 
 ### Details
 
-npm stores cache data in an opaque directory within the configured `cache`,
-named `_cacache`. This directory is a
-[`cacache`](http://npm.im/cacache)-based content-addressable cache that
-stores all http request data as well as other package-related data. This
-directory is primarily accessed through `pacote`, the library responsible
-for all package fetching as of npm@5.
+npm stores cache data in an opaque directory within the configured `cache`, named `_cacache`. This directory is a [`cacache`](http://npm.im/cacache)-based content-addressable cache that stores all http request data as well as other package-related data. This directory is primarily accessed through `pacote`, the library responsible for all package fetching as of npm@5.
 
-All data that passes through the cache is fully verified for integrity on
-both insertion and extraction. Cache corruption will either trigger an
-error, or signal to `pacote` that the data must be refetched, which it will
-do automatically. For this reason, it should never be necessary to clear
-the cache for any reason other than reclaiming disk space, thus why `clean`
-now requires `--force` to run.
+All data that passes through the cache is fully verified for integrity on both insertion and extraction. Cache corruption will either trigger an error, or signal to `pacote` that the data must be refetched, which it will do automatically. For this reason, it should never be necessary to clear the cache for any reason other than reclaiming disk space, thus why `clean` now requires `--force` to run.
 
-There is currently no method exposed through npm to inspect or directly
-manage the contents of this cache. In order to access it, `cacache` must be
-used directly.
+There is currently no method exposed through npm to inspect or directly manage the contents of this cache. In order to access it, `cacache` must be used directly.
 
-npm will not remove data by itself: the cache will grow as new packages are
-installed.
+npm will not remove data by itself: the cache will grow as new packages are installed.
 
 ### A note about the cache's design
 
-The npm cache is strictly a cache: it should not be relied upon as a
-persistent and reliable data store for package data. npm makes no guarantee
-that a previously-cached piece of data will be available later, and will
-automatically delete corrupted contents. The primary guarantee that the
-cache makes is that, if it does return data, that data will be exactly the
-data that was inserted.
+The npm cache is strictly a cache: it should not be relied upon as a persistent and reliable data store for package data. npm makes no guarantee that a previously-cached piece of data will be available later, and will automatically delete corrupted contents. The primary guarantee that the cache makes is that, if it does return data, that data will be exactly the data that was inserted.
 
 To run an offline verification of existing cache contents, use `npm cache
 verify`.
@@ -74,6 +68,7 @@ verify`.
 * [npm install](/commands/npm-install)
 * [npm publish](/commands/npm-publish)
 * [npm pack](/commands/npm-pack)
+* [npm exec](/commands/npm-exec)
 * https://npm.im/cacache
 * https://npm.im/pacote
 * https://npm.im/@npmcli/arborist

lib/commands/cache.js

@@ -1,13 +1,15 @@
-const cacache = require('cacache')
-const pacote = require('pacote')
 const fs = require('node:fs/promises')
 const { join } = require('node:path')
+const cacache = require('cacache')
+const pacote = require('pacote')
 const semver = require('semver')
-const BaseCommand = require('../base-cmd.js')
 const npa = require('npm-package-arg')
 const jsonParse = require('json-parse-even-better-errors')
 const localeCompare = require('@isaacs/string-locale-compare')('en')
 const { log, output } = require('proc-log')
+const PkgJson = require('@npmcli/package-json')
+const BaseCommand = require('../base-cmd.js')
+const abbrev = require('abbrev')
 
 const searchCachePackage = async (path, parsed, cacheKeys) => {
   const searchMFH = new RegExp(`^make-fetch-happen:request-cache:.*(?<!/[@a-zA-Z]+)/${parsed.name}/-/(${parsed.name}[^/]+.tgz)$`)
@@ -62,20 +64,23 @@ const searchCachePackage = async (path, parsed, cacheKeys) => {
 }
 
 class Cache extends BaseCommand {
-  static description = 'Manipulates packages cache'
+  static description = 'Manipulates packages and npx cache'
   static name = 'cache'
   static params = ['cache']
   static usage = [
     'add <package-spec>',
     'clean [<key>]',
     'ls [<name>@<version>]',
     'verify',
+    'npx ls',
+    'npx rm [<key>]',
+    'npx info [<key>]',
   ]
 
   static async completion (opts) {
     const argv = opts.conf.argv.remain
     if (argv.length === 2) {
-      return ['add', 'clean', 'verify', 'ls']
+      return ['add', 'clean', 'verify', 'ls', 'npx']
     }
 
     // TODO - eventually...
@@ -99,14 +104,31 @@ class Cache extends BaseCommand {
         return await this.verify()
       case 'ls':
         return await this.ls(args)
+      case 'npx':
+        return await this.npx(args)
+      default:
+        throw this.usageError()
+    }
+  }
+
+  // npm cache npx
+  async npx ([cmd, ...keys]) {
+    switch (cmd) {
+      case 'ls':
+        return await this.npxLs(keys)
+      case 'rm':
+        return await this.npxRm(keys)
+      case 'info':
+        return await this.npxInfo(keys)
       default:
         throw this.usageError()
     }
   }
 
-  // npm cache clean [pkg]*
+  // npm cache clean [spec]*
   async clean (args) {
-    const cachePath = join(this.npm.cache, '_cacache')
+    // this is a derived value
+    const cachePath = this.npm.flatOptions.cache
     if (args.length === 0) {
       if (!this.npm.config.get('force')) {
         throw new Error(`As of npm@5, the npm cache self-heals from corruption issues
@@ -169,11 +191,12 @@ class Cache extends BaseCommand {
   }
 
   async verify () {
-    const cache = join(this.npm.cache, '_cacache')
-    const prefix = cache.indexOf(process.env.HOME) === 0
-      ? `~${cache.slice(process.env.HOME.length)}`
-      : cache
-    const stats = await cacache.verify(cache)
+    // this is a derived value
+    const cachePath = this.npm.flatOptions.cache
+    const prefix = cachePath.indexOf(process.env.HOME) === 0
+      ? `~${cachePath.slice(process.env.HOME.length)}`
+      : cachePath
+    const stats = await cacache.verify(cachePath)
     output.standard(`Cache verified and compressed (${prefix})`)
     output.standard(`Content verified: ${stats.verifiedContent} (${stats.keptSize} bytes)`)
     if (stats.badContentCount) {
@@ -189,9 +212,10 @@ class Cache extends BaseCommand {
     output.standard(`Finished in ${stats.runTime.total / 1000}s`)
   }
 
-  // npm cache ls [--package <spec> ...]
+  // npm cache ls [<spec> ...]
   async ls (specs) {
-    const cachePath = join(this.npm.cache, '_cacache')
+    // This is a derived value
+    const { cache: cachePath } = this.npm.flatOptions
     const cacheKeys = Object.keys(await cacache.ls(cachePath))
     if (specs.length > 0) {
       // get results for each package spec specified
@@ -211,6 +235,136 @@ class Cache extends BaseCommand {
     }
     cacheKeys.sort(localeCompare).forEach(key => output.standard(key))
   }
+
+  async #npxCache (keys = []) {
+    // This is a derived value
+    const { npxCache } = this.npm.flatOptions
+    let dirs
+    try {
+      dirs = await fs.readdir(npxCache, { encoding: 'utf-8' })
+    } catch {
+      output.standard('npx cache does not exist')
+      return
+    }
+    const cache = {}
+    const { default: pMap } = await import('p-map')
+    await pMap(dirs, async e => {
+      const pkgPath = join(npxCache, e)
+      cache[e] = {
+        hash: e,
+        path: pkgPath,
+        valid: false,
+      }
+      try {
+        const pkgJson = await PkgJson.load(pkgPath)
+        cache[e].package = pkgJson.content
+        cache[e].valid = true
+      } catch {
+        // Defaults to not valid already
+      }
+    }, { concurrency: 20 })
+    if (!keys.length) {
+      return cache
+    }
+    const result = {}
+    const abbrevs = abbrev(Object.keys(cache))
+    for (const key of keys) {
+      if (!abbrevs[key]) {
+        throw this.usageError(`Invalid npx key ${key}`)
+      }
+      result[abbrevs[key]] = cache[abbrevs[key]]
+    }
+    return result
+  }
+
+  async npxLs () {
+    const cache = await this.#npxCache()
+    for (const key in cache) {
+      const { hash, valid, package: pkg } = cache[key]
+      let result = `${hash}:`
+      if (!valid) {
+        result = `${result} (empty/invalid)`
+      } else if (pkg?._npx) {
+        result = `${result} ${pkg._npx.packages.join(', ')}`
+      } else {
+        result = `${result} (unknown)`
+      }
+      output.standard(result)
+    }
+  }
+
+  async npxRm (keys) {
+    if (!keys.length) {
+      if (!this.npm.config.get('force')) {
+        throw this.usageError('Please use --force to remove entire npx cache')
+      }
+      const { npxCache } = this.npm.flatOptions
+      if (!this.npm.config.get('dry-run')) {
+        return fs.rm(npxCache, { recursive: true, force: true })
+      }
+    }
+
+    const cache = await this.#npxCache(keys)
+    for (const key in cache) {
+      const { path: cachePath } = cache[key]
+      output.standard(`Removing npx key at ${cachePath}`)
+      if (!this.npm.config.get('dry-run')) {
+        await fs.rm(cachePath, { recursive: true })
+      }
+    }
+  }
+
+  async npxInfo (keys) {
+    const chalk = this.npm.chalk
+    if (!keys.length) {
+      throw this.usageError()
+    }
+    const cache = await this.#npxCache(keys)
+    const Arborist = require('@npmcli/arborist')
+    for (const key in cache) {
+      const { hash, path, package: pkg } = cache[key]
+      let valid = cache[key].valid
+      const results = []
+      try {
+        if (valid) {
+          const arb = new Arborist({ path })
+          const tree = await arb.loadVirtual()
+          if (pkg._npx) {
+            results.push('packages:')
+            for (const p of pkg._npx.packages) {
+              const parsed = npa(p)
+              if (parsed.type === 'directory') {
+                // in the tree the spec is relative, even if the dependency spec is absolute, so we can't find it by name or spec.
+                results.push(`- ${chalk.cyan(p)}`)
+              } else {
+                results.push(`- ${chalk.cyan(p)} (${chalk.blue(tree.children.get(parsed.name).pkgid)})`)
+              }
+            }
+          } else {
+            results.push('packages: (unknown)')
+            results.push(`dependencies:`)
+            for (const dep in pkg.dependencies) {
+              const child = tree.children.get(dep)
+              if (child.isLink) {
+                results.push(`- ${chalk.cyan(child.realpath)}`)
+              } else {
+                results.push(`- ${chalk.cyan(child.pkgid)}`)
+              }
+            }
+          }
+        }
+      } catch (ex) {
+        valid = false
+      }
+      const v = valid ? chalk.green('valid') : chalk.red('invalid')
+      output.standard(`${v} npx cache entry with key ${chalk.blue(hash)}`)
+      output.standard(`location: ${chalk.blue(path)}`)
+      if (valid) {
+        output.standard(results.join('\n'))
+      }
+      output.standard('')
+    }
+  }
 }
 
 module.exports = Cache

package-lock.json

@@ -18864,6 +18864,7 @@
       "license": "ISC",
       "dependencies": {
         "@npmcli/arborist": "^9.0.0",
+        "@npmcli/package-json": "^6.1.1",
         "@npmcli/run-script": "^9.0.1",
         "ci-info": "^4.0.0",
         "npm-package-arg": "^12.0.0",