feat: add support for npm modules

.eslintrc.cjs

@@ -12,6 +12,7 @@ module.exports = {
     complexity: 'off',
     'import/extensions': 'off',
     'max-lines': 'off',
+    'max-lines-per-function': 'off',
     'max-statements': 'off',
     'node/no-missing-import': 'off',
     'no-magic-numbers': 'off',

.gitignore

@@ -2,6 +2,7 @@
 *.swp
 npm-debug.log
 node_modules
+!test/fixtures/**/node_modules
 /core
 .eslintcache
 .npmrc

deno/bundle.ts

@@ -1,6 +1,6 @@
 import { writeStage2 } from './lib/stage2.ts'
 
 const [payload] = Deno.args
-const { basePath, destPath, externals, functions, importMapData } = JSON.parse(payload)
+const { basePath, destPath, externals, functions, importMapData, vendorDirectory } = JSON.parse(payload)
 
-await writeStage2({ basePath, destPath, externals, functions, importMapData })
+await writeStage2({ basePath, destPath, externals, functions, importMapData, vendorDirectory })

deno/lib/stage2.ts

@@ -3,7 +3,7 @@ import { build, LoadResponse } from 'https://deno.land/x/[email protected]/mod.ts'
 import * as path from 'https://deno.land/[email protected]/path/mod.ts'
 
 import type { InputFunction, WriteStage2Options } from '../../shared/stage2.ts'
-import { importMapSpecifier, virtualRoot } from '../../shared/consts.ts'
+import { importMapSpecifier, virtualRoot, virtualVendorRoot } from '../../shared/consts.ts'
 import { PUBLIC_SPECIFIER, STAGE2_SPECIFIER } from './consts.ts'
 import { inlineModule, loadFromVirtualRoot, loadWithRetry } from './common.ts'
 
@@ -63,7 +63,13 @@ const getVirtualPath = (basePath: string, filePath: string) => {
   return url
 }
 
-const stage2Loader = (basePath: string, functions: InputFunction[], externals: Set<string>, importMapData?: string) => {
+const stage2Loader = (
+  basePath: string,
+  functions: InputFunction[],
+  externals: Set<string>,
+  importMapData: string | undefined,
+  vendorDirectory?: string,
+) => {
   return async (specifier: string): Promise<LoadResponse | undefined> => {
     if (specifier === STAGE2_SPECIFIER) {
       const stage2Entry = getStage2Entry(basePath, functions)
@@ -86,13 +92,24 @@ const stage2Loader = (basePath: string, functions: InputFunction[], externals: S
       return loadFromVirtualRoot(specifier, virtualRoot, basePath)
     }
 
+    if (vendorDirectory !== undefined && specifier.startsWith(virtualVendorRoot)) {
+      return loadFromVirtualRoot(specifier, virtualVendorRoot, vendorDirectory)
+    }
+
     return await loadWithRetry(specifier)
   }
 }
 
-const writeStage2 = async ({ basePath, destPath, externals, functions, importMapData }: WriteStage2Options) => {
+const writeStage2 = async ({
+  basePath,
+  destPath,
+  externals,
+  functions,
+  importMapData,
+  vendorDirectory,
+}: WriteStage2Options) => {
   const importMapURL = importMapData ? importMapSpecifier : undefined
-  const loader = stage2Loader(basePath, functions, new Set(externals), importMapData)
+  const loader = stage2Loader(basePath, functions, new Set(externals), importMapData, vendorDirectory)
   const bytes = await build([STAGE2_SPECIFIER], loader, importMapURL)
   const directory = path.dirname(destPath)
 

node/bundler.test.ts

@@ -131,7 +131,7 @@ test('Prints a nice error message when user tries importing NPM module', async (
   } catch (error) {
     expect(error).toBeInstanceOf(BundleError)
     expect((error as BundleError).message).toEqual(
-      `It seems like you're trying to import an npm module. This is only supported via CDNs like esm.sh. Have you tried 'import mod from "https://esm.sh/p-retry"'?`,
+      `It seems like you're trying to import an npm module. This is only supported via CDNs like esm.sh. Have you tried 'import mod from "https://esm.sh/parent-1"'?`,
     )
   } finally {
     await cleanup()
@@ -446,3 +446,32 @@ test('Handles imports with the `node:` prefix', async () => {
 
   await cleanup()
 })
+
+test('Loads npm modules from bare specifiers with and without the `npm:` prefix', async () => {
+  const { basePath, cleanup, distPath } = await useFixture('imports_npm_module')
+  const sourceDirectory = join(basePath, 'functions')
+  const declarations: Declaration[] = [
+    {
+      function: 'func1',
+      path: '/func1',
+    },
+  ]
+  const vendorDirectory = await tmp.dir()
+
+  await bundle([sourceDirectory], distPath, declarations, {
+    basePath,
+    featureFlags: { edge_functions_npm_modules: true },
+    vendorDirectory: vendorDirectory.path,
+  })
+  const manifestFile = await readFile(resolve(distPath, 'manifest.json'), 'utf8')
+  const manifest = JSON.parse(manifestFile)
+  const bundlePath = join(distPath, manifest.bundles[0].asset)
+  const { func1 } = await runESZIP(bundlePath, vendorDirectory.path)
+
+  expect(func1).toBe(
+    `<parent-1><child-1>JavaScript</child-1></parent-1>, <parent-2><child-2><grandchild-1>APIs<cwd>${process.cwd()}</cwd></grandchild-1></child-2></parent-2>, <parent-3><child-2><grandchild-1>Markup<cwd>${process.cwd()}</cwd></grandchild-1></child-2></parent-3>`,
+  )
+
+  await cleanup()
+  await rm(vendorDirectory.path, { force: true, recursive: true })
+})

node/bundler.ts

@@ -17,40 +17,43 @@ import { bundle as bundleESZIP } from './formats/eszip.js'
 import { ImportMap } from './import_map.js'
 import { getLogger, LogFunction } from './logger.js'
 import { writeManifest } from './manifest.js'
+import { vendorNPMSpecifiers } from './npm_dependencies.js'
 import { ensureLatestTypes } from './types.js'
 
-interface BundleOptions {
+export interface BundleOptions {
   basePath?: string
+  bootstrapURL?: string
   cacheDirectory?: string
   configPath?: string
   debug?: boolean
   distImportMapPath?: string
   featureFlags?: FeatureFlags
   importMapPaths?: (string | undefined)[]
+  internalSrcFolder?: string
   onAfterDownload?: OnAfterDownloadHook
   onBeforeDownload?: OnBeforeDownloadHook
   systemLogger?: LogFunction
-  internalSrcFolder?: string
-  bootstrapURL?: string
+  vendorDirectory?: string
 }
 
-const bundle = async (
+export const bundle = async (
   sourceDirectories: string[],
   distDirectory: string,
   tomlDeclarations: Declaration[] = [],
   {
     basePath: inputBasePath,
+    bootstrapURL = 'https://edge.netlify.com/bootstrap/index-combined.ts',
     cacheDirectory,
     configPath,
     debug,
     distImportMapPath,
     featureFlags: inputFeatureFlags,
     importMapPaths = [],
+    internalSrcFolder,
     onAfterDownload,
     onBeforeDownload,
     systemLogger,
-    internalSrcFolder,
-    bootstrapURL = 'https://edge.netlify.com/bootstrap/index-combined.ts',
+    vendorDirectory,
   }: BundleOptions = {},
 ) => {
   const logger = getLogger(systemLogger, debug)
@@ -95,6 +98,18 @@ const bundle = async (
   const internalFunctions = internalSrcFolder ? await findFunctions([internalSrcFolder]) : []
   const functions = [...internalFunctions, ...userFunctions]
 
+  const vendor =
+    featureFlags.edge_functions_npm_modules &&
+    (await vendorNPMSpecifiers(
+      basePath,
+      functions.map(({ path }) => path),
+      vendorDirectory,
+    ))
+
+  if (vendor) {
+    importMap.add(vendor.importMap)
+  }
+
   const functionBundle = await bundleESZIP({
     basePath,
     buildID,
@@ -105,6 +120,7 @@ const bundle = async (
     functions,
     featureFlags,
     importMap,
+    vendorDirectory: vendor ? vendor.directory : undefined,
   })
 
   // The final file name of the bundles contains a SHA256 hash of the contents,
@@ -117,7 +133,6 @@ const bundle = async (
   const internalConfigPromises = internalFunctions.map(
     async (func) => [func.name, await getFunctionConfig({ func, importMap, deno, log: logger, bootstrapURL })] as const,
   )
-
   const userConfigPromises = userFunctions.map(
     async (func) => [func.name, await getFunctionConfig({ func, importMap, deno, log: logger, bootstrapURL })] as const,
   )
@@ -153,6 +168,10 @@ const bundle = async (
     layers: deployConfig.layers,
   })
 
+  if (vendor) {
+    await vendor.cleanup()
+  }
+
   if (distImportMapPath) {
     await importMap.writeToFile(distImportMapPath)
   }
@@ -224,6 +243,3 @@ const createFunctionConfig = ({ internalFunctionsWithConfig, declarations }: Cre
       [functionName]: addGeneratorFallback(mergedConfigFields),
     }
   }, {} as Record<string, FunctionConfig>)
-
-export { bundle }
-export type { BundleOptions }

node/feature_flags.ts

@@ -1,6 +1,7 @@
 const defaultFlags = {
   edge_functions_fail_unsupported_regex: false,
   edge_functions_path_urlpattern: false,
+  edge_functions_npm_modules: false,
 }
 
 type FeatureFlag = keyof typeof defaultFlags

node/formats/eszip.ts

@@ -1,7 +1,7 @@
 import { join } from 'path'
 import { pathToFileURL } from 'url'
 
-import { virtualRoot } from '../../shared/consts.js'
+import { virtualRoot, virtualVendorRoot } from '../../shared/consts.js'
 import type { WriteStage2Options } from '../../shared/stage2.js'
 import { DenoBridge } from '../bridge.js'
 import { Bundle, BundleFormat } from '../bundle.js'
@@ -23,6 +23,7 @@ interface BundleESZIPOptions {
   featureFlags: FeatureFlags
   functions: EdgeFunction[]
   importMap: ImportMap
+  vendorDirectory?: string
 }
 
 const bundleESZIP = async ({
@@ -34,23 +35,27 @@ const bundleESZIP = async ({
   externals,
   functions,
   importMap,
+  vendorDirectory,
 }: BundleESZIPOptions): Promise<Bundle> => {
   const extension = '.eszip'
   const destPath = join(distDirectory, `${buildID}${extension}`)
-  const { bundler, importMap: bundlerImportMap } = getESZIPPaths()
-
-  // Transforming all paths under `basePath` to use the virtual root prefix.
   const importMapPrefixes: Record<string, string> = {
     [`${pathToFileURL(basePath)}/`]: virtualRoot,
   }
-  const importMapData = importMap.getContents(importMapPrefixes)
 
+  if (vendorDirectory !== undefined) {
+    importMapPrefixes[`${pathToFileURL(vendorDirectory)}/`] = virtualVendorRoot
+  }
+
+  const { bundler, importMap: bundlerImportMap } = getESZIPPaths()
+  const importMapData = JSON.stringify(importMap.getContents(importMapPrefixes))
   const payload: WriteStage2Options = {
     basePath,
     destPath,
     externals,
     functions,
-    importMapData: JSON.stringify(importMapData),
+    importMapData,
+    vendorDirectory,
   }
   const flags = ['--allow-all', '--no-config', `--import-map=${bundlerImportMap}`]
 

node/node_builtins.ts

@@ -0,0 +1,53 @@
+// From https://github.com/denoland/deno/tree/main/ext/node/polyfills.
+export const builtInModules = [
+  'assert',
+  'assert/strict',
+  'async_hooks',
+  'buffer',
+  'child_process',
+  'cluster',
+  'console',
+  'constants',
+  'crypto',
+  'dgram',
+  'diagnostics_channel',
+  'dns',
+  'events',
+  'fs',
+  'fs/promises',
+  'http',
+  'http2',
+  'https',
+  'inspector',
+  'module',
+  'net',
+  'os',
+  'path',
+  'path/posix',
+  'path/win32',
+  'perf_hooks',
+  'process',
+  'punycode',
+  'querystring',
+  'readline',
+  'repl',
+  'stream',
+  'stream/promises',
+  'stream/web',
+  'string_decoder',
+  'sys',
+  'timers',
+  'timers/promises',
+  'tls',
+  'trace_events',
+  'tty',
+  'url',
+  'util',
+  'util/types',
+  'v8',
+  'vm',
+  'wasi',
+  'webcrypto',
+  'worker_threads',
+  'zlib',
+]