feat(ses,module-source): Dynamic import (#2639)

Closes: #291

## Description

This change adds support for dynamic `import` to `ses` and
`@endo/module-source`, such that `import(x)` in the scope of a module
loaded from source (using `ModuleSource` in a `Compartment` module
loading hook).

### Security Considerations

This change builds on prior work to ensure that dynamic import is
facilitated by a hidden lexical name injected by `ses`. The dynamic
import mechanism is isolated to the surrounding compartment and
specifiers are resolved on the surrounding module.

### Scaling Considerations

This change introduces a static analysis for the presence of a dynamic
`import` in the module source, which it communicates on the module
source, even if marshaled through JSON, to `ses` that it should create
an `import` closure bound to the calling module’s base specifier. This
avoids an allocation for most modules.

### Documentation Considerations

Only news included. Dynamic import is a language feature that is
expected to work in general and documented where JavaScript is
documented as a language.

### Testing Considerations

This change includes a minimal happy path test that verifies that
dynamic import produces a promise that settles on a module namespace
object. Note that dynamic import does not box namespaces regardless of
the `__noNamespaceBox__` compartment option.

### Compatibility Considerations

Backward compatible.

### Upgrade Considerations

None.

packages/compartment-mapper/test/optional.test.js

@@ -15,7 +15,8 @@ const fixtureOptionalDepsCjs = new URL(
 
 scaffold(
   'optionalDependencies/esm',
-  // this test fails because it relies on dynamic import
+  // fails for archives because dynamic import cannot reach modules not
+  // discovered during archival
   test,
   fixtureOptionalDepsEsm,
   async (t, { namespace }) => {
@@ -41,7 +42,7 @@ scaffold(
     );
   },
   4,
-  { knownFailure: true },
+  { knownArchiveFailure: true },
 );
 
 scaffold(

packages/compartment-mapper/test/scaffold.js

@@ -93,6 +93,7 @@ export function scaffold(
     addGlobals = {},
     policy,
     knownFailure = false,
+    knownArchiveFailure = false,
     tags = undefined,
     conditions = tags,
     strict = false,
@@ -111,7 +112,10 @@ export function scaffold(
   // wrapping each time allows for convenient use of test.only
   const wrap = (testFunc, testCategoryHint) => (title, implementation) => {
     // mark as known failure if available (but fallback to support test.only)
-    if (knownFailure) {
+    if (
+      knownFailure ||
+      (knownArchiveFailure && testCategoryHint === 'Archive')
+    ) {
       testFunc = testFunc.failing || testFunc;
     }
     return testFunc(title, async t => {

packages/module-source/NEWS.md

@@ -2,6 +2,10 @@ User-visible changes in `@endo/module-source`:
 
 # Next release
 
+- Supports dynamic `import` within a `ModuleSource` in conjunction with
+  a related change in `ses`.
+  For example, `await import(specifier)` can now call through to the
+  surrounding compartment's `importHook` to load and evaluate further modules.
 - Provides an XS-specific variant of `@endo/module-source` that adapts the
   native `ModuleSource` instead of entraining Babel.
 

packages/module-source/src/babelPlugin.js

@@ -49,6 +49,7 @@ function makeModulePlugins(options) {
     reexportMap,
     liveExportMap,
     importMeta,
+    dynamicImport,
   } = options;
 
   if (sourceType !== 'module') {
@@ -286,6 +287,7 @@ function makeModulePlugins(options) {
         CallExpression(path) {
           // import(FOO) -> $h_import(FOO)
           if (path.node.callee.type === 'Import') {
+            dynamicImport.present = true;
             path.node.callee = hiddenIdentifier(h.HIDDEN_IMPORT);
           }
         },

packages/module-source/src/module-source.js

@@ -82,6 +82,7 @@ export function ModuleSource(source, opts = {}) {
     reexportMap,
     fixedExportMap,
     exportAlls,
+    needsImport,
     needsImportMeta,
   } = analyzeModule(source, opts);
   this.imports = freeze([...keys(imports)]);
@@ -97,6 +98,7 @@ export function ModuleSource(source, opts = {}) {
   this.__liveExportMap__ = liveExportMap;
   this.__reexportMap__ = reexportMap;
   this.__fixedExportMap__ = fixedExportMap;
+  this.__needsImport__ = needsImport;
   this.__needsImportMeta__ = needsImportMeta;
   freeze(this);
 }

packages/module-source/src/transform-analyze.js

@@ -41,6 +41,7 @@ const makeCreateStaticRecord = transformSource =>
       hoistedDecls: [],
       importSources: Object.create(null),
       importDecls: [],
+      dynamicImport: { present: false },
       // enables passing import.meta usage hints up.
       importMeta: { present: false },
     };
@@ -103,6 +104,7 @@ const makeCreateStaticRecord = transformSource =>
   imports: ${h.HIDDEN_IMPORTS}, \
   liveVar: ${h.HIDDEN_LIVE}, \
   onceVar: ${h.HIDDEN_ONCE}, \
+  import: ${h.HIDDEN_IMPORT}, \
   importMeta: ${h.HIDDEN_META}, \
 }) => (function () { 'use strict'; \
   ${preamble} \
@@ -119,6 +121,7 @@ const makeCreateStaticRecord = transformSource =>
       liveExportMap: freeze(sourceOptions.liveExportMap),
       fixedExportMap: freeze(sourceOptions.fixedExportMap),
       reexportMap: freeze(sourceOptions.reexportMap),
+      needsImport: sourceOptions.dynamicImport.present,
       needsImportMeta: sourceOptions.importMeta.present,
       functorSource,
     });

packages/ses/NEWS.md

@@ -2,7 +2,11 @@ User-visible changes in `ses`:
 
 # Next version
 
-- Specifying the long discontinued `mathTaming` or `dateTaming` options logs a warning.
+- Adds support for dynamic `import` in conjunction with an update to
+  `@endo/module-source`.
+
+- Specifying the long-discontinued `mathTaming` or `dateTaming` options logs a
+  warning.
 
 # v1.10.0 (2024-11-13)
 

packages/ses/src/compartment.js

@@ -19,15 +19,15 @@ import {
   setGlobalObjectMutableProperties,
   setGlobalObjectEvaluators,
 } from './global-object.js';
-import { assert, assertEqual } from './error/assert.js';
+import { assert, assertEqual, q } from './error/assert.js';
 import { sharedGlobalPropertyNames } from './permits.js';
 import { load, loadNow } from './module-load.js';
 import { link } from './module-link.js';
 import { getDeferredExports } from './module-proxy.js';
 import { compartmentEvaluate } from './compartment-evaluate.js';
 import { makeSafeEvaluator } from './make-safe-evaluator.js';
 
-/** @import {ModuleDescriptor} from '../types.js' */
+/** @import {ModuleDescriptor, ModuleExportsNamespace} from '../types.js' */
 
 // moduleAliases associates every public module exports namespace with its
 // corresponding compartment and specifier so they can be used to link modules
@@ -297,6 +297,43 @@ export const makeCompartmentConstructor = (
 
     assign(globalObject, endowments);
 
+    /**
+     * In support dynamic import in a module source loaded by this compartment,
+     * like `await import(importSpecifier)`, induces this compartment to import
+     * a module, returning a promise for the resulting module exports
+     * namespace.
+     * Unlike `compartment.import`, never creates a box object for the
+     * namespace as that behavior is deprecated and inconsistent with the
+     * standard behavior of dynamic import.
+     * Obliges the caller to resolve import specifiers to their corresponding
+     * full specifier.
+     * That is, every module must have its own dynamic import function that
+     * closes over the surrounding module's full module specifier and calls
+     * through to this function.
+     * @param {string} fullSpecifier - A full specifier is a key in the
+     * compartment's module memo.
+     * The method `compartment.import` accepts a full specifier, but dynamic
+     * import accepts an import specifier and resolves it to a full specifier
+     * relative to the calling module's full specifier.
+     * @returns {Promise<ModuleExportsNamespace>}
+     */
+    const compartmentImport = async fullSpecifier => {
+      if (typeof resolveHook !== 'function') {
+        throw new TypeError(
+          `Compartment does not support dynamic import: no configured resolveHook for compartment ${q(name)}`,
+        );
+      }
+      await load(privateFields, moduleAliases, this, fullSpecifier);
+      const { execute, exportsProxy } = link(
+        privateFields,
+        moduleAliases,
+        this,
+        fullSpecifier,
+      );
+      execute();
+      return exportsProxy;
+    };
+
     weakmapSet(privateFields, this, {
       name: `${name}`,
       globalTransforms,
@@ -314,6 +351,7 @@ export const makeCompartmentConstructor = (
       instances,
       parentCompartment,
       noNamespaceBox,
+      compartmentImport,
     });
   }
 

packages/ses/src/module-instance.js

@@ -1,3 +1,5 @@
+/** @import {ModuleExportsNamespace} from '../types.js' */
+
 import { assert } from './error/assert.js';
 import { getDeferredExports } from './module-proxy.js';
 import {
@@ -131,13 +133,15 @@ export const makeModuleInstance = (
     __fixedExportMap__: fixedExportMap = {},
     __liveExportMap__: liveExportMap = {},
     __reexportMap__: reexportMap = {},
+    __needsImport__: needsImport = false,
     __needsImportMeta__: needsImportMeta = false,
     __syncModuleFunctor__,
   } = moduleSource;
 
   const compartmentFields = weakmapGet(privateFields, compartment);
 
-  const { __shimTransforms__, importMetaHook } = compartmentFields;
+  const { __shimTransforms__, resolveHook, importMetaHook, compartmentImport } =
+    compartmentFields;
 
   const { exportsProxy, exportsTarget, activate } = getDeferredExports(
     compartment,
@@ -171,6 +175,14 @@ export const makeModuleInstance = (
     importMetaHook(moduleSpecifier, importMeta);
   }
 
+  /** @type {(fullSpecifier: string) => Promise<ModuleExportsNamespace>} */
+  let dynamicImport;
+  if (needsImport) {
+    /** @param {string} importSpecifier */
+    dynamicImport = async importSpecifier =>
+      compartmentImport(resolveHook(importSpecifier, moduleSpecifier));
+  }
+
   // {_localName_: [{get, set, notify}]} used to merge all the export updaters.
   const localGetNotify = create(null);
 
@@ -462,6 +474,7 @@ export const makeModuleInstance = (
             imports: freeze(imports),
             onceVar: freeze(onceVar),
             liveVar: freeze(liveVar),
+            import: dynamicImport,
             importMeta,
           }),
         );

packages/ses/test/import.test.js

@@ -4,6 +4,7 @@
 /* eslint max-lines: 0 */
 
 import test from 'ava';
+import { ModuleSource } from '@endo/module-source';
 import '../index.js';
 import { resolveNode, makeNodeImporter } from './_node.js';
 import { makeImporter, makeStaticRetriever } from './_import-commons.js';
@@ -600,3 +601,21 @@ test('importMetaHook and meta from record', async t => {
   const { default: metaurl } = await compartment.import('./index.js');
   t.is(metaurl, 'https://example.com/index.js?foo');
 });
+
+test('dynamic import from source', async t => {
+  const c = new Compartment({
+    __options__: true,
+    __noNamespaceBox__: true,
+    resolveHook: s => s,
+    modules: {
+      '-': {
+        source: new ModuleSource(`
+          export const dynamic = import('-');
+        `),
+      },
+    },
+  });
+  const namespace = await c.import('-');
+  const namespace2 = await namespace.dynamic;
+  t.is(namespace, namespace2);
+});