Obtain wasm packages from emscripten-forge when building deployment (#61

)

* Obtain wasm packages from emscripten-forge when building deployment

* Fix CI

* Increase delay at start of tests

.github/workflows/test.yml

@@ -36,9 +36,6 @@ jobs:
         run: |
           npm install
 
-      - name: Fetch wasm dependencies
-        run: npm run fetch:wasm
-
       - name: Build
         run: |
           npm run build

.gitignore

@@ -9,5 +9,8 @@ test/package-lock.json
 *.js
 *.wasm
 cockle_wasm_env/
+cockle-config.json
 demo/package-lock.json
+demo/assets/*.json
+test/assets/*.json
 src/version.ts

README.md

@@ -18,7 +18,6 @@ a `micromamba` environment as part of the `npm prepack` process.
 micromamba env create -f environment-dev.yml -y
 micromamba activate cockle
 npm install
-npm run fetch:wasm
 npm run build
 npm run lint:check
 ```

cockle-config-base.json

@@ -0,0 +1,23 @@
+[
+  {
+    "package": "cockle_fs",
+    "modules": [
+      {
+        "name": "fs",
+        "commands": ""
+      }
+    ]
+  },
+  {
+    "package": "coreutils",
+    "modules": [
+      {
+        "name": "coreutils",
+        "commands": "basename,cat,chmod,cp,cut,date,dir,dircolors,dirname,echo,env,expr,head,id,join,ln,logname,ls,md5sum,mkdir,mv,nl,pwd,realpath,rm,rmdir,seq,sha1sum,sha224sum,sha256sum,sha384sum,sha512sum,sleep,sort,stat,stty,tail,touch,tr,tty,uname,uniq,vdir,wc"
+      }
+    ]
+  },
+  {
+    "package": "grep"
+  }
+]

demo/cockle-config-in.json

@@ -0,0 +1,5 @@
+[
+  {
+    "package": "lua"
+  }
+]

demo/package.json

@@ -8,8 +8,8 @@
     "types": "lib/index.d.ts",
     "private": true,
     "scripts": {
-        "build": "npm run build:wasm && rspack build",
-        "build:wasm": "cp node_modules/@jupyterlite/cockle/lib/wasm/*.js assets/ && cp node_modules/@jupyterlite/cockle/lib/wasm/*.wasm assets/",
+        "build": "rspack build",
+        "postbuild": "node node_modules/@jupyterlite/cockle/lib/tools/prepare_wasm.js --copy assets",
         "serve": "rspack serve"
     },
     "devDependencies": {

package.json

@@ -14,22 +14,20 @@
     },
     "files": [
         "lib/**/*.{d.ts,eot,gif,html,jpg,js,js.map,json,png,svg,wasm,woff2,ttf}",
-        "src/**/*.ts"
+        "src/**/*.ts",
+        "cockle-config-base.json"
     ],
     "main": "lib/index.js",
     "types": "lib/index.d.ts",
     "scripts": {
-        "fetch:wasm:create-env": "micromamba create -p $(pwd)/cockle_wasm_env -y cockle_fs grep coreutils lua --platform=emscripten-wasm32 -c https://repo.mamba.pm/emscripten-forge -c https://repo.mamba.pm/conda-forge",
-        "fetch:wasm:copy": "mkdir -p src/wasm && cp $(pwd)/cockle_wasm_env/bin/*.js src/wasm/ && cp $(pwd)/cockle_wasm_env/bin/*.wasm src/wasm/",
-        "fetch:wasm": "npm run fetch:wasm:create-env && npm run fetch:wasm:copy",
-        "build": "tsc && npm run build:worker && cp src/wasm/*.wasm lib/wasm/",
+        "build": "tsc && npm run build:worker",
         "build:worker": "rspack --config worker.rspack.config.js --mode=development",
         "eslint": "npm run eslint:check -- --fix",
         "eslint:check": "eslint . --cache --ext .ts,.tsx",
         "lint": "npm run prettier && npm run eslint",
         "lint:check": "npm run prettier:check && npm run eslint:check",
         "prebuild": "node -p \"'export const COCKLE_VERSION = ' + JSON.stringify(require('./package.json').version) + ';'\" > src/version.ts",
-        "prepack": "npm install && npm run fetch:wasm && npm run build",
+        "prepack": "npm install && npm run build",
         "prettier": "prettier --list-different --write \"**/*{.ts,.tsx,.js,.jsx,.css,.json,.md,.yml}\"",
         "prettier:check": "prettier --list-different \"**/*{.ts,.tsx,.js,.jsx,.css,.json,.md,.yml}\""
     },

src/command_registry.ts

@@ -1,27 +1,10 @@
 import { ICommandRunner } from './commands/command_runner';
-import { CoreutilsCommandRunner } from './commands/coreutils_command_runner';
-import { GrepCommandRunner } from './commands/grep_command_runner';
-import { LuaCommandRunner } from './commands/lua_command_runner';
 import * as AllBuiltinCommands from './builtin';
 import { WasmLoader } from './wasm_loader';
 
 export class CommandRegistry {
   constructor(wasmLoader: WasmLoader) {
     this.registerBuiltinCommands(AllBuiltinCommands);
-
-    this._commandRunners = [
-      new CoreutilsCommandRunner(wasmLoader),
-      new GrepCommandRunner(wasmLoader),
-      new LuaCommandRunner(wasmLoader)
-    ];
-
-    // Command name -> runner mapping
-    // Should probably check not overwriting any command names
-    for (const runner of this._commandRunners) {
-      for (const name of runner.names()) {
-        this._map.set(name, runner);
-      }
-    }
   }
 
   get(name: string): ICommandRunner | null {
@@ -33,10 +16,13 @@ export class CommandRegistry {
   }
 
   /**
-   * Register a command runner under a single name.
+   * Register a command runner under all of its names.
    */
-  register(name: string, commandRunner: ICommandRunner) {
-    this._map.set(name, commandRunner);
+  register(commandRunner: ICommandRunner) {
+    // Should probably check not overwriting any command names
+    for (const name of commandRunner.names()) {
+      this._map.set(name, commandRunner);
+    }
   }
 
   registerBuiltinCommands(commands: any) {
@@ -47,14 +33,13 @@ export class CommandRegistry {
       try {
         const obj = new (cls as any)();
         if (obj instanceof AllBuiltinCommands.BuiltinCommand) {
-          this.register(obj.name, obj);
+          this.register(obj);
         }
       } catch {
         // If there is any problem registering a command runner this way, silently fail.
       }
     }
   }
 
-  private _commandRunners: ICommandRunner[];
   private _map: Map<string, ICommandRunner> = new Map();
 }

src/commands/wasm_command_runner.ts

@@ -3,12 +3,20 @@ import { Context } from '../context';
 import { ExitCode } from '../exit_code';
 import { WasmLoader } from '../wasm_loader';
 
-export abstract class WasmCommandRunner implements ICommandRunner {
-  constructor(readonly wasmLoader: WasmLoader) {}
-
-  abstract moduleName(): string;
+export class WasmCommandRunner implements ICommandRunner {
+  constructor(
+    readonly wasmLoader: WasmLoader,
+    readonly _moduleName: string,
+    readonly _commandNames: string[]
+  ) {}
+
+  moduleName(): string {
+    return this._moduleName;
+  }
 
-  abstract names(): string[];
+  names(): string[] {
+    return this._commandNames;
+  }
 
   async run(cmdName: string, context: Context): Promise<number> {
     const { args, fileSystem, mountpoint, stdin, stdout, stderr } = context;

src/shell_impl.ts

@@ -14,6 +14,7 @@ import { FileInput, FileOutput, IInput, IOutput, Pipe, TerminalInput, TerminalOu
 import { CommandNode, PipeNode, parse } from './parse';
 import { longestStartsWith, toColumns } from './utils';
 import { WasmLoader } from './wasm_loader';
+import { WasmCommandRunner } from './commands/wasm_command_runner';
 
 /**
  * Shell implementation.
@@ -38,6 +39,7 @@ export class ShellImpl implements IShell {
   }
 
   async initialize() {
+    await this._initWasmPackages();
     await this._initFilesystem();
   }
 
@@ -219,6 +221,38 @@ export class ShellImpl implements IShell {
     }
   }
 
+  private async _initWasmPackages(): Promise<void> {
+    const url = this.options.wasmBaseUrl + 'cockle-config.json';
+    const response = await fetch(url);
+    if (!response.ok) {
+      // Would be nice to report this via the terminal.
+      console.error(`Failed to fetch ${url}, terminal cannot function without it`);
+    }
+
+    const cockleConfig = await response.json();
+    // Check JSON follows schema?
+    // May want to store JSON config.
+
+    const packageNames = cockleConfig.map((x: any) => x.package);
+    const fsPackage = 'cockle_fs';
+    if (!packageNames.includes(fsPackage)) {
+      console.error(`cockle-config.json does not include required package '${fsPackage}'`);
+    }
+
+    // Create a command runners for each wasm module of each emscripten-forge package.
+    for (const pkgConfig of cockleConfig) {
+      if (pkgConfig.package === fsPackage) {
+        continue;
+      }
+
+      for (const module of pkgConfig.modules) {
+        const commands = module.commands.split(',');
+        const runner = new WasmCommandRunner(this._wasmLoader, module.name, commands);
+        this._commandRegistry.register(runner);
+      }
+    }
+  }
+
   private async _runCommands(cmdText: string): Promise<void> {
     this.options.enableBufferedStdinCallback(true);