Merge pull request #22 from pufferfish101007/rewrite-2024 - mega rewrite

Mega rewrite.

Notable changes:

modularisation
proper unit tests to ensure wasm (and imported js) is valid (resolves unit tests are not proper unit tests #18)
simplifies adding new blocks
boxed values use NaN boxing rather than tags
ints are i32 rather than i64
casting should always pick the best type to cast to
clones a bit less (resolves clone is potentially overused #20)
add compiler settings (currently mostly unused)
auto track implemented blocks (resolves Auto-track implemented blocks #15)
converts nans to zeros (resolves NaNs should be treated as 0 in mathematical operations #17)
makes output types more specific (e.g. +ve/-ve, nan-ness) - based off Update improved type analysis to develop TurboWarp/scratch-vm#216
adds own internal wasm! macro to generate code (with special pseudo-instructions for things like zeroing nans)
github actions deploys each branch to its own gh pages subdirectory
warped custom blocks with arguments supported
turbo mode works properly now
modules optimised using wasm-opt

Author: pufferfish101007

.github/workflows/ci.yaml renamed to .github/workflows/ci-checks.yaml

@@ -1,6 +1,6 @@
-on: [push, workflow_dispatch]
+on: [push, workflow_dispatch, pull_request]
 
-name: CI
+name: CI checks
 
 jobs:
   build:
@@ -16,18 +16,20 @@ jobs:
           profile: minimal
           toolchain: nightly
           override: true
+          target: wasm32-unknown-unknown
 
       - name: Run cargo check
         uses: actions-rs/cargo@v1
         with:
           command: check
+          args: --target=wasm32-unknown-unknown
 
   clippy:
     name: Lint (clippy)
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Install nightly toolchain with clippy available
         uses: actions-rs/toolchain@v1
@@ -48,7 +50,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Install nightly toolchain with rustfmt available
         uses: actions-rs/toolchain@v1
@@ -60,14 +62,12 @@ jobs:
 
       - name: Run cargo fmt
         uses: actions-rs/cargo@v1
-        continue-on-error: true  # WARNING: only for this example, remove it!
         with:
           command: fmt
           args: --all -- --check
-
-  buildwasm:
-    name: Build WASM & website
-    if: github.ref_name == 'main'
+          
+  test:
+    name: Run unit tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
@@ -79,34 +79,26 @@ jobs:
           profile: minimal
           toolchain: nightly
           override: true
-          target: wasm32-unknown-unknown
-      
-      - name: Install wasm-bindgen
+
+      - name: Run cargo test
         uses: actions-rs/cargo@v1
         with:
-          command: install
-          args: -f wasm-bindgen-cli
-          
-      - name: Install binaryen
-        run: sudo apt-get install binaryen
-      
-      - name: Install node
-        uses: actions/setup-node@v3
-        with:
-          node-version: "20.x"
-          
-      - name: Run npm install
-        run: npm install
-      
-      - name: Build
-        run: chmod +x build.sh && ./build.sh -pWV
-      
-      - name: Move files for gh pages
-        run: mv ./playground/dist docs && cp docs/index.html docs/404.html
-      
-      - name: Commit changes
-        uses: stefanzweifel/git-auto-commit-action@v4
-        with:
-          branch: gh-pages
-          create_branch: true
-          push_options: '--force'
+          command: test
+          args: >-
+            -- 
+            --skip cast::float::js_functions_match_declared_types 
+            --skip cast::int::js_functions_match_declared_types 
+            --skip cast::string::js_functions_match_declared_types 
+            --skip join::tests::js_functions_match_declared_types 
+            --skip lt::tests::js_functions_match_declared_types 
+            --skip gt::tests::js_functions_match_declared_types 
+            --skip equals::tests::js_functions_match_declared_types 
+            --skip length::tests::js_functions_match_declared_types 
+            --skip letter_of::tests::js_functions_match_declared_types 
+            --skip contains::tests::js_functions_match_declared_types
+            --skip dayssince2000::tests::js_functions_match_declared_types 
+            --skip looks::say::tests_debug::js_functions_match_declared_types
+            --skip looks::say::tests_non_debug::js_functions_match_declared_types
+            --skip looks::think::tests_debug::js_functions_match_declared_types
+            --skip looks::think::tests_non_debug::js_functions_match_declared_types
+ 

.github/workflows/deploy.yml

@@ -0,0 +1,76 @@
+on: [push, workflow_dispatch]
+
+name: Deploy
+
+jobs:
+  deploy:
+    name: Build WASM & website
+    runs-on: ubuntu-latest
+    env:
+      BRANCH_NAME: ${{ github.head_ref || github.ref_name }} 
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Install nightly toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          target: wasm32-unknown-unknown
+      
+      - name: Install wasm-bindgen
+        uses: actions-rs/cargo@v1
+        with:
+          command: install
+          args: -f wasm-bindgen-cli
+          
+      - name: Install cargo-outdir
+        uses: actions-rs/cargo@v1
+        with:
+          command: install
+          args: cargo-outdir
+
+      - name: Install binaryen
+        run: sudo apt-get install binaryen
+      
+      - name: Install node
+        uses: actions/setup-node@v3
+        with:
+          node-version: "20.x"
+          
+      - name: Run npm install
+        run: |
+          npm install
+          npm i -g vite
+          npm i -g binaryen@nightly
+      
+      - name: Build
+        env:
+          VITE_HASH_HISTORY: true
+        run: |
+          chmod +x build.sh && ./build.sh -Wpz
+          vite build --base=/hyperquark/$BRANCH_NAME/
+      
+      - name: Move files to tmp
+        run: mv ./playground/dist /tmp/hq-dist
+      
+      - name: checkout gh-pages
+        uses: actions/checkout@v4
+        with:
+          ref: gh-pages
+      
+      - name: move file to gh-pages
+        run: |
+          rm -rf ./$BRANCH_NAME
+          mv /tmp/hq-dist ./$BRANCH_NAME
+          #mv ./main/* ./
+      
+      - name: Commit changes
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          branch: gh-pages
+          push_options: '--force-with-lease'

.gitignore

@@ -4,8 +4,12 @@
 /bad.mjs
 /bad.wat
 /wabt*
-/js/
+/js/compiler
+/js/no-compiler
+/js/imports.ts
+/js/opcodes.js
 /node_modules/
 /package-lock.json
 /playground/dist/
-/.vite/
+/.vite/
+/.vscode/

Cargo.toml

@@ -8,20 +8,37 @@ publish = false
 serde = { version = "1.0", default-features = false, features = ["derive", "alloc"] }
 serde_json = { version = "1.0", default-features = false, features = ["alloc"] }
 enum-field-getter = { path = "enum-field-getter" }
-wasm-encoder = "0.214.0"
-wasm-bindgen = "0.2.92"
+wasm-encoder = "0.226.0"
 indexmap = { version = "2.0.0", default-features = false }
 hashers = "1.0.1"
 uuid = { version = "1.4.1", default-features = false, features = ["v4", "js"] }
 regex = "1.10.5"
 lazy-regex = "3.2.0"
+bitmask-enum = "2.2.5"
+itertools = { version = "0.13.0", default-features = false, features = ["use_alloc"] }
+split_exact = "1.1.0"
+wasm-bindgen = "0.2.92"
+serde-wasm-bindgen = "0.6.5"
+wasm-gen = { path = "wasm-gen" }
 
-#[dev-dependencies]
+[dev-dependencies]
+wasmparser = "0.226.0"
+wasmprinter = "0.226.0"
 #reqwest = { version = "0.11", features = ["blocking"] }
 
+[target.'cfg(not(target_family = "wasm"))'.dev-dependencies]
+ezno-checker = { git = "https://github.com/kaleidawave/ezno.git", rev = "96d5058bdbb0cde924be008ca1e5a67fe39f46b9" }
+
 [lib]
 crate-type = ["cdylib", "rlib"]
 
 [profile.release]
 lto = true
 opt-level = "z"
+
+[build-dependencies]
+convert_case = "0.6.0"
+
+[features]
+compiler = [] # if we only want to access flags, we don't want to additionally have all the compiler machinery
+default = ["compiler"]

README.md

@@ -5,22 +5,42 @@ Compile scratch projects to WASM
 
 ## Prerequisites
 
-- [Rust](https://rust-lang.org) (v1.65.0 or later)
+- [Rust](https://rust-lang.org) v1.65.0 or later
 - the `wasm32-unknown-unknown` target (`rustup target add wasm32-unknown-unknown`)
 - wasm-bindgen-cli (`cargo install -f wasm-bindgen-cli`)
 - wasm-opt (install binaryen using whatever package manager you use)
+- `cargo-outdir` (`cargo install cargo-outdir`)
 
 ## Building
 
 ```bash
-./build.sh -pVW # use -dVW for a debug build without optimisation
+./build.sh -Wp # use -Wd for a debug build without optimisation
 ```
 
 You may need to run `chmod +x build.sh` if it says it doesn't have permission.
 
 The build script has additonal configuration options; run `./build.sh -h` for info on these.
 
-If you experience runtime stack overflow errors in debug mode, try using the `-O` option to enable wasm-opt.
+If you experience runtime stack overflow errors in debug mode, try using the `-s` or `-z` options to enable wasm-opt; weird wasm errors in production mode may conversely be solved by *disabling* wasm-opt using the `-o` flag.
+
+## Adding a new block
+
+To add a new block named `category_opcode`, if it cannot be reduced to simpler blocks:
+1. create `src/instructions/category/opcode.rs`. Make sure to `use super::super::prelude::*` and create the relevant `pub` items:
+- (optional) `pub struct Fields` (must be `Debug` and `Clone`)
+- `pub fn wasm(func: &StepFunc, inputs: Rc<[IrType]>, (fields: &Fields)?) -> HQResult<Vec<InternalInstruction>>;`
+- - wasm is generated using the `wasm_gen::wasm` macro. See [its README](./wasm-gen/README.md) for usage instructions, or e.g. [say.rs](./src/instructions/looks/say.rs) for an example.
+- `pub fn acceptable_inputs() -> Rc<[IrType]>;`
+- - these should really be base types (see BASE_TYPES in [types.rs](./src/ir/types.rs))
+- `pub fn output_type(inputs: Rc<[IrType]>, (fields: &Fields)?) -> HQResult<Option<IrType>>;`
+- - the output type should be as restrictive as possible; loose output types can cause us to lose out on some optimisations
+- ensure to add relevant `instructions_test!`s - see [instructions/tests.rs](./src/instructions/tests.rs) for usage
+2. add `pub mod opcode;` to `src/instructions/category.rs`, creating the file if needed
+- if you're creating the category file, add `mod category;` to `src/instructions.rs`
+3. add the block to `from_normal_block` in `src/ir/blocks.rs`; in most cases this should be a direct mapping of `BlockOpcode::category_opcode => IrOpcode::category_opcode`
+4. add the block's input names to `input_names` in `src/ir/blocks.rs`
+
+If the block *can* be reduced to simpler steps, only carry out steps 3 and 4 above.
 
 ## generared WASM module memory layout