test: improve test infrastructure

.github/workflows/main.yml

@@ -125,17 +125,13 @@ jobs:
         export WASIP1_DIR=$(realpath $($CLANG_DIR/clang -print-resource-dir)/lib/wasip1/)
         export WASIP2_DIR=$(realpath $($CLANG_DIR/clang -print-resource-dir)/lib/wasip2/)
         mkdir -p $WASI_DIR $WASIP1_DIR $WASIP2_DIR
-        cp download/libclang_rt.builtins-wasm32.a $WASI_DIR
-        cp download/libclang_rt.builtins-wasm32.a $WASIP1_DIR
-        cp download/libclang_rt.builtins-wasm32.a $WASIP2_DIR
+        cp build/download/libclang_rt.builtins-wasm32.a $WASI_DIR
+        cp build/download/libclang_rt.builtins-wasm32.a $WASIP1_DIR
+        cp build/download/libclang_rt.builtins-wasm32.a $WASIP2_DIR
         TARGET_TRIPLE=wasm32-wasi make test
-        rm -r build
         TARGET_TRIPLE=wasm32-wasip1 make test
-        rm -r build
         TARGET_TRIPLE=wasm32-wasip2 make test
-        rm -r build
         TARGET_TRIPLE=wasm32-wasi-threads make test
-        rm -r build
         TARGET_TRIPLE=wasm32-wasip1-threads make test
       # The older version of Clang does not provide the expected symbol for the
       # test entrypoints: `undefined symbol: __main_argc_argv`.

test/.gitignore

@@ -1,5 +1,2 @@
 build
-download
 run
-
-smoke/*.dir

test/Makefile

@@ -9,37 +9,43 @@
 # - `run`: execute the benchmarks with a Wasm `$(ENGINE)` of choice (e.g.,
 #   Wasmtime)
 
-test: run run_smoke
-
-# Unlike the `libc-test` directory, we will output all artifacts to the `build`
-# directory (keeping with the `wasi-libc` conventions).
-OBJDIR ?= $(CURDIR)/build
-DOWNDIR ?= $(CURDIR)/download
+test: run
 
+# Decide which target to build for and which libc to use.
 TARGET_TRIPLE ?= wasm32-wasi
 
+# Setup various paths used by the tests.
+OBJDIR ?= build/$(TARGET_TRIPLE)
+DOWNDIR ?= build/download
+SRCDIR ?= src
+RUNDIR ?= run/$(TARGET_TRIPLE)
+
+# We also need to know the location the wasi-libc sysroot we're building
+# against.
+SYSROOT_DIR ?= ../sysroot
+SYSROOT := $(SYSROOT_DIR)/lib/$(TARGET_TRIPLE)
+$(SYSROOT):
+	@echo "No sysroot for $(TARGET_TRIPLE) available at $(SYSROOT_DIR); to build it, e.g.:"
+	@echo "  cd $(dir $(SYSROOT_DIR))"
+	@echo "  make TARGET_TRIPLE=$(TARGET_TRIPLE)"
+	@exit 1
+
+
 ##### DOWNLOAD #################################################################
 
 LIBC_TEST_URL ?= https://github.com/bytecodealliance/libc-test
 LIBC_TEST = $(DOWNDIR)/libc-test
 LIBRT_URL ?= https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/libclang_rt.builtins-wasm32-wasi-24.0.tar.gz
 LIBRT = $(DOWNDIR)/libclang_rt.builtins-wasm32.a
 WASMTIME_URL ?= https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasmtime-v26.0.1-x86_64-linux.tar.xz
-WASMTIME = $(DOWNDIR)/$(shell basename $(WASMTIME_URL) .tar.xz)/wasmtime
+WASMTIME = $(abspath $(DOWNDIR)/$(shell basename $(WASMTIME_URL) .tar.xz)/wasmtime)
 WASM_TOOLS_URL ?= https://github.com/bytecodealliance/wasm-tools/releases/download/v1.220.0/wasm-tools-1.220.0-x86_64-linux.tar.gz
 WASM_TOOLS = $(DOWNDIR)/$(shell basename $(WASM_TOOLS_URL) .tar.gz)/wasm-tools
 ADAPTER_URL ?= https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasi_snapshot_preview1.command.wasm
 ADAPTER = $(DOWNDIR)/wasi_snapshot_preview1.command.wasm
 
-TO_DOWNLOAD = $(LIBC_TEST) $(LIBRT) $(WASMTIME)
-ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
-TO_DOWNLOAD += $(ADAPTER) $(WASM_TOOLS)
-endif
-
-download: $(TO_DOWNLOAD)
-
 $(DOWNDIR):
-	mkdir -p download
+	@mkdir -p $@
 
 $(LIBC_TEST): | $(DOWNDIR)
 	git clone --depth 1 $(LIBC_TEST_URL) $@
@@ -61,151 +67,149 @@ $(WASM_TOOLS): | $(DOWNDIR)
 $(ADAPTER): | $(DOWNDIR)
 	wget --no-clobber --directory-prefix=$(DOWNDIR) $(ADAPTER_URL)
 
+# Target to download all necessary dependencies.
+TO_DOWNLOAD = $(LIBC_TEST) $(LIBRT) $(WASMTIME)
+ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
+TO_DOWNLOAD += $(ADAPTER) $(WASM_TOOLS)
+endif
+DOWNLOADED := $(DOWNDIR)/downloaded.stamp
+$(DOWNLOADED): $(TO_DOWNLOAD)
+	touch $@
+download: $(DOWNLOADED)
+
 clean::
 	rm -rf $(DOWNDIR)
 
-##### BUILD ####################################################################
+##### INFRA ####################################################################
 
-# For now, we list out the tests that we can currently build and run. This is
-# heavily focused on the functional tests; in the future it would be good to
-# fill out the missing tests and also include some `src/api` and `src/math`
-# tests (TODO).
-TESTS := \
-	$(LIBC_TEST)/src/functional/argv.c \
-	$(LIBC_TEST)/src/functional/basename.c \
-	$(LIBC_TEST)/src/functional/clocale_mbfuncs.c \
-	$(LIBC_TEST)/src/functional/clock_gettime.c \
-	$(LIBC_TEST)/src/functional/crypt.c \
-	$(LIBC_TEST)/src/functional/dirname.c \
-	$(LIBC_TEST)/src/functional/env.c \
-	$(LIBC_TEST)/src/functional/fnmatch.c \
-	$(LIBC_TEST)/src/functional/iconv_open.c \
-	$(LIBC_TEST)/src/functional/mbc.c \
-	$(LIBC_TEST)/src/functional/memstream.c \
-	$(LIBC_TEST)/src/functional/qsort.c \
-	$(LIBC_TEST)/src/functional/random.c \
-	$(LIBC_TEST)/src/functional/search_hsearch.c \
-	$(LIBC_TEST)/src/functional/search_insque.c \
-	$(LIBC_TEST)/src/functional/search_lsearch.c \
-	$(LIBC_TEST)/src/functional/search_tsearch.c \
-	$(LIBC_TEST)/src/functional/snprintf.c \
-	$(LIBC_TEST)/src/functional/sscanf.c \
-	$(LIBC_TEST)/src/functional/strftime.c \
-	$(LIBC_TEST)/src/functional/string.c \
-	$(LIBC_TEST)/src/functional/string_memcpy.c \
-	$(LIBC_TEST)/src/functional/string_memmem.c \
-	$(LIBC_TEST)/src/functional/string_memset.c \
-	$(LIBC_TEST)/src/functional/string_strchr.c \
-	$(LIBC_TEST)/src/functional/string_strcspn.c \
-	$(LIBC_TEST)/src/functional/string_strstr.c \
-	$(LIBC_TEST)/src/functional/strtod.c \
-	$(LIBC_TEST)/src/functional/strtod_long.c \
-	$(LIBC_TEST)/src/functional/strtod_simple.c \
-	$(LIBC_TEST)/src/functional/strtof.c \
-	$(LIBC_TEST)/src/functional/strtol.c \
-	$(LIBC_TEST)/src/functional/swprintf.c \
-	$(LIBC_TEST)/src/functional/tgmath.c \
-	$(LIBC_TEST)/src/functional/udiv.c \
-	$(LIBC_TEST)/src/functional/wcsstr.c \
-	$(LIBC_TEST)/src/functional/wcstol.c
-
-# Part of the problem including more tests is that the `libc-test`
-# infrastructure code is not all Wasm-compilable. As we include more tests
-# above, this list will also likely need to grow.
-COMMON_TEST_INFRA = \
-	$(LIBC_TEST)/src/common/path.c \
+INFRA_OBJDIR := $(OBJDIR)/common
+$(INFRA_OBJDIR):
+	mkdir -p $@
+
+# Build the common test infrastructure. Part of the problem including more tests
+# is that the `libc-test` infrastructure code is not all Wasm-compilable. As we
+# include more tests above, this list will also likely need to grow.
+INFRA_FILES = $(LIBC_TEST)/src/common/path.c \
 	$(LIBC_TEST)/src/common/print.c \
 	$(LIBC_TEST)/src/common/rand.c \
 	$(LIBC_TEST)/src/common/utf8.c
+$(INFRA_FILES): $(DOWNLOADED)
+INFRA_WASM_OBJS := $(patsubst $(LIBC_TEST)/src/common/%.c,$(OBJDIR)/common/%.wasm.o,$(INFRA_FILES))
+$(OBJDIR)/common/%.wasm.o: $(LIBC_TEST)/src/common/%.c | $(INFRA_OBJDIR)
+	$(CC) $(CFLAGS) -c $< -o $@
+
+# Also, include the `libc-test` infrastructure headers.
+INFRA_HEADERS_DIR := $(LIBC_TEST)/src/common
+INFRA_HEADERS := $(shell find $(INFRA_HEADERS_DIR) -name '*.h')
+$(INFRA_HEADERS): $(DOWNLOADED)
+
+##### BUILD ####################################################################
 
 # Create various lists containing the various artifacts to be built: mainly,
 # $(WASM_OBJS) are compiled in the $(OBJDIRS) and then linked together to form
 # the $(WASMS) tests.
-NAMES := $(TESTS:$(LIBC_TEST)/src/%.c=%)
-WASMS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.core.wasm)
-WASM_OBJS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o)
-INFRA_WASM_OBJS := $(COMMON_TEST_INFRA:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o)
+ALL_TESTS := $(shell find $(SRCDIR) -name '*.c')
+TESTS := $(shell TARGET_TRIPLE=$(TARGET_TRIPLE) scripts/filter.py $(ALL_TESTS))
+WASM_OBJS := $(TESTS:$(SRCDIR)/%.c=$(OBJDIR)/%.wasm.o)
 WASM_OBJS += $(INFRA_WASM_OBJS)
-DIRS := $(patsubst $(OBJDIR)/%/,%,$(sort $(dir $(WASM_OBJS))))
-OBJDIRS := $(DIRS:%=$(OBJDIR)/%)
+ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
+WASMS := $(TESTS:$(SRCDIR)/%.c=$(OBJDIR)/%.component.wasm)
+else
+WASMS := $(TESTS:$(SRCDIR)/%.c=$(OBJDIR)/%.core.wasm)
+endif
+
 
-# Allow $(CC) to be set from the command line; ?= doesn't work for CC because
-# make has a default value for it.
+# Setup the compiler. We allow $(CC) to be set from the command line; ?= doesn't
+# work for CC because make has a default value for it.
 ifeq ($(origin CC), default)
 CC := clang
 endif
 LDFLAGS ?=
-CFLAGS ?= --target=$(TARGET_TRIPLE) --sysroot=../sysroot
+CFLAGS ?= --target=$(TARGET_TRIPLE) --sysroot=$(SYSROOT_DIR)
 # Always include the `libc-test` infrastructure headers.
-CFLAGS += -I$(LIBC_TEST)/src/common
+CFLAGS += -I$(INFRA_HEADERS_DIR)
 ifneq ($(findstring -threads,$(TARGET_TRIPLE)),)
 CFLAGS += -pthread
 endif
 
-# Compile each selected test using Clang. Note that failures here are likely
-# due to a missing `libclang_rt.builtins-wasm32.a` in the Clang lib directory.
-# This location is system-dependent, but could be fixed by something like:
-#  $ sudo mkdir /usr/lib64/clang/14.0.5/lib/wasi
-#  $ sudo cp download/libclang_rt.builtins-wasm32.a /usr/lib64/clang/14.0.5/lib/wasi/
-build: download $(WASMS)
+# Build up all the `*.wasm.o` object files; these are the same regardless of
+# whether we're building core modules or components.
+$(WASM_OBJS): $(INFRA_HEADERS)
+$(OBJDIR)/%.wasm.o: $(SRCDIR)/%.c $(DOWNLOADED) $(SYSROOT)
+	@mkdir -p $(@D)
+	$(CC) $(CFLAGS) $(shell scripts/add-flags.py CFLAGS $<) -c $< -o $@
 
-$(WASMS): | $(OBJDIRS)
+# Build up all the `*.wasm` files.
+obj_to_c = $(patsubst $(OBJDIR)/%.wasm.o,$(SRCDIR)/%.c,$1)
 $(OBJDIR)/%.core.wasm: $(OBJDIR)/%.wasm.o $(INFRA_WASM_OBJS)
-	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
+	@mkdir -p $(@D)
+	$(CC) $(CFLAGS) $(LDFLAGS) $(shell scripts/add-flags.py LDFLAGS $(call obj_to_c,$<)) $^ -o $@
 
-ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
-$(OBJDIR)/%.wasm: $(OBJDIR)/%.core.wasm
+# For wasip2, we include an additional step to wrap up the core module into
+# a component.
+$(OBJDIR)/%.component.wasm: $(OBJDIR)/%.core.wasm
 	$(WASM_TOOLS) component new --adapt $(ADAPTER) $< -o $@
-endif
 
-$(WASM_OBJS): $(LIBC_TEST)/src/common/test.h | $(OBJDIRS)
-$(OBJDIR)/%.wasm.o: $(LIBC_TEST)/src/%.c
-	$(CC) $(CFLAGS) -c -o $@ $<
-
-$(OBJDIRS):
-	mkdir -p $@
+# Compile each selected test using Clang. Note that failures here are likely
+# due to a missing `libclang_rt.builtins-wasm32.a` in the Clang lib directory.
+# This location is system-dependent, but could be fixed by something like:
+#  $ sudo mkdir /usr/lib64/clang/14.0.5/lib/wasi
+#  $ sudo cp download/libclang_rt.builtins-wasm32.a /usr/lib64/clang/14.0.5/lib/wasi/
+build: $(DOWNLOADED) $(WASMS)
 
 clean::
 	rm -rf $(OBJDIR)
 
-##### RUN ######################################################################
+##### GENERATE #################################################################
 
-ENGINE ?= $(WASMTIME) run
-ERRS:=$(WASMS:%.core.wasm=%.wasm.err)
+# Not all of the downloaded `libc-test` tests can be built and run with
+# `wasi-libc`. Thus, we only include the subset that can be in `src/libc-test`
+# as stub files that `#include` the original test files. When we want to add
+# more tests, though, the `generate-stubs` target will generate stubs for the
+# missing tests which we can delete or alter as needed.
 
-# Use the provided Wasm engine to execute each test, emitting its output into
-# a `.err` file.
-run: build $(ERRS)
-	@echo "Tests passed"
+STUBDIR := $(SRCDIR)/libc-test
+generate-stubs:
+	FROM_DIR=$(LIBC_TEST) TO_DIR=$(STUBDIR) scripts/generate-stubs.sh
 
-$(ERRS): | $(OBJDIRS)
+##### RUN ######################################################################
 
+ENGINE ?= $(WASMTIME) run
 ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
-%.wasm.err: %.wasm
-	$(ENGINE) --wasm component-model $< >$@
+ENGINE += --wasm component-model
+OBJPAT := $(OBJDIR)/%.component.wasm
 else
-%.wasm.err: %.core.wasm
-	$(ENGINE) $< >$@
+OBJPAT := $(OBJDIR)/%.core.wasm
 endif
 
-clean::
-	rm -rf $(OBJDIR)/*/*.err
+# Each Wasm test is run every time, generating a folder containing a `cmd.sh`
+# script and an `output.log` file (see `scripts/run-test.sh` for details). The
+# `success` file is never generated, which means the test will rerun every time.
+# To ignore a test temporarily, `touch .../success:`.
+RUNTESTS:=$(WASMS:$(OBJPAT)=$(RUNDIR)/%/success)
+wasm_to_c = $(patsubst $(OBJPAT),$(SRCDIR)/%.c,$1)
+$(RUNDIR)/%/success: $(OBJPAT)
+	@mkdir -p $(@D)
+	@DIR="$(abspath $(@D))" \
+	  WASM="$(abspath $<)" \
+	  ENGINE="$(ENGINE) $(shell scripts/add-flags.py RUN $(call wasm_to_c,$<))" \
+	    scripts/run-test.sh
 
-##### SMOKE TEST SUITE #########################################################
+# Use the provided Wasm engine to execute each test, emitting its output into
+# a `.err` file.
+run: build $(RUNTESTS)
+	@if scripts/failed-tests.sh $(RUNDIR); then \
+		echo "Tests passed"; \
+	else \
+		echo "Tests failed:"; \
+		VERBOSE=1 scripts/failed-tests.sh $(RUNDIR); \
+	fi
 
-include smoke/smoke.mk
+clean::
+	rm -rf $(RUNDIR)
 
 ##### MISC #####################################################################
 
 # Note: the `clean` target has been built up by all of the previous sections.
 
-debug:
-	@echo NAMES $(NAMES)
-	@echo TESTS $(TESTS)
-	@echo WASMS $(WASMS)
-	@echo WASM_OBJS $(WASM_OBJS)
-	@echo ERRS $(ERRS)
-	@echo DIRS $(DIRS)
-	@echo OBJDIRS $(OBJDIRS)
-
-.PHONY: test download build run clean
+.PHONY: test download build run generate-stubs clean