diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b7e5c910..40d5a25a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -125,17 +125,13 @@ jobs: export WASIP1_DIR=$(realpath $($CLANG_DIR/clang -print-resource-dir)/lib/wasip1/) export WASIP2_DIR=$(realpath $($CLANG_DIR/clang -print-resource-dir)/lib/wasip2/) mkdir -p $WASI_DIR $WASIP1_DIR $WASIP2_DIR - cp download/libclang_rt.builtins-wasm32.a $WASI_DIR - cp download/libclang_rt.builtins-wasm32.a $WASIP1_DIR - cp download/libclang_rt.builtins-wasm32.a $WASIP2_DIR + cp build/download/libclang_rt.builtins-wasm32.a $WASI_DIR + cp build/download/libclang_rt.builtins-wasm32.a $WASIP1_DIR + cp build/download/libclang_rt.builtins-wasm32.a $WASIP2_DIR TARGET_TRIPLE=wasm32-wasi make test - rm -r build TARGET_TRIPLE=wasm32-wasip1 make test - rm -r build TARGET_TRIPLE=wasm32-wasip2 make test - rm -r build TARGET_TRIPLE=wasm32-wasi-threads make test - rm -r build TARGET_TRIPLE=wasm32-wasip1-threads make test # The older version of Clang does not provide the expected symbol for the # test entrypoints: `undefined symbol: __main_argc_argv`. diff --git a/test/.gitignore b/test/.gitignore index 469544f6..266ca8d5 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -1,5 +1,2 @@ build -download run - -smoke/*.dir diff --git a/test/Makefile b/test/Makefile index 31ff39f4..bf656782 100644 --- a/test/Makefile +++ b/test/Makefile @@ -9,15 +9,28 @@ # - `run`: execute the benchmarks with a Wasm `$(ENGINE)` of choice (e.g., # Wasmtime) -test: run run_smoke - -# Unlike the `libc-test` directory, we will output all artifacts to the `build` -# directory (keeping with the `wasi-libc` conventions). -OBJDIR ?= $(CURDIR)/build -DOWNDIR ?= $(CURDIR)/download +test: run +# Decide which target to build for and which libc to use. TARGET_TRIPLE ?= wasm32-wasi +# Setup various paths used by the tests. +OBJDIR ?= build/$(TARGET_TRIPLE) +DOWNDIR ?= build/download +SRCDIR ?= src +RUNDIR ?= run/$(TARGET_TRIPLE) + +# We also need to know the location the wasi-libc sysroot we're building +# against. +SYSROOT_DIR ?= ../sysroot +SYSROOT := $(SYSROOT_DIR)/lib/$(TARGET_TRIPLE) +$(SYSROOT): + @echo "No sysroot for $(TARGET_TRIPLE) available at $(SYSROOT_DIR); to build it, e.g.:" + @echo " cd $(dir $(SYSROOT_DIR))" + @echo " make TARGET_TRIPLE=$(TARGET_TRIPLE)" + @exit 1 + + ##### DOWNLOAD ################################################################# LIBC_TEST_URL ?= https://github.com/bytecodealliance/libc-test @@ -25,21 +38,14 @@ LIBC_TEST = $(DOWNDIR)/libc-test LIBRT_URL ?= https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/libclang_rt.builtins-wasm32-wasi-24.0.tar.gz LIBRT = $(DOWNDIR)/libclang_rt.builtins-wasm32.a WASMTIME_URL ?= https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasmtime-v26.0.1-x86_64-linux.tar.xz -WASMTIME = $(DOWNDIR)/$(shell basename $(WASMTIME_URL) .tar.xz)/wasmtime +WASMTIME = $(abspath $(DOWNDIR)/$(shell basename $(WASMTIME_URL) .tar.xz)/wasmtime) WASM_TOOLS_URL ?= https://github.com/bytecodealliance/wasm-tools/releases/download/v1.220.0/wasm-tools-1.220.0-x86_64-linux.tar.gz WASM_TOOLS = $(DOWNDIR)/$(shell basename $(WASM_TOOLS_URL) .tar.gz)/wasm-tools ADAPTER_URL ?= https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasi_snapshot_preview1.command.wasm ADAPTER = $(DOWNDIR)/wasi_snapshot_preview1.command.wasm -TO_DOWNLOAD = $(LIBC_TEST) $(LIBRT) $(WASMTIME) -ifeq ($(TARGET_TRIPLE), wasm32-wasip2) -TO_DOWNLOAD += $(ADAPTER) $(WASM_TOOLS) -endif - -download: $(TO_DOWNLOAD) - $(DOWNDIR): - mkdir -p download + @mkdir -p $@ $(LIBC_TEST): | $(DOWNDIR) git clone --depth 1 $(LIBC_TEST_URL) $@ @@ -61,151 +67,149 @@ $(WASM_TOOLS): | $(DOWNDIR) $(ADAPTER): | $(DOWNDIR) wget --no-clobber --directory-prefix=$(DOWNDIR) $(ADAPTER_URL) +# Target to download all necessary dependencies. +TO_DOWNLOAD = $(LIBC_TEST) $(LIBRT) $(WASMTIME) +ifeq ($(TARGET_TRIPLE), wasm32-wasip2) +TO_DOWNLOAD += $(ADAPTER) $(WASM_TOOLS) +endif +DOWNLOADED := $(DOWNDIR)/downloaded.stamp +$(DOWNLOADED): $(TO_DOWNLOAD) + touch $@ +download: $(DOWNLOADED) + clean:: rm -rf $(DOWNDIR) -##### BUILD #################################################################### +##### INFRA #################################################################### -# For now, we list out the tests that we can currently build and run. This is -# heavily focused on the functional tests; in the future it would be good to -# fill out the missing tests and also include some `src/api` and `src/math` -# tests (TODO). -TESTS := \ - $(LIBC_TEST)/src/functional/argv.c \ - $(LIBC_TEST)/src/functional/basename.c \ - $(LIBC_TEST)/src/functional/clocale_mbfuncs.c \ - $(LIBC_TEST)/src/functional/clock_gettime.c \ - $(LIBC_TEST)/src/functional/crypt.c \ - $(LIBC_TEST)/src/functional/dirname.c \ - $(LIBC_TEST)/src/functional/env.c \ - $(LIBC_TEST)/src/functional/fnmatch.c \ - $(LIBC_TEST)/src/functional/iconv_open.c \ - $(LIBC_TEST)/src/functional/mbc.c \ - $(LIBC_TEST)/src/functional/memstream.c \ - $(LIBC_TEST)/src/functional/qsort.c \ - $(LIBC_TEST)/src/functional/random.c \ - $(LIBC_TEST)/src/functional/search_hsearch.c \ - $(LIBC_TEST)/src/functional/search_insque.c \ - $(LIBC_TEST)/src/functional/search_lsearch.c \ - $(LIBC_TEST)/src/functional/search_tsearch.c \ - $(LIBC_TEST)/src/functional/snprintf.c \ - $(LIBC_TEST)/src/functional/sscanf.c \ - $(LIBC_TEST)/src/functional/strftime.c \ - $(LIBC_TEST)/src/functional/string.c \ - $(LIBC_TEST)/src/functional/string_memcpy.c \ - $(LIBC_TEST)/src/functional/string_memmem.c \ - $(LIBC_TEST)/src/functional/string_memset.c \ - $(LIBC_TEST)/src/functional/string_strchr.c \ - $(LIBC_TEST)/src/functional/string_strcspn.c \ - $(LIBC_TEST)/src/functional/string_strstr.c \ - $(LIBC_TEST)/src/functional/strtod.c \ - $(LIBC_TEST)/src/functional/strtod_long.c \ - $(LIBC_TEST)/src/functional/strtod_simple.c \ - $(LIBC_TEST)/src/functional/strtof.c \ - $(LIBC_TEST)/src/functional/strtol.c \ - $(LIBC_TEST)/src/functional/swprintf.c \ - $(LIBC_TEST)/src/functional/tgmath.c \ - $(LIBC_TEST)/src/functional/udiv.c \ - $(LIBC_TEST)/src/functional/wcsstr.c \ - $(LIBC_TEST)/src/functional/wcstol.c - -# Part of the problem including more tests is that the `libc-test` -# infrastructure code is not all Wasm-compilable. As we include more tests -# above, this list will also likely need to grow. -COMMON_TEST_INFRA = \ - $(LIBC_TEST)/src/common/path.c \ +INFRA_OBJDIR := $(OBJDIR)/common +$(INFRA_OBJDIR): + mkdir -p $@ + +# Build the common test infrastructure. Part of the problem including more tests +# is that the `libc-test` infrastructure code is not all Wasm-compilable. As we +# include more tests above, this list will also likely need to grow. +INFRA_FILES = $(LIBC_TEST)/src/common/path.c \ $(LIBC_TEST)/src/common/print.c \ $(LIBC_TEST)/src/common/rand.c \ $(LIBC_TEST)/src/common/utf8.c +$(INFRA_FILES): $(DOWNLOADED) +INFRA_WASM_OBJS := $(patsubst $(LIBC_TEST)/src/common/%.c,$(OBJDIR)/common/%.wasm.o,$(INFRA_FILES)) +$(OBJDIR)/common/%.wasm.o: $(LIBC_TEST)/src/common/%.c | $(INFRA_OBJDIR) + $(CC) $(CFLAGS) -c $< -o $@ + +# Also, include the `libc-test` infrastructure headers. +INFRA_HEADERS_DIR := $(LIBC_TEST)/src/common +INFRA_HEADERS := $(shell find $(INFRA_HEADERS_DIR) -name '*.h') +$(INFRA_HEADERS): $(DOWNLOADED) + +##### BUILD #################################################################### # Create various lists containing the various artifacts to be built: mainly, # $(WASM_OBJS) are compiled in the $(OBJDIRS) and then linked together to form # the $(WASMS) tests. -NAMES := $(TESTS:$(LIBC_TEST)/src/%.c=%) -WASMS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.core.wasm) -WASM_OBJS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o) -INFRA_WASM_OBJS := $(COMMON_TEST_INFRA:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o) +ALL_TESTS := $(shell find $(SRCDIR) -name '*.c') +TESTS := $(shell TARGET_TRIPLE=$(TARGET_TRIPLE) scripts/filter.py $(ALL_TESTS)) +WASM_OBJS := $(TESTS:$(SRCDIR)/%.c=$(OBJDIR)/%.wasm.o) WASM_OBJS += $(INFRA_WASM_OBJS) -DIRS := $(patsubst $(OBJDIR)/%/,%,$(sort $(dir $(WASM_OBJS)))) -OBJDIRS := $(DIRS:%=$(OBJDIR)/%) +ifeq ($(TARGET_TRIPLE), wasm32-wasip2) +WASMS := $(TESTS:$(SRCDIR)/%.c=$(OBJDIR)/%.component.wasm) +else +WASMS := $(TESTS:$(SRCDIR)/%.c=$(OBJDIR)/%.core.wasm) +endif + -# Allow $(CC) to be set from the command line; ?= doesn't work for CC because -# make has a default value for it. +# Setup the compiler. We allow $(CC) to be set from the command line; ?= doesn't +# work for CC because make has a default value for it. ifeq ($(origin CC), default) CC := clang endif LDFLAGS ?= -CFLAGS ?= --target=$(TARGET_TRIPLE) --sysroot=../sysroot +CFLAGS ?= --target=$(TARGET_TRIPLE) --sysroot=$(SYSROOT_DIR) # Always include the `libc-test` infrastructure headers. -CFLAGS += -I$(LIBC_TEST)/src/common +CFLAGS += -I$(INFRA_HEADERS_DIR) ifneq ($(findstring -threads,$(TARGET_TRIPLE)),) CFLAGS += -pthread endif -# Compile each selected test using Clang. Note that failures here are likely -# due to a missing `libclang_rt.builtins-wasm32.a` in the Clang lib directory. -# This location is system-dependent, but could be fixed by something like: -# $ sudo mkdir /usr/lib64/clang/14.0.5/lib/wasi -# $ sudo cp download/libclang_rt.builtins-wasm32.a /usr/lib64/clang/14.0.5/lib/wasi/ -build: download $(WASMS) +# Build up all the `*.wasm.o` object files; these are the same regardless of +# whether we're building core modules or components. +$(WASM_OBJS): $(INFRA_HEADERS) +$(OBJDIR)/%.wasm.o: $(SRCDIR)/%.c $(DOWNLOADED) $(SYSROOT) + @mkdir -p $(@D) + $(CC) $(CFLAGS) $(shell scripts/add-flags.py CFLAGS $<) -c $< -o $@ -$(WASMS): | $(OBJDIRS) +# Build up all the `*.wasm` files. +obj_to_c = $(patsubst $(OBJDIR)/%.wasm.o,$(SRCDIR)/%.c,$1) $(OBJDIR)/%.core.wasm: $(OBJDIR)/%.wasm.o $(INFRA_WASM_OBJS) - $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^ + @mkdir -p $(@D) + $(CC) $(CFLAGS) $(LDFLAGS) $(shell scripts/add-flags.py LDFLAGS $(call obj_to_c,$<)) $^ -o $@ -ifeq ($(TARGET_TRIPLE), wasm32-wasip2) -$(OBJDIR)/%.wasm: $(OBJDIR)/%.core.wasm +# For wasip2, we include an additional step to wrap up the core module into +# a component. +$(OBJDIR)/%.component.wasm: $(OBJDIR)/%.core.wasm $(WASM_TOOLS) component new --adapt $(ADAPTER) $< -o $@ -endif -$(WASM_OBJS): $(LIBC_TEST)/src/common/test.h | $(OBJDIRS) -$(OBJDIR)/%.wasm.o: $(LIBC_TEST)/src/%.c - $(CC) $(CFLAGS) -c -o $@ $< - -$(OBJDIRS): - mkdir -p $@ +# Compile each selected test using Clang. Note that failures here are likely +# due to a missing `libclang_rt.builtins-wasm32.a` in the Clang lib directory. +# This location is system-dependent, but could be fixed by something like: +# $ sudo mkdir /usr/lib64/clang/14.0.5/lib/wasi +# $ sudo cp download/libclang_rt.builtins-wasm32.a /usr/lib64/clang/14.0.5/lib/wasi/ +build: $(DOWNLOADED) $(WASMS) clean:: rm -rf $(OBJDIR) -##### RUN ###################################################################### +##### GENERATE ################################################################# -ENGINE ?= $(WASMTIME) run -ERRS:=$(WASMS:%.core.wasm=%.wasm.err) +# Not all of the downloaded `libc-test` tests can be built and run with +# `wasi-libc`. Thus, we only include the subset that can be in `src/libc-test` +# as stub files that `#include` the original test files. When we want to add +# more tests, though, the `generate-stubs` target will generate stubs for the +# missing tests which we can delete or alter as needed. -# Use the provided Wasm engine to execute each test, emitting its output into -# a `.err` file. -run: build $(ERRS) - @echo "Tests passed" +STUBDIR := $(SRCDIR)/libc-test +generate-stubs: + FROM_DIR=$(LIBC_TEST) TO_DIR=$(STUBDIR) scripts/generate-stubs.sh -$(ERRS): | $(OBJDIRS) +##### RUN ###################################################################### +ENGINE ?= $(WASMTIME) run ifeq ($(TARGET_TRIPLE), wasm32-wasip2) -%.wasm.err: %.wasm - $(ENGINE) --wasm component-model $< >$@ +ENGINE += --wasm component-model +OBJPAT := $(OBJDIR)/%.component.wasm else -%.wasm.err: %.core.wasm - $(ENGINE) $< >$@ +OBJPAT := $(OBJDIR)/%.core.wasm endif -clean:: - rm -rf $(OBJDIR)/*/*.err +# Each Wasm test is run every time, generating a folder containing a `cmd.sh` +# script and an `output.log` file (see `scripts/run-test.sh` for details). The +# `success` file is never generated, which means the test will rerun every time. +# To ignore a test temporarily, `touch .../success:`. +RUNTESTS:=$(WASMS:$(OBJPAT)=$(RUNDIR)/%/success) +wasm_to_c = $(patsubst $(OBJPAT),$(SRCDIR)/%.c,$1) +$(RUNDIR)/%/success: $(OBJPAT) + @mkdir -p $(@D) + @DIR="$(abspath $(@D))" \ + WASM="$(abspath $<)" \ + ENGINE="$(ENGINE) $(shell scripts/add-flags.py RUN $(call wasm_to_c,$<))" \ + scripts/run-test.sh -##### SMOKE TEST SUITE ######################################################### +# Use the provided Wasm engine to execute each test, emitting its output into +# a `.err` file. +run: build $(RUNTESTS) + @if scripts/failed-tests.sh $(RUNDIR); then \ + echo "Tests passed"; \ + else \ + echo "Tests failed:"; \ + VERBOSE=1 scripts/failed-tests.sh $(RUNDIR); \ + fi -include smoke/smoke.mk +clean:: + rm -rf $(RUNDIR) ##### MISC ##################################################################### # Note: the `clean` target has been built up by all of the previous sections. -debug: - @echo NAMES $(NAMES) - @echo TESTS $(TESTS) - @echo WASMS $(WASMS) - @echo WASM_OBJS $(WASM_OBJS) - @echo ERRS $(ERRS) - @echo DIRS $(DIRS) - @echo OBJDIRS $(OBJDIRS) - -.PHONY: test download build run clean +.PHONY: test download build run generate-stubs clean diff --git a/test/README.md b/test/README.md index 590c2d4d..2ba40e19 100644 --- a/test/README.md +++ b/test/README.md @@ -1,6 +1,85 @@ # Test -This directory runs a subset of libc-test using the sysroot produced by -`wasi-libc`. +This directory runs C tests compiled to WebAssembly against `wasi-libc` to check +its functionality. It enables a subset of [libc-test] as well as adding custom C +tests; all enabled tests are contained in the [`src`] directory. + +### Pre-requisites + +- Clang +- [`libc-test`] +- `libclang_rt.builtins-wasm32.a` +- a WebAssembly engine +- other WebAssembly tools, especially for `wasm32-wasip2` support (see the + [`Makefile] for a complete list) + +All but Clang are downloaded automatically by the `make download` target. + +### Build and run + +To build and run all tests: + +```sh +$ make TARGET_TRIPLE=... +Tests passed +``` + +Note that `wasm-ld` must be available, so an additional +`CC=/bin/clang` may be necessary. Each test runs in a directory that +looks like (see [`run-test.sh`]): + +```sh +$ ls run/$TARGET_TRIPLE/misc/some-test +cmd.sh # the exact command used to run the test +fs # a directory containing any test-created files +output.log # the captured printed output--only for errors +``` + +### Adding tests + +To add a test, create a new C file in [`src/misc`]: + +```c +//! filter.py(TARGET_TRIPLE): !wasm32-wasip2 +//! add-flags.py(CFLAGS): ... +//! add-flags.py(LDFLAGS): ... +//! add-flags.py(RUN): ... +void main() { ... } +``` + +- to pass, the `main` function must exit successfully and avoid printing output +- the `filter.py` directive controls when the test builds and runs (e.g., not + for `wasip2`) +- the `add-flags.py` directive adds extra information for building or running + the test (see the [`Makefile`] for precise use). + +### Enabling more [libc-test] tests + +[libc-test] has more tests available that are not yet enabled (e.g., to count +the enabled subset, `find src -name *.c | wc -l`). Each enabled test contains a +stub file in [`src/libc-test`] that `#include`s its downloaded version and adds +various `filter.py` and `add-flags.py` directives. + +To quickly create stub files for not-yet-enabled tests: + +```sh +$ make generate-stubs +$ git status +... +src/libc-test/functional/tls_align.c +src/libc-test/functional/tls_align_dlopen.c +src/libc-test/functional/tls_align_dso.c +src/libc-test/functional/tls_init.c +``` + +Then modify the directives for these new stub files to get the new tests to +compile and successfully run. + + [libc-test]: https://wiki.musl-libc.org/libc-test.html +[`Makefile`]: Makefile +[`run-test.sh`]: scripts/run-test.sh +[`src`]: src +[`src/libc-test`]: src/libc-test +[`src/misc`]: src/misc diff --git a/test/scripts/add-flags.py b/test/scripts/add-flags.py new file mode 100755 index 00000000..643a922b --- /dev/null +++ b/test/scripts/add-flags.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +# Find additional compilation flags specified in test files. +# +# This script accepts a single file as an argument and looks for a comment like +# the following: `// add-flags.py(): `. If found, the `` are +# printed to stdout. +# +# Example: +# ``` +# $ head one.c +# //! add-flags.py(CFLAGS): -DFOO +# $ ./add-flags.py CFLAGS one.c +# -DFOO + +import sys +import os +import re +import logging + +""" +Match a C comment like the following: `//! add-flags.py: `. +""" +PATTERN = re.compile('\\s*//\\!\\s*add-flags\\.py\\(([^)]+)\\):\\s*(.*)') + + +def find_flags(name, file): + with open(file, 'r') as f: + for lineno, line in enumerate(f, start=1): + match = PATTERN.match(line) + if match and match[1] == name: + pos = f'[{file}:{lineno}]' + logging.debug(f'{pos} found flags') + return match[2].strip() + + +def main(name, file): + flags = find_flags(name, file) + if flags: + print(flags) + + +if __name__ == "__main__": + logging.getLogger().name = os.path.basename(__file__) + if os.environ.get('VERBOSE'): + logging.basicConfig(level=logging.DEBUG) + if len(sys.argv) != 3: + print(f'usage: {sys.argv[0]} ') + sys.exit(1) + main(sys.argv[1], sys.argv[2]) diff --git a/test/scripts/failed-tests.sh b/test/scripts/failed-tests.sh new file mode 100755 index 00000000..381218a1 --- /dev/null +++ b/test/scripts/failed-tests.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# This script checks for failed tests in the given directory. Failed tests will +# include text in a *.log file. +# +# Usage: failed-tests.sh [TESTDIR] +# +# Optionally set the VERBOSE environment variable to see the list of failed +# tests. + +set -e + +TESTDIR=${1:-.} +FAILED=$(find $TESTDIR -type f -and -name *.log -and -not -size 0) + +if [[ -n "$VERBOSE" ]]; then + for failed in "$FAILED"; do + echo "$failed"; + done +fi + +if [[ -n "$FAILED" ]]; then + exit 1 +fi diff --git a/test/scripts/filter.py b/test/scripts/filter.py new file mode 100755 index 00000000..6df11717 --- /dev/null +++ b/test/scripts/filter.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python + +# Filter out test files that do not match a configuration variable. +# +# This script accepts a list of files as arguments and looks for a comment like +# the following in each: `//! filter.py(): `. The +# `VARIABLE` value is looked up in the environment and compared with the +# expression. An `EXPRESSION` is a comma-separated list of terms: either +# `` or `!`. If the `VARIABLE` value matches the expression, the +# file is printed; otherwise, it is filtered out. +# +# Example: +# ``` +# $ head one.c +# //! filter.py(FOO): bar +# $ head two.c +# //! filter.py(FOO):!bar, baz +# $ FOO=bar ./filter.py one.c two.c +# one.c +# $ FOO=baz ./filter.py one.c two.c +# two.c +# $ ./filter.py one.c two.c +# one.c two.c +# ``` + +import sys +import os +import re +import logging + + +def parse_expression(expr): + """ + An expression is a comma-separated list of terms: either or + !. + """ + return [s.strip() for s in expr.split(',')] + + +def matches_expression(value, expr, pos): + """ + A value matches an expression if it is in the list of strings or if it is + not explicitly rejected; order matters, because the first term that matches, + wins. + """ + for e in expr: + if e.startswith('!') and value == e[1:]: + logging.debug(f'{pos} {value} != {expr}, REJECTED') + return False + elif value == e: + logging.debug(f'{pos} {value} == {expr}, ALLOWED') + return True + logging.debug(f'{pos} {value} not in {expr}, REJECTED') + return False + + +""" +Match a C comment like the following: `//! filter.py(): `. +""" +PATTERN = re.compile('\\s*//\\!\\s*filter\\.py\\(([^)]+)\\):\\s*(.*)') + + +def line_matches(line, env, pos): + match = PATTERN.match(line) + if match: + value = env.get(match[1]) + if value is None: + logging.debug(f'{pos} no value for {match[1]}, ALLOWED') + return True + else: + expr = parse_expression(match[2]) + return matches_expression(value, expr, pos) + return True + + +def file_matches(file, env): + with open(file, 'r') as f: + print = True + for lineno, line in enumerate(f, start=1): + print &= line_matches(line, env, f'[{file}:{lineno}]') + return print + + +def main(env, files): + filtered = [file for file in files if file_matches(file, env)] + print(' '.join(filtered)) + + +if __name__ == "__main__": + logging.getLogger().name = os.path.basename(__file__) + if os.environ.get('VERBOSE'): + logging.basicConfig(level=logging.DEBUG) + main(os.environ, sys.argv[1:]) diff --git a/test/scripts/generate-stubs.sh b/test/scripts/generate-stubs.sh new file mode 100755 index 00000000..42f01423 --- /dev/null +++ b/test/scripts/generate-stubs.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Create stub tests in `$TO_DIR` that `#include` a test in the `$FROM_DIR` +# directory. If the stub already exists, do not overwrite it. +# +# Usage: FROM_DIR=... TO_DIR=... ./generate-stubs.sh + +FROM_DIR="${FROM_DIR:-build/download/libc-test}" +TO_DIR="${TO_DIR:-src/libc-test}" +# For now, only retrieve the functional tests. +FUNCTIONAL_TESTS=$(find $FROM_DIR/src/math -name '*.c') + +for from in $FUNCTIONAL_TESTS; do + to="${from/"$FROM_DIR/src"/"$TO_DIR"}" + if [ ! -f $to ]; then \ + mkdir -p $(dirname $to) + echo '// add-flags.py(CFLAGS): -I.' > $to + echo "#include \"$from\"" >> $to + fi +done diff --git a/test/scripts/run-test.sh b/test/scripts/run-test.sh new file mode 100755 index 00000000..820b7a26 --- /dev/null +++ b/test/scripts/run-test.sh @@ -0,0 +1,20 @@ +#/usr/bin/env bash + +# Run a previously built test, generating in `$DIR`: +# - a `cmd.sh` script containing the actual command used to run the test in an +# engine +# - a `fs` directory which the test may have used for file IO +# - an `output.log` file containing the output of the test +# +# Usage: DIR=... WASM=... ENGINE=... ./run-test.sh + +ENGINE="${ENGINE:-wasmtime}" +[ -n "$WASM" ] || (echo "missing WASM variable" && exit 1) +[ -n "$DIR" ] || (echo "missing DIR variable" && exit 1) + +cd $DIR +mkdir -p fs +echo "$ENGINE $WASM" > cmd.sh +chmod +x cmd.sh +./cmd.sh &> output.log +[ $? -eq 0 ] || echo "Test failed" >> output.log diff --git a/test/smoke/smoke.mk b/test/smoke/smoke.mk deleted file mode 100644 index ad9e9b89..00000000 --- a/test/smoke/smoke.mk +++ /dev/null @@ -1,35 +0,0 @@ -# Smoke test suite specific to wasi-libc -# -# This Makefile is included by the parent Makefile - -SMOKE_TESTS_DIR := $(dir $(lastword $(MAKEFILE_LIST))) -SMOKE_TESTS := $(wildcard $(SMOKE_TESTS_DIR)/*.c) -SMOKE_OBJDIR := $(OBJDIR)/smoke -SMOKE_WASMS := $(SMOKE_TESTS:$(SMOKE_TESTS_DIR)/%.c=$(SMOKE_OBJDIR)/%.core.wasm) -SMOKE_ERRS := $(SMOKE_WASMS:%.core.wasm=%.wasm.err) - -$(SMOKE_OBJDIR): - mkdir -p $@ - -$(SMOKE_OBJDIR)/%.core.wasm: $(SMOKE_TESTS_DIR)/%.c | $(SMOKE_OBJDIR) - $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^ - -ifeq ($(TARGET_TRIPLE), wasm32-wasip2) -$(SMOKE_OBJDIR)/%.wasm: $(SMOKE_OBJDIR)/%.core.wasm - $(WASM_TOOLS) component new --adapt $(ADAPTER) $< -o $@ -endif - -ifeq ($(TARGET_TRIPLE), wasm32-wasip2) -$(SMOKE_OBJDIR)/%.wasm.err: $(SMOKE_OBJDIR)/%.wasm - rm -rf $(SMOKE_TESTS_DIR)/$*.dir - mkdir -p $(SMOKE_TESTS_DIR)/$*.dir - $(ENGINE) --dir $(SMOKE_TESTS_DIR)/$*.dir::/ --wasm component-model $< >$@ -else -$(SMOKE_OBJDIR)/%.wasm.err: $(SMOKE_OBJDIR)/%.core.wasm - rm -rf $(SMOKE_TESTS_DIR)/$*.dir - mkdir -p $(SMOKE_TESTS_DIR)/$*.dir - $(ENGINE) --dir $(SMOKE_TESTS_DIR)/$*.dir::/ $< >$@ -endif - -run_smoke: $(SMOKE_ERRS) - @echo "Smoke tests passed" diff --git a/test/src/libc-test/functional/argv.c b/test/src/libc-test/functional/argv.c new file mode 100644 index 00000000..41dda8ae --- /dev/null +++ b/test/src/libc-test/functional/argv.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/argv.c" diff --git a/test/src/libc-test/functional/basename.c b/test/src/libc-test/functional/basename.c new file mode 100644 index 00000000..feb49eef --- /dev/null +++ b/test/src/libc-test/functional/basename.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/basename.c" diff --git a/test/src/libc-test/functional/clocale_mbfuncs.c b/test/src/libc-test/functional/clocale_mbfuncs.c new file mode 100644 index 00000000..d2bc5752 --- /dev/null +++ b/test/src/libc-test/functional/clocale_mbfuncs.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/clocale_mbfuncs.c" diff --git a/test/src/libc-test/functional/clock_gettime.c b/test/src/libc-test/functional/clock_gettime.c new file mode 100644 index 00000000..1744dfb7 --- /dev/null +++ b/test/src/libc-test/functional/clock_gettime.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/clock_gettime.c" diff --git a/test/src/libc-test/functional/crypt.c b/test/src/libc-test/functional/crypt.c new file mode 100644 index 00000000..a49662cd --- /dev/null +++ b/test/src/libc-test/functional/crypt.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/crypt.c" diff --git a/test/src/libc-test/functional/dirname.c b/test/src/libc-test/functional/dirname.c new file mode 100644 index 00000000..df9ea4ef --- /dev/null +++ b/test/src/libc-test/functional/dirname.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/dirname.c" diff --git a/test/src/libc-test/functional/env.c b/test/src/libc-test/functional/env.c new file mode 100644 index 00000000..9dba36a5 --- /dev/null +++ b/test/src/libc-test/functional/env.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/env.c" diff --git a/test/src/libc-test/functional/fnmatch.c b/test/src/libc-test/functional/fnmatch.c new file mode 100644 index 00000000..26ae512e --- /dev/null +++ b/test/src/libc-test/functional/fnmatch.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/fnmatch.c" diff --git a/test/src/libc-test/functional/iconv_open.c b/test/src/libc-test/functional/iconv_open.c new file mode 100644 index 00000000..945c5726 --- /dev/null +++ b/test/src/libc-test/functional/iconv_open.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/iconv_open.c" diff --git a/test/src/libc-test/functional/mbc.c b/test/src/libc-test/functional/mbc.c new file mode 100644 index 00000000..7b353ec2 --- /dev/null +++ b/test/src/libc-test/functional/mbc.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/mbc.c" diff --git a/test/src/libc-test/functional/memstream.c b/test/src/libc-test/functional/memstream.c new file mode 100644 index 00000000..c3fccf3a --- /dev/null +++ b/test/src/libc-test/functional/memstream.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/memstream.c" diff --git a/test/src/libc-test/functional/qsort.c b/test/src/libc-test/functional/qsort.c new file mode 100644 index 00000000..ba8debf8 --- /dev/null +++ b/test/src/libc-test/functional/qsort.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/qsort.c" diff --git a/test/src/libc-test/functional/random.c b/test/src/libc-test/functional/random.c new file mode 100644 index 00000000..b6282112 --- /dev/null +++ b/test/src/libc-test/functional/random.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/random.c" diff --git a/test/src/libc-test/functional/search_hsearch.c b/test/src/libc-test/functional/search_hsearch.c new file mode 100644 index 00000000..9a5eb9ad --- /dev/null +++ b/test/src/libc-test/functional/search_hsearch.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/search_hsearch.c" diff --git a/test/src/libc-test/functional/search_insque.c b/test/src/libc-test/functional/search_insque.c new file mode 100644 index 00000000..21bf2cde --- /dev/null +++ b/test/src/libc-test/functional/search_insque.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/search_insque.c" diff --git a/test/src/libc-test/functional/search_lsearch.c b/test/src/libc-test/functional/search_lsearch.c new file mode 100644 index 00000000..e681fb54 --- /dev/null +++ b/test/src/libc-test/functional/search_lsearch.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/search_lsearch.c" diff --git a/test/src/libc-test/functional/search_tsearch.c b/test/src/libc-test/functional/search_tsearch.c new file mode 100644 index 00000000..45804c5a --- /dev/null +++ b/test/src/libc-test/functional/search_tsearch.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/search_tsearch.c" diff --git a/test/src/libc-test/functional/snprintf.c b/test/src/libc-test/functional/snprintf.c new file mode 100644 index 00000000..67f64da3 --- /dev/null +++ b/test/src/libc-test/functional/snprintf.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/snprintf.c" diff --git a/test/src/libc-test/functional/sscanf.c b/test/src/libc-test/functional/sscanf.c new file mode 100644 index 00000000..ad63e275 --- /dev/null +++ b/test/src/libc-test/functional/sscanf.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/sscanf.c" diff --git a/test/src/libc-test/functional/strftime.c b/test/src/libc-test/functional/strftime.c new file mode 100644 index 00000000..787f7e90 --- /dev/null +++ b/test/src/libc-test/functional/strftime.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/strftime.c" diff --git a/test/src/libc-test/functional/string.c b/test/src/libc-test/functional/string.c new file mode 100644 index 00000000..52e181ef --- /dev/null +++ b/test/src/libc-test/functional/string.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/string.c" diff --git a/test/src/libc-test/functional/string_memcpy.c b/test/src/libc-test/functional/string_memcpy.c new file mode 100644 index 00000000..cfc94784 --- /dev/null +++ b/test/src/libc-test/functional/string_memcpy.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/string_memcpy.c" diff --git a/test/src/libc-test/functional/string_memmem.c b/test/src/libc-test/functional/string_memmem.c new file mode 100644 index 00000000..d2362c4c --- /dev/null +++ b/test/src/libc-test/functional/string_memmem.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/string_memmem.c" diff --git a/test/src/libc-test/functional/string_memset.c b/test/src/libc-test/functional/string_memset.c new file mode 100644 index 00000000..d5672dac --- /dev/null +++ b/test/src/libc-test/functional/string_memset.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/string_memset.c" diff --git a/test/src/libc-test/functional/string_strchr.c b/test/src/libc-test/functional/string_strchr.c new file mode 100644 index 00000000..c9c2d269 --- /dev/null +++ b/test/src/libc-test/functional/string_strchr.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/string_strchr.c" diff --git a/test/src/libc-test/functional/string_strcspn.c b/test/src/libc-test/functional/string_strcspn.c new file mode 100644 index 00000000..052e1e68 --- /dev/null +++ b/test/src/libc-test/functional/string_strcspn.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/string_strcspn.c" diff --git a/test/src/libc-test/functional/string_strstr.c b/test/src/libc-test/functional/string_strstr.c new file mode 100644 index 00000000..f0d2dbc7 --- /dev/null +++ b/test/src/libc-test/functional/string_strstr.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/string_strstr.c" diff --git a/test/src/libc-test/functional/strtod.c b/test/src/libc-test/functional/strtod.c new file mode 100644 index 00000000..9bb36cff --- /dev/null +++ b/test/src/libc-test/functional/strtod.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/strtod.c" diff --git a/test/src/libc-test/functional/strtod_long.c b/test/src/libc-test/functional/strtod_long.c new file mode 100644 index 00000000..5779e277 --- /dev/null +++ b/test/src/libc-test/functional/strtod_long.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/strtod_long.c" diff --git a/test/src/libc-test/functional/strtod_simple.c b/test/src/libc-test/functional/strtod_simple.c new file mode 100644 index 00000000..bbd817a1 --- /dev/null +++ b/test/src/libc-test/functional/strtod_simple.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/strtod_simple.c" diff --git a/test/src/libc-test/functional/strtof.c b/test/src/libc-test/functional/strtof.c new file mode 100644 index 00000000..a5682c13 --- /dev/null +++ b/test/src/libc-test/functional/strtof.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/strtof.c" diff --git a/test/src/libc-test/functional/strtol.c b/test/src/libc-test/functional/strtol.c new file mode 100644 index 00000000..b874d302 --- /dev/null +++ b/test/src/libc-test/functional/strtol.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/strtol.c" diff --git a/test/src/libc-test/functional/strtold.c b/test/src/libc-test/functional/strtold.c new file mode 100644 index 00000000..8f467fed --- /dev/null +++ b/test/src/libc-test/functional/strtold.c @@ -0,0 +1,3 @@ +//! add-flags.py(CFLAGS): -I. +//! add-flags.py(LDFLAGS): -lc-printscan-long-double +#include "build/download/libc-test/src/functional/strtold.c" diff --git a/test/src/libc-test/functional/swprintf.c b/test/src/libc-test/functional/swprintf.c new file mode 100644 index 00000000..278b177d --- /dev/null +++ b/test/src/libc-test/functional/swprintf.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/swprintf.c" diff --git a/test/src/libc-test/functional/tgmath.c b/test/src/libc-test/functional/tgmath.c new file mode 100644 index 00000000..3a9d3c33 --- /dev/null +++ b/test/src/libc-test/functional/tgmath.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/tgmath.c" diff --git a/test/src/libc-test/functional/udiv.c b/test/src/libc-test/functional/udiv.c new file mode 100644 index 00000000..162b6002 --- /dev/null +++ b/test/src/libc-test/functional/udiv.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/udiv.c" diff --git a/test/src/libc-test/functional/wcsstr.c b/test/src/libc-test/functional/wcsstr.c new file mode 100644 index 00000000..9c5581dd --- /dev/null +++ b/test/src/libc-test/functional/wcsstr.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/wcsstr.c" diff --git a/test/src/libc-test/functional/wcstol.c b/test/src/libc-test/functional/wcstol.c new file mode 100644 index 00000000..04d19dd1 --- /dev/null +++ b/test/src/libc-test/functional/wcstol.c @@ -0,0 +1,2 @@ +//! add-flags.py(CFLAGS): -I. +#include "build/download/libc-test/src/functional/wcstol.c" diff --git a/test/smoke/test_fts.c b/test/src/misc/fts.c similarity index 99% rename from test/smoke/test_fts.c rename to test/src/misc/fts.c index 240bb3d1..1126f9b7 100644 --- a/test/smoke/test_fts.c +++ b/test/src/misc/fts.c @@ -1,3 +1,5 @@ +//! add-flags.py(RUN): --dir fs::/ + /* * We modified musl-fts not to use fchdir() and we made FTS_NOCHDIR * the default behavior. This test is to make sure that the modified