test: run a subset of tests using libc-test (#346)

* test: run a subset of tests from `libc-test`

This change introduces a `test` directory that retrieves the `libc-test`
suite, compiles a subset of the tests using `wasi-libc`, and runs them
with Wasmtime.

* ci: run tests during CI

This change includes some fixups to the filesystem to place Clang's
runtime library for `wasm32-wasi` in the right location. Note that this
CI action is limited to a single OS--Linux.

.github/workflows/main.yml

@@ -58,6 +58,7 @@ jobs:
         curl -sSfL https://github.com/llvm/llvm-project/releases/download/llvmorg-${{ matrix.clang_version }}/clang+llvm-${{ matrix.clang_version }}-x86_64-linux-gnu-ubuntu-18.04.tar.xz | tar xJf -
         export CLANG_DIR=`pwd`/clang+llvm-${{ matrix.clang_version }}-x86_64-linux-gnu-ubuntu-18.04/bin
         echo "$CLANG_DIR" >> $GITHUB_PATH
+        echo "CLANG_DIR=$CLANG_DIR" >> $GITHUB_ENV
         echo "CC=$CLANG_DIR/clang" >> $GITHUB_ENV
         echo "AR=$CLANG_DIR/llvm-ar" >> $GITHUB_ENV
         echo "NM=$CLANG_DIR/llvm-nm" >> $GITHUB_ENV
@@ -67,6 +68,22 @@ jobs:
       shell: bash
       run: make -j4
 
+    - name: Test
+      shell: bash
+      # For Clang linking to work correctly, we need to place Clang's runtime
+      # library for `wasm32-wasi` in the right location (i.e., the `mkdir` and
+      # `cp` below).
+      run: |
+        cd test
+        make download
+        export WASI_DIR=$(realpath $CLANG_DIR/../lib/clang/${{ matrix.clang_version }}/lib/wasi/)
+        mkdir -p $WASI_DIR
+        cp download/lib/wasi/libclang_rt.builtins-wasm32.a $WASI_DIR
+        make test
+      # The older version of Clang does not provide the expected symbol for the
+      # test entrypoints: `undefined symbol: __main_argc_argv`.
+      if: matrix.os == 'ubuntu-latest' && matrix.clang_version != '10.0.0'
+
     - uses: actions/upload-artifact@v1
       with:
         # Upload the sysroot folder. Give it a name according to the OS it was built for.

Makefile

@@ -19,7 +19,7 @@ THREAD_MODEL ?= single
 MALLOC_IMPL ?= dlmalloc
 # yes or no
 BUILD_LIBC_TOP_HALF ?= yes
-# The directory where we're store intermediate artifacts.
+# The directory where we will store intermediate artifacts.
 OBJDIR ?= $(CURDIR)/build
 
 # When the length is no larger than this threshold, we consider the

test/.gitignore

@@ -0,0 +1,3 @@
+build
+download
+run

test/Makefile

@@ -0,0 +1,175 @@
+# Build the `libc-test` tests as Wasm programs and run them with the selected
+# engine. Contributors beware! This Makefile follows the style of the
+# `libc-test` Makefile and uses some more exotic features of `make`.
+#
+# The top-level `test` target is composed of a chain of several phony
+# sub-targets:
+# - `download`: retrieve the `libc-test` source from a Git `$(MIRROR)`
+# - `build`: construct Wasm modules for a subset of the `libc-test` tests
+# - `run`: execute the benchmarks with a Wasm `$(ENGINE)` of choice (e.g.,
+#   Wasmtime)
+
+test: run
+
+# Unlike the `libc-test` directory, we will output all artifacts to the `build`
+# directory (keeping with the `wasi-libc` conventions).
+OBJDIR ?= $(CURDIR)/build
+DOWNDIR ?= $(CURDIR)/download
+
+##### DOWNLOAD #################################################################
+
+LIBC_TEST_URL ?= https://github.com/bytecodealliance/libc-test
+LIBC_TEST = $(DOWNDIR)/libc-test
+LIBRT_URL ?= https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-16/libclang_rt.builtins-wasm32-wasi-16.0.tar.gz
+LIBRT = $(DOWNDIR)/lib/wasi/libclang_rt.builtins-wasm32.a
+WASMTIME_URL ?= https://github.com/bytecodealliance/wasmtime/releases/download/v3.0.0/wasmtime-v3.0.0-x86_64-linux.tar.xz
+WASMTIME = $(DOWNDIR)/$(shell basename $(WASMTIME_URL) .tar.xz)/wasmtime
+
+download: $(LIBC_TEST) $(LIBRT) $(WASMTIME)
+
+$(DOWNDIR):
+	mkdir -p download
+
+$(LIBC_TEST): | $(DOWNDIR)
+	git clone --depth 1 $(LIBC_TEST_URL) $@
+
+# TODO install target to place into...
+$(LIBRT): | $(DOWNDIR)
+	wget --no-clobber --directory-prefix=$(DOWNDIR) $(LIBRT_URL)
+	tar --extract --file=$(DOWNDIR)/$(shell basename $(LIBRT_URL)) --directory=$(DOWNDIR)/	
+
+$(WASMTIME): | $(DOWNDIR)
+	wget --no-clobber --directory-prefix=$(DOWNDIR) $(WASMTIME_URL)
+	tar --extract --file=$(DOWNDIR)/$(shell basename $(WASMTIME_URL)) --directory=$(DOWNDIR)/
+
+clean::
+	rm -rf download
+
+##### BUILD ####################################################################
+
+# For now, we list out the tests that we can currently build and run. This is
+# heavily focused on the functional tests; in the future it would be good to
+# fill out the missing tests and also include some `src/api` and `src/math`
+# tests (TODO).
+TESTS := \
+	$(LIBC_TEST)/src/functional/argv.c \
+	$(LIBC_TEST)/src/functional/basename.c \
+	$(LIBC_TEST)/src/functional/clocale_mbfuncs.c \
+	$(LIBC_TEST)/src/functional/clock_gettime.c \
+	$(LIBC_TEST)/src/functional/crypt.c \
+	$(LIBC_TEST)/src/functional/dirname.c \
+	$(LIBC_TEST)/src/functional/env.c \
+	$(LIBC_TEST)/src/functional/fnmatch.c \
+	$(LIBC_TEST)/src/functional/iconv_open.c \
+	$(LIBC_TEST)/src/functional/mbc.c \
+	$(LIBC_TEST)/src/functional/memstream.c \
+	$(LIBC_TEST)/src/functional/qsort.c \
+	$(LIBC_TEST)/src/functional/random.c \
+	$(LIBC_TEST)/src/functional/search_hsearch.c \
+	$(LIBC_TEST)/src/functional/search_insque.c \
+	$(LIBC_TEST)/src/functional/search_lsearch.c \
+	$(LIBC_TEST)/src/functional/search_tsearch.c \
+	$(LIBC_TEST)/src/functional/snprintf.c \
+	$(LIBC_TEST)/src/functional/sscanf.c \
+	$(LIBC_TEST)/src/functional/strftime.c \
+	$(LIBC_TEST)/src/functional/string.c \
+	$(LIBC_TEST)/src/functional/string_memcpy.c \
+	$(LIBC_TEST)/src/functional/string_memmem.c \
+	$(LIBC_TEST)/src/functional/string_memset.c \
+	$(LIBC_TEST)/src/functional/string_strchr.c \
+	$(LIBC_TEST)/src/functional/string_strcspn.c \
+	$(LIBC_TEST)/src/functional/string_strstr.c \
+	$(LIBC_TEST)/src/functional/strtod.c \
+	$(LIBC_TEST)/src/functional/strtod_long.c \
+	$(LIBC_TEST)/src/functional/strtod_simple.c \
+	$(LIBC_TEST)/src/functional/strtof.c \
+	$(LIBC_TEST)/src/functional/strtol.c \
+	$(LIBC_TEST)/src/functional/swprintf.c \
+	$(LIBC_TEST)/src/functional/tgmath.c \
+	$(LIBC_TEST)/src/functional/udiv.c \
+	$(LIBC_TEST)/src/functional/wcsstr.c \
+	$(LIBC_TEST)/src/functional/wcstol.c
+
+# Part of the problem including more tests is that the `libc-test`
+# infrastructure code is not all Wasm-compilable. As we include more tests
+# above, this list will also likely need to grow.
+COMMON_TEST_INFRA = \
+	$(LIBC_TEST)/src/common/path.c \
+	$(LIBC_TEST)/src/common/print.c \
+	$(LIBC_TEST)/src/common/rand.c \
+	$(LIBC_TEST)/src/common/utf8.c
+
+# Create various lists containing the various artifacts to be built: mainly,
+# $(WASM_OBJS) are compiled in the $(OBJDIRS) and then linked together to form
+# the $(WASMS) tests.
+NAMES := $(TESTS:$(LIBC_TEST)/src/%.c=%)
+WASMS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm)
+WASM_OBJS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o)
+INFRA_WASM_OBJS := $(COMMON_TEST_INFRA:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o)
+WASM_OBJS += $(INFRA_WASM_OBJS)
+DIRS := $(patsubst $(OBJDIR)/%/,%,$(sort $(dir $(WASM_OBJS))))
+OBJDIRS := $(DIRS:%=$(OBJDIR)/%)
+
+# Allow $(CC) to be set from the command line; ?= doesn't work for CC because
+# make has a default value for it.
+ifeq ($(origin CC), default)
+CC := clang
+endif
+LDFLAGS ?=
+CFLAGS ?= --target=wasm32-wasi --sysroot=../sysroot
+# Always include the `libc-test` infrastructure headers.
+CFLAGS += -I$(LIBC_TEST)/src/common
+
+# Compile each selected test using Clang. Note that failures here are likely
+# due to a missing `libclang_rt.builtins-wasm32.a` in the Clang lib directory.
+# This location is system-dependent, but could be fixed by something like:
+#  $ sudo mkdir /usr/lib64/clang/14.0.5/lib/wasi
+#  $ sudo cp download/lib/wasi/libclang_rt.builtins-wasm32.a /usr/lib64/clang/14.0.5/lib/wasi/
+build: download $(WASMS)
+
+$(WASMS): | $(OBJDIRS)
+$(OBJDIR)/%.wasm: $(OBJDIR)/%.wasm.o $(INFRA_WASM_OBJS)
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^ 
+
+$(WASM_OBJS): $(LIBC_TEST)/src/common/test.h | $(OBJDIRS)
+$(OBJDIR)/%.wasm.o: $(LIBC_TEST)/src/%.c
+	$(CC) $(CFLAGS) -c -o $@ $< 
+
+$(OBJDIRS):
+	mkdir -p $@
+
+clean::
+	rm -rf $(OBJDIR)
+
+##### RUN ######################################################################
+
+ENGINE ?= $(WASMTIME) run
+ERRS:=$(WASMS:%.wasm=%.wasm.err)
+
+# Use the provided Wasm engine to execute each test, emitting its output into
+# a `.err` file.
+run: build $(ERRS)
+	@echo "Tests passed"
+
+$(ERRS): | $(OBJDIRS)
+
+%.wasm.err: %.wasm
+	$(ENGINE) $< >$@
+
+clean::
+	rm -rf $(OBJDIR)/*/*.err
+
+##### MISC #####################################################################
+
+# Note: the `clean` target has been built up by all of the previous sections.
+
+debug:
+	@echo NAMES $(NAMES)
+	@echo TESTS $(TESTS)
+	@echo WASMS $(WASMS)
+	@echo WASM_OBJS $(WASM_OBJS)
+	@echo ERRS $(ERRS)
+	@echo DIRS $(DIRS)
+	@echo OBJDIRS $(OBJDIRS)
+
+.PHONY: test download build run clean

test/README.md

@@ -0,0 +1,6 @@
+# Test
+
+This directory runs a subset of libc-test using the sysroot produced by
+`wasi-libc`.
+
+[libc-test]: https://wiki.musl-libc.org/libc-test.html