diff --git a/.github/actions/binary-compatible-builds/main.js b/.github/actions/binary-compatible-builds/main.js index 378c5c202c9c..6230126fcb81 100755 --- a/.github/actions/binary-compatible-builds/main.js +++ b/.github/actions/binary-compatible-builds/main.js @@ -34,7 +34,7 @@ if (process.env.CENTOS !== undefined) { return; } -const name = process.env.INPUT_NAME; +const name = process.env.INPUT_NAME.replace(/-min$/, ''); child_process.execFileSync('docker', [ 'build', diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2df6de8fa74d..92c5ae06e819 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -116,6 +116,7 @@ jobs: outputs: run-full: ${{ steps.calculate.outputs.run-full }} test-matrix: ${{ steps.calculate.outputs.test-matrix }} + build-matrix: ${{ steps.calculate.outputs.build-matrix }} test-capi: ${{ steps.calculate.outputs.test-capi }} build-fuzz: ${{ steps.calculate.outputs.build-fuzz }} audit: ${{ steps.calculate.outputs.audit }} @@ -161,6 +162,9 @@ jobs: echo "test-matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT echo "$matrix" + matrix="$(node ./ci/build-build-matrix.js)" + echo "build-matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT + if [ "$run_full" = "true" ]; then echo run-full=true >> $GITHUB_OUTPUT echo test-capi=true >> $GITHUB_OUTPUT @@ -658,40 +662,28 @@ jobs: # Perform release builds of `wasmtime` and `libwasmtime.so`. Builds a variety # of platforms and architectures and then uploads the release artifacts to # this workflow run's list of artifacts. + # + # Note that the full matrix is computed by `ci/build-build-matrix.js`. build: needs: determine if: needs.determine.outputs.run-full name: Release build for ${{ matrix.build }} runs-on: ${{ matrix.os }} strategy: - matrix: - include: - - build: x86_64-linux - os: ubuntu-latest - - build: x86_64-macos - os: macos-latest - - build: aarch64-macos - os: macos-latest - target: aarch64-apple-darwin - - build: x86_64-windows - os: windows-latest - - build: x86_64-mingw - os: windows-latest - target: x86_64-pc-windows-gnu - - build: aarch64-linux - os: ubuntu-latest - target: aarch64-unknown-linux-gnu - - build: s390x-linux - os: ubuntu-latest - target: s390x-unknown-linux-gnu - - build: riscv64gc-linux - os: ubuntu-latest - target: riscv64gc-unknown-linux-gnu + fail-fast: false + matrix: ${{ fromJson(needs.determine.outputs.build-matrix) }} steps: - uses: actions/checkout@v3 with: submodules: true + - uses: ./.github/actions/install-rust + with: + toolchain: ${{ matrix.rust }} + - run: | + rustup component add rust-src + rustup target add ${{ matrix.target }} + # On one builder produce the source tarball since there's no need to produce # it everywhere - run: ./ci/build-src-tarball.sh @@ -699,21 +691,13 @@ jobs: - uses: ./.github/actions/binary-compatible-builds with: name: ${{ matrix.build }} - - run: | - echo CARGO_BUILD_TARGET=${{ matrix.target }} >> $GITHUB_ENV - rustup target add ${{ matrix.target }} - if: matrix.target != '' - - # Build `wasmtime` and executables. Note that we include some non-default - # features so the # release artifacts can be maximally feature-ful. - - run: $CENTOS cargo build --release --bin wasmtime --features all-arch,component-model - # Build `libwasmtime.so` - - run: $CENTOS cargo build --release --manifest-path crates/c-api/Cargo.toml + - run: $CENTOS ./ci/build-release-artifacts.sh "${{ matrix.build }}" "${{ matrix.target }}" - # Assemble release artifats appropriate for this platform, then upload them + # Assemble release artifacts appropriate for this platform, then upload them # unconditionally to this workflow's files so we have a copy of them. - run: ./ci/build-tarballs.sh "${{ matrix.build }}" "${{ matrix.target }}" + - uses: actions/upload-artifact@v3 with: name: bins-${{ matrix.build }} diff --git a/.github/workflows/publish-artifacts.yml b/.github/workflows/publish-artifacts.yml index e036474d24eb..8c149003dc91 100644 --- a/.github/workflows/publish-artifacts.yml +++ b/.github/workflows/publish-artifacts.yml @@ -28,6 +28,8 @@ jobs: env: GH_TOKEN: ${{ github.token }} + - run: ./ci/merge-artifacts.sh + # Deploy the `gh-pages.tar.gz` artifact to the `gh-pages` branch. - run: tar xf gh-pages.tar.gz working-directory: gh-pages @@ -41,10 +43,6 @@ jobs: - run: npm install --production working-directory: .github/actions/github-release - - run: | - mkdir dist - mv -t dist bins-*/*.tar.* - mv -t dist bins-*/*.{zip,msi,wasm} - name: Publish Release uses: ./.github/actions/github-release with: diff --git a/ci/build-build-matrix.js b/ci/build-build-matrix.js new file mode 100644 index 000000000000..23c2b6ebb060 --- /dev/null +++ b/ci/build-build-matrix.js @@ -0,0 +1,68 @@ +// Small script used to calculate the matrix of builds that are going to be +// done if CI decides to do a release build. +// +// This is a separate script primarily to write out all the release +// targets/platforms once and then duplicate them all with a "min" build. + +const array = [ + { + // The name of the build which shows up in the name of the artifact for + // Wasmtime's github releases. + "build": "x86_64-linux", + // The GitHub Actions platform that this build runs on + "os": "ubuntu-latest", + // The Rust target that will be used for the build. + "target": "x86_64-unknown-linux-gnu", + }, + { + "build": "aarch64-linux", + "os": "ubuntu-latest", + "target": "aarch64-unknown-linux-gnu", + }, + { + "build": "s390x-linux", + "os": "ubuntu-latest", + "target": "s390x-unknown-linux-gnu", + }, + { + "build": "riscv64gc-linux", + "os": "ubuntu-latest", + "target": "riscv64gc-unknown-linux-gnu", + }, + { + "build": "x86_64-macos", + "os": "macos-latest", + "target": "x86_64-apple-darwin", + }, + { + "build": "aarch64-macos", + "os": "macos-latest", + "target": "aarch64-apple-darwin", + }, + { + "build": "x86_64-windows", + "os": "windows-latest", + "target": "x86_64-pc-windows-msvc", + }, + { + "build": "x86_64-mingw", + "os": "windows-latest", + "target": "x86_64-pc-windows-gnu", + }, +]; + +const builds = []; +for (let build of array) { + // Perform a "deep clone" roundtripping through JSON for a copy of the build + // that's normal + build.rust = 'stable'; + builds.push(JSON.parse(JSON.stringify(build))); + + // Next generate a "min" build and add it to the builds list. Min builds + // require Nightly rust due to some nightly build options that are configured. + build.build += '-min'; + build.rust = 'nightly-2023-10-10'; + builds.push(build); +} + +console.log(JSON.stringify(builds)); diff --git a/ci/build-release-artifacts.sh b/ci/build-release-artifacts.sh new file mode 100755 index 000000000000..3b3b2b1abaa1 --- /dev/null +++ b/ci/build-release-artifacts.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# A script to build the release artifacts of Wasmtime into the `target` +# directory. For now this is the CLI and the C API. Note that this script only +# produces the artifacts through Cargo and doesn't package things up. That's +# intended for the `build-tarballs.sh` script. +# +# This script takes a Rust target as its first input and optionally a parameter +# afterwards which can be "-min" to indicate that a minimal build should be +# produced with as many features as possible stripped out. + +set -ex + +build=$1 +target=$2 + +# Default build flags for release artifacts. Leave debugging for +# builds-from-source which have richer information anyway, and additionally the +# CLI won't benefit from catching unwinds and neither will the C API so use +# panic=abort in both situations. +export CARGO_PROFILE_RELEASE_STRIP=debuginfo +export CARGO_PROFILE_RELEASE_PANIC=abort + +if [[ "$build" = *-min ]]; then + # Configure a whole bunch of compile-time options which help reduce the size + # of the binary artifact produced. + export CARGO_PROFILE_RELEASE_OPT_LEVEL=s + export RUSTFLAGS=-Zlocation-detail=none + export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1 + export CARGO_PROFILE_RELEASE_LTO=true + flags="-Zbuild-std=std,panic_abort --no-default-features -Zbuild-std-features=" + flags="$flags --features disable-logging" +else + # For release builds the CLI is built a bit more feature-ful than the Cargo + # defaults to provide artifacts that can do as much as possible. + bin_flags="--features all-arch,component-model" +fi + +cargo build --release $flags --target $target -p wasmtime-cli $bin_flags +cargo build --release $flags --target $target -p wasmtime-c-api diff --git a/ci/build-src-tarball.sh b/ci/build-src-tarball.sh index 571fcc47a3a7..c5d3c10fa9b7 100755 --- a/ci/build-src-tarball.sh +++ b/ci/build-src-tarball.sh @@ -18,3 +18,5 @@ cargo vendor > .cargo/config.toml tar -czf /tmp/$pkgname.tar.gz --transform "s/^\./$pkgname/S" --exclude=.git . mkdir -p dist mv /tmp/$pkgname.tar.gz dist/ + +rm .cargo/config.toml diff --git a/ci/build-tarballs.sh b/ci/build-tarballs.sh index eb6197c2a5ac..d8a249f113b1 100755 --- a/ci/build-tarballs.sh +++ b/ci/build-tarballs.sh @@ -1,19 +1,18 @@ #!/bin/bash # A small script used for assembling release tarballs for both the `wasmtime` -# binary and the C API. This is executed with two arguments, mostly coming from -# the CI matrix. +# binary and the C API. This is executed with two arguments, mostly coming +# from the CI matrix. # -# * The first argument is the name of the platform, used to name the release -# * The second argument is the "target", if present, currently only for -# cross-compiles +# * The first argument is the name of the "build", used to name the release. +# * The second argument is the Rust target that the build was performed for. # # This expects the build to already be done and will assemble release artifacts # in `dist/` set -ex -platform=$1 +build=$1 target=$2 rm -rf tmp @@ -25,8 +24,15 @@ if [[ $GITHUB_REF == refs/heads/release-* ]]; then tag=v$(./ci/print-current-version.sh) fi -bin_pkgname=wasmtime-$tag-$platform -api_pkgname=wasmtime-$tag-$platform-c-api +# For *-min builds produce the same named artifacts as the normal build and +# they'll get unioned together in a later step in the CI. +build_pkgname=$build +if [[ $build == *-min ]]; then + build_pkgname=${build%-min} +fi + +bin_pkgname=wasmtime-$tag-$build_pkgname +api_pkgname=wasmtime-$tag-$build_pkgname-c-api mkdir tmp/$api_pkgname mkdir tmp/$api_pkgname/lib @@ -37,47 +43,61 @@ cp LICENSE README.md tmp/$bin_pkgname cp -r crates/c-api/include tmp/$api_pkgname cp crates/c-api/wasm-c-api/include/wasm.h tmp/$api_pkgname/include -fmt=tar -if [ "$platform" = "x86_64-windows" ]; then - cp target/release/wasmtime.exe tmp/$bin_pkgname - cp target/release/{wasmtime.dll,wasmtime.lib,wasmtime.dll.lib} tmp/$api_pkgname/lib - fmt=zip - - # Generate a `*.msi` installer for Windows as well - export WT_VERSION=`cat Cargo.toml | sed -n 's/^version = "\([^"]*\)".*/\1/p'` - "$WIX/bin/candle" -arch x64 -out target/wasmtime.wixobj ci/wasmtime.wxs - "$WIX/bin/light" -out dist/$bin_pkgname.msi target/wasmtime.wixobj -ext WixUtilExtension - rm dist/$bin_pkgname.wixpdb -elif [ "$platform" = "x86_64-mingw" ]; then - cp target/x86_64-pc-windows-gnu/release/wasmtime.exe tmp/$bin_pkgname - cp target/x86_64-pc-windows-gnu/release/{wasmtime.dll,libwasmtime.a,libwasmtime.dll.a} tmp/$api_pkgname/lib - fmt=zip -elif [ "$platform" = "x86_64-macos" ]; then - # Postprocess the macOS dylib a bit to have a more reasonable `LC_ID_DYLIB` - # directive than the default one that comes out of the linker when typically - # doing `cargo build`. For more info see #984 - install_name_tool -id "@rpath/libwasmtime.dylib" target/release/libwasmtime.dylib - cp target/release/wasmtime tmp/$bin_pkgname - cp target/release/libwasmtime.{a,dylib} tmp/$api_pkgname/lib -elif [ "$platform" = "aarch64-macos" ]; then - install_name_tool -id "@rpath/libwasmtime.dylib" target/aarch64-apple-darwin/release/libwasmtime.dylib - cp target/aarch64-apple-darwin/release/wasmtime tmp/$bin_pkgname - cp target/aarch64-apple-darwin/release/libwasmtime.{a,dylib} tmp/$api_pkgname/lib -elif [ "$target" = "" ]; then - cp target/release/wasmtime tmp/$bin_pkgname - cp target/release/libwasmtime.{a,so} tmp/$api_pkgname/lib -else - cp target/$target/release/wasmtime tmp/$bin_pkgname - cp target/$target/release/libwasmtime.{a,so} tmp/$api_pkgname/lib +# For *-min builds rename artifacts with a `-min` suffix to avoid eventual +# clashes with the normal builds when the tarballs are unioned together. +if [[ $build == *-min ]]; then + min="-min" fi +fmt=tar + +case $build in + x86_64-windows*) + cp target/$target/release/wasmtime.exe tmp/$bin_pkgname/wasmtime$min.exe + cp target/$target/release/wasmtime.dll tmp/$api_pkgname/lib/wasmtime$min.dll + cp target/$target/release/wasmtime.lib tmp/$api_pkgname/lib/wasmtime$min.lib + cp target/$target/release/wasmtime.dll.lib tmp/$api_pkgname/lib/wasmtime$min.dll.lib + fmt=zip + + if [ "$min" = "" ]; then + # Generate a `*.msi` installer for Windows as well + export WT_VERSION=`cat Cargo.toml | sed -n 's/^version = "\([^"]*\)".*/\1/p'` + "$WIX/bin/candle" -arch x64 -out target/wasmtime.wixobj ci/wasmtime.wxs + "$WIX/bin/light" -out dist/$bin_pkgname.msi target/wasmtime.wixobj -ext WixUtilExtension + rm dist/$bin_pkgname.wixpdb + fi + ;; + + x86_64-mingw*) + cp target/$target/release/wasmtime.exe tmp/$bin_pkgname/wasmtime$min.exe + cp target/$target/release/wasmtime.dll tmp/$api_pkgname/lib/wasmtime$min.dll + cp target/$target/release/libwasmtime.a tmp/$api_pkgname/lib/libwasmtime$min.a + cp target/$target/release/libwasmtime.dll.a tmp/$api_pkgname/lib/libwasmtime$min.dll.a + fmt=zip + ;; + + *-macos*) + # Postprocess the macOS dylib a bit to have a more reasonable `LC_ID_DYLIB` + # directive than the default one that comes out of the linker when typically + # doing `cargo build`. For more info see #984 + install_name_tool -id "@rpath/libwasmtime$min.dylib" target/$target/release/libwasmtime.dylib + cp target/$target/release/wasmtime tmp/$bin_pkgname/wasmtime$min + cp target/$target/release/libwasmtime.a tmp/$api_pkgname/lib/libwasmtime$min.a + cp target/$target/release/libwasmtime.dylib tmp/$api_pkgname/lib/libwasmtime$min.dylib + ;; + + *) + cp target/$target/release/wasmtime tmp/$bin_pkgname/wasmtime$min + cp target/$target/release/libwasmtime.a tmp/$api_pkgname/lib/libwasmtime$min.a + cp target/$target/release/libwasmtime.so tmp/$api_pkgname/lib/libwasmtime$min.so + ;; +esac + mktarball() { dir=$1 if [ "$fmt" = "tar" ]; then - # this is a bit wonky, but the goal is to use `xz` with threaded compression - # to ideally get better performance with the `-T0` flag. - tar -cvf - -C tmp $dir | xz -9 -T0 > dist/$dir.tar.xz + tar -czvf dist/$dir.tar.gz -C tmp $dir else # Note that this runs on Windows, and it looks like GitHub Actions doesn't # have a `zip` tool there, so we use something else diff --git a/ci/merge-artifacts.sh b/ci/merge-artifacts.sh new file mode 100755 index 000000000000..aece95e6c4ad --- /dev/null +++ b/ci/merge-artifacts.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Script to merge the outputs of a run on github actions to github releases. +# This is invoked from `.github/workflows/publish-artifacts.yml`. All previous +# artifacts from builds are located in `bins-*` folders. The main purpose of +# this script is to take the "min" build and merge it into the "normal" build to +# produce one final tarball. This means that the final artifacts will have both +# a normal and a min build in them for comparison and usage. + +set -ex + +# Prepare the upload folder and move all aritfacts that aren't being merged into +# this folder, e.g. the MSI installer and adapter wasm files. +rm -rf dist +mkdir dist +mv -t dist bins-*/*.{msi,wasm} + +# Merge tarballs and zips by searching for `*-min` builds, unpacking the +# min/normal builds, into the same destination, and then repacking into a +# tarball. +# +# Note that for now xz compression is used for the final artifact to try to get +# small artifacts, but it's left at the default level since a lot of artifacts +# are processed here and turning it up to the max 9 compression might take +# quite awhile on CI for this one builder to process. +for min in bins-*-min/*.tar.*; do + normal=${min/-min\//\/} + filename=$(basename $normal) + dir=${filename%.tar.gz} + + rm -rf tmp + mkdir tmp + tar xf $min -C tmp + tar xf $normal -C tmp + tar -cf - -C tmp $dir | xz -T0 > dist/$dir.tar.xz + rm $min $normal +done + +for min in bins-*-min/*.zip; do + normal=${min/-min\//\/} + filename=$(basename $normal) + dir=${filename%.zip} + + rm -rf tmp + mkdir tmp + (cd tmp && unzip -o ../$min) + (cd tmp && unzip -o ../$normal) + (cd tmp && 7z a ../dist/$dir.zip $dir/) + rm $min $normal +done + +# Copy over remaining source tarball into the dist folder +mv -t dist bins-*/*.tar.* diff --git a/ci/wasmtime.wxs b/ci/wasmtime.wxs index ec3a58f9aa13..0e947135d555 100644 --- a/ci/wasmtime.wxs +++ b/ci/wasmtime.wxs @@ -62,7 +62,7 @@ - + diff --git a/crates/runtime/src/trampolines/riscv64.rs b/crates/runtime/src/trampolines/riscv64.rs index e3b06863b217..b43afcbf4518 100644 --- a/crates/runtime/src/trampolines/riscv64.rs +++ b/crates/runtime/src/trampolines/riscv64.rs @@ -17,7 +17,9 @@ macro_rules! wasm_to_libcall_trampoline { sd ra, 32(t0) // Tail call to the actual implementation of this libcall. - j {} + .Lhi_{0}: + auipc t0, %pcrel_hi({0}) + jalr x0, %pcrel_lo(.Lhi_{0})(t0) .cfi_endproc ", diff --git a/docs/contributing-ci.md b/docs/contributing-ci.md index 5b7306f66160..fc69f243b0fd 100644 --- a/docs/contributing-ci.md +++ b/docs/contributing-ci.md @@ -13,14 +13,32 @@ directory][dir] of the repository. ## PRs and CI -Currently the full CI test suite runs on every Pull Request. All PRs need to -have that lovely green checkmark before being candidates for being merged. If a -test is failing you'll want to check out the logs on CI and fix it before the PR -can be merged. +Currently on sample of the full CI test suite is run on every Pull Request. CI +on PRs is intended to be relatively quick and catch the majority of mistakes and +errors. By default the test suite is run on x86\_64 Linux but this may change +depending on what files the PR is modifying. The intention is to run "mostly +relevant" CI on a PR by default. PR authors are expected to fix CI failures in their PR, unless the CI failure is systemic and unrelated to the PR. In that case other maintainers should be -alerted to ensure that the problem can be addressed. +alerted to ensure that the problem can be addressed. Some reviewers may also +wait to perform a review until CI is green on the PR as otherwise it may +indicate changes are needed. + +The Wasmtime repository uses GitHub's Merge Queue feature to merge PRs which. +Entry in to the merge queue requires green CI on the PR beforehand. Maintainers +who have approved a PR will flag it for entry into the merge queue, and the PR +will automatically enter the merge queue once CI is green. + +When entering the merge queue a PR will have the full test suite executed which +may include tests that weren't previously run on the PR. This may surface new +failures, and contributors are expected to fix these failures as well. + +To force PRs to execute the full test suite, which takes longer than the default +test suite for PRs, then contributors can place the string "prtest:full" +somewhere in any commit of the PR. From that point on the PR will automatically +run the full test suite as-if it were in the merge queue. Note that when going +through the merge queue this will rerun tests. ## Tests run on CI @@ -64,22 +82,23 @@ documentation of Wasmtime and Cranelift. Currently this consists of: Linux build in a really old CentOS container to have a very low glibc requirement. -* Tarballs of the Python extension - also produced on the main three platforms - these wheels are compiled on each commit. +* Tarballs of the Wasmtime C API - produced for the same set of platforms as the + CLI above. * Book and API documentation - the book is rendered with `mdbook` and we also build all documentation with `cargo doc`. -Artifacts are produced for every single commit and every single PR. You should -be able to find a downloadable version of all artifacts produced on the "runs" -page in GitHub Actions. For example [here's an example -job](https://github.com/bytecodealliance/wasmtime/actions/runs/50372673), and if -you're looking at [a specific -builder](https://github.com/bytecodealliance/wasmtime/runs/488719677?check_suite_focus=true) -you can see the artifacts link in the top right. Note that artifacts don't -become available until the whole run finishes. - -Commits merged into the `main` branch will rerun CI and will also produce -artifacts as usual. On the `main` branch, however, documentation is pushed to -the `gh-pages` branch as well, and binaries are pushed to the `dev` release on -GitHub. Finally, tagged commits get a whole dedicated release to them too. +* A source code tarball which is entirely self-contained. This source tarball + has all dependencies vendored so the network is not needed to build it. + +* WebAssembly adapters for the component model to translate + `wasi_snapshot_preview1` to WASI Preview 2. + +Artifacts are produced as part of the full CI suite. This means that artifacts +are not produced on a PR by default but can be requested via "prtest:full". All +runs through the merge queue though, which means all merges to `main`, will +produce a full suite of artifacts. The latest artifacts are available through +Wasmtime's [`dev` release][dev] and downloads are also available for recent CI +runs through the CI page in GitHub Actions. + +[dev]: https://github.com/bytecodealliance/wasmtime/releases/tag/dev diff --git a/docs/examples-minimal.md b/docs/examples-minimal.md index 3f9e9891f53d..6c0d11cd9453 100644 --- a/docs/examples-minimal.md +++ b/docs/examples-minimal.md @@ -7,8 +7,11 @@ of Wasmtime and how to best produce a minimal build of Wasmtime. ## Building a minimal CLI > *Note*: the exact numbers in this section were last updated on 2023-10-18 on a -> macOS aarch64 host. They should provide a general ballpark estimate but should -> be confirmed locally again before being totally relied upon. +> macOS aarch64 host. For up-to-date numbers consult the artifacts in the [`dev` +> release of Wasmtime][dev] where the `wasmtime-min` executable represents the +> culmination of these steps. + +[dev]: https://github.com/bytecodealliance/wasmtime/releases/tag/dev Many Wasmtime embeddings go through the `wasmtime` crate as opposed to the `wasmtime` CLI executable, but to start out let's take a look at minimizing the