This repository has been archived by the owner on Jan 24, 2024. It is now read-only.
Update to latest threads spectests (#5) #13
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Standard Library Integration Tests | |
on: | |
pull_request: | |
branches: [main] | |
paths-ignore: # ignore docs as they are built with Netlify. | |
- '**/*.md' | |
- 'site/**' | |
- 'netlify.toml' | |
push: | |
branches: [main] | |
paths-ignore: # ignore docs as they are built with Netlify. | |
- '**/*.md' | |
- 'site/**' | |
- 'netlify.toml' | |
defaults: | |
run: # use bash for all operating systems unless overridden | |
shell: bash | |
env: # Update this prior to requiring a higher minor version in go.mod | |
GO_VERSION: "1.21" # 1.xx == latest patch of 1.xx | |
TINYGO_VERSION: "0.30.0" | |
ZIG_VERSION: "0.11.0" | |
concurrency: | |
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-using-concurrency-to-cancel-any-in-progress-job-or-run | |
group: ${{ github.ref }}-${{ github.workflow }}-${{ github.actor }} | |
cancel-in-progress: true | |
jobs: | |
# This builds a zig test binary only if the same version tag hasn't been build before. | |
# This saves time as we rarely update the zig version. | |
build_zig_test_binary: | |
name: Build Zig test binary | |
runs-on: ubuntu-22.04 | |
env: | |
ZIG_INSTALL: ~/zig-install | |
ZIG_SOURCE: ~/zig-source | |
steps: | |
- name: Checkout wazero | |
uses: actions/checkout@v3 | |
- uses: actions/cache@v3 | |
id: binary-cache | |
with: | |
# Use share the cache containing archives across OSes. | |
enableCrossOsArchive: true | |
key: zig-stdlib-test-binary-${{ env.ZIG_VERSION }} | |
path: | |
./zigbin/ | |
- name: Install Zig build | |
if: steps.binary-cache.outputs.cache-hit != 'true' | |
run: | | |
mkdir -p ${{ env.ZIG_INSTALL }} | |
curl -sSL https://ziglang.org/download/${{ env.ZIG_VERSION }}/zig-linux-x86_64-${{ env.ZIG_VERSION }}.tar.xz | tar -xJ --strip-components=1 -C ${{ env.ZIG_INSTALL }} | |
- name: Download Zig source code | |
if: steps.binary-cache.outputs.cache-hit != 'true' | |
run: | | |
mkdir -p ${{ env.ZIG_SOURCE }} | |
curl -sSL https://ziglang.org/download/${{ env.ZIG_VERSION }}/zig-${{ env.ZIG_VERSION }}.tar.xz | tar -xJ --strip-components=1 -C ${{ env.ZIG_SOURCE }} | |
- name: Build Stdlib test binary | |
if: steps.binary-cache.outputs.cache-hit != 'true' | |
# --test-no-exec allows building of the test Wasm binary without executing command. | |
# We use find because the test.wasm will be something like ./zig-cache/o/dd6df1361b2134adc5eee9d027495436/test.wasm | |
run: | | |
mkdir ${{ github.workspace }}/zigbin | |
cd ${{ env.ZIG_SOURCE }} | |
${{ env.ZIG_INSTALL }}/zig test --test-no-exec -target wasm32-wasi --zig-lib-dir ./lib ./lib/std/std.zig | |
_ZIG_TEST_BINARY_PATH=$(find . -name test.wasm) | |
cp ${_ZIG_TEST_BINARY_PATH} ${{ github.workspace }}/zigbin/test.wasm | |
zig: | |
needs: build_zig_test_binary | |
name: Zig (${{ matrix.os }}, ${{ matrix.arch }}, ${{ matrix.compiler }}-compiler) | |
runs-on: ${{ matrix.os }} | |
strategy: | |
fail-fast: false # don't fail fast as sometimes failures are arch/OS specific | |
matrix: | |
os: [ubuntu-22.04, macos-12, windows-2022] | |
compiler: [baseline] | |
arch: [amd64] | |
include: | |
- os: ubuntu-22.04 | |
compiler: optimizing | |
arch: "arm64" | |
steps: | |
- name: Checkout wazero | |
uses: actions/checkout@v3 | |
- uses: actions/cache@v3 | |
id: binary-cache | |
with: | |
# Use share the cache containing archives across OSes. | |
enableCrossOsArchive: true | |
# We need this cache to run tests. | |
fail-on-cache-miss: true | |
key: zig-stdlib-test-binary-${{ env.ZIG_VERSION }} | |
path: | |
./zigbin/ | |
- uses: actions/setup-go@v4 | |
with: | |
go-version: ${{ env.GO_VERSION }} | |
- name: Build wazero | |
run: go build -o ./wazerocli ./cmd/wazero | |
env: | |
GOARCH: ${{ matrix.arch }} | |
- name: Set up QEMU | |
if: ${{ matrix.compiler == 'optimizing' && matrix.arch == 'arm64' }} | |
uses: docker/setup-qemu-action@v2 | |
with: # Avoid docker.io rate-limits; built with internal-images.yml | |
image: ghcr.io/tetratelabs/wazero/internal-binfmt | |
platforms: ${{ matrix.arch }} | |
- name: Build scratch container | |
if: ${{ matrix.compiler == 'optimizing' }} | |
run: | | |
echo 'FROM scratch' >> Dockerfile | |
echo 'CMD ["/test"]' >> Dockerfile | |
docker buildx build -t wazero:test --platform linux/${{ matrix.arch }} . | |
# The following two steps run the previously compiled Zig tests with wazero. | |
# If you need to troubleshoot any of the test, you can add "-hostlogging=filesystem" after | |
# adding filter argument either to step "Run built test binaries" or | |
# "Run built test binaries (container)". | |
# e.g. --test-filter "Dir.Iterator but dir is deleted during iteration" | |
- name: Run built test binaries (container) | |
if: ${{ matrix.compiler == 'optimizing' }} | |
run: | | |
docker run --platform linux/${{ matrix.arch }} -v $(pwd)/zigbin:/test -v $(pwd)/wazerocli:/wazero -e WAZEROCLI=/wazero --tmpfs /tmp --rm -t wazero:test \ | |
/wazero run -optimizing-compiler -mount=:/ ./test/test.wasm | |
- name: Run built test binaries | |
if: ${{ matrix.compiler != 'optimizing' }} | |
run: ./wazerocli run -mount=:/ ./zigbin/test.wasm | |
build_tinygo_test_binary: | |
name: Build TinyGo test binary | |
runs-on: ubuntu-22.04 | |
steps: | |
- name: Checkout wazero | |
uses: actions/checkout@v3 | |
- uses: actions/cache@v3 | |
id: binary-cache | |
with: | |
# Use share the cache containing archives across OSes. | |
enableCrossOsArchive: true | |
key: tinygo-test-binaries-${{ env.TINYGO_VERSION }} | |
path: | |
./tinygobin/ | |
- name: Install TinyGo | |
if: steps.binary-cache.outputs.cache-hit != 'true' | |
run: | # installing via curl so commands are similar on OS/x | |
tinygo_version=${{ env.TINYGO_VERSION }} | |
curl -sSL https://github.com/tinygo-org/tinygo/releases/download/v${tinygo_version}/tinygo${tinygo_version}.linux-amd64.tar.gz | sudo tar -C /usr/local -xzf - | |
echo "TINYGOROOT=/usr/local/tinygo" >> $GITHUB_ENV | |
echo "/usr/local/tinygo/bin" >> $GITHUB_PATH | |
- uses: actions/setup-go@v4 | |
if: steps.binary-cache.outputs.cache-hit != 'true' | |
with: | |
go-version: ${{ env.GO_VERSION }} | |
- name: Build Test Binaries | |
if: steps.binary-cache.outputs.cache-hit != 'true' | |
# The following list of packages is derived from: | |
# https://github.com/tinygo-org/tinygo/blob/v0.28.1/Makefile#L281-L322 | |
# Note: | |
# - index/suffixarray is extremely slow, so skip it. | |
# - compress/zlib is skipped as it depends on the local files https://github.com/golang/go/blob/go1.20/src/compress/zlib/writer_test.go#L16-L30 | |
# - debug/macho is skipped as it depends on the local files https://github.com/golang/go/blob/go1.20/src/debug/macho/file_test.go#L25 | |
run: | | |
mkdir ./tinygobin | |
for value in container/heap \ | |
container/list \ | |
container/ring \ | |
crypto/des \ | |
crypto/md5 \ | |
crypto/rc4 \ | |
crypto/sha1 \ | |
crypto/sha256 \ | |
crypto/sha512 \ | |
embed/internal/embedtest \ | |
encoding \ | |
encoding/ascii85 \ | |
encoding/base32 \ | |
encoding/csv \ | |
encoding/hex \ | |
go/scanner \ | |
hash \ | |
hash/adler32 \ | |
hash/crc64 \ | |
hash/fnv \ | |
html \ | |
internal/itoa \ | |
internal/profile \ | |
math \ | |
math/cmplx \ | |
net \ | |
net/http/internal/ascii \ | |
net/mail \ | |
os \ | |
path \ | |
reflect \ | |
sync \ | |
testing \ | |
testing/iotest \ | |
text/scanner \ | |
unicode \ | |
unicode/utf16 \ | |
unicode/utf8 | |
do | |
tinygo test -target wasi -c -o ./tinygobin/${value/\//_}.test $value | |
done | |
tinygo: | |
needs: build_tinygo_test_binary | |
name: TinyGo (${{ matrix.os }}, ${{ matrix.arch }}, ${{ matrix.compiler }}-compiler) | |
runs-on: ${{ matrix.os }} | |
strategy: | |
fail-fast: false # don't fail fast as sometimes failures are arch/OS specific | |
matrix: | |
os: [ubuntu-22.04, macos-12, windows-2022] | |
compiler: [baseline] | |
arch: [amd64] | |
include: | |
- os: ubuntu-22.04 | |
compiler: optimizing | |
arch: "arm64" | |
steps: | |
- name: Checkout wazero | |
uses: actions/checkout@v3 | |
- uses: actions/cache@v3 | |
id: binary-cache | |
with: | |
# Use share the cache containing archives across OSes. | |
enableCrossOsArchive: true | |
# We need this cache to run tests. | |
fail-on-cache-miss: true | |
key: tinygo-test-binaries-${{ env.TINYGO_VERSION }} | |
path: | |
./tinygobin/ | |
- uses: actions/setup-go@v4 | |
with: | |
go-version: ${{ env.GO_VERSION }} | |
- name: Build wazero | |
run: go build -o ~/wazerocli ./cmd/wazero | |
env: | |
GOARCH: ${{ matrix.arch }} | |
- name: Set up QEMU | |
if: ${{ matrix.compiler == 'optimizing' && matrix.arch == 'arm64' }} | |
uses: docker/setup-qemu-action@v2 | |
with: # Avoid docker.io rate-limits; built with internal-images.yml | |
image: ghcr.io/tetratelabs/wazero/internal-binfmt | |
platforms: ${{ matrix.arch }} | |
- name: Build scratch container | |
if: ${{ matrix.compiler == 'optimizing' }} | |
run: | | |
echo 'FROM scratch' >> Dockerfile | |
echo 'CMD ["/test"]' >> Dockerfile | |
docker buildx build -t wazero:test --platform linux/${{ matrix.arch }} . | |
# The following steps run the previously compiled TinyGo tests with wazero. If you need | |
# to troubleshoot one, you can add "-hostlogging=filesystem" and also a | |
# trailing argument narrowing which test to execute. | |
# e.g. "-test.run '^TestStatBadDir$'" | |
- name: Run test binaries (container) | |
if: ${{ matrix.compiler == 'optimizing' }} | |
# This runs all tests compiled above in sequence. Note: This mounts /tmp to allow t.TempDir() in tests. | |
run: | | |
cd ./tinygobin | |
for bin in *.test; do | |
echo $bin | |
docker run --platform linux/${{ matrix.arch }} -v $(pwd):/test -v ~/wazerocli:/wazero -e WAZEROCLI=/wazero --tmpfs /tmp --rm -t wazero:test \ | |
/wazero run -optimizing-compiler -mount=:/ /test/$bin -- -test.v | |
done | |
- name: Run test binaries | |
if: ${{ matrix.compiler != 'optimizing' }} | |
run: | | |
cd ./tinygobin | |
for bin in *.test; do | |
echo $bin | |
~/wazerocli run -mount=:/ -mount=:/tmp $bin -- -test.v | |
done | |
wasi-testsuite: | |
name: wasi-testsuite | |
runs-on: ${{ matrix.os }} | |
strategy: | |
fail-fast: false # don't fail fast as sometimes failures are arch/OS specific | |
matrix: | |
os: [ubuntu-22.04, macos-12, windows-2022] | |
steps: | |
- uses: actions/cache@v3 | |
id: cache | |
with: | |
path: | |
~/go/pkg/mod | |
key: integration-test-wasi-testsuite-${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} | |
- uses: actions/setup-go@v4 | |
with: | |
go-version: ${{ env.GO_VERSION }} | |
- name: Checkout wazero | |
uses: actions/checkout@v3 | |
- name: Install wazero | |
run: go install ./cmd/wazero | |
- name: Checkout wasi-testsuite | |
uses: actions/checkout@v3 | |
with: | |
repository: WebAssembly/wasi-testsuite | |
# prod/testsuite-base branch, as of May 12, 2023. | |
# TODO: once the wasi-testsuite is stable, we should use the latest tag instead of a branch. | |
ref: c9c751586fd86b321d595bbef13f2c7403cfdbc5 | |
path: wasi-testsuite | |
- name: Initialize Python environment | |
uses: actions/setup-python@v4 | |
with: | |
python-version: '3.11' # latest version of python 3 | |
cache: pip | |
- name: Install dependencies | |
working-directory: wasi-testsuite/test-runner | |
run: | | |
python3 -m pip install -r requirements.txt | |
- name: Run all wasi-testsuite | |
working-directory: wasi-testsuite | |
run: | | |
python3 test-runner/wasi_test_runner.py \ | |
-t ./tests/assemblyscript/testsuite/ \ | |
./tests/c/testsuite/ \ | |
./tests/rust/testsuite/ \ | |
-f ../.github/wasi_testsuite_skip.json \ | |
-r ../.github/wasi_testsuite_adapter.py | |
gojs_stdlib: | |
name: Go (js) (${{ matrix.os }}) | |
runs-on: ${{ matrix.os }} | |
strategy: | |
fail-fast: false # don't fail fast as sometimes failures are arch/OS specific | |
matrix: | |
os: [ubuntu-22.04, macos-12] # GOOS=js isn't supposed to work on windows. See #1222 | |
steps: | |
- uses: actions/setup-go@v4 | |
with: | |
go-version: ${{ env.GO_VERSION }} | |
- name: Checkout wazero | |
uses: actions/checkout@v3 | |
- name: Install wazero | |
run: go install ./cmd/wazero | |
- name: Build gojs test binaries | |
env: | |
GOOS: js | |
GOARCH: wasm | |
run: | # Only test os package as this is being replaced by GOOS=wasip1 | |
mkdir ~/bin && cd ~/bin | |
go test -c -o os.wasm os | |
- name: Run tests | |
run: | # skip tests that use functionality not also used in GOOS=wasip1 | |
cd $(go env GOROOT)/src/os; wazero run -mount=/:/ ~/bin/os.wasm -test.v -test.skip '^Test(Chmod|Truncate|LongPath|Chown|FileChown).*$' | |
go_tests: | |
# Due to the embedding of the GOROOT of the building env(https://github.com/golang/go/blob/3c59639b902fada0a2e5a6a35bafd10fc9183b89/src/os/os_test.go#L112), | |
# we have to build and cache on each OS unlike others in this file. | |
name: Go (${{ matrix.os }}, Go-${{ matrix.go-version }}, ${{ matrix.arch }}, ${{ matrix.compiler }}-compiler) | |
runs-on: ${{ matrix.os }} | |
strategy: | |
fail-fast: false # don't fail fast as sometimes failures are arch/OS specific | |
matrix: | |
os: [ubuntu-22.04, macos-12, windows-2022] | |
compiler: [baseline] | |
arch: [amd64] | |
go-version: | |
- "1.21" # Current Go version && The only version that supports wasip1. | |
include: | |
- os: ubuntu-22.04 | |
compiler: optimizing | |
arch: "arm64" | |
go-version: "1.21" | |
steps: | |
- id: setup-go | |
uses: actions/setup-go@v4 | |
with: | |
go-version: ${{ matrix.go-version }} | |
- name: Cache Go test binaries | |
id: cache-go-test-binaries | |
uses: actions/cache@v3 | |
with: | |
path: ~/tests | |
# Use precise Go version from setup-go as patch version differences can effect tests. | |
key: go-wasip1-binaries-${{ matrix.os }}-${{ steps.setup-go.outputs.go-version }} | |
- if: ${{ steps.cache-go-test-binaries.outputs.cache-hit != 'true' }} | |
name: Build Test Binaries | |
run: | | |
mkdir ~/tests | |
cd $(go env GOROOT) | |
# Choose important packages to limit execution time. | |
for value in src/archive/tar \ | |
src/bufio \ | |
src/bytes \ | |
src/context \ | |
src/encoding/ascii85 \ | |
src/encoding/asn1 \ | |
src/encoding/base32 \ | |
src/encoding/base64 \ | |
src/encoding/binary \ | |
src/encoding/csv \ | |
src/encoding/gob \ | |
src/encoding/hex \ | |
src/encoding/json \ | |
src/encoding/pem \ | |
src/encoding/xml \ | |
src/errors \ | |
src/expvar \ | |
src/flag \ | |
src/fmt \ | |
src/hash \ | |
src/hash/adler32 \ | |
src/hash/crc32 \ | |
src/hash/crc64 \ | |
src/hash/fnv \ | |
src/hash/maphash \ | |
src/io \ | |
src/io/fs \ | |
src/io/ioutil \ | |
src/log \ | |
src/log/syslog \ | |
src/maps \ | |
src/math \ | |
src/math/big \ | |
src/math/bits \ | |
src/math/cmplx \ | |
src/math/rand \ | |
src/mime \ | |
src/mime/multipart \ | |
src/mime/quotedprintable \ | |
src/os \ | |
src/os/exec \ | |
src/os/signal \ | |
src/os/user \ | |
src/path \ | |
src/reflect \ | |
src/regexp \ | |
src/regexp/syntax \ | |
src/runtime \ | |
src/runtime/internal/atomic \ | |
src/runtime/internal/math \ | |
src/runtime/internal/sys \ | |
src/slices \ | |
src/sort \ | |
src/strconv \ | |
src/strings \ | |
src/sync \ | |
src/sync/atomic \ | |
src/syscall \ | |
src/testing \ | |
src/testing/fstest \ | |
src/testing/iotest \ | |
src/testing/quick \ | |
src/time | |
do | |
echo "GOOS=wasip1 GOARCH=wasm go test -v -c -o ~/tests/${value//\//_}.test ./$value" | |
GOOS=wasip1 GOARCH=wasm go test -v -c -o ~/tests/${value//\//_}.test ./$value | |
done | |
- name: Checkout wazero | |
uses: actions/checkout@v3 | |
- name: Build wazero | |
run: go build -o ~/wazerocli ./cmd/wazero | |
env: | |
GOARCH: ${{ matrix.arch }} | |
- name: Set up QEMU | |
if: ${{ matrix.compiler == 'optimizing' && matrix.arch == 'arm64' }} | |
uses: docker/setup-qemu-action@v2 | |
with: # Avoid docker.io rate-limits; built with internal-images.yml | |
image: ghcr.io/tetratelabs/wazero/internal-binfmt | |
platforms: ${{ matrix.arch }} | |
- name: Build scratch container | |
if: ${{ matrix.compiler == 'optimizing' }} | |
run: | | |
echo 'FROM scratch' >> Dockerfile | |
echo 'CMD ["/test"]' >> Dockerfile | |
docker buildx build -t wazero:test --platform linux/${{ matrix.arch }} . | |
- if: ${{ matrix.compiler == 'optimizing' }} | |
name: Run test binaries (container) | |
run: | | |
echo "Running $(find ~/tests -name *.test | wc -l) test binaries" | |
# Skip tests that are hideously slow (mostly compilation time) for running inside QEMU. | |
skip_targets="src_encoding_json|src_runtime|src_os_exec|src_math_big|src_encoding_gob|src_time|src_archive_tar|src_math_rand|src_expvar|src_testing|src_os" | |
# Go tests often look for files relative to the source. Change to the corresponding directory. | |
for bin in $(find ~/tests -name "*.test" | grep -vE "$skip_targets"); do | |
dir=$(basename $bin); dir=${dir%.test}; dir=${dir//_/\/} | |
pushd $(go env GOROOT)/$dir | |
# Mount / as /ROOT in docker and then remount /ROOT as / in wazero; $bin is now in /ROOT/$bin. | |
docker run --platform linux/arm64 -v /:/ROOT -v ~/wazerocli:/wazero -e WAZEROCLI=/wazero --tmpfs /tmp --rm -t wazero:test \ | |
/wazero run -optimizing-compiler -mount=/ROOT:/ -mount=/tmp:/tmp -env PWD=$PWD /ROOT/$bin -- -test.short -test.v | |
popd | |
done | |
- if: ${{ matrix.compiler != 'optimizing' && runner.os != 'Windows' }} | |
name: Run test binaries | |
run: | | |
echo "Running $(find ~/tests -name *.test | wc -l) test binaries" | |
# Go tests often look for files relative to the source. Change to the corresponding directory. | |
for bin in ~/tests/*.test; do | |
dir=$(basename $bin); dir=${dir%.test}; dir=${dir//_/\/} | |
pushd $(go env GOROOT)/$dir | |
~/wazerocli run -mount=/:/ -env PWD=$PWD $bin -- -test.short -test.v | |
popd | |
done | |
- if: ${{ runner.os == 'Windows' }} | |
name: Run test binaries (Windows) | |
# Ack failures on Windows. https://github.com/tetratelabs/wazero/issues/1410 | |
continue-on-error: true | |
run: | | |
GOOS=$(go env GOOS) | |
echo "Running $(find ~/tests -name *.test | wc -l) test binaries" | |
MOUNT=c:\\:/ | |
SCRIPT="$HOME/tests.cmd" | |
# Trim `/c` from the in-Wasm GOROOT. | |
REAL_GOROOT=$(go env GOROOT) | |
GOROOT=$(cygpath -u $REAL_GOROOT | cut -c 3-) | |
# Append early exit on cmd. | |
POSTFIX="if %errorlevel% neq 0 exit /b %errorlevel%" | |
RUNNER="cmd //c %USERPROFILE%\tests.cmd" | |
EXTRAPARAMS="-mount=%TEMP%:/tmp" | |
# Go tests often look for files relative to the source. Change to the corresponding directory. | |
for bin in ~/tests/*.test; do | |
dir=$(basename $bin); dir=${dir%.test}; dir=${dir//_/\/} | |
pushd $REAL_GOROOT/$dir | |
# Trim `/c` from the in-Wasm pwd. | |
IN_WASM_PWD=$(pwd | cut -c 3-) | |
# Convert to a Windows path. | |
bin=`cygpath -w $bin` | |
# Create a script with all the tests (do not run yet). | |
echo ${MOUNT} ${IN_WASM_PWD} $GOROOT/$dir | |
COMMAND="~/wazerocli run -mount=${MOUNT} ${EXTRAPARAMS} -hostlogging=filesystem -env PWD=${IN_WASM_PWD} -env GOROOT=${GOROOT} -env GOOS=wasip1 $bin -- -test.short -test.v" | |
echo $COMMAND >> $SCRIPT | |
# Uncomment the following line for early exit on error on Windows. | |
# Otherwise the tests will report are successful evne on failure. | |
# echo $POSTFIX >> $SCRIPT | |
popd | |
done | |
# Run all the tests in their own shell. | |
$RUNNER |