Import upstream and rename imports (#4)

.github/workflows/integration.yaml

@@ -75,12 +75,18 @@ jobs:
 
   zig:
     needs: build_zig_test_binary
-    name: Zig (${{ matrix.os }})
+    name: Zig (${{ matrix.os }}, ${{ matrix.arch }}, ${{ matrix.compiler }}-compiler)
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false # don't fail fast as sometimes failures are arch/OS specific
       matrix:
-        os: [ubuntu-22.04, macos-12, windows-2022]
+        os:       [ubuntu-22.04, macos-12, windows-2022]
+        compiler: [baseline]
+        arch:     [amd64]
+        include:
+          - os:       ubuntu-22.04
+            compiler: optimizing
+            arch:     "arm64"
 
     steps:
       - name: Checkout wazero
@@ -101,15 +107,41 @@ jobs:
         with:
           go-version: ${{ env.GO_VERSION }}
 
-      - name: Install wazero
-        run: go install ./cmd/wazero
+      - name: Build wazero
+        run: go build -o ./wazerocli ./cmd/wazero
+        env:
+          GOARCH: ${{ matrix.arch }}
+
+      - name: Set up QEMU
+        if: ${{ matrix.compiler == 'optimizing' && matrix.arch == 'arm64' }}
+        uses: docker/setup-qemu-action@v2
+        with: # Avoid docker.io rate-limits; built with internal-images.yml
+          image: ghcr.io/tetratelabs/wazero/internal-binfmt
+          platforms: ${{ matrix.arch }}
 
-      # This runs the previously compiled Zig tests with wazero. If you need
-      # to troubleshoot one, you can add "-hostlogging=filesystem" after
-      # adding filter argument to the "Build Stdlib test binary" step.
+      - name: Build scratch container
+        if: ${{ matrix.compiler == 'optimizing' }}
+        run: |
+          echo 'FROM scratch' >> Dockerfile
+          echo 'CMD ["/test"]' >> Dockerfile
+          docker buildx build -t wazero:test --platform linux/${{ matrix.arch }} .
+
+
+      # The following two steps run the previously compiled Zig tests with wazero.
+      # If you need to troubleshoot any of the test, you can add "-hostlogging=filesystem" after
+      # adding filter argument either to step "Run built test binaries" or
+      # "Run built test binaries (container)".
       # e.g. --test-filter "Dir.Iterator but dir is deleted during iteration"
-      - name: Run the test binary with wazero CLI
-        run: wazero run -mount=:/ ./zigbin/test.wasm
+
+      - name: Run built test binaries (container)
+        if: ${{ matrix.compiler == 'optimizing' }}
+        run: |
+          docker run --platform linux/${{ matrix.arch }} -v $(pwd)/zigbin:/test -v $(pwd)/wazerocli:/wazero -e WAZEROCLI=/wazero --tmpfs /tmp --rm -t wazero:test \
+            /wazero run -optimizing-compiler -mount=:/ ./test/test.wasm
+
+      - name: Run built test binaries
+        if: ${{ matrix.compiler != 'optimizing' }}
+        run: ./wazerocli run -mount=:/ ./zigbin/test.wasm
 
   build_tinygo_test_binary:
     name: Build TinyGo test binary
@@ -194,12 +226,18 @@ jobs:
 
   tinygo:
     needs: build_tinygo_test_binary
-    name: TinyGo (${{ matrix.os }})
+    name: TinyGo (${{ matrix.os }}, ${{ matrix.arch }}, ${{ matrix.compiler }}-compiler)
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false # don't fail fast as sometimes failures are arch/OS specific
       matrix:
-        os: [ubuntu-22.04, macos-12, windows-2022]
+        os:       [ubuntu-22.04, macos-12, windows-2022]
+        compiler: [baseline]
+        arch:     [amd64]
+        include:
+          - os:       ubuntu-22.04
+            compiler: optimizing
+            arch:     "arm64"
 
     steps:
       - name: Checkout wazero
@@ -220,19 +258,48 @@ jobs:
         with:
           go-version: ${{ env.GO_VERSION }}
 
-      - name: Install wazero
-        run: go install ./cmd/wazero
+      - name: Build wazero
+        run: go build -o ~/wazerocli ./cmd/wazero
+        env:
+          GOARCH: ${{ matrix.arch }}
+
+      - name: Set up QEMU
+        if: ${{ matrix.compiler == 'optimizing' && matrix.arch == 'arm64' }}
+        uses: docker/setup-qemu-action@v2
+        with: # Avoid docker.io rate-limits; built with internal-images.yml
+          image: ghcr.io/tetratelabs/wazero/internal-binfmt
+          platforms: ${{ matrix.arch }}
 
-      # This runs the previously compiled TinyGo tests with wazero. If you need
+      - name: Build scratch container
+        if: ${{ matrix.compiler == 'optimizing' }}
+        run: |
+          echo 'FROM scratch' >> Dockerfile
+          echo 'CMD ["/test"]' >> Dockerfile
+          docker buildx build -t wazero:test --platform linux/${{ matrix.arch }} .
+
+      # The following steps run the previously compiled TinyGo tests with wazero. If you need
       # to troubleshoot one, you can add "-hostlogging=filesystem" and also a
       # trailing argument narrowing which test to execute.
       # e.g. "-test.run '^TestStatBadDir$'"
-      - name: Run standard library tests
+
+      - name: Run test binaries (container)
+        if: ${{ matrix.compiler == 'optimizing' }}
+        # This runs all tests compiled above in sequence. Note: This mounts /tmp to allow t.TempDir() in tests.
+        run: |
+          cd ./tinygobin
+          for bin in *.test; do
+            echo $bin
+            docker run --platform linux/${{ matrix.arch }} -v $(pwd):/test -v ~/wazerocli:/wazero -e WAZEROCLI=/wazero --tmpfs /tmp --rm -t wazero:test \
+              /wazero run -optimizing-compiler -mount=:/  /test/$bin -- -test.v
+          done
+
+      - name: Run test binaries
+        if: ${{ matrix.compiler != 'optimizing' }}
         run: |
           cd ./tinygobin
           for bin in *.test; do
             echo $bin
-            wazero run -mount=:/ -mount=:/tmp $bin -- -test.v
+            ~/wazerocli run -mount=:/ -mount=:/tmp $bin -- -test.v
           done
 
   wasi-testsuite:
@@ -325,15 +392,21 @@ jobs:
   go_tests:
     # Due to the embedding of the GOROOT of the building env(https://github.com/golang/go/blob/3c59639b902fada0a2e5a6a35bafd10fc9183b89/src/os/os_test.go#L112),
     # we have to build and cache on each OS unlike others in this file.
-    name: Go (${{ matrix.os }}, Go-${{ matrix.go-version }})
+    name: Go (${{ matrix.os }}, Go-${{ matrix.go-version }}, ${{ matrix.arch }}, ${{ matrix.compiler }}-compiler)
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false # don't fail fast as sometimes failures are arch/OS specific
       matrix:
-        os: [ubuntu-22.04, macos-12, windows-2022]
+        os:       [ubuntu-22.04, macos-12, windows-2022]
+        compiler: [baseline]
+        arch:     [amd64]
         go-version:
           - "1.21"  # Current Go version && The only version that supports wasip1.
-
+        include:
+          - os:         ubuntu-22.04
+            compiler:   optimizing
+            arch:       "arm64"
+            go-version: "1.21"
     steps:
       - id: setup-go
         uses: actions/setup-go@v4
@@ -425,24 +498,57 @@ jobs:
       - name: Checkout wazero
         uses: actions/checkout@v3
 
-      - name: Install wazero
-        run: go install ./cmd/wazero
+      - name: Build wazero
+        run: go build -o ~/wazerocli ./cmd/wazero
+        env:
+          GOARCH: ${{ matrix.arch }}
+
+      - name: Set up QEMU
+        if: ${{ matrix.compiler == 'optimizing' && matrix.arch == 'arm64' }}
+        uses: docker/setup-qemu-action@v2
+        with: # Avoid docker.io rate-limits; built with internal-images.yml
+          image: ghcr.io/tetratelabs/wazero/internal-binfmt
+          platforms: ${{ matrix.arch }}
+
+      - name: Build scratch container
+        if: ${{ matrix.compiler == 'optimizing' }}
+        run: |
+          echo 'FROM scratch' >> Dockerfile
+          echo 'CMD ["/test"]' >> Dockerfile
+          docker buildx build -t wazero:test --platform linux/${{ matrix.arch }} .
+
+      - if: ${{ matrix.compiler == 'optimizing' }}
+        name: Run test binaries (container)
+        run: |
+          echo "Running $(find ~/tests -name *.test | wc -l) test binaries"
+
+          # Skip tests that are hideously slow (mostly compilation time) for running inside QEMU.
+          skip_targets="src_encoding_json|src_runtime|src_os_exec|src_math_big|src_encoding_gob|src_time|src_archive_tar|src_math_rand|src_expvar|src_testing|src_os"
+          # Go tests often look for files relative to the source. Change to the corresponding directory.
+          for bin in $(find ~/tests -name "*.test" | grep -vE "$skip_targets"); do
+            dir=$(basename $bin); dir=${dir%.test}; dir=${dir//_/\/}
+            pushd $(go env GOROOT)/$dir
+            # Mount / as /ROOT in docker and then remount /ROOT as / in wazero; $bin is now in /ROOT/$bin.
+            docker run --platform linux/arm64 -v /:/ROOT -v ~/wazerocli:/wazero -e WAZEROCLI=/wazero --tmpfs /tmp --rm -t wazero:test \
+              /wazero run -optimizing-compiler -mount=/ROOT:/ -mount=/tmp:/tmp -env PWD=$PWD /ROOT/$bin -- -test.short -test.v
+            popd
+          done
 
-      - if: ${{ runner.os != 'Windows' }}
-        name: Run standard library tests
+      - if: ${{ matrix.compiler != 'optimizing' && runner.os != 'Windows' }}
+        name: Run test binaries
         run: |
           echo "Running $(find ~/tests -name *.test | wc -l) test binaries"
 
           # Go tests often look for files relative to the source. Change to the corresponding directory.
           for bin in ~/tests/*.test; do
             dir=$(basename $bin); dir=${dir%.test}; dir=${dir//_/\/}
             pushd $(go env GOROOT)/$dir
-            wazero run -mount=/:/ -env PWD=$PWD $bin -- -test.short -test.v
+            ~/wazerocli run -mount=/:/ -env PWD=$PWD $bin -- -test.short -test.v
             popd
           done
 
       - if: ${{ runner.os == 'Windows' }}
-        name: Run standard library tests
+        name: Run test binaries (Windows)
         # Ack failures on Windows. https://github.com/tetratelabs/wazero/issues/1410
         continue-on-error: true
         run: |
@@ -471,7 +577,7 @@ jobs:
 
             # Create a script with all the tests (do not run yet).
             echo ${MOUNT} ${IN_WASM_PWD} $GOROOT/$dir
-            COMMAND="wazero run -mount=${MOUNT} ${EXTRAPARAMS} -hostlogging=filesystem -env PWD=${IN_WASM_PWD} -env GOROOT=${GOROOT} -env GOOS=wasip1 $bin -- -test.short -test.v"
+            COMMAND="~/wazerocli run -mount=${MOUNT} ${EXTRAPARAMS} -hostlogging=filesystem -env PWD=${IN_WASM_PWD} -env GOROOT=${GOROOT} -env GOOS=wasip1 $bin -- -test.short -test.v"
             echo $COMMAND >> $SCRIPT
             # Uncomment the following line for early exit on error on Windows.
             # Otherwise the tests will report are successful evne on failure.

Makefile

@@ -278,6 +278,7 @@ clean: ## Ensure a clean build
 fuzz_timeout_seconds ?= 10
 .PHONY: fuzz
 fuzz:
+	@cd internal/integration_test/fuzz && cargo test
 	@cd internal/integration_test/fuzz && cargo fuzz run no_diff --sanitizer=none -- -rss_limit_mb=8192 -max_total_time=$(fuzz_timeout_seconds)
 	@cd internal/integration_test/fuzz && cargo fuzz run memory_no_diff --sanitizer=none -- -rss_limit_mb=8192 -max_total_time=$(fuzz_timeout_seconds)
 	@cd internal/integration_test/fuzz && cargo fuzz run validation --sanitizer=none -- -rss_limit_mb=8192 -max_total_time=$(fuzz_timeout_seconds)

api/wasm.go

@@ -374,6 +374,7 @@ type Function interface {
 	// Call is not goroutine-safe, therefore it is recommended to create
 	// another Function if you want to invoke the same function concurrently.
 	// On the other hand, sequential invocations of Call is allowed.
+	// However, this should not be called multiple times until the previous Call returns.
 	//
 	// To safely encode/decode params/results expressed as uint64, users are encouraged to
 	// use api.EncodeXXX or DecodeXXX functions. See the docs on api.ValueType.

cmd/wazero/wazero.go

@@ -20,6 +20,7 @@ import (
 	"github.com/wasilibs/wazerox/experimental"
 	"github.com/wasilibs/wazerox/experimental/gojs"
 	"github.com/wasilibs/wazerox/experimental/logging"
+	"github.com/wasilibs/wazerox/experimental/opt"
 	"github.com/wasilibs/wazerox/experimental/sock"
 	"github.com/wasilibs/wazerox/experimental/sysfs"
 	"github.com/wasilibs/wazerox/imports/wasi_snapshot_preview1"
@@ -161,6 +162,10 @@ func doRun(args []string, stdOut io.Writer, stdErr logging.Writer) int {
 	flags.BoolVar(&useInterpreter, "interpreter", false,
 		"Interprets WebAssembly modules instead of compiling them into native code.")
 
+	var useOptimizingCompiler bool
+	flags.BoolVar(&useOptimizingCompiler, "optimizing-compiler", false,
+		"[Experimental] Compiles WebAssembly modules using the optimizing compiler.")
+
 	var envs sliceFlag
 	flags.Var(&envs, "env", "key=value pair of environment variable to expose to the binary. "+
 		"Can be specified multiple times.")
@@ -269,6 +274,8 @@ func doRun(args []string, stdOut io.Writer, stdErr logging.Writer) int {
 	var rtc wazero.RuntimeConfig
 	if useInterpreter {
 		rtc = wazero.NewRuntimeConfigInterpreter()
+	} else if useOptimizingCompiler {
+		rtc = opt.NewRuntimeConfigOptimizingCompiler()
 	} else {
 		rtc = wazero.NewRuntimeConfig()
 	}

config.go

@@ -14,6 +14,7 @@ import (
 	experimentalsys "github.com/wasilibs/wazerox/experimental/sys"
 	"github.com/wasilibs/wazerox/internal/engine/compiler"
 	"github.com/wasilibs/wazerox/internal/engine/interpreter"
+	"github.com/wasilibs/wazerox/internal/engine/wazevo"
 	"github.com/wasilibs/wazerox/internal/filecache"
 	"github.com/wasilibs/wazerox/internal/internalapi"
 	"github.com/wasilibs/wazerox/internal/platform"
@@ -135,7 +136,7 @@ type RuntimeConfig interface {
 	// However, if you use this in tests of a package not named as `main`, then wazero cannot obtain the correct
 	// version of wazero due to the known issue of debug.BuildInfo function: https://github.com/golang/go/issues/33976.
 	// As a consequence, your cache won't contain the correct version information and always be treated as `dev` version.
-	// To avoid this issue, you can pass -ldflags "-X github.com/tetratelabs/wazero/internal/version.version=foo" when running tests.
+	// To avoid this issue, you can pass -ldflags "-X github.com/wasilibs/wazerox/internal/version.version=foo" when running tests.
 	WithCompilationCache(CompilationCache) RuntimeConfig
 
 	// WithCustomSections toggles parsing of "custom sections". Defaults to false.
@@ -190,6 +191,11 @@ type runtimeConfig struct {
 	ensureTermination     bool
 }
 
+// EnableOptimizingCompiler implements experimental/opt/enabler.EnableOptimizingCompiler.
+func (c *runtimeConfig) EnableOptimizingCompiler() {
+	c.newEngine = wazevo.NewEngine
+}
+
 // engineLessConfig helps avoid copy/pasting the wrong defaults.
 var engineLessConfig = &runtimeConfig{
 	enabledFeatures:       api.CoreFeaturesV2,

experimental/gojs/README.md

@@ -3,7 +3,7 @@
 When `GOOS=js` and `GOARCH=wasm`, Go's compiler targets WebAssembly Binary
 format (%.wasm).
 
-Wazero's "github.com/tetratelabs/wazero/experimental/gojs" package allows you to run
+Wazero's "github.com/wasilibs/wazerox/experimental/gojs" package allows you to run
 a `%.wasm` file compiled by Go.  This is similar to what is implemented in
 [wasm_exec.js][1]. See https://wazero.io/languages/go/ for more.
 

experimental/opt/opt.go

@@ -0,0 +1,23 @@
+package opt
+
+import (
+	"runtime"
+
+	wazero "github.com/wasilibs/wazerox"
+)
+
+type enabler interface {
+	// EnableOptimizingCompiler enables the optimizing compiler.
+	// This is only implemented the internal type of wazero.runtimeConfig.
+	EnableOptimizingCompiler()
+}
+
+// NewRuntimeConfigOptimizingCompiler returns a new RuntimeConfig with the optimizing compiler enabled.
+func NewRuntimeConfigOptimizingCompiler() wazero.RuntimeConfig {
+	if runtime.GOARCH != "arm64" {
+		panic("UseOptimizingCompiler is only supported on arm64")
+	}
+	c := wazero.NewRuntimeConfig()
+	c.(enabler).EnableOptimizingCompiler()
+	return c
+}

experimental/opt/opt_test.go

@@ -0,0 +1,20 @@
+package opt_test
+
+import (
+	"context"
+	"runtime"
+	"testing"
+
+	wazero "github.com/wasilibs/wazerox"
+	"github.com/wasilibs/wazerox/experimental/opt"
+	"github.com/wasilibs/wazerox/internal/testing/require"
+)
+
+func TestUseOptimizingCompiler(t *testing.T) {
+	if runtime.GOARCH != "arm64" {
+		return
+	}
+	c := opt.NewRuntimeConfigOptimizingCompiler()
+	r := wazero.NewRuntimeWithConfig(context.Background(), c)
+	require.NoError(t, r.Close(context.Background()))
+}

imports/wasi_snapshot_preview1/fs.go

@@ -464,15 +464,15 @@ var fdFilestatSetSize = newHostFunc(wasip1.FdFilestatSetSizeName, fdFilestatSetS
 
 func fdFilestatSetSizeFn(_ context.Context, mod api.Module, params []uint64) experimentalsys.Errno {
 	fd := int32(params[0])
-	size := uint32(params[1])
+	size := int64(params[1])
 
 	fsc := mod.(*wasm.ModuleInstance).Sys.FS()
 
 	// Check to see if the file descriptor is available
 	if f, ok := fsc.LookupFile(fd); !ok {
 		return experimentalsys.EBADF
 	} else {
-		return f.File.Truncate(int64(size))
+		return f.File.Truncate(size)
 	}
 }