opt: encoder VM

.github/workflows/benchmark-linux-x64.yml → .github/workflows/benchmark-linux.yml

@@ -4,7 +4,10 @@ on: pull_request
 
 jobs:
   build:
-    runs-on: [self-hosted, X64]
+    strategy:
+      matrix:
+        os: [X64, arm]
+    runs-on: ${{ matrix.os }}
     steps:
       - name: Clear repository
         run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE

.github/workflows/compatibility_test.yml

@@ -6,7 +6,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.15.x, 1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
+        go-version: [1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
         os: [arm, X64]
     runs-on: ${{ matrix.os }}
     steps:
@@ -28,10 +28,16 @@ jobs:
             ${{ runner.os }}-go-
 
       - name: main
-        run: GOMAXPROCS=4 go test -v -gcflags="all=-l" -race github.com/bytedance/sonic
+        run: go test -race -v -gcflags="all=-l" github.com/bytedance/sonic
+
+      - name: decoder
+        run: go test -race -v -gcflags="all=-l" github.com/bytedance/sonic/decoder
+
+      - name: encoder
+        run: go test -race -v -gcflags="all=-l" github.com/bytedance/sonic/encoder
 
       - name: ast
-        run: GOMAXPROCS=4 go test -v -gcflags="all=-l" -race github.com/bytedance/sonic/ast
+        run: go test -race -v -gcflags="all=-l" github.com/bytedance/sonic/ast
 
       - name: qemu
         run: sh scripts/qemu.sh

.github/workflows/fuzzing-linux-x64.yml

@@ -4,7 +4,10 @@ on: pull_request
 
 jobs:
   build:
-    runs-on: [self-hosted, X64]
+    strategy:
+        matrix:
+          os: [arm, X64]
+    runs-on: ${{ matrix.os }}
     steps:
       - name: Clear repository
         run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
@@ -14,7 +17,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.20.x
 
       - uses: actions/cache@v2
         with:

.github/workflows/unit_test-linux-x64.yml

@@ -6,7 +6,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
+        go-version: [1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
     runs-on: [self-hosted, X64]
     steps:
       - name: Clear repository
@@ -30,17 +30,17 @@ jobs:
 
       - name: Unit Test
         run: |
-          go test -race -covermode=atomic -coverprofile=coverage.txt ./...
+          GOMAXPROCS=4 go test -race -covermode=atomic -coverprofile=coverage.txt ./...
 
       - name: external
         run: |
           cd ./external_jsonlib_test
-          GOMAXPROCS=4 go test -v -race ./...
+          go test -v -race ./...
 
       - name: external
         run: |
           cd ./loader
-          GOMAXPROCS=4 go test -v -race ./...
+          go test -v -race ./...
       
       - name: Codecov
         run: bash <(curl -s https://codecov.io/bash)

ast/api.go

@@ -1,5 +1,4 @@
-//go:build (amd64 && go1.16 && !go1.23) || (arm64 && go1.20 && !go1.23)
-// +build amd64,go1.16,!go1.23 arm64,go1.20,!go1.23
+// +build amd64,go1.17,!go1.23 arm64,go1.20,!go1.23
 
 /*
  * Copyright 2022 ByteDance Inc.

ast/api_compat.go

@@ -1,4 +1,4 @@
-// +build !amd64,!arm64 go1.23 !go1.16 arm64,!go1.20
+// +build !amd64,!arm64 go1.23 !go1.17 arm64,!go1.20
 
 /*
 * Copyright 2022 ByteDance Inc.

ast/api_native_test.go

@@ -1,5 +1,4 @@
-//go:build (amd64 && go1.16 && !go1.23) || (arm64 && go1.20 && !go1.23)
-// +build amd64,go1.16,!go1.23 arm64,go1.20,!go1.23
+// +build amd64,go1.17,!go1.23 arm64,go1.20,!go1.23
 
 /*
  * Copyright 2022 ByteDance Inc.

compat.go

@@ -1,4 +1,4 @@
-// +build !amd64 !go1.16 go1.23
+// +build !amd64 go1.23 !go1.17
 
 /*
  * Copyright 2021 ByteDance Inc.

decode_test.go

@@ -1,4 +1,4 @@
-// +build amd64,go1.16,!go1.23
+// +build amd64,go1.17,!go1.23
 
 /*
  * Copyright 2021 ByteDance Inc.

decoder/decoder_amd64.go

@@ -1,4 +1,4 @@
-// +build amd64,go1.16,!go1.23
+// +build amd64,go1.17,!go1.23
 
 /*
 * Copyright 2023 ByteDance Inc.

decoder/decoder_amd64_test.go

@@ -1,4 +1,4 @@
-// +build amd64,go1.16,!go1.23
+// +build amd64,go1.17,!go1.23
 
 /*
  * Copyright 2021 ByteDance Inc.

decoder/decoder_compat.go

@@ -1,4 +1,4 @@
-// +build !amd64 !go1.16 go1.23
+// +build !amd64 !go1.17 go1.23
 
 /*
 * Copyright 2023 ByteDance Inc.
@@ -30,7 +30,7 @@ import (
 )
 
 func init() {
-     println("WARNING: sonic only supports Go1.16~1.22 && CPU amd64, but your environment is not suitable")
+     println("WARNING: sonic only supports Go1.17~1.22 && CPU amd64, but your environment is not suitable")
 }
 
 const (

encode_test.go

@@ -1,4 +1,4 @@
-// +build amd64,go1.16,!go1.23
+// +build amd64,go1.17,!go1.23
 
 /*
  * Copyright 2021 ByteDance Inc.

encoder/encoder_compat.go

@@ -1,4 +1,4 @@
-// +build !amd64 !go1.16 go1.23
+// +build !amd64,!arm64 go1.23 !go1.17 arm64,!go1.20
 
 /*
 * Copyright 2023 ByteDance Inc.
@@ -28,7 +28,7 @@ import (
 )
 
 func init() {
-    println("WARNING:(encoder) sonic only supports Go1.16~1.22 && CPU amd64, but your environment is not suitable")
+    println("WARNING:(encoder) sonic only supports Go1.17~1.22 && CPU amd64, but your environment is not suitable")
 }
 
 // EnableFallback indicates if encoder use fallback

encoder/encoder_amd64.go → encoder/encoder_native.go

@@ -1,4 +1,4 @@
-// +build amd64,go1.16,!go1.23
+// +build amd64,go1.17,!go1.23 arm64,go1.20,!go1.23
 
 /*
  * Copyright 2023 ByteDance Inc.

encoder/encoder_amd64_test.go → encoder/encoder_native_test.go

@@ -1,4 +1,4 @@
-// +build amd64,go1.16,!go1.23
+// +build amd64,go1.17,!go1.23 arm64,go1.20,!go1.23
 
 /*
  * Copyright 2021 ByteDance Inc.

encoder/encoder_test.go

@@ -246,7 +246,7 @@ func TestEncoder_Marshal_EscapeHTML_LargeJson(t *testing.T) {
     require.NoError(t, err1)
     buf2, err2 :=json.Marshal(&_BindingValue)
     require.NoError(t, err2)
-    require.Equal(t, buf1, buf2)
+    require.Equal(t, string(buf1), string(buf2))
 }
 
 var _GenericValue interface{}

fuzz/Makefile

@@ -9,7 +9,7 @@ fuzz:
 	file2fuzz -o ${corpusdir} ./go-fuzz-corpus/json/corpus/*  ./corpus/* 
 
 run:
-	SONIC_FUZZ_MEM_LIMIT=2 GOMAXPROCS=4 GOARCH=amd64 go test -fuzz=${testname} -v -fuzztime 5m
+	SONIC_FUZZ_MEM_LIMIT=2 GOMAXPROCS=4 go test -fuzz=${testname} -v -fuzztime 5m
 
 clean:
 	rm -rf ./go-fuzz-corpus/

internal/base64/b64_amd64.go

@@ -0,0 +1,46 @@
+// +build amd64,go1.16
+
+/**
+ * Copyright 2023 ByteDance Inc.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package base64 
+
+import (
+	"github.com/cloudwego/base64x"
+)
+
+func DecodeBase64(src string) ([]byte, error) {
+	return base64x.StdEncoding.DecodeString(src)
+}
+
+func EncodeBase64(buf []byte, src []byte) []byte {
+	if len(src) == 0 {
+		return append(buf, '"', '"')
+	}
+	buf = append(buf, '"')
+	need := base64x.StdEncoding.EncodedLen(len(src))
+	if cap(buf) - len(buf) < need {
+		tmp := make([]byte, len(buf), len(buf) + need*2)
+		copy(tmp, buf)
+		buf = tmp
+	}
+	base64x.StdEncoding.Encode(buf[len(buf):cap(buf)], src)
+	buf = buf[:len(buf) + need]
+	buf = append(buf, '"')
+	return buf
+}
+
+

internal/base64/b64_compat.go

@@ -0,0 +1,44 @@
+// +build !amd64 !go1.16
+
+/*
+ * Copyright 2022 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package base64 
+
+import (
+	"encoding/base64"
+)
+
+func EncodeBase64(buf []byte, src []byte) []byte {
+	if len(src) == 0 {
+		return append(buf, '"', '"')
+	}
+	buf = append(buf, '"')
+	need := base64.StdEncoding.EncodedLen(len(src))
+	if cap(buf) - len(buf) < need {
+		tmp := make([]byte, len(buf), len(buf) + need*2)
+		copy(tmp, buf)
+		buf = tmp
+	}
+	base64.StdEncoding.Encode(buf[len(buf):cap(buf)], src)
+	buf = buf[:len(buf) + need]
+	buf = append(buf, '"')
+	return buf
+}
+
+func DecodeBase64(src string) ([]byte, error) {
+	return base64.StdEncoding.DecodeString(src)
+}