diff --git a/.github/workflows/benchmark-linux-arm64.yml b/.github/workflows/benchmark-linux-arm64.yml index 8027e6077..9a14088e2 100644 --- a/.github/workflows/benchmark-linux-arm64.yml +++ b/.github/workflows/benchmark-linux-arm64.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.17.1 + go-version: 1.21.1 - uses: actions/cache@v2 with: diff --git a/.github/workflows/benchmark-linux-x64.yml b/.github/workflows/benchmark-linux-x64.yml index 41f271434..512fa4ae2 100644 --- a/.github/workflows/benchmark-linux-x64.yml +++ b/.github/workflows/benchmark-linux-x64.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.17.1 + go-version: 1.21.1 - uses: actions/cache@v2 with: diff --git a/.github/workflows/compatibility_test.yml b/.github/workflows/compatibility_test.yml index 954f5a2b7..6fa0ed775 100644 --- a/.github/workflows/compatibility_test.yml +++ b/.github/workflows/compatibility_test.yml @@ -6,7 +6,7 @@ jobs: build: strategy: matrix: - go-version: [1.15.x, 1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x] + go-version: [1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x] os: [arm, X64] runs-on: ${{ matrix.os }} steps: @@ -28,10 +28,16 @@ jobs: ${{ runner.os }}-go- - name: main - run: GOMAXPROCS=4 go test -v -gcflags="all=-l" -race github.com/bytedance/sonic + run: GOMAXPROCS=4 go test -v -race github.com/bytedance/sonic + + - name: encoder + run: GOMAXPROCS=4 go test -v -race github.com/bytedance/sonic/internal/encoder + + - name: issues + run: GOMAXPROCS=4 go test -v -race github.com/bytedance/sonic/issue_test - name: ast - run: GOMAXPROCS=4 go test -v -gcflags="all=-l" -race github.com/bytedance/sonic/ast + run: GOMAXPROCS=4 go test -v -race github.com/bytedance/sonic/ast - name: qemu run: sh scripts/qemu.sh \ No newline at end of file diff --git a/.github/workflows/fuzzing-linux-x64.yml b/.github/workflows/fuzzing-linux-x64.yml index d7246b2f1..57fe28dea 100644 --- a/.github/workflows/fuzzing-linux-x64.yml +++ b/.github/workflows/fuzzing-linux-x64.yml @@ -1,10 +1,13 @@ -name: Fuzz Test Linux-X64 +name: Fuzz Test Linux on: pull_request jobs: build: - runs-on: [self-hosted, X64] + strategy: + matrix: + os: [arm, X64] + runs-on: ${{ matrix.os }} steps: - name: Clear repository run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE diff --git a/.github/workflows/unit_test-linux-x64.yml b/.github/workflows/unit_test-linux-x64.yml index bdc597aff..cb247f620 100644 --- a/.github/workflows/unit_test-linux-x64.yml +++ b/.github/workflows/unit_test-linux-x64.yml @@ -6,7 +6,7 @@ jobs: build: strategy: matrix: - go-version: [1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x] + go-version: [1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x] runs-on: [self-hosted, X64] steps: - name: Clear repository diff --git a/compat.go b/compat.go deleted file mode 100644 index 57de53a23..000000000 --- a/compat.go +++ /dev/null @@ -1,141 +0,0 @@ -// +build !amd64 !go1.16 go1.22 - -/* - * Copyright 2021 ByteDance Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package sonic - -import ( - `bytes` - `encoding/json` - `io` - `reflect` - - `github.com/bytedance/sonic/dev/decoder` - `github.com/bytedance/sonic/option` -) - -type frozenConfig struct { - Config - // encoderOpts encoder.Options - decoderOpts decoder.Options -} - -// Froze convert the Config to API -func (cfg Config) Froze() API { - api := &frozenConfig{Config: cfg} - - // configure decoder options: - if cfg.UseInt64 { - api.decoderOpts |= decoder.OptionUseInt64 - } - if cfg.UseNumber { - api.decoderOpts |= decoder.OptionUseNumber - } - if cfg.DisallowUnknownFields { - api.decoderOpts |= decoder.OptionDisableUnknown - } - if cfg.CopyString { - api.decoderOpts |= decoder.OptionCopyString - } - if cfg.ValidateString { - api.decoderOpts |= decoder.OptionValidateString - } - return api -} - -func (cfg frozenConfig) marshalOptions(val interface{}, prefix, indent string) ([]byte, error) { - w := bytes.NewBuffer([]byte{}) - enc := json.NewEncoder(w) - enc.SetEscapeHTML(cfg.EscapeHTML) - enc.SetIndent(prefix, indent) - err := enc.Encode(val) - out := w.Bytes() - - // json.Encoder always appends '\n' after encoding, - // which is not same with json.Marshal() - if len(out) > 0 && out[len(out)-1] == '\n' { - out = out[:len(out)-1] - } - return out, err -} - -// Marshal is implemented by sonic -func (cfg frozenConfig) Marshal(val interface{}) ([]byte, error) { - if !cfg.EscapeHTML { - return cfg.marshalOptions(val, "", "") - } - return json.Marshal(val) -} - -// MarshalToString is implemented by sonic -func (cfg frozenConfig) MarshalToString(val interface{}) (string, error) { - out, err := cfg.Marshal(val) - return string(out), err -} - -// MarshalIndent is implemented by sonic -func (cfg frozenConfig) MarshalIndent(val interface{}, prefix, indent string) ([]byte, error) { - if !cfg.EscapeHTML { - return cfg.marshalOptions(val, prefix, indent) - } - return json.MarshalIndent(val, prefix, indent) -} - -// UnmarshalFromString is implemented by sonic -func (cfg frozenConfig) UnmarshalFromString(buf string, val interface{}) error { - dec := decoder.NewDecoder(buf) - dec.SetOptions(cfg.decoderOpts) - return dec.Decode(val) -} - -// Unmarshal is implemented by sonic -func (cfg frozenConfig) Unmarshal(buf []byte, val interface{}) error { - return cfg.UnmarshalFromString(string(buf), val) -} - -// NewEncoder is implemented by sonic -func (cfg frozenConfig) NewEncoder(writer io.Writer) Encoder { - enc := json.NewEncoder(writer) - if !cfg.EscapeHTML { - enc.SetEscapeHTML(cfg.EscapeHTML) - } - return enc -} - -// TODO: use dev.NewDecoder -// NewDecoder is implemented by sonic -func (cfg frozenConfig) NewDecoder(reader io.Reader) Decoder { - dec := decoder.NewStreamDecoder(reader) - dec.Decoder.SetOptions(cfg.decoderOpts) - return dec -} - -// Valid is implemented by sonic -func (cfg frozenConfig) Valid(data []byte) bool { - return json.Valid(data) -} - -// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in -// order to reduce the first-hit latency at **amd64** Arch. -// Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is -// a compile option to set the depth of recursive compile for the nested struct type. -// * This is the none implement for !amd64. -// It will be useful for someone who develop with !amd64 arch,like Mac M1. -func Pretouch(vt reflect.Type, opts ...option.CompileOption) error { - return nil -} - diff --git a/decoder/decoder_compat.go b/decoder/decoder_compat.go index c6d869e0f..90c9e0334 100644 --- a/decoder/decoder_compat.go +++ b/decoder/decoder_compat.go @@ -19,7 +19,6 @@ package decoder import ( - `encoding/json` `io` `reflect` @@ -66,11 +65,11 @@ func Pretouch(vt reflect.Type, opts ...option.CompileOption) error { } // TODO: replace stream decoder with dev/decoder -type StreamDecoder = json.Decoder +type StreamDecoder = decoder.StreamDecoder // NewStreamDecoder adapts to encoding/json.NewDecoder API. // // NewStreamDecoder returns a new decoder that reads from r. func NewStreamDecoder(r io.Reader) *StreamDecoder { - return json.NewDecoder(r) + return decoder.NewStreamDecoder(r) } \ No newline at end of file diff --git a/encoder/encoder_amd64.go b/encoder/encoder.go similarity index 99% rename from encoder/encoder_amd64.go rename to encoder/encoder.go index 1cbef7b24..fedb79978 100644 --- a/encoder/encoder_amd64.go +++ b/encoder/encoder.go @@ -1,5 +1,3 @@ -// +build amd64,go1.16,!go1.22 - /* * Copyright 2023 ByteDance Inc. * diff --git a/encoder/encoder_compat.go b/encoder/encoder_compat.go deleted file mode 100644 index 2d3cb27a9..000000000 --- a/encoder/encoder_compat.go +++ /dev/null @@ -1,254 +0,0 @@ -// +build !amd64 !go1.16 go1.22 - -/* -* Copyright 2023 ByteDance Inc. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package encoder - -import ( - `io` - `bytes` - `encoding/json` - `reflect` - - `github.com/bytedance/sonic/option` -) - -func init() { - println("WARNING: sonic only supports Go1.16~1.21 && CPU amd64, but your environment is not suitable") -} - -// Options is a set of encoding options. -type Options uint64 - -const ( - bitSortMapKeys = iota - bitEscapeHTML - bitCompactMarshaler - bitNoQuoteTextMarshaler - bitNoNullSliceOrMap - bitValidateString - bitNoValidateJSONMarshaler - bitNoEncoderNewline - - // used for recursive compile - bitPointerValue = 63 -) - -const ( - // SortMapKeys indicates that the keys of a map needs to be sorted - // before serializing into JSON. - // WARNING: This hurts performance A LOT, USE WITH CARE. - SortMapKeys Options = 1 << bitSortMapKeys - - // EscapeHTML indicates encoder to escape all HTML characters - // after serializing into JSON (see https://pkg.go.dev/encoding/json#HTMLEscape). - // WARNING: This hurts performance A LOT, USE WITH CARE. - EscapeHTML Options = 1 << bitEscapeHTML - - // CompactMarshaler indicates that the output JSON from json.Marshaler - // is always compact and needs no validation - CompactMarshaler Options = 1 << bitCompactMarshaler - - // NoQuoteTextMarshaler indicates that the output text from encoding.TextMarshaler - // is always escaped string and needs no quoting - NoQuoteTextMarshaler Options = 1 << bitNoQuoteTextMarshaler - - // NoNullSliceOrMap indicates all empty Array or Object are encoded as '[]' or '{}', - // instead of 'null' - NoNullSliceOrMap Options = 1 << bitNoNullSliceOrMap - - // ValidateString indicates that encoder should validate the input string - // before encoding it into JSON. - ValidateString Options = 1 << bitValidateString - - // NoValidateJSONMarshaler indicates that the encoder should not validate the output string - // after encoding the JSONMarshaler to JSON. - NoValidateJSONMarshaler Options = 1 << bitNoValidateJSONMarshaler - - // NoEncoderNewline indicates that the encoder should not add a newline after every message - NoEncoderNewline Options = 1 << bitNoEncoderNewline - - // CompatibleWithStd is used to be compatible with std encoder. - CompatibleWithStd Options = SortMapKeys | EscapeHTML | CompactMarshaler -) - -// Encoder represents a specific set of encoder configurations. -type Encoder struct { - Opts Options - prefix string - indent string -} - -// Encode returns the JSON encoding of v. -func (self *Encoder) Encode(v interface{}) ([]byte, error) { - if self.indent != "" || self.prefix != "" { - return EncodeIndented(v, self.prefix, self.indent, self.Opts) - } - return Encode(v, self.Opts) -} - -// SortKeys enables the SortMapKeys option. -func (self *Encoder) SortKeys() *Encoder { - self.Opts |= SortMapKeys - return self -} - -// SetEscapeHTML specifies if option EscapeHTML opens -func (self *Encoder) SetEscapeHTML(f bool) { - if f { - self.Opts |= EscapeHTML - } else { - self.Opts &= ^EscapeHTML - } -} - -// SetValidateString specifies if option ValidateString opens -func (self *Encoder) SetValidateString(f bool) { - if f { - self.Opts |= ValidateString - } else { - self.Opts &= ^ValidateString - } -} - -// SetNoValidateJSONMarshaler specifies if option NoValidateJSONMarshaler opens -func (self *Encoder) SetNoValidateJSONMarshaler(f bool) { - if f { - self.Opts |= NoValidateJSONMarshaler - } else { - self.Opts &= ^NoValidateJSONMarshaler - } -} - -// SetNoEncoderNewline specifies if option NoEncoderNewline opens -func (self *Encoder) SetNoEncoderNewline(f bool) { - if f { - self.Opts |= NoEncoderNewline - } else { - self.Opts &= ^NoEncoderNewline - } -} - -// SetCompactMarshaler specifies if option CompactMarshaler opens -func (self *Encoder) SetCompactMarshaler(f bool) { - if f { - self.Opts |= CompactMarshaler - } else { - self.Opts &= ^CompactMarshaler - } -} - -// SetNoQuoteTextMarshaler specifies if option NoQuoteTextMarshaler opens -func (self *Encoder) SetNoQuoteTextMarshaler(f bool) { - if f { - self.Opts |= NoQuoteTextMarshaler - } else { - self.Opts &= ^NoQuoteTextMarshaler - } -} - -// SetIndent instructs the encoder to format each subsequent encoded -// value as if indented by the package-level function EncodeIndent(). -// Calling SetIndent("", "") disables indentation. -func (enc *Encoder) SetIndent(prefix, indent string) { - enc.prefix = prefix - enc.indent = indent -} - -// Quote returns the JSON-quoted version of s. -func Quote(s string) string { - /* check for empty string */ - if s == "" { - return `""` - } - - out, _ := json.Marshal(s) - return string(out) -} - -// Encode returns the JSON encoding of val, encoded with opts. -func Encode(val interface{}, opts Options) ([]byte, error) { - return json.Marshal(val) -} - -// EncodeInto is like Encode but uses a user-supplied buffer instead of allocating -// a new one. -func EncodeInto(buf *[]byte, val interface{}, opts Options) error { - if buf == nil { - panic("user-supplied buffer buf is nil") - } - w := bytes.NewBuffer(*buf) - enc := json.NewEncoder(w) - enc.SetEscapeHTML((opts & EscapeHTML) != 0) - err := enc.Encode(val) - *buf = w.Bytes() - return err -} - -// HTMLEscape appends to dst the JSON-encoded src with <, >, &, U+2028 and U+2029 -// characters inside string literals changed to \u003c, \u003e, \u0026, \u2028, \u2029 -// so that the JSON will be safe to embed inside HTML