bug: steam IO encode performance is unexpect, freeBytes is not done

[201430098137]

recently, I use stram IO to reduct memory usage, but memory usage is higher then sonic.Marshal, I bench it.
I encode map[string]interface{} to json and write it to file,

var cases = []map[string]interface{}{
	{
		"dsds": "dsdds",
		"dsalkjlsd":   "sasasas",
		"dsalkjlsdd":  "sasasas",
		"dsalkjlsds":  "sasasas",
		"dsalkjlasd":  "sasasas",
		"dsalkjlsdsf": "sasasas",
	},
	{
		"dsdsd": "dsdsd",
		"dsalkjalsd":   "sasasas",
		"dsalkjlsdsdd":  "sasasas",
		"dsalkjlsdss":  "sasasas",
		"dsalkjlasdsd":  "sasasas",
		"dsalkjlssfddsf": "sasasas",
		"dsalkjlssdfd":   "sasasas",
		"dsalkjlssdd":  "sasasas",
		"dsalkjlssds":  "sasasas",
		"dsalkjlassd":  "sasasas",
		"dsalkjlscdsf": "sasasas",
		"dsalkjlsdsd":   "sasasas",
		"dsalkjlsdd":  "sasasas",
		"dsalkjlsds":  "sasasas",
		"dsalkjlasd":  "sasasas",
		"dsalkjlsdsf": "sasasas",
	},
	{
		"dsdsd": "dsdsd",
		"dsalkjalsd":   "sasasas",
		"dsalkjlsdsdd":  "sasasas",
		"dsalkjlsdss":  "sasasas",
		"dsalkjlasdsd":  "sasasas",
		"dsalkjlssfddsf": "sasasas",
		"dsalkjlssdfd":   "sasasas",
		"dsalkjlssdd":  "sasasas",
		"dsalkjlssds":  "sasasas",
		"dsalkjlassd":  "sasasas",
		"dsalkjlscdsf": "sasasas",
		"dsalkjlsdsd":   "sasasas",
		"dsalkjlsdd":  "sasasas",
		"dsalkjlsds":  "sasasas",
		"dsalkjlasd":  "sasasas",
		"dsalkjlsdsf": "sasasas",
		"sasdas":"asa",
		"saasdss":"asa",
		"sassdc":"asa",
		"sasvsdsdas":"asa",
		"sasjyja":"asa",
		"sagncs":"asa",
		"sabdfss":"asa",
		"santys":"asa",
		"saabfdss":"asa",
		"sasbdfc":"asa",
		"sasbdfas":"asa",
		"sabdfsa":"asa",
		"sacdbfas":"asa",
		"sasss":"asa",
		"sasaasf":"asa",
		"saasdass":"asa",
		"sasavsasc":"asa",
		"sfda":"asa",
		"saasfsa":"asa",
		"sasafscs":"asa",
		"asf":"asa",
		"vxxz":"asa",
		"zxc":"asa",
		"nrt":"asa",
		"dbf":"asa",
		"sabwdfsa":"asa",
		"bdf":"asa",
		"sazxvss":"asa",
		"sasvzv":"asa",
		"saaczczxss":map[string]interface{}{

			"dsdsd": "dsdsd",
			"dsalkjalsd":   "sasasas",
			"dsalkjlsdsdd":  "sasasas",
			"dsalkjlsdss":  "sasasas",
			"dsalkjlasdsd":  "sasasas",
			"dsalkjlssfddsf": "sasasas",
			"dsalkjlssdfd":   "sasasas",
			"dsalkjlssdd":  "sasasas",
			"dsalkjlssds":  "sasasas",
			"dsalkjlassd":  "sasasas",
			"dsalkjlscdsf": "sasasas",
			"dsalkjlsdsd":   "sasasas",
			"dsalkjlsdd":  "sasasas",
			"dsalkjlsds":  "sasasas",
			"dsalkjlasd":  "sasasas",
			"dsalkjlsdsf": "sasasas",
		},
		"cz":"asa",
		"sasvzxas":map[string]interface{}{

			"dsdsd": "dsdsd",
			"dsalkjalsd":   "sasasas",
			"dsalkjlsdsdd":  "sasasas",
			"dsalkjlsdss":  "sasasas",
			"dsalkjlasdsd":  "sasasas",
			"dsalkjlssfddsf": "sasasas",
			"dsalkjlssdfd":   "sasasas",
			"dsalkjlssdd":  "sasasas",
			"dsalkjlssds":  "sasasas",
			"dsalkjlassd":  "sasasas",
			"dsalkjlscdsf": "sasasas",
			"dsalkjlsdsd":   "sasasas",
			"dsalkjlsdd":  "sasasas",
			"dsalkjlsds":  "sasasas",
			"dsalkjlasd":  "sasasas",
			"dsalkjlsdsf": "sasasas",
		},
		"vz":map[string]interface{}{

			"dsdsd": "dsdsd",
			"dsalkjalsd":   "sasasas",
			"dsalkjlsdsdd":  "sasasas",
			"dsalkjlsdss":  "sasasas",
			"dsalkjlasdsd":  "sasasas",
			"dsalkjlssfddsf": "sasasas",
			"dsalkjlssdfd":   "sasasas",
			"dsalkjlssdd":  "sasasas",
			"dsalkjlssds":  "sasasas",
			"dsalkjlassd":  "sasasas",
			"dsalkjlscdsf": "sasasas",
			"dsalkjlsdsd":   "sasasas",
			"dsalkjlsdd":  "sasasas",
			"dsalkjlsds":  "sasasas",
			"dsalkjlasd":  "sasasas",
			"dsalkjlsdsf": "sasasas",
		},
		"sazxvcs":map[string]interface{}{

			"dsdsd": "dsdsd",
			"dsalkjalsd":   "sasasas",
			"dsalkjlsdsdd":  "sasasas",
			"dsalkjlsdss":  "sasasas",
			"dsalkjlasdsd":  "sasasas",
			"dsalkjlssfddsf": "sasasas",
			"dsalkjlssdfd":   "sasasas",
			"dsalkjlssdd":  "sasasas",
			"dsalkjlssds":  "sasasas",
			"dsalkjlassd":  "sasasas",
			"dsalkjlscdsf": "sasasas",
			"dsalkjlsdsd":   "sasasas",
			"dsalkjlsdd":  "sasasas",
			"dsalkjlsds":  "sasasas",
			"dsalkjlasd":  "sasasas",
			"dsalkjlsdsf": "sasasas",
		},
		"sasvzxs":map[string]interface{}{

			"dsdsd": "dsdsd",
			"dsalkjalsd":   "sasasas",
			"dsalkjlsdsdd":  "sasasas",
			"dsalkjlsdss":  "sasasas",
			"dsalkjlasdsd":  "sasasas",
			"dsalkjlssfddsf": "sasasas",
			"dsalkjlssdfd":   "sasasas",
			"dsalkjlssdd":  "sasasas",
			"dsalkjlssds":  "sasasas",
			"dsalkjlassd":  "sasasas",
			"dsalkjlscdsf": "sasasas",
			"dsalkjlsdsd":   "sasasas",
			"dsalkjlsdd":  "sasasas",
			"dsalkjlsds":  "sasasas",
			"dsalkjlasd":  "sasasas",
			"dsalkjlsdsf": "sasasas",
		},
		"sl;kas":map[string]interface{}{

				"dsdsd": "dsdsd",
				"dsalkjalsd":   "sasasas",
				"dsalkjlsdsdd":  "sasasas",
				"dsalkjlsdss":  "sasasas",
				"dsalkjlasdsd":  "sasasas",
				"dsalkjlssfddsf": "sasasas",
				"dsalkjlssdfd":   "sasasas",
				"dsalkjlssdd":  "sasasas",
				"dsalkjlssds":  "sasasas",
				"dsalkjlassd":  "sasasas",
				"dsalkjlscdsf": "sasasas",
				"dsalkjlsdsd":   "sasasas",
				"dsalkjlsdd":  "sasasas",
				"dsalkjlsds":  "sasasas",
				"dsalkjlasd":  "sasasas",
				"dsalkjlsdsf": "sasasas",
		},

	},
}
func BenchmarkEncode(b *testing.B) {
	fd, err := os.OpenFile("/tmp/aa.json", os.O_RDWR|os.O_CREATE|os.O_APPEND, os.ModePerm)
	if err != nil {
		b.Errorf("打开文件错误：%s", err.Error())
	}
	var enc = sonic.ConfigDefault.NewEncoder(fd)
	for i := 0; i < b.N; i++ {
		err := enc.Encode(cases[i%len(cases)])
		if err != nil {
			b.Errorf("enc.Encode错误：%s", err.Error())
		}
	}
}

func BenchmarkMarshal(b *testing.B) {
	fd, err := os.OpenFile("/tmp/bb.json", os.O_RDWR|os.O_CREATE|os.O_APPEND, os.ModePerm)
	if err != nil {
		b.Errorf("打开文件错误：%s", err.Error())
	}

	for i := 0; i < b.N; i++ {
		bytes, err := sonic.Marshal(cases[i%len(cases)])
		if err != nil {
			b.Errorf("sonic.Marshal错误：%s", err.Error())
		}
		_, err = fd.Write(bytes)
		if err != nil {
			b.Errorf("Write错误：%s", err.Error())
		}
		_, err = fd.Write([]byte{'\n'})
		if err != nil {
			b.Errorf("Write错误：%s", err.Error())
		}
	}

}

profile memory
stream IO

sonic.Marshal

Interestingly, stream IO use more memory then sonic.Marshal.
I debugger that code, I discover the issue.

line 71 out is set out[n:], data of out is unaccess, line 81 freeBytes is useless.

[201430098137]

code: github.com/bytedance/[email protected]/internal/encoder/stream.go:70

[liuq19]

Thanks. we will investigate it. cc @AsterDY

[AsterDY]

fixed by #547