diff --git a/loader/funcdata_go115.go b/loader/funcdata_go115.go index f35f03d56..a2e3e65f9 100644 --- a/loader/funcdata_go115.go +++ b/loader/funcdata_go115.go @@ -1,5 +1,5 @@ -// go:build go1.15 && !go1.18 -// +build go1.15,!go1.18 +//go:build go1.15 && !go1.16 +// +build go1.15,!go1.16 /* * Copyright 2021 ByteDance Inc. @@ -28,7 +28,7 @@ import ( ) const ( - _Magic uint32 = 0xfffffff0 + _Magic uint32 = 0xfffffffa ) type pcHeader struct { @@ -38,7 +38,6 @@ type pcHeader struct { ptrSize uint8 // size of a ptr in bytes nfunc int // number of functions in the module nfiles uint // number of entries in the file tab - textStart uintptr // base for function entry PC offsets in this module, equal to moduledata.text funcnameOffset uintptr // offset to the funcnametab variable from pcHeader cuOffset uintptr // offset to the cutab variable from pcHeader filetabOffset uintptr // offset to the filetab variable from pcHeader @@ -64,9 +63,7 @@ type moduledata struct { noptrbss, enoptrbss uintptr end, gcdata, gcbss uintptr types, etypes uintptr - rodata uintptr - gofunc uintptr // go.func.* is actual funcinfo object in image - + textsectmap []textSection // see runtime/symtab.go: textAddr() typelinks []int32 // offsets from types itablinks []*rt.GoItab @@ -91,7 +88,7 @@ type moduledata struct { } type _func struct { - entryOff uint32 // start pc, as offset from moduledata.text/pcHeader.textStart + entry uintptr // start pc, as offset from moduledata.text/pcHeader.textStart nameOff int32 // function name, as index into moduledata.funcnametab. args int32 // in/out args size @@ -103,8 +100,7 @@ type _func struct { npcdata uint32 cuOffset uint32 // runtime.cutab offset of this function's CU funcID uint8 // set for certain special runtime functions - flag uint8 - _ [1]byte // pad + _ [2]byte // pad nfuncdata uint8 // // The end of the struct is followed immediately by two variable-length @@ -131,8 +127,8 @@ type _func struct { } type funcTab struct { - entry uint32 - funcoff uint32 + entry uintptr + funcoff uintptr } type bitVector struct { @@ -170,6 +166,11 @@ type findfuncbucket struct { _SUBBUCKETS [16]byte } + +type compilationUnit struct { + fileNames []string +} + // func name table format: // nameOff[0] -> namePartA namePartB namePartC \x00 // nameOff[1] -> namePartA namePartB namePartC \x00 @@ -192,10 +193,6 @@ func makeFuncnameTab(funcs []Func) (tab []byte, offs []int32) { return } -type compilationUnit struct { - fileNames []string -} - // CU table format: // cuOffsets[0] -> filetabOffset[0] filetabOffset[1] ... filetabOffset[len(CUs[0].fileNames)-1] // cuOffsets[1] -> filetabOffset[len(CUs[0].fileNames)] ... filetabOffset[len(CUs[0].fileNames) + len(CUs[1].fileNames)-1] @@ -266,68 +263,76 @@ func writeFuncdata(out *[]byte, funcs []Func) (fstart int, funcdataOffs [][]uint return } -func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab) { +func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab, pclntabSize int64, startLocations []uint32) { // Allocate space for the pc->func table. This structure consists of a pc offset // and an offset to the func structure. After that, we have a single pc // value that marks the end of the last function in the binary. - var size int64 = int64(len(funcs)*2*4 + 4) - var startLocations = make([]uint32, len(funcs)) + pclntabSize = int64(len(funcs)*2*int(_PtrSize) + int(_PtrSize)) + startLocations = make([]uint32, len(funcs)) for i, f := range funcs { - size = rnd(size, int64(_PtrSize)) + pclntabSize = rnd(pclntabSize, int64(_PtrSize)) //writePCToFunc - startLocations[i] = uint32(size) - size += int64(uint8(_FUNC_SIZE)+f.nfuncdata*4+uint8(f.npcdata)*4) + startLocations[i] = uint32(pclntabSize) + pclntabSize += int64(uint8(_FUNC_SIZE) + f.nfuncdata*_PtrSize + uint8(f.npcdata)*4) } - ftab = make([]funcTab, 0, len(funcs)+1) // write a map of pc->func info offsets for i, f := range funcs { - ftab = append(ftab, funcTab{uint32(f.entryOff), uint32(startLocations[i])}) + ftab = append(ftab, funcTab{uintptr(f.entry), uintptr(startLocations[i])}) } // Final entry of table is just end pc offset. lastFunc := funcs[len(funcs)-1] - ftab = append(ftab, funcTab{uint32(lastFunc.entryOff + lastFuncSize), 0}) + ftab = append(ftab, funcTab{lastFunc.entry + uintptr(lastFuncSize), 0}) return } // Pcln table format: [...]funcTab + [...]_Func -func makePclntable(funcs []_func, lastFuncSize uint32, pcdataOffs [][]uint32, funcdataOffs [][]uint32) (pclntab []byte) { - // Allocate space for the pc->func table. This structure consists of a pc offset - // and an offset to the func structure. After that, we have a single pc - // value that marks the end of the last function in the binary. - var size int64 = int64(len(funcs)*2*4 + 4) - var startLocations = make([]uint32, len(funcs)) - for i := range funcs { - size = rnd(size, int64(_PtrSize)) - //writePCToFunc - startLocations[i] = uint32(size) - size += int64(int(_FUNC_SIZE)+len(funcdataOffs[i])*4+len(pcdataOffs[i])*4) - } - +func makePclntable(size int64, startLocations []uint32, funcs []_func, lastFuncSize uint32, pcdataOffs [][]uint32, funcdataAddr uintptr, funcdataOffs [][]uint32) (pclntab []byte) { pclntab = make([]byte, size, size) // write a map of pc->func info offsets offs := 0 for i, f := range funcs { - byteOrder.PutUint32(pclntab[offs:offs+4], uint32(f.entryOff)) - byteOrder.PutUint32(pclntab[offs+4:offs+8], uint32(startLocations[i])) - offs += 8 + byteOrder.PutUint64(pclntab[offs:offs+8], uint64(f.entry)) + byteOrder.PutUint64(pclntab[offs+8:offs+16], uint64(startLocations[i])) + offs += 16 } // Final entry of table is just end pc offset. lastFunc := funcs[len(funcs)-1] - byteOrder.PutUint32(pclntab[offs:offs+4], uint32(lastFunc.entryOff+lastFuncSize)) + byteOrder.PutUint64(pclntab[offs:offs+8], uint64(lastFunc.entry)+uint64(lastFuncSize)) + offs += 8 // write func info table for i, f := range funcs { off := startLocations[i] // write _func structure to pclntab - fb := rt.BytesFrom(unsafe.Pointer(&f), int(_FUNC_SIZE), int(_FUNC_SIZE)) - copy(pclntab[off:off+uint32(_FUNC_SIZE)], fb) - off += uint32(_FUNC_SIZE) + byteOrder.PutUint64(pclntab[off:off+8], uint64(f.entry)) + off += 8 + byteOrder.PutUint32(pclntab[off:off+4], uint32(f.nameOff)) + off += 4 + byteOrder.PutUint32(pclntab[off:off+4], uint32(f.args)) + off += 4 + byteOrder.PutUint32(pclntab[off:off+4], uint32(f.deferreturn)) + off += 4 + byteOrder.PutUint32(pclntab[off:off+4], uint32(f.pcsp)) + off += 4 + byteOrder.PutUint32(pclntab[off:off+4], uint32(f.pcfile)) + off += 4 + byteOrder.PutUint32(pclntab[off:off+4], uint32(f.pcln)) + off += 4 + byteOrder.PutUint32(pclntab[off:off+4], uint32(f.npcdata)) + off += 4 + byteOrder.PutUint32(pclntab[off:off+4], uint32(f.cuOffset)) + off += 4 + pclntab[off] = f.funcID + // NOTICE: _[2]byte alignment + off += 3 + pclntab[off] = f.nfuncdata + off += 1 // NOTICE: _func.pcdata always starts from PcUnsafePoint, which is index 3 for j := 3; j < len(pcdataOffs[i]); j++ { @@ -335,12 +340,17 @@ func makePclntable(funcs []_func, lastFuncSize uint32, pcdataOffs [][]uint32, fu off += 4 } + off = uint32(rnd(int64(off), int64(_PtrSize))) + // funcdata refs as offsets from gofunc for _, funcdata := range funcdataOffs[i] { - byteOrder.PutUint32(pclntab[off:off+4], uint32(funcdata)) - off += 4 + if funcdata == _INVALID_FUNCDATA_OFFSET { + byteOrder.PutUint64(pclntab[off:off+8], 0) + } else { + byteOrder.PutUint64(pclntab[off:off+8], uint64(funcdataAddr)+uint64(funcdata)) + } + off += 8 } - } return @@ -364,14 +374,14 @@ func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) { tab := make([]findfuncbucket, 0, nbuckets) var s, e = 0, 0 for i := 0; ifunc (binary search) func table lastFuncsize := funcs[len(funcs)-1].TextSize - ftab := makeFtab(_funcs, lastFuncsize) + ftab, pclntSize, startLocations := makeFtab(_funcs, lastFuncsize) mod.ftab = ftab // write pc->func (modmap) findfunc table ffstart := writeFindfunctab(&cache, ftab) + // cache funcdata and findfuncbucket + moduleCache.Lock() + moduleCache.m[mod] = cache + moduleCache.Unlock() + mod.findfunctab = uintptr(rt.IndexByte(cache, ffstart)) + funcdataAddr := uintptr(rt.IndexByte(cache, fstart)) + // make pclnt table - pclntab := makePclntable(_funcs, lastFuncsize, pcdataOffs, funcdataOffs) + pclntab := makePclntable(pclntSize, startLocations, _funcs, lastFuncsize, pcdataOffs, funcdataAddr, funcdataOffs) mod.pclntable = pclntab - // mmap() text and funcdata segements - p := os.Getpagesize() - size := int(rnd(int64(len(text)), int64(p))) - addr := mmap(size) - // copy the machine code - s := rt.BytesFrom(unsafe.Pointer(addr), len(text), size) - copy(s, text) - // make it executable - mprotect(addr, size) - // assign addresses mod.text = addr mod.etext = addr + uintptr(size) mod.minpc = addr mod.maxpc = addr + uintptr(len(text)) - // cache funcdata and findfuncbucket - moduleCache.Lock() - moduleCache.m[mod] = cache - moduleCache.Unlock() - mod.gofunc = uintptr(unsafe.Pointer(&cache[fstart])) - mod.findfunctab = uintptr(unsafe.Pointer(&cache[ffstart])) - // make pc header mod.pcHeader = &pcHeader { magic : _Magic, @@ -461,7 +471,6 @@ func makeModuledata(name string, filenames []string, funcs []Func, text []byte) ptrSize : _PtrSize, nfunc : len(funcs), nfiles: uint(len(cu)), - textStart: mod.text, funcnameOffset: getOffsetOf(moduledata{}, "funcnametab"), cuOffset: getOffsetOf(moduledata{}, "cutab"), filetabOffset: getOffsetOf(moduledata{}, "filetab"), @@ -478,7 +487,7 @@ func makeModuledata(name string, filenames []string, funcs []Func, text []byte) // makePctab generates pcdelta->valuedelta tables for functions, // and returns the table and the entry offset of every kind pcdata in the table. -func makePctab(funcs []Func, cuOffset []uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) { +func makePctab(funcs []Func, addr uintptr, cuOffset []uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) { _funcs = make([]_func, len(funcs)) // Pctab offsets of 0 are considered invalid in the runtime. We respect @@ -523,7 +532,7 @@ func makePctab(funcs []Func, cuOffset []uint32, nameOffset []int32) (pctab []byt writer(f.PcInlTreeIndex) writer(f.PcArgLiveIndex) - _f.entryOff = f.EntryOff + _f.entry = addr + uintptr(f.EntryOff) _f.nameOff = nameOffset[i] _f.args = f.ArgsSize _f.deferreturn = f.DeferReturn @@ -531,7 +540,6 @@ func makePctab(funcs []Func, cuOffset []uint32, nameOffset []int32) (pctab []byt _f.npcdata = uint32(_N_PCDATA) _f.cuOffset = cuOffset[i] _f.funcID = f.ID - _f.flag = f.Flag _f.nfuncdata = uint8(_N_FUNCDATA) } diff --git a/loader/funcdata_go116.go b/loader/funcdata_go116.go new file mode 100644 index 000000000..508268e7d --- /dev/null +++ b/loader/funcdata_go116.go @@ -0,0 +1,549 @@ +//go:build go1.16 && !go1.18 +// +build go1.16,!go1.18 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +import ( + `encoding` + `os` + `unsafe` + + `github.com/bytedance/sonic/internal/rt` +) + +const ( + _Magic uint32 = 0xfffffffa +) + +type pcHeader struct { + magic uint32 // 0xFFFFFFF0 + pad1, pad2 uint8 // 0,0 + minLC uint8 // min instruction size + ptrSize uint8 // size of a ptr in bytes + nfunc int // number of functions in the module + nfiles uint // number of entries in the file tab + funcnameOffset uintptr // offset to the funcnametab variable from pcHeader + cuOffset uintptr // offset to the cutab variable from pcHeader + filetabOffset uintptr // offset to the filetab variable from pcHeader + pctabOffset uintptr // offset to the pctab variable from pcHeader + pclnOffset uintptr // offset to the pclntab variable from pcHeader +} + +type moduledata struct { + pcHeader *pcHeader + funcnametab []byte + cutab []uint32 + filetab []byte + pctab []byte + pclntable []byte + ftab []funcTab + findfunctab uintptr + minpc, maxpc uintptr // first func address, last func address + last func size + + text, etext uintptr // start/end of text, (etext-text) must be greater than MIN_FUNC + noptrdata, enoptrdata uintptr + data, edata uintptr + bss, ebss uintptr + noptrbss, enoptrbss uintptr + end, gcdata, gcbss uintptr + types, etypes uintptr + + textsectmap []textSection // see runtime/symtab.go: textAddr() + typelinks []int32 // offsets from types + itablinks []*rt.GoItab + + ptab []ptabEntry + + pluginpath string + pkghashes []modulehash + + modulename string + modulehashes []modulehash + + hasmain uint8 // 1 if module contains the main function, 0 otherwise + + gcdatamask, gcbssmask bitVector + + typemap map[int32]*rt.GoType // offset to *_rtype in previous module + + bad bool // module failed to load and should be ignored + + next *moduledata +} + +type _func struct { + entry uintptr // start pc, as offset from moduledata.text/pcHeader.textStart + nameOff int32 // function name, as index into moduledata.funcnametab. + + args int32 // in/out args size + deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any. + + pcsp uint32 + pcfile uint32 + pcln uint32 + npcdata uint32 + cuOffset uint32 // runtime.cutab offset of this function's CU + funcID uint8 // set for certain special runtime functions + _ [2]byte // pad + nfuncdata uint8 // + + // The end of the struct is followed immediately by two variable-length + // arrays that reference the pcdata and funcdata locations for this + // function. + + // pcdata contains the offset into moduledata.pctab for the start of + // that index's table. e.g., + // &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of + // the unsafe point table. + // + // An offset of 0 indicates that there is no table. + // + // pcdata [npcdata]uint32 + + // funcdata contains the offset past moduledata.gofunc which contains a + // pointer to that index's funcdata. e.g., + // *(moduledata.gofunc + _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is + // the argument pointer map. + // + // An offset of ^uint32(0) indicates that there is no entry. + // + // funcdata [nfuncdata]uint32 +} + +type funcTab struct { + entry uintptr + funcoff uintptr +} + +type bitVector struct { + n int32 // # of bits + bytedata *uint8 +} + +type ptabEntry struct { + name int32 + typ int32 +} + +type textSection struct { + vaddr uintptr // prelinked section vaddr + end uintptr // vaddr + section length + baseaddr uintptr // relocated section address +} + +type modulehash struct { + modulename string + linktimehash string + runtimehash *string +} + +// findfuncbucket is an array of these structures. +// Each bucket represents 4096 bytes of the text segment. +// Each subbucket represents 256 bytes of the text segment. +// To find a function given a pc, locate the bucket and subbucket for +// that pc. Add together the idx and subbucket value to obtain a +// function index. Then scan the functab array starting at that +// index to find the target function. +// This table uses 20 bytes for every 4096 bytes of code, or ~0.5% overhead. +type findfuncbucket struct { + idx uint32 + _SUBBUCKETS [16]byte +} + + +type compilationUnit struct { + fileNames []string +} + +// func name table format: +// nameOff[0] -> namePartA namePartB namePartC \x00 +// nameOff[1] -> namePartA namePartB namePartC \x00 +// ... +func makeFuncnameTab(funcs []Func) (tab []byte, offs []int32) { + offs = make([]int32, len(funcs)) + offset := 0 + + for i, f := range funcs { + offs[i] = int32(offset) + + a, b, c := funcNameParts(f.Name) + tab = append(tab, a...) + tab = append(tab, b...) + tab = append(tab, c...) + tab = append(tab, 0) + offset += len(a) + len(b) + len(c) + 1 + } + + return +} + +// CU table format: +// cuOffsets[0] -> filetabOffset[0] filetabOffset[1] ... filetabOffset[len(CUs[0].fileNames)-1] +// cuOffsets[1] -> filetabOffset[len(CUs[0].fileNames)] ... filetabOffset[len(CUs[0].fileNames) + len(CUs[1].fileNames)-1] +// ... +// +// file name table format: +// filetabOffset[0] -> CUs[0].fileNames[0] \x00 +// ... +// filetabOffset[len(CUs[0]-1)] -> CUs[0].fileNames[len(CUs[0].fileNames)-1] \x00 +// ... +// filetabOffset[SUM(CUs,fileNames)-1] -> CUs[len(CU)-1].fileNames[len(CUs[len(CU)-1].fileNames)-1] \x00 +func makeFilenametab(cus []compilationUnit) (cutab []uint32, filetab []byte, cuOffsets []uint32) { + cuOffsets = make([]uint32, len(cus)) + cuOffset := 0 + fileOffset := 0 + + for i, cu := range cus { + cuOffsets[i] = uint32(cuOffset) + + for _, name := range cu.fileNames { + cutab = append(cutab, uint32(fileOffset)) + + fileOffset += len(name) + 1 + filetab = append(filetab, name...) + filetab = append(filetab, 0) + } + + cuOffset += len(cu.fileNames) + } + + return +} + +func writeFuncdata(out *[]byte, funcs []Func) (fstart int, funcdataOffs [][]uint32) { + fstart = len(*out) + *out = append(*out, byte(0)) + offs := uint32(1) + + funcdataOffs = make([][]uint32, len(funcs)) + for i, f := range funcs { + + var writer = func(fd encoding.BinaryMarshaler) { + var ab []byte + var err error + if fd != nil { + ab, err = fd.MarshalBinary() + if err != nil { + panic(err) + } + funcdataOffs[i] = append(funcdataOffs[i], offs) + } else { + ab = []byte{0} + funcdataOffs[i] = append(funcdataOffs[i], _INVALID_FUNCDATA_OFFSET) + } + *out = append(*out, ab...) + offs += uint32(len(ab)) + } + + writer(f.ArgsPointerMaps) + writer(f.LocalsPointerMaps) + writer(f.StackObjects) + writer(f.InlTree) + writer(f.OpenCodedDeferInfo) + writer(f.ArgInfo) + writer(f.ArgLiveInfo) + writer(f.WrapInfo) + } + return +} + +func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab, pclntabSize int64, startLocations []uint32) { + // Allocate space for the pc->func table. This structure consists of a pc offset + // and an offset to the func structure. After that, we have a single pc + // value that marks the end of the last function in the binary. + pclntabSize = int64(len(funcs)*2*int(_PtrSize) + int(_PtrSize)) + startLocations = make([]uint32, len(funcs)) + for i, f := range funcs { + pclntabSize = rnd(pclntabSize, int64(_PtrSize)) + //writePCToFunc + startLocations[i] = uint32(pclntabSize) + pclntabSize += int64(uint8(_FUNC_SIZE) + f.nfuncdata*_PtrSize + uint8(f.npcdata)*4) + } + ftab = make([]funcTab, 0, len(funcs)+1) + + // write a map of pc->func info offsets + for i, f := range funcs { + ftab = append(ftab, funcTab{uintptr(f.entry), uintptr(startLocations[i])}) + } + + // Final entry of table is just end pc offset. + lastFunc := funcs[len(funcs)-1] + ftab = append(ftab, funcTab{lastFunc.entry + uintptr(lastFuncSize), 0}) + + return +} + +// Pcln table format: [...]funcTab + [...]_Func +func makePclntable(size int64, startLocations []uint32, funcs []_func, lastFuncSize uint32, pcdataOffs [][]uint32, funcdataAddr uintptr, funcdataOffs [][]uint32) (pclntab []byte) { + pclntab = make([]byte, size, size) + + // write a map of pc->func info offsets + offs := 0 + for i, f := range funcs { + byteOrder.PutUint64(pclntab[offs:offs+8], uint64(f.entry)) + byteOrder.PutUint64(pclntab[offs+8:offs+16], uint64(startLocations[i])) + offs += 16 + } + // Final entry of table is just end pc offset. + lastFunc := funcs[len(funcs)-1] + byteOrder.PutUint64(pclntab[offs:offs+8], uint64(lastFunc.entry)+uint64(lastFuncSize)) + offs += 8 + + // write func info table + for i, f := range funcs { + off := startLocations[i] + + // write _func structure to pclntab + byteOrder.PutUint64(pclntab[off:off+8], uint64(f.entry)) + off += 8 + byteOrder.PutUint32(pclntab[off:off+4], uint32(f.nameOff)) + off += 4 + byteOrder.PutUint32(pclntab[off:off+4], uint32(f.args)) + off += 4 + byteOrder.PutUint32(pclntab[off:off+4], uint32(f.deferreturn)) + off += 4 + byteOrder.PutUint32(pclntab[off:off+4], uint32(f.pcsp)) + off += 4 + byteOrder.PutUint32(pclntab[off:off+4], uint32(f.pcfile)) + off += 4 + byteOrder.PutUint32(pclntab[off:off+4], uint32(f.pcln)) + off += 4 + byteOrder.PutUint32(pclntab[off:off+4], uint32(f.npcdata)) + off += 4 + byteOrder.PutUint32(pclntab[off:off+4], uint32(f.cuOffset)) + off += 4 + pclntab[off] = f.funcID + // NOTICE: _[2]byte alignment + off += 3 + pclntab[off] = f.nfuncdata + off += 1 + + // NOTICE: _func.pcdata always starts from PcUnsafePoint, which is index 3 + for j := 3; j < len(pcdataOffs[i]); j++ { + byteOrder.PutUint32(pclntab[off:off+4], uint32(pcdataOffs[i][j])) + off += 4 + } + + off = uint32(rnd(int64(off), int64(_PtrSize))) + + // funcdata refs as offsets from gofunc + for _, funcdata := range funcdataOffs[i] { + if funcdata == _INVALID_FUNCDATA_OFFSET { + byteOrder.PutUint64(pclntab[off:off+8], 0) + } else { + byteOrder.PutUint64(pclntab[off:off+8], uint64(funcdataAddr)+uint64(funcdata)) + } + off += 8 + } + } + + return +} + +// findfunc table used to map pc to belonging func, +// returns the index in the func table. +// +// All text section are divided into buckets sized _BUCKETSIZE(4K): +// every bucket is divided into _SUBBUCKETS sized _SUB_BUCKETSIZE(64), +// and it has a base idx to plus the offset stored in jth subbucket. +// see findfunc() in runtime/symtab.go +func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) { + start = len(*out) + + max := ftab[len(ftab)-1].entry + min := ftab[0].entry + nbuckets := (max - min + _BUCKETSIZE - 1) / _BUCKETSIZE + n := (max - min + _SUB_BUCKETSIZE - 1) / _SUB_BUCKETSIZE + + tab := make([]findfuncbucket, 0, nbuckets) + var s, e = 0, 0 + for i := 0; i 0 { + size := int(unsafe.Sizeof(findfuncbucket{}))*len(tab) + *out = append(*out, rt.BytesFrom(unsafe.Pointer(&tab[0]), size, size)...) + } + return +} + +func makeModuledata(name string, filenames []string, funcs []Func, text []byte) (mod *moduledata) { + mod = new(moduledata) + mod.modulename = name + + // make filename table + cu := make([]string, 0, len(filenames)) + for _, f := range filenames { + cu = append(cu, f) + } + cutab, filetab, cuOffs := makeFilenametab([]compilationUnit{{cu}}) + mod.cutab = cutab + mod.filetab = filetab + + // make funcname table + funcnametab, nameOffs := makeFuncnameTab(funcs) + mod.funcnametab = funcnametab + + // mmap() text and funcdata segements + p := os.Getpagesize() + size := int(rnd(int64(len(text)), int64(p))) + addr := mmap(size) + // copy the machine code + s := rt.BytesFrom(unsafe.Pointer(addr), len(text), size) + copy(s, text) + // make it executable + mprotect(addr, size) + + // make pcdata table + // NOTICE: _func only use offset to index pcdata, thus no need mmap() pcdata + pctab, pcdataOffs, _funcs := makePctab(funcs, addr, cuOffs, nameOffs) + mod.pctab = pctab + + // write func data + // NOTICE: _func use mod.gofunc+offset to directly point funcdata, thus need cache funcdata + // TODO: estimate accurate capacity + cache := make([]byte, 0, len(funcs)*int(_PtrSize)) + fstart, funcdataOffs := writeFuncdata(&cache, funcs) + + // make pc->func (binary search) func table + lastFuncsize := funcs[len(funcs)-1].TextSize + ftab, pclntSize, startLocations := makeFtab(_funcs, lastFuncsize) + mod.ftab = ftab + + // write pc->func (modmap) findfunc table + ffstart := writeFindfunctab(&cache, ftab) + + // cache funcdata and findfuncbucket + moduleCache.Lock() + moduleCache.m[mod] = cache + moduleCache.Unlock() + mod.findfunctab = uintptr(rt.IndexByte(cache, ffstart)) + funcdataAddr := uintptr(rt.IndexByte(cache, fstart)) + + // make pclnt table + pclntab := makePclntable(pclntSize, startLocations, _funcs, lastFuncsize, pcdataOffs, funcdataAddr, funcdataOffs) + mod.pclntable = pclntab + + // assign addresses + mod.text = addr + mod.etext = addr + uintptr(size) + mod.minpc = addr + mod.maxpc = addr + uintptr(len(text)) + + // make pc header + mod.pcHeader = &pcHeader { + magic : _Magic, + minLC : _MinLC, + ptrSize : _PtrSize, + nfunc : len(funcs), + nfiles: uint(len(cu)), + funcnameOffset: getOffsetOf(moduledata{}, "funcnametab"), + cuOffset: getOffsetOf(moduledata{}, "cutab"), + filetabOffset: getOffsetOf(moduledata{}, "filetab"), + pctabOffset: getOffsetOf(moduledata{}, "pctab"), + pclnOffset: getOffsetOf(moduledata{}, "pclntable"), + } + + // sepecial case: gcdata and gcbss must by non-empty + mod.gcdata = uintptr(unsafe.Pointer(&emptyByte)) + mod.gcbss = uintptr(unsafe.Pointer(&emptyByte)) + + return +} + +// makePctab generates pcdelta->valuedelta tables for functions, +// and returns the table and the entry offset of every kind pcdata in the table. +func makePctab(funcs []Func, addr uintptr, cuOffset []uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) { + _funcs = make([]_func, len(funcs)) + + // Pctab offsets of 0 are considered invalid in the runtime. We respect + // that by just padding a single byte at the beginning of runtime.pctab, + // that way no real offsets can be zero. + pctab = make([]byte, 1, 12*len(funcs)+1) + pcdataOffs = make([][]uint32, len(funcs)) + + for i, f := range funcs { + _f := &_funcs[i] + + var writer = func(pc *Pcdata) { + var ab []byte + var err error + if pc != nil { + ab, err = pc.MarshalBinary() + if err != nil { + panic(err) + } + pcdataOffs[i] = append(pcdataOffs[i], uint32(len(pctab))) + } else { + ab = []byte{0} + pcdataOffs[i] = append(pcdataOffs[i], _PCDATA_INVALID_OFFSET) + } + pctab = append(pctab, ab...) + } + + if f.Pcsp != nil { + _f.pcsp = uint32(len(pctab)) + } + writer(f.Pcsp) + if f.Pcfile != nil { + _f.pcfile = uint32(len(pctab)) + } + writer(f.Pcfile) + if f.Pcline != nil { + _f.pcln = uint32(len(pctab)) + } + writer(f.Pcline) + writer(f.PcUnsafePoint) + writer(f.PcStackMapIndex) + writer(f.PcInlTreeIndex) + writer(f.PcArgLiveIndex) + + _f.entry = addr + uintptr(f.EntryOff) + _f.nameOff = nameOffset[i] + _f.args = f.ArgsSize + _f.deferreturn = f.DeferReturn + // NOTICE: _func.pcdata is always as [PCDATA_UnsafePoint(0) : PCDATA_ArgLiveIndex(3)] + _f.npcdata = uint32(_N_PCDATA) + _f.cuOffset = cuOffset[i] + _f.funcID = f.ID + _f.nfuncdata = uint8(_N_FUNCDATA) + } + + return +} + +func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {} \ No newline at end of file diff --git a/loader/loader_go115.go b/loader/loader_go115.go index 1e44b36a6..a1d4d7892 100644 --- a/loader/loader_go115.go +++ b/loader/loader_go115.go @@ -1,5 +1,5 @@ -//go:build go1.15 && !go1.18 -// +build go1.15,!go1.18 +//go:build go1.15 && !go1.16 +// +build go1.15,!go1.16 /* * Copyright 2021 ByteDance Inc. @@ -20,14 +20,9 @@ package loader import ( - `github.com/bytedance/sonic/internal/loader` ) func (self Loader) LoadOne(text []byte, funcName string, frameSize int, argSize int, argStackmap []bool, localStackmap []bool) Function { return Function(loader.Loader(text).Load(funcName, frameSize, argSize, argStackmap, localStackmap)) -} - -func Load(modulename string, filenames []string, funcs []Func, text []byte) (out []Function) { - panic("not implemented") } \ No newline at end of file diff --git a/loader/loader_go118.go b/loader/loader_go116.go similarity index 98% rename from loader/loader_go118.go rename to loader/loader_go116.go index 9a0fe4843..ea30ec9a5 100644 --- a/loader/loader_go118.go +++ b/loader/loader_go116.go @@ -1,5 +1,5 @@ -//go:build go1.18 && !go1.21 -// +build go1.18,!go1.21 +//go:build go1.16 && !go1.21 +// +build go1.16,!go1.21 /* * Copyright 2021 ByteDance Inc. diff --git a/loader/loader_go116_test.go b/loader/loader_go116_test.go new file mode 100644 index 000000000..0505bc872 --- /dev/null +++ b/loader/loader_go116_test.go @@ -0,0 +1,142 @@ +//go:build go1.16 && !go1.17 +// +build go1.16,!go1.17 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +import ( + `fmt` + `runtime` + `runtime/debug` + `strconv` + `testing` + `unsafe` + + `github.com/bytedance/sonic/internal/rt` + `github.com/stretchr/testify/require` +) + +func TestLoad(t *testing.T) { + // defer func() { + // if r := recover(); r != nil { + // runtime.GC() + // if r != "hook1" { + // t.Fatal("not right panic:" + r.(string)) + // } + // } else { + // t.Fatal("not panic") + // } + // }() + + var hstr string + + type TestFunc func(i *int, hook func(i *int)) int + var hook = func(i *int) { + runtime.GC() + debug.FreeOSMemory() + hstr = ("hook" + strconv.Itoa(*i)) + runtime.GC() + debug.FreeOSMemory() + } + // var f TestFunc = func(i *int, hook func(i *int)) int { + // var t = *i + // hook(i) + // return t + *i + // } + bc := []byte { + 0x48, 0x83, 0xec, 0x18, // (0x00) subq $24, %rsp + 0x48, 0x89, 0x6c, 0x24, 0x10, // (0x04) movq %rbp, 16(%rsp) + 0x48, 0x8d, 0x6c, 0x24, 0x10, // (0x09) leaq 16(%rsp), %rbp + 0x48, 0x8b, 0x44, 0x24, 0x20, // (0x0e) movq 32(%rsp), %rax + 0x48, 0x8b, 0x08, // (0x13) movq (%rax), %rcx + 0x48, 0x89, 0x4c, 0x24, 0x08, // (0x16) movq %rcx, 8(%rsp) + 0x48, 0x8b, 0x54, 0x24, 0x28, // (0x1b) movq 40(%rsp), %rdx + 0x48, 0x8b, 0x1a, // (0x20) movq (%rdx), %rbx + 0x48, 0x89, 0x04, 0x24, // (0x23) movq %rax, (%rsp) + 0xff, 0xd3, // (0x27) callq %rbx + 0x48, 0x8b, 0x44, 0x24, 0x08, // (0x29) movq 8(%rsp), %rax + 0x48, 0x8b, 0x4c, 0x24, 0x20, // (0x2e) movq 32(%rsp), %rcx + 0x48, 0x03, 0x01, // (0x33) addq (%rcx), %rax + 0x48, 0x89, 0x44, 0x24, 0x30, // (0x36) movq %rax, 48(%rsp) + 0x48, 0x8b, 0x6c, 0x24, 0x10, // (0x3b) movq 16(%rsp), %rbp + 0x48, 0x83, 0xc4, 0x18, // (0x40) addq $24, %rsp + 0xc3, // (0x44) ret + } + + size := uint32(len(bc)) + fn := Func{ + ID: 0, + Flag: 0, + ArgsSize: 16, + EntryOff: 0, + TextSize: size, + DeferReturn: 0, + FileIndex: 0, + Name: "dummy", + } + + fn.Pcsp = &Pcdata{ + {PC: 0x04, Val: 0}, + {PC: size, Val: 24}, + } + + fn.Pcline = &Pcdata{ + {PC: 0x13, Val: 0}, + {PC: 0x1b, Val: 1}, + {PC: 0x23, Val: 2}, + {PC: size, Val: 3}, + } + + fn.Pcfile = &Pcdata{ + {PC: size, Val: 0}, + } + + fn.PcUnsafePoint = &Pcdata{ + {PC: size, Val: PCDATA_UnsafePointUnsafe}, + } + + fn.PcStackMapIndex = &Pcdata{ + {PC: size, Val: 0}, + } + + args := rt.StackMapBuilder{} + args.AddField(true) + args.AddField(true) + fn.ArgsPointerMaps = args.Build() + ab, _ := fn.ArgsPointerMaps.MarshalBinary() + fmt.Printf("args: %+v\n", ab) + + locals := rt.StackMapBuilder{} + locals.AddField(false) + locals.AddField(false) + fn.LocalsPointerMaps = locals.Build() + lb, _ := fn.LocalsPointerMaps.MarshalBinary() + fmt.Printf("locals: %+v\n", lb) + + rets := Load(bc, []Func{fn}, "dummy_module", []string{"github.com/bytedance/sonic/dummy.go"}) + println("func address ", *(*unsafe.Pointer)(rets[0])) + // for k, _ := range moduleCache.m { + // spew.Dump(k) + // } + f := *(*TestFunc)(unsafe.Pointer(&rets[0])) + i := 1 + j := f(&i, hook) + require.Equal(t, 2, j) + require.Equal(t, "hook1", hstr) + +} \ No newline at end of file diff --git a/loader/loader_test.go b/loader/loader_go117_test.go similarity index 99% rename from loader/loader_test.go rename to loader/loader_go117_test.go index f18640695..d5712f805 100644 --- a/loader/loader_test.go +++ b/loader/loader_go117_test.go @@ -1,5 +1,5 @@ -//go:build go1.18 -// +build go1.18 +//go:build go1.17 +// +build go1.17 /* * Copyright 2021 ByteDance Inc.