diff --git a/loader/go.mod b/loader/go.mod index 8f27f97f1..1d334f54a 100644 --- a/loader/go.mod +++ b/loader/go.mod @@ -3,6 +3,6 @@ module github.com/bytedance/sonic/loader go 1.16 require ( - github.com/cloudwego/iasm v0.2.0 + github.com/davecgh/go-spew v1.1.1 github.com/stretchr/testify v1.8.1 ) diff --git a/loader/go.sum b/loader/go.sum index a4a6fd4ef..2ec90f70f 100644 --- a/loader/go.sum +++ b/loader/go.sum @@ -1,16 +1,11 @@ -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= -github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= @@ -20,4 +15,3 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/loader/internal/abi/abi_amd64.go b/loader/internal/abi/abi_amd64.go index c2b45a8e1..2969c3bba 100644 --- a/loader/internal/abi/abi_amd64.go +++ b/loader/internal/abi/abi_amd64.go @@ -17,266 +17,285 @@ package abi import ( - `fmt` - `reflect` - `unsafe` + "fmt" + "reflect" + "unsafe" - . `github.com/cloudwego/iasm/x86_64` + x64 "github.com/bytedance/sonic/loader/internal/iasm/x86_64" +) + +type ( + Register = x64.Register + Register64 = x64.Register64 + XMMRegister = x64.XMMRegister + Program = x64.Program + MemoryOperand = x64.MemoryOperand + Label = x64.Label +) + +var ( + Ptr = x64.Ptr + DefaultArch = x64.DefaultArch + CreateLabel = x64.CreateLabel ) const ( - PtrSize = 8 // pointer size - PtrAlign = 8 // pointer alignment + RAX = x64.RAX + RSP = x64.RSP + RBP = x64.RBP + R12 = x64.R12 + R14 = x64.R14 + R15 = x64.R15 +) + +const ( + PtrSize = 8 // pointer size + PtrAlign = 8 // pointer alignment ) var iregOrderC = []Register{ - RDI, - RSI, - RDX, - RCX, - R8, - R9, + x64.RDI, + x64.RSI, + x64.RDX, + x64.RCX, + x64.R8, + x64.R9, } var xregOrderC = []Register{ - XMM0, - XMM1, - XMM2, - XMM3, - XMM4, - XMM5, - XMM6, - XMM7, + x64.XMM0, + x64.XMM1, + x64.XMM2, + x64.XMM3, + x64.XMM4, + x64.XMM5, + x64.XMM6, + x64.XMM7, } var ( - intType = reflect.TypeOf(0) - ptrType = reflect.TypeOf(unsafe.Pointer(nil)) + intType = reflect.TypeOf(0) + ptrType = reflect.TypeOf(unsafe.Pointer(nil)) ) func (self *Frame) argv(i int) *MemoryOperand { - return Ptr(RSP, int32(self.Prev() + self.desc.Args[i].Mem)) + return Ptr(RSP, int32(self.Prev()+self.desc.Args[i].Mem)) } // spillv is used for growstack spill registers func (self *Frame) spillv(i int) *MemoryOperand { - // remain one slot for caller return pc - return Ptr(RSP, PtrSize + int32(self.desc.Args[i].Mem)) + // remain one slot for caller return pc + return Ptr(RSP, PtrSize+int32(self.desc.Args[i].Mem)) } func (self *Frame) retv(i int) *MemoryOperand { - return Ptr(RSP, int32(self.Prev() + self.desc.Rets[i].Mem)) + return Ptr(RSP, int32(self.Prev()+self.desc.Rets[i].Mem)) } func (self *Frame) resv(i int) *MemoryOperand { - return Ptr(RSP, int32(self.Offs() - uint32((i+1) * PtrSize))) + return Ptr(RSP, int32(self.Offs()-uint32((i+1)*PtrSize))) } func (self *Frame) emitGrowStack(p *Program, entry *Label) { - // spill all register arguments - for i, v := range self.desc.Args { - if v.InRegister { - if v.IsFloat == floatKind64 { - p.MOVSD(v.Reg, self.spillv(i)) - } else if v.IsFloat == floatKind32 { - p.MOVSS(v.Reg, self.spillv(i)) - }else { - p.MOVQ(v.Reg, self.spillv(i)) - } - } - } - - // call runtime.morestack_noctxt - p.MOVQ(F_morestack_noctxt, R12) - p.CALLQ(R12) - // load all register arguments - for i, v := range self.desc.Args { - if v.InRegister { - if v.IsFloat == floatKind64 { - p.MOVSD(self.spillv(i), v.Reg) - } else if v.IsFloat == floatKind32 { - p.MOVSS(self.spillv(i), v.Reg) - }else { - p.MOVQ(self.spillv(i), v.Reg) - } - } - } - - // jump back to the function entry - p.JMP(entry) + // spill all register arguments + for i, v := range self.desc.Args { + if v.InRegister { + if v.IsFloat == floatKind64 { + p.MOVSD(v.Reg, self.spillv(i)) + } else if v.IsFloat == floatKind32 { + p.MOVSS(v.Reg, self.spillv(i)) + } else { + p.MOVQ(v.Reg, self.spillv(i)) + } + } + } + + // call runtime.morestack_noctxt + p.MOVQ(F_morestack_noctxt, R12) + p.CALLQ(R12) + // load all register arguments + for i, v := range self.desc.Args { + if v.InRegister { + if v.IsFloat == floatKind64 { + p.MOVSD(self.spillv(i), v.Reg) + } else if v.IsFloat == floatKind32 { + p.MOVSS(self.spillv(i), v.Reg) + } else { + p.MOVQ(self.spillv(i), v.Reg) + } + } + } + + // jump back to the function entry + p.JMP(entry) } func (self *Frame) GrowStackTextSize() uint32 { - p := DefaultArch.CreateProgram() - // spill all register arguments - for i, v := range self.desc.Args { - if v.InRegister { - if v.IsFloat == floatKind64 { - p.MOVSD(v.Reg, self.spillv(i)) - } else if v.IsFloat == floatKind32 { - p.MOVSS(v.Reg, self.spillv(i)) - }else { - p.MOVQ(v.Reg, self.spillv(i)) - } - } - } - - // call runtime.morestack_noctxt - p.MOVQ(F_morestack_noctxt, R12) - p.CALLQ(R12) - // load all register arguments - for i, v := range self.desc.Args { - if v.InRegister { - if v.IsFloat == floatKind64 { - p.MOVSD(self.spillv(i), v.Reg) - } else if v.IsFloat == floatKind32 { - p.MOVSS(self.spillv(i), v.Reg) - } else { - p.MOVQ(self.spillv(i), v.Reg) - } - } - } - - // jump back to the function entry - l := CreateLabel("") - p.Link(l) - p.JMP(l) - - return uint32(len(p.Assemble(0))) + p := DefaultArch.CreateProgram() + // spill all register arguments + for i, v := range self.desc.Args { + if v.InRegister { + if v.IsFloat == floatKind64 { + p.MOVSD(v.Reg, self.spillv(i)) + } else if v.IsFloat == floatKind32 { + p.MOVSS(v.Reg, self.spillv(i)) + } else { + p.MOVQ(v.Reg, self.spillv(i)) + } + } + } + + // call runtime.morestack_noctxt + p.MOVQ(F_morestack_noctxt, R12) + p.CALLQ(R12) + // load all register arguments + for i, v := range self.desc.Args { + if v.InRegister { + if v.IsFloat == floatKind64 { + p.MOVSD(self.spillv(i), v.Reg) + } else if v.IsFloat == floatKind32 { + p.MOVSS(self.spillv(i), v.Reg) + } else { + p.MOVQ(self.spillv(i), v.Reg) + } + } + } + + // jump back to the function entry + l := CreateLabel("") + p.Link(l) + p.JMP(l) + + return uint32(len(p.Assemble(0))) } func (self *Frame) emitPrologue(p *Program) { - p.SUBQ(self.Size(), RSP) - p.MOVQ(RBP, Ptr(RSP, int32(self.Offs()))) - p.LEAQ(Ptr(RSP, int32(self.Offs())), RBP) + p.SUBQ(self.Size(), RSP) + p.MOVQ(RBP, Ptr(RSP, int32(self.Offs()))) + p.LEAQ(Ptr(RSP, int32(self.Offs())), RBP) } func (self *Frame) emitEpilogue(p *Program) { - p.MOVQ(Ptr(RSP, int32(self.Offs())), RBP) - p.ADDQ(self.Size(), RSP) - p.RET() + p.MOVQ(Ptr(RSP, int32(self.Offs())), RBP) + p.ADDQ(self.Size(), RSP) + p.RET() } func (self *Frame) emitReserveRegs(p *Program) { - // spill reserved registers - for i, r := range ReservedRegs(self.ccall) { - switch r.(type) { - case Register64: - p.MOVQ(r, self.resv(i)) - case XMMRegister: - p.MOVSD(r, self.resv(i)) - default: - panic(fmt.Sprintf("unsupported register type %t to reserve", r)) - } - } + // spill reserved registers + for i, r := range ReservedRegs(self.ccall) { + switch r.(type) { + case Register64: + p.MOVQ(r, self.resv(i)) + case XMMRegister: + p.MOVSD(r, self.resv(i)) + default: + panic(fmt.Sprintf("unsupported register type %t to reserve", r)) + } + } } func (self *Frame) emitSpillPtrs(p *Program) { - // spill pointer argument registers - for i, r := range self.desc.Args { - if r.InRegister && r.IsPointer { - p.MOVQ(r.Reg, self.argv(i)) - } - } + // spill pointer argument registers + for i, r := range self.desc.Args { + if r.InRegister && r.IsPointer { + p.MOVQ(r.Reg, self.argv(i)) + } + } } func (self *Frame) emitClearPtrs(p *Program) { - // spill pointer argument registers - for i, r := range self.desc.Args { - if r.InRegister && r.IsPointer { - p.MOVQ(int64(0), self.argv(i)) - } - } + // spill pointer argument registers + for i, r := range self.desc.Args { + if r.InRegister && r.IsPointer { + p.MOVQ(int64(0), self.argv(i)) + } + } } func (self *Frame) emitCallC(p *Program, addr uintptr) { - p.MOVQ(addr, RAX) - p.CALLQ(RAX) + p.MOVQ(addr, RAX) + p.CALLQ(RAX) } type floatKind uint8 const ( - notFloatKind floatKind = iota - floatKind32 - floatKind64 + notFloatKind floatKind = iota + floatKind32 + floatKind64 ) type Parameter struct { - InRegister bool - IsPointer bool - IsFloat floatKind - Reg Register - Mem uint32 - Type reflect.Type + InRegister bool + IsPointer bool + IsFloat floatKind + Reg Register + Mem uint32 + Type reflect.Type } func mkIReg(vt reflect.Type, reg Register64) (p Parameter) { - p.Reg = reg - p.Type = vt - p.InRegister = true - p.IsPointer = isPointer(vt) - return + p.Reg = reg + p.Type = vt + p.InRegister = true + p.IsPointer = isPointer(vt) + return } func isFloat(vt reflect.Type) floatKind { - switch vt.Kind() { - case reflect.Float32: - return floatKind32 - case reflect.Float64: - return floatKind64 - default: - return notFloatKind - } + switch vt.Kind() { + case reflect.Float32: + return floatKind32 + case reflect.Float64: + return floatKind64 + default: + return notFloatKind + } } func mkXReg(vt reflect.Type, reg XMMRegister) (p Parameter) { - p.Reg = reg - p.Type = vt - p.InRegister = true - p.IsFloat = isFloat(vt) - return + p.Reg = reg + p.Type = vt + p.InRegister = true + p.IsFloat = isFloat(vt) + return } func mkStack(vt reflect.Type, mem uint32) (p Parameter) { - p.Mem = mem - p.Type = vt - p.InRegister = false - p.IsPointer = isPointer(vt) - p.IsFloat = isFloat(vt) - return + p.Mem = mem + p.Type = vt + p.InRegister = false + p.IsPointer = isPointer(vt) + p.IsFloat = isFloat(vt) + return } func (self Parameter) String() string { - if self.InRegister { - return fmt.Sprintf("[%%%s, Pointer(%v), Float(%v)]", self.Reg, self.IsPointer, self.IsFloat) - } else { - return fmt.Sprintf("[%d(FP), Pointer(%v), Float(%v)]", self.Mem, self.IsPointer, self.IsFloat) - } + if self.InRegister { + return fmt.Sprintf("[%%%s, Pointer(%v), Float(%v)]", self.Reg, self.IsPointer, self.IsFloat) + } else { + return fmt.Sprintf("[%d(FP), Pointer(%v), Float(%v)]", self.Mem, self.IsPointer, self.IsFloat) + } } func CallC(addr uintptr, fr Frame, maxStack uintptr) []byte { - p := DefaultArch.CreateProgram() - - stack := CreateLabel("_stack_grow") - entry := CreateLabel("_entry") - p.Link(entry) - fr.emitStackCheck(p, stack, maxStack) - fr.emitPrologue(p) - fr.emitReserveRegs(p) - fr.emitSpillPtrs(p) - fr.emitExchangeArgs(p) - fr.emitCallC(p, addr) - fr.emitExchangeRets(p) - fr.emitRestoreRegs(p) - fr.emitEpilogue(p) - p.Link(stack) - fr.emitGrowStack(p, entry) - - return p.Assemble(0) + p := DefaultArch.CreateProgram() + + stack := CreateLabel("_stack_grow") + entry := CreateLabel("_entry") + p.Link(entry) + fr.emitStackCheck(p, stack, maxStack) + fr.emitPrologue(p) + fr.emitReserveRegs(p) + fr.emitSpillPtrs(p) + fr.emitExchangeArgs(p) + fr.emitCallC(p, addr) + fr.emitExchangeRets(p) + fr.emitRestoreRegs(p) + fr.emitEpilogue(p) + p.Link(stack) + fr.emitGrowStack(p, entry) + + return p.Assemble(0) } - - -func (self *Frame) emitDebug(p *Program) { - p.INT(3) -} \ No newline at end of file diff --git a/loader/internal/abi/abi_legacy_amd64.go b/loader/internal/abi/abi_legacy_amd64.go index 298c48178..722c0696f 100644 --- a/loader/internal/abi/abi_legacy_amd64.go +++ b/loader/internal/abi/abi_legacy_amd64.go @@ -20,163 +20,196 @@ package abi import ( - `fmt` - `reflect` - `runtime` - - . `github.com/cloudwego/iasm/x86_64` + "fmt" + "reflect" + "runtime" ) func ReservedRegs(callc bool) []Register { - return nil + return nil } func salloc(p []Parameter, sp uint32, vt reflect.Type) (uint32, []Parameter) { - switch vt.Kind() { - case reflect.Bool : return sp + 8, append(p, mkStack(reflect.TypeOf(false), sp)) - case reflect.Int : return sp + 8, append(p, mkStack(intType, sp)) - case reflect.Int8 : return sp + 8, append(p, mkStack(reflect.TypeOf(int8(0)), sp)) - case reflect.Int16 : return sp + 8, append(p, mkStack(reflect.TypeOf(int16(0)), sp)) - case reflect.Int32 : return sp + 8, append(p, mkStack(reflect.TypeOf(int32(0)), sp)) - case reflect.Int64 : return sp + 8, append(p, mkStack(reflect.TypeOf(int64(0)), sp)) - case reflect.Uint : return sp + 8, append(p, mkStack(reflect.TypeOf(uint(0)), sp)) - case reflect.Uint8 : return sp + 8, append(p, mkStack(reflect.TypeOf(uint8(0)), sp)) - case reflect.Uint16 : return sp + 8, append(p, mkStack(reflect.TypeOf(uint16(0)), sp)) - case reflect.Uint32 : return sp + 8, append(p, mkStack(reflect.TypeOf(uint32(0)), sp)) - case reflect.Uint64 : return sp + 8, append(p, mkStack(reflect.TypeOf(uint64(0)), sp)) - case reflect.Uintptr : return sp + 8, append(p, mkStack(reflect.TypeOf(uintptr(0)), sp)) - case reflect.Float32 : return sp + 8, append(p, mkStack(reflect.TypeOf(float32(0)), sp)) - case reflect.Float64 : return sp + 8, append(p, mkStack(reflect.TypeOf(float64(0)), sp)) - case reflect.Complex64 : panic("abi: go116: not implemented: complex64") - case reflect.Complex128 : panic("abi: go116: not implemented: complex128") - case reflect.Array : panic("abi: go116: not implemented: arrays") - case reflect.Chan : return sp + 8, append(p, mkStack(reflect.TypeOf((chan int)(nil)), sp)) - case reflect.Func : return sp + 8, append(p, mkStack(reflect.TypeOf((func())(nil)), sp)) - case reflect.Map : return sp + 8, append(p, mkStack(reflect.TypeOf((map[int]int)(nil)), sp)) - case reflect.Ptr : return sp + 8, append(p, mkStack(reflect.TypeOf((*int)(nil)), sp)) - case reflect.UnsafePointer : return sp + 8, append(p, mkStack(ptrType, sp)) - case reflect.Interface : return sp + 16, append(p, mkStack(ptrType, sp), mkStack(ptrType, sp + 8)) - case reflect.Slice : return sp + 24, append(p, mkStack(ptrType, sp), mkStack(intType, sp + 8), mkStack(intType, sp + 16)) - case reflect.String : return sp + 16, append(p, mkStack(ptrType, sp), mkStack(intType, sp + 8)) - case reflect.Struct : panic("abi: go116: not implemented: structs") - default : panic("abi: invalid value type") - } + switch vt.Kind() { + case reflect.Bool: + return sp + 8, append(p, mkStack(reflect.TypeOf(false), sp)) + case reflect.Int: + return sp + 8, append(p, mkStack(intType, sp)) + case reflect.Int8: + return sp + 8, append(p, mkStack(reflect.TypeOf(int8(0)), sp)) + case reflect.Int16: + return sp + 8, append(p, mkStack(reflect.TypeOf(int16(0)), sp)) + case reflect.Int32: + return sp + 8, append(p, mkStack(reflect.TypeOf(int32(0)), sp)) + case reflect.Int64: + return sp + 8, append(p, mkStack(reflect.TypeOf(int64(0)), sp)) + case reflect.Uint: + return sp + 8, append(p, mkStack(reflect.TypeOf(uint(0)), sp)) + case reflect.Uint8: + return sp + 8, append(p, mkStack(reflect.TypeOf(uint8(0)), sp)) + case reflect.Uint16: + return sp + 8, append(p, mkStack(reflect.TypeOf(uint16(0)), sp)) + case reflect.Uint32: + return sp + 8, append(p, mkStack(reflect.TypeOf(uint32(0)), sp)) + case reflect.Uint64: + return sp + 8, append(p, mkStack(reflect.TypeOf(uint64(0)), sp)) + case reflect.Uintptr: + return sp + 8, append(p, mkStack(reflect.TypeOf(uintptr(0)), sp)) + case reflect.Float32: + return sp + 8, append(p, mkStack(reflect.TypeOf(float32(0)), sp)) + case reflect.Float64: + return sp + 8, append(p, mkStack(reflect.TypeOf(float64(0)), sp)) + case reflect.Complex64: + panic("abi: go116: not implemented: complex64") + case reflect.Complex128: + panic("abi: go116: not implemented: complex128") + case reflect.Array: + panic("abi: go116: not implemented: arrays") + case reflect.Chan: + return sp + 8, append(p, mkStack(reflect.TypeOf((chan int)(nil)), sp)) + case reflect.Func: + return sp + 8, append(p, mkStack(reflect.TypeOf((func())(nil)), sp)) + case reflect.Map: + return sp + 8, append(p, mkStack(reflect.TypeOf((map[int]int)(nil)), sp)) + case reflect.Ptr: + return sp + 8, append(p, mkStack(reflect.TypeOf((*int)(nil)), sp)) + case reflect.UnsafePointer: + return sp + 8, append(p, mkStack(ptrType, sp)) + case reflect.Interface: + return sp + 16, append(p, mkStack(ptrType, sp), mkStack(ptrType, sp+8)) + case reflect.Slice: + return sp + 24, append(p, mkStack(ptrType, sp), mkStack(intType, sp+8), mkStack(intType, sp+16)) + case reflect.String: + return sp + 16, append(p, mkStack(ptrType, sp), mkStack(intType, sp+8)) + case reflect.Struct: + panic("abi: go116: not implemented: structs") + default: + panic("abi: invalid value type") + } } func NewFunctionLayout(ft reflect.Type) FunctionLayout { - var sp uint32 - var fn FunctionLayout - - /* assign every arguments */ - for i := 0; i < ft.NumIn(); i++ { - sp, fn.Args = salloc(fn.Args, sp, ft.In(i)) - } - - /* assign every return value */ - for i := 0; i < ft.NumOut(); i++ { - sp, fn.Rets = salloc(fn.Rets, sp, ft.Out(i)) - } - - /* update function ID and stack pointer */ - fn.FP = sp - return fn + var sp uint32 + var fn FunctionLayout + + /* assign every arguments */ + for i := 0; i < ft.NumIn(); i++ { + sp, fn.Args = salloc(fn.Args, sp, ft.In(i)) + } + + /* assign every return value */ + for i := 0; i < ft.NumOut(); i++ { + sp, fn.Rets = salloc(fn.Rets, sp, ft.Out(i)) + } + + /* update function ID and stack pointer */ + fn.FP = sp + return fn } func (self *Frame) emitExchangeArgs(p *Program) { - iregArgs, xregArgs := 0, 0 - for _, v := range self.desc.Args { - if v.IsFloat != notFloatKind { - xregArgs += 1 - } else { - iregArgs += 1 - } - } - - if iregArgs > len(iregOrderC) { - panic("too many arguments, only support at most 6 integer arguments now") - } - if xregArgs > len(xregOrderC) { - panic("too many arguments, only support at most 8 float arguments now") - } - - ic, xc := iregArgs, xregArgs - for i := 0; i < len(self.desc.Args); i++ { - arg := self.desc.Args[i] - if arg.IsFloat == floatKind64 { - p.MOVSD(self.argv(i), xregOrderC[xregArgs - xc]) - xc -= 1 - } else if arg.IsFloat == floatKind32 { - p.MOVSS(self.argv(i), xregOrderC[xregArgs - xc]) - xc -= 1 - } else { - p.MOVQ(self.argv(i), iregOrderC[iregArgs - ic]) - ic -= 1 - } - } + iregArgs, xregArgs := 0, 0 + for _, v := range self.desc.Args { + if v.IsFloat != notFloatKind { + xregArgs += 1 + } else { + iregArgs += 1 + } + } + + if iregArgs > len(iregOrderC) { + panic("too many arguments, only support at most 6 integer arguments now") + } + if xregArgs > len(xregOrderC) { + panic("too many arguments, only support at most 8 float arguments now") + } + + ic, xc := iregArgs, xregArgs + for i := 0; i < len(self.desc.Args); i++ { + arg := self.desc.Args[i] + if arg.IsFloat == floatKind64 { + p.MOVSD(self.argv(i), xregOrderC[xregArgs-xc]) + xc -= 1 + } else if arg.IsFloat == floatKind32 { + p.MOVSS(self.argv(i), xregOrderC[xregArgs-xc]) + xc -= 1 + } else { + p.MOVQ(self.argv(i), iregOrderC[iregArgs-ic]) + ic -= 1 + } + } } func (self *Frame) emitStackCheck(p *Program, to *Label, maxStack uintptr) { - // get the current goroutine - switch runtime.GOOS { - case "linux" : p.MOVQ(Abs(-8), R14).FS() - case "darwin" : p.MOVQ(Abs(0x30), R14).GS() - case "windows": break // windows always stores G pointer at R14 - default : panic("unsupported operating system") - } - - // check the stack guard - p.LEAQ(Ptr(RSP, -int32(self.Size() + uint32(maxStack))), RAX) - p.CMPQ(Ptr(R14, _G_stackguard0), RAX) - p.JBE(to) + // get the current goroutine + switch runtime.GOOS { + case "linux": + p.MOVQ(Abs(-8), R14).FS() + case "darwin": + p.MOVQ(Abs(0x30), R14).GS() + case "windows": + break // windows always stores G pointer at R14 + default: + panic("unsupported operating system") + } + + // check the stack guard + p.LEAQ(Ptr(RSP, -int32(self.Size()+uint32(maxStack))), RAX) + p.CMPQ(Ptr(R14, _G_stackguard0), RAX) + p.JBE(to) } func (self *Frame) StackCheckTextSize() uint32 { - p := DefaultArch.CreateProgram() - - // get the current goroutine - switch runtime.GOOS { - case "linux" : p.MOVQ(Abs(-8), R14).FS() - case "darwin" : p.MOVQ(Abs(0x30), R14).GS() - case "windows": break // windows always stores G pointer at R14 - default : panic("unsupported operating system") - } - - // check the stack guard - p.LEAQ(Ptr(RSP, -int32(self.Size())), RAX) - p.CMPQ(Ptr(R14, _G_stackguard0), RAX) - l := CreateLabel("") - p.Link(l) - p.JBE(l) - - return uint32(len(p.Assemble(0))) + p := DefaultArch.CreateProgram() + + // get the current goroutine + switch runtime.GOOS { + case "linux": + p.MOVQ(Abs(-8), R14).FS() + case "darwin": + p.MOVQ(Abs(0x30), R14).GS() + case "windows": + break // windows always stores G pointer at R14 + default: + panic("unsupported operating system") + } + + // check the stack guard + p.LEAQ(Ptr(RSP, -int32(self.Size())), RAX) + p.CMPQ(Ptr(R14, _G_stackguard0), RAX) + l := CreateLabel("") + p.Link(l) + p.JBE(l) + + return uint32(len(p.Assemble(0))) } func (self *Frame) emitExchangeRets(p *Program) { - if len(self.desc.Rets) > 1 { - panic("too many results, only support one result now") - } - // store result - if len(self.desc.Rets) ==1 { - if self.desc.Rets[0].IsFloat == floatKind64 { - p.MOVSD(xregOrderC[0], self.retv(0)) - } else if self.desc.Rets[0].IsFloat == floatKind32 { - p.MOVSS(xregOrderC[0], self.retv(0)) - } else { - p.MOVQ(RAX, self.retv(0)) - } - } + if len(self.desc.Rets) > 1 { + panic("too many results, only support one result now") + } + // store result + if len(self.desc.Rets) == 1 { + if self.desc.Rets[0].IsFloat == floatKind64 { + p.MOVSD(xregOrderC[0], self.retv(0)) + } else if self.desc.Rets[0].IsFloat == floatKind32 { + p.MOVSS(xregOrderC[0], self.retv(0)) + } else { + p.MOVQ(RAX, self.retv(0)) + } + } } func (self *Frame) emitRestoreRegs(p *Program) { - // load reserved registers - for i, r := range ReservedRegs(self.ccall) { - switch r.(type) { - case Register64: - p.MOVQ(self.resv(i), r) - case XMMRegister: - p.MOVSD(self.resv(i), r) - default: - panic(fmt.Sprintf("unsupported register type %t to reserve", r)) - } - } -} \ No newline at end of file + // load reserved registers + for i, r := range ReservedRegs(self.ccall) { + switch r.(type) { + case Register64: + p.MOVQ(self.resv(i), r) + case XMMRegister: + p.MOVSD(self.resv(i), r) + default: + panic(fmt.Sprintf("unsupported register type %t to reserve", r)) + } + } +} diff --git a/loader/internal/abi/abi_regabi_amd64.go b/loader/internal/abi/abi_regabi_amd64.go index 5a31dea89..d4c940de3 100644 --- a/loader/internal/abi/abi_regabi_amd64.go +++ b/loader/internal/abi/abi_regabi_amd64.go @@ -26,10 +26,10 @@ package abi import ( - `fmt` - `reflect` + "fmt" + "reflect" - . `github.com/cloudwego/iasm/x86_64` + x64 "github.com/bytedance/sonic/loader/internal/iasm/x86_64" ) /** Frame Structure of the Generated Function @@ -59,258 +59,287 @@ offs() -------------------------------| RSP -------------------------------|↓ lower addresses */ -const zeroRegGo = XMM15 - -var iregOrderGo = [...]Register64 { - RAX,// RDI - RBX,// RSI - RCX,// RDX - RDI,// RCX - RSI,// R8 - R8, // R9 - R9, - R10, - R11, +const zeroRegGo = x64.XMM15 + +var iregOrderGo = [...]Register64{ + x64.RAX, // RDI + x64.RBX, // RSI + x64.RCX, // RDX + x64.RDI, // RCX + x64.RSI, // R8 + x64.R8, // R9 + x64.R9, + x64.R10, + x64.R11, } -var xregOrderGo = [...]XMMRegister { - XMM0, - XMM1, - XMM2, - XMM3, - XMM4, - XMM5, - XMM6, - XMM7, - XMM8, - XMM9, - XMM10, - XMM11, - XMM12, - XMM13, - XMM14, +var xregOrderGo = [...]XMMRegister{ + x64.XMM0, + x64.XMM1, + x64.XMM2, + x64.XMM3, + x64.XMM4, + x64.XMM5, + x64.XMM6, + x64.XMM7, + x64.XMM8, + x64.XMM9, + x64.XMM10, + x64.XMM11, + x64.XMM12, + x64.XMM13, + x64.XMM14, } func ReservedRegs(callc bool) []Register { - if callc { - return nil - } - return []Register { - R14, // current goroutine - R15, // GOT reference - } + if callc { + return nil + } + return []Register{ + R14, // current goroutine + R15, // GOT reference + } } type stackAlloc struct { - s uint32 - i int - x int + s uint32 + i int + x int } func (self *stackAlloc) reset() { - self.i, self.x = 0, 0 + self.i, self.x = 0, 0 } func (self *stackAlloc) ireg(vt reflect.Type) (p Parameter) { - p = mkIReg(vt, iregOrderGo[self.i]) - self.i++ - return + p = mkIReg(vt, iregOrderGo[self.i]) + self.i++ + return } func (self *stackAlloc) xreg(vt reflect.Type) (p Parameter) { - p = mkXReg(vt, xregOrderGo[self.x]) - self.x++ - return + p = mkXReg(vt, xregOrderGo[self.x]) + self.x++ + return } func (self *stackAlloc) stack(vt reflect.Type) (p Parameter) { - p = mkStack(vt, self.s) - self.s += uint32(vt.Size()) - return + p = mkStack(vt, self.s) + self.s += uint32(vt.Size()) + return } func (self *stackAlloc) spill(n uint32, a int) uint32 { - self.s = alignUp(self.s, a) + n - return self.s + self.s = alignUp(self.s, a) + n + return self.s } func (self *stackAlloc) alloc(p []Parameter, vt reflect.Type) []Parameter { - nb := vt.Size() - vk := vt.Kind() - - /* zero-sized objects are allocated on stack */ - if nb == 0 { - return append(p, mkStack(intType, self.s)) - } - - /* check for value type */ - switch vk { - case reflect.Bool : return self.valloc(p, reflect.TypeOf(false)) - case reflect.Int : return self.valloc(p, intType) - case reflect.Int8 : return self.valloc(p, reflect.TypeOf(int8(0))) - case reflect.Int16 : return self.valloc(p, reflect.TypeOf(int16(0))) - case reflect.Int32 : return self.valloc(p, reflect.TypeOf(uint32(0))) - case reflect.Int64 : return self.valloc(p, reflect.TypeOf(int64(0))) - case reflect.Uint : return self.valloc(p, reflect.TypeOf(uint(0))) - case reflect.Uint8 : return self.valloc(p, reflect.TypeOf(uint8(0))) - case reflect.Uint16 : return self.valloc(p, reflect.TypeOf(uint16(0))) - case reflect.Uint32 : return self.valloc(p, reflect.TypeOf(uint32(0))) - case reflect.Uint64 : return self.valloc(p, reflect.TypeOf(uint64(0))) - case reflect.Uintptr : return self.valloc(p, reflect.TypeOf(uintptr(0))) - case reflect.Float32 : return self.valloc(p, reflect.TypeOf(float32(0))) - case reflect.Float64 : return self.valloc(p, reflect.TypeOf(float64(0))) - case reflect.Complex64 : panic("abi: go117: not implemented: complex64") - case reflect.Complex128 : panic("abi: go117: not implemented: complex128") - case reflect.Array : panic("abi: go117: not implemented: arrays") - case reflect.Chan : return self.valloc(p, reflect.TypeOf((chan int)(nil))) - case reflect.Func : return self.valloc(p, reflect.TypeOf((func())(nil))) - case reflect.Map : return self.valloc(p, reflect.TypeOf((map[int]int)(nil))) - case reflect.Ptr : return self.valloc(p, reflect.TypeOf((*int)(nil))) - case reflect.UnsafePointer : return self.valloc(p, ptrType) - case reflect.Interface : return self.valloc(p, ptrType, ptrType) - case reflect.Slice : return self.valloc(p, ptrType, intType, intType) - case reflect.String : return self.valloc(p, ptrType, intType) - case reflect.Struct : panic("abi: go117: not implemented: structs") - default : panic("abi: invalid value type") - } + nb := vt.Size() + vk := vt.Kind() + + /* zero-sized objects are allocated on stack */ + if nb == 0 { + return append(p, mkStack(intType, self.s)) + } + + /* check for value type */ + switch vk { + case reflect.Bool: + return self.valloc(p, reflect.TypeOf(false)) + case reflect.Int: + return self.valloc(p, intType) + case reflect.Int8: + return self.valloc(p, reflect.TypeOf(int8(0))) + case reflect.Int16: + return self.valloc(p, reflect.TypeOf(int16(0))) + case reflect.Int32: + return self.valloc(p, reflect.TypeOf(uint32(0))) + case reflect.Int64: + return self.valloc(p, reflect.TypeOf(int64(0))) + case reflect.Uint: + return self.valloc(p, reflect.TypeOf(uint(0))) + case reflect.Uint8: + return self.valloc(p, reflect.TypeOf(uint8(0))) + case reflect.Uint16: + return self.valloc(p, reflect.TypeOf(uint16(0))) + case reflect.Uint32: + return self.valloc(p, reflect.TypeOf(uint32(0))) + case reflect.Uint64: + return self.valloc(p, reflect.TypeOf(uint64(0))) + case reflect.Uintptr: + return self.valloc(p, reflect.TypeOf(uintptr(0))) + case reflect.Float32: + return self.valloc(p, reflect.TypeOf(float32(0))) + case reflect.Float64: + return self.valloc(p, reflect.TypeOf(float64(0))) + case reflect.Complex64: + panic("abi: go117: not implemented: complex64") + case reflect.Complex128: + panic("abi: go117: not implemented: complex128") + case reflect.Array: + panic("abi: go117: not implemented: arrays") + case reflect.Chan: + return self.valloc(p, reflect.TypeOf((chan int)(nil))) + case reflect.Func: + return self.valloc(p, reflect.TypeOf((func())(nil))) + case reflect.Map: + return self.valloc(p, reflect.TypeOf((map[int]int)(nil))) + case reflect.Ptr: + return self.valloc(p, reflect.TypeOf((*int)(nil))) + case reflect.UnsafePointer: + return self.valloc(p, ptrType) + case reflect.Interface: + return self.valloc(p, ptrType, ptrType) + case reflect.Slice: + return self.valloc(p, ptrType, intType, intType) + case reflect.String: + return self.valloc(p, ptrType, intType) + case reflect.Struct: + panic("abi: go117: not implemented: structs") + default: + panic("abi: invalid value type") + } } func (self *stackAlloc) valloc(p []Parameter, vts ...reflect.Type) []Parameter { - for _, vt := range vts { - enum := isFloat(vt) - if enum != notFloatKind && self.x < len(xregOrderGo) { - p = append(p, self.xreg(vt)) - } else if enum == notFloatKind && self.i < len(iregOrderGo) { - p = append(p, self.ireg(vt)) - } else { - p = append(p, self.stack(vt)) - } - } - return p + for _, vt := range vts { + enum := isFloat(vt) + if enum != notFloatKind && self.x < len(xregOrderGo) { + p = append(p, self.xreg(vt)) + } else if enum == notFloatKind && self.i < len(iregOrderGo) { + p = append(p, self.ireg(vt)) + } else { + p = append(p, self.stack(vt)) + } + } + return p } func NewFunctionLayout(ft reflect.Type) FunctionLayout { - var sa stackAlloc - var fn FunctionLayout - - /* assign every arguments */ - for i := 0; i < ft.NumIn(); i++ { - fn.Args = sa.alloc(fn.Args, ft.In(i)) - } - - /* reset the register counter, and add a pointer alignment field */ - sa.reset() - - /* assign every return value */ - for i := 0; i < ft.NumOut(); i++ { - fn.Rets = sa.alloc(fn.Rets, ft.Out(i)) - } - - sa.spill(0, PtrAlign) - - /* assign spill slots */ - for i := 0; i < len(fn.Args); i++ { - if fn.Args[i].InRegister { - fn.Args[i].Mem = sa.spill(PtrSize, PtrAlign) - PtrSize - } - } - - /* add the final pointer alignment field */ - fn.FP = sa.spill(0, PtrAlign) - return fn + var sa stackAlloc + var fn FunctionLayout + + /* assign every arguments */ + for i := 0; i < ft.NumIn(); i++ { + fn.Args = sa.alloc(fn.Args, ft.In(i)) + } + + /* reset the register counter, and add a pointer alignment field */ + sa.reset() + + /* assign every return value */ + for i := 0; i < ft.NumOut(); i++ { + fn.Rets = sa.alloc(fn.Rets, ft.Out(i)) + } + + sa.spill(0, PtrAlign) + + /* assign spill slots */ + for i := 0; i < len(fn.Args); i++ { + if fn.Args[i].InRegister { + fn.Args[i].Mem = sa.spill(PtrSize, PtrAlign) - PtrSize + } + } + + /* add the final pointer alignment field */ + fn.FP = sa.spill(0, PtrAlign) + return fn } func (self *Frame) emitExchangeArgs(p *Program) { - iregArgs := make([]Parameter, 0, len(self.desc.Args)) - xregArgs := 0 - for _, v := range self.desc.Args { - if v.InRegister { - if v.IsFloat != notFloatKind { - xregArgs += 1 - } else { - iregArgs = append(iregArgs, v) - } - } else { - panic("not support stack-assgined arguments now") - } - } - if xregArgs > len(xregOrderC) { - panic("too many arguments, only support at most 8 integer register arguments now") - } - - switch len(iregArgs) { - case 0, 1, 2, 3: { - //Fast-Path: when arguments count are less than four, just exchange the registers - for i := 0; i < len(iregArgs); i++ { - p.MOVQ(iregOrderGo[i], iregOrderC[i]) - } - } - case 4, 5, 6: { - // need to spill 3th ~ regArgs registers before exchange - for i := 3; i < len(iregArgs); i++ { - arg := iregArgs[i] - // pointer args have already been spilled - if !arg.IsPointer { - p.MOVQ(iregOrderGo[i], Ptr(RSP, int32(self.Prev() + arg.Mem))) - } - } - p.MOVQ(iregOrderGo[0], iregOrderC[0]) - p.MOVQ(iregOrderGo[1], iregOrderC[1]) - p.MOVQ(iregOrderGo[2], iregOrderC[2]) - for i := 3; i < len(iregArgs); i++ { - arg := iregArgs[i] - p.MOVQ(Ptr(RSP, int32(self.Prev() + arg.Mem)), iregOrderC[i]) - } - } - default: - panic("too many arguments, only support at most 6 integer register arguments now") - } + iregArgs := make([]Parameter, 0, len(self.desc.Args)) + xregArgs := 0 + for _, v := range self.desc.Args { + if v.InRegister { + if v.IsFloat != notFloatKind { + xregArgs += 1 + } else { + iregArgs = append(iregArgs, v) + } + } else { + panic("not support stack-assgined arguments now") + } + } + if xregArgs > len(xregOrderC) { + panic("too many arguments, only support at most 8 integer register arguments now") + } + + switch len(iregArgs) { + case 0, 1, 2, 3: + { + //Fast-Path: when arguments count are less than four, just exchange the registers + for i := 0; i < len(iregArgs); i++ { + p.MOVQ(iregOrderGo[i], iregOrderC[i]) + } + } + case 4, 5, 6: + { + // need to spill 3th ~ regArgs registers before exchange + for i := 3; i < len(iregArgs); i++ { + arg := iregArgs[i] + // pointer args have already been spilled + if !arg.IsPointer { + p.MOVQ(iregOrderGo[i], Ptr(RSP, int32(self.Prev()+arg.Mem))) + } + } + p.MOVQ(iregOrderGo[0], iregOrderC[0]) + p.MOVQ(iregOrderGo[1], iregOrderC[1]) + p.MOVQ(iregOrderGo[2], iregOrderC[2]) + for i := 3; i < len(iregArgs); i++ { + arg := iregArgs[i] + p.MOVQ(Ptr(RSP, int32(self.Prev()+arg.Mem)), iregOrderC[i]) + } + } + default: + panic("too many arguments, only support at most 6 integer register arguments now") + } } func (self *Frame) emitStackCheck(p *Program, to *Label, maxStack uintptr) { - p.LEAQ(Ptr(RSP, int32(-(self.Size() + uint32(maxStack)))), R12) - p.CMPQ(Ptr(R14, _G_stackguard0), R12) - p.JBE(to) + p.LEAQ(Ptr(RSP, int32(-(self.Size()+uint32(maxStack)))), R12) + p.CMPQ(Ptr(R14, _G_stackguard0), R12) + p.JBE(to) } func (self *Frame) StackCheckTextSize() uint32 { - p := DefaultArch.CreateProgram() - p.LEAQ(Ptr(RSP, int32(-(self.Size()))), R12) - p.CMPQ(Ptr(R14, _G_stackguard0), R12) - to := CreateLabel("") - p.Link(to) - p.JBE(to) - return uint32(len(p.Assemble(0))) + p := DefaultArch.CreateProgram() + p.LEAQ(Ptr(RSP, int32(-(self.Size()))), R12) + p.CMPQ(Ptr(R14, _G_stackguard0), R12) + to := CreateLabel("") + p.Link(to) + p.JBE(to) + return uint32(len(p.Assemble(0))) } func (self *Frame) emitExchangeRets(p *Program) { - if len(self.desc.Rets) > 1 { - panic("too many results, only support one result now") - } - // store result - if len(self.desc.Rets) == 1 && !self.desc.Rets[0].InRegister { - if self.desc.Rets[0].IsFloat == floatKind64 { - p.MOVSD(xregOrderC[0], self.retv(0)) - } else if self.desc.Rets[0].IsFloat == floatKind32 { - p.MOVSS(xregOrderC[0], self.retv(0)) - } else { - p.MOVQ(RAX, self.retv(0)) - } - } + if len(self.desc.Rets) > 1 { + panic("too many results, only support one result now") + } + // store result + if len(self.desc.Rets) == 1 && !self.desc.Rets[0].InRegister { + if self.desc.Rets[0].IsFloat == floatKind64 { + p.MOVSD(xregOrderC[0], self.retv(0)) + } else if self.desc.Rets[0].IsFloat == floatKind32 { + p.MOVSS(xregOrderC[0], self.retv(0)) + } else { + p.MOVQ(RAX, self.retv(0)) + } + } } func (self *Frame) emitRestoreRegs(p *Program) { - // load reserved registers - for i, r := range ReservedRegs(self.ccall) { - switch r.(type) { - case Register64: - p.MOVQ(self.resv(i), r) - case XMMRegister: - p.MOVSD(self.resv(i), r) - default: - panic(fmt.Sprintf("unsupported register type %t to reserve", r)) - } - } - // zero xmm15 for go abi - p.XORPS(zeroRegGo, zeroRegGo) -} \ No newline at end of file + // load reserved registers + for i, r := range ReservedRegs(self.ccall) { + switch r.(type) { + case Register64: + p.MOVQ(self.resv(i), r) + case XMMRegister: + p.MOVSD(self.resv(i), r) + default: + panic(fmt.Sprintf("unsupported register type %t to reserve", r)) + } + } + // zero xmm15 for go abi + p.XORPS(zeroRegGo, zeroRegGo) +} diff --git a/loader/internal/iasm/expr/ast.go b/loader/internal/iasm/expr/ast.go new file mode 100644 index 000000000..d340c5c55 --- /dev/null +++ b/loader/internal/iasm/expr/ast.go @@ -0,0 +1,273 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package expr + +import ( + "fmt" +) + +// Type is tyep expression type. +type Type int + +const ( + // CONST indicates that the expression is a constant. + CONST Type = iota + + // TERM indicates that the expression is a Term reference. + TERM + + // EXPR indicates that the expression is a unary or binary expression. + EXPR +) + +var typeNames = map[Type]string{ + EXPR: "Expr", + TERM: "Term", + CONST: "Const", +} + +// String returns the string representation of a Type. +func (self Type) String() string { + if v, ok := typeNames[self]; ok { + return v + } else { + return fmt.Sprintf("expr.Type(%d)", self) + } +} + +// Operator represents an operation to perform when Type is EXPR. +type Operator uint8 + +const ( + // ADD performs "Add Expr.Left and Expr.Right". + ADD Operator = iota + + // SUB performs "Subtract Expr.Left by Expr.Right". + SUB + + // MUL performs "Multiply Expr.Left by Expr.Right". + MUL + + // DIV performs "Divide Expr.Left by Expr.Right". + DIV + + // MOD performs "Modulo Expr.Left by Expr.Right". + MOD + + // AND performs "Bitwise AND Expr.Left and Expr.Right". + AND + + // OR performs "Bitwise OR Expr.Left and Expr.Right". + OR + + // XOR performs "Bitwise XOR Expr.Left and Expr.Right". + XOR + + // SHL performs "Bitwise Shift Expr.Left to the Left by Expr.Right Bits". + SHL + + // SHR performs "Bitwise Shift Expr.Left to the Right by Expr.Right Bits". + SHR + + // POW performs "Raise Expr.Left to the power of Expr.Right" + POW + + // NOT performs "Bitwise Invert Expr.Left". + NOT + + // NEG performs "Negate Expr.Left". + NEG +) + +var operatorNames = map[Operator]string{ + ADD: "Add", + SUB: "Subtract", + MUL: "Multiply", + DIV: "Divide", + MOD: "Modulo", + AND: "And", + OR: "Or", + XOR: "ExclusiveOr", + SHL: "ShiftLeft", + SHR: "ShiftRight", + POW: "Power", + NOT: "Invert", + NEG: "Negate", +} + +// String returns the string representation of a Type. +func (self Operator) String() string { + if v, ok := operatorNames[self]; ok { + return v + } else { + return fmt.Sprintf("expr.Operator(%d)", self) + } +} + +// Expr represents an expression node. +type Expr struct { + Type Type + Term Term + Op Operator + Left *Expr + Right *Expr + Const int64 +} + +// Ref creates an expression from a Term. +func Ref(t Term) (p *Expr) { + p = newExpression() + p.Term = t + p.Type = TERM + return +} + +// Int creates an expression from an integer. +func Int(v int64) (p *Expr) { + p = newExpression() + p.Type = CONST + p.Const = v + return +} + +func (self *Expr) clear() { + if self.Term != nil { + self.Term.Free() + } + if self.Left != nil { + self.Left.Free() + } + if self.Right != nil { + self.Right.Free() + } +} + +// Free returns the Expr into pool. +// Any operation performed after Free is undefined behavior. +func (self *Expr) Free() { + self.clear() + freeExpression(self) +} + +// Evaluate evaluates the expression into an integer. +// It also implements the Term interface. +func (self *Expr) Evaluate() (int64, error) { + switch self.Type { + case EXPR: + return self.eval() + case TERM: + return self.Term.Evaluate() + case CONST: + return self.Const, nil + default: + panic("invalid expression type: " + self.Type.String()) + } +} + +/** Expression Combinator **/ + +func combine(a *Expr, op Operator, b *Expr) (r *Expr) { + r = newExpression() + r.Op = op + r.Type = EXPR + r.Left = a + r.Right = b + return +} + +func (self *Expr) Add(v *Expr) *Expr { return combine(self, ADD, v) } +func (self *Expr) Sub(v *Expr) *Expr { return combine(self, SUB, v) } +func (self *Expr) Mul(v *Expr) *Expr { return combine(self, MUL, v) } +func (self *Expr) Div(v *Expr) *Expr { return combine(self, DIV, v) } +func (self *Expr) Mod(v *Expr) *Expr { return combine(self, MOD, v) } +func (self *Expr) And(v *Expr) *Expr { return combine(self, AND, v) } +func (self *Expr) Or(v *Expr) *Expr { return combine(self, OR, v) } +func (self *Expr) Xor(v *Expr) *Expr { return combine(self, XOR, v) } +func (self *Expr) Shl(v *Expr) *Expr { return combine(self, SHL, v) } +func (self *Expr) Shr(v *Expr) *Expr { return combine(self, SHR, v) } +func (self *Expr) Pow(v *Expr) *Expr { return combine(self, POW, v) } +func (self *Expr) Not() *Expr { return combine(self, NOT, nil) } +func (self *Expr) Neg() *Expr { return combine(self, NEG, nil) } + +/** Expression Evaluator **/ + +var binaryEvaluators = [256]func(int64, int64) (int64, error){ + ADD: func(a, b int64) (int64, error) { return a + b, nil }, + SUB: func(a, b int64) (int64, error) { return a - b, nil }, + MUL: func(a, b int64) (int64, error) { return a * b, nil }, + DIV: idiv, + MOD: imod, + AND: func(a, b int64) (int64, error) { return a & b, nil }, + OR: func(a, b int64) (int64, error) { return a | b, nil }, + XOR: func(a, b int64) (int64, error) { return a ^ b, nil }, + SHL: func(a, b int64) (int64, error) { return a << b, nil }, + SHR: func(a, b int64) (int64, error) { return a >> b, nil }, + POW: ipow, +} + +func (self *Expr) eval() (int64, error) { + var lhs int64 + var rhs int64 + var err error + var vfn func(int64, int64) (int64, error) + + /* evaluate LHS */ + if lhs, err = self.Left.Evaluate(); err != nil { + return 0, err + } + + /* check for unary operators */ + switch self.Op { + case NOT: + return self.unaryNot(lhs) + case NEG: + return self.unaryNeg(lhs) + } + + /* check for operators */ + if vfn = binaryEvaluators[self.Op]; vfn == nil { + panic("invalid operator: " + self.Op.String()) + } + + /* must be a binary expression */ + if self.Right == nil { + panic("operator " + self.Op.String() + " is a binary operator") + } + + /* evaluate RHS, and call the operator */ + if rhs, err = self.Right.Evaluate(); err != nil { + return 0, err + } else { + return vfn(lhs, rhs) + } +} + +func (self *Expr) unaryNot(v int64) (int64, error) { + if self.Right == nil { + return ^v, nil + } else { + panic("operator Invert is an unary operator") + } +} + +func (self *Expr) unaryNeg(v int64) (int64, error) { + if self.Right == nil { + return -v, nil + } else { + panic("operator Negate is an unary operator") + } +} diff --git a/loader/internal/iasm/expr/errors.go b/loader/internal/iasm/expr/errors.go new file mode 100644 index 000000000..791c18bd6 --- /dev/null +++ b/loader/internal/iasm/expr/errors.go @@ -0,0 +1,53 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package expr + +import ( + "fmt" +) + +// SyntaxError represents a syntax error in the expression. +type SyntaxError struct { + Pos int + Reason string +} + +func newSyntaxError(pos int, reason string) *SyntaxError { + return &SyntaxError{ + Pos: pos, + Reason: reason, + } +} + +func (self *SyntaxError) Error() string { + return fmt.Sprintf("Syntax error at position %d: %s", self.Pos, self.Reason) +} + +// RuntimeError is an error which would occure at run time. +type RuntimeError struct { + Reason string +} + +func newRuntimeError(reason string) *RuntimeError { + return &RuntimeError{ + Reason: reason, + } +} + +func (self *RuntimeError) Error() string { + return "Runtime error: " + self.Reason +} diff --git a/loader/internal/iasm/expr/ops.go b/loader/internal/iasm/expr/ops.go new file mode 100644 index 000000000..8502dbc9d --- /dev/null +++ b/loader/internal/iasm/expr/ops.go @@ -0,0 +1,67 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package expr + +import ( + "fmt" +) + +func idiv(v int64, d int64) (int64, error) { + if d != 0 { + return v / d, nil + } else { + return 0, newRuntimeError("division by zero") + } +} + +func imod(v int64, d int64) (int64, error) { + if d != 0 { + return v % d, nil + } else { + return 0, newRuntimeError("division by zero") + } +} + +func ipow(v int64, e int64) (int64, error) { + mul := v + ret := int64(1) + + /* value must be 0 or positive */ + if v < 0 { + return 0, newRuntimeError(fmt.Sprintf("negative base value: %d", v)) + } + + /* exponent must be non-negative */ + if e < 0 { + return 0, newRuntimeError(fmt.Sprintf("negative exponent: %d", e)) + } + + /* fast power first round */ + if (e & 1) != 0 { + ret *= mul + } + + /* fast power remaining rounds */ + for e >>= 1; e != 0; e >>= 1 { + if mul *= mul; (e & 1) != 0 { + ret *= mul + } + } + + /* all done */ + return ret, nil +} diff --git a/loader/internal/iasm/expr/parser.go b/loader/internal/iasm/expr/parser.go new file mode 100644 index 000000000..98d5db68c --- /dev/null +++ b/loader/internal/iasm/expr/parser.go @@ -0,0 +1,331 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package expr + +import ( + "strconv" + "unicode" + "unsafe" +) + +type _TokenKind uint8 + +const ( + _T_end _TokenKind = iota + 1 + _T_int + _T_punc + _T_name +) + +const ( + _OP2 = 0x80 + _POW = _OP2 | '*' + _SHL = _OP2 | '<' + _SHR = _OP2 | '>' +) + +type _Slice struct { + p unsafe.Pointer + n int + c int +} + +type _Token struct { + pos int + ptr *rune + u64 uint64 + tag _TokenKind +} + +func (self _Token) str() (v string) { + return string(self.rbuf()) +} + +func (self _Token) rbuf() (v []rune) { + (*_Slice)(unsafe.Pointer(&v)).c = int(self.u64) + (*_Slice)(unsafe.Pointer(&v)).n = int(self.u64) + (*_Slice)(unsafe.Pointer(&v)).p = unsafe.Pointer(self.ptr) + return +} + +func tokenEnd(p int) _Token { + return _Token{ + pos: p, + tag: _T_end, + } +} + +func tokenInt(p int, v uint64) _Token { + return _Token{ + pos: p, + u64: v, + tag: _T_int, + } +} + +func tokenPunc(p int, v rune) _Token { + return _Token{ + pos: p, + tag: _T_punc, + u64: uint64(v), + } +} + +func tokenName(p int, v []rune) _Token { + return _Token{ + pos: p, + ptr: &v[0], + tag: _T_name, + u64: uint64(len(v)), + } +} + +// Repository represents a repository of Term's. +type Repository interface { + Get(name string) (Term, error) +} + +// Parser parses an expression string to it's AST representation. +type Parser struct { + pos int + src []rune +} + +var binaryOps = [...]func(*Expr, *Expr) *Expr{ + '+': (*Expr).Add, + '-': (*Expr).Sub, + '*': (*Expr).Mul, + '/': (*Expr).Div, + '%': (*Expr).Mod, + '&': (*Expr).And, + '^': (*Expr).Xor, + '|': (*Expr).Or, + _SHL: (*Expr).Shl, + _SHR: (*Expr).Shr, + _POW: (*Expr).Pow, +} + +var precedence = [...]map[int]bool{ + {_SHL: true, _SHR: true}, + {'|': true}, + {'^': true}, + {'&': true}, + {'+': true, '-': true}, + {'*': true, '/': true, '%': true}, + {_POW: true}, +} + +func (self *Parser) ch() rune { + return self.src[self.pos] +} + +func (self *Parser) eof() bool { + return self.pos >= len(self.src) +} + +func (self *Parser) rch() (v rune) { + v, self.pos = self.src[self.pos], self.pos+1 + return +} + +func (self *Parser) hex(ss []rune) bool { + if len(ss) == 1 && ss[0] == '0' { + return unicode.ToLower(self.ch()) == 'x' + } else if len(ss) <= 1 || unicode.ToLower(ss[1]) != 'x' { + return unicode.IsDigit(self.ch()) + } else { + return ishexdigit(self.ch()) + } +} + +func (self *Parser) int(p int, ss []rune) (_Token, error) { + var err error + var val uint64 + + /* find all the digits */ + for !self.eof() && self.hex(ss) { + ss = append(ss, self.rch()) + } + + /* parse the value */ + if val, err = strconv.ParseUint(string(ss), 0, 64); err != nil { + return _Token{}, err + } else { + return tokenInt(p, val), nil + } +} + +func (self *Parser) name(p int, ss []rune) _Token { + for !self.eof() && isident(self.ch()) { + ss = append(ss, self.rch()) + } + return tokenName(p, ss) +} + +func (self *Parser) read(p int, ch rune) (_Token, error) { + if isdigit(ch) { + return self.int(p, []rune{ch}) + } else if isident0(ch) { + return self.name(p, []rune{ch}), nil + } else if isop2ch(ch) && !self.eof() && self.ch() == ch { + return tokenPunc(p, _OP2|self.rch()), nil + } else if isop1ch(ch) { + return tokenPunc(p, ch), nil + } else { + return _Token{}, newSyntaxError(self.pos, "invalid character "+strconv.QuoteRuneToASCII(ch)) + } +} + +func (self *Parser) next() (_Token, error) { + for { + var p int + var c rune + + /* check for EOF */ + if self.eof() { + return tokenEnd(self.pos), nil + } + + /* read the next char */ + p = self.pos + c = self.rch() + + /* parse the token if not a space */ + if !unicode.IsSpace(c) { + return self.read(p, c) + } + } +} + +func (self *Parser) grab(tk _Token, repo Repository) (*Expr, error) { + if repo == nil { + return nil, newSyntaxError(tk.pos, "unresolved symbol: "+tk.str()) + } else if term, err := repo.Get(tk.str()); err != nil { + return nil, err + } else { + return Ref(term), nil + } +} + +func (self *Parser) nest(nest int, repo Repository) (*Expr, error) { + var err error + var ret *Expr + var ntk _Token + + /* evaluate the nested expression */ + if ret, err = self.expr(0, nest+1, repo); err != nil { + return nil, err + } + + /* must follows with a ')' */ + if ntk, err = self.next(); err != nil { + return nil, err + } else if ntk.tag != _T_punc || ntk.u64 != ')' { + return nil, newSyntaxError(ntk.pos, "')' expected") + } else { + return ret, nil + } +} + +func (self *Parser) unit(nest int, repo Repository) (*Expr, error) { + if tk, err := self.next(); err != nil { + return nil, err + } else if tk.tag == _T_int { + return Int(int64(tk.u64)), nil + } else if tk.tag == _T_name { + return self.grab(tk, repo) + } else if tk.tag == _T_punc && tk.u64 == '(' { + return self.nest(nest, repo) + } else if tk.tag == _T_punc && tk.u64 == '+' { + return self.unit(nest, repo) + } else if tk.tag == _T_punc && tk.u64 == '-' { + return neg2(self.unit(nest, repo)) + } else if tk.tag == _T_punc && tk.u64 == '~' { + return not2(self.unit(nest, repo)) + } else { + return nil, newSyntaxError(tk.pos, "integer, unary operator or nested expression expected") + } +} + +func (self *Parser) term(prec int, nest int, repo Repository) (*Expr, error) { + var err error + var val *Expr + + /* parse the LHS operand */ + if val, err = self.expr(prec+1, nest, repo); err != nil { + return nil, err + } + + /* parse all the operators of the same precedence */ + for { + var op int + var rv *Expr + var tk _Token + + /* peek the next token */ + pp := self.pos + tk, err = self.next() + + /* check for errors */ + if err != nil { + return nil, err + } + + /* encountered EOF */ + if tk.tag == _T_end { + return val, nil + } + + /* must be an operator */ + if tk.tag != _T_punc { + return nil, newSyntaxError(tk.pos, "operators expected") + } + + /* check for the operator precedence */ + if op = int(tk.u64); !precedence[prec][op] { + self.pos = pp + return val, nil + } + + /* evaluate the RHS operand, and combine the value */ + if rv, err = self.expr(prec+1, nest, repo); err != nil { + return nil, err + } else { + val = binaryOps[op](val, rv) + } + } +} + +func (self *Parser) expr(prec int, nest int, repo Repository) (*Expr, error) { + if prec >= len(precedence) { + return self.unit(nest, repo) + } else { + return self.term(prec, nest, repo) + } +} + +// Parse parses the expression, and returns it's AST tree. +func (self *Parser) Parse(repo Repository) (*Expr, error) { + return self.expr(0, 0, repo) +} + +// SetSource resets the expression parser and sets the expression source. +func (self *Parser) SetSource(src string) *Parser { + self.pos = 0 + self.src = []rune(src) + return self +} diff --git a/loader/internal/iasm/expr/parser_test.go b/loader/internal/iasm/expr/parser_test.go new file mode 100644 index 000000000..ecb893904 --- /dev/null +++ b/loader/internal/iasm/expr/parser_test.go @@ -0,0 +1,33 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package expr + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParser_Eval(t *testing.T) { + p := new(Parser).SetSource(`3 - 2 * (5 + 6) ** 4 / 7 + (1 << (1234 % 23)) & 0x5436 ^ 0x5a5a - 2 | 1`) + v, err := p.Parse(nil) + require.NoError(t, err) + r, err := v.Evaluate() + require.NoError(t, err) + assert.Equal(t, int64(7805), r) +} diff --git a/loader/internal/iasm/expr/pools.go b/loader/internal/iasm/expr/pools.go new file mode 100644 index 000000000..77410b4be --- /dev/null +++ b/loader/internal/iasm/expr/pools.go @@ -0,0 +1,42 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package expr + +import ( + "sync" +) + +var ( + expressionPool sync.Pool +) + +func newExpression() *Expr { + if v := expressionPool.Get(); v == nil { + return new(Expr) + } else { + return resetExpression(v.(*Expr)) + } +} + +func freeExpression(p *Expr) { + expressionPool.Put(p) +} + +func resetExpression(p *Expr) *Expr { + *p = Expr{} + return p +} diff --git a/loader/internal/iasm/expr/term.go b/loader/internal/iasm/expr/term.go new file mode 100644 index 000000000..4fe723989 --- /dev/null +++ b/loader/internal/iasm/expr/term.go @@ -0,0 +1,23 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package expr + +// Term represents a value that can Evaluate() into an integer. +type Term interface { + Free() + Evaluate() (int64, error) +} diff --git a/loader/internal/iasm/expr/utils.go b/loader/internal/iasm/expr/utils.go new file mode 100644 index 000000000..57b4fd842 --- /dev/null +++ b/loader/internal/iasm/expr/utils.go @@ -0,0 +1,77 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package expr + +var op1ch = [...]bool{ + '+': true, + '-': true, + '*': true, + '/': true, + '%': true, + '&': true, + '|': true, + '^': true, + '~': true, + '(': true, + ')': true, +} + +var op2ch = [...]bool{ + '*': true, + '<': true, + '>': true, +} + +func neg2(v *Expr, err error) (*Expr, error) { + if err != nil { + return nil, err + } else { + return v.Neg(), nil + } +} + +func not2(v *Expr, err error) (*Expr, error) { + if err != nil { + return nil, err + } else { + return v.Not(), nil + } +} + +func isop1ch(ch rune) bool { + return ch >= 0 && int(ch) < len(op1ch) && op1ch[ch] +} + +func isop2ch(ch rune) bool { + return ch >= 0 && int(ch) < len(op2ch) && op2ch[ch] +} + +func isdigit(ch rune) bool { + return ch >= '0' && ch <= '9' +} + +func isident(ch rune) bool { + return isdigit(ch) || isident0(ch) +} + +func isident0(ch rune) bool { + return (ch == '_') || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') +} + +func ishexdigit(ch rune) bool { + return isdigit(ch) || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F') +} diff --git a/loader/internal/iasm/obj/macho.go b/loader/internal/iasm/obj/macho.go new file mode 100644 index 000000000..10cbaf9dd --- /dev/null +++ b/loader/internal/iasm/obj/macho.go @@ -0,0 +1,252 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package obj + +import ( + "encoding/binary" + "io" + "unsafe" +) + +type MachOHeader struct { + Magic uint32 + CPUType uint32 + CPUSubType uint32 + FileType uint32 + CmdCount uint32 + CmdSize uint32 + Flags uint32 + _ uint32 +} + +type SegmentCommand struct { + Cmd uint32 + Size uint32 + Name [16]byte + VMAddr uint64 + VMSize uint64 + FileOffset uint64 + FileSize uint64 + MaxProtect uint32 + InitProtect uint32 + SectionCount uint32 + Flags uint32 +} + +type SegmentSection struct { + Name [16]byte + SegName [16]byte + Addr uint64 + Size uint64 + Offset uint32 + Align uint32 + RelOffset uint32 + RelCount uint32 + Flags uint32 + _ [3]uint32 +} + +type Registers struct { + RAX uint64 + RBX uint64 + RCX uint64 + RDX uint64 + RDI uint64 + RSI uint64 + RBP uint64 + RSP uint64 + R8 uint64 + R9 uint64 + R10 uint64 + R11 uint64 + R12 uint64 + R13 uint64 + R14 uint64 + R15 uint64 + RIP uint64 + RFLAGS uint64 + CS uint64 + FS uint64 + GS uint64 +} + +type UnixThreadCommand struct { + Cmd uint32 + Size uint32 + Flavor uint32 + Count uint32 + Regs Registers +} + +const ( + _MH_MAGIC_64 = 0xfeedfacf + _MH_EXECUTE = 0x02 + _MH_NOUNDEFS = 0x01 +) + +const ( + _CPU_TYPE_I386 = 0x00000007 + _CPU_ARCH_ABI64 = 0x01000000 +) + +const ( + _CPU_SUBTYPE_LIB64 = 0x80000000 + _CPU_SUBTYPE_I386_ALL = 0x00000003 +) + +const ( + _LC_SEGMENT_64 = 0x19 + _LC_UNIXTHREAD = 0x05 +) + +const ( + _VM_PROT_READ = 0x01 + _VM_PROT_WRITE = 0x02 + _VM_PROT_EXECUTE = 0x04 +) + +const ( + _S_ATTR_SOME_INSTRUCTIONS = 0x00000400 + _S_ATTR_PURE_INSTRUCTIONS = 0x80000000 +) + +const ( + _x86_THREAD_STATE64 = 0x04 + _x86_EXCEPTION_STATE64_COUNT = 42 +) + +const ( + _MACHO_SIZE = uint32(unsafe.Sizeof(MachOHeader{})) + _SEGMENT_SIZE = uint32(unsafe.Sizeof(SegmentCommand{})) + _SECTION_SIZE = uint32(unsafe.Sizeof(SegmentSection{})) + _UNIXTHREAD_SIZE = uint32(unsafe.Sizeof(UnixThreadCommand{})) +) + +const ( + _IMAGE_SIZE = 4096 + _IMAGE_BASE = 0x04000000 +) + +const ( + _HDR_SIZE = _MACHO_SIZE + _SEGMENT_SIZE*2 + _SECTION_SIZE + _UNIXTHREAD_SIZE + _ZERO_SIZE = (_IMAGE_SIZE - _HDR_SIZE%_IMAGE_SIZE) % _IMAGE_SIZE +) + +var ( + zeroBytes = [_ZERO_SIZE]byte{} +) + +func assembleMachO(w io.Writer, code []byte, base uint64, entry uint64) error { + var p0name [16]byte + var txname [16]byte + var tsname [16]byte + + /* segment names */ + copy(tsname[:], "__text") + copy(txname[:], "__TEXT") + copy(p0name[:], "__PAGEZERO") + + /* calculate size of code */ + clen := uint64(len(code)) + hlen := uint64(_HDR_SIZE + _ZERO_SIZE) + + /* Mach-O does not allow image base at zero */ + if base == 0 { + base = _IMAGE_BASE + entry += _IMAGE_BASE + } + + /* Page-0 Segment */ + p0 := SegmentCommand{ + Cmd: _LC_SEGMENT_64, + Size: _SEGMENT_SIZE, + Name: p0name, + VMSize: base, + } + + /* TEXT Segment */ + text := SegmentCommand{ + Cmd: _LC_SEGMENT_64, + Size: _SEGMENT_SIZE + _SECTION_SIZE, + Name: txname, + VMAddr: base, + VMSize: hlen + clen, + FileSize: hlen + clen, + MaxProtect: _VM_PROT_READ | _VM_PROT_WRITE | _VM_PROT_EXECUTE, + InitProtect: _VM_PROT_READ | _VM_PROT_EXECUTE, + SectionCount: 1, + } + + /* __TEXT.__text section */ + tsec := SegmentSection{ + Name: tsname, + SegName: txname, + Addr: base + hlen, + Size: clen, + Offset: uint32(hlen), + Flags: _S_ATTR_SOME_INSTRUCTIONS | _S_ATTR_PURE_INSTRUCTIONS, + } + + /* UNIX Thread Metadata */ + unix := UnixThreadCommand{ + Cmd: _LC_UNIXTHREAD, + Size: _UNIXTHREAD_SIZE, + Flavor: _x86_THREAD_STATE64, + Count: _x86_EXCEPTION_STATE64_COUNT, + Regs: Registers{RIP: hlen + entry}, + } + + /* Mach-O Header */ + macho := MachOHeader{ + Magic: _MH_MAGIC_64, + CPUType: _CPU_ARCH_ABI64 | _CPU_TYPE_I386, + CPUSubType: _CPU_SUBTYPE_LIB64 | _CPU_SUBTYPE_I386_ALL, + FileType: _MH_EXECUTE, + CmdCount: 3, + CmdSize: _SEGMENT_SIZE*2 + _SECTION_SIZE + _UNIXTHREAD_SIZE, + Flags: _MH_NOUNDEFS, + } + + /* write the headers */ + if err := binary.Write(w, binary.LittleEndian, &macho); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, &p0); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, &text); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, &tsec); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, &unix); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, &zeroBytes); err != nil { + return err + } + + /* write the code */ + if n, err := w.Write(code); err != nil { + return err + } else if n != len(code) { + return io.ErrShortWrite + } else { + return nil + } +} diff --git a/loader/internal/iasm/obj/macho_test.go b/loader/internal/iasm/obj/macho_test.go new file mode 100644 index 000000000..83bfb4721 --- /dev/null +++ b/loader/internal/iasm/obj/macho_test.go @@ -0,0 +1,59 @@ +//go:build darwin +// +build darwin + +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package obj + +import ( + "os" + "os/exec" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/stretchr/testify/require" +) + +func TestMachO_Create(t *testing.T) { + fp, err := os.CreateTemp("", "macho_out-") + require.NoError(t, err) + code := []byte{ + 0x48, 0xc7, 0xc7, 0x01, 0x00, 0x00, 0x00, // MOVQ $1, %rdi + 0x48, 0x8d, 0x35, 0x1b, 0x00, 0x00, 0x00, // LEAQ 0x1b(%rip), %rsi + 0x48, 0xc7, 0xc2, 0x0e, 0x00, 0x00, 0x00, // MOVQ $14, %rdx + 0x48, 0xc7, 0xc0, 0x04, 0x00, 0x00, 0x02, // MOVQ $0x02000004, %rax + 0x0f, 0x05, // SYSCALL + 0x31, 0xff, // XORL %edi, %edi + 0x48, 0xc7, 0xc0, 0x01, 0x00, 0x00, 0x02, // MOVQ $0x02000001, %rax + 0x0f, 0x05, // SYSCALL + 'h', 'e', 'l', 'l', 'o', ',', ' ', + 'w', 'o', 'r', 'l', 'd', '\r', '\n', + } + err = assembleMachO(fp, code, 0, 0) + require.NoError(t, err) + err = fp.Close() + require.NoError(t, err) + err = os.Chmod(fp.Name(), 0755) + require.NoError(t, err) + println("Saved to", fp.Name()) + out, err := exec.Command(fp.Name()).Output() + require.NoError(t, err) + spew.Dump(out) + require.Equal(t, []byte("hello, world\r\n"), out) + err = os.Remove(fp.Name()) + require.NoError(t, err) +} diff --git a/loader/internal/iasm/obj/obj.go b/loader/internal/iasm/obj/obj.go new file mode 100644 index 000000000..3b891925f --- /dev/null +++ b/loader/internal/iasm/obj/obj.go @@ -0,0 +1,85 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package obj + +import ( + "fmt" + "io" + "os" +) + +// Format represents the saved binary file format. +type Format int + +const ( + // ELF indicates it should be saved as an ELF binary. + ELF Format = iota + + // MachO indicates it should be saved as a Mach-O binary. + MachO +) + +var formatTab = [...]func(w io.Writer, code []byte, base uint64, entry uint64) error{ + ELF: nil, + MachO: assembleMachO, +} + +var formatNames = map[Format]string{ + ELF: "ELF", + MachO: "Mach-O", +} + +// String returns the name of a specified format. +func (self Format) String() string { + if v, ok := formatNames[self]; ok { + return v + } else { + return fmt.Sprintf("Format(%d)", self) + } +} + +// Write assembles a binary executable. +func (self Format) Write(w io.Writer, code []byte, base uint64, entry uint64) error { + if self >= 0 && int(self) < len(formatTab) && formatTab[self] != nil { + return formatTab[self](w, code, base, entry) + } else { + return fmt.Errorf("unsupported format: %s", self) + } +} + +// Generate generates a binary executable file from the specified code. +func (self Format) Generate(name string, code []byte, base uint64, entry uint64) error { + var fp *os.File + var err error + + /* create the output file */ + if fp, err = os.Create(name); err != nil { + return err + } + + /* generate the code */ + if err = self.Write(fp, code, base, entry); err != nil { + _ = fp.Close() + _ = os.Remove(name) + return err + } + + /* close the file and make it executable */ + _ = fp.Close() + _ = os.Chmod(name, 0755) + return nil +} diff --git a/loader/internal/iasm/sync.sh b/loader/internal/iasm/sync.sh new file mode 100755 index 000000000..9423a7585 --- /dev/null +++ b/loader/internal/iasm/sync.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# Copyright 2025 ByteDance Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This script is used to synchronize the code of cloudwego/iasm, +# and trim it for minimizing the size of the compiled binary file. + +set -e + +echo "Cloning cloudwego/iasm into 'tmp'..." + +IASM_TAG="v0.2.0" +rm -rf tmp +if git clone --branch $IASM_TAG --depth 1 git@github.com:cloudwego/iasm.git tmp >/dev/null 2>&1; then + echo "done" +else + echo "git clone failed" + exit 1 +fi + +rm -rf expr +cp -r tmp/expr . + +rm -rf obj +cp -r tmp/obj . + +rm -rf x86_64 +cp -r tmp/x86_64 . + +rm -rf tmp + +# rm unused code +rm ./x86_64/assembler* + +# replace import +sed -i.bak 's:github.com/cloudwego:github.com/bytedance/sonic/loader/internal:g' ./x86_64/*.go && rm ./x86_64/*.bak + +# trim file methods +./trim.py + +gofmt -w */*.go + +echo "done" diff --git a/loader/internal/iasm/trim.py b/loader/internal/iasm/trim.py new file mode 100755 index 000000000..f3e01ed86 --- /dev/null +++ b/loader/internal/iasm/trim.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# +# Copyright 2025 ByteDance Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This file is used by sync.sh for trimming x86_64/instructions.go and x86_64/instructions_table.go + +filename = "x86_64/instructions.go" +print("trimming", filename) + +keep_methods = { + "ADDQ", + "CALLQ", + "CMPQ", + "JBE", + "JMP", + "LEAQ", + "MOVQ", + "MOVSD", + "MOVSS", + "RET", + "SUBQ", + "XORPS", + + # for _test.go + "VPERMIL2PD", + "MOVSLQ", + "JMPQ", +} +lines = [] +keep = True + +for line in open(filename): + if line.startswith("//") and "performs" in line: + method = line.split(" ")[1] + if method in keep_methods: + keep = True + keep = method in keep_methods + if keep: + lines.append(line.rstrip()) + +with open(filename, "w") as f: + f.write("\n".join(lines)) + + +filename = "x86_64/instructions_table.go" +print("trimming", filename) + +keep_before = "// Instructions" +lines = [] +keep = True + +for line in open(filename): + if keep and keep_before in line: + keep = False + if keep: + lines.append(line.rstrip()) + +with open(filename, "w") as f: + f.write("\n".join(lines)) diff --git a/loader/internal/iasm/x86_64/arch.go b/loader/internal/iasm/x86_64/arch.go new file mode 100644 index 000000000..f33cc9ca6 --- /dev/null +++ b/loader/internal/iasm/x86_64/arch.go @@ -0,0 +1,251 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "fmt" +) + +// ISA represents an extension to x86-64 instruction set. +type ISA uint64 + +const ( + ISA_CPUID ISA = 1 << iota + ISA_RDTSC + ISA_RDTSCP + ISA_CMOV + ISA_MOVBE + ISA_POPCNT + ISA_LZCNT + ISA_TBM + ISA_BMI + ISA_BMI2 + ISA_ADX + ISA_MMX + ISA_MMX_PLUS + ISA_FEMMS + ISA_3DNOW + ISA_3DNOW_PLUS + ISA_SSE + ISA_SSE2 + ISA_SSE3 + ISA_SSSE3 + ISA_SSE4A + ISA_SSE4_1 + ISA_SSE4_2 + ISA_FMA3 + ISA_FMA4 + ISA_XOP + ISA_F16C + ISA_AVX + ISA_AVX2 + ISA_AVX512F + ISA_AVX512BW + ISA_AVX512DQ + ISA_AVX512VL + ISA_AVX512PF + ISA_AVX512ER + ISA_AVX512CD + ISA_AVX512VBMI + ISA_AVX512IFMA + ISA_AVX512VPOPCNTDQ + ISA_AVX512_4VNNIW + ISA_AVX512_4FMAPS + ISA_PREFETCH + ISA_PREFETCHW + ISA_PREFETCHWT1 + ISA_CLFLUSH + ISA_CLFLUSHOPT + ISA_CLWB + ISA_CLZERO + ISA_RDRAND + ISA_RDSEED + ISA_PCLMULQDQ + ISA_AES + ISA_SHA + ISA_MONITOR + ISA_MONITORX + ISA_ALL = ^ISA(0) +) + +var _ISA_NAMES = map[ISA]string{ + ISA_CPUID: "CPUID", + ISA_RDTSC: "RDTSC", + ISA_RDTSCP: "RDTSCP", + ISA_CMOV: "CMOV", + ISA_MOVBE: "MOVBE", + ISA_POPCNT: "POPCNT", + ISA_LZCNT: "LZCNT", + ISA_TBM: "TBM", + ISA_BMI: "BMI", + ISA_BMI2: "BMI2", + ISA_ADX: "ADX", + ISA_MMX: "MMX", + ISA_MMX_PLUS: "MMX+", + ISA_FEMMS: "FEMMS", + ISA_3DNOW: "3dnow!", + ISA_3DNOW_PLUS: "3dnow!+", + ISA_SSE: "SSE", + ISA_SSE2: "SSE2", + ISA_SSE3: "SSE3", + ISA_SSSE3: "SSSE3", + ISA_SSE4A: "SSE4A", + ISA_SSE4_1: "SSE4.1", + ISA_SSE4_2: "SSE4.2", + ISA_FMA3: "FMA3", + ISA_FMA4: "FMA4", + ISA_XOP: "XOP", + ISA_F16C: "F16C", + ISA_AVX: "AVX", + ISA_AVX2: "AVX2", + ISA_AVX512F: "AVX512F", + ISA_AVX512BW: "AVX512BW", + ISA_AVX512DQ: "AVX512DQ", + ISA_AVX512VL: "AVX512VL", + ISA_AVX512PF: "AVX512PF", + ISA_AVX512ER: "AVX512ER", + ISA_AVX512CD: "AVX512CD", + ISA_AVX512VBMI: "AVX512VBMI", + ISA_AVX512IFMA: "AVX512IFMA", + ISA_AVX512VPOPCNTDQ: "AVX512VPOPCNTDQ", + ISA_AVX512_4VNNIW: "AVX512_4VNNIW", + ISA_AVX512_4FMAPS: "AVX512_4FMAPS", + ISA_PREFETCH: "PREFETCH", + ISA_PREFETCHW: "PREFETCHW", + ISA_PREFETCHWT1: "PREFETCHWT1", + ISA_CLFLUSH: "CLFLUSH", + ISA_CLFLUSHOPT: "CLFLUSHOPT", + ISA_CLWB: "CLWB", + ISA_CLZERO: "CLZERO", + ISA_RDRAND: "RDRAND", + ISA_RDSEED: "RDSEED", + ISA_PCLMULQDQ: "PCLMULQDQ", + ISA_AES: "AES", + ISA_SHA: "SHA", + ISA_MONITOR: "MONITOR", + ISA_MONITORX: "MONITORX", +} + +var _ISA_MAPPING = map[string]ISA{ + "CPUID": ISA_CPUID, + "RDTSC": ISA_RDTSC, + "RDTSCP": ISA_RDTSCP, + "CMOV": ISA_CMOV, + "MOVBE": ISA_MOVBE, + "POPCNT": ISA_POPCNT, + "LZCNT": ISA_LZCNT, + "TBM": ISA_TBM, + "BMI": ISA_BMI, + "BMI2": ISA_BMI2, + "ADX": ISA_ADX, + "MMX": ISA_MMX, + "MMX+": ISA_MMX_PLUS, + "FEMMS": ISA_FEMMS, + "3dnow!": ISA_3DNOW, + "3dnow!+": ISA_3DNOW_PLUS, + "SSE": ISA_SSE, + "SSE2": ISA_SSE2, + "SSE3": ISA_SSE3, + "SSSE3": ISA_SSSE3, + "SSE4A": ISA_SSE4A, + "SSE4.1": ISA_SSE4_1, + "SSE4.2": ISA_SSE4_2, + "FMA3": ISA_FMA3, + "FMA4": ISA_FMA4, + "XOP": ISA_XOP, + "F16C": ISA_F16C, + "AVX": ISA_AVX, + "AVX2": ISA_AVX2, + "AVX512F": ISA_AVX512F, + "AVX512BW": ISA_AVX512BW, + "AVX512DQ": ISA_AVX512DQ, + "AVX512VL": ISA_AVX512VL, + "AVX512PF": ISA_AVX512PF, + "AVX512ER": ISA_AVX512ER, + "AVX512CD": ISA_AVX512CD, + "AVX512VBMI": ISA_AVX512VBMI, + "AVX512IFMA": ISA_AVX512IFMA, + "AVX512VPOPCNTDQ": ISA_AVX512VPOPCNTDQ, + "AVX512_4VNNIW": ISA_AVX512_4VNNIW, + "AVX512_4FMAPS": ISA_AVX512_4FMAPS, + "PREFETCH": ISA_PREFETCH, + "PREFETCHW": ISA_PREFETCHW, + "PREFETCHWT1": ISA_PREFETCHWT1, + "CLFLUSH": ISA_CLFLUSH, + "CLFLUSHOPT": ISA_CLFLUSHOPT, + "CLWB": ISA_CLWB, + "CLZERO": ISA_CLZERO, + "RDRAND": ISA_RDRAND, + "RDSEED": ISA_RDSEED, + "PCLMULQDQ": ISA_PCLMULQDQ, + "AES": ISA_AES, + "SHA": ISA_SHA, + "MONITOR": ISA_MONITOR, + "MONITORX": ISA_MONITORX, +} + +func (self ISA) String() string { + if v, ok := _ISA_NAMES[self]; ok { + return v + } else { + return fmt.Sprintf("(invalid: %#x)", uint64(self)) + } +} + +// ParseISA parses name into ISA, it will panic if the name is invalid. +func ParseISA(name string) ISA { + if v, ok := _ISA_MAPPING[name]; ok { + return v + } else { + panic("invalid ISA name: " + name) + } +} + +// Arch represents the x86_64 architecture. +type Arch struct { + isa ISA +} + +// DefaultArch is the default architecture with all ISA enabled. +var DefaultArch = CreateArch() + +// CreateArch creates a new Arch with all ISA enabled. +func CreateArch() *Arch { + return new(Arch).EnableISA(ISA_ALL) +} + +// HasISA checks if a particular ISA was enabled. +func (self *Arch) HasISA(isa ISA) bool { + return (self.isa & isa) != 0 +} + +// EnableISA enables a particular ISA. +func (self *Arch) EnableISA(isa ISA) *Arch { + self.isa |= isa + return self +} + +// DisableISA disables a particular ISA. +func (self *Arch) DisableISA(isa ISA) *Arch { + self.isa &^= isa + return self +} + +// CreateProgram creates a new empty program. +func (self *Arch) CreateProgram() *Program { + return newProgram(self) +} diff --git a/loader/internal/iasm/x86_64/asm.s b/loader/internal/iasm/x86_64/asm.s new file mode 100644 index 000000000..a00c41dfc --- /dev/null +++ b/loader/internal/iasm/x86_64/asm.s @@ -0,0 +1,16 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + diff --git a/loader/internal/iasm/x86_64/eface.go b/loader/internal/iasm/x86_64/eface.go new file mode 100644 index 000000000..eb7f3c406 --- /dev/null +++ b/loader/internal/iasm/x86_64/eface.go @@ -0,0 +1,79 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "reflect" + "unsafe" +) + +type _GoType struct { + size uintptr + pdata uintptr + hash uint32 + flags uint8 + align uint8 + falign uint8 + kflags uint8 + traits unsafe.Pointer + gcdata *byte + str int32 + ptrx int32 +} + +const ( + _KindMask = (1 << 5) - 1 +) + +func (self *_GoType) kind() reflect.Kind { + return reflect.Kind(self.kflags & _KindMask) +} + +type _GoSlice struct { + ptr unsafe.Pointer + len int + cap int +} + +type _GoEface struct { + vt *_GoType + ptr unsafe.Pointer +} + +func (self *_GoEface) kind() reflect.Kind { + if self.vt != nil { + return self.vt.kind() + } else { + return reflect.Invalid + } +} + +func (self *_GoEface) toInt64() int64 { + if self.vt.size == 8 { + return *(*int64)(self.ptr) + } else if self.vt.size == 4 { + return int64(*(*int32)(self.ptr)) + } else if self.vt.size == 2 { + return int64(*(*int16)(self.ptr)) + } else { + return int64(*(*int8)(self.ptr)) + } +} + +func efaceOf(v interface{}) _GoEface { + return *(*_GoEface)(unsafe.Pointer(&v)) +} diff --git a/loader/internal/iasm/x86_64/encodings.go b/loader/internal/iasm/x86_64/encodings.go new file mode 100644 index 000000000..f003be927 --- /dev/null +++ b/loader/internal/iasm/x86_64/encodings.go @@ -0,0 +1,836 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "encoding/binary" + "math" +) + +/** Operand Encoding Helpers **/ + +func imml(v interface{}) byte { + return byte(toImmAny(v) & 0x0f) +} + +func relv(v interface{}) int64 { + switch r := v.(type) { + case *Label: + return 0 + case RelativeOffset: + return int64(r) + default: + panic("invalid relative offset") + } +} + +func addr(v interface{}) interface{} { + switch a := v.(*MemoryOperand).Addr; a.Type { + case Memory: + return a.Memory + case Offset: + return a.Offset + case Reference: + return a.Reference + default: + panic("invalid memory operand type") + } +} + +func bcode(v interface{}) byte { + if m, ok := v.(*MemoryOperand); !ok { + panic("v is not a memory operand") + } else if m.Broadcast == 0 { + return 0 + } else { + return 1 + } +} + +func vcode(v interface{}) byte { + switch r := v.(type) { + case XMMRegister: + return byte(r) + case YMMRegister: + return byte(r) + case ZMMRegister: + return byte(r) + case MaskedRegister: + return vcode(r.Reg) + default: + panic("v is not a vector register") + } +} + +func kcode(v interface{}) byte { + switch r := v.(type) { + case KRegister: + return byte(r) + case XMMRegister: + return 0 + case YMMRegister: + return 0 + case ZMMRegister: + return 0 + case RegisterMask: + return byte(r.K) + case MaskedRegister: + return byte(r.Mask.K) + case *MemoryOperand: + return toKcodeMem(r) + default: + panic("v is not a maskable operand") + } +} + +func zcode(v interface{}) byte { + switch r := v.(type) { + case KRegister: + return 0 + case XMMRegister: + return 0 + case YMMRegister: + return 0 + case ZMMRegister: + return 0 + case RegisterMask: + return toZcodeRegM(r) + case MaskedRegister: + return toZcodeRegM(r.Mask) + case *MemoryOperand: + return toZcodeMem(r) + default: + panic("v is not a maskable operand") + } +} + +func lcode(v interface{}) byte { + switch r := v.(type) { + case Register8: + return byte(r & 0x07) + case Register16: + return byte(r & 0x07) + case Register32: + return byte(r & 0x07) + case Register64: + return byte(r & 0x07) + case KRegister: + return byte(r & 0x07) + case MMRegister: + return byte(r & 0x07) + case XMMRegister: + return byte(r & 0x07) + case YMMRegister: + return byte(r & 0x07) + case ZMMRegister: + return byte(r & 0x07) + case MaskedRegister: + return lcode(r.Reg) + default: + panic("v is not a register") + } +} + +func hcode(v interface{}) byte { + switch r := v.(type) { + case Register8: + return byte(r>>3) & 1 + case Register16: + return byte(r>>3) & 1 + case Register32: + return byte(r>>3) & 1 + case Register64: + return byte(r>>3) & 1 + case KRegister: + return byte(r>>3) & 1 + case MMRegister: + return byte(r>>3) & 1 + case XMMRegister: + return byte(r>>3) & 1 + case YMMRegister: + return byte(r>>3) & 1 + case ZMMRegister: + return byte(r>>3) & 1 + case MaskedRegister: + return hcode(r.Reg) + default: + panic("v is not a register") + } +} + +func ecode(v interface{}) byte { + switch r := v.(type) { + case Register8: + return byte(r>>4) & 1 + case Register16: + return byte(r>>4) & 1 + case Register32: + return byte(r>>4) & 1 + case Register64: + return byte(r>>4) & 1 + case KRegister: + return byte(r>>4) & 1 + case MMRegister: + return byte(r>>4) & 1 + case XMMRegister: + return byte(r>>4) & 1 + case YMMRegister: + return byte(r>>4) & 1 + case ZMMRegister: + return byte(r>>4) & 1 + case MaskedRegister: + return ecode(r.Reg) + default: + panic("v is not a register") + } +} + +func hlcode(v interface{}) byte { + switch r := v.(type) { + case Register8: + return toHLcodeReg8(r) + case Register16: + return byte(r & 0x0f) + case Register32: + return byte(r & 0x0f) + case Register64: + return byte(r & 0x0f) + case KRegister: + return byte(r & 0x0f) + case MMRegister: + return byte(r & 0x0f) + case XMMRegister: + return byte(r & 0x0f) + case YMMRegister: + return byte(r & 0x0f) + case ZMMRegister: + return byte(r & 0x0f) + case MaskedRegister: + return hlcode(r.Reg) + default: + panic("v is not a register") + } +} + +func ehcode(v interface{}) byte { + switch r := v.(type) { + case Register8: + return byte(r>>3) & 0x03 + case Register16: + return byte(r>>3) & 0x03 + case Register32: + return byte(r>>3) & 0x03 + case Register64: + return byte(r>>3) & 0x03 + case KRegister: + return byte(r>>3) & 0x03 + case MMRegister: + return byte(r>>3) & 0x03 + case XMMRegister: + return byte(r>>3) & 0x03 + case YMMRegister: + return byte(r>>3) & 0x03 + case ZMMRegister: + return byte(r>>3) & 0x03 + case MaskedRegister: + return ehcode(r.Reg) + default: + panic("v is not a register") + } +} + +func toImmAny(v interface{}) int64 { + if x, ok := asInt64(v); ok { + return x + } else { + panic("value is not an integer") + } +} + +func toHcodeOpt(v interface{}) byte { + if v == nil { + return 0 + } else { + return hcode(v) + } +} + +func toEcodeVMM(v interface{}, x byte) byte { + switch r := v.(type) { + case XMMRegister: + return ecode(r) + case YMMRegister: + return ecode(r) + case ZMMRegister: + return ecode(r) + default: + return x + } +} + +func toKcodeMem(v *MemoryOperand) byte { + if !v.Masked { + return 0 + } else { + return byte(v.Mask.K) + } +} + +func toZcodeMem(v *MemoryOperand) byte { + if !v.Masked || v.Mask.Z { + return 0 + } else { + return 1 + } +} + +func toZcodeRegM(v RegisterMask) byte { + if v.Z { + return 1 + } else { + return 0 + } +} + +func toHLcodeReg8(v Register8) byte { + switch v { + case AH: + fallthrough + case BH: + fallthrough + case CH: + fallthrough + case DH: + panic("ah/bh/ch/dh registers never use 4-bit encoding") + default: + return byte(v & 0x0f) + } +} + +/** Instruction Encoding Helpers **/ + +const ( + _N_inst = 16 +) + +const ( + _F_rel1 = 1 << iota + _F_rel4 +) + +type _Encoding struct { + len int + flags int + bytes [_N_inst]byte + encoder func(m *_Encoding, v []interface{}) +} + +// buf ensures len + n <= len(bytes). +func (self *_Encoding) buf(n int) []byte { + if i := self.len; i+n > _N_inst { + panic("instruction too long") + } else { + return self.bytes[i:] + } +} + +// emit encodes a single byte. +func (self *_Encoding) emit(v byte) { + self.buf(1)[0] = v + self.len++ +} + +// imm1 encodes a single byte immediate value. +func (self *_Encoding) imm1(v int64) { + self.emit(byte(v)) +} + +// imm2 encodes a two-byte immediate value in little-endian. +func (self *_Encoding) imm2(v int64) { + binary.LittleEndian.PutUint16(self.buf(2), uint16(v)) + self.len += 2 +} + +// imm4 encodes a 4-byte immediate value in little-endian. +func (self *_Encoding) imm4(v int64) { + binary.LittleEndian.PutUint32(self.buf(4), uint32(v)) + self.len += 4 +} + +// imm8 encodes an 8-byte immediate value in little-endian. +func (self *_Encoding) imm8(v int64) { + binary.LittleEndian.PutUint64(self.buf(8), uint64(v)) + self.len += 8 +} + +// vex2 encodes a 2-byte or 3-byte VEX prefix. +// +// 2-byte VEX prefix: +// +// Requires: VEX.W = 0, VEX.mmmmm = 0b00001 and VEX.B = VEX.X = 0 +// +// +----------------+ +// +// Byte 0: | Bits 0-7: 0xc5 | +// +// +----------------+ +// +// +-----------+----------------+----------+--------------+ +// +// Byte 1: | Bit 7: ~R | Bits 3-6 ~vvvv | Bit 2: L | Bits 0-1: pp | +// +// +-----------+----------------+----------+--------------+ +// +// 3-byte VEX prefix: +// +----------------+ +// +// Byte 0: | Bits 0-7: 0xc4 | +// +// +----------------+ +// +// +-----------+-----------+-----------+-------------------+ +// +// Byte 1: | Bit 7: ~R | Bit 6: ~X | Bit 5: ~B | Bits 0-4: 0b00001 | +// +// +-----------+-----------+-----------+-------------------+ +// +// +----------+-----------------+----------+--------------+ +// +// Byte 2: | Bit 7: 0 | Bits 3-6: ~vvvv | Bit 2: L | Bits 0-1: pp | +// +// +----------+-----------------+----------+--------------+ +func (self *_Encoding) vex2(lpp byte, r byte, rm interface{}, vvvv byte) { + var b byte + var x byte + + /* VEX.R must be a single-bit mask */ + if r > 1 { + panic("VEX.R must be a 1-bit mask") + } + + /* VEX.Lpp must be a 3-bit mask */ + if lpp&^0b111 != 0 { + panic("VEX.Lpp must be a 3-bit mask") + } + + /* VEX.vvvv must be a 4-bit mask */ + if vvvv&^0b1111 != 0 { + panic("VEX.vvvv must be a 4-bit mask") + } + + /* encode the RM bits if any */ + if rm != nil { + switch v := rm.(type) { + case *Label: + break + case Register: + b = hcode(v) + case MemoryAddress: + b, x = toHcodeOpt(v.Base), toHcodeOpt(v.Index) + case RelativeOffset: + break + default: + panic("rm is expected to be a register or a memory address") + } + } + + /* if VEX.B and VEX.X are zeroes, 2-byte VEX prefix can be used */ + if x == 0 && b == 0 { + self.emit(0xc5) + self.emit(0xf8 ^ (r << 7) ^ (vvvv << 3) ^ lpp) + } else { + self.emit(0xc4) + self.emit(0xe1 ^ (r << 7) ^ (x << 6) ^ (b << 5)) + self.emit(0x78 ^ (vvvv << 3) ^ lpp) + } +} + +// vex3 encodes a 3-byte VEX or XOP prefix. +// +// 3-byte VEX/XOP prefix +// +-----------------------------------+ +// +// Byte 0: | Bits 0-7: 0xc4 (VEX) / 0x8f (XOP) | +// +// +-----------------------------------+ +// +// +-----------+-----------+-----------+-----------------+ +// +// Byte 1: | Bit 7: ~R | Bit 6: ~X | Bit 5: ~B | Bits 0-4: mmmmm | +// +// +-----------+-----------+-----------+-----------------+ +// +// +----------+-----------------+----------+--------------+ +// +// Byte 2: | Bit 7: W | Bits 3-6: ~vvvv | Bit 2: L | Bits 0-1: pp | +// +// +----------+-----------------+----------+--------------+ +func (self *_Encoding) vex3(esc byte, mmmmm byte, wlpp byte, r byte, rm interface{}, vvvv byte) { + var b byte + var x byte + + /* VEX.R must be a single-bit mask */ + if r > 1 { + panic("VEX.R must be a 1-bit mask") + } + + /* VEX.vvvv must be a 4-bit mask */ + if vvvv&^0b1111 != 0 { + panic("VEX.vvvv must be a 4-bit mask") + } + + /* escape must be a 3-byte VEX (0xc4) or XOP (0x8f) prefix */ + if esc != 0xc4 && esc != 0x8f { + panic("escape must be a 3-byte VEX (0xc4) or XOP (0x8f) prefix") + } + + /* VEX.W____Lpp is expected to have no bits set except 0, 1, 2 and 7 */ + if wlpp&^0b10000111 != 0 { + panic("VEX.W____Lpp is expected to have no bits set except 0, 1, 2 and 7") + } + + /* VEX.m-mmmm is expected to be a 5-bit mask */ + if mmmmm&^0b11111 != 0 { + panic("VEX.m-mmmm is expected to be a 5-bit mask") + } + + /* encode the RM bits */ + switch v := rm.(type) { + case *Label: + break + case MemoryAddress: + b, x = toHcodeOpt(v.Base), toHcodeOpt(v.Index) + case RelativeOffset: + break + default: + panic("rm is expected to be a register or a memory address") + } + + /* encode the 3-byte VEX or XOP prefix */ + self.emit(esc) + self.emit(0xe0 ^ (r << 7) ^ (x << 6) ^ (b << 5) ^ mmmmm) + self.emit(0x78 ^ (vvvv << 3) ^ wlpp) +} + +// evex encodes a 4-byte EVEX prefix. +func (self *_Encoding) evex(mm byte, w1pp byte, ll byte, rr byte, rm interface{}, vvvvv byte, aaa byte, zz byte, bb byte) { + var b byte + var x byte + + /* EVEX.b must be a single-bit mask */ + if bb > 1 { + panic("EVEX.b must be a 1-bit mask") + } + + /* EVEX.z must be a single-bit mask */ + if zz > 1 { + panic("EVEX.z must be a 1-bit mask") + } + + /* EVEX.mm must be a 2-bit mask */ + if mm&^0b11 != 0 { + panic("EVEX.mm must be a 2-bit mask") + } + + /* EVEX.L'L must be a 2-bit mask */ + if ll&^0b11 != 0 { + panic("EVEX.L'L must be a 2-bit mask") + } + + /* EVEX.R'R must be a 2-bit mask */ + if rr&^0b11 != 0 { + panic("EVEX.R'R must be a 2-bit mask") + } + + /* EVEX.aaa must be a 3-bit mask */ + if aaa&^0b111 != 0 { + panic("EVEX.aaa must be a 3-bit mask") + } + + /* EVEX.v'vvvv must be a 5-bit mask */ + if vvvvv&^0b11111 != 0 { + panic("EVEX.v'vvvv must be a 5-bit mask") + } + + /* EVEX.W____1pp is expected to have no bits set except 0, 1, 2, and 7 */ + if w1pp&^0b10000011 != 0b100 { + panic("EVEX.W____1pp is expected to have no bits set except 0, 1, 2, and 7") + } + + /* extract bits from EVEX.R'R and EVEX.v'vvvv */ + r1, r0 := rr>>1, rr&1 + v1, v0 := vvvvv>>4, vvvvv&0b1111 + + /* encode the RM bits if any */ + if rm != nil { + switch m := rm.(type) { + case *Label: + break + case Register: + b, x = hcode(m), ecode(m) + case MemoryAddress: + b, x, v1 = toHcodeOpt(m.Base), toHcodeOpt(m.Index), toEcodeVMM(m.Index, v1) + case RelativeOffset: + break + default: + panic("rm is expected to be a register or a memory address") + } + } + + /* EVEX prefix bytes */ + p0 := (r0 << 7) | (x << 6) | (b << 5) | (r1 << 4) | mm + p1 := (v0 << 3) | w1pp + p2 := (zz << 7) | (ll << 5) | (b << 4) | (v1 << 3) | aaa + + /* p0: invert RXBR' (bits 4-7) + * p1: invert vvvv (bits 3-6) + * p2: invert V' (bit 3) */ + self.emit(0x62) + self.emit(p0 ^ 0xf0) + self.emit(p1 ^ 0x78) + self.emit(p2 ^ 0x08) +} + +// rexm encodes a mandatory REX prefix. +func (self *_Encoding) rexm(w byte, r byte, rm interface{}) { + var b byte + var x byte + + /* REX.R must be 0 or 1 */ + if r != 0 && r != 1 { + panic("REX.R must be 0 or 1") + } + + /* REX.W must be 0 or 1 */ + if w != 0 && w != 1 { + panic("REX.W must be 0 or 1") + } + + /* encode the RM bits */ + switch v := rm.(type) { + case *Label: + break + case MemoryAddress: + b, x = toHcodeOpt(v.Base), toHcodeOpt(v.Index) + case RelativeOffset: + break + default: + panic("rm is expected to be a register or a memory address") + } + + /* encode the REX prefix */ + self.emit(0x40 | (w << 3) | (r << 2) | (x << 1) | b) +} + +// rexo encodes an optional REX prefix. +func (self *_Encoding) rexo(r byte, rm interface{}, force bool) { + var b byte + var x byte + + /* REX.R must be 0 or 1 */ + if r != 0 && r != 1 { + panic("REX.R must be 0 or 1") + } + + /* encode the RM bits */ + switch v := rm.(type) { + case *Label: + break + case Register: + b = hcode(v) + case MemoryAddress: + b, x = toHcodeOpt(v.Base), toHcodeOpt(v.Index) + case RelativeOffset: + break + default: + panic("rm is expected to be a register or a memory address") + } + + /* if REX.R, REX.X, and REX.B are all zeroes, REX prefix can be omitted */ + if force || r != 0 || x != 0 || b != 0 { + self.emit(0x40 | (r << 2) | (x << 1) | b) + } +} + +// mrsd encodes ModR/M, SIB and Displacement. +// +// ModR/M byte +// +// +----------------+---------------+---------------+ +// | Bits 6-7: Mode | Bits 3-5: Reg | Bits 0-2: R/M | +// +----------------+---------------+---------------+ +// +// SIB byte +// +// +-----------------+-----------------+----------------+ +// | Bits 6-7: Scale | Bits 3-5: Index | Bits 0-2: Base | +// +-----------------+-----------------+----------------+ +func (self *_Encoding) mrsd(reg byte, rm interface{}, disp8v int32) { + var ok bool + var mm MemoryAddress + var ro RelativeOffset + + /* ModRM encodes the lower 3-bit of the register */ + if reg > 7 { + panic("invalid register bits") + } + + /* check the displacement scale */ + switch disp8v { + case 1: + break + case 2: + break + case 4: + break + case 8: + break + case 16: + break + case 32: + break + case 64: + break + default: + panic("invalid displacement size") + } + + /* special case: unresolved labels, assuming a zero offset */ + if _, ok = rm.(*Label); ok { + self.emit(0x05 | (reg << 3)) + self.imm4(0) + return + } + + /* special case: RIP-relative offset + * ModRM.Mode == 0 and ModeRM.R/M == 5 indicates (rip + disp32) addressing */ + if ro, ok = rm.(RelativeOffset); ok { + self.emit(0x05 | (reg << 3)) + self.imm4(int64(ro)) + return + } + + /* must be a generic memory address */ + if mm, ok = rm.(MemoryAddress); !ok { + panic("rm must be a memory address") + } + + /* absolute addressing, encoded as disp(%rbp,%rsp,1) */ + if mm.Base == nil && mm.Index == nil { + self.emit(0x04 | (reg << 3)) + self.emit(0x25) + self.imm4(int64(mm.Displacement)) + return + } + + /* no SIB byte */ + if mm.Index == nil && lcode(mm.Base) != 0b100 { + cc := lcode(mm.Base) + dv := mm.Displacement + + /* ModRM.Mode == 0 (no displacement) */ + if dv == 0 && mm.Base != RBP && mm.Base != R13 { + if cc == 0b101 { + panic("rbp/r13 is not encodable as a base register (interpreted as disp32 address)") + } else { + self.emit((reg << 3) | cc) + return + } + } + + /* ModRM.Mode == 1 (8-bit displacement) */ + if dq := dv / disp8v; dq >= math.MinInt8 && dq <= math.MaxInt8 && dv%disp8v == 0 { + self.emit(0x40 | (reg << 3) | cc) + self.imm1(int64(dq)) + return + } + + /* ModRM.Mode == 2 (32-bit displacement) */ + self.emit(0x80 | (reg << 3) | cc) + self.imm4(int64(mm.Displacement)) + return + } + + /* all encodings below use ModRM.R/M = 4 (0b100) to indicate the presence of SIB */ + if mm.Index == RSP { + panic("rsp is not encodable as an index register (interpreted as no index)") + } + + /* index = 4 (0b100) denotes no-index encoding */ + var scale byte + var index byte = 0x04 + + /* encode the scale byte */ + if mm.Scale != 0 { + switch mm.Scale { + case 1: + scale = 0 + case 2: + scale = 1 + case 4: + scale = 2 + case 8: + scale = 3 + default: + panic("invalid scale value") + } + } + + /* encode the index byte */ + if mm.Index != nil { + index = lcode(mm.Index) + } + + /* SIB.Base = 5 (0b101) and ModRM.Mode = 0 indicates no-base encoding with disp32 */ + if mm.Base == nil { + self.emit((reg << 3) | 0b100) + self.emit((scale << 6) | (index << 3) | 0b101) + self.imm4(int64(mm.Displacement)) + return + } + + /* base L-code & displacement value */ + cc := lcode(mm.Base) + dv := mm.Displacement + + /* ModRM.Mode == 0 (no displacement) */ + if dv == 0 && cc != 0b101 { + self.emit((reg << 3) | 0b100) + self.emit((scale << 6) | (index << 3) | cc) + return + } + + /* ModRM.Mode == 1 (8-bit displacement) */ + if dq := dv / disp8v; dq >= math.MinInt8 && dq <= math.MaxInt8 && dv%disp8v == 0 { + self.emit(0x44 | (reg << 3)) + self.emit((scale << 6) | (index << 3) | cc) + self.imm1(int64(dq)) + return + } + + /* ModRM.Mode == 2 (32-bit displacement) */ + self.emit(0x84 | (reg << 3)) + self.emit((scale << 6) | (index << 3) | cc) + self.imm4(int64(mm.Displacement)) +} + +// encode invokes the encoder to encode this instruction. +func (self *_Encoding) encode(v []interface{}) int { + self.len = 0 + self.encoder(self, v) + return self.len +} diff --git a/loader/internal/iasm/x86_64/instructions.go b/loader/internal/iasm/x86_64/instructions.go new file mode 100644 index 000000000..836e1807f --- /dev/null +++ b/loader/internal/iasm/x86_64/instructions.go @@ -0,0 +1,1077 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Code generated by "mkasm_amd64.py", DO NOT EDIT. + +package x86_64 + +// ADDQ performs "Add". +// +// Mnemonic : ADD +// Supported forms : (8 forms) +// +// - ADDQ imm32, rax +// - ADDQ imm8, r64 +// - ADDQ imm32, r64 +// - ADDQ r64, r64 +// - ADDQ m64, r64 +// - ADDQ imm8, m64 +// - ADDQ imm32, m64 +// - ADDQ r64, m64 +func (self *Program) ADDQ(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("ADDQ", 2, Operands{v0, v1}) + // ADDQ imm32, rax + if isImm32(v0) && v1 == RAX { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48) + m.emit(0x05) + m.imm4(toImmAny(v[0])) + }) + } + // ADDQ imm8, r64 + if isImm8Ext(v0, 8) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])) + m.emit(0x83) + m.emit(0xc0 | lcode(v[1])) + m.imm1(toImmAny(v[0])) + }) + } + // ADDQ imm32, r64 + if isImm32Ext(v0, 8) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])) + m.emit(0x81) + m.emit(0xc0 | lcode(v[1])) + m.imm4(toImmAny(v[0])) + }) + } + // ADDQ r64, r64 + if isReg64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[0])<<2 | hcode(v[1])) + m.emit(0x01) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])<<2 | hcode(v[0])) + m.emit(0x03) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + } + // ADDQ m64, r64 + if isM64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[1]), addr(v[0])) + m.emit(0x03) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + // ADDQ imm8, m64 + if isImm8Ext(v0, 8) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, 0, addr(v[1])) + m.emit(0x83) + m.mrsd(0, addr(v[1]), 1) + m.imm1(toImmAny(v[0])) + }) + } + // ADDQ imm32, m64 + if isImm32Ext(v0, 8) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, 0, addr(v[1])) + m.emit(0x81) + m.mrsd(0, addr(v[1]), 1) + m.imm4(toImmAny(v[0])) + }) + } + // ADDQ r64, m64 + if isReg64(v0) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[0]), addr(v[1])) + m.emit(0x01) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for ADDQ") + } + return p +} + +// CALLQ performs "Call Procedure". +// +// Mnemonic : CALL +// Supported forms : (2 forms) +// +// - CALLQ r64 +// - CALLQ m64 +func (self *Program) CALLQ(v0 interface{}) *Instruction { + p := self.alloc("CALLQ", 1, Operands{v0}) + // CALLQ r64 + if isReg64(v0) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(0, v[0], false) + m.emit(0xff) + m.emit(0xd0 | lcode(v[0])) + }) + } + // CALLQ m64 + if isM64(v0) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(0, addr(v[0]), false) + m.emit(0xff) + m.mrsd(2, addr(v[0]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for CALLQ") + } + return p +} + +// CMPQ performs "Compare Two Operands". +// +// Mnemonic : CMP +// Supported forms : (8 forms) +// +// - CMPQ imm32, rax +// - CMPQ imm8, r64 +// - CMPQ imm32, r64 +// - CMPQ r64, r64 +// - CMPQ m64, r64 +// - CMPQ imm8, m64 +// - CMPQ imm32, m64 +// - CMPQ r64, m64 +func (self *Program) CMPQ(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("CMPQ", 2, Operands{v0, v1}) + // CMPQ imm32, rax + if isImm32(v0) && v1 == RAX { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48) + m.emit(0x3d) + m.imm4(toImmAny(v[0])) + }) + } + // CMPQ imm8, r64 + if isImm8Ext(v0, 8) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])) + m.emit(0x83) + m.emit(0xf8 | lcode(v[1])) + m.imm1(toImmAny(v[0])) + }) + } + // CMPQ imm32, r64 + if isImm32Ext(v0, 8) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])) + m.emit(0x81) + m.emit(0xf8 | lcode(v[1])) + m.imm4(toImmAny(v[0])) + }) + } + // CMPQ r64, r64 + if isReg64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[0])<<2 | hcode(v[1])) + m.emit(0x39) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])<<2 | hcode(v[0])) + m.emit(0x3b) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + } + // CMPQ m64, r64 + if isM64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[1]), addr(v[0])) + m.emit(0x3b) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + // CMPQ imm8, m64 + if isImm8Ext(v0, 8) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, 0, addr(v[1])) + m.emit(0x83) + m.mrsd(7, addr(v[1]), 1) + m.imm1(toImmAny(v[0])) + }) + } + // CMPQ imm32, m64 + if isImm32Ext(v0, 8) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, 0, addr(v[1])) + m.emit(0x81) + m.mrsd(7, addr(v[1]), 1) + m.imm4(toImmAny(v[0])) + }) + } + // CMPQ r64, m64 + if isReg64(v0) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[0]), addr(v[1])) + m.emit(0x39) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for CMPQ") + } + return p +} + +// JBE performs "Jump if below or equal (CF == 1 or ZF == 1)". +// +// Mnemonic : JBE +// Supported forms : (2 forms) +// +// - JBE rel8 +// - JBE rel32 +func (self *Program) JBE(v0 interface{}) *Instruction { + p := self.alloc("JBE", 1, Operands{v0}) + p.branch = _B_conditional + // JBE rel8 + if isRel8(v0) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x76) + m.imm1(relv(v[0])) + }) + } + // JBE rel32 + if isRel32(v0) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x0f) + m.emit(0x86) + m.imm4(relv(v[0])) + }) + } + // JBE label + if isLabel(v0) { + p.add(_F_rel1, func(m *_Encoding, v []interface{}) { + m.emit(0x76) + m.imm1(relv(v[0])) + }) + p.add(_F_rel4, func(m *_Encoding, v []interface{}) { + m.emit(0x0f) + m.emit(0x86) + m.imm4(relv(v[0])) + }) + } + if p.len == 0 { + panic("invalid operands for JBE") + } + return p +} + +// JMP performs "Jump Unconditionally". +// +// Mnemonic : JMP +// Supported forms : (2 forms) +// +// - JMP rel8 +// - JMP rel32 +func (self *Program) JMP(v0 interface{}) *Instruction { + p := self.alloc("JMP", 1, Operands{v0}) + p.branch = _B_unconditional + // JMP rel8 + if isRel8(v0) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xeb) + m.imm1(relv(v[0])) + }) + } + // JMP rel32 + if isRel32(v0) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xe9) + m.imm4(relv(v[0])) + }) + } + // JMP label + if isLabel(v0) { + p.add(_F_rel1, func(m *_Encoding, v []interface{}) { + m.emit(0xeb) + m.imm1(relv(v[0])) + }) + p.add(_F_rel4, func(m *_Encoding, v []interface{}) { + m.emit(0xe9) + m.imm4(relv(v[0])) + }) + } + if p.len == 0 { + panic("invalid operands for JMP") + } + return p +} + +// JMPQ performs "Jump Unconditionally". +// +// Mnemonic : JMP +// Supported forms : (2 forms) +// +// - JMPQ r64 +// - JMPQ m64 +func (self *Program) JMPQ(v0 interface{}) *Instruction { + p := self.alloc("JMPQ", 1, Operands{v0}) + // JMPQ r64 + if isReg64(v0) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(0, v[0], false) + m.emit(0xff) + m.emit(0xe0 | lcode(v[0])) + }) + } + // JMPQ m64 + if isM64(v0) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(0, addr(v[0]), false) + m.emit(0xff) + m.mrsd(4, addr(v[0]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for JMPQ") + } + return p +} + +// LEAQ performs "Load Effective Address". +// +// Mnemonic : LEA +// Supported forms : (1 form) +// +// - LEAQ m, r64 +func (self *Program) LEAQ(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("LEAQ", 2, Operands{v0, v1}) + // LEAQ m, r64 + if isM(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[1]), addr(v[0])) + m.emit(0x8d) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for LEAQ") + } + return p +} + +// MOVQ performs "Move". +// +// Mnemonic : MOV +// Supported forms : (16 forms) +// +// - MOVQ imm32, r64 +// - MOVQ imm64, r64 +// - MOVQ r64, r64 +// - MOVQ m64, r64 +// - MOVQ imm32, m64 +// - MOVQ r64, m64 +// - MOVQ mm, r64 [MMX] +// - MOVQ r64, mm [MMX] +// - MOVQ mm, mm [MMX] +// - MOVQ m64, mm [MMX] +// - MOVQ mm, m64 [MMX] +// - MOVQ xmm, r64 [SSE2] +// - MOVQ r64, xmm [SSE2] +// - MOVQ xmm, xmm [SSE2] +// - MOVQ m64, xmm [SSE2] +// - MOVQ xmm, m64 [SSE2] +func (self *Program) MOVQ(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("MOVQ", 2, Operands{v0, v1}) + // MOVQ imm32, r64 + if isImm32Ext(v0, 8) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])) + m.emit(0xc7) + m.emit(0xc0 | lcode(v[1])) + m.imm4(toImmAny(v[0])) + }) + } + // MOVQ imm64, r64 + if isImm64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])) + m.emit(0xb8 | lcode(v[1])) + m.imm8(toImmAny(v[0])) + }) + } + // MOVQ r64, r64 + if isReg64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[0])<<2 | hcode(v[1])) + m.emit(0x89) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])<<2 | hcode(v[0])) + m.emit(0x8b) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + } + // MOVQ m64, r64 + if isM64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[1]), addr(v[0])) + m.emit(0x8b) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + // MOVQ imm32, m64 + if isImm32Ext(v0, 8) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, 0, addr(v[1])) + m.emit(0xc7) + m.mrsd(0, addr(v[1]), 1) + m.imm4(toImmAny(v[0])) + }) + } + // MOVQ r64, m64 + if isReg64(v0) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[0]), addr(v[1])) + m.emit(0x89) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + } + // MOVQ mm, r64 + if isMM(v0) && isReg64(v1) { + self.require(ISA_MMX) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[0])<<2 | hcode(v[1])) + m.emit(0x0f) + m.emit(0x7e) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + } + // MOVQ r64, mm + if isReg64(v0) && isMM(v1) { + self.require(ISA_MMX) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])<<2 | hcode(v[0])) + m.emit(0x0f) + m.emit(0x6e) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + } + // MOVQ mm, mm + if isMM(v0) && isMM(v1) { + self.require(ISA_MMX) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(hcode(v[1]), v[0], false) + m.emit(0x0f) + m.emit(0x6f) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(hcode(v[0]), v[1], false) + m.emit(0x0f) + m.emit(0x7f) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + } + // MOVQ m64, mm + if isM64(v0) && isMM(v1) { + self.require(ISA_MMX) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(hcode(v[1]), addr(v[0]), false) + m.emit(0x0f) + m.emit(0x6f) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[1]), addr(v[0])) + m.emit(0x0f) + m.emit(0x6e) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + // MOVQ mm, m64 + if isMM(v0) && isM64(v1) { + self.require(ISA_MMX) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(hcode(v[0]), addr(v[1]), false) + m.emit(0x0f) + m.emit(0x7f) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[0]), addr(v[1])) + m.emit(0x0f) + m.emit(0x7e) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + } + // MOVQ xmm, r64 + if isXMM(v0) && isReg64(v1) { + self.require(ISA_SSE2) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x66) + m.emit(0x48 | hcode(v[0])<<2 | hcode(v[1])) + m.emit(0x0f) + m.emit(0x7e) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + } + // MOVQ r64, xmm + if isReg64(v0) && isXMM(v1) { + self.require(ISA_SSE2) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x66) + m.emit(0x48 | hcode(v[1])<<2 | hcode(v[0])) + m.emit(0x0f) + m.emit(0x6e) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + } + // MOVQ xmm, xmm + if isXMM(v0) && isXMM(v1) { + self.require(ISA_SSE2) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf3) + m.rexo(hcode(v[1]), v[0], false) + m.emit(0x0f) + m.emit(0x7e) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x66) + m.rexo(hcode(v[0]), v[1], false) + m.emit(0x0f) + m.emit(0xd6) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + } + // MOVQ m64, xmm + if isM64(v0) && isXMM(v1) { + self.require(ISA_SSE2) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf3) + m.rexo(hcode(v[1]), addr(v[0]), false) + m.emit(0x0f) + m.emit(0x7e) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x66) + m.rexm(1, hcode(v[1]), addr(v[0])) + m.emit(0x0f) + m.emit(0x6e) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + // MOVQ xmm, m64 + if isXMM(v0) && isM64(v1) { + self.require(ISA_SSE2) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x66) + m.rexo(hcode(v[0]), addr(v[1]), false) + m.emit(0x0f) + m.emit(0xd6) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x66) + m.rexm(1, hcode(v[0]), addr(v[1])) + m.emit(0x0f) + m.emit(0x7e) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for MOVQ") + } + return p +} + +// MOVSD performs "Move Scalar Double-Precision Floating-Point Value". +// +// Mnemonic : MOVSD +// Supported forms : (3 forms) +// +// - MOVSD xmm, xmm [SSE2] +// - MOVSD m64, xmm [SSE2] +// - MOVSD xmm, m64 [SSE2] +func (self *Program) MOVSD(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("MOVSD", 2, Operands{v0, v1}) + // MOVSD xmm, xmm + if isXMM(v0) && isXMM(v1) { + self.require(ISA_SSE2) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf2) + m.rexo(hcode(v[1]), v[0], false) + m.emit(0x0f) + m.emit(0x10) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf2) + m.rexo(hcode(v[0]), v[1], false) + m.emit(0x0f) + m.emit(0x11) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + } + // MOVSD m64, xmm + if isM64(v0) && isXMM(v1) { + self.require(ISA_SSE2) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf2) + m.rexo(hcode(v[1]), addr(v[0]), false) + m.emit(0x0f) + m.emit(0x10) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + // MOVSD xmm, m64 + if isXMM(v0) && isM64(v1) { + self.require(ISA_SSE2) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf2) + m.rexo(hcode(v[0]), addr(v[1]), false) + m.emit(0x0f) + m.emit(0x11) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for MOVSD") + } + return p +} + +// MOVSLQ performs "Move Doubleword to Quadword with Sign-Extension". +// +// Mnemonic : MOVSXD +// Supported forms : (2 forms) +// +// - MOVSLQ r32, r64 +// - MOVSLQ m32, r64 +func (self *Program) MOVSLQ(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("MOVSLQ", 2, Operands{v0, v1}) + // MOVSLQ r32, r64 + if isReg32(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])<<2 | hcode(v[0])) + m.emit(0x63) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + } + // MOVSLQ m32, r64 + if isM32(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[1]), addr(v[0])) + m.emit(0x63) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for MOVSLQ") + } + return p +} + +// MOVSS performs "Move Scalar Single-Precision Floating-Point Values". +// +// Mnemonic : MOVSS +// Supported forms : (3 forms) +// +// - MOVSS xmm, xmm [SSE] +// - MOVSS m32, xmm [SSE] +// - MOVSS xmm, m32 [SSE] +func (self *Program) MOVSS(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("MOVSS", 2, Operands{v0, v1}) + // MOVSS xmm, xmm + if isXMM(v0) && isXMM(v1) { + self.require(ISA_SSE) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf3) + m.rexo(hcode(v[1]), v[0], false) + m.emit(0x0f) + m.emit(0x10) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf3) + m.rexo(hcode(v[0]), v[1], false) + m.emit(0x0f) + m.emit(0x11) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + } + // MOVSS m32, xmm + if isM32(v0) && isXMM(v1) { + self.require(ISA_SSE) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf3) + m.rexo(hcode(v[1]), addr(v[0]), false) + m.emit(0x0f) + m.emit(0x10) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + // MOVSS xmm, m32 + if isXMM(v0) && isM32(v1) { + self.require(ISA_SSE) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xf3) + m.rexo(hcode(v[0]), addr(v[1]), false) + m.emit(0x0f) + m.emit(0x11) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for MOVSS") + } + return p +} + +// RET performs "Return from Procedure". +// +// Mnemonic : RET +// Supported forms : (2 forms) +// +// - RET +// - RET imm16 +func (self *Program) RET(vv ...interface{}) *Instruction { + var p *Instruction + switch len(vv) { + case 0: + p = self.alloc("RET", 0, Operands{}) + case 1: + p = self.alloc("RET", 1, Operands{vv[0]}) + default: + panic("instruction RET takes 0 or 1 operands") + } + // RET + if len(vv) == 0 { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xc3) + }) + } + // RET imm16 + if len(vv) == 1 && isImm16(vv[0]) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xc2) + m.imm2(toImmAny(v[0])) + }) + } + if p.len == 0 { + panic("invalid operands for RET") + } + return p +} + +// SUBQ performs "Subtract". +// +// Mnemonic : SUB +// Supported forms : (8 forms) +// +// - SUBQ imm32, rax +// - SUBQ imm8, r64 +// - SUBQ imm32, r64 +// - SUBQ r64, r64 +// - SUBQ m64, r64 +// - SUBQ imm8, m64 +// - SUBQ imm32, m64 +// - SUBQ r64, m64 +func (self *Program) SUBQ(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("SUBQ", 2, Operands{v0, v1}) + // SUBQ imm32, rax + if isImm32(v0) && v1 == RAX { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48) + m.emit(0x2d) + m.imm4(toImmAny(v[0])) + }) + } + // SUBQ imm8, r64 + if isImm8Ext(v0, 8) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])) + m.emit(0x83) + m.emit(0xe8 | lcode(v[1])) + m.imm1(toImmAny(v[0])) + }) + } + // SUBQ imm32, r64 + if isImm32Ext(v0, 8) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])) + m.emit(0x81) + m.emit(0xe8 | lcode(v[1])) + m.imm4(toImmAny(v[0])) + }) + } + // SUBQ r64, r64 + if isReg64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[0])<<2 | hcode(v[1])) + m.emit(0x29) + m.emit(0xc0 | lcode(v[0])<<3 | lcode(v[1])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0x48 | hcode(v[1])<<2 | hcode(v[0])) + m.emit(0x2b) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + } + // SUBQ m64, r64 + if isM64(v0) && isReg64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[1]), addr(v[0])) + m.emit(0x2b) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + // SUBQ imm8, m64 + if isImm8Ext(v0, 8) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, 0, addr(v[1])) + m.emit(0x83) + m.mrsd(5, addr(v[1]), 1) + m.imm1(toImmAny(v[0])) + }) + } + // SUBQ imm32, m64 + if isImm32Ext(v0, 8) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, 0, addr(v[1])) + m.emit(0x81) + m.mrsd(5, addr(v[1]), 1) + m.imm4(toImmAny(v[0])) + }) + } + // SUBQ r64, m64 + if isReg64(v0) && isM64(v1) { + p.domain = DomainGeneric + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexm(1, hcode(v[0]), addr(v[1])) + m.emit(0x29) + m.mrsd(lcode(v[0]), addr(v[1]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for SUBQ") + } + return p +} + +// VPERMIL2PD performs "Permute Two-Source Double-Precision Floating-Point Vectors". +// +// Mnemonic : VPERMIL2PD +// Supported forms : (6 forms) +// +// - VPERMIL2PD imm4, xmm, xmm, xmm, xmm [XOP] +// - VPERMIL2PD imm4, m128, xmm, xmm, xmm [XOP] +// - VPERMIL2PD imm4, xmm, m128, xmm, xmm [XOP] +// - VPERMIL2PD imm4, ymm, ymm, ymm, ymm [XOP] +// - VPERMIL2PD imm4, m256, ymm, ymm, ymm [XOP] +// - VPERMIL2PD imm4, ymm, m256, ymm, ymm [XOP] +func (self *Program) VPERMIL2PD(v0 interface{}, v1 interface{}, v2 interface{}, v3 interface{}, v4 interface{}) *Instruction { + p := self.alloc("VPERMIL2PD", 5, Operands{v0, v1, v2, v3, v4}) + // VPERMIL2PD imm4, xmm, xmm, xmm, xmm + if isImm4(v0) && isXMM(v1) && isXMM(v2) && isXMM(v3) && isXMM(v4) { + self.require(ISA_XOP) + p.domain = DomainAMDSpecific + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xc4) + m.emit(0xe3 ^ (hcode(v[4]) << 7) ^ (hcode(v[2]) << 5)) + m.emit(0x79 ^ (hlcode(v[3]) << 3)) + m.emit(0x49) + m.emit(0xc0 | lcode(v[4])<<3 | lcode(v[2])) + m.emit((hlcode(v[1]) << 4) | imml(v[0])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xc4) + m.emit(0xe3 ^ (hcode(v[4]) << 7) ^ (hcode(v[1]) << 5)) + m.emit(0xf9 ^ (hlcode(v[3]) << 3)) + m.emit(0x49) + m.emit(0xc0 | lcode(v[4])<<3 | lcode(v[1])) + m.emit((hlcode(v[2]) << 4) | imml(v[0])) + }) + } + // VPERMIL2PD imm4, m128, xmm, xmm, xmm + if isImm4(v0) && isM128(v1) && isXMM(v2) && isXMM(v3) && isXMM(v4) { + self.require(ISA_XOP) + p.domain = DomainAMDSpecific + p.add(0, func(m *_Encoding, v []interface{}) { + m.vex3(0xc4, 0b11, 0x81, hcode(v[4]), addr(v[1]), hlcode(v[3])) + m.emit(0x49) + m.mrsd(lcode(v[4]), addr(v[1]), 1) + m.emit((hlcode(v[2]) << 4) | imml(v[0])) + }) + } + // VPERMIL2PD imm4, xmm, m128, xmm, xmm + if isImm4(v0) && isXMM(v1) && isM128(v2) && isXMM(v3) && isXMM(v4) { + self.require(ISA_XOP) + p.domain = DomainAMDSpecific + p.add(0, func(m *_Encoding, v []interface{}) { + m.vex3(0xc4, 0b11, 0x01, hcode(v[4]), addr(v[2]), hlcode(v[3])) + m.emit(0x49) + m.mrsd(lcode(v[4]), addr(v[2]), 1) + m.emit((hlcode(v[1]) << 4) | imml(v[0])) + }) + } + // VPERMIL2PD imm4, ymm, ymm, ymm, ymm + if isImm4(v0) && isYMM(v1) && isYMM(v2) && isYMM(v3) && isYMM(v4) { + self.require(ISA_XOP) + p.domain = DomainAMDSpecific + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xc4) + m.emit(0xe3 ^ (hcode(v[4]) << 7) ^ (hcode(v[2]) << 5)) + m.emit(0x7d ^ (hlcode(v[3]) << 3)) + m.emit(0x49) + m.emit(0xc0 | lcode(v[4])<<3 | lcode(v[2])) + m.emit((hlcode(v[1]) << 4) | imml(v[0])) + }) + p.add(0, func(m *_Encoding, v []interface{}) { + m.emit(0xc4) + m.emit(0xe3 ^ (hcode(v[4]) << 7) ^ (hcode(v[1]) << 5)) + m.emit(0xfd ^ (hlcode(v[3]) << 3)) + m.emit(0x49) + m.emit(0xc0 | lcode(v[4])<<3 | lcode(v[1])) + m.emit((hlcode(v[2]) << 4) | imml(v[0])) + }) + } + // VPERMIL2PD imm4, m256, ymm, ymm, ymm + if isImm4(v0) && isM256(v1) && isYMM(v2) && isYMM(v3) && isYMM(v4) { + self.require(ISA_XOP) + p.domain = DomainAMDSpecific + p.add(0, func(m *_Encoding, v []interface{}) { + m.vex3(0xc4, 0b11, 0x85, hcode(v[4]), addr(v[1]), hlcode(v[3])) + m.emit(0x49) + m.mrsd(lcode(v[4]), addr(v[1]), 1) + m.emit((hlcode(v[2]) << 4) | imml(v[0])) + }) + } + // VPERMIL2PD imm4, ymm, m256, ymm, ymm + if isImm4(v0) && isYMM(v1) && isM256(v2) && isYMM(v3) && isYMM(v4) { + self.require(ISA_XOP) + p.domain = DomainAMDSpecific + p.add(0, func(m *_Encoding, v []interface{}) { + m.vex3(0xc4, 0b11, 0x05, hcode(v[4]), addr(v[2]), hlcode(v[3])) + m.emit(0x49) + m.mrsd(lcode(v[4]), addr(v[2]), 1) + m.emit((hlcode(v[1]) << 4) | imml(v[0])) + }) + } + if p.len == 0 { + panic("invalid operands for VPERMIL2PD") + } + return p +} + +// XORPS performs "Bitwise Logical XOR for Single-Precision Floating-Point Values". +// +// Mnemonic : XORPS +// Supported forms : (2 forms) +// +// - XORPS xmm, xmm [SSE] +// - XORPS m128, xmm [SSE] +func (self *Program) XORPS(v0 interface{}, v1 interface{}) *Instruction { + p := self.alloc("XORPS", 2, Operands{v0, v1}) + // XORPS xmm, xmm + if isXMM(v0) && isXMM(v1) { + self.require(ISA_SSE) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(hcode(v[1]), v[0], false) + m.emit(0x0f) + m.emit(0x57) + m.emit(0xc0 | lcode(v[1])<<3 | lcode(v[0])) + }) + } + // XORPS m128, xmm + if isM128(v0) && isXMM(v1) { + self.require(ISA_SSE) + p.domain = DomainMMXSSE + p.add(0, func(m *_Encoding, v []interface{}) { + m.rexo(hcode(v[1]), addr(v[0]), false) + m.emit(0x0f) + m.emit(0x57) + m.mrsd(lcode(v[1]), addr(v[0]), 1) + }) + } + if p.len == 0 { + panic("invalid operands for XORPS") + } + return p +} diff --git a/loader/internal/iasm/x86_64/instructions_table.go b/loader/internal/iasm/x86_64/instructions_table.go new file mode 100644 index 000000000..2becd9d00 --- /dev/null +++ b/loader/internal/iasm/x86_64/instructions_table.go @@ -0,0 +1,24 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Code generated by "mkasm_amd64.py", DO NOT EDIT. + +package x86_64 + +const ( + _N_args = 5 + _N_forms = 23 +) diff --git a/loader/internal/iasm/x86_64/instructions_test.go b/loader/internal/iasm/x86_64/instructions_test.go new file mode 100644 index 000000000..ed7f8876f --- /dev/null +++ b/loader/internal/iasm/x86_64/instructions_test.go @@ -0,0 +1,55 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "testing" + + "github.com/davecgh/go-spew/spew" +) + +func TestInstr_Encode(t *testing.T) { + m := []byte(nil) + a := CreateArch() + p := a.CreateProgram() + p.VPERMIL2PD(7, Sib(R8, R9, 1, 12345), YMM1, YMM2, YMM3).encode(&m) + spew.Dump(m) +} + +func TestInstr_EncodeSegment(t *testing.T) { + m := []byte(nil) + a := CreateArch() + p := a.CreateProgram() + p.MOVQ(Abs(0x30), RCX).GS().encode(&m) + spew.Dump(m) +} + +func BenchmarkInstr_Encode(b *testing.B) { + a := CreateArch() + m := make([]byte, 0, 16) + p := a.CreateProgram() + p.VPERMIL2PD(7, Sib(R8, R9, 1, 12345), YMM1, YMM2, YMM3).encode(&m) + p.Free() + b.SetBytes(int64(len(m))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + m = m[:0] + p = a.CreateProgram() + p.VPERMIL2PD(7, Sib(R8, R9, 1, 12345), YMM1, YMM2, YMM3).encode(&m) + p.Free() + } +} diff --git a/loader/internal/iasm/x86_64/operands.go b/loader/internal/iasm/x86_64/operands.go new file mode 100644 index 000000000..61adbf83b --- /dev/null +++ b/loader/internal/iasm/x86_64/operands.go @@ -0,0 +1,665 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "errors" + "fmt" + "math" + "reflect" + "strconv" + "strings" + "sync/atomic" +) + +// RelativeOffset represents an RIP-relative offset. +type RelativeOffset int32 + +// String implements the fmt.Stringer interface. +func (self RelativeOffset) String() string { + if self == 0 { + return "(%rip)" + } else { + return fmt.Sprintf("%d(%%rip)", self) + } +} + +// RoundingControl represents a floating-point rounding option. +type RoundingControl uint8 + +const ( + // RN_SAE represents "Round Nearest", which is the default rounding option. + RN_SAE RoundingControl = iota + + // RD_SAE represents "Round Down". + RD_SAE + + // RU_SAE represents "Round Up". + RU_SAE + + // RZ_SAE represents "Round towards Zero". + RZ_SAE +) + +var _RC_NAMES = map[RoundingControl]string{ + RN_SAE: "rn-sae", + RD_SAE: "rd-sae", + RU_SAE: "ru-sae", + RZ_SAE: "rz-sae", +} + +func (self RoundingControl) String() string { + if v, ok := _RC_NAMES[self]; ok { + return v + } else { + panic("invalid RoundingControl value") + } +} + +// ExceptionControl represents the "Suppress All Exceptions" flag. +type ExceptionControl uint8 + +const ( + // SAE represents the flag "Suppress All Exceptions" for floating point operations. + SAE ExceptionControl = iota +) + +func (ExceptionControl) String() string { + return "sae" +} + +// AddressType indicates which kind of value that an Addressable object contains. +type AddressType uint + +const ( + // None indicates the Addressable does not contain any addressable value. + None AddressType = iota + + // Memory indicates the Addressable contains a memory address. + Memory + + // Offset indicates the Addressable contains an RIP-relative offset. + Offset + + // Reference indicates the Addressable contains a label reference. + Reference +) + +// Disposable is a type of object that can be Free'd manually. +type Disposable interface { + Free() +} + +// Label represents a location within the program. +type Label struct { + refs int64 + Name string + Dest *Instruction +} + +func (self *Label) offset(p uintptr, n int) RelativeOffset { + if self.Dest == nil { + panic("unresolved label: " + self.Name) + } else { + return RelativeOffset(self.Dest.pc - p - uintptr(n)) + } +} + +// Free decreases the reference count of a Label, if the +// refcount drops to 0, the Label will be recycled. +func (self *Label) Free() { + if atomic.AddInt64(&self.refs, -1) == 0 { + //freeLabel(self) + } +} + +// String implements the fmt.Stringer interface. +func (self *Label) String() string { + if self.Dest == nil { + return fmt.Sprintf("%s(%%rip)", self.Name) + } else { + return fmt.Sprintf("%s(%%rip)@%#x", self.Name, self.Dest.pc) + } +} + +// Retain increases the reference count of a Label. +func (self *Label) Retain() *Label { + atomic.AddInt64(&self.refs, 1) + return self +} + +// Evaluate implements the interface expr.Term. +func (self *Label) Evaluate() (int64, error) { + if self.Dest != nil { + return int64(self.Dest.pc), nil + } else { + return 0, errors.New("unresolved label: " + self.Name) + } +} + +// Addressable is a union to represent an addressable operand. +type Addressable struct { + Type AddressType + Memory MemoryAddress + Offset RelativeOffset + Reference *Label +} + +// String implements the fmt.Stringer interface. +func (self *Addressable) String() string { + switch self.Type { + case None: + return "(not addressable)" + case Memory: + return self.Memory.String() + case Offset: + return self.Offset.String() + case Reference: + return self.Reference.String() + default: + return "(invalid addressable)" + } +} + +// MemoryOperand represents a memory operand for an instruction. +type MemoryOperand struct { + refs int64 + Size int + Addr Addressable + Mask RegisterMask + Masked bool + Broadcast uint8 +} + +const ( + _Sizes = 0b10000000100010111 // bit-mask for valid sizes (0, 1, 2, 4, 8, 16) +) + +func (self *MemoryOperand) isVMX(evex bool) bool { + return self.Addr.Type == Memory && self.Addr.Memory.isVMX(evex) +} + +func (self *MemoryOperand) isVMY(evex bool) bool { + return self.Addr.Type == Memory && self.Addr.Memory.isVMY(evex) +} + +func (self *MemoryOperand) isVMZ() bool { + return self.Addr.Type == Memory && self.Addr.Memory.isVMZ() +} + +func (self *MemoryOperand) isMem() bool { + if (_Sizes & (1 << self.Broadcast)) == 0 { + return false + } else if self.Addr.Type == Memory { + return self.Addr.Memory.isMem() + } else if self.Addr.Type == Offset { + return true + } else if self.Addr.Type == Reference { + return true + } else { + return false + } +} + +func (self *MemoryOperand) isSize(n int) bool { + return self.Size == 0 || self.Size == n +} + +func (self *MemoryOperand) isBroadcast(n int, b uint8) bool { + return self.Size == n && self.Broadcast == b +} + +func (self *MemoryOperand) formatMask() string { + if !self.Masked { + return "" + } else { + return self.Mask.String() + } +} + +func (self *MemoryOperand) formatBroadcast() string { + if self.Broadcast == 0 { + return "" + } else { + return fmt.Sprintf("{1to%d}", self.Broadcast) + } +} + +func (self *MemoryOperand) ensureAddrValid() { + switch self.Addr.Type { + case None: + break + case Memory: + self.Addr.Memory.EnsureValid() + case Offset: + break + case Reference: + break + default: + panic("invalid address type") + } +} + +func (self *MemoryOperand) ensureSizeValid() { + if (_Sizes & (1 << self.Size)) == 0 { + panic("invalid memory operand size") + } +} + +func (self *MemoryOperand) ensureBroadcastValid() { + if (_Sizes & (1 << self.Broadcast)) == 0 { + panic("invalid memory operand broadcast") + } +} + +// Free decreases the reference count of a MemoryOperand, if the +// refcount drops to 0, the Label will be recycled. +func (self *MemoryOperand) Free() { + if atomic.AddInt64(&self.refs, -1) == 0 { + //freeMemoryOperand(self) + } +} + +// String implements the fmt.Stringer interface. +func (self *MemoryOperand) String() string { + return self.Addr.String() + self.formatMask() + self.formatBroadcast() +} + +// Retain increases the reference count of a MemoryOperand. +func (self *MemoryOperand) Retain() *MemoryOperand { + atomic.AddInt64(&self.refs, 1) + return self +} + +// EnsureValid checks if the memory operand is valid, if not, it panics. +func (self *MemoryOperand) EnsureValid() { + self.ensureAddrValid() + self.ensureSizeValid() + self.ensureBroadcastValid() +} + +// MemoryAddress represents a memory address. +type MemoryAddress struct { + Base Register + Index Register + Scale uint8 + Displacement int32 +} + +const ( + _Scales = 0b100010111 // bit-mask for valid scales (0, 1, 2, 4, 8) +) + +func (self *MemoryAddress) isVMX(evex bool) bool { + return self.isMemBase() && (self.Index == nil || isXMM(self.Index) || (evex && isEVEXXMM(self.Index))) +} + +func (self *MemoryAddress) isVMY(evex bool) bool { + return self.isMemBase() && (self.Index == nil || isYMM(self.Index) || (evex && isEVEXYMM(self.Index))) +} + +func (self *MemoryAddress) isVMZ() bool { + return self.isMemBase() && (self.Index == nil || isZMM(self.Index)) +} + +func (self *MemoryAddress) isMem() bool { + return self.isMemBase() && (self.Index == nil || isReg64(self.Index)) +} + +func (self *MemoryAddress) isMemBase() bool { + return (self.Base == nil || isReg64(self.Base)) && // `Base` must be 64-bit if present + (self.Scale == 0) == (self.Index == nil) && // `Scale` and `Index` depends on each other + (_Scales&(1<= 2 { + sb.WriteByte(',') + sb.WriteString(strconv.Itoa(int(self.Scale))) + } + } + + /* close the bracket */ + sb.WriteByte(')') + return sb.String() +} + +// EnsureValid checks if the memory address is valid, if not, it panics. +func (self *MemoryAddress) EnsureValid() { + if !self.isMemBase() || (self.Index != nil && !isIndexable(self.Index)) { + panic("not a valid memory address") + } +} + +// Ref constructs a memory reference to a label. +func Ref(ref *Label) (v *MemoryOperand) { + v = CreateMemoryOperand() + v.Addr.Type = Reference + v.Addr.Reference = ref + return +} + +// Abs construct a simple memory address that represents absolute addressing. +func Abs(disp int32) *MemoryOperand { + return Sib(nil, nil, 0, disp) +} + +// Ptr constructs a simple memory operand with base and displacement. +func Ptr(base Register, disp int32) *MemoryOperand { + return Sib(base, nil, 0, disp) +} + +// Sib constructs a simple memory operand that represents a complete memory address. +func Sib(base Register, index Register, scale uint8, disp int32) (v *MemoryOperand) { + v = CreateMemoryOperand() + v.Addr.Type = Memory + v.Addr.Memory.Base = base + v.Addr.Memory.Index = index + v.Addr.Memory.Scale = scale + v.Addr.Memory.Displacement = disp + v.EnsureValid() + return +} + +/** Operand Matching Helpers **/ + +const _IntMask = (1 << reflect.Int) | + (1 << reflect.Int8) | + (1 << reflect.Int16) | + (1 << reflect.Int32) | + (1 << reflect.Int64) | + (1 << reflect.Uint) | + (1 << reflect.Uint8) | + (1 << reflect.Uint16) | + (1 << reflect.Uint32) | + (1 << reflect.Uint64) | + (1 << reflect.Uintptr) + +func isInt(k reflect.Kind) bool { + return (_IntMask & (1 << k)) != 0 +} + +func asInt64(v interface{}) (int64, bool) { + if isSpecial(v) { + return 0, false + } else if x := efaceOf(v); isInt(x.kind()) { + return x.toInt64(), true + } else { + return 0, false + } +} + +func inRange(v interface{}, low int64, high int64) bool { + x, ok := asInt64(v) + return ok && x >= low && x <= high +} + +func isSpecial(v interface{}) bool { + switch v.(type) { + case Register8: + return true + case Register16: + return true + case Register32: + return true + case Register64: + return true + case KRegister: + return true + case MMRegister: + return true + case XMMRegister: + return true + case YMMRegister: + return true + case ZMMRegister: + return true + case RelativeOffset: + return true + case RoundingControl: + return true + case ExceptionControl: + return true + default: + return false + } +} + +func isIndexable(v interface{}) bool { + return isZMM(v) || isReg64(v) || isEVEXXMM(v) || isEVEXYMM(v) +} + +func isImm4(v interface{}) bool { return inRange(v, 0, 15) } +func isImm8(v interface{}) bool { return inRange(v, math.MinInt8, math.MaxUint8) } +func isImm16(v interface{}) bool { return inRange(v, math.MinInt16, math.MaxUint16) } +func isImm32(v interface{}) bool { return inRange(v, math.MinInt32, math.MaxUint32) } +func isImm64(v interface{}) bool { _, r := asInt64(v); return r } +func isConst1(v interface{}) bool { x, r := asInt64(v); return r && x == 1 } +func isConst3(v interface{}) bool { x, r := asInt64(v); return r && x == 3 } +func isRel8(v interface{}) bool { + x, r := v.(RelativeOffset) + return r && x >= math.MinInt8 && x <= math.MaxInt8 +} +func isRel32(v interface{}) bool { _, r := v.(RelativeOffset); return r } +func isLabel(v interface{}) bool { _, r := v.(*Label); return r } +func isReg8(v interface{}) bool { _, r := v.(Register8); return r } +func isReg8REX(v interface{}) bool { + x, r := v.(Register8) + return r && (x&0x80) == 0 && x >= SPL +} +func isReg16(v interface{}) bool { _, r := v.(Register16); return r } +func isReg32(v interface{}) bool { _, r := v.(Register32); return r } +func isReg64(v interface{}) bool { _, r := v.(Register64); return r } +func isMM(v interface{}) bool { _, r := v.(MMRegister); return r } +func isXMM(v interface{}) bool { x, r := v.(XMMRegister); return r && x <= XMM15 } +func isEVEXXMM(v interface{}) bool { _, r := v.(XMMRegister); return r } +func isXMMk(v interface{}) bool { + x, r := v.(MaskedRegister) + return isXMM(v) || (r && isXMM(x.Reg) && !x.Mask.Z) +} +func isXMMkz(v interface{}) bool { + x, r := v.(MaskedRegister) + return isXMM(v) || (r && isXMM(x.Reg)) +} +func isYMM(v interface{}) bool { x, r := v.(YMMRegister); return r && x <= YMM15 } +func isEVEXYMM(v interface{}) bool { _, r := v.(YMMRegister); return r } +func isYMMk(v interface{}) bool { + x, r := v.(MaskedRegister) + return isYMM(v) || (r && isYMM(x.Reg) && !x.Mask.Z) +} +func isYMMkz(v interface{}) bool { + x, r := v.(MaskedRegister) + return isYMM(v) || (r && isYMM(x.Reg)) +} +func isZMM(v interface{}) bool { _, r := v.(ZMMRegister); return r } +func isZMMk(v interface{}) bool { + x, r := v.(MaskedRegister) + return isZMM(v) || (r && isZMM(x.Reg) && !x.Mask.Z) +} +func isZMMkz(v interface{}) bool { + x, r := v.(MaskedRegister) + return isZMM(v) || (r && isZMM(x.Reg)) +} +func isK(v interface{}) bool { _, r := v.(KRegister); return r } +func isKk(v interface{}) bool { + x, r := v.(MaskedRegister) + return isK(v) || (r && isK(x.Reg) && !x.Mask.Z) +} +func isM(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && x.isMem() && x.Broadcast == 0 && !x.Masked +} +func isMk(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && x.isMem() && x.Broadcast == 0 && !(x.Masked && x.Mask.Z) +} +func isMkz(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && x.isMem() && x.Broadcast == 0 +} +func isM8(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isM(v) && x.isSize(1) +} +func isM16(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isM(v) && x.isSize(2) +} +func isM16kz(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isMkz(v) && x.isSize(2) +} +func isM32(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isM(v) && x.isSize(4) +} +func isM32k(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isMk(v) && x.isSize(4) +} +func isM32kz(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isMkz(v) && x.isSize(4) +} +func isM64(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isM(v) && x.isSize(8) +} +func isM64k(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isMk(v) && x.isSize(8) +} +func isM64kz(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isMkz(v) && x.isSize(8) +} +func isM128(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isM(v) && x.isSize(16) +} +func isM128kz(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isMkz(v) && x.isSize(16) +} +func isM256(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isM(v) && x.isSize(32) +} +func isM256kz(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isMkz(v) && x.isSize(32) +} +func isM512(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isM(v) && x.isSize(64) +} +func isM512kz(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && isMkz(v) && x.isSize(64) +} +func isM64M32bcst(v interface{}) bool { + x, r := v.(*MemoryOperand) + return isM64(v) || (r && x.isBroadcast(4, 2)) +} +func isM128M32bcst(v interface{}) bool { + x, r := v.(*MemoryOperand) + return isM128(v) || (r && x.isBroadcast(4, 4)) +} +func isM256M32bcst(v interface{}) bool { + x, r := v.(*MemoryOperand) + return isM256(v) || (r && x.isBroadcast(4, 8)) +} +func isM512M32bcst(v interface{}) bool { + x, r := v.(*MemoryOperand) + return isM512(v) || (r && x.isBroadcast(4, 16)) +} +func isM128M64bcst(v interface{}) bool { + x, r := v.(*MemoryOperand) + return isM128(v) || (r && x.isBroadcast(8, 2)) +} +func isM256M64bcst(v interface{}) bool { + x, r := v.(*MemoryOperand) + return isM256(v) || (r && x.isBroadcast(8, 4)) +} +func isM512M64bcst(v interface{}) bool { + x, r := v.(*MemoryOperand) + return isM512(v) || (r && x.isBroadcast(8, 8)) +} +func isVMX(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && x.isVMX(false) && !x.Masked +} +func isEVEXVMX(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && x.isVMX(true) && !x.Masked +} +func isVMXk(v interface{}) bool { x, r := v.(*MemoryOperand); return r && x.isVMX(true) } +func isVMY(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && x.isVMY(false) && !x.Masked +} +func isEVEXVMY(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && x.isVMY(true) && !x.Masked +} +func isVMYk(v interface{}) bool { x, r := v.(*MemoryOperand); return r && x.isVMY(true) } +func isVMZ(v interface{}) bool { + x, r := v.(*MemoryOperand) + return r && x.isVMZ() && !x.Masked +} +func isVMZk(v interface{}) bool { x, r := v.(*MemoryOperand); return r && x.isVMZ() } +func isSAE(v interface{}) bool { _, r := v.(ExceptionControl); return r } +func isER(v interface{}) bool { _, r := v.(RoundingControl); return r } + +func isImmExt(v interface{}, ext int, min int64, max int64) bool { + if x, ok := asInt64(v); !ok { + return false + } else if m := int64(1) << (8 * ext); x < m && x >= m+min { + return true + } else { + return x <= max && x >= min + } +} + +func isImm8Ext(v interface{}, ext int) bool { + return isImmExt(v, ext, math.MinInt8, math.MaxInt8) +} + +func isImm32Ext(v interface{}, ext int) bool { + return isImmExt(v, ext, math.MinInt32, math.MaxInt32) +} diff --git a/loader/internal/iasm/x86_64/pools.go b/loader/internal/iasm/x86_64/pools.go new file mode 100644 index 000000000..690db5b7a --- /dev/null +++ b/loader/internal/iasm/x86_64/pools.go @@ -0,0 +1,54 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +// CreateLabel creates a new Label, it may allocate a new one or grab one from a pool. +func CreateLabel(name string) *Label { + p := new(Label) + + /* initialize the label */ + p.refs = 1 + p.Name = name + return p +} + +func newProgram(arch *Arch) *Program { + p := new(Program) + + /* initialize the program */ + p.arch = arch + return p +} + +func newInstruction(name string, argc int, argv Operands) *Instruction { + p := new(Instruction) + + /* initialize the instruction */ + p.name = name + p.argc = argc + p.argv = argv + return p +} + +// CreateMemoryOperand creates a new MemoryOperand, it may allocate a new one or grab one from a pool. +func CreateMemoryOperand() *MemoryOperand { + p := new(MemoryOperand) + + /* initialize the memory operand */ + p.refs = 1 + return p +} diff --git a/loader/internal/iasm/x86_64/program.go b/loader/internal/iasm/x86_64/program.go new file mode 100644 index 000000000..bf7d3a1dc --- /dev/null +++ b/loader/internal/iasm/x86_64/program.go @@ -0,0 +1,584 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "fmt" + "math" + "math/bits" + + "github.com/bytedance/sonic/loader/internal/iasm/expr" +) + +type ( + _PseudoType int + _InstructionEncoder func(*Program, ...interface{}) *Instruction +) + +const ( + _PseudoNop _PseudoType = iota + 1 + _PseudoByte + _PseudoWord + _PseudoLong + _PseudoQuad + _PseudoData + _PseudoAlign +) + +func (self _PseudoType) String() string { + switch self { + case _PseudoNop: + return ".nop" + case _PseudoByte: + return ".byte" + case _PseudoWord: + return ".word" + case _PseudoLong: + return ".long" + case _PseudoQuad: + return ".quad" + case _PseudoData: + return ".data" + case _PseudoAlign: + return ".align" + default: + panic("unreachable") + } +} + +type _Pseudo struct { + kind _PseudoType + data []byte + uint uint64 + expr *expr.Expr +} + +func (self *_Pseudo) free() { + if self.expr != nil { + self.expr.Free() + } +} + +func (self *_Pseudo) encode(m *[]byte, pc uintptr) int { + switch self.kind { + case _PseudoNop: + return 0 + case _PseudoByte: + self.encodeByte(m) + return 1 + case _PseudoWord: + self.encodeWord(m) + return 2 + case _PseudoLong: + self.encodeLong(m) + return 4 + case _PseudoQuad: + self.encodeQuad(m) + return 8 + case _PseudoData: + self.encodeData(m) + return len(self.data) + case _PseudoAlign: + self.encodeAlign(m, pc) + return self.alignSize(pc) + default: + panic("invalid pseudo instruction") + } +} + +func (self *_Pseudo) evalExpr(low int64, high int64) int64 { + if v, err := self.expr.Evaluate(); err != nil { + panic(err) + } else if v < low || v > high { + panic(fmt.Sprintf("expression out of range [%d, %d]: %d", low, high, v)) + } else { + return v + } +} + +func (self *_Pseudo) alignSize(pc uintptr) int { + if !ispow2(self.uint) { + panic(fmt.Sprintf("aligment should be a power of 2, not %d", self.uint)) + } else { + return align(int(pc), bits.TrailingZeros64(self.uint)) - int(pc) + } +} + +func (self *_Pseudo) encodeData(m *[]byte) { + if m != nil { + *m = append(*m, self.data...) + } +} + +func (self *_Pseudo) encodeByte(m *[]byte) { + if m != nil { + append8(m, byte(self.evalExpr(math.MinInt8, math.MaxUint8))) + } +} + +func (self *_Pseudo) encodeWord(m *[]byte) { + if m != nil { + append16(m, uint16(self.evalExpr(math.MinInt16, math.MaxUint16))) + } +} + +func (self *_Pseudo) encodeLong(m *[]byte) { + if m != nil { + append32(m, uint32(self.evalExpr(math.MinInt32, math.MaxUint32))) + } +} + +func (self *_Pseudo) encodeQuad(m *[]byte) { + if m != nil { + if v, err := self.expr.Evaluate(); err != nil { + panic(err) + } else { + append64(m, uint64(v)) + } + } +} + +func (self *_Pseudo) encodeAlign(m *[]byte, pc uintptr) { + if m != nil { + if self.expr == nil { + expandmm(m, self.alignSize(pc), 0) + } else { + expandmm(m, self.alignSize(pc), byte(self.evalExpr(math.MinInt8, math.MaxUint8))) + } + } +} + +// Operands represents a sequence of operand required by an instruction. +type Operands [_N_args]interface{} + +// InstructionDomain represents the domain of an instruction. +type InstructionDomain uint8 + +const ( + DomainGeneric InstructionDomain = iota + DomainMMXSSE + DomainAVX + DomainFMA + DomainCrypto + DomainMask + DomainAMDSpecific + DomainMisc + DomainPseudo +) + +type ( + _BranchType uint8 +) + +const ( + _B_none _BranchType = iota + _B_conditional + _B_unconditional +) + +// Instruction represents an unencoded instruction. +type Instruction struct { + next *Instruction + pc uintptr + nb int + len int + argc int + name string + argv Operands + forms [_N_forms]_Encoding + pseudo _Pseudo + branch _BranchType + domain InstructionDomain + prefix []byte +} + +func (self *Instruction) add(flags int, encoder func(m *_Encoding, v []interface{})) { + self.forms[self.len].flags = flags + self.forms[self.len].encoder = encoder + self.len++ +} + +func (self *Instruction) free() { + self.clear() + self.pseudo.free() + //freeInstruction(self) +} + +func (self *Instruction) clear() { + for i := 0; i < self.argc; i++ { + if v, ok := self.argv[i].(Disposable); ok { + v.Free() + } + } +} + +func (self *Instruction) check(e *_Encoding) bool { + if (e.flags & _F_rel1) != 0 { + return isRel8(self.argv[0]) + } else if (e.flags & _F_rel4) != 0 { + return isRel32(self.argv[0]) || isLabel(self.argv[0]) + } else { + return true + } +} + +func (self *Instruction) encode(m *[]byte) int { + n := math.MaxInt64 + p := (*_Encoding)(nil) + + /* encode prefixes if any */ + if self.nb = len(self.prefix); m != nil { + *m = append(*m, self.prefix...) + } + + /* check for pseudo-instructions */ + if self.pseudo.kind != 0 { + self.nb += self.pseudo.encode(m, self.pc) + return self.nb + } + + /* find the shortest encoding */ + for i := 0; i < self.len; i++ { + if e := &self.forms[i]; self.check(e) { + if v := e.encode(self.argv[:self.argc]); v < n { + n = v + p = e + } + } + } + + /* add to buffer if needed */ + if m != nil { + *m = append(*m, p.bytes[:n]...) + } + + /* update the instruction length */ + self.nb += n + return self.nb +} + +/** Instruction Prefixes **/ + +const ( + _P_cs = 0x2e + _P_ds = 0x3e + _P_es = 0x26 + _P_fs = 0x64 + _P_gs = 0x65 + _P_ss = 0x36 + _P_lock = 0xf0 +) + +// CS overrides the memory operation of this instruction to CS. +func (self *Instruction) CS() *Instruction { + self.prefix = append(self.prefix, _P_cs) + return self +} + +// DS overrides the memory operation of this instruction to DS, +// this is the default section for most instructions if not specified. +func (self *Instruction) DS() *Instruction { + self.prefix = append(self.prefix, _P_ds) + return self +} + +// ES overrides the memory operation of this instruction to ES. +func (self *Instruction) ES() *Instruction { + self.prefix = append(self.prefix, _P_es) + return self +} + +// FS overrides the memory operation of this instruction to FS. +func (self *Instruction) FS() *Instruction { + self.prefix = append(self.prefix, _P_fs) + return self +} + +// GS overrides the memory operation of this instruction to GS. +func (self *Instruction) GS() *Instruction { + self.prefix = append(self.prefix, _P_gs) + return self +} + +// SS overrides the memory operation of this instruction to SS. +func (self *Instruction) SS() *Instruction { + self.prefix = append(self.prefix, _P_ss) + return self +} + +// LOCK causes the processor's LOCK# signal to be asserted during execution of +// the accompanying instruction (turns the instruction into an atomic instruction). +// In a multiprocessor environment, the LOCK# signal insures that the processor +// has exclusive use of any shared memory while the signal is asserted. +func (self *Instruction) LOCK() *Instruction { + self.prefix = append(self.prefix, _P_lock) + return self +} + +/** Basic Instruction Properties **/ + +// Name returns the instruction name. +func (self *Instruction) Name() string { + return self.name +} + +// Domain returns the domain of this instruction. +func (self *Instruction) Domain() InstructionDomain { + return self.domain +} + +// Operands returns the operands of this instruction. +func (self *Instruction) Operands() []interface{} { + return self.argv[:self.argc] +} + +// Program represents a sequence of instructions. +type Program struct { + arch *Arch + head *Instruction + tail *Instruction +} + +const ( + _N_near = 2 // near-branch (-128 ~ +127) takes 2 bytes to encode + _N_far_cond = 6 // conditional far-branch takes 6 bytes to encode + _N_far_uncond = 5 // unconditional far-branch takes 5 bytes to encode +) + +func (self *Program) clear() { + for p, q := self.head, self.head; p != nil; p = q { + q = p.next + p.free() + } +} + +func (self *Program) alloc(name string, argc int, argv Operands) *Instruction { + p := self.tail + q := newInstruction(name, argc, argv) + + /* attach to tail if any */ + if p != nil { + p.next = q + } else { + self.head = q + } + + /* set the new tail */ + self.tail = q + return q +} + +func (self *Program) pseudo(kind _PseudoType) (p *Instruction) { + p = self.alloc(kind.String(), 0, Operands{}) + p.domain = DomainPseudo + p.pseudo.kind = kind + return +} + +func (self *Program) require(isa ISA) { + if !self.arch.HasISA(isa) { + panic("ISA '" + isa.String() + "' was not enabled") + } +} + +func (self *Program) branchSize(p *Instruction) int { + switch p.branch { + case _B_none: + panic("p is not a branch") + case _B_conditional: + return _N_far_cond + case _B_unconditional: + return _N_far_uncond + default: + panic("invalid instruction") + } +} + +/** Pseudo-Instructions **/ + +// Byte is a pseudo-instruction to add raw byte to the assembled code. +func (self *Program) Byte(v *expr.Expr) (p *Instruction) { + p = self.pseudo(_PseudoByte) + p.pseudo.expr = v + return +} + +// Word is a pseudo-instruction to add raw uint16 as little-endian to the assembled code. +func (self *Program) Word(v *expr.Expr) (p *Instruction) { + p = self.pseudo(_PseudoWord) + p.pseudo.expr = v + return +} + +// Long is a pseudo-instruction to add raw uint32 as little-endian to the assembled code. +func (self *Program) Long(v *expr.Expr) (p *Instruction) { + p = self.pseudo(_PseudoLong) + p.pseudo.expr = v + return +} + +// Quad is a pseudo-instruction to add raw uint64 as little-endian to the assembled code. +func (self *Program) Quad(v *expr.Expr) (p *Instruction) { + p = self.pseudo(_PseudoQuad) + p.pseudo.expr = v + return +} + +// Data is a pseudo-instruction to add raw bytes to the assembled code. +func (self *Program) Data(v []byte) (p *Instruction) { + p = self.pseudo(_PseudoData) + p.pseudo.data = v + return +} + +// Align is a pseudo-instruction to ensure the PC is aligned to a certain value. +func (self *Program) Align(align uint64, padding *expr.Expr) (p *Instruction) { + p = self.pseudo(_PseudoAlign) + p.pseudo.uint = align + p.pseudo.expr = padding + return +} + +/** Program Assembler **/ + +// Free returns the Program object into pool. +// Any operation performed after Free is undefined behavior. +// +// NOTE: This also frees all the instructions, labels, memory +// +// operands and expressions associated with this program. +func (self *Program) Free() { + self.clear() + //freeProgram(self) +} + +// Link pins a label at the current position. +func (self *Program) Link(p *Label) { + if p.Dest != nil { + panic("lable was alreay linked") + } else { + p.Dest = self.pseudo(_PseudoNop) + } +} + +// Assemble assembles and links the entire program into machine code. +func (self *Program) Assemble(pc uintptr) (ret []byte) { + orig := pc + next := true + offs := uintptr(0) + + /* Pass 0: PC-precompute, assume all labeled branches are far-branches. */ + for p := self.head; p != nil; p = p.next { + if p.pc = pc; !isLabel(p.argv[0]) || p.branch == _B_none { + pc += uintptr(p.encode(nil)) + } else { + pc += uintptr(self.branchSize(p)) + } + } + + /* allocate space for the machine code */ + nb := int(pc - orig) + ret = make([]byte, 0, nb) + + /* Pass 1: adjust all the jumps */ + for next { + next = false + offs = uintptr(0) + + /* scan all the branches */ + for p := self.head; p != nil; p = p.next { + var ok bool + var lb *Label + + /* re-calculate the alignment here */ + if nb = p.nb; p.pseudo.kind == _PseudoAlign { + p.pc -= offs + offs += uintptr(nb - p.encode(nil)) + continue + } + + /* adjust the program counter */ + p.pc -= offs + lb, ok = p.argv[0].(*Label) + + /* only care about labeled far-branches */ + if !ok || p.nb == _N_near || p.branch == _B_none { + continue + } + + /* calculate the jump offset */ + size := self.branchSize(p) + diff := lb.offset(p.pc, size) + + /* too far to be a near jump */ + if diff > 127 || diff < -128 { + p.nb = size + continue + } + + /* a far jump becomes a near jump, calculate + * the PC adjustment value and assemble again */ + next = true + p.nb = _N_near + offs += uintptr(size - _N_near) + } + } + + /* Pass 3: link all the cross-references */ + for p := self.head; p != nil; p = p.next { + for i := 0; i < p.argc; i++ { + var ok bool + var lb *Label + var op *MemoryOperand + + /* resolve labels */ + if lb, ok = p.argv[i].(*Label); ok { + p.argv[i] = lb.offset(p.pc, p.nb) + continue + } + + /* check for memory operands */ + if op, ok = p.argv[i].(*MemoryOperand); !ok { + continue + } + + /* check for label references */ + if op.Addr.Type != Reference { + continue + } + + /* replace the label with the real offset */ + op.Addr.Type = Offset + op.Addr.Offset = op.Addr.Reference.offset(p.pc, p.nb) + } + } + + /* Pass 4: actually encode all the instructions */ + for p := self.head; p != nil; p = p.next { + p.encode(&ret) + } + + /* all done */ + return ret +} + +// AssembleAndFree is like Assemble, but it frees the Program after assembling. +func (self *Program) AssembleAndFree(pc uintptr) (ret []byte) { + ret = self.Assemble(pc) + self.Free() + return +} diff --git a/loader/internal/iasm/x86_64/program_test.go b/loader/internal/iasm/x86_64/program_test.go new file mode 100644 index 000000000..c31d87f19 --- /dev/null +++ b/loader/internal/iasm/x86_64/program_test.go @@ -0,0 +1,49 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "bytes" + "testing" + + "github.com/bytedance/sonic/loader/internal/iasm/expr" + "github.com/davecgh/go-spew/spew" +) + +func TestProgram_Assemble(t *testing.T) { + a := CreateArch() + b := CreateLabel("bak") + s := CreateLabel("tab") + j := CreateLabel("jmp") + p := a.CreateProgram() + p.JMP(j) + p.JMP(j) + p.Link(b) + p.Data(bytes.Repeat([]byte{0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00}, 15)) + p.Data([]byte{0x0f, 0x1f, 0x00}) + p.JMP(b) + p.Link(j) + p.LEAQ(Ref(s), RDI) + p.MOVSLQ(Sib(RDI, RAX, 4, -4), RAX) + p.ADDQ(RDI, RAX) + p.JMPQ(RAX) + p.Align(32, expr.Int(0xcc)) + p.Link(s) + p.Long(expr.Ref(s.Retain()).Sub(expr.Ref(j.Retain()))) + p.Long(expr.Ref(s.Retain()).Sub(expr.Ref(b.Retain()))) + spew.Dump(p.AssembleAndFree(0)) +} diff --git a/loader/internal/iasm/x86_64/registers.go b/loader/internal/iasm/x86_64/registers.go new file mode 100644 index 000000000..265575a26 --- /dev/null +++ b/loader/internal/iasm/x86_64/registers.go @@ -0,0 +1,747 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "fmt" +) + +// Register represents a hardware register. +type Register interface { + fmt.Stringer + implRegister() +} + +type ( + Register8 byte + Register16 byte + Register32 byte + Register64 byte +) + +type ( + KRegister byte + MMRegister byte + XMMRegister byte + YMMRegister byte + ZMMRegister byte +) + +// RegisterMask is a KRegister used to mask another register. +type RegisterMask struct { + Z bool + K KRegister +} + +// String implements the fmt.Stringer interface. +func (self RegisterMask) String() string { + if !self.Z { + return fmt.Sprintf("{%%%s}", self.K) + } else { + return fmt.Sprintf("{%%%s}{z}", self.K) + } +} + +// MaskedRegister is a Register masked by a RegisterMask. +type MaskedRegister struct { + Reg Register + Mask RegisterMask +} + +// String implements the fmt.Stringer interface. +func (self MaskedRegister) String() string { + return self.Reg.String() + self.Mask.String() +} + +const ( + AL Register8 = iota + CL + DL + BL + SPL + BPL + SIL + DIL + R8b + R9b + R10b + R11b + R12b + R13b + R14b + R15b +) + +const ( + AH = SPL | 0x80 + CH = BPL | 0x80 + DH = SIL | 0x80 + BH = DIL | 0x80 +) + +const ( + AX Register16 = iota + CX + DX + BX + SP + BP + SI + DI + R8w + R9w + R10w + R11w + R12w + R13w + R14w + R15w +) + +const ( + EAX Register32 = iota + ECX + EDX + EBX + ESP + EBP + ESI + EDI + R8d + R9d + R10d + R11d + R12d + R13d + R14d + R15d +) + +const ( + RAX Register64 = iota + RCX + RDX + RBX + RSP + RBP + RSI + RDI + R8 + R9 + R10 + R11 + R12 + R13 + R14 + R15 +) + +const ( + K0 KRegister = iota + K1 + K2 + K3 + K4 + K5 + K6 + K7 +) + +const ( + MM0 MMRegister = iota + MM1 + MM2 + MM3 + MM4 + MM5 + MM6 + MM7 +) + +const ( + XMM0 XMMRegister = iota + XMM1 + XMM2 + XMM3 + XMM4 + XMM5 + XMM6 + XMM7 + XMM8 + XMM9 + XMM10 + XMM11 + XMM12 + XMM13 + XMM14 + XMM15 + XMM16 + XMM17 + XMM18 + XMM19 + XMM20 + XMM21 + XMM22 + XMM23 + XMM24 + XMM25 + XMM26 + XMM27 + XMM28 + XMM29 + XMM30 + XMM31 +) + +const ( + YMM0 YMMRegister = iota + YMM1 + YMM2 + YMM3 + YMM4 + YMM5 + YMM6 + YMM7 + YMM8 + YMM9 + YMM10 + YMM11 + YMM12 + YMM13 + YMM14 + YMM15 + YMM16 + YMM17 + YMM18 + YMM19 + YMM20 + YMM21 + YMM22 + YMM23 + YMM24 + YMM25 + YMM26 + YMM27 + YMM28 + YMM29 + YMM30 + YMM31 +) + +const ( + ZMM0 ZMMRegister = iota + ZMM1 + ZMM2 + ZMM3 + ZMM4 + ZMM5 + ZMM6 + ZMM7 + ZMM8 + ZMM9 + ZMM10 + ZMM11 + ZMM12 + ZMM13 + ZMM14 + ZMM15 + ZMM16 + ZMM17 + ZMM18 + ZMM19 + ZMM20 + ZMM21 + ZMM22 + ZMM23 + ZMM24 + ZMM25 + ZMM26 + ZMM27 + ZMM28 + ZMM29 + ZMM30 + ZMM31 +) + +func (self Register8) implRegister() {} +func (self Register16) implRegister() {} +func (self Register32) implRegister() {} +func (self Register64) implRegister() {} + +func (self KRegister) implRegister() {} +func (self MMRegister) implRegister() {} +func (self XMMRegister) implRegister() {} +func (self YMMRegister) implRegister() {} +func (self ZMMRegister) implRegister() {} + +func (self Register8) String() string { + if int(self) >= len(r8names) { + return "???" + } else { + return r8names[self] + } +} +func (self Register16) String() string { + if int(self) >= len(r16names) { + return "???" + } else { + return r16names[self] + } +} +func (self Register32) String() string { + if int(self) >= len(r32names) { + return "???" + } else { + return r32names[self] + } +} +func (self Register64) String() string { + if int(self) >= len(r64names) { + return "???" + } else { + return r64names[self] + } +} + +func (self KRegister) String() string { + if int(self) >= len(knames) { + return "???" + } else { + return knames[self] + } +} +func (self MMRegister) String() string { + if int(self) >= len(mmnames) { + return "???" + } else { + return mmnames[self] + } +} +func (self XMMRegister) String() string { + if int(self) >= len(xmmnames) { + return "???" + } else { + return xmmnames[self] + } +} +func (self YMMRegister) String() string { + if int(self) >= len(ymmnames) { + return "???" + } else { + return ymmnames[self] + } +} +func (self ZMMRegister) String() string { + if int(self) >= len(zmmnames) { + return "???" + } else { + return zmmnames[self] + } +} + +// Registers maps register name into Register instances. +var Registers = map[string]Register{ + "al": AL, + "cl": CL, + "dl": DL, + "bl": BL, + "spl": SPL, + "bpl": BPL, + "sil": SIL, + "dil": DIL, + "r8b": R8b, + "r9b": R9b, + "r10b": R10b, + "r11b": R11b, + "r12b": R12b, + "r13b": R13b, + "r14b": R14b, + "r15b": R15b, + "ah": AH, + "ch": CH, + "dh": DH, + "bh": BH, + "ax": AX, + "cx": CX, + "dx": DX, + "bx": BX, + "sp": SP, + "bp": BP, + "si": SI, + "di": DI, + "r8w": R8w, + "r9w": R9w, + "r10w": R10w, + "r11w": R11w, + "r12w": R12w, + "r13w": R13w, + "r14w": R14w, + "r15w": R15w, + "eax": EAX, + "ecx": ECX, + "edx": EDX, + "ebx": EBX, + "esp": ESP, + "ebp": EBP, + "esi": ESI, + "edi": EDI, + "r8d": R8d, + "r9d": R9d, + "r10d": R10d, + "r11d": R11d, + "r12d": R12d, + "r13d": R13d, + "r14d": R14d, + "r15d": R15d, + "rax": RAX, + "rcx": RCX, + "rdx": RDX, + "rbx": RBX, + "rsp": RSP, + "rbp": RBP, + "rsi": RSI, + "rdi": RDI, + "r8": R8, + "r9": R9, + "r10": R10, + "r11": R11, + "r12": R12, + "r13": R13, + "r14": R14, + "r15": R15, + "k0": K0, + "k1": K1, + "k2": K2, + "k3": K3, + "k4": K4, + "k5": K5, + "k6": K6, + "k7": K7, + "mm0": MM0, + "mm1": MM1, + "mm2": MM2, + "mm3": MM3, + "mm4": MM4, + "mm5": MM5, + "mm6": MM6, + "mm7": MM7, + "xmm0": XMM0, + "xmm1": XMM1, + "xmm2": XMM2, + "xmm3": XMM3, + "xmm4": XMM4, + "xmm5": XMM5, + "xmm6": XMM6, + "xmm7": XMM7, + "xmm8": XMM8, + "xmm9": XMM9, + "xmm10": XMM10, + "xmm11": XMM11, + "xmm12": XMM12, + "xmm13": XMM13, + "xmm14": XMM14, + "xmm15": XMM15, + "xmm16": XMM16, + "xmm17": XMM17, + "xmm18": XMM18, + "xmm19": XMM19, + "xmm20": XMM20, + "xmm21": XMM21, + "xmm22": XMM22, + "xmm23": XMM23, + "xmm24": XMM24, + "xmm25": XMM25, + "xmm26": XMM26, + "xmm27": XMM27, + "xmm28": XMM28, + "xmm29": XMM29, + "xmm30": XMM30, + "xmm31": XMM31, + "ymm0": YMM0, + "ymm1": YMM1, + "ymm2": YMM2, + "ymm3": YMM3, + "ymm4": YMM4, + "ymm5": YMM5, + "ymm6": YMM6, + "ymm7": YMM7, + "ymm8": YMM8, + "ymm9": YMM9, + "ymm10": YMM10, + "ymm11": YMM11, + "ymm12": YMM12, + "ymm13": YMM13, + "ymm14": YMM14, + "ymm15": YMM15, + "ymm16": YMM16, + "ymm17": YMM17, + "ymm18": YMM18, + "ymm19": YMM19, + "ymm20": YMM20, + "ymm21": YMM21, + "ymm22": YMM22, + "ymm23": YMM23, + "ymm24": YMM24, + "ymm25": YMM25, + "ymm26": YMM26, + "ymm27": YMM27, + "ymm28": YMM28, + "ymm29": YMM29, + "ymm30": YMM30, + "ymm31": YMM31, + "zmm0": ZMM0, + "zmm1": ZMM1, + "zmm2": ZMM2, + "zmm3": ZMM3, + "zmm4": ZMM4, + "zmm5": ZMM5, + "zmm6": ZMM6, + "zmm7": ZMM7, + "zmm8": ZMM8, + "zmm9": ZMM9, + "zmm10": ZMM10, + "zmm11": ZMM11, + "zmm12": ZMM12, + "zmm13": ZMM13, + "zmm14": ZMM14, + "zmm15": ZMM15, + "zmm16": ZMM16, + "zmm17": ZMM17, + "zmm18": ZMM18, + "zmm19": ZMM19, + "zmm20": ZMM20, + "zmm21": ZMM21, + "zmm22": ZMM22, + "zmm23": ZMM23, + "zmm24": ZMM24, + "zmm25": ZMM25, + "zmm26": ZMM26, + "zmm27": ZMM27, + "zmm28": ZMM28, + "zmm29": ZMM29, + "zmm30": ZMM30, + "zmm31": ZMM31, +} + +/** Register Name Tables **/ + +var r8names = [...]string{ + AL: "al", + CL: "cl", + DL: "dl", + BL: "bl", + SPL: "spl", + BPL: "bpl", + SIL: "sil", + DIL: "dil", + R8b: "r8b", + R9b: "r9b", + R10b: "r10b", + R11b: "r11b", + R12b: "r12b", + R13b: "r13b", + R14b: "r14b", + R15b: "r15b", + AH: "ah", + CH: "ch", + DH: "dh", + BH: "bh", +} + +var r16names = [...]string{ + AX: "ax", + CX: "cx", + DX: "dx", + BX: "bx", + SP: "sp", + BP: "bp", + SI: "si", + DI: "di", + R8w: "r8w", + R9w: "r9w", + R10w: "r10w", + R11w: "r11w", + R12w: "r12w", + R13w: "r13w", + R14w: "r14w", + R15w: "r15w", +} + +var r32names = [...]string{ + EAX: "eax", + ECX: "ecx", + EDX: "edx", + EBX: "ebx", + ESP: "esp", + EBP: "ebp", + ESI: "esi", + EDI: "edi", + R8d: "r8d", + R9d: "r9d", + R10d: "r10d", + R11d: "r11d", + R12d: "r12d", + R13d: "r13d", + R14d: "r14d", + R15d: "r15d", +} + +var r64names = [...]string{ + RAX: "rax", + RCX: "rcx", + RDX: "rdx", + RBX: "rbx", + RSP: "rsp", + RBP: "rbp", + RSI: "rsi", + RDI: "rdi", + R8: "r8", + R9: "r9", + R10: "r10", + R11: "r11", + R12: "r12", + R13: "r13", + R14: "r14", + R15: "r15", +} + +var knames = [...]string{ + K0: "k0", + K1: "k1", + K2: "k2", + K3: "k3", + K4: "k4", + K5: "k5", + K6: "k6", + K7: "k7", +} + +var mmnames = [...]string{ + MM0: "mm0", + MM1: "mm1", + MM2: "mm2", + MM3: "mm3", + MM4: "mm4", + MM5: "mm5", + MM6: "mm6", + MM7: "mm7", +} + +var xmmnames = [...]string{ + XMM0: "xmm0", + XMM1: "xmm1", + XMM2: "xmm2", + XMM3: "xmm3", + XMM4: "xmm4", + XMM5: "xmm5", + XMM6: "xmm6", + XMM7: "xmm7", + XMM8: "xmm8", + XMM9: "xmm9", + XMM10: "xmm10", + XMM11: "xmm11", + XMM12: "xmm12", + XMM13: "xmm13", + XMM14: "xmm14", + XMM15: "xmm15", + XMM16: "xmm16", + XMM17: "xmm17", + XMM18: "xmm18", + XMM19: "xmm19", + XMM20: "xmm20", + XMM21: "xmm21", + XMM22: "xmm22", + XMM23: "xmm23", + XMM24: "xmm24", + XMM25: "xmm25", + XMM26: "xmm26", + XMM27: "xmm27", + XMM28: "xmm28", + XMM29: "xmm29", + XMM30: "xmm30", + XMM31: "xmm31", +} + +var ymmnames = [...]string{ + YMM0: "ymm0", + YMM1: "ymm1", + YMM2: "ymm2", + YMM3: "ymm3", + YMM4: "ymm4", + YMM5: "ymm5", + YMM6: "ymm6", + YMM7: "ymm7", + YMM8: "ymm8", + YMM9: "ymm9", + YMM10: "ymm10", + YMM11: "ymm11", + YMM12: "ymm12", + YMM13: "ymm13", + YMM14: "ymm14", + YMM15: "ymm15", + YMM16: "ymm16", + YMM17: "ymm17", + YMM18: "ymm18", + YMM19: "ymm19", + YMM20: "ymm20", + YMM21: "ymm21", + YMM22: "ymm22", + YMM23: "ymm23", + YMM24: "ymm24", + YMM25: "ymm25", + YMM26: "ymm26", + YMM27: "ymm27", + YMM28: "ymm28", + YMM29: "ymm29", + YMM30: "ymm30", + YMM31: "ymm31", +} + +var zmmnames = [...]string{ + ZMM0: "zmm0", + ZMM1: "zmm1", + ZMM2: "zmm2", + ZMM3: "zmm3", + ZMM4: "zmm4", + ZMM5: "zmm5", + ZMM6: "zmm6", + ZMM7: "zmm7", + ZMM8: "zmm8", + ZMM9: "zmm9", + ZMM10: "zmm10", + ZMM11: "zmm11", + ZMM12: "zmm12", + ZMM13: "zmm13", + ZMM14: "zmm14", + ZMM15: "zmm15", + ZMM16: "zmm16", + ZMM17: "zmm17", + ZMM18: "zmm18", + ZMM19: "zmm19", + ZMM20: "zmm20", + ZMM21: "zmm21", + ZMM22: "zmm22", + ZMM23: "zmm23", + ZMM24: "zmm24", + ZMM25: "zmm25", + ZMM26: "zmm26", + ZMM27: "zmm27", + ZMM28: "zmm28", + ZMM29: "zmm29", + ZMM30: "zmm30", + ZMM31: "zmm31", +} diff --git a/loader/internal/iasm/x86_64/utils.go b/loader/internal/iasm/x86_64/utils.go new file mode 100644 index 000000000..107dfb3cd --- /dev/null +++ b/loader/internal/iasm/x86_64/utils.go @@ -0,0 +1,147 @@ +// +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package x86_64 + +import ( + "encoding/binary" + "errors" + "reflect" + "strconv" + "unicode/utf8" + "unsafe" +) + +const ( + _CC_digit = 1 << iota + _CC_ident + _CC_ident0 + _CC_number +) + +func ispow2(v uint64) bool { + return (v & (v - 1)) == 0 +} + +func isdigit(cc rune) bool { + return '0' <= cc && cc <= '9' +} + +func isalpha(cc rune) bool { + return (cc >= 'a' && cc <= 'z') || (cc >= 'A' && cc <= 'Z') +} + +func isident(cc rune) bool { + return cc == '_' || isalpha(cc) || isdigit(cc) +} + +func isident0(cc rune) bool { + return cc == '_' || isalpha(cc) +} + +func isnumber(cc rune) bool { + return (cc == 'b' || cc == 'B') || + (cc == 'o' || cc == 'O') || + (cc == 'x' || cc == 'X') || + (cc >= '0' && cc <= '9') || + (cc >= 'a' && cc <= 'f') || + (cc >= 'A' && cc <= 'F') +} + +func align(v int, n int) int { + return (((v - 1) >> n) + 1) << n +} + +func append8(m *[]byte, v byte) { + *m = append(*m, v) +} + +func append16(m *[]byte, v uint16) { + p := len(*m) + *m = append(*m, 0, 0) + binary.LittleEndian.PutUint16((*m)[p:], v) +} + +func append32(m *[]byte, v uint32) { + p := len(*m) + *m = append(*m, 0, 0, 0, 0) + binary.LittleEndian.PutUint32((*m)[p:], v) +} + +func append64(m *[]byte, v uint64) { + p := len(*m) + *m = append(*m, 0, 0, 0, 0, 0, 0, 0, 0) + binary.LittleEndian.PutUint64((*m)[p:], v) +} + +func expandmm(m *[]byte, n int, v byte) { + sl := (*_GoSlice)(unsafe.Pointer(m)) + nb := sl.len + n + + /* grow as needed */ + if nb > cap(*m) { + *m = growslice(byteType, *m, nb) + } + + /* fill the new area */ + memset(unsafe.Pointer(uintptr(sl.ptr)+uintptr(sl.len)), v, uintptr(n)) + sl.len = nb +} + +func memset(p unsafe.Pointer, c byte, n uintptr) { + if c != 0 { + memsetv(p, c, n) + } else { + memclrNoHeapPointers(p, n) + } +} + +func memsetv(p unsafe.Pointer, c byte, n uintptr) { + for i := uintptr(0); i < n; i++ { + *(*byte)(unsafe.Pointer(uintptr(p) + i)) = c + } +} + +func literal64(v string) (uint64, error) { + var nb int + var ch rune + var ex error + var mm [12]byte + + /* unquote the runes */ + for v != "" { + if ch, _, v, ex = strconv.UnquoteChar(v, '\''); ex != nil { + return 0, ex + } else if nb += utf8.EncodeRune(mm[nb:], ch); nb > 8 { + return 0, errors.New("multi-char constant too large") + } + } + + /* convert to uint64 */ + return *(*uint64)(unsafe.Pointer(&mm)), nil +} + +var ( + byteWrap = reflect.TypeOf(byte(0)) + byteType = (*_GoType)(efaceOf(byteWrap).ptr) +) + +//go:linkname growslice runtime.growslice +func growslice(_ *_GoType, _ []byte, _ int) []byte + +//go:noescape +//go:linkname memclrNoHeapPointers runtime.memclrNoHeapPointers +func memclrNoHeapPointers(_ unsafe.Pointer, _ uintptr)