diff --git a/collection/lscq/asm.go b/collection/lscq/asm_amd64.go similarity index 100% rename from collection/lscq/asm.go rename to collection/lscq/asm_amd64.go diff --git a/collection/lscq/asm.s b/collection/lscq/asm_amd64.s similarity index 100% rename from collection/lscq/asm.s rename to collection/lscq/asm_amd64.s diff --git a/collection/lscq/asm_loong64.go b/collection/lscq/asm_loong64.go new file mode 100644 index 00000000..fd3a0cca --- /dev/null +++ b/collection/lscq/asm_loong64.go @@ -0,0 +1,90 @@ +// Copyright 2022 Loongson Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build loong64 && !gccgo && !appengine +// +build loong64,!gccgo,!appengine + +package lscq + +import ( + "sync" + "unsafe" +) + +type uint128 struct { + data [2]uint64 + mu sync.Mutex + _ [128 - unsafe.Sizeof(sync.Mutex{})]byte +} + +func compareAndSwapUint128(addr *uint128, old1, old2, new1, new2 uint64) (swapped bool) { + addr.mu.Lock() + defer addr.mu.Unlock() + if addr.data[0] == old1 && addr.data[1] == old2 { + addr.data[0] = new1 + addr.data[1] = new2 + return true + } else { + return false + } +} + +func loadUint128(addr *uint128) (val uint128) { + addr.mu.Lock() + defer addr.mu.Unlock() + val = loadUint128Asm(addr) + return +} + +func loadUint128Asm(addr *uint128) (val uint128) + +func loadSCQNodePointer(addr unsafe.Pointer) (val scqNodePointer) + +func loadSCQNodeUint64(addr unsafe.Pointer) (val scqNodeUint64) + +func atomicTestAndSetFirstBit(addr *uint64) (val uint64) + +func atomicTestAndSetSecondBit(addr *uint64) (val uint64) + +func resetNode(addr unsafe.Pointer) + +//go:nosplit +func compareAndSwapSCQNodePointer(addr *scqNodePointer, old, new scqNodePointer) (swapped bool) { + // Ref: src/runtime/atomic_pointer.go:sync_atomic_CompareAndSwapPointer + if runtimeEnableWriteBarrier() { + runtimeatomicwb(&addr.data, new.data) + } + return compareAndSwapUint128((*uint128)(runtimenoescape(unsafe.Pointer(addr))), old.flags, uint64(uintptr(old.data)), new.flags, uint64(uintptr(new.data))) +} + +func compareAndSwapSCQNodeUint64(addr *scqNodeUint64, old, new scqNodeUint64) (swapped bool) { + return compareAndSwapUint128((*uint128)(unsafe.Pointer(addr)), old.flags, old.data, new.flags, new.data) +} + +func runtimeEnableWriteBarrier() bool + +//go:linkname runtimeatomicwb runtime.atomicwb +//go:noescape +func runtimeatomicwb(ptr *unsafe.Pointer, new unsafe.Pointer) + +//go:linkname runtimenoescape runtime.noescape +func runtimenoescape(p unsafe.Pointer) unsafe.Pointer + +//go:nosplit +func atomicWriteBarrier(ptr *unsafe.Pointer) { + // For SCQ dequeue only. (fastpath) + if runtimeEnableWriteBarrier() { + runtimeatomicwb(ptr, nil) + } +} diff --git a/collection/lscq/asm_loong64.s b/collection/lscq/asm_loong64.s new file mode 100644 index 00000000..9e7ce769 --- /dev/null +++ b/collection/lscq/asm_loong64.s @@ -0,0 +1,88 @@ +// Copyright 2022 Loongson Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build loong64,!gccgo,!appengine + +#include "textflag.h" + +//func loadUint128(addr *uint128) (ret uint128) +TEXT ·loadUint128Asm(SB),NOSPLIT,$0-24 + MOVV addr+0(FP), R4 + DBAR + MOVV 0(R4), R5 + MOVV 8(R4), R6 + DBAR + MOVV R5, ret+8(FP) + MOVV R6, ret+16(FP) + DBAR + RET + +// func loadSCQNodeUint64(addr unsafe.Pointer)(val scqNodeUint64) +TEXT ·loadSCQNodeUint64(SB),NOSPLIT,$0 + JMP ·loadUint128(SB) + +// func loadSCQNodePointer(addr unsafe.Pointer)(val scqNodePointer) +TEXT ·loadSCQNodePointer(SB),NOSPLIT,$0 + JMP ·loadUint128(SB) + +// func atomicTestAndSetFirstBit(addr *uint64)(val uint64) +TEXT ·atomicTestAndSetFirstBit(SB),NOSPLIT,$0 + MOVV addr+0(FP), R4 + MOVV $(1<<63), R6 + DBAR +testAndSetFirstBit_again: + LLV (R4), R5 + OR R6, R5, R5 + MOVV R5, R7 + SCV R5, (R4) + BEQ R5, testAndSetFirstBit_again + DBAR + MOVV R7, val+8(FP) + RET + +// func atomicTestAndSetSecondBit(addr *uint64)(val uint64) +TEXT ·atomicTestAndSetSecondBit(SB),NOSPLIT,$0 + MOVV addr+0(FP), R4 + MOVV $(1<<62), R6 + DBAR +testAndSetSecBit_again: + LLV (R4), R5 + OR R6, R5, R5 + MOVV R5, R7 + SCV R5, (R4) + BEQ R5, testAndSetSecBit_again + DBAR + MOVV R7, val+8(FP) + RET + +// func resetNode(addr unsafe.Pointer) +TEXT ·resetNode(SB),NOSPLIT,$0 + MOVV addr+0(FP), R4 + MOVV R0, 8(R4) + MOVV $(1<<62), R6 + DBAR +resetNode_again: + LLV (R4), R5 + OR R6, R5, R5 + SCV R5, (R4) + BEQ R5, resetNode_again + DBAR + RET + +// func runtimeEnableWriteBarrier() bool +TEXT ·runtimeEnableWriteBarrier(SB),NOSPLIT,$0 + MOVW runtime·writeBarrier(SB), R0 + DBAR + MOVB R0, ret+0(FP) + RET diff --git a/collection/lscq/lscq.go b/collection/lscq/lscq.go index 6f42c80b..a0b844fc 100644 --- a/collection/lscq/lscq.go +++ b/collection/lscq/lscq.go @@ -131,6 +131,8 @@ type pointerSCQ struct { type scqNodePointer struct { flags uint64 // isSafe 1-bit + isEmpty 1-bit + cycle 62-bit data unsafe.Pointer + _ sync.Mutex // Only for architectures that do not support 128bit atomic operations + _ [128 - unsafe.Sizeof(sync.Mutex{})]byte // The x86_64 instruction CMPXCHG16B requires 128bit alignment } func (q *pointerSCQ) Enqueue(data unsafe.Pointer) bool { diff --git a/collection/lscq/types.go b/collection/lscq/types.go index a30d7350..72087de5 100644 --- a/collection/lscq/types.go +++ b/collection/lscq/types.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -131,6 +131,8 @@ type uint64SCQ struct { type scqNodeUint64 struct { flags uint64 // isSafe 1-bit + isEmpty 1-bit + cycle 62-bit data uint64 + _ sync.Mutex // Only for architectures that do not support 128bit atomic operations + _ [128 - unsafe.Sizeof(sync.Mutex{})]byte // The x86_64 instruction CMPXCHG16B requires 128bit alignment } func (q *uint64SCQ) Enqueue(data uint64) bool { diff --git a/go.mod b/go.mod index 2abadac4..0ee485fe 100644 --- a/go.mod +++ b/go.mod @@ -5,5 +5,5 @@ go 1.16 require ( github.com/stretchr/testify v1.7.0 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20220110181412-a018aaa089fe + golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 ) diff --git a/go.sum b/go.sum index 963d7a88..95bf4c8a 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20220110181412-a018aaa089fe h1:W8vbETX/n8S6EmY0Pu4Ix7VvpsJUESTwl0oCK8MJOgk= -golang.org/x/sys v0.0.0-20220110181412-a018aaa089fe/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= diff --git a/internal/runtimex/readunaligned.go b/internal/runtimex/readunaligned.go index 6e545073..13689347 100644 --- a/internal/runtimex/readunaligned.go +++ b/internal/runtimex/readunaligned.go @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build 386 || amd64 || arm || arm64 || ppc64le || mips64le || mipsle || riscv64 || wasm -// +build 386 amd64 arm arm64 ppc64le mips64le mipsle riscv64 wasm +//go:build 386 || amd64 || arm || arm64 || ppc64le || mips64le || mipsle || riscv64 || wasm || loong64 +// +build 386 amd64 arm arm64 ppc64le mips64le mipsle riscv64 wasm loong64 // // from golang-go/src/os/endian_big.go