Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

getGlobalTSO may call Recv concurrently on a stream #9091

Closed
rleungx opened this issue Feb 21, 2025 · 1 comment · Fixed by #9095
Closed

getGlobalTSO may call Recv concurrently on a stream #9091

rleungx opened this issue Feb 21, 2025 · 1 comment · Fixed by #9095
Labels
affects-7.1 This bug affects the 7.1.x(LTS) versions. affects-7.5 This bug affects the 7.5.x(LTS) versions. affects-8.1 This bug affects the 8.1.x(LTS) versions. affects-8.5 This bug affects the 8.5.x(LTS) versions. severity/major type/bug The issue is confirmed as a bug.

Comments

@rleungx
Copy link
Member

rleungx commented Feb 21, 2025

Bug Report

panic: runtime error: invalid memory address or nil pointer dereference

[signal SIGSEGV: segmentation violation code=0x1 addr=0x8 pc=0x47288c]


goroutine 31644974457 [running]:
bytes.(*Buffer).Len(...)
	/usr/local/go/src/bytes/buffer.go:79
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0x405a28df90, {0x408bece880?, 0x4044348ca8?, 0x40b1744ae0?})
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:166 +0xcc
google.golang.org/grpc/internal/transport.(*transportReader).Read(0x408bece840, {0x408bece880?, 0x40b1744b68?, 0x473d74?})
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:525 +0x34
io.ReadAtLeast({0x1849060, 0x408bece840}, {0x408bece880, 0x5, 0x5}, 0x5)
	/usr/local/go/src/io/io.go:335 +0xa0
io.ReadFull(...)
	/usr/local/go/src/io/io.go:354
google.golang.org/grpc/internal/transport.(*Stream).Read(0x40989826c0, {0x408bece880, 0x5, 0x5})
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:509 +0x94
google.golang.org/grpc.(*parser).recvMsg(0x408bece870, 0x400000)
	/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:614 +0x48
google.golang.org/grpc.recvAndDecompress(0x408bece870, 0x40989826c0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0})
	/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:753 +0x58
google.golang.org/grpc.recv(0x1c7daf7?, {0xfffd2c698290, 0x260dfe0}, 0x1be9546?, {0x0?, 0x0?}, {0x14b7f80, 0x40a56238c0}, 0x402209aa80?, 0x14b7e20?, ...)
	/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:833 +0x64
google.golang.org/grpc.(*csAttempt).recvMsg(0x40801c21a0, {0x14b7f80?, 0x40a56238c0}, 0x1f1d8?)
	/go/pkg/mod/google.golang.org/[email protected]/stream.go:1085 +0x208
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x40b1744f38?)
	/go/pkg/mod/google.golang.org/[email protected]/stream.go:928 +0x28
google.golang.org/grpc.(*clientStream).withRetry(0x4098982480, 0x40b1744fe8, 0x40b1744fd8)
	/go/pkg/mod/google.golang.org/[email protected]/stream.go:761 +0x4e8
google.golang.org/grpc.(*clientStream).RecvMsg(0x4098982480, {0x14b7f80?, 0x40a56238c0?})
	/go/pkg/mod/google.golang.org/[email protected]/stream.go:927 +0xc0
github.com/pingcap/kvproto/pkg/tsopb.(*tSOTsoClient).Recv(0x40c92169d0)
	/go/pkg/mod/github.com/pingcap/[email protected]/pkg/tsopb/tsopb.pb.go:1011 +0x58
github.com/tikv/pd/server.(*GrpcServer).getGlobalTSO(0x403fe9e4e0, {0x185caa8, 0x40990a6cc0})
	/go/src/github.com/tikv/pd/server/forward.go:468 +0x438
github.com/tikv/pd/server.(*GrpcServer).UpdateServiceSafePointV2(0x403fe9e4e0, {0x185caa8, 0x40990a6cc0}, 0x40547bcf60)
	/go/src/github.com/tikv/pd/server/gc_service.go:109 +0x9c
github.com/pingcap/kvproto/pkg/pdpb._PD_UpdateServiceSafePointV2_Handler.func1({0x185caa8, 0x40990a6cc0}, {0x14cd340?, 0x40547bcf60})
	/go/pkg/mod/github.com/pingcap/[email protected]/pkg/pdpb/pdpb.pb.go:10124 +0x74
github.com/grpc-ecosystem/go-grpc-prometheus.init.(*ServerMetrics).UnaryServerInterceptor.func3({0x185caa8, 0x40990a6cc0}, {0x14cd340, 0x40547bcf60}, 0x0?, 0x40ce2a59e0)
	/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/server_metrics.go:107 +0x74
go.etcd.io/etcd/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6.1({0x185caa8?, 0x40990a6cc0?}, {0x14cd340?, 0x40547bcf60?})
	/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:48 +0x50
go.etcd.io/etcd/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func4({0x185caa8, 0x40990a6cc0}, {0x14cd340, 0x40547bcf60}, 0x40b1745798?, 0x40a5623240)
	/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/api/v3rpc/interceptor.go:71 +0x19c
go.etcd.io/etcd/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6.1({0x185caa8?, 0x40990a6cc0?}, {0x14cd340?, 0x40547bcf60?})
	/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:48 +0x50
go.etcd.io/etcd/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func3({0x185caa8?, 0x40990a6cc0}, {0x14cd340?, 0x40547bcf60}, 0x4062e1e980, 0x40a5623280)
	/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/api/v3rpc/interceptor.go:78 +0x88
go.etcd.io/etcd/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6({0x185caa8, 0x40990a6cc0}, {0x14cd340, 0x40547bcf60}, 0x4062e1e980, 0x4da90c?)
	/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:53 +0x128
github.com/pingcap/kvproto/pkg/pdpb._PD_UpdateServiceSafePointV2_Handler({0x1552140?, 0x403fe9e4e0}, {0x185caa8, 0x40990a6cc0}, 0x4073cfe080, 0x403d55bec0)
	/go/pkg/mod/github.com/pingcap/[email protected]/pkg/pdpb/pdpb.pb.go:10126 +0x12c
google.golang.org/grpc.(*Server).processUnaryRPC(0x40436ddc00, {0x185caa8, 0x40ea13be90}, {0x1867640, 0x4089106b60}, 0x401f045b00, 0x403feb2660, 0x25c4d90, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1386 +0xba0
google.golang.org/grpc.(*Server).handleStream(0x40436ddc00, {0x1867640, 0x4089106b60}, 0x401f045b00)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1797 +0xc70
google.golang.org/grpc.(*Server).serveStreams.func2.1()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1027 +0x8c
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 31644974454
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1038 +0x150

After confirming with @ystaticy, it's possible to call UpdateServiceSafePointV2 concurrently, which results in calling Recv concurrently on a stream.
@rleungx rleungx added the type/bug The issue is confirmed as a bug. label Feb 21, 2025
@lhy1024 lhy1024 mentioned this issue Feb 24, 2025
Merged
@lhy1024 lhy1024 added affects-7.1 This bug affects the 7.1.x(LTS) versions. affects-7.5 This bug affects the 7.5.x(LTS) versions. affects-8.1 This bug affects the 8.1.x(LTS) versions. affects-8.5 This bug affects the 8.5.x(LTS) versions. and removed may-affects-6.1 may-affects-6.5 may-affects-7.1 may-affects-7.5 may-affects-8.1 may-affects-8.5 labels Feb 25, 2025
@ti-chi-bot ti-chi-bot bot closed this as completed in 0c13897 Feb 26, 2025
ti-chi-bot pushed a commit to ti-chi-bot/pd that referenced this issue Feb 27, 2025
@lhy1024 @ti-chi-bot
This is an automated cherry-pick of tikv#9095
1fc6e50 
close tikv#9091

Signed-off-by: ti-chi-bot <[email protected]>
This was referenced Feb 27, 2025
Open
Merged
Merged
ti-chi-bot pushed a commit to ti-chi-bot/pd that referenced this issue Feb 27, 2025
@lhy1024 @ti-chi-bot
This is an automated cherry-pick of tikv#9095
5d3d23c 
close tikv#9091

Signed-off-by: ti-chi-bot <[email protected]>
@ti-chi-bot ti-chi-bot mentioned this issue Feb 27, 2025
Merged
@lhy1024
Copy link
Contributor

lhy1024 commented Feb 27, 2025

Before #9095, if run TestForwardTsoConcurrently will cause

panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x18 pc=0x1ad31c6]
goroutine 22151 [running]:
bytes.(*Buffer).Len(...)
	/usr/lib/go/src/bytes/buffer.go:79
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc005966780, {0xc00b1b4dc0?, 0xc00b08e0a8?, 0xc013a96e40?})
	/home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:166 +0xc6
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00b1b4cf0, {0xc00b1b4dc0?, 0xc013a96eb8?, 0x1abad65?})
	/home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:525 +0x2c
io.ReadAtLeast({0x423f3e0, 0xc00b1b4cf0}, {0xc00b1b4dc0, 0x5, 0x5}, 0x5)
	/usr/lib/go/src/io/io.go:335 +0x90
io.ReadFull(...)
	/usr/lib/go/src/io/io.go:354
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001edcea0, {0xc00b1b4dc0, 0x5, 0x5})
	/home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:509 +0x96
google.golang.org/grpc.(*parser).recvMsg(0xc00b1b4db0, 0x400000)
	/home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:614 +0x46
google.golang.org/grpc.recvAndDecompress(0xc00b1b4db0, 0xc001edcea0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0})
	/home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:753 +0x85
google.golang.org/grpc.recv(0xc013a97130?, {0x7f47f0679d20, 0x5851e60}, 0x3367e40?, {0x0?, 0x0?}, {0x34ee020, 0xc01d675050}, 0x0?, 0x0, ...)
	/home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:833 +0x7d
google.golang.org/grpc.(*csAttempt).recvMsg(0xc01367e000, {0x34ee020, 0xc01d675050}, 0x18?)
	/home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/stream.go:1085 +0x289
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x117444b?)
	/home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/stream.go:928 +0x1f
google.golang.org/grpc.(*clientStream).withRetry(0xc0121ef8c0, 0xc013a972e0, 0xc013a97328)
	/home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/stream.go:761 +0x3ae
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0121ef8c0, {0x34ee020?, 0xc01d675050?})
	/home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/stream.go:927 +0xf8
github.com/pingcap/kvproto/pkg/tsopb.(*tSOTsoClient).Recv(0xc00cfe7800)
	/home/lhy1024/.go/pkg/mod/github.com/pingcap/[email protected]/pkg/tsopb/tsopb.pb.go:972 +0x46
github.com/tikv/pd/server.(*GrpcServer).getGlobalTSO(0xc0017212c0, {0x425d9a0, 0xc01d4ffa70})
	/home/lhy1024/pd/server/forward.go:474 +0x2e9
github.com/tikv/pd/server.(*GrpcServer).UpdateServiceGCSafePoint(0xc0017212c0, {0x425d9a0, 0xc01d4ffa70}, 0xc01d4ffaa0)
	/home/lhy1024/pd/server/grpc_service.go:2217 +0x4c5
github.com/pingcap/kvproto/pkg/pdpb._PD_UpdateServiceGCSafePoint_Handler.func1({0x425d9a0?, 0xc01d4ffa70?}, {0x34fb060?, 0xc01d4ffaa0?})
	/home/lhy1024/.go/pkg/mod/github.com/pingcap/[email protected]/pkg/pdpb/pdpb.pb.go:10267 +0xce
github.com/grpc-ecosystem/go-grpc-prometheus.init.(*ServerMetrics).UnaryServerInterceptor.func3({0x425d9a0, 0xc01d4ffa70}, {0x34fb060, 0xc01d4ffaa0}, 0x35f38cc?, 0xc01d4f1698)
	/home/lhy1024/.go/pkg/mod/github.com/grpc-ecosystem/[email protected]/server_metrics.go:107 +0x7d
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6.1({0x425d9a0?, 0xc01d4ffa70?}, {0x34fb060?, 0xc01d4ffaa0?})
	/home/lhy1024/.go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:48 +0x45
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func4({0x425d9a0, 0xc01d4ffa70}, {0x34fb060, 0xc01d4ffaa0}, 0xc013a977f0?, 0xc01d4e9bc0)
	/home/lhy1024/.go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/api/v3rpc/interceptor.go:71 +0x1b4
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6.1({0x425d9a0?, 0xc01d4ffa70?}, {0x34fb060?, 0xc01d4ffaa0?})
	/home/lhy1024/.go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:48 +0x45
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func3({0x425d9a0, 0xc01d4ffa70}, {0x34fb060, 0xc01d4ffaa0}, 0xc0109694e0, 0xc01d4e9c00)
	/home/lhy1024/.go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/api/v3rpc/interceptor.go:78 +0xb2
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6({0x425d9a0, 0xc01d4ffa70}, {0x34fb060, 0xc01d4ffaa0}, 0xc0109694e0, 0x80?)
	/home/lhy1024/.go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:53 +0x123
github.com/pingcap/kvproto/pkg/pdpb._PD_UpdateServiceGCSafePoint_Handler({0x35c5720, 0xc0017212c0}, {0x425d9a0, 0xc01d4ffa70}, 0xc01d499c80, 0xc00211a7e0)
	/home/lhy1024/.go/pkg/mod/github.com/pingcap/[email protected]/pkg/pdpb/pdpb.pb.go:10269 +0x143
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00010e200, {0x425d9a0, 0xc01d4ff9e0}, {0x426cc60, 0xc00dda0000}, 0xc01d4fb320, 0xc00211af30, 0x574c160, 0x0)
	/home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/server.go:1386 +0xdf8
google.golang.org/grpc.(*Server).handleStream(0xc00010e200, {0x426cc60, 0xc00dda0000}, 0xc01d4fb320)
	/home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/server.go:1797 +0xe87
google.golang.org/grpc.(*Server).serveStreams.func2.1()
	/home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/server.go:1027 +0x7f
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 5481
	/home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/server.go:1038 +0x125
FAIL	github.com/tikv/pd/tests/integrations/mcs/tso	5.855s
FAIL

WARNING: DATA RACE
Write at 0x00c0090c99b0 by goroutine 2230:
  google.golang.org/grpc.(*csAttempt).recvMsg()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/stream.go:1083 +0x38f
  google.golang.org/grpc.(*clientStream).RecvMsg.func1()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/stream.go:928 +0x50
  google.golang.org/grpc.(*clientStream).withRetry()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/stream.go:777 +0x197
  google.golang.org/grpc.(*clientStream).RecvMsg()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/stream.go:927 +0x1c6
  github.com/pingcap/kvproto/pkg/tsopb.(*tSOTsoClient).Recv()
      /home/lhy1024/.go/pkg/mod/github.com/pingcap/[email protected]/pkg/tsopb/tsopb.pb.go:972 +0x65
  github.com/tikv/pd/server.(*GrpcServer).getGlobalTSO()
      /home/lhy1024/pd/server/forward.go:473 +0x438
  github.com/tikv/pd/server.(*GrpcServer).UpdateServiceGCSafePoint()
      /home/lhy1024/pd/server/grpc_service.go:2217 +0x5af
  github.com/pingcap/kvproto/pkg/pdpb._PD_UpdateServiceGCSafePoint_Handler.func1()
      /home/lhy1024/.go/pkg/mod/github.com/pingcap/[email protected]/pkg/pdpb/pdpb.pb.go:10267 +0xc1
  github.com/grpc-ecosystem/go-grpc-prometheus.init.(*ServerMetrics).UnaryServerInterceptor.func3()
      /home/lhy1024/.go/pkg/mod/github.com/grpc-ecosystem/[email protected]/server_metrics.go:107 +0xbb
  go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6.1()
      /home/lhy1024/.go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:48 +0xd0
  go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func4()
      /home/lhy1024/.go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/api/v3rpc/interceptor.go:71 +0x2ba
  go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6.1()
      /home/lhy1024/.go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:48 +0xd0
  go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func3()
      /home/lhy1024/.go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/api/v3rpc/interceptor.go:78 +0xbb
  go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6()
      /home/lhy1024/.go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:53 +0x22c
  github.com/pingcap/kvproto/pkg/pdpb._PD_UpdateServiceGCSafePoint_Handler()
      /home/lhy1024/.go/pkg/mod/github.com/pingcap/[email protected]/pkg/pdpb/pdpb.pb.go:10269 +0x1f3
  google.golang.org/grpc.(*Server).processUnaryRPC()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/server.go:1386 +0x1a2c
  google.golang.org/grpc.(*Server).handleStream()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/server.go:1797 +0x1811
  google.golang.org/grpc.(*Server).serveStreams.func2.1()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/server.go:1027 +0x158
Previous read at 0x00c0090c99b0 by goroutine 2248:
  google.golang.org/grpc.(*csAttempt).recvMsg()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/stream.go:1069 +0x12c
  google.golang.org/grpc.(*clientStream).RecvMsg.func1()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/stream.go:928 +0x50
  google.golang.org/grpc.(*clientStream).withRetry()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/stream.go:777 +0x197
  google.golang.org/grpc.(*clientStream).RecvMsg()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/stream.go:927 +0x1c6
  github.com/pingcap/kvproto/pkg/tsopb.(*tSOTsoClient).Recv()
      /home/lhy1024/.go/pkg/mod/github.com/pingcap/[email protected]/pkg/tsopb/tsopb.pb.go:972 +0x65
  github.com/tikv/pd/server.(*GrpcServer).getGlobalTSO()
      /home/lhy1024/pd/server/forward.go:473 +0x438
  github.com/tikv/pd/server.(*GrpcServer).UpdateServiceGCSafePoint()
      /home/lhy1024/pd/server/grpc_service.go:2217 +0x5af
  github.com/pingcap/kvproto/pkg/pdpb._PD_UpdateServiceGCSafePoint_Handler.func1()
      /home/lhy1024/.go/pkg/mod/github.com/pingcap/[email protected]/pkg/pdpb/pdpb.pb.go:10267 +0xc1
  github.com/grpc-ecosystem/go-grpc-prometheus.init.(*ServerMetrics).UnaryServerInterceptor.func3()
      /home/lhy1024/.go/pkg/mod/github.com/grpc-ecosystem/[email protected]/server_metrics.go:107 +0xbb
  go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6.1()
      /home/lhy1024/.go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:48 +0xd0
  go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func4()
      /home/lhy1024/.go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/api/v3rpc/interceptor.go:71 +0x2ba
  go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6.1()
      /home/lhy1024/.go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:48 +0xd0
  go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func3()
      /home/lhy1024/.go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/api/v3rpc/interceptor.go:78 +0xbb
  go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6()
      /home/lhy1024/.go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:53 +0x22c
  github.com/pingcap/kvproto/pkg/pdpb._PD_UpdateServiceGCSafePoint_Handler()
      /home/lhy1024/.go/pkg/mod/github.com/pingcap/[email protected]/pkg/pdpb/pdpb.pb.go:10269 +0x1f3
  google.golang.org/grpc.(*Server).processUnaryRPC()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/server.go:1386 +0x1a2c
  google.golang.org/grpc.(*Server).handleStream()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/server.go:1797 +0x1811
  google.golang.org/grpc.(*Server).serveStreams.func2.1()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/server.go:1027 +0x158
Goroutine 2230 (running) created at:
  google.golang.org/grpc.(*Server).serveStreams.func2()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/server.go:1038 +0x224
  google.golang.org/grpc/internal/transport.(*http2Server).operateHeaders()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:619 +0x3941
  google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:660 +0x404
  google.golang.org/grpc.(*Server).serveStreams()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/server.go:1021 +0x69b
  google.golang.org/grpc.(*Server).handleRawConn.func1()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/server.go:957 +0x86
Goroutine 2248 (running) created at:
  google.golang.org/grpc.(*Server).serveStreams.func2()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/server.go:1038 +0x224
  google.golang.org/grpc/internal/transport.(*http2Server).operateHeaders()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:619 +0x3941
  google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:660 +0x404
  google.golang.org/grpc.(*Server).serveStreams()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/server.go:1021 +0x69b
  google.golang.org/grpc.(*Server).handleRawConn.func1()
      /home/lhy1024/.go/pkg/mod/google.golang.org/[email protected]/server.go:957 +0x86
==================

ti-chi-bot bot pushed a commit that referenced this issue Feb 28, 2025
@ti-chi-bot @lhy1024
mcs: add lock for forward tso stream (#9095) (#9106)
2da8160 
close #9091

Signed-off-by: lhy1024 <[email protected]>

Co-authored-by: lhy1024 <[email protected]>
ti-chi-bot bot pushed a commit that referenced this issue Mar 3, 2025
@ti-chi-bot @lhy1024
mcs: add lock for forward tso stream (#9095) (#9105)
6ebc2df 
close #9091

Signed-off-by: lhy1024 <[email protected]>

Co-authored-by: lhy1024 <[email protected]>
ti-chi-bot bot pushed a commit that referenced this issue Mar 3, 2025
@ti-chi-bot @lhy1024
mcs: add lock for forward tso stream (#9095) (#9107)
4a40114 
close #9091

Signed-off-by: ti-chi-bot <[email protected]>
Signed-off-by: lhy1024 <[email protected]>

Co-authored-by: lhy1024 <[email protected]>
Co-authored-by: lhy1024 <[email protected]>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
affects-7.1 This bug affects the 7.1.x(LTS) versions. affects-7.5 This bug affects the 7.5.x(LTS) versions. affects-8.1 This bug affects the 8.1.x(LTS) versions. affects-8.5 This bug affects the 8.5.x(LTS) versions. severity/major type/bug The issue is confirmed as a bug.
Projects
None yet
Milestone
No milestone
Development

Successfully merging a pull request may close this issue.

mcs: add lock for forward tso stream lhy1024/pd
2 participants
@lhy1024 @rleungx