discv5: try to increase the UDP buffer size

dennis-tra · Nov 5, 2024 · 1fb9976 · 1fb9976
1 parent 404da78
commit 1fb9976
Show file tree

Hide file tree

Showing 4 changed files with 57 additions and 28 deletions.
diff --git a/cmd/nebula/cmd_crawl.go b/cmd/nebula/cmd_crawl.go
@@ -356,6 +356,7 @@ func CrawlAction(c *cli.Context) error {
 			TracerProvider: cfg.Root.TracerProvider,
 			MeterProvider:  cfg.Root.MeterProvider,
 			LogErrors:      cfg.Root.LogErrors,
+			UDPBufferSize:  cfg.Root.UDPBufferSize,
 			UDPRespTimeout: cfg.UDPRespTimeout,
 		}
 

diff --git a/discv4/driver_crawler.go b/discv4/driver_crawler.go
@@ -7,7 +7,6 @@ import (
 	"net"
 	"net/netip"
 	"sync"
-	"syscall"
 	"time"
 
 	ethcrypto "github.com/ethereum/go-ethereum/crypto"
@@ -265,7 +264,7 @@ func (d *CrawlDriver) NewWorker() (core.Worker[PeerInfo, core.CrawlResult[PeerIn
 		log.Warnln("Failed to set read buffer size on UDP listener", err)
 	}
 
-	rcvbuf, sndbuf, err := getUDPBufferSize(conn)
+	rcvbuf, sndbuf, err := utils.GetUDPBufferSize(conn)
 	logOnce.Do(func() {
 		logEntry := log.WithFields(log.Fields{
 			"rcvbuf": rcvbuf,
@@ -402,27 +401,3 @@ func (d *CrawlDriver) monitorUnhandledPackets() {
 		}
 	}()
 }
-
-// getUDPBufferSize reads the receive and send buffer sizes from the system
-func getUDPBufferSize(conn *net.UDPConn) (rcvbuf int, sndbuf int, err error) {
-	rawConn, err := conn.SyscallConn()
-	if err != nil {
-		return 0, 0, err
-	}
-
-	var (
-		rcverr error
-		snderr error
-	)
-	err = rawConn.Control(func(fd uintptr) {
-		rcvbuf, rcverr = syscall.GetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF)
-		sndbuf, snderr = syscall.GetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF)
-	})
-	if rcverr != nil {
-		err = rcverr
-	} else if snderr != nil {
-		err = snderr
-	}
-
-	return
-}
diff --git a/discv5/driver_crawler.go b/discv5/driver_crawler.go
@@ -8,6 +8,7 @@ import (
 	"io"
 	"net"
 	"runtime"
+	"sync"
 	"time"
 
 	secp256k1v4 "github.com/decred/dcrd/dcrec/secp256k1/v4"
@@ -149,6 +150,7 @@ type CrawlDriverConfig struct {
 	MeterProvider  metric.MeterProvider
 	TracerProvider trace.TracerProvider
 	LogErrors      bool
+	UDPBufferSize  int
 	UDPRespTimeout time.Duration
 }
 
@@ -221,18 +223,44 @@ func NewCrawlDriver(dbc db.Client, crawl *models.Crawl, cfg *CrawlDriverConfig)
 	}, nil
 }
 
+// NewWorker is called multiple times but only log the configured buffer sizes once
+var logOnce sync.Once
+
 func (d *CrawlDriver) NewWorker() (core.Worker[PeerInfo, core.CrawlResult[PeerInfo]], error) {
 	// If I'm not using the below elliptic curve, some Ethereum clients will reject communication
 	priv, err := ecdsa.GenerateKey(ethcrypto.S256(), crand.Reader)
 	if err != nil {
 		return nil, fmt.Errorf("new ethereum ecdsa key: %w", err)
 	}
 
-	conn, err := net.ListenUDP("udp4", nil)
+	laddr := &net.UDPAddr{
+		IP:   net.ParseIP("0.0.0.0"),
+		Port: 0,
+	}
+
+	conn, err := net.ListenUDP("udp4", laddr)
 	if err != nil {
-		return nil, fmt.Errorf("listen on udp port: %w", err)
+		return nil, fmt.Errorf("listen on udp4 port: %w", err)
+	}
+
+	if err = conn.SetReadBuffer(d.cfg.UDPBufferSize); err != nil {
+		log.Warnln("Failed to set read buffer size on UDP listener", err)
 	}
 
+	rcvbuf, sndbuf, err := utils.GetUDPBufferSize(conn)
+	logOnce.Do(func() {
+		logEntry := log.WithFields(log.Fields{
+			"rcvbuf": rcvbuf,
+			"sndbuf": sndbuf,
+			"rcvtgt": d.cfg.UDPBufferSize, // receive target
+		})
+		if rcvbuf < d.cfg.UDPBufferSize {
+			logEntry.Warnln("Failed to increase UDP buffer sizes, using default")
+		} else {
+			logEntry.Infoln("Configured UDP buffer sizes")
+		}
+	})
+
 	ethNode := enode.NewLocalNode(d.peerstore, priv)
 
 	// I'm not really sure if the below is strictly necessary.

diff --git a/utils/utils.go b/utils/utils.go
@@ -6,6 +6,7 @@ import (
 	"net"
 	"strconv"
 	"strings"
+	"syscall"
 
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -214,3 +215,27 @@ func (nebulaIdentityScheme) NodeAddr(r *enr.Record) []byte {
 	math.ReadBits(pubkey.Y, buf[32:])
 	return crypto.Keccak256(buf)
 }
+
+// GetUDPBufferSize reads the receive and send buffer sizes from the system
+func GetUDPBufferSize(conn *net.UDPConn) (rcvbuf int, sndbuf int, err error) {
+	rawConn, err := conn.SyscallConn()
+	if err != nil {
+		return 0, 0, err
+	}
+
+	var (
+		rcverr error
+		snderr error
+	)
+	err = rawConn.Control(func(fd uintptr) {
+		rcvbuf, rcverr = syscall.GetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF)
+		sndbuf, snderr = syscall.GetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF)
+	})
+	if rcverr != nil {
+		err = rcverr
+	} else if snderr != nil {
+		err = snderr
+	}
+
+	return
+}