From 191211fbd2523b1cf373b53f7dac0ce131507013 Mon Sep 17 00:00:00 2001 From: thinhC98 Date: Wed, 27 Nov 2024 16:08:18 +0700 Subject: [PATCH 1/5] enable pprofFlag --- internal/debug/flags.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 7e6539fb26..af80d1a82a 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -91,10 +91,10 @@ var Flags = []cli.Flag{ //vmoduleFlag, //backtraceAtFlag, //debugFlag, - //pprofFlag, - //pprofAddrFlag, - //pprofPortFlag, - //memprofilerateFlag, + pprofFlag, + pprofAddrFlag, + pprofPortFlag, + memprofilerateFlag, //blockprofilerateFlag, //cpuprofileFlag, //traceFlag, From 842f746589f640dbad1707cb47b10688f45238d6 Mon Sep 17 00:00:00 2001 From: thinhC98 Date: Wed, 27 Nov 2024 16:14:21 +0700 Subject: [PATCH 2/5] Set cache allowance cap for GC --- cmd/tomo/consolecmd.go | 1 + cmd/tomo/main.go | 26 +++++++++++++++++++++++--- cmd/utils/flags.go | 23 +++++++++++++++++++++++ go.mod | 5 ++++- go.sum | 8 ++++++++ 5 files changed, 59 insertions(+), 4 deletions(-) diff --git a/cmd/tomo/consolecmd.go b/cmd/tomo/consolecmd.go index 10a36b54d7..a68f586c6a 100644 --- a/cmd/tomo/consolecmd.go +++ b/cmd/tomo/consolecmd.go @@ -77,6 +77,7 @@ JavaScript API. See https://github.com/tomochain/tomochain/wiki/JavaScript-Conso // same time. func localConsole(ctx *cli.Context) error { // Create and start the node based on the CLI flags + prepare(ctx) node, cfg := makeFullNode(ctx) startNode(ctx, node, cfg) defer node.Stop() diff --git a/cmd/tomo/main.go b/cmd/tomo/main.go index 0bd21952a7..dcfb07cc59 100644 --- a/cmd/tomo/main.go +++ b/cmd/tomo/main.go @@ -21,6 +21,7 @@ import ( "os" "runtime" "sort" + "strconv" "strings" "time" @@ -88,9 +89,9 @@ var ( //utils.LightServFlag, //utils.LightPeersFlag, //utils.LightKDFFlag, - //utils.CacheFlag, - //utils.CacheDatabaseFlag, - //utils.CacheGCFlag, + utils.CacheFlag, + // utils.CacheDatabaseFlag, + // utils.CacheGCFlag, //utils.TrieCacheGenFlag, utils.ListenPortFlag, utils.MaxPeersFlag, @@ -213,12 +214,31 @@ func main() { // It creates a default node based on the command line arguments and runs it in // blocking mode, waiting for it to be shut down. func tomo(ctx *cli.Context) error { + prepare(ctx) node, cfg := makeFullNode(ctx) startNode(ctx, node, cfg) node.Wait() return nil } +// prepare manipulates memory cache allowance and setups metric system. +// This function should be called before launching devp2p stack. +func prepare(ctx *cli.Context) { + + // If we're a full node on mainnet without --cache specified, bump default cache allowance + if !ctx.IsSet(utils.CacheFlag.Name) { + log.Info("Bumping default cache on mainnet", "provided", ctx.Int(utils.CacheFlag.Name), "updated", 4096) + ctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096)) + } + // Note: This features below will be improved and updated in future releases. + // Start metrics export if enabled + // utils.SetupMetrics(ctx) + + // Start system runtime metrics collection + // go metrics.CollectProcessMetrics(3 * time.Second) + +} + // startNode boots up the system node and all registered protocols, after which // it unlocks any requested accounts, and starts the RPC/IPC interfaces and the // miner. diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b1f0f92763..fad29a61e7 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -21,13 +21,16 @@ import ( "crypto/ecdsa" "fmt" "io/ioutil" + "math" "math/big" "os" "path/filepath" "runtime" + godebug "runtime/debug" "strconv" "strings" + gopsutil "github.com/shirou/gopsutil/mem" "github.com/tomochain/tomochain/accounts" "github.com/tomochain/tomochain/accounts/keystore" "github.com/tomochain/tomochain/common" @@ -1094,6 +1097,26 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { setTxPool(ctx, &cfg.TxPool) setEthash(ctx, cfg) + // Cap the cache allowance and tune the garbage collector + mem, err := gopsutil.VirtualMemory() + if err == nil { + if 32<<(^uintptr(0)>>63) == 32 && mem.Total > 2*1024*1024*1024 { + log.Warn("Lowering memory allowance on 32bit arch", "available", mem.Total/1024/1024, "addressable", 2*1024) + mem.Total = 2 * 1024 * 1024 * 1024 + } + allowance := int(mem.Total / 1024 / 1024 / 3) + if cache := ctx.Int(CacheFlag.Name); cache > allowance { + log.Warn("Sanitizing cache to Go's GC limits", "provided", cache, "updated", allowance) + ctx.Set(CacheFlag.Name, strconv.Itoa(allowance)) + } + } + // Ensure Go's GC ignores the database cache for trigger percentage + cache := ctx.Int(CacheFlag.Name) + gogc := math.Max(20, math.Min(100, 100/(float64(cache)/1024))) + + log.Debug("Sanitizing Go's GC trigger", "percent", int(gogc)) + godebug.SetGCPercent(int(gogc)) + switch { case ctx.GlobalIsSet(SyncModeFlag.Name): cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode) diff --git a/go.mod b/go.mod index 037caaaa85..36132b2d59 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( golang.org/x/crypto v0.0.0-20191105034135-c7e5f84aec59 golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2 golang.org/x/sync v0.0.0-20190423024810-112230192c58 - golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8 + golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 golang.org/x/tools v0.0.0-20191104232314-dc038396d1f0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 gopkg.in/karalabe/cookiejar.v2 v2.0.0-20150724131613-8dcd6a7f4951 @@ -52,6 +52,7 @@ require ( require ( github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/dlclark/regexp2 v1.2.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect github.com/google/go-cmp v0.3.1 // indirect github.com/google/uuid v1.0.0 // indirect @@ -65,7 +66,9 @@ require ( github.com/naoina/go-stringutil v0.1.0 // indirect github.com/nsf/termbox-go v0.0.0-20170211012700-3540b76b9c77 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect golang.org/x/text v0.3.2 // indirect gopkg.in/yaml.v2 v2.2.2 // indirect gotest.tools v2.2.0+incompatible // indirect diff --git a/go.sum b/go.sum index d055669717..bed28892f1 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,8 @@ github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= @@ -170,6 +172,8 @@ github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa github.com/rs/cors v1.6.0 h1:G9tHG9lebljV9mfp9SNPDL36nCDxmo3zTlAf1YgvzmI= github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= @@ -190,6 +194,8 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhe github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xtaci/kcp-go v5.4.5+incompatible/go.mod h1:bN6vIwHQbfHaHtFpEssmWsN45a+AZwO7eyRCmEIbtvE= github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= @@ -221,6 +227,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8 h1:41hwlulw1prEMBxLQSlMSux1zxJf07B3WPsdjJlKZxE= golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= From 4a5bd220162f00701a3ab89a8b1bc51b45d10d28 Mon Sep 17 00:00:00 2001 From: khangnt-coin98 Date: Thu, 28 Nov 2024 15:26:04 +0700 Subject: [PATCH 3/5] feat: increase default cache allowance to 4096 MB for mainnet --- cmd/tomo/main.go | 4 ++-- cmd/utils/flags.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/tomo/main.go b/cmd/tomo/main.go index dcfb07cc59..3faa587d5e 100644 --- a/cmd/tomo/main.go +++ b/cmd/tomo/main.go @@ -227,8 +227,8 @@ func prepare(ctx *cli.Context) { // If we're a full node on mainnet without --cache specified, bump default cache allowance if !ctx.IsSet(utils.CacheFlag.Name) { - log.Info("Bumping default cache on mainnet", "provided", ctx.Int(utils.CacheFlag.Name), "updated", 4096) - ctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096)) + log.Info("Bumping default cache on mainnet", "value", utils.CacheFlag.Value) + ctx.Set(utils.CacheFlag.Name, strconv.Itoa(utils.CacheFlag.Value)) } // Note: This features below will be improved and updated in future releases. // Start metrics export if enabled diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index fad29a61e7..6582756352 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -296,7 +296,7 @@ var ( CacheFlag = cli.IntFlag{ Name: "cache", Usage: "Megabytes of memory allocated to internal caching", - Value: 1024, + Value: 4096, } CacheDatabaseFlag = cli.IntFlag{ Name: "cache.database", From 9cfdf934650725bad3d1c5595a2c8bd2b5114c78 Mon Sep 17 00:00:00 2001 From: thinhC98 Date: Sat, 30 Nov 2024 20:11:16 +0700 Subject: [PATCH 4/5] disable pprof flag --- internal/debug/flags.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/debug/flags.go b/internal/debug/flags.go index af80d1a82a..c3048672d4 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -91,10 +91,10 @@ var Flags = []cli.Flag{ //vmoduleFlag, //backtraceAtFlag, //debugFlag, - pprofFlag, - pprofAddrFlag, - pprofPortFlag, - memprofilerateFlag, + // pprofFlag, + // pprofAddrFlag, + // pprofPortFlag, + // memprofilerateFlag, //blockprofilerateFlag, //cpuprofileFlag, //traceFlag, From 5c3c1d1174aa87ef1c68322586cf8d98d86644ba Mon Sep 17 00:00:00 2001 From: thinhC98 Date: Sat, 30 Nov 2024 20:12:39 +0700 Subject: [PATCH 5/5] refactor --- internal/debug/flags.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/debug/flags.go b/internal/debug/flags.go index c3048672d4..7e6539fb26 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -91,10 +91,10 @@ var Flags = []cli.Flag{ //vmoduleFlag, //backtraceAtFlag, //debugFlag, - // pprofFlag, - // pprofAddrFlag, - // pprofPortFlag, - // memprofilerateFlag, + //pprofFlag, + //pprofAddrFlag, + //pprofPortFlag, + //memprofilerateFlag, //blockprofilerateFlag, //cpuprofileFlag, //traceFlag,