Enable performance based relayer selection (#1226)

# Related Github tickets - #1220 - #422 # Background This change introduces fundamental changes to the relayer selection process. Live values are now gathered from the metrics module, and individual relayer fees are taken into account. Going forward, it's paramount for relayers to make sure they have their fees set and configured for each chain, otherwise they will **not be eligible for relaying anymore**. The majority of the changes are in `msg_assigner.go`, that's where most of your attention will be needed. It works by collecting metric and fee data once per block for all validators and ranking them. Then each message during that block will be filtered depending on its requirements, and the top ranking eligible validator will be picked (and removed from the cache). They become eligible again next block. There are still some issues with this algorithm though. At the moment, the logic will prioritize the few top ranking validators only, with little way for the mid tier to rank up. This will need to be revisited in the future for sure. # Testing completed - [ ] test coverage exists or has been added/updated - [ ] tested in a private testnet # Breaking changes - [ ] I have checked my code for breaking changes - [ ] If there are breaking changes, there is a supporting migration. --------- Co-authored-by: Taariq Lewis <[email protected]> Co-authored-by: Luis Carvalho <[email protected]>
VolumeFi · Jul 22, 2024 · e4a98d9 · e4a98d9
1 parent f90a63f
commit e4a98d9
Show file tree

Hide file tree

Showing 40 changed files with 2,445 additions and 1,370 deletions.
diff --git a/app/app.go b/app/app.go
@@ -582,6 +582,8 @@ func New(
 		app.ConsensusKeeper,
 		app.ValsetKeeper,
 		authcodec.NewBech32Codec(chainparams.ValidatorAddressPrefix),
+		app.MetrixKeeper,
+		&app.TreasuryKeeper, // only iniditalised later in code, so we pass the pointer
 	)
 	app.ValsetKeeper.SnapshotListeners = []valsetmoduletypes.OnSnapshotBuiltListener{
 		&app.EvmKeeper,
@@ -656,7 +658,6 @@ func New(
 		app.GetSubspace(schedulermoduletypes.ModuleName),
 		app.BankKeeper,
 		app.AccountKeeper,
-		app.SchedulerKeeper,
 		app.EvmKeeper,
 	)
 

diff --git a/proto/palomachain/paloma/treasury/query.proto b/proto/palomachain/paloma/treasury/query.proto
@@ -26,6 +26,11 @@ service Query {
   rpc RelayerFee(QueryRelayerFeeRequest) returns (RelayerFeeSetting) {
     option (google.api.http).get = "/palomachain/paloma/treasury/relayer_fee";
   }
+
+  // RelayerFee queries the fee settings of all validators for a given external chain.
+  rpc RelayerFees(QueryRelayerFeesRequest) returns (QueryRelayerFeesResponse) {
+    option (google.api.http).get = "/palomachain/paloma/treasury/relayer_fees";
+  }
 }
 
 // QueryParamsRequest is request type for the Query/Params RPC method.
@@ -46,3 +51,10 @@ message QueryRelayerFeeRequest {
   string val_address = 1 [ (cosmos_proto.scalar) = "cosmos.AddressString" ];
 }
 
+message QueryRelayerFeesRequest {
+    string chain_reference_id = 1;
+}
+
+message QueryRelayerFeesResponse {
+    repeated RelayerFeeSetting relayer_fees = 1 [(gogoproto.nullable) = false];
+}
diff --git a/tests/integration/evm/keeper/genesis_test.go b/tests/integration/evm/keeper/genesis_test.go
@@ -97,6 +97,7 @@ var _ = g.Describe("genesis", func() {
 							Uptime:        "1.0",
 							SuccessRate:   "1.0",
 							ExecutionTime: "1.0",
+							FeatureSet:    "1.0",
 						},
 					},
 					{
@@ -110,6 +111,7 @@ var _ = g.Describe("genesis", func() {
 							Uptime:        "1.0",
 							SuccessRate:   "1.0",
 							ExecutionTime: "1.0",
+							FeatureSet:    "1.0",
 						},
 					},
 				},

diff --git a/tests/integration/evm/keeper/keeper_integration_test.go b/tests/integration/evm/keeper/keeper_integration_test.go
@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"cosmossdk.io/core/header"
+	"cosmossdk.io/math"
 	"github.com/VolumeFi/whoops"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
@@ -24,6 +25,7 @@ import (
 	consensustypes "github.com/palomachain/paloma/x/consensus/types"
 	"github.com/palomachain/paloma/x/evm/keeper"
 	"github.com/palomachain/paloma/x/evm/types"
+	treasurytypes "github.com/palomachain/paloma/x/treasury/types"
 	valsettypes "github.com/palomachain/paloma/x/valset/types"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -88,10 +90,25 @@ func TestEndToEndForEvmArbitraryCall(t *testing.T) {
 			},
 		})
 		require.NoError(t, err)
+		err = f.treasuryKeeper.SetRelayerFee(ctx, sdk.ValAddress(operator), &treasurytypes.RelayerFeeSetting{
+			ValAddress: sdk.ValAddress(operator).String(),
+			Fees: []treasurytypes.RelayerFeeSetting_FeeSetting{
+				{
+					Multiplicator:    math.LegacyMustNewDecFromStr("1.1"),
+					ChainReferenceId: chainReferenceID,
+				},
+				{
+					Multiplicator:    math.LegacyMustNewDecFromStr("1.1"),
+					ChainReferenceId: newChain.ChainReferenceID,
+				},
+			},
+		})
+		require.NoError(t, err)
 	}
 
 	_, err = f.valsetKeeper.TriggerSnapshotBuild(ctx)
 	require.NoError(t, err)
+	f.metrixKeeper.UpdateUptime(ctx)
 
 	smartContractAddr := common.BytesToAddress(rand.Bytes(5))
 	_, err = f.evmKeeper.AddSmartContractExecutionToConsensus(
@@ -814,8 +831,23 @@ var _ = Describe("evm", func() {
 						},
 					})
 					Expect(err).To(BeNil())
+					err = a.treasuryKeeper.SetRelayerFee(ctx, valAddr, &treasurytypes.RelayerFeeSetting{
+						ValAddress: sdk.ValAddress(valAddr).String(),
+						Fees: []treasurytypes.RelayerFeeSetting_FeeSetting{
+							{
+								Multiplicator:    math.LegacyMustNewDecFromStr("1.1"),
+								ChainReferenceId: chain1.ChainReferenceID,
+							},
+							{
+								Multiplicator:    math.LegacyMustNewDecFromStr("1.1"),
+								ChainReferenceId: chain2.ChainReferenceID,
+							},
+						},
+					})
+					require.NoError(t, err)
 				}
 				_, err := a.valsetKeeper.TriggerSnapshotBuild(ctx)
+				a.metrixKeeper.UpdateUptime(ctx)
 				Expect(err).To(BeNil())
 			})
 
@@ -957,10 +989,25 @@ var _ = Describe("evm", func() {
 							},
 						})
 						Expect(err).To(BeNil())
+						err = a.treasuryKeeper.SetRelayerFee(ctx, valAddr, &treasurytypes.RelayerFeeSetting{
+							ValAddress: sdk.ValAddress(valAddr).String(),
+							Fees: []treasurytypes.RelayerFeeSetting_FeeSetting{
+								{
+									Multiplicator:    math.LegacyMustNewDecFromStr("1.1"),
+									ChainReferenceId: newChain.ChainReferenceID,
+								},
+								{
+									Multiplicator:    math.LegacyMustNewDecFromStr("1.1"),
+									ChainReferenceId: "new-chain",
+								},
+							},
+						})
+						require.NoError(t, err)
 					}
 					var err error
 					snapshot, err = a.valsetKeeper.TriggerSnapshotBuild(ctx)
 					Expect(err).To(BeNil())
+					a.metrixKeeper.UpdateUptime(ctx)
 				})
 
 				BeforeEach(func() {
@@ -1076,7 +1123,22 @@ var _ = Describe("evm", func() {
 								},
 							})
 							Expect(err).To(BeNil())
+							err = a.treasuryKeeper.SetRelayerFee(ctx, valAddr, &treasurytypes.RelayerFeeSetting{
+								ValAddress: sdk.ValAddress(valAddr).String(),
+								Fees: []treasurytypes.RelayerFeeSetting_FeeSetting{
+									{
+										Multiplicator:    math.LegacyMustNewDecFromStr("1.1"),
+										ChainReferenceId: newChain.ChainReferenceID,
+									},
+									{
+										Multiplicator:    math.LegacyMustNewDecFromStr("1.1"),
+										ChainReferenceId: "new-chain",
+									},
+								},
+							})
+							require.NoError(t, err)
 						}
+						a.metrixKeeper.UpdateUptime(ctx)
 					})
 					BeforeEach(func() {
 						msgs, err := a.consensusKeeper.GetMessagesFromQueue(ctx, "evm/new-chain/evm-turnstone-message", 5)
@@ -1238,6 +1300,7 @@ var _ = Describe("change relay weights", func() {
 				Uptime:        "1.0",
 				SuccessRate:   "1.0",
 				ExecutionTime: "1.0",
+				FeatureSet:    "1.0",
 			}))
 		})
 
@@ -1247,6 +1310,7 @@ var _ = Describe("change relay weights", func() {
 				Uptime:        "0.34",
 				SuccessRate:   "0.56",
 				ExecutionTime: "0.78",
+				FeatureSet:    "0.99",
 			}
 			err := a.evmKeeper.SetRelayWeights(ctx, newChain.GetChainReferenceID(), newWeights)
 			Expect(err).To(BeNil())
@@ -1265,6 +1329,7 @@ var _ = Describe("change relay weights", func() {
 				Uptime:        "0.34",
 				SuccessRate:   "0.56",
 				ExecutionTime: "0.78",
+				FeatureSet:    "0.99",
 			})
 			Expect(err).To(MatchError(keeper.ErrChainNotFound))
 		})

diff --git a/tests/integration/evm/keeper/test_helpers_test.go b/tests/integration/evm/keeper/test_helpers_test.go
@@ -63,13 +63,16 @@ import (
 	"github.com/palomachain/paloma/x/evm"
 	evmmodulekeeper "github.com/palomachain/paloma/x/evm/keeper"
 	evmmoduletypes "github.com/palomachain/paloma/x/evm/types"
+	metrixmodulekeeper "github.com/palomachain/paloma/x/metrix/keeper"
+	metrixmoduletypes "github.com/palomachain/paloma/x/metrix/types"
 	"github.com/palomachain/paloma/x/paloma"
 	palomamoduletypes "github.com/palomachain/paloma/x/paloma/types"
 	"github.com/palomachain/paloma/x/scheduler"
 	"github.com/palomachain/paloma/x/scheduler/keeper"
 	schedulertypes "github.com/palomachain/paloma/x/scheduler/types"
 	skywaymodulekeeper "github.com/palomachain/paloma/x/skyway/keeper"
 	skywaymoduletypes "github.com/palomachain/paloma/x/skyway/types"
+	treasurykeeper "github.com/palomachain/paloma/x/treasury/keeper"
 	treasurymoduletypes "github.com/palomachain/paloma/x/treasury/types"
 	"github.com/palomachain/paloma/x/valset"
 	valsetmodulekeeper "github.com/palomachain/paloma/x/valset/keeper"
@@ -88,6 +91,8 @@ type fixture struct {
 	schedulerKeeper   schedulertypes.Keeper
 	stakingKeeper     stakingkeeper.Keeper
 	paramsKeeper      paramskeeper.Keeper
+	metrixKeeper      *metrixmodulekeeper.Keeper
+	treasuryKeeper    *treasurykeeper.Keeper
 }
 
 func init() {
@@ -118,6 +123,7 @@ func initFixture(t ginkgo.FullGinkgoTInterface) *fixture {
 		treasurymoduletypes.StoreKey,
 		evmmoduletypes.StoreKey,
 		skywaymoduletypes.StoreKey,
+		metrixmoduletypes.StoreKey,
 	)
 	cdc := moduletestutil.MakeTestEncodingConfig(
 		auth.AppModuleBasic{},
@@ -153,6 +159,7 @@ func initFixture(t ginkgo.FullGinkgoTInterface) *fixture {
 		evmmoduletypes.MemStoreKey,
 		treasurymoduletypes.MemStoreKey,
 		palomamoduletypes.MemStoreKey,
+		metrixmoduletypes.MemStoreKey,
 	)
 
 	accountKeeper := authkeeper.NewAccountKeeper(
@@ -215,12 +222,31 @@ func initFixture(t ginkgo.FullGinkgoTInterface) *fixture {
 		consensusRegistry,
 	)
 
-	evmKeeper := *evmmodulekeeper.NewKeeper(
+	var evmKeeper *evmmodulekeeper.Keeper = &evmmodulekeeper.Keeper{}
+	treasurykeeper := *treasurykeeper.NewKeeper(
+		appCodec,
+		runtime.NewKVStoreService(keys[treasurymoduletypes.StoreKey]),
+		helper.GetSubspace(treasurymoduletypes.ModuleName, paramsKeeper),
+		bankKeeper,
+		accountKeeper,
+		evmKeeper,
+	)
+	metrixKeeper := metrixmodulekeeper.NewKeeper(
+		appCodec,
+		runtime.NewKVStoreService(keys[metrixmoduletypes.StoreKey]),
+		helper.GetSubspace(metrixmoduletypes.ModuleName, paramsKeeper),
+		slashingKeeper,
+		stakingKeeper,
+		authcodec.NewBech32Codec(params2.ValidatorAddressPrefix),
+	)
+	*evmKeeper = *evmmodulekeeper.NewKeeper(
 		appCodec,
 		runtime.NewKVStoreService(keys[evmmoduletypes.StoreKey]),
 		consensusKeeper,
 		valsetKeeper,
 		authcodec.NewBech32Codec(params2.ValidatorAddressPrefix),
+		&metrixKeeper,
+		treasurykeeper,
 	)
 	capabilityKeeper := capabilitykeeper.NewKeeper(
 		appCodec,
@@ -321,6 +347,7 @@ func initFixture(t ginkgo.FullGinkgoTInterface) *fixture {
 		evmKeeper,
 	)
 	valsetKeeper.EvmKeeper = evmKeeper
+	valsetKeeper.SnapshotListeners = []valsetmoduletypes.OnSnapshotBuiltListener{evmKeeper, &metrixKeeper}
 	schedulerKeeper := *keeper.NewKeeper(
 		appCodec,
 		runtime.NewKVStoreService(keys[schedulertypes.StoreKey]),
@@ -334,7 +361,7 @@ func initFixture(t ginkgo.FullGinkgoTInterface) *fixture {
 	authModule := auth.NewAppModule(cdc, accountKeeper, authsims.RandomGenesisAccounts, nil)
 	bankModule := bank.NewAppModule(cdc, bankKeeper, accountKeeper, nil)
 	schedulerModule := scheduler.NewAppModule(cdc, schedulerKeeper, accountKeeper, bankKeeper)
-	evmModule := evm.NewAppModule(cdc, evmKeeper, accountKeeper, bankKeeper)
+	evmModule := evm.NewAppModule(cdc, *evmKeeper, accountKeeper, bankKeeper)
 	stakingModule := staking.NewAppModule(cdc, stakingKeeper, accountKeeper, bankKeeper, helper.GetSubspace(stakingtypes.ModuleName, paramsKeeper))
 	integrationApp := integration.NewIntegrationApp(newCtx, logger, keys, cdc, map[string]appmodule.AppModule{
 		authtypes.ModuleName:      authModule,
@@ -348,6 +375,7 @@ func initFixture(t ginkgo.FullGinkgoTInterface) *fixture {
 
 	queryClient := v1.NewQueryClient(integrationApp.QueryHelper())
 	legacyQueryClient := v1beta1.NewQueryClient(integrationApp.QueryHelper())
+	metrixKeeper.UpdateUptime(integrationApp.Context())
 	return &fixture{
 		ctx:               sdkCtx,
 		codec:             appCodec,
@@ -357,7 +385,9 @@ func initFixture(t ginkgo.FullGinkgoTInterface) *fixture {
 		valsetKeeper:      valsetKeeper,
 		schedulerKeeper:   schedulerKeeper,
 		paramsKeeper:      paramsKeeper,
-		evmKeeper:         evmKeeper,
+		evmKeeper:         *evmKeeper,
 		stakingKeeper:     *stakingKeeper,
+		metrixKeeper:      &metrixKeeper,
+		treasuryKeeper:    &treasurykeeper,
 	}
 }
diff --git a/tests/integration/paloma/keeper/test_helpers_test.go b/tests/integration/paloma/keeper/test_helpers_test.go
@@ -57,12 +57,16 @@ import (
 	"github.com/palomachain/paloma/x/evm"
 	evmmodulekeeper "github.com/palomachain/paloma/x/evm/keeper"
 	evmmoduletypes "github.com/palomachain/paloma/x/evm/types"
+	metrixmodulekeeper "github.com/palomachain/paloma/x/metrix/keeper"
+	metrixmoduletypes "github.com/palomachain/paloma/x/metrix/types"
 	"github.com/palomachain/paloma/x/paloma"
 	palomakeeper "github.com/palomachain/paloma/x/paloma/keeper"
 	palomamoduletypes "github.com/palomachain/paloma/x/paloma/types"
 	"github.com/palomachain/paloma/x/scheduler"
 	"github.com/palomachain/paloma/x/scheduler/keeper"
 	schedulertypes "github.com/palomachain/paloma/x/scheduler/types"
+	treasurykeeper "github.com/palomachain/paloma/x/treasury/keeper"
+	treasurymoduletypes "github.com/palomachain/paloma/x/treasury/types"
 	"github.com/palomachain/paloma/x/valset"
 	valsetmodulekeeper "github.com/palomachain/paloma/x/valset/keeper"
 	valsetmoduletypes "github.com/palomachain/paloma/x/valset/types"
@@ -184,12 +188,31 @@ func initFixture(t ginkgo.FullGinkgoTInterface) *fixture {
 		valsetKeeper,
 		consensusRegistry,
 	)
-	evmKeeper := *evmmodulekeeper.NewKeeper(
+	var evmKeeper *evmmodulekeeper.Keeper = &evmmodulekeeper.Keeper{}
+	treasurykeeper := *treasurykeeper.NewKeeper(
+		appCodec,
+		runtime.NewKVStoreService(keys[treasurymoduletypes.StoreKey]),
+		helper.GetSubspace(treasurymoduletypes.ModuleName, paramsKeeper),
+		bankKeeper,
+		accountKeeper,
+		evmKeeper,
+	)
+	metrixKeeper := metrixmodulekeeper.NewKeeper(
+		appCodec,
+		runtime.NewKVStoreService(keys[metrixmoduletypes.StoreKey]),
+		helper.GetSubspace(metrixmoduletypes.ModuleName, paramsKeeper),
+		slashingKeeper,
+		stakingKeeper,
+		authcodec.NewBech32Codec(params2.ValidatorAddressPrefix),
+	)
+	*evmKeeper = *evmmodulekeeper.NewKeeper(
 		appCodec,
 		runtime.NewKVStoreService(keys[evmmoduletypes.StoreKey]),
 		consensusKeeper,
 		valsetKeeper,
 		authcodec.NewBech32Codec(params2.ValidatorAddressPrefix),
+		metrixKeeper,
+		treasurykeeper,
 	)
 	consensusRegistry.Add(
 		evmKeeper,
@@ -264,7 +287,7 @@ func initFixture(t ginkgo.FullGinkgoTInterface) *fixture {
 	authModule := auth.NewAppModule(cdc, accountKeeper, authsims.RandomGenesisAccounts, nil)
 	bankModule := bank.NewAppModule(cdc, bankKeeper, accountKeeper, nil)
 	schedulerModule := scheduler.NewAppModule(cdc, schedulerKeeper, accountKeeper, bankKeeper)
-	evmModule := evm.NewAppModule(cdc, evmKeeper, accountKeeper, bankKeeper)
+	evmModule := evm.NewAppModule(cdc, *evmKeeper, accountKeeper, bankKeeper)
 	upgradeModule := upgrade.NewAppModule(&upgradeKeeper, address.NewBech32Codec("cosmos"))
 	palomaModule := paloma.NewAppModule(cdc, *palomaKeeper, accountKeeper, bankKeeper)
 
@@ -294,7 +317,7 @@ func initFixture(t ginkgo.FullGinkgoTInterface) *fixture {
 		valsetKeeper:      valsetKeeper,
 		schedulerKeeper:   schedulerKeeper,
 		paramsKeeper:      paramsKeeper,
-		evmKeeper:         evmKeeper,
+		evmKeeper:         *evmKeeper,
 		stakingKeeper:     *stakingKeeper,
 		palomaKeeper:      *palomaKeeper,
 		upgradeKeeper:     upgradeKeeper,