From e32848cee0e4d00242821ea3a79ffce6bff666a7 Mon Sep 17 00:00:00 2001 From: wjrjerome Date: Fri, 8 Nov 2024 23:26:30 +1100 Subject: [PATCH 1/2] fix: retry when block not found --- internal/clients/bbnclient/bbnclient.go | 58 ++++++++++++++++++++----- internal/clients/bbnclient/interface.go | 4 +- internal/services/bootstrap.go | 2 +- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/internal/clients/bbnclient/bbnclient.go b/internal/clients/bbnclient/bbnclient.go index d391c8a..d0e8aab 100644 --- a/internal/clients/bbnclient/bbnclient.go +++ b/internal/clients/bbnclient/bbnclient.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "strings" + "time" "github.com/babylonlabs-io/babylon-staking-indexer/internal/types" "github.com/babylonlabs-io/babylon/client/config" @@ -14,6 +15,13 @@ import ( "github.com/rs/zerolog/log" ) +const ( + // Backoff parameters for retries for getting BBN block result + initialBackoff = 500 * time.Millisecond // Start with 500ms + backoffFactor = 2 // Exponential backoff factor + maxRetries = 10 // 8 minutes in worst case +) + type BbnClient struct { queryClient *query.QueryClient } @@ -38,18 +46,6 @@ func (c *BbnClient) GetLatestBlockNumber(ctx context.Context) (int64, *types.Err return status.SyncInfo.LatestBlockHeight, nil } -func (c *BbnClient) GetBlockResults(ctx context.Context, blockHeight int64) (*ctypes.ResultBlockResults, *types.Error) { - resp, err := c.queryClient.RPCClient.BlockResults(ctx, &blockHeight) - if err != nil { - return nil, types.NewErrorWithMsg( - http.StatusInternalServerError, - types.InternalServiceError, - fmt.Sprintf("failed to get block results for block %d: %s", blockHeight, err.Error()), - ) - } - return resp, nil -} - func (c *BbnClient) GetCheckpointParams(ctx context.Context) (*CheckpointParams, *types.Error) { params, err := c.queryClient.BTCCheckpointParams() if err != nil { @@ -107,6 +103,44 @@ func (c *BbnClient) GetAllStakingParams(ctx context.Context) (map[uint32]*Stakin return allParams, nil } +// GetBlockResultsWithRetry retries the `getBlockResults` method with exponential backoff +// when the block is not yet available. +func (c *BbnClient) GetBlockResultsWithRetry( + ctx context.Context, blockHeight *int64, +) (*ctypes.ResultBlockResults, *types.Error) { + backoff := initialBackoff + var resp *ctypes.ResultBlockResults + var err *types.Error + + for i := 0; i < maxRetries; i++ { + resp, err = c.getBlockResults(ctx, blockHeight) + if err == nil { + return resp, nil + } + + if strings.Contains( + err.Err.Error(), + "must be less than or equal to the current blockchain height", + ) { + log.Debug(). + Str("block_height", fmt.Sprintf("%d", *blockHeight)). + Str("backoff", backoff.String()). + Msg("Block not yet available, retrying...") + time.Sleep(backoff) + backoff *= backoffFactor + continue + } + return nil, err + } + + // If we exhaust retries, return a not found error + return nil, types.NewErrorWithMsg( + http.StatusNotFound, + types.NotFound, + fmt.Sprintf("Block height %d not found after retries", *blockHeight), + ) +} + func (c *BbnClient) Subscribe(subscriber, query string, outCapacity ...int) (out <-chan ctypes.ResultEvent, err error) { return c.queryClient.RPCClient.Subscribe(context.Background(), subscriber, query, outCapacity...) } diff --git a/internal/clients/bbnclient/interface.go b/internal/clients/bbnclient/interface.go index 022a762..f53c176 100644 --- a/internal/clients/bbnclient/interface.go +++ b/internal/clients/bbnclient/interface.go @@ -11,8 +11,8 @@ type BbnInterface interface { GetCheckpointParams(ctx context.Context) (*CheckpointParams, *types.Error) GetAllStakingParams(ctx context.Context) (map[uint32]*StakingParams, *types.Error) GetLatestBlockNumber(ctx context.Context) (int64, *types.Error) - GetBlockResults( - ctx context.Context, blockHeight int64, + GetBlockResultsWithRetry( + ctx context.Context, blockHeight *int64, ) (*ctypes.ResultBlockResults, *types.Error) Subscribe(subscriber, query string, outCapacity ...int) (out <-chan ctypes.ResultEvent, err error) UnsubscribeAll(subscriber string) error diff --git a/internal/services/bootstrap.go b/internal/services/bootstrap.go index df5b7fc..bcea1d2 100644 --- a/internal/services/bootstrap.go +++ b/internal/services/bootstrap.go @@ -105,7 +105,7 @@ func (s *Service) getEventsFromBlock( ctx context.Context, blockHeight int64, ) ([]BbnEvent, *types.Error) { events := make([]BbnEvent, 0) - blockResult, err := s.bbn.GetBlockResults(ctx, blockHeight) + blockResult, err := s.bbn.GetBlockResultsWithRetry(ctx, &blockHeight) if err != nil { return nil, err } From 250d4215f9848d3d54f5ac7f18d9330f42752cf9 Mon Sep 17 00:00:00 2001 From: Gurjot Date: Mon, 11 Nov 2024 11:24:39 +0530 Subject: [PATCH 2/2] fix interface and mocks --- internal/clients/btcclient/interface.go | 2 +- internal/services/service.go | 4 ++-- tests/mocks/mock_bbn_client.go | 27 ++++++++++++------------- tests/mocks/mock_btc_client.go | 14 ++++++------- 4 files changed, 23 insertions(+), 24 deletions(-) diff --git a/internal/clients/btcclient/interface.go b/internal/clients/btcclient/interface.go index 1a6a73b..a6f89c6 100644 --- a/internal/clients/btcclient/interface.go +++ b/internal/clients/btcclient/interface.go @@ -1,5 +1,5 @@ package btcclient -type Client interface { +type BtcInterface interface { GetTipHeight() (uint64, error) } diff --git a/internal/services/service.go b/internal/services/service.go index 4843e44..bbe8120 100644 --- a/internal/services/service.go +++ b/internal/services/service.go @@ -20,7 +20,7 @@ type Service struct { cfg *config.Config db db.DbInterface - btc btcclient.Client + btc btcclient.BtcInterface btcNotifier notifier.ChainNotifier bbn bbnclient.BbnInterface queueManager *queue.QueueManager @@ -31,7 +31,7 @@ type Service struct { func NewService( cfg *config.Config, db db.DbInterface, - btc btcclient.Client, + btc btcclient.BtcInterface, btcNotifier notifier.ChainNotifier, bbn bbnclient.BbnInterface, qm *queue.QueueManager, diff --git a/tests/mocks/mock_bbn_client.go b/tests/mocks/mock_bbn_client.go index 12f9684..2d67d0f 100644 --- a/tests/mocks/mock_bbn_client.go +++ b/tests/mocks/mock_bbn_client.go @@ -3,11 +3,10 @@ package mocks import ( - bbnclient "github.com/babylonlabs-io/babylon-staking-indexer/internal/clients/bbnclient" - btccheckpointtypes "github.com/babylonlabs-io/babylon/x/btccheckpoint/types" - context "context" + bbnclient "github.com/babylonlabs-io/babylon-staking-indexer/internal/clients/bbnclient" + coretypes "github.com/cometbft/cometbft/rpc/core/types" mock "github.com/stretchr/testify/mock" @@ -52,20 +51,20 @@ func (_m *BbnInterface) GetAllStakingParams(ctx context.Context) (map[uint32]*bb return r0, r1 } -// GetBlockResults provides a mock function with given fields: ctx, blockHeight -func (_m *BbnInterface) GetBlockResults(ctx context.Context, blockHeight int64) (*coretypes.ResultBlockResults, *types.Error) { +// GetBlockResultsWithRetry provides a mock function with given fields: ctx, blockHeight +func (_m *BbnInterface) GetBlockResultsWithRetry(ctx context.Context, blockHeight *int64) (*coretypes.ResultBlockResults, *types.Error) { ret := _m.Called(ctx, blockHeight) if len(ret) == 0 { - panic("no return value specified for GetBlockResults") + panic("no return value specified for GetBlockResultsWithRetry") } var r0 *coretypes.ResultBlockResults var r1 *types.Error - if rf, ok := ret.Get(0).(func(context.Context, int64) (*coretypes.ResultBlockResults, *types.Error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *int64) (*coretypes.ResultBlockResults, *types.Error)); ok { return rf(ctx, blockHeight) } - if rf, ok := ret.Get(0).(func(context.Context, int64) *coretypes.ResultBlockResults); ok { + if rf, ok := ret.Get(0).(func(context.Context, *int64) *coretypes.ResultBlockResults); ok { r0 = rf(ctx, blockHeight) } else { if ret.Get(0) != nil { @@ -73,7 +72,7 @@ func (_m *BbnInterface) GetBlockResults(ctx context.Context, blockHeight int64) } } - if rf, ok := ret.Get(1).(func(context.Context, int64) *types.Error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *int64) *types.Error); ok { r1 = rf(ctx, blockHeight) } else { if ret.Get(1) != nil { @@ -85,23 +84,23 @@ func (_m *BbnInterface) GetBlockResults(ctx context.Context, blockHeight int64) } // GetCheckpointParams provides a mock function with given fields: ctx -func (_m *BbnInterface) GetCheckpointParams(ctx context.Context) (*btccheckpointtypes.Params, *types.Error) { +func (_m *BbnInterface) GetCheckpointParams(ctx context.Context) (*bbnclient.CheckpointParams, *types.Error) { ret := _m.Called(ctx) if len(ret) == 0 { panic("no return value specified for GetCheckpointParams") } - var r0 *btccheckpointtypes.Params + var r0 *bbnclient.CheckpointParams var r1 *types.Error - if rf, ok := ret.Get(0).(func(context.Context) (*btccheckpointtypes.Params, *types.Error)); ok { + if rf, ok := ret.Get(0).(func(context.Context) (*bbnclient.CheckpointParams, *types.Error)); ok { return rf(ctx) } - if rf, ok := ret.Get(0).(func(context.Context) *btccheckpointtypes.Params); ok { + if rf, ok := ret.Get(0).(func(context.Context) *bbnclient.CheckpointParams); ok { r0 = rf(ctx) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*btccheckpointtypes.Params) + r0 = ret.Get(0).(*bbnclient.CheckpointParams) } } diff --git a/tests/mocks/mock_btc_client.go b/tests/mocks/mock_btc_client.go index 83c0ef2..a97c702 100644 --- a/tests/mocks/mock_btc_client.go +++ b/tests/mocks/mock_btc_client.go @@ -9,23 +9,23 @@ type BtcInterface struct { mock.Mock } -// GetBlockCount provides a mock function with given fields: -func (_m *BtcInterface) GetBlockCount() (int64, error) { +// GetTipHeight provides a mock function with given fields: +func (_m *BtcInterface) GetTipHeight() (uint64, error) { ret := _m.Called() if len(ret) == 0 { - panic("no return value specified for GetBlockCount") + panic("no return value specified for GetTipHeight") } - var r0 int64 + var r0 uint64 var r1 error - if rf, ok := ret.Get(0).(func() (int64, error)); ok { + if rf, ok := ret.Get(0).(func() (uint64, error)); ok { return rf() } - if rf, ok := ret.Get(0).(func() int64); ok { + if rf, ok := ret.Get(0).(func() uint64); ok { r0 = rf() } else { - r0 = ret.Get(0).(int64) + r0 = ret.Get(0).(uint64) } if rf, ok := ret.Get(1).(func() error); ok {