From 2ead657a2051f9895a5455fae4a88b7444ed7a77 Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 13 Sep 2024 19:38:34 +0300 Subject: [PATCH] Add refactoring on caching. --- blockchain/arbitrum_one/arbitrum_one.go | 250 +++++------------ .../arbitrum_sepolia/arbitrum_sepolia.go | 250 +++++------------ blockchain/blockchain.go.tmpl | 254 +++++------------- blockchain/common/decoding.go | 7 + blockchain/ethereum/ethereum.go | 250 +++++------------ .../game7_orbit_arbitrum_sepolia.go | 250 +++++------------ blockchain/game7_testnet/game7_testnet.go | 250 +++++------------ blockchain/handlers.go | 4 +- blockchain/imx_zkevm/imx_zkevm.go | 250 +++++------------ .../imx_zkevm_sepolia/imx_zkevm_sepolia.go | 250 +++++------------ blockchain/mantle/mantle.go | 250 +++++------------ blockchain/mantle_sepolia/mantle_sepolia.go | 250 +++++------------ blockchain/polygon/polygon.go | 105 +++++--- blockchain/sepolia/sepolia.go | 250 +++++------------ blockchain/xai/xai.go | 250 +++++------------ blockchain/xai_sepolia/xai_sepolia.go | 250 +++++------------ cmd.go | 2 +- indexer/db.go | 27 +- synchronizer/synchronizer.go | 69 +++-- 19 files changed, 1055 insertions(+), 2413 deletions(-) diff --git a/blockchain/arbitrum_one/arbitrum_one.go b/blockchain/arbitrum_one/arbitrum_one.go index 40039a7..dbfc662 100644 --- a/blockchain/arbitrum_one/arbitrum_one.go +++ b/blockchain/arbitrum_one/arbitrum_one.go @@ -878,170 +878,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1049,6 +925,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1058,13 +936,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1083,7 +961,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1096,17 +974,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1121,7 +989,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1131,23 +999,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1155,9 +1028,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1208,24 +1080,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1241,7 +1129,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go index 641bcc2..bf8bac9 100644 --- a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go +++ b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go @@ -878,170 +878,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1049,6 +925,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1058,13 +936,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1083,7 +961,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1096,17 +974,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1121,7 +989,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1131,23 +999,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1155,9 +1028,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1208,24 +1080,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1241,7 +1129,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/blockchain.go.tmpl b/blockchain/blockchain.go.tmpl index 4103158..80cfc05 100644 --- a/blockchain/blockchain.go.tmpl +++ b/blockchain/blockchain.go.tmpl @@ -881,172 +881,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } - -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1054,6 +928,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1063,13 +939,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1088,7 +964,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1101,17 +977,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1126,7 +992,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1136,23 +1002,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1160,9 +1031,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1213,24 +1083,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1246,7 +1132,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } @@ -1256,4 +1142,4 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return eventsLabels, nil -} \ No newline at end of file +} diff --git a/blockchain/common/decoding.go b/blockchain/common/decoding.go index ff41dca..a4008f0 100644 --- a/blockchain/common/decoding.go +++ b/blockchain/common/decoding.go @@ -101,6 +101,13 @@ type QueryFilter struct { Topics [][]string `json:"topics"` } +type BlockWithTransactions struct { + BlockNumber uint64 + BlockHash string + BlockTimestamp uint64 + Transactions map[string]TransactionJson +} + // ReadJsonBlocks reads blocks from a JSON file func ReadJsonBlocks() []*BlockJson { file, err := os.Open("data/blocks_go.json") diff --git a/blockchain/ethereum/ethereum.go b/blockchain/ethereum/ethereum.go index bfbc7d0..0d7190c 100644 --- a/blockchain/ethereum/ethereum.go +++ b/blockchain/ethereum/ethereum.go @@ -868,170 +868,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1039,6 +915,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1048,13 +926,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1073,7 +951,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1086,17 +964,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1111,7 +979,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1121,23 +989,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1145,9 +1018,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1198,24 +1070,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1231,7 +1119,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go index a434a3a..f25885a 100644 --- a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go +++ b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go @@ -878,170 +878,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1049,6 +925,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1058,13 +936,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1083,7 +961,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1096,17 +974,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1121,7 +989,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1131,23 +999,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1155,9 +1028,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1208,24 +1080,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1241,7 +1129,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/game7_testnet/game7_testnet.go b/blockchain/game7_testnet/game7_testnet.go index 7911b46..8f98bda 100644 --- a/blockchain/game7_testnet/game7_testnet.go +++ b/blockchain/game7_testnet/game7_testnet.go @@ -878,170 +878,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1049,6 +925,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1058,13 +936,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1083,7 +961,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1096,17 +974,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1121,7 +989,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1131,23 +999,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1155,9 +1028,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1208,24 +1080,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1241,7 +1129,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/handlers.go b/blockchain/handlers.go index 030fdc9..94f6ca4 100644 --- a/blockchain/handlers.go +++ b/blockchain/handlers.go @@ -92,8 +92,8 @@ type BlockchainClient interface { DecodeProtoTransactionsToLabels([]string, map[uint64]uint64, map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) ChainType() string GetCode(context.Context, common.Address, uint64) ([]byte, error) - GetTransactionsLabels(uint64, uint64, map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) - GetEventsLabels(uint64, uint64, map[string]map[string]map[string]string) ([]indexer.EventLabel, error) + GetTransactionsLabels(uint64, uint64, map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) + GetEventsLabels(uint64, uint64, map[string]map[string]map[string]string, map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) } func GetLatestBlockNumberWithRetry(client BlockchainClient, retryAttempts int, retryWaitTime time.Duration) (*big.Int, error) { diff --git a/blockchain/imx_zkevm/imx_zkevm.go b/blockchain/imx_zkevm/imx_zkevm.go index 79cfa2b..feae909 100644 --- a/blockchain/imx_zkevm/imx_zkevm.go +++ b/blockchain/imx_zkevm/imx_zkevm.go @@ -868,170 +868,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1039,6 +915,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1048,13 +926,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1073,7 +951,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1086,17 +964,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1111,7 +979,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1121,23 +989,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1145,9 +1018,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1198,24 +1070,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1231,7 +1119,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go index 30cae97..9fb847b 100644 --- a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go +++ b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go @@ -868,170 +868,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1039,6 +915,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1048,13 +926,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1073,7 +951,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1086,17 +964,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1111,7 +979,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1121,23 +989,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1145,9 +1018,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1198,24 +1070,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1231,7 +1119,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/mantle/mantle.go b/blockchain/mantle/mantle.go index fd37ca0..78cded5 100644 --- a/blockchain/mantle/mantle.go +++ b/blockchain/mantle/mantle.go @@ -868,170 +868,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1039,6 +915,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1048,13 +926,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1073,7 +951,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1086,17 +964,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1111,7 +979,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1121,23 +989,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1145,9 +1018,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1198,24 +1070,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1231,7 +1119,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/mantle_sepolia/mantle_sepolia.go b/blockchain/mantle_sepolia/mantle_sepolia.go index 537ab1e..c7c87e8 100644 --- a/blockchain/mantle_sepolia/mantle_sepolia.go +++ b/blockchain/mantle_sepolia/mantle_sepolia.go @@ -868,170 +868,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1039,6 +915,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1048,13 +926,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1073,7 +951,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1086,17 +964,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1111,7 +979,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1121,23 +989,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1145,9 +1018,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1198,24 +1070,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1231,7 +1119,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/polygon/polygon.go b/blockchain/polygon/polygon.go index dfc4a59..335c83a 100644 --- a/blockchain/polygon/polygon.go +++ b/blockchain/polygon/polygon.go @@ -874,19 +874,40 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -894,6 +915,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -903,13 +926,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -928,7 +951,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -941,17 +964,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -966,7 +979,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -976,23 +989,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1000,9 +1018,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1053,24 +1070,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1086,7 +1119,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/sepolia/sepolia.go b/blockchain/sepolia/sepolia.go index 6618085..d4c116d 100644 --- a/blockchain/sepolia/sepolia.go +++ b/blockchain/sepolia/sepolia.go @@ -868,170 +868,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1039,6 +915,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1048,13 +926,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1073,7 +951,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1086,17 +964,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1111,7 +979,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1121,23 +989,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1145,9 +1018,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1198,24 +1070,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1231,7 +1119,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/xai/xai.go b/blockchain/xai/xai.go index 248b668..903e27e 100644 --- a/blockchain/xai/xai.go +++ b/blockchain/xai/xai.go @@ -878,170 +878,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1049,6 +925,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1058,13 +936,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1083,7 +961,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1096,17 +974,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1121,7 +989,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1131,23 +999,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1155,9 +1028,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1208,24 +1080,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1241,7 +1129,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/xai_sepolia/xai_sepolia.go b/blockchain/xai_sepolia/xai_sepolia.go index 68290d3..7403f4d 100644 --- a/blockchain/xai_sepolia/xai_sepolia.go +++ b/blockchain/xai_sepolia/xai_sepolia.go @@ -878,170 +878,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1049,6 +925,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1058,13 +936,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1083,7 +961,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1096,17 +974,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1121,7 +989,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1131,23 +999,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1155,9 +1028,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1208,24 +1080,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1241,7 +1129,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/cmd.go b/cmd.go index 602f85c..d4a5e21 100644 --- a/cmd.go +++ b/cmd.go @@ -906,7 +906,7 @@ func CreateHistoricalSyncCommand() *cobra.Command { return synchonizerErr } - err := newSynchronizer.HistoricalSyncRef(customerDbUriFlag, customerIds, addresses, batchSize, auto, threads) + err := newSynchronizer.HistoricalSyncRef(customerDbUriFlag, addresses, customerIds, batchSize, auto, threads) if err != nil { return err diff --git a/indexer/db.go b/indexer/db.go index 6d325a6..09e29b9 100644 --- a/indexer/db.go +++ b/indexer/db.go @@ -1152,18 +1152,37 @@ func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, cus WHERE chain = @chain AND ((abi::jsonb)->>'type' = 'function' or (abi::jsonb)->>'type' = 'event') `) + fmt.Println("Addresses:", addresses) + if len(addresses) > 0 { queryBuilder.WriteString(" AND address = ANY(@addresses) ") - queryArgs["addresses"] = addresses + + // decode addresses + addressesBytes := make([][]byte, len(addresses)) + for i, address := range addresses { + addressBytes, err := decodeAddress(address) + if err != nil { + return nil, nil, err + } + addressesBytes[i] = addressBytes // Assign directly to the index + } + + queryArgs["addresses"] = addressesBytes } + fmt.Println("Customer IDs:", customersIds) + if len(customersIds) > 0 { queryBuilder.WriteString(" AND customer_id = ANY(@customer_ids) ") queryArgs["customer_ids"] = customersIds } + fmt.Println("Query:", queryBuilder.String()) + fmt.Println("Query Args:", queryArgs) + rows, err := conn.Query(context.Background(), queryBuilder.String(), queryArgs) if err != nil { + log.Println("Error querying ABI jobs from database", err) return nil, nil, err } @@ -1200,8 +1219,14 @@ func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, cus "abi_type": abiJob.AbiType, } + if abiJob.DeploymentBlockNumber == nil { + value := uint64(1) + abiJob.DeploymentBlockNumber = &value + } + // Retrieve the struct from the map deployInfo, exists := addressDeployBlockDict[address] + if !exists { // Initialize the struct if it doesn't exist deployInfo = AbiJobsDeployInfo{ diff --git a/synchronizer/synchronizer.go b/synchronizer/synchronizer.go index 20976fa..0f2a25b 100644 --- a/synchronizer/synchronizer.go +++ b/synchronizer/synchronizer.go @@ -14,6 +14,7 @@ import ( "time" seer_blockchain "github.com/moonstream-to/seer/blockchain" + "github.com/moonstream-to/seer/blockchain/common" "github.com/moonstream-to/seer/crawler" "github.com/moonstream-to/seer/indexer" "github.com/moonstream-to/seer/storage" @@ -410,6 +411,7 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s return fmt.Errorf("error getting latest block number: %w", err) } d.startBlock = indexedLatestBlock + fmt.Printf("Start block is %d\n", d.startBlock) } // Automatically update ABI jobs as active if auto mode is enabled @@ -425,9 +427,13 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s return fmt.Errorf("error selecting ABI jobs: %w", err) } + fmt.Printf("Found %d customer updates\n", len(customerUpdates)) + // Filter out blocks more for address, abisInfo := range addressesAbisInfo { + fmt.Printf("Address %s, block %d\n", address, abisInfo.DeployedBlockNumber) if abisInfo.DeployedBlockNumber > d.startBlock { + fmt.Printf("Deleting address %s\n", address, abisInfo.DeployedBlockNumber) delete(addressesAbisInfo, address) } } @@ -485,10 +491,14 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s } if useRPC { - d.endBlock = d.startBlock - batchSize - if d.endBlock == 0 { - d.endBlock = 1 + // Calculate the end block + if d.startBlock > batchSize { + d.endBlock = d.startBlock - batchSize + } else { + d.endBlock = 1 // Prevent underflow by setting endBlock to 1 if startBlock < batchSize } + + fmt.Printf("Start block is %d, end block is %d\n", d.startBlock, d.endBlock) } // Read raw data from storage or via RPC @@ -529,7 +539,11 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s for _, update := range customerUpdates { wg.Add(1) - go d.processProtoCustomerUpdate(update, rawData, customerDBConnections, sem, errChan, &wg) + if useRPC { + go d.processRPCCustomerUpdate(update, customerDBConnections, sem, errChan, &wg) + } else { + go d.processProtoCustomerUpdate(update, rawData, customerDBConnections, sem, errChan, &wg) + } } wg.Wait() @@ -631,7 +645,8 @@ func (d *Synchronizer) processRPCCustomerUpdate( for address, selectorMap := range update.Abis { for selector, abiInfo := range selectorMap { - if abiInfo["type"] == "event" { + + if abiInfo["abi_type"] == "event" { if _, ok := eventAbis[address]; !ok { eventAbis[address] = make(map[string]map[string]string) } @@ -646,30 +661,50 @@ func (d *Synchronizer) processRPCCustomerUpdate( } - // Read transactions and events from the blockchain - // Transactions - transactions, err := d.Client.GetTransactionsLabels(d.startBlock, d.endBlock, transactionAbis) + var transactions []indexer.TransactionLabel + var blocksCache map[uint64]common.BlockWithTransactions - if err != nil { - errChan <- fmt.Errorf("error getting transactions for customer %s: %w", update.CustomerID, err) - <-sem // Release semaphore - return + if len(transactionAbis) != 0 { + + transactions, blocksCache, err = d.Client.GetTransactionsLabels(d.startBlock, d.endBlock, transactionAbis) + + if err != nil { + log.Println("Error getting transactions for customer", update.CustomerID, ":", err) + errChan <- fmt.Errorf("error getting transactions for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + } + } else { + transactions = make([]indexer.TransactionLabel, 0) } + fmt.Printf("Transactions length: %d\n", len(transactions)) + // Events - events, err := d.Client.GetEventsLabels(d.startBlock, d.endBlock, eventAbis) + var events []indexer.EventLabel - if err != nil { + if len(eventAbis) != 0 { - errChan <- fmt.Errorf("error getting events for customer %s: %w", update.CustomerID, err) - <-sem // Release semaphore - return + events, err = d.Client.GetEventsLabels(d.endBlock, d.startBlock, eventAbis, blocksCache) + if err != nil { + + log.Println("Error getting events for customer", update.CustomerID, ":", err) + + errChan <- fmt.Errorf("error getting events for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + + } + } else { + events = make([]indexer.EventLabel, 0) } + fmt.Printf("Events length: %d\n", len(events)) + customer.Pgx.WriteLabes(d.blockchain, transactions, events) <-sem // Release semaphore