Skip to content

Commit

Permalink
Add refactoring on caching.
Browse files Browse the repository at this point in the history
  • Loading branch information
Andrey committed Sep 13, 2024
1 parent a25e1f4 commit 2ead657
Show file tree
Hide file tree
Showing 19 changed files with 1,055 additions and 2,413 deletions.
250 changes: 69 additions & 181 deletions blockchain/arbitrum_one/arbitrum_one.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -878,177 +878,55 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
return labels, nil
}

// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) {
// var labels []indexer.TransactionLabel
// var decodedArgs map[string]interface{}
// var decodeErr error

// for _, transaction := range transactions {
// var transactionJson seer_common.TransactionJson
// err := json.Unmarshal([]byte(transaction), &transactionJson)
// if err != nil {
// return nil, err
// }

// label := indexer.SeerCrawlerLabel

// selector := transactionJson.Input[:10]

// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"]))

// if err != nil {
// return nil, err
// }

// inputData, err := hex.DecodeString(transactionJson.Input[2:])
// if err != nil {
// fmt.Println("Error decoding input data: ", err)
// return nil, err
// }

// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData)

// if decodeErr != nil {
// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr)
// decodedArgs = map[string]interface{}{
// "input_raw": transactionJson,
// "abi": abiMap[transactionJson.ToAddress][selector]["abi"],
// "selector": selector,
// "error": decodeErr,
// }
// label = indexer.SeerCrawlerRawLabel
// }

// labelDataBytes, err := json.Marshal(decodedArgs)
// if err != nil {
// fmt.Println("Error converting decodedArgs to JSON: ", err)
// return nil, err
// }

// // Convert JSON byte slice to string
// labelDataString := string(labelDataBytes)

// // Convert transaction to label
// transactionLabel := indexer.TransactionLabel{
// Address: transactionJson.ToAddress,
// BlockNumber: transactionJson.BlockNumber,
// BlockHash: transactionJson.BlockHash,
// CallerAddress: transactionJson.FromAddress,
// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"],
// LabelType: "tx_call",
// OriginAddress: transactionJson.FromAddress,
// Label: label,
// TransactionHash: transactionJson.Hash,
// LabelData: labelDataString,
// BlockTimestamp: transactionJson.BlockTimestamp,
// }

// labels = append(labels, transactionLabel)

// }

// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) {
// var labels []indexer.EventLabel
// var decodedArgs map[string]interface{}
// var decodeErr error

// for _, event := range events {
// var eventJson seer_common.EventJson
// err := json.Unmarshal([]byte(event), &eventJson)
// if err != nil {
// return nil, err
// }

// label := indexer.SeerCrawlerLabel

// var topicSelector string

// if len(eventJson.Topics) > 0 {
// topicSelector = eventJson.Topics[0]
// } else {
// // 0x0 is the default topic selector
// topicSelector = "0x0"
// }

// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil {
// continue
// }

// // Get the ABI string
// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"]))
// if err != nil {
// fmt.Println("Error initializing contract ABI: ", err)
// return nil, err
// }

// // Decode the event data
// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data)
// if decodeErr != nil {
// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr)
// decodedArgs = map[string]interface{}{
// "input_raw": eventJson,
// "abi": abiMap[eventJson.Address][topicSelector]["abi"],
// "selector": topicSelector,
// "error": decodeErr,
// }
// label = indexer.SeerCrawlerRawLabel
// }

// // Convert decodedArgs map to JSON
// labelDataBytes, err := json.Marshal(decodedArgs)
// if err != nil {
// fmt.Println("Error converting decodedArgs to JSON: ", err)
// return nil, err
// }

// // Convert event to label
// eventLabel := indexer.EventLabel{
// Label: label,
// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"],
// LabelType: "event",
// BlockNumber: eventJson.BlockNumber,
// BlockHash: eventJson.BlockHash,
// Address: eventJson.Address,
// OriginAddress: eventJson.OriginAddress,
// TransactionHash: eventJson.TransactionHash,
// LabelData: string(labelDataBytes), // Convert JSON byte slice to string
// BlockTimestamp: eventJson.BlockTimestamp,
// LogIndex: eventJson.LogIndex,
// }

// labels = append(labels, eventLabel)

// }

// return labels, nil
// }

func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) {
var tx *seer_common.TransactionJson
err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash)
return tx, err
}

func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) {
func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) {
var transactionsLabels []indexer.TransactionLabel

var blocksCache map[uint64]seer_common.BlockWithTransactions

// Get blocks in range
blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1)

if err != nil {
return nil, err
return nil, nil, err
}

// Get transactions in range

for _, block := range blocks {

blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64)
if err != nil {
log.Fatalf("Failed to convert BlockNumber to uint64: %v", err)
}

blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64)

if err != nil {
log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err)
}

blocksCache[blockNumber] = seer_common.BlockWithTransactions{
BlockNumber: blockNumber,
BlockHash: block.Hash,
BlockTimestamp: blockTimestamp,
Transactions: make(map[string]seer_common.TransactionJson),
}

for _, tx := range block.Transactions {

label := indexer.SeerCrawlerLabel

if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer
continue
}
// Fill blocks cache
blocksCache[blockNumber].Transactions[tx.Hash] = tx

// Process transaction labels

Expand All @@ -1058,13 +936,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa
txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"]))
if err != nil {
fmt.Println("Error initializing contract ABI transactions: ", err)
return nil, err
return nil, nil, err
}

inputData, err := hex.DecodeString(tx.Input[2:])
if err != nil {
fmt.Println("Error decoding input data: ", err)
return nil, err
return nil, nil, err
}

decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData)
Expand All @@ -1083,7 +961,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa

if err != nil {
fmt.Println("Error fetching transaction receipt: ", err)
return nil, err
return nil, nil, err
}

// check if the transaction was successful
Expand All @@ -1096,17 +974,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa
txLabelDataBytes, err := json.Marshal(decodedArgsTx)
if err != nil {
fmt.Println("Error converting decodedArgsTx to JSON: ", err)
return nil, err
}

blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64)
if err != nil {
log.Fatalf("Failed to convert BlockNumber to uint64: %v", err)
}

uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64)
if err != nil {
log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err)
return nil, nil, err
}

// Convert transaction to label
Expand All @@ -1121,7 +989,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa
Label: label,
TransactionHash: tx.Hash,
LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string
BlockTimestamp: uintBlockTimestamp,
BlockTimestamp: blockTimestamp,
}

transactionsLabels = append(transactionsLabels, transactionLabel)
Expand All @@ -1131,33 +999,37 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa

}

return transactionsLabels, nil
return transactionsLabels, blocksCache, nil

}

func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) {
func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) {
var eventsLabels []indexer.EventLabel

if blocksCache == nil {
blocksCache = make(map[uint64]seer_common.BlockWithTransactions)
}

// Get events in range

var addresses []common.Address
var topics [][]common.Hash
var topics []common.Hash

for address, selectorMap := range abiMap {
for selector, _ := range selectorMap {
addresses = append(addresses, common.HexToAddress(address))
topics = append(topics, []common.Hash{common.HexToHash(selector)})
topics = append(topics, common.HexToHash(selector))
}

addresses = append(addresses, common.HexToAddress(address))
}

// query filter from abiMap
filter := ethereum.FilterQuery{
FromBlock: big.NewInt(int64(startBlock)),
ToBlock: big.NewInt(int64(endBlock)),
Addresses: addresses,
Topics: topics,
Topics: [][]common.Hash{topics},
}

logs, err := c.ClientFilterLogs(context.Background(), filter, false)

if err != nil {
Expand Down Expand Up @@ -1208,24 +1080,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[
return nil, err
}

// get transaction information
transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash)

blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64)
if err != nil {
return nil, err
}

blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64)
if err != nil {
return nil, err
}
if _, ok := blocksCache[blockNumber]; !ok {

// get block from rpc
block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true)
if err != nil {
return nil, err
}

blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64)
if err != nil {
return nil, err
}

blocksCache[blockNumber] = seer_common.BlockWithTransactions{
BlockNumber: blockNumber,
BlockHash: block.Hash,
BlockTimestamp: blockTimestamp,
Transactions: make(map[string]seer_common.TransactionJson),
}

for _, tx := range block.Transactions {
blocksCache[blockNumber].Transactions[tx.Hash] = tx
}

blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64)
if err != nil {
return nil, err
}

logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64)
transaction := blocksCache[blockNumber].Transactions[log.TransactionHash]

logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64)
if err != nil {
return nil, err
}
Expand All @@ -1241,7 +1129,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[
OriginAddress: transaction.FromAddress,
TransactionHash: log.TransactionHash,
LabelData: string(labelDataBytes), // Convert JSON byte slice to string
BlockTimestamp: blockTimestamp,
BlockTimestamp: blocksCache[blockNumber].BlockTimestamp,
LogIndex: logIndex,
}

Expand Down
Loading

0 comments on commit 2ead657

Please sign in to comment.