diff --git a/blockchain/arbitrum_one/arbitrum_one.go b/blockchain/arbitrum_one/arbitrum_one.go index a867fc7..5ae4996 100644 --- a/blockchain/arbitrum_one/arbitrum_one.go +++ b/blockchain/arbitrum_one/arbitrum_one.go @@ -9,13 +9,14 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +72,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +106,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +207,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -214,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -224,7 +246,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -342,7 +364,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("arbitrum_one", @@ -373,14 +394,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &ArbitrumOneBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *ArbitrumOneBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -457,10 +478,10 @@ func ToEntireBlocksBatchFromLogProto(obj *ArbitrumOneBlocksBatch) *seer_common.B BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -491,10 +512,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *ArbitrumOneBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -612,49 +633,381 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*ArbitrumOneBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch ArbitrumOneBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch ArbitrumOneBlocksBatch dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *ArbitrumOneBlock) { + defer wg.Done() + defer func() { <-semaphoreChan }() + + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel + + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + localTxLabels = append(localTxLabels, transactionLabel) + } + + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel + + var topicSelector string + + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } + + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } + + localEventLabels = append(localEventLabels, eventLabel) + } + } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } + + return labels, txLabels, nil +} + +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { + + decodedTransactions, err := c.DecodeProtoTransactions(transactions) + + if err != nil { + return nil, err + } + + var labels []indexer.TransactionLabel + var decodedArgs map[string]interface{} + var decodeErr error + + for _, transaction := range decodedTransactions { + + label := indexer.SeerCrawlerLabel + + selector := transaction.Input[:10] + + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } + } + + inputData, err := hex.DecodeString(transaction.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) + + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) + decodedArgs = map[string]interface{}{ + "input_raw": transaction, + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + labelDataBytes, err := json.Marshal(decodedArgs) + if err != nil { + fmt.Println("Error converting decodedArgs to JSON: ", err) + return nil, err + } + + // Convert JSON byte slice to string + labelDataString := string(labelDataBytes) + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: transaction.ToAddress, + BlockNumber: transaction.BlockNumber, + BlockHash: transaction.BlockHash, + CallerAddress: transaction.FromAddress, + LabelName: abiMap[transaction.ToAddress][selector].AbiName, + LabelType: "tx_call", + OriginAddress: transaction.FromAddress, + Label: label, + TransactionHash: transaction.Hash, + LabelData: labelDataString, + BlockTimestamp: blocksCache[transaction.BlockNumber], + } + + labels = append(labels, transactionLabel) + + } + + return labels, nil +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels + selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) + + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return + } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) return nil, nil, err } @@ -664,18 +1017,32 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiEntryTx.AbiJSON, + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -685,161 +1052,169 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Convert transaction to label transactionLabel := indexer.TransactionLabel{ Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, + BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + BlockTimestamp: blockTimestamp, } - txLabels = append(txLabels, transactionLabel) + transactionsLabels = append(transactionsLabels, transactionLabel) } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel - - var topicSelector string + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + return transactionsLabels, blocksCache, nil - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } +} +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Get events in range - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + var addresses []common.Address + var topics []common.Hash - labels = append(labels, eventLabel) - } + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) } - } - return labels, txLabels, nil -} - -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + addresses = append(addresses, common.HexToAddress(address)) + } - decodedTransactions, err := c.DecodeProtoTransactions(transactions) + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { return nil, err } - var labels []indexer.TransactionLabel - var decodedArgs map[string]interface{} - var decodeErr error + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + var topicSelector string - for _, transaction := range decodedTransactions { + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - label := indexer.SeerCrawlerLabel + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } - selector := transaction.Input[:10] + abiEntryLog := abiMap[log.Address][topicSelector] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) return nil, err } - inputData, err := hex.DecodeString(transaction.Input[2:]) + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { - fmt.Println("Error decoding input data: ", err) return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + if _, ok := blocksCache[blockNumber]; !ok { - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) - decodedArgs = map[string]interface{}{ - "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err } - label = indexer.SeerCrawlerRawLabel + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + } - labelDataBytes, err := json.Marshal(decodedArgs) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { - fmt.Println("Error converting decodedArgs to JSON: ", err) return nil, err } - // Convert JSON byte slice to string - labelDataString := string(labelDataBytes) - - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: transaction.ToAddress, - BlockNumber: transaction.BlockNumber, - BlockHash: transaction.BlockHash, - CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: transaction.FromAddress, + // Convert event to label + eventLabel := indexer.EventLabel{ Label: label, - TransactionHash: transaction.Hash, - LabelData: labelDataString, - BlockTimestamp: blocksCache[transaction.BlockNumber], + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, } - labels = append(labels, transactionLabel) + eventsLabels = append(eventsLabels, eventLabel) } - return labels, nil -} \ No newline at end of file + return eventsLabels, nil + +} diff --git a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go index 868ca3d..bdfd94f 100644 --- a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go +++ b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go @@ -9,13 +9,14 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +72,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +106,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +207,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -214,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -224,7 +246,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -342,7 +364,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("arbitrum_sepolia", @@ -373,14 +394,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &ArbitrumSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *ArbitrumSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -457,10 +478,10 @@ func ToEntireBlocksBatchFromLogProto(obj *ArbitrumSepoliaBlocksBatch) *seer_comm BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -491,10 +512,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *ArbitrumSepoliaBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -612,49 +633,381 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*ArbitrumSepoliaBlock, erro func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch ArbitrumSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch ArbitrumSepoliaBlocksBatch dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *ArbitrumSepoliaBlock) { + defer wg.Done() + defer func() { <-semaphoreChan }() + + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel + + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + localTxLabels = append(localTxLabels, transactionLabel) + } + + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel + + var topicSelector string + + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } + + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } + + localEventLabels = append(localEventLabels, eventLabel) + } + } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } + + return labels, txLabels, nil +} + +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { + + decodedTransactions, err := c.DecodeProtoTransactions(transactions) + + if err != nil { + return nil, err + } + + var labels []indexer.TransactionLabel + var decodedArgs map[string]interface{} + var decodeErr error + + for _, transaction := range decodedTransactions { + + label := indexer.SeerCrawlerLabel + + selector := transaction.Input[:10] + + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } + } + + inputData, err := hex.DecodeString(transaction.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) + + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) + decodedArgs = map[string]interface{}{ + "input_raw": transaction, + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + labelDataBytes, err := json.Marshal(decodedArgs) + if err != nil { + fmt.Println("Error converting decodedArgs to JSON: ", err) + return nil, err + } + + // Convert JSON byte slice to string + labelDataString := string(labelDataBytes) + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: transaction.ToAddress, + BlockNumber: transaction.BlockNumber, + BlockHash: transaction.BlockHash, + CallerAddress: transaction.FromAddress, + LabelName: abiMap[transaction.ToAddress][selector].AbiName, + LabelType: "tx_call", + OriginAddress: transaction.FromAddress, + Label: label, + TransactionHash: transaction.Hash, + LabelData: labelDataString, + BlockTimestamp: blocksCache[transaction.BlockNumber], + } + + labels = append(labels, transactionLabel) + + } + + return labels, nil +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels + selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) + + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return + } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) return nil, nil, err } @@ -664,18 +1017,32 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiEntryTx.AbiJSON, + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -685,161 +1052,169 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Convert transaction to label transactionLabel := indexer.TransactionLabel{ Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, + BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + BlockTimestamp: blockTimestamp, } - txLabels = append(txLabels, transactionLabel) + transactionsLabels = append(transactionsLabels, transactionLabel) } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel - - var topicSelector string + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + return transactionsLabels, blocksCache, nil - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } +} +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Get events in range - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + var addresses []common.Address + var topics []common.Hash - labels = append(labels, eventLabel) - } + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) } - } - return labels, txLabels, nil -} - -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + addresses = append(addresses, common.HexToAddress(address)) + } - decodedTransactions, err := c.DecodeProtoTransactions(transactions) + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { return nil, err } - var labels []indexer.TransactionLabel - var decodedArgs map[string]interface{} - var decodeErr error + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + var topicSelector string - for _, transaction := range decodedTransactions { + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - label := indexer.SeerCrawlerLabel + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } - selector := transaction.Input[:10] + abiEntryLog := abiMap[log.Address][topicSelector] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) return nil, err } - inputData, err := hex.DecodeString(transaction.Input[2:]) + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { - fmt.Println("Error decoding input data: ", err) return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + if _, ok := blocksCache[blockNumber]; !ok { - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) - decodedArgs = map[string]interface{}{ - "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err } - label = indexer.SeerCrawlerRawLabel + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + } - labelDataBytes, err := json.Marshal(decodedArgs) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { - fmt.Println("Error converting decodedArgs to JSON: ", err) return nil, err } - // Convert JSON byte slice to string - labelDataString := string(labelDataBytes) - - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: transaction.ToAddress, - BlockNumber: transaction.BlockNumber, - BlockHash: transaction.BlockHash, - CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: transaction.FromAddress, + // Convert event to label + eventLabel := indexer.EventLabel{ Label: label, - TransactionHash: transaction.Hash, - LabelData: labelDataString, - BlockTimestamp: blocksCache[transaction.BlockNumber], + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, } - labels = append(labels, transactionLabel) + eventsLabels = append(eventsLabels, eventLabel) } - return labels, nil -} \ No newline at end of file + return eventsLabels, nil + +} diff --git a/blockchain/b3/b3.go b/blockchain/b3/b3.go index f2a9390..602a58f 100644 --- a/blockchain/b3/b3.go +++ b/blockchain/b3/b3.go @@ -9,13 +9,14 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +72,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +106,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +207,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -214,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -224,7 +246,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -342,7 +364,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("b3", @@ -373,14 +394,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &B3BlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *B3BlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -457,11 +478,6 @@ func ToEntireBlocksBatchFromLogProto(obj *B3BlocksBatch) *seer_common.BlocksBatc BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -490,11 +506,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *B3Block { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -612,49 +623,381 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*B3Block, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch B3BlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch B3BlocksBatch dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *B3Block) { + defer wg.Done() + defer func() { <-semaphoreChan }() + + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel + + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + localTxLabels = append(localTxLabels, transactionLabel) + } + + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel + + var topicSelector string + + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } + + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } + + localEventLabels = append(localEventLabels, eventLabel) + } + } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } + + return labels, txLabels, nil +} + +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { + + decodedTransactions, err := c.DecodeProtoTransactions(transactions) + + if err != nil { + return nil, err + } + + var labels []indexer.TransactionLabel + var decodedArgs map[string]interface{} + var decodeErr error + + for _, transaction := range decodedTransactions { + + label := indexer.SeerCrawlerLabel + + selector := transaction.Input[:10] + + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } + } + + inputData, err := hex.DecodeString(transaction.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) + + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) + decodedArgs = map[string]interface{}{ + "input_raw": transaction, + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + labelDataBytes, err := json.Marshal(decodedArgs) + if err != nil { + fmt.Println("Error converting decodedArgs to JSON: ", err) + return nil, err + } + + // Convert JSON byte slice to string + labelDataString := string(labelDataBytes) + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: transaction.ToAddress, + BlockNumber: transaction.BlockNumber, + BlockHash: transaction.BlockHash, + CallerAddress: transaction.FromAddress, + LabelName: abiMap[transaction.ToAddress][selector].AbiName, + LabelType: "tx_call", + OriginAddress: transaction.FromAddress, + Label: label, + TransactionHash: transaction.Hash, + LabelData: labelDataString, + BlockTimestamp: blocksCache[transaction.BlockNumber], + } + + labels = append(labels, transactionLabel) + + } + + return labels, nil +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels + selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) + + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return + } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) return nil, nil, err } @@ -664,18 +1007,32 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiEntryTx.AbiJSON, + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -685,161 +1042,169 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Convert transaction to label transactionLabel := indexer.TransactionLabel{ Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, + BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + BlockTimestamp: blockTimestamp, } - txLabels = append(txLabels, transactionLabel) + transactionsLabels = append(transactionsLabels, transactionLabel) } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + } - var topicSelector string + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + return transactionsLabels, blocksCache, nil - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } +} - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + // Get events in range - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + var addresses []common.Address + var topics []common.Hash - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } - - labels = append(labels, eventLabel) - } + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) } - } - return labels, txLabels, nil -} - -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + addresses = append(addresses, common.HexToAddress(address)) + } - decodedTransactions, err := c.DecodeProtoTransactions(transactions) + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { return nil, err } - var labels []indexer.TransactionLabel - var decodedArgs map[string]interface{} - var decodeErr error + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + var topicSelector string - for _, transaction := range decodedTransactions { + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - label := indexer.SeerCrawlerLabel + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } - selector := transaction.Input[:10] + abiEntryLog := abiMap[log.Address][topicSelector] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) return nil, err } - inputData, err := hex.DecodeString(transaction.Input[2:]) + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { - fmt.Println("Error decoding input data: ", err) return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + if _, ok := blocksCache[blockNumber]; !ok { - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) - decodedArgs = map[string]interface{}{ - "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err } - label = indexer.SeerCrawlerRawLabel + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + } - labelDataBytes, err := json.Marshal(decodedArgs) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { - fmt.Println("Error converting decodedArgs to JSON: ", err) return nil, err } - // Convert JSON byte slice to string - labelDataString := string(labelDataBytes) - - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: transaction.ToAddress, - BlockNumber: transaction.BlockNumber, - BlockHash: transaction.BlockHash, - CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: transaction.FromAddress, + // Convert event to label + eventLabel := indexer.EventLabel{ Label: label, - TransactionHash: transaction.Hash, - LabelData: labelDataString, - BlockTimestamp: blocksCache[transaction.BlockNumber], + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, } - labels = append(labels, transactionLabel) + eventsLabels = append(eventsLabels, eventLabel) } - return labels, nil -} \ No newline at end of file + return eventsLabels, nil + +} diff --git a/blockchain/b3_sepolia/b3_sepolia.go b/blockchain/b3_sepolia/b3_sepolia.go index 58f71d2..f881c64 100644 --- a/blockchain/b3_sepolia/b3_sepolia.go +++ b/blockchain/b3_sepolia/b3_sepolia.go @@ -9,13 +9,14 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +72,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +106,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +207,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -214,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -224,7 +246,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -342,7 +364,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("b3_sepolia", @@ -373,14 +394,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &B3SepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *B3SepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -457,11 +478,6 @@ func ToEntireBlocksBatchFromLogProto(obj *B3SepoliaBlocksBatch) *seer_common.Blo BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -490,11 +506,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *B3SepoliaBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -612,49 +623,381 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*B3SepoliaBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch B3SepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch B3SepoliaBlocksBatch dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *B3SepoliaBlock) { + defer wg.Done() + defer func() { <-semaphoreChan }() + + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel + + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + localTxLabels = append(localTxLabels, transactionLabel) + } + + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel + + var topicSelector string + + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } + + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } + + localEventLabels = append(localEventLabels, eventLabel) + } + } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } + + return labels, txLabels, nil +} + +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { + + decodedTransactions, err := c.DecodeProtoTransactions(transactions) + + if err != nil { + return nil, err + } + + var labels []indexer.TransactionLabel + var decodedArgs map[string]interface{} + var decodeErr error + + for _, transaction := range decodedTransactions { + + label := indexer.SeerCrawlerLabel + + selector := transaction.Input[:10] + + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } + } + + inputData, err := hex.DecodeString(transaction.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) + + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) + decodedArgs = map[string]interface{}{ + "input_raw": transaction, + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + labelDataBytes, err := json.Marshal(decodedArgs) + if err != nil { + fmt.Println("Error converting decodedArgs to JSON: ", err) + return nil, err + } + + // Convert JSON byte slice to string + labelDataString := string(labelDataBytes) + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: transaction.ToAddress, + BlockNumber: transaction.BlockNumber, + BlockHash: transaction.BlockHash, + CallerAddress: transaction.FromAddress, + LabelName: abiMap[transaction.ToAddress][selector].AbiName, + LabelType: "tx_call", + OriginAddress: transaction.FromAddress, + Label: label, + TransactionHash: transaction.Hash, + LabelData: labelDataString, + BlockTimestamp: blocksCache[transaction.BlockNumber], + } + + labels = append(labels, transactionLabel) + + } + + return labels, nil +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels + selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) + + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return + } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) return nil, nil, err } @@ -664,18 +1007,32 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiEntryTx.AbiJSON, + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -685,161 +1042,169 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Convert transaction to label transactionLabel := indexer.TransactionLabel{ Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, + BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + BlockTimestamp: blockTimestamp, } - txLabels = append(txLabels, transactionLabel) + transactionsLabels = append(transactionsLabels, transactionLabel) } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + } - var topicSelector string + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + return transactionsLabels, blocksCache, nil - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } +} - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + // Get events in range - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + var addresses []common.Address + var topics []common.Hash - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } - - labels = append(labels, eventLabel) - } + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) } - } - return labels, txLabels, nil -} - -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + addresses = append(addresses, common.HexToAddress(address)) + } - decodedTransactions, err := c.DecodeProtoTransactions(transactions) + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { return nil, err } - var labels []indexer.TransactionLabel - var decodedArgs map[string]interface{} - var decodeErr error + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + var topicSelector string - for _, transaction := range decodedTransactions { + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - label := indexer.SeerCrawlerLabel + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } - selector := transaction.Input[:10] + abiEntryLog := abiMap[log.Address][topicSelector] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) return nil, err } - inputData, err := hex.DecodeString(transaction.Input[2:]) + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { - fmt.Println("Error decoding input data: ", err) return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + if _, ok := blocksCache[blockNumber]; !ok { - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) - decodedArgs = map[string]interface{}{ - "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err } - label = indexer.SeerCrawlerRawLabel + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + } - labelDataBytes, err := json.Marshal(decodedArgs) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { - fmt.Println("Error converting decodedArgs to JSON: ", err) return nil, err } - // Convert JSON byte slice to string - labelDataString := string(labelDataBytes) - - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: transaction.ToAddress, - BlockNumber: transaction.BlockNumber, - BlockHash: transaction.BlockHash, - CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: transaction.FromAddress, + // Convert event to label + eventLabel := indexer.EventLabel{ Label: label, - TransactionHash: transaction.Hash, - LabelData: labelDataString, - BlockTimestamp: blocksCache[transaction.BlockNumber], + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, } - labels = append(labels, transactionLabel) + eventsLabels = append(eventsLabels, eventLabel) } - return labels, nil -} \ No newline at end of file + return eventsLabels, nil + +} diff --git a/blockchain/blockchain.go.tmpl b/blockchain/blockchain.go.tmpl index 4401d37..bbd5a70 100644 --- a/blockchain/blockchain.go.tmpl +++ b/blockchain/blockchain.go.tmpl @@ -9,13 +9,14 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +72,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +106,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +207,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -224,7 +246,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -624,7 +646,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch {{.BlockchainName}}BlocksBatch dataBytes := rawData.Bytes() @@ -634,142 +656,212 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error - for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + var wg sync.WaitGroup - label := indexer.SeerCrawlerLabel + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) - // Process transaction labels - selector := tx.Input[:10] - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err - } + // Iterate over blocks and launch goroutines + for _, b := range protoBlocksBatch.Blocks { + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *{{.BlockchainName}}Block) { + defer wg.Done() + defer func() { <-semaphoreChan }() - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } + + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + label := indexer.SeerCrawlerLabel - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - txLabels = append(txLabels, transactionLabel) - } + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - var topicSelector string + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" + localTxLabels = append(localTxLabels, transactionLabel) } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -788,10 +880,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -800,13 +894,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -828,7 +922,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -842,4 +936,291 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil -} \ No newline at end of file +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return + } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiEntryTx.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, nil, err + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiEntryTx.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, blocksCache, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + // Get events in range + + var addresses []common.Address + var topics []common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) + } + + addresses = append(addresses, common.HexToAddress(address)) + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) + if err != nil { + return nil, err + } + + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + + } + + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + +} diff --git a/blockchain/common/decoding.go b/blockchain/common/decoding.go index ff41dca..a4008f0 100644 --- a/blockchain/common/decoding.go +++ b/blockchain/common/decoding.go @@ -101,6 +101,13 @@ type QueryFilter struct { Topics [][]string `json:"topics"` } +type BlockWithTransactions struct { + BlockNumber uint64 + BlockHash string + BlockTimestamp uint64 + Transactions map[string]TransactionJson +} + // ReadJsonBlocks reads blocks from a JSON file func ReadJsonBlocks() []*BlockJson { file, err := os.Open("data/blocks_go.json") diff --git a/blockchain/ethereum/ethereum.go b/blockchain/ethereum/ethereum.go index ab0676e..a9e960c 100644 --- a/blockchain/ethereum/ethereum.go +++ b/blockchain/ethereum/ethereum.go @@ -9,13 +9,14 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +72,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +106,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +207,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -214,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -224,7 +246,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -342,7 +364,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("ethereum", @@ -373,14 +394,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &EthereumBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *EthereumBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -457,11 +478,6 @@ func ToEntireBlocksBatchFromLogProto(obj *EthereumBlocksBatch) *seer_common.Bloc BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -490,11 +506,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *EthereumBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -612,49 +623,381 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*EthereumBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch EthereumBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch EthereumBlocksBatch dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *EthereumBlock) { + defer wg.Done() + defer func() { <-semaphoreChan }() + + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel + + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + localTxLabels = append(localTxLabels, transactionLabel) + } + + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel + + var topicSelector string + + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } + + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } + + localEventLabels = append(localEventLabels, eventLabel) + } + } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } + + return labels, txLabels, nil +} + +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { + + decodedTransactions, err := c.DecodeProtoTransactions(transactions) + + if err != nil { + return nil, err + } + + var labels []indexer.TransactionLabel + var decodedArgs map[string]interface{} + var decodeErr error + + for _, transaction := range decodedTransactions { + + label := indexer.SeerCrawlerLabel + + selector := transaction.Input[:10] + + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } + } + + inputData, err := hex.DecodeString(transaction.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) + + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) + decodedArgs = map[string]interface{}{ + "input_raw": transaction, + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + labelDataBytes, err := json.Marshal(decodedArgs) + if err != nil { + fmt.Println("Error converting decodedArgs to JSON: ", err) + return nil, err + } + + // Convert JSON byte slice to string + labelDataString := string(labelDataBytes) + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: transaction.ToAddress, + BlockNumber: transaction.BlockNumber, + BlockHash: transaction.BlockHash, + CallerAddress: transaction.FromAddress, + LabelName: abiMap[transaction.ToAddress][selector].AbiName, + LabelType: "tx_call", + OriginAddress: transaction.FromAddress, + Label: label, + TransactionHash: transaction.Hash, + LabelData: labelDataString, + BlockTimestamp: blocksCache[transaction.BlockNumber], + } + + labels = append(labels, transactionLabel) + + } + + return labels, nil +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels + selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) + + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return + } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) return nil, nil, err } @@ -664,18 +1007,32 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiEntryTx.AbiJSON, + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -685,161 +1042,169 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Convert transaction to label transactionLabel := indexer.TransactionLabel{ Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, + BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + BlockTimestamp: blockTimestamp, } - txLabels = append(txLabels, transactionLabel) + transactionsLabels = append(transactionsLabels, transactionLabel) } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + } - var topicSelector string + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + return transactionsLabels, blocksCache, nil - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } +} - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + // Get events in range - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + var addresses []common.Address + var topics []common.Hash - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } - - labels = append(labels, eventLabel) - } + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) } - } - return labels, txLabels, nil -} - -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + addresses = append(addresses, common.HexToAddress(address)) + } - decodedTransactions, err := c.DecodeProtoTransactions(transactions) + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { return nil, err } - var labels []indexer.TransactionLabel - var decodedArgs map[string]interface{} - var decodeErr error + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + var topicSelector string - for _, transaction := range decodedTransactions { + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - label := indexer.SeerCrawlerLabel + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } - selector := transaction.Input[:10] + abiEntryLog := abiMap[log.Address][topicSelector] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) return nil, err } - inputData, err := hex.DecodeString(transaction.Input[2:]) + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { - fmt.Println("Error decoding input data: ", err) return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + if _, ok := blocksCache[blockNumber]; !ok { - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) - decodedArgs = map[string]interface{}{ - "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err } - label = indexer.SeerCrawlerRawLabel + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + } - labelDataBytes, err := json.Marshal(decodedArgs) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { - fmt.Println("Error converting decodedArgs to JSON: ", err) return nil, err } - // Convert JSON byte slice to string - labelDataString := string(labelDataBytes) - - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: transaction.ToAddress, - BlockNumber: transaction.BlockNumber, - BlockHash: transaction.BlockHash, - CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: transaction.FromAddress, + // Convert event to label + eventLabel := indexer.EventLabel{ Label: label, - TransactionHash: transaction.Hash, - LabelData: labelDataString, - BlockTimestamp: blocksCache[transaction.BlockNumber], + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, } - labels = append(labels, transactionLabel) + eventsLabels = append(eventsLabels, eventLabel) } - return labels, nil -} \ No newline at end of file + return eventsLabels, nil + +} diff --git a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go index 680ff22..946c2f7 100644 --- a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go +++ b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go @@ -9,13 +9,14 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +72,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +106,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +207,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -214,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -224,7 +246,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -342,7 +364,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("game7_orbit_arbitrum_sepolia", @@ -373,14 +394,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &Game7OrbitArbitrumSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *Game7OrbitArbitrumSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -457,10 +478,10 @@ func ToEntireBlocksBatchFromLogProto(obj *Game7OrbitArbitrumSepoliaBlocksBatch) BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -491,10 +512,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *Game7OrbitArbitrumSepoliaBl TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -612,49 +633,381 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*Game7OrbitArbitrumSepoliaB func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch Game7OrbitArbitrumSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch Game7OrbitArbitrumSepoliaBlocksBatch dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *Game7OrbitArbitrumSepoliaBlock) { + defer wg.Done() + defer func() { <-semaphoreChan }() + + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel + + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + localTxLabels = append(localTxLabels, transactionLabel) + } + + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel + + var topicSelector string + + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } + + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } + + localEventLabels = append(localEventLabels, eventLabel) + } + } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } + + return labels, txLabels, nil +} + +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { + + decodedTransactions, err := c.DecodeProtoTransactions(transactions) + + if err != nil { + return nil, err + } + + var labels []indexer.TransactionLabel + var decodedArgs map[string]interface{} + var decodeErr error + + for _, transaction := range decodedTransactions { + + label := indexer.SeerCrawlerLabel + + selector := transaction.Input[:10] + + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } + } + + inputData, err := hex.DecodeString(transaction.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) + + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) + decodedArgs = map[string]interface{}{ + "input_raw": transaction, + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + labelDataBytes, err := json.Marshal(decodedArgs) + if err != nil { + fmt.Println("Error converting decodedArgs to JSON: ", err) + return nil, err + } + + // Convert JSON byte slice to string + labelDataString := string(labelDataBytes) + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: transaction.ToAddress, + BlockNumber: transaction.BlockNumber, + BlockHash: transaction.BlockHash, + CallerAddress: transaction.FromAddress, + LabelName: abiMap[transaction.ToAddress][selector].AbiName, + LabelType: "tx_call", + OriginAddress: transaction.FromAddress, + Label: label, + TransactionHash: transaction.Hash, + LabelData: labelDataString, + BlockTimestamp: blocksCache[transaction.BlockNumber], + } + + labels = append(labels, transactionLabel) + + } + + return labels, nil +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels + selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) + + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return + } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) return nil, nil, err } @@ -664,18 +1017,32 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiEntryTx.AbiJSON, + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -685,161 +1052,169 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Convert transaction to label transactionLabel := indexer.TransactionLabel{ Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, + BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + BlockTimestamp: blockTimestamp, } - txLabels = append(txLabels, transactionLabel) + transactionsLabels = append(transactionsLabels, transactionLabel) } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel - - var topicSelector string + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + return transactionsLabels, blocksCache, nil - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } +} +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Get events in range - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + var addresses []common.Address + var topics []common.Hash - labels = append(labels, eventLabel) - } + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) } - } - return labels, txLabels, nil -} - -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + addresses = append(addresses, common.HexToAddress(address)) + } - decodedTransactions, err := c.DecodeProtoTransactions(transactions) + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { return nil, err } - var labels []indexer.TransactionLabel - var decodedArgs map[string]interface{} - var decodeErr error + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + var topicSelector string - for _, transaction := range decodedTransactions { + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - label := indexer.SeerCrawlerLabel + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } - selector := transaction.Input[:10] + abiEntryLog := abiMap[log.Address][topicSelector] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) return nil, err } - inputData, err := hex.DecodeString(transaction.Input[2:]) + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { - fmt.Println("Error decoding input data: ", err) return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + if _, ok := blocksCache[blockNumber]; !ok { - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) - decodedArgs = map[string]interface{}{ - "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err } - label = indexer.SeerCrawlerRawLabel + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + } - labelDataBytes, err := json.Marshal(decodedArgs) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { - fmt.Println("Error converting decodedArgs to JSON: ", err) return nil, err } - // Convert JSON byte slice to string - labelDataString := string(labelDataBytes) - - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: transaction.ToAddress, - BlockNumber: transaction.BlockNumber, - BlockHash: transaction.BlockHash, - CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: transaction.FromAddress, + // Convert event to label + eventLabel := indexer.EventLabel{ Label: label, - TransactionHash: transaction.Hash, - LabelData: labelDataString, - BlockTimestamp: blocksCache[transaction.BlockNumber], + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, } - labels = append(labels, transactionLabel) + eventsLabels = append(eventsLabels, eventLabel) } - return labels, nil -} \ No newline at end of file + return eventsLabels, nil + +} diff --git a/blockchain/game7_testnet/game7_testnet.go b/blockchain/game7_testnet/game7_testnet.go index 90ccdaa..9928402 100644 --- a/blockchain/game7_testnet/game7_testnet.go +++ b/blockchain/game7_testnet/game7_testnet.go @@ -9,13 +9,14 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +72,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +106,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +207,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -214,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -224,7 +246,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -342,7 +364,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("game7_testnet", @@ -373,14 +394,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &Game7TestnetBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *Game7TestnetBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -457,10 +478,10 @@ func ToEntireBlocksBatchFromLogProto(obj *Game7TestnetBlocksBatch) *seer_common. BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -491,10 +512,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *Game7TestnetBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -612,49 +633,381 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*Game7TestnetBlock, error) func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch Game7TestnetBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch Game7TestnetBlocksBatch dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *Game7TestnetBlock) { + defer wg.Done() + defer func() { <-semaphoreChan }() + + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel + + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + localTxLabels = append(localTxLabels, transactionLabel) + } + + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel + + var topicSelector string + + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } + + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } + + localEventLabels = append(localEventLabels, eventLabel) + } + } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } + + return labels, txLabels, nil +} + +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { + + decodedTransactions, err := c.DecodeProtoTransactions(transactions) + + if err != nil { + return nil, err + } + + var labels []indexer.TransactionLabel + var decodedArgs map[string]interface{} + var decodeErr error + + for _, transaction := range decodedTransactions { + + label := indexer.SeerCrawlerLabel + + selector := transaction.Input[:10] + + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } + } + + inputData, err := hex.DecodeString(transaction.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) + + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) + decodedArgs = map[string]interface{}{ + "input_raw": transaction, + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + labelDataBytes, err := json.Marshal(decodedArgs) + if err != nil { + fmt.Println("Error converting decodedArgs to JSON: ", err) + return nil, err + } + + // Convert JSON byte slice to string + labelDataString := string(labelDataBytes) + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: transaction.ToAddress, + BlockNumber: transaction.BlockNumber, + BlockHash: transaction.BlockHash, + CallerAddress: transaction.FromAddress, + LabelName: abiMap[transaction.ToAddress][selector].AbiName, + LabelType: "tx_call", + OriginAddress: transaction.FromAddress, + Label: label, + TransactionHash: transaction.Hash, + LabelData: labelDataString, + BlockTimestamp: blocksCache[transaction.BlockNumber], + } + + labels = append(labels, transactionLabel) + + } + + return labels, nil +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels + selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) + + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return + } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) return nil, nil, err } @@ -664,18 +1017,32 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiEntryTx.AbiJSON, + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -685,161 +1052,169 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Convert transaction to label transactionLabel := indexer.TransactionLabel{ Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, + BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + BlockTimestamp: blockTimestamp, } - txLabels = append(txLabels, transactionLabel) + transactionsLabels = append(transactionsLabels, transactionLabel) } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel - - var topicSelector string + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + return transactionsLabels, blocksCache, nil - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } +} +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Get events in range - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + var addresses []common.Address + var topics []common.Hash - labels = append(labels, eventLabel) - } + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) } - } - return labels, txLabels, nil -} - -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + addresses = append(addresses, common.HexToAddress(address)) + } - decodedTransactions, err := c.DecodeProtoTransactions(transactions) + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { return nil, err } - var labels []indexer.TransactionLabel - var decodedArgs map[string]interface{} - var decodeErr error + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + var topicSelector string - for _, transaction := range decodedTransactions { + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - label := indexer.SeerCrawlerLabel + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } - selector := transaction.Input[:10] + abiEntryLog := abiMap[log.Address][topicSelector] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) return nil, err } - inputData, err := hex.DecodeString(transaction.Input[2:]) + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { - fmt.Println("Error decoding input data: ", err) return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + if _, ok := blocksCache[blockNumber]; !ok { - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) - decodedArgs = map[string]interface{}{ - "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err } - label = indexer.SeerCrawlerRawLabel + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + } - labelDataBytes, err := json.Marshal(decodedArgs) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { - fmt.Println("Error converting decodedArgs to JSON: ", err) return nil, err } - // Convert JSON byte slice to string - labelDataString := string(labelDataBytes) - - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: transaction.ToAddress, - BlockNumber: transaction.BlockNumber, - BlockHash: transaction.BlockHash, - CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: transaction.FromAddress, + // Convert event to label + eventLabel := indexer.EventLabel{ Label: label, - TransactionHash: transaction.Hash, - LabelData: labelDataString, - BlockTimestamp: blocksCache[transaction.BlockNumber], + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, } - labels = append(labels, transactionLabel) + eventsLabels = append(eventsLabels, eventLabel) } - return labels, nil -} \ No newline at end of file + return eventsLabels, nil + +} diff --git a/blockchain/handlers.go b/blockchain/handlers.go index dbc44c1..812c575 100644 --- a/blockchain/handlers.go +++ b/blockchain/handlers.go @@ -2,13 +2,16 @@ package blockchain import ( "bytes" + "context" "errors" "fmt" "log" "math/big" + "sync" "time" "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/common" "github.com/moonstream-to/seer/blockchain/arbitrum_one" "github.com/moonstream-to/seer/blockchain/arbitrum_sepolia" "github.com/moonstream-to/seer/blockchain/b3" @@ -93,9 +96,12 @@ type BlockchainClient interface { FetchAsProtoBlocksWithEvents(*big.Int, *big.Int, bool, int) ([]proto.Message, []indexer.BlockIndex, uint64, error) ProcessBlocksToBatch([]proto.Message) (proto.Message, error) DecodeProtoEntireBlockToJson(*bytes.Buffer) (*seer_common.BlocksBatchJson, error) - DecodeProtoEntireBlockToLabels(*bytes.Buffer, map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) - DecodeProtoTransactionsToLabels([]string, map[uint64]uint64, map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) + DecodeProtoEntireBlockToLabels(*bytes.Buffer, map[string]map[string]*indexer.AbiEntry, int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) + DecodeProtoTransactionsToLabels([]string, map[uint64]uint64, map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) ChainType() string + GetCode(context.Context, common.Address, uint64) ([]byte, error) + GetTransactionsLabels(uint64, uint64, map[string]map[string]*indexer.AbiEntry, int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) + GetEventsLabels(uint64, uint64, map[string]map[string]*indexer.AbiEntry, map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) } func GetLatestBlockNumberWithRetry(client BlockchainClient, retryAttempts int, retryWaitTime time.Duration) (*big.Int, error) { @@ -144,3 +150,127 @@ func DecodeTransactionInputData(contractABI *abi.ABI, data []byte) { fmt.Printf("Method Name: %s\n", method.Name) fmt.Printf("Method inputs: %v\n", inputsMap) } + +func DeployBlocksLookUpAndUpdate(blockchain string) error { + + // get all abi jobs without deployed block + + chainsAddresses, err := indexer.DBConnection.GetAbiJobsWithoutDeployBlocks(blockchain) + + if err != nil { + log.Printf("Failed to get abi jobs without deployed blocks: %v", err) + return err + } + + if len(chainsAddresses) == 0 { + log.Printf("No abi jobs without deployed blocks") + return nil + } + + for chain, addresses := range chainsAddresses { + + var wg sync.WaitGroup + + sem := make(chan struct{}, 5) // Semaphore to control + errChan := make(chan error, 1) // Buffered channel for error handling + + log.Printf("Processing chain: %s with amount of addresses: %d\n", chain, len(addresses)) + + for address, ids := range addresses { + + wg.Add(1) + go func(address string, chain string, ids []string) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + + client, err := NewClient(chain, BlockchainURLs[chain], 4) + + if err != nil { + errChan <- err + return + } + + // get all abi jobs without deployed block + deployedBlock, err := FindDeployedBlock(client, address) + + if err != nil { + errChan <- err + return + } + + log.Printf("Deployed block: %d for address: %s in chain: %s\n", deployedBlock, address, chain) + + if deployedBlock != 0 { + // update abi job with deployed block + err := indexer.DBConnection.UpdateAbiJobsDeployBlock(deployedBlock, ids) + if err != nil { + errChan <- err + return + } + } + + }(address, chain, ids) + + } + + go func() { + wg.Wait() + close(errChan) + }() + + for err := range errChan { + if err != nil { + log.Printf("Failed to get deployed block: %v", err) + return err + } + } + + } + + return nil + +} + +func FindDeployedBlock(client BlockchainClient, address string) (uint64, error) { + + // Binary search by get code + + ctx := context.Background() + + // with timeout + + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + + defer cancel() + + latestBlockNumber, err := client.GetLatestBlockNumber() + + if err != nil { + return 0, err + } + + var left uint64 = 1 + var right uint64 = latestBlockNumber.Uint64() + var code []byte + + for left < right { + mid := (left + right) / 2 + + code, err = client.GetCode(ctx, common.HexToAddress(address), mid) + + if err != nil { + log.Printf("Failed to get code: %v", err) + return 0, err + } + + if len(code) == 0 { + left = mid + 1 + } else { + right = mid + } + + } + + return left, nil +} diff --git a/blockchain/imx_zkevm/imx_zkevm.go b/blockchain/imx_zkevm/imx_zkevm.go index f6f0df6..986c600 100644 --- a/blockchain/imx_zkevm/imx_zkevm.go +++ b/blockchain/imx_zkevm/imx_zkevm.go @@ -9,13 +9,14 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +72,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +106,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +207,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -214,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -224,7 +246,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -342,7 +364,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("imx_zkevm", @@ -373,14 +394,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &ImxZkevmBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *ImxZkevmBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -457,11 +478,6 @@ func ToEntireBlocksBatchFromLogProto(obj *ImxZkevmBlocksBatch) *seer_common.Bloc BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -490,11 +506,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *ImxZkevmBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -612,49 +623,381 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*ImxZkevmBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch ImxZkevmBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch ImxZkevmBlocksBatch dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *ImxZkevmBlock) { + defer wg.Done() + defer func() { <-semaphoreChan }() + + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel + + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + localTxLabels = append(localTxLabels, transactionLabel) + } + + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel + + var topicSelector string + + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } + + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } + + localEventLabels = append(localEventLabels, eventLabel) + } + } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } + + return labels, txLabels, nil +} + +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { + + decodedTransactions, err := c.DecodeProtoTransactions(transactions) + + if err != nil { + return nil, err + } + + var labels []indexer.TransactionLabel + var decodedArgs map[string]interface{} + var decodeErr error + + for _, transaction := range decodedTransactions { + + label := indexer.SeerCrawlerLabel + + selector := transaction.Input[:10] + + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } + } + + inputData, err := hex.DecodeString(transaction.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) + + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) + decodedArgs = map[string]interface{}{ + "input_raw": transaction, + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + labelDataBytes, err := json.Marshal(decodedArgs) + if err != nil { + fmt.Println("Error converting decodedArgs to JSON: ", err) + return nil, err + } + + // Convert JSON byte slice to string + labelDataString := string(labelDataBytes) + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: transaction.ToAddress, + BlockNumber: transaction.BlockNumber, + BlockHash: transaction.BlockHash, + CallerAddress: transaction.FromAddress, + LabelName: abiMap[transaction.ToAddress][selector].AbiName, + LabelType: "tx_call", + OriginAddress: transaction.FromAddress, + Label: label, + TransactionHash: transaction.Hash, + LabelData: labelDataString, + BlockTimestamp: blocksCache[transaction.BlockNumber], + } + + labels = append(labels, transactionLabel) + + } + + return labels, nil +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels + selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) + + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return + } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) return nil, nil, err } @@ -664,18 +1007,32 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiEntryTx.AbiJSON, + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -685,161 +1042,169 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Convert transaction to label transactionLabel := indexer.TransactionLabel{ Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, + BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + BlockTimestamp: blockTimestamp, } - txLabels = append(txLabels, transactionLabel) + transactionsLabels = append(transactionsLabels, transactionLabel) } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + } - var topicSelector string + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + return transactionsLabels, blocksCache, nil - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } +} - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + // Get events in range - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + var addresses []common.Address + var topics []common.Hash - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } - - labels = append(labels, eventLabel) - } + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) } - } - return labels, txLabels, nil -} - -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + addresses = append(addresses, common.HexToAddress(address)) + } - decodedTransactions, err := c.DecodeProtoTransactions(transactions) + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { return nil, err } - var labels []indexer.TransactionLabel - var decodedArgs map[string]interface{} - var decodeErr error + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + var topicSelector string - for _, transaction := range decodedTransactions { + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - label := indexer.SeerCrawlerLabel + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } - selector := transaction.Input[:10] + abiEntryLog := abiMap[log.Address][topicSelector] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) return nil, err } - inputData, err := hex.DecodeString(transaction.Input[2:]) + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { - fmt.Println("Error decoding input data: ", err) return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + if _, ok := blocksCache[blockNumber]; !ok { - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) - decodedArgs = map[string]interface{}{ - "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err } - label = indexer.SeerCrawlerRawLabel + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + } - labelDataBytes, err := json.Marshal(decodedArgs) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { - fmt.Println("Error converting decodedArgs to JSON: ", err) return nil, err } - // Convert JSON byte slice to string - labelDataString := string(labelDataBytes) - - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: transaction.ToAddress, - BlockNumber: transaction.BlockNumber, - BlockHash: transaction.BlockHash, - CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: transaction.FromAddress, + // Convert event to label + eventLabel := indexer.EventLabel{ Label: label, - TransactionHash: transaction.Hash, - LabelData: labelDataString, - BlockTimestamp: blocksCache[transaction.BlockNumber], + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, } - labels = append(labels, transactionLabel) + eventsLabels = append(eventsLabels, eventLabel) } - return labels, nil -} \ No newline at end of file + return eventsLabels, nil + +} diff --git a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go index d99fab5..64ad7ae 100644 --- a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go +++ b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go @@ -9,13 +9,14 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +72,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +106,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +207,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -214,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -224,7 +246,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -342,7 +364,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("imx_zkevm_sepolia", @@ -373,14 +394,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &ImxZkevmSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *ImxZkevmSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -457,11 +478,6 @@ func ToEntireBlocksBatchFromLogProto(obj *ImxZkevmSepoliaBlocksBatch) *seer_comm BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -490,11 +506,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *ImxZkevmSepoliaBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -612,49 +623,381 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*ImxZkevmSepoliaBlock, erro func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch ImxZkevmSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch ImxZkevmSepoliaBlocksBatch dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *ImxZkevmSepoliaBlock) { + defer wg.Done() + defer func() { <-semaphoreChan }() + + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel + + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + localTxLabels = append(localTxLabels, transactionLabel) + } + + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel + + var topicSelector string + + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } + + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } + + localEventLabels = append(localEventLabels, eventLabel) + } + } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } + + return labels, txLabels, nil +} + +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { + + decodedTransactions, err := c.DecodeProtoTransactions(transactions) + + if err != nil { + return nil, err + } + + var labels []indexer.TransactionLabel + var decodedArgs map[string]interface{} + var decodeErr error + + for _, transaction := range decodedTransactions { + + label := indexer.SeerCrawlerLabel + + selector := transaction.Input[:10] + + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } + } + + inputData, err := hex.DecodeString(transaction.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) + + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) + decodedArgs = map[string]interface{}{ + "input_raw": transaction, + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + labelDataBytes, err := json.Marshal(decodedArgs) + if err != nil { + fmt.Println("Error converting decodedArgs to JSON: ", err) + return nil, err + } + + // Convert JSON byte slice to string + labelDataString := string(labelDataBytes) + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: transaction.ToAddress, + BlockNumber: transaction.BlockNumber, + BlockHash: transaction.BlockHash, + CallerAddress: transaction.FromAddress, + LabelName: abiMap[transaction.ToAddress][selector].AbiName, + LabelType: "tx_call", + OriginAddress: transaction.FromAddress, + Label: label, + TransactionHash: transaction.Hash, + LabelData: labelDataString, + BlockTimestamp: blocksCache[transaction.BlockNumber], + } + + labels = append(labels, transactionLabel) + + } + + return labels, nil +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels + selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) + + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return + } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) return nil, nil, err } @@ -664,18 +1007,32 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiEntryTx.AbiJSON, + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -685,161 +1042,169 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Convert transaction to label transactionLabel := indexer.TransactionLabel{ Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, + BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + BlockTimestamp: blockTimestamp, } - txLabels = append(txLabels, transactionLabel) + transactionsLabels = append(transactionsLabels, transactionLabel) } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + } - var topicSelector string + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + return transactionsLabels, blocksCache, nil - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } +} - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + // Get events in range - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + var addresses []common.Address + var topics []common.Hash - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } - - labels = append(labels, eventLabel) - } + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) } - } - return labels, txLabels, nil -} - -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + addresses = append(addresses, common.HexToAddress(address)) + } - decodedTransactions, err := c.DecodeProtoTransactions(transactions) + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { return nil, err } - var labels []indexer.TransactionLabel - var decodedArgs map[string]interface{} - var decodeErr error + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + var topicSelector string - for _, transaction := range decodedTransactions { + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - label := indexer.SeerCrawlerLabel + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } - selector := transaction.Input[:10] + abiEntryLog := abiMap[log.Address][topicSelector] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) return nil, err } - inputData, err := hex.DecodeString(transaction.Input[2:]) + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { - fmt.Println("Error decoding input data: ", err) return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + if _, ok := blocksCache[blockNumber]; !ok { - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) - decodedArgs = map[string]interface{}{ - "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err } - label = indexer.SeerCrawlerRawLabel + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + } - labelDataBytes, err := json.Marshal(decodedArgs) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { - fmt.Println("Error converting decodedArgs to JSON: ", err) return nil, err } - // Convert JSON byte slice to string - labelDataString := string(labelDataBytes) - - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: transaction.ToAddress, - BlockNumber: transaction.BlockNumber, - BlockHash: transaction.BlockHash, - CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: transaction.FromAddress, + // Convert event to label + eventLabel := indexer.EventLabel{ Label: label, - TransactionHash: transaction.Hash, - LabelData: labelDataString, - BlockTimestamp: blocksCache[transaction.BlockNumber], + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, } - labels = append(labels, transactionLabel) + eventsLabels = append(eventsLabels, eventLabel) } - return labels, nil -} \ No newline at end of file + return eventsLabels, nil + +} diff --git a/blockchain/mantle/mantle.go b/blockchain/mantle/mantle.go index 0f44560..7add238 100644 --- a/blockchain/mantle/mantle.go +++ b/blockchain/mantle/mantle.go @@ -9,13 +9,14 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +72,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +106,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +207,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -214,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -224,7 +246,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -342,7 +364,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("mantle", @@ -373,14 +394,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &MantleBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *MantleBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -457,11 +478,6 @@ func ToEntireBlocksBatchFromLogProto(obj *MantleBlocksBatch) *seer_common.Blocks BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -490,11 +506,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *MantleBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -612,49 +623,381 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*MantleBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch MantleBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch MantleBlocksBatch dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *MantleBlock) { + defer wg.Done() + defer func() { <-semaphoreChan }() + + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel + + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + localTxLabels = append(localTxLabels, transactionLabel) + } + + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel + + var topicSelector string + + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } + + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } + + localEventLabels = append(localEventLabels, eventLabel) + } + } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } + + return labels, txLabels, nil +} + +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { + + decodedTransactions, err := c.DecodeProtoTransactions(transactions) + + if err != nil { + return nil, err + } + + var labels []indexer.TransactionLabel + var decodedArgs map[string]interface{} + var decodeErr error + + for _, transaction := range decodedTransactions { + + label := indexer.SeerCrawlerLabel + + selector := transaction.Input[:10] + + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } + } + + inputData, err := hex.DecodeString(transaction.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) + + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) + decodedArgs = map[string]interface{}{ + "input_raw": transaction, + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + labelDataBytes, err := json.Marshal(decodedArgs) + if err != nil { + fmt.Println("Error converting decodedArgs to JSON: ", err) + return nil, err + } + + // Convert JSON byte slice to string + labelDataString := string(labelDataBytes) + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: transaction.ToAddress, + BlockNumber: transaction.BlockNumber, + BlockHash: transaction.BlockHash, + CallerAddress: transaction.FromAddress, + LabelName: abiMap[transaction.ToAddress][selector].AbiName, + LabelType: "tx_call", + OriginAddress: transaction.FromAddress, + Label: label, + TransactionHash: transaction.Hash, + LabelData: labelDataString, + BlockTimestamp: blocksCache[transaction.BlockNumber], + } + + labels = append(labels, transactionLabel) + + } + + return labels, nil +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels + selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) + + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return + } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) return nil, nil, err } @@ -664,18 +1007,32 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiEntryTx.AbiJSON, + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -685,161 +1042,169 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Convert transaction to label transactionLabel := indexer.TransactionLabel{ Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, + BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + BlockTimestamp: blockTimestamp, } - txLabels = append(txLabels, transactionLabel) + transactionsLabels = append(transactionsLabels, transactionLabel) } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + } - var topicSelector string + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + return transactionsLabels, blocksCache, nil - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } +} - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + // Get events in range - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + var addresses []common.Address + var topics []common.Hash - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } - - labels = append(labels, eventLabel) - } + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) } - } - return labels, txLabels, nil -} - -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + addresses = append(addresses, common.HexToAddress(address)) + } - decodedTransactions, err := c.DecodeProtoTransactions(transactions) + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { return nil, err } - var labels []indexer.TransactionLabel - var decodedArgs map[string]interface{} - var decodeErr error + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + var topicSelector string - for _, transaction := range decodedTransactions { + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - label := indexer.SeerCrawlerLabel + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } - selector := transaction.Input[:10] + abiEntryLog := abiMap[log.Address][topicSelector] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) return nil, err } - inputData, err := hex.DecodeString(transaction.Input[2:]) + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { - fmt.Println("Error decoding input data: ", err) return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + if _, ok := blocksCache[blockNumber]; !ok { - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) - decodedArgs = map[string]interface{}{ - "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err } - label = indexer.SeerCrawlerRawLabel + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + } - labelDataBytes, err := json.Marshal(decodedArgs) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { - fmt.Println("Error converting decodedArgs to JSON: ", err) return nil, err } - // Convert JSON byte slice to string - labelDataString := string(labelDataBytes) - - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: transaction.ToAddress, - BlockNumber: transaction.BlockNumber, - BlockHash: transaction.BlockHash, - CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: transaction.FromAddress, + // Convert event to label + eventLabel := indexer.EventLabel{ Label: label, - TransactionHash: transaction.Hash, - LabelData: labelDataString, - BlockTimestamp: blocksCache[transaction.BlockNumber], + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, } - labels = append(labels, transactionLabel) + eventsLabels = append(eventsLabels, eventLabel) } - return labels, nil -} \ No newline at end of file + return eventsLabels, nil + +} diff --git a/blockchain/mantle_sepolia/mantle_sepolia.go b/blockchain/mantle_sepolia/mantle_sepolia.go index a140ca5..e85f6fa 100644 --- a/blockchain/mantle_sepolia/mantle_sepolia.go +++ b/blockchain/mantle_sepolia/mantle_sepolia.go @@ -9,13 +9,14 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +72,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +106,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +207,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -214,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -224,7 +246,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -342,7 +364,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("mantle_sepolia", @@ -373,14 +394,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &MantleSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *MantleSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -457,11 +478,6 @@ func ToEntireBlocksBatchFromLogProto(obj *MantleSepoliaBlocksBatch) *seer_common BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -490,11 +506,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *MantleSepoliaBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -612,49 +623,381 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*MantleSepoliaBlock, error) func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch MantleSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch MantleSepoliaBlocksBatch dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *MantleSepoliaBlock) { + defer wg.Done() + defer func() { <-semaphoreChan }() + + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel + + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + localTxLabels = append(localTxLabels, transactionLabel) + } + + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel + + var topicSelector string + + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } + + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } + + localEventLabels = append(localEventLabels, eventLabel) + } + } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } + + return labels, txLabels, nil +} + +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { + + decodedTransactions, err := c.DecodeProtoTransactions(transactions) + + if err != nil { + return nil, err + } + + var labels []indexer.TransactionLabel + var decodedArgs map[string]interface{} + var decodeErr error + + for _, transaction := range decodedTransactions { + + label := indexer.SeerCrawlerLabel + + selector := transaction.Input[:10] + + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } + } + + inputData, err := hex.DecodeString(transaction.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) + + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) + decodedArgs = map[string]interface{}{ + "input_raw": transaction, + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + labelDataBytes, err := json.Marshal(decodedArgs) + if err != nil { + fmt.Println("Error converting decodedArgs to JSON: ", err) + return nil, err + } + + // Convert JSON byte slice to string + labelDataString := string(labelDataBytes) + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: transaction.ToAddress, + BlockNumber: transaction.BlockNumber, + BlockHash: transaction.BlockHash, + CallerAddress: transaction.FromAddress, + LabelName: abiMap[transaction.ToAddress][selector].AbiName, + LabelType: "tx_call", + OriginAddress: transaction.FromAddress, + Label: label, + TransactionHash: transaction.Hash, + LabelData: labelDataString, + BlockTimestamp: blocksCache[transaction.BlockNumber], + } + + labels = append(labels, transactionLabel) + + } + + return labels, nil +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels + selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) + + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return + } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) return nil, nil, err } @@ -664,18 +1007,32 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiEntryTx.AbiJSON, + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -685,161 +1042,169 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Convert transaction to label transactionLabel := indexer.TransactionLabel{ Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, + BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + BlockTimestamp: blockTimestamp, } - txLabels = append(txLabels, transactionLabel) + transactionsLabels = append(transactionsLabels, transactionLabel) } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + } - var topicSelector string + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + return transactionsLabels, blocksCache, nil - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } +} - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + // Get events in range - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + var addresses []common.Address + var topics []common.Hash - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } - - labels = append(labels, eventLabel) - } + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) } - } - return labels, txLabels, nil -} - -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + addresses = append(addresses, common.HexToAddress(address)) + } - decodedTransactions, err := c.DecodeProtoTransactions(transactions) + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { return nil, err } - var labels []indexer.TransactionLabel - var decodedArgs map[string]interface{} - var decodeErr error + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + var topicSelector string - for _, transaction := range decodedTransactions { + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - label := indexer.SeerCrawlerLabel + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } - selector := transaction.Input[:10] + abiEntryLog := abiMap[log.Address][topicSelector] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) return nil, err } - inputData, err := hex.DecodeString(transaction.Input[2:]) + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { - fmt.Println("Error decoding input data: ", err) return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + if _, ok := blocksCache[blockNumber]; !ok { - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) - decodedArgs = map[string]interface{}{ - "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err } - label = indexer.SeerCrawlerRawLabel + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + } - labelDataBytes, err := json.Marshal(decodedArgs) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { - fmt.Println("Error converting decodedArgs to JSON: ", err) return nil, err } - // Convert JSON byte slice to string - labelDataString := string(labelDataBytes) - - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: transaction.ToAddress, - BlockNumber: transaction.BlockNumber, - BlockHash: transaction.BlockHash, - CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: transaction.FromAddress, + // Convert event to label + eventLabel := indexer.EventLabel{ Label: label, - TransactionHash: transaction.Hash, - LabelData: labelDataString, - BlockTimestamp: blocksCache[transaction.BlockNumber], + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, } - labels = append(labels, transactionLabel) + eventsLabels = append(eventsLabels, eventLabel) } - return labels, nil -} \ No newline at end of file + return eventsLabels, nil + +} diff --git a/blockchain/polygon/polygon.go b/blockchain/polygon/polygon.go index ff5671e..07acc96 100644 --- a/blockchain/polygon/polygon.go +++ b/blockchain/polygon/polygon.go @@ -9,13 +9,14 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +72,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +106,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +207,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -214,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -224,7 +246,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -342,7 +364,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("polygon", @@ -373,14 +394,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &PolygonBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *PolygonBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -457,11 +478,6 @@ func ToEntireBlocksBatchFromLogProto(obj *PolygonBlocksBatch) *seer_common.Block BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -490,11 +506,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *PolygonBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -612,49 +623,381 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*PolygonBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch PolygonBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch PolygonBlocksBatch dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *PolygonBlock) { + defer wg.Done() + defer func() { <-semaphoreChan }() + + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel + + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + localTxLabels = append(localTxLabels, transactionLabel) + } + + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel + + var topicSelector string + + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } + + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } + + localEventLabels = append(localEventLabels, eventLabel) + } + } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } + + return labels, txLabels, nil +} + +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { + + decodedTransactions, err := c.DecodeProtoTransactions(transactions) + + if err != nil { + return nil, err + } + + var labels []indexer.TransactionLabel + var decodedArgs map[string]interface{} + var decodeErr error + + for _, transaction := range decodedTransactions { + + label := indexer.SeerCrawlerLabel + + selector := transaction.Input[:10] + + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } + } + + inputData, err := hex.DecodeString(transaction.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) + + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) + decodedArgs = map[string]interface{}{ + "input_raw": transaction, + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + labelDataBytes, err := json.Marshal(decodedArgs) + if err != nil { + fmt.Println("Error converting decodedArgs to JSON: ", err) + return nil, err + } + + // Convert JSON byte slice to string + labelDataString := string(labelDataBytes) + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: transaction.ToAddress, + BlockNumber: transaction.BlockNumber, + BlockHash: transaction.BlockHash, + CallerAddress: transaction.FromAddress, + LabelName: abiMap[transaction.ToAddress][selector].AbiName, + LabelType: "tx_call", + OriginAddress: transaction.FromAddress, + Label: label, + TransactionHash: transaction.Hash, + LabelData: labelDataString, + BlockTimestamp: blocksCache[transaction.BlockNumber], + } + + labels = append(labels, transactionLabel) + + } + + return labels, nil +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels + selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) + + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return + } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) return nil, nil, err } @@ -664,18 +1007,32 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiEntryTx.AbiJSON, + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -685,161 +1042,169 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Convert transaction to label transactionLabel := indexer.TransactionLabel{ Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, + BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + BlockTimestamp: blockTimestamp, } - txLabels = append(txLabels, transactionLabel) + transactionsLabels = append(transactionsLabels, transactionLabel) } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + } - var topicSelector string + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + return transactionsLabels, blocksCache, nil - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } +} - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + // Get events in range - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + var addresses []common.Address + var topics []common.Hash - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } - - labels = append(labels, eventLabel) - } + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) } - } - return labels, txLabels, nil -} - -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + addresses = append(addresses, common.HexToAddress(address)) + } - decodedTransactions, err := c.DecodeProtoTransactions(transactions) + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { return nil, err } - var labels []indexer.TransactionLabel - var decodedArgs map[string]interface{} - var decodeErr error + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + var topicSelector string - for _, transaction := range decodedTransactions { + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - label := indexer.SeerCrawlerLabel + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } - selector := transaction.Input[:10] + abiEntryLog := abiMap[log.Address][topicSelector] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) return nil, err } - inputData, err := hex.DecodeString(transaction.Input[2:]) + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { - fmt.Println("Error decoding input data: ", err) return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + if _, ok := blocksCache[blockNumber]; !ok { - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) - decodedArgs = map[string]interface{}{ - "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err } - label = indexer.SeerCrawlerRawLabel + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + } - labelDataBytes, err := json.Marshal(decodedArgs) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { - fmt.Println("Error converting decodedArgs to JSON: ", err) return nil, err } - // Convert JSON byte slice to string - labelDataString := string(labelDataBytes) - - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: transaction.ToAddress, - BlockNumber: transaction.BlockNumber, - BlockHash: transaction.BlockHash, - CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: transaction.FromAddress, + // Convert event to label + eventLabel := indexer.EventLabel{ Label: label, - TransactionHash: transaction.Hash, - LabelData: labelDataString, - BlockTimestamp: blocksCache[transaction.BlockNumber], + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, } - labels = append(labels, transactionLabel) + eventsLabels = append(eventsLabels, eventLabel) } - return labels, nil -} \ No newline at end of file + return eventsLabels, nil + +} diff --git a/blockchain/sepolia/sepolia.go b/blockchain/sepolia/sepolia.go index 89b9f43..097adde 100644 --- a/blockchain/sepolia/sepolia.go +++ b/blockchain/sepolia/sepolia.go @@ -9,13 +9,14 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +72,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +106,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +207,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -214,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -224,7 +246,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -342,7 +364,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("sepolia", @@ -373,14 +394,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &SepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *SepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -457,11 +478,6 @@ func ToEntireBlocksBatchFromLogProto(obj *SepoliaBlocksBatch) *seer_common.Block BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -490,11 +506,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *SepoliaBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -612,49 +623,381 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*SepoliaBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch SepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch SepoliaBlocksBatch dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *SepoliaBlock) { + defer wg.Done() + defer func() { <-semaphoreChan }() + + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel + + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + localTxLabels = append(localTxLabels, transactionLabel) + } + + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel + + var topicSelector string + + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } + + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } + + localEventLabels = append(localEventLabels, eventLabel) + } + } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } + + return labels, txLabels, nil +} + +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { + + decodedTransactions, err := c.DecodeProtoTransactions(transactions) + + if err != nil { + return nil, err + } + + var labels []indexer.TransactionLabel + var decodedArgs map[string]interface{} + var decodeErr error + + for _, transaction := range decodedTransactions { + + label := indexer.SeerCrawlerLabel + + selector := transaction.Input[:10] + + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } + } + + inputData, err := hex.DecodeString(transaction.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) + + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) + decodedArgs = map[string]interface{}{ + "input_raw": transaction, + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + labelDataBytes, err := json.Marshal(decodedArgs) + if err != nil { + fmt.Println("Error converting decodedArgs to JSON: ", err) + return nil, err + } + + // Convert JSON byte slice to string + labelDataString := string(labelDataBytes) + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: transaction.ToAddress, + BlockNumber: transaction.BlockNumber, + BlockHash: transaction.BlockHash, + CallerAddress: transaction.FromAddress, + LabelName: abiMap[transaction.ToAddress][selector].AbiName, + LabelType: "tx_call", + OriginAddress: transaction.FromAddress, + Label: label, + TransactionHash: transaction.Hash, + LabelData: labelDataString, + BlockTimestamp: blocksCache[transaction.BlockNumber], + } + + labels = append(labels, transactionLabel) + + } + + return labels, nil +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels + selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) + + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return + } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) return nil, nil, err } @@ -664,18 +1007,32 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiEntryTx.AbiJSON, + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -685,161 +1042,169 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Convert transaction to label transactionLabel := indexer.TransactionLabel{ Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, + BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + BlockTimestamp: blockTimestamp, } - txLabels = append(txLabels, transactionLabel) + transactionsLabels = append(transactionsLabels, transactionLabel) } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + } - var topicSelector string + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + return transactionsLabels, blocksCache, nil - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } +} - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + // Get events in range - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + var addresses []common.Address + var topics []common.Hash - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } - - labels = append(labels, eventLabel) - } + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) } - } - return labels, txLabels, nil -} - -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + addresses = append(addresses, common.HexToAddress(address)) + } - decodedTransactions, err := c.DecodeProtoTransactions(transactions) + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { return nil, err } - var labels []indexer.TransactionLabel - var decodedArgs map[string]interface{} - var decodeErr error + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + var topicSelector string - for _, transaction := range decodedTransactions { + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - label := indexer.SeerCrawlerLabel + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } - selector := transaction.Input[:10] + abiEntryLog := abiMap[log.Address][topicSelector] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) return nil, err } - inputData, err := hex.DecodeString(transaction.Input[2:]) + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { - fmt.Println("Error decoding input data: ", err) return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + if _, ok := blocksCache[blockNumber]; !ok { - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) - decodedArgs = map[string]interface{}{ - "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err } - label = indexer.SeerCrawlerRawLabel + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + } - labelDataBytes, err := json.Marshal(decodedArgs) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { - fmt.Println("Error converting decodedArgs to JSON: ", err) return nil, err } - // Convert JSON byte slice to string - labelDataString := string(labelDataBytes) - - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: transaction.ToAddress, - BlockNumber: transaction.BlockNumber, - BlockHash: transaction.BlockHash, - CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: transaction.FromAddress, + // Convert event to label + eventLabel := indexer.EventLabel{ Label: label, - TransactionHash: transaction.Hash, - LabelData: labelDataString, - BlockTimestamp: blocksCache[transaction.BlockNumber], + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, } - labels = append(labels, transactionLabel) + eventsLabels = append(eventsLabels, eventLabel) } - return labels, nil -} \ No newline at end of file + return eventsLabels, nil + +} diff --git a/blockchain/xai/xai.go b/blockchain/xai/xai.go index 85cbbf2..4dbfa92 100644 --- a/blockchain/xai/xai.go +++ b/blockchain/xai/xai.go @@ -9,13 +9,14 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +72,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +106,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +207,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -214,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -224,7 +246,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -342,7 +364,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("xai", @@ -373,14 +394,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &XaiBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *XaiBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -457,10 +478,10 @@ func ToEntireBlocksBatchFromLogProto(obj *XaiBlocksBatch) *seer_common.BlocksBat BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -491,10 +512,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *XaiBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -612,49 +633,381 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*XaiBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch XaiBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch XaiBlocksBatch dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *XaiBlock) { + defer wg.Done() + defer func() { <-semaphoreChan }() + + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel + + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + localTxLabels = append(localTxLabels, transactionLabel) + } + + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel + + var topicSelector string + + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } + + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } + + localEventLabels = append(localEventLabels, eventLabel) + } + } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } + + return labels, txLabels, nil +} + +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { + + decodedTransactions, err := c.DecodeProtoTransactions(transactions) + + if err != nil { + return nil, err + } + + var labels []indexer.TransactionLabel + var decodedArgs map[string]interface{} + var decodeErr error + + for _, transaction := range decodedTransactions { + + label := indexer.SeerCrawlerLabel + + selector := transaction.Input[:10] + + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } + } + + inputData, err := hex.DecodeString(transaction.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) + + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) + decodedArgs = map[string]interface{}{ + "input_raw": transaction, + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + labelDataBytes, err := json.Marshal(decodedArgs) + if err != nil { + fmt.Println("Error converting decodedArgs to JSON: ", err) + return nil, err + } + + // Convert JSON byte slice to string + labelDataString := string(labelDataBytes) + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: transaction.ToAddress, + BlockNumber: transaction.BlockNumber, + BlockHash: transaction.BlockHash, + CallerAddress: transaction.FromAddress, + LabelName: abiMap[transaction.ToAddress][selector].AbiName, + LabelType: "tx_call", + OriginAddress: transaction.FromAddress, + Label: label, + TransactionHash: transaction.Hash, + LabelData: labelDataString, + BlockTimestamp: blocksCache[transaction.BlockNumber], + } + + labels = append(labels, transactionLabel) + + } + + return labels, nil +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels + selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) + + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return + } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) return nil, nil, err } @@ -664,18 +1017,32 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiEntryTx.AbiJSON, + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -685,161 +1052,169 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Convert transaction to label transactionLabel := indexer.TransactionLabel{ Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, + BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + BlockTimestamp: blockTimestamp, } - txLabels = append(txLabels, transactionLabel) + transactionsLabels = append(transactionsLabels, transactionLabel) } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel - - var topicSelector string + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + return transactionsLabels, blocksCache, nil - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } +} +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Get events in range - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + var addresses []common.Address + var topics []common.Hash - labels = append(labels, eventLabel) - } + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) } - } - return labels, txLabels, nil -} - -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + addresses = append(addresses, common.HexToAddress(address)) + } - decodedTransactions, err := c.DecodeProtoTransactions(transactions) + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { return nil, err } - var labels []indexer.TransactionLabel - var decodedArgs map[string]interface{} - var decodeErr error + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + var topicSelector string - for _, transaction := range decodedTransactions { + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - label := indexer.SeerCrawlerLabel + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } - selector := transaction.Input[:10] + abiEntryLog := abiMap[log.Address][topicSelector] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) return nil, err } - inputData, err := hex.DecodeString(transaction.Input[2:]) + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { - fmt.Println("Error decoding input data: ", err) return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + if _, ok := blocksCache[blockNumber]; !ok { - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) - decodedArgs = map[string]interface{}{ - "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err } - label = indexer.SeerCrawlerRawLabel + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + } - labelDataBytes, err := json.Marshal(decodedArgs) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { - fmt.Println("Error converting decodedArgs to JSON: ", err) return nil, err } - // Convert JSON byte slice to string - labelDataString := string(labelDataBytes) - - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: transaction.ToAddress, - BlockNumber: transaction.BlockNumber, - BlockHash: transaction.BlockHash, - CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: transaction.FromAddress, + // Convert event to label + eventLabel := indexer.EventLabel{ Label: label, - TransactionHash: transaction.Hash, - LabelData: labelDataString, - BlockTimestamp: blocksCache[transaction.BlockNumber], + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, } - labels = append(labels, transactionLabel) + eventsLabels = append(eventsLabels, eventLabel) } - return labels, nil -} \ No newline at end of file + return eventsLabels, nil + +} diff --git a/blockchain/xai_sepolia/xai_sepolia.go b/blockchain/xai_sepolia/xai_sepolia.go index 98d475b..08bf180 100644 --- a/blockchain/xai_sepolia/xai_sepolia.go +++ b/blockchain/xai_sepolia/xai_sepolia.go @@ -9,13 +9,14 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +72,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +106,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +207,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -214,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -224,7 +246,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -342,7 +364,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("xai_sepolia", @@ -373,14 +394,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &XaiSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *XaiSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -457,10 +478,10 @@ func ToEntireBlocksBatchFromLogProto(obj *XaiSepoliaBlocksBatch) *seer_common.Bl BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -491,10 +512,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *XaiSepoliaBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -612,49 +633,381 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*XaiSepoliaBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch XaiSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch XaiSepoliaBlocksBatch dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *XaiSepoliaBlock) { + defer wg.Done() + defer func() { <-semaphoreChan }() + + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel + + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + localTxLabels = append(localTxLabels, transactionLabel) + } + + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel + + var topicSelector string + + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } + + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } + + localEventLabels = append(localEventLabels, eventLabel) + } + } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } + + return labels, txLabels, nil +} + +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { + + decodedTransactions, err := c.DecodeProtoTransactions(transactions) + + if err != nil { + return nil, err + } + + var labels []indexer.TransactionLabel + var decodedArgs map[string]interface{} + var decodeErr error + + for _, transaction := range decodedTransactions { + + label := indexer.SeerCrawlerLabel + + selector := transaction.Input[:10] + + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } + } + + inputData, err := hex.DecodeString(transaction.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) + + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) + decodedArgs = map[string]interface{}{ + "input_raw": transaction, + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + labelDataBytes, err := json.Marshal(decodedArgs) + if err != nil { + fmt.Println("Error converting decodedArgs to JSON: ", err) + return nil, err + } + + // Convert JSON byte slice to string + labelDataString := string(labelDataBytes) + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: transaction.ToAddress, + BlockNumber: transaction.BlockNumber, + BlockHash: transaction.BlockHash, + CallerAddress: transaction.FromAddress, + LabelName: abiMap[transaction.ToAddress][selector].AbiName, + LabelType: "tx_call", + OriginAddress: transaction.FromAddress, + Label: label, + TransactionHash: transaction.Hash, + LabelData: labelDataString, + BlockTimestamp: blocksCache[transaction.BlockNumber], + } + + labels = append(labels, transactionLabel) + + } + + return labels, nil +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels + selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) + + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return + } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) return nil, nil, err } @@ -664,18 +1017,32 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiEntryTx.AbiJSON, + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -685,161 +1052,169 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Convert transaction to label transactionLabel := indexer.TransactionLabel{ Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, + BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + BlockTimestamp: blockTimestamp, } - txLabels = append(txLabels, transactionLabel) + transactionsLabels = append(transactionsLabels, transactionLabel) } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel - - var topicSelector string + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + return transactionsLabels, blocksCache, nil - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } +} +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Get events in range - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + var addresses []common.Address + var topics []common.Hash - labels = append(labels, eventLabel) - } + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) } - } - return labels, txLabels, nil -} - -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + addresses = append(addresses, common.HexToAddress(address)) + } - decodedTransactions, err := c.DecodeProtoTransactions(transactions) + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { return nil, err } - var labels []indexer.TransactionLabel - var decodedArgs map[string]interface{} - var decodeErr error + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + var topicSelector string - for _, transaction := range decodedTransactions { + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - label := indexer.SeerCrawlerLabel + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } - selector := transaction.Input[:10] + abiEntryLog := abiMap[log.Address][topicSelector] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) return nil, err } - inputData, err := hex.DecodeString(transaction.Input[2:]) + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { - fmt.Println("Error decoding input data: ", err) return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + if _, ok := blocksCache[blockNumber]; !ok { - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) - decodedArgs = map[string]interface{}{ - "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err } - label = indexer.SeerCrawlerRawLabel + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + } - labelDataBytes, err := json.Marshal(decodedArgs) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { - fmt.Println("Error converting decodedArgs to JSON: ", err) return nil, err } - // Convert JSON byte slice to string - labelDataString := string(labelDataBytes) - - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: transaction.ToAddress, - BlockNumber: transaction.BlockNumber, - BlockHash: transaction.BlockHash, - CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: transaction.FromAddress, + // Convert event to label + eventLabel := indexer.EventLabel{ Label: label, - TransactionHash: transaction.Hash, - LabelData: labelDataString, - BlockTimestamp: blocksCache[transaction.BlockNumber], + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, } - labels = append(labels, transactionLabel) + eventsLabels = append(eventsLabels, eventLabel) } - return labels, nil -} \ No newline at end of file + return eventsLabels, nil + +} diff --git a/cmd.go b/cmd.go index 1ade926..c658762 100644 --- a/cmd.go +++ b/cmd.go @@ -46,7 +46,8 @@ func CreateRootCommand() *cobra.Command { synchronizerCmd := CreateSynchronizerCommand() abiCmd := CreateAbiCommand() dbCmd := CreateDatabaseOperationCommand() - rootCmd.AddCommand(completionCmd, versionCmd, blockchainCmd, starknetCmd, evmCmd, crawlerCmd, inspectorCmd, synchronizerCmd, abiCmd, dbCmd) + historicalSyncCmd := CreateHistoricalSyncCommand() + rootCmd.AddCommand(completionCmd, versionCmd, blockchainCmd, starknetCmd, evmCmd, crawlerCmd, inspectorCmd, synchronizerCmd, abiCmd, dbCmd, historicalSyncCmd) // By default, cobra Command objects write to stderr. We have to forcibly set them to output to // stdout. @@ -243,6 +244,11 @@ func CreateCrawlerCommand() *cobra.Command { return crawlerErr } + blockchainErr := seer_blockchain.CheckVariablesForBlockchains() + if blockchainErr != nil { + return blockchainErr + } + return nil }, RunE: func(cmd *cobra.Command, args []string) error { @@ -289,7 +295,7 @@ func CreateCrawlerCommand() *cobra.Command { func CreateSynchronizerCommand() *cobra.Command { var startBlock, endBlock, batchSize uint64 - var timeout int + var timeout, threads int var chain, baseDir, customerDbUriFlag string synchronizerCmd := &cobra.Command{ @@ -316,6 +322,11 @@ func CreateSynchronizerCommand() *cobra.Command { return syncErr } + blockchainErr := seer_blockchain.CheckVariablesForBlockchains() + if blockchainErr != nil { + return blockchainErr + } + if chain == "" { return fmt.Errorf("blockchain is required via --chain") } @@ -325,7 +336,7 @@ func CreateSynchronizerCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { indexer.InitDBConnection() - newSynchronizer, synchonizerErr := synchronizer.NewSynchronizer(chain, baseDir, startBlock, endBlock, batchSize, timeout) + newSynchronizer, synchonizerErr := synchronizer.NewSynchronizer(chain, baseDir, startBlock, endBlock, batchSize, timeout, threads) if synchonizerErr != nil { return synchonizerErr } @@ -354,6 +365,7 @@ func CreateSynchronizerCommand() *cobra.Command { synchronizerCmd.Flags().IntVar(&timeout, "timeout", 30, "The timeout for the crawler in seconds (default: 30)") synchronizerCmd.Flags().Uint64Var(&batchSize, "batch-size", 100, "The number of blocks to crawl in each batch (default: 100)") synchronizerCmd.Flags().StringVar(&customerDbUriFlag, "customer-db-uri", "", "Set customer database URI for development. This workflow bypass fetching customer IDs and its database URL connection strings from mdb-v3-controller API") + synchronizerCmd.Flags().IntVar(&threads, "threads", 5, "Number of go-routines for concurrent decoding") return synchronizerCmd } @@ -698,7 +710,7 @@ func CreateAbiEnsureSelectorsCommand() *cobra.Command { indexer.InitDBConnection() - updateErr := indexer.DBConnection.EnsureCorrectSelectors(chain, WriteToDB, outFilePath) + updateErr := indexer.DBConnection.EnsureCorrectSelectors(chain, WriteToDB, outFilePath, []string{}) if updateErr != nil { return updateErr } @@ -762,11 +774,173 @@ func CreateDatabaseOperationCommand() *cobra.Command { indexCommand.AddCommand(cleanCommand) + deploymentBlocksCommand := &cobra.Command{ + Use: "deployment-blocks", + Short: "Get deployment blocks from address in abi jobs", + PreRunE: func(cmd *cobra.Command, args []string) error { + indexerErr := indexer.CheckVariablesForIndexer() + if indexerErr != nil { + return indexerErr + } + blockchainErr := seer_blockchain.CheckVariablesForBlockchains() + if blockchainErr != nil { + return blockchainErr + } + indexer.InitDBConnection() + + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + + deploymentBlocksErr := seer_blockchain.DeployBlocksLookUpAndUpdate(chain) + if deploymentBlocksErr != nil { + return deploymentBlocksErr + } + + return nil + }, + } + + deploymentBlocksCommand.Flags().StringVar(&chain, "chain", "ethereum", "The blockchain to crawl (default: ethereum)") + + var jobChain, address, abiFile, customerId, userId string + var deployBlock uint64 + + createJobsCommand := &cobra.Command{ + Use: "create-jobs", + Short: "Create jobs for ABI", + PreRunE: func(cmd *cobra.Command, args []string) error { + indexerErr := indexer.CheckVariablesForIndexer() + if indexerErr != nil { + return indexerErr + } + + indexer.InitDBConnection() + + blockchainErr := seer_blockchain.CheckVariablesForBlockchains() + if blockchainErr != nil { + return blockchainErr + } + + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + + // check if the chain is supported + if _, ok := seer_blockchain.BlockchainURLs[jobChain]; !ok { + return fmt.Errorf("chain %s is not supported", jobChain) + } + + client, clientErr := seer_blockchain.NewClient(jobChain, seer_blockchain.BlockchainURLs[jobChain], 30) + if clientErr != nil { + return clientErr + } + + // detect deploy block + if deployBlock == 0 { + fmt.Println("Deploy block is not provided, trying to find it from chain") + deployBlockFromChain, deployErr := seer_blockchain.FindDeployedBlock(client, address) + + if deployErr != nil { + return deployErr + } + deployBlock = deployBlockFromChain + } + + createJobsErr := indexer.DBConnection.CreateJobsFromAbi(jobChain, address, abiFile, customerId, userId, deployBlock) + if createJobsErr != nil { + return createJobsErr + } + + return nil + }, + } + + createJobsCommand.Flags().StringVar(&jobChain, "jobChain", "ethereum", "The blockchain to crawl (default: ethereum)") + createJobsCommand.Flags().StringVar(&address, "address", "", "The address to create jobs for") + createJobsCommand.Flags().StringVar(&abiFile, "abi-file", "", "The path to the ABI file") + createJobsCommand.Flags().StringVar(&customerId, "customer-id", "", "The customer ID to create jobs for (default: '')") + createJobsCommand.Flags().StringVar(&userId, "user-id", "00000000-0000-0000-0000-000000000000", "The user ID to create jobs for (default: '00000000-0000-0000-0000-000000000000')") + createJobsCommand.Flags().Uint64Var(&deployBlock, "deploy-block", 0, "The block number to deploy contract (default: 0)") + + indexCommand.AddCommand(deploymentBlocksCommand) + indexCommand.AddCommand(createJobsCommand) databaseCmd.AddCommand(indexCommand) return databaseCmd } +func CreateHistoricalSyncCommand() *cobra.Command { + + var chain, baseDir, customerDbUriFlag string + var addresses, customerIds []string + var startBlock, endBlock, batchSize uint64 + var timeout, threads int + var auto bool + + historicalSyncCmd := &cobra.Command{ + Use: "historical-sync", + Short: "Decode the historical data from various blockchains", + PreRunE: func(cmd *cobra.Command, args []string) error { + indexerErr := indexer.CheckVariablesForIndexer() + if indexerErr != nil { + return indexerErr + } + + storageErr := storage.CheckVariablesForStorage() + if storageErr != nil { + return storageErr + } + + crawlerErr := crawler.CheckVariablesForCrawler() + if crawlerErr != nil { + return crawlerErr + } + + syncErr := synchronizer.CheckVariablesForSynchronizer() + if syncErr != nil { + return syncErr + } + + if chain == "" { + return fmt.Errorf("blockchain is required via --chain") + } + + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + indexer.InitDBConnection() + + newSynchronizer, synchonizerErr := synchronizer.NewSynchronizer(chain, baseDir, startBlock, endBlock, batchSize, timeout, threads) + if synchonizerErr != nil { + return synchonizerErr + } + + err := newSynchronizer.HistoricalSyncRef(customerDbUriFlag, addresses, customerIds, batchSize, auto) + + if err != nil { + return err + } + + return nil + }, + } + + historicalSyncCmd.Flags().StringVar(&chain, "chain", "ethereum", "The blockchain to crawl (default: ethereum)") + historicalSyncCmd.Flags().StringVar(&baseDir, "base-dir", "", "The base directory to store the crawled data (default: '')") + historicalSyncCmd.Flags().Uint64Var(&startBlock, "start-block", 0, "The block number to start decoding from (default: latest block)") + historicalSyncCmd.Flags().Uint64Var(&endBlock, "end-block", 0, "The block number to end decoding at (default: latest block)") + historicalSyncCmd.Flags().IntVar(&timeout, "timeout", 30, "The timeout for the crawler in seconds (default: 30)") + historicalSyncCmd.Flags().Uint64Var(&batchSize, "batch-size", 100, "The number of blocks to crawl in each batch (default: 100)") + historicalSyncCmd.Flags().StringVar(&customerDbUriFlag, "customer-db-uri", "", "Set customer database URI for development. This workflow bypass fetching customer IDs and its database URL connection strings from mdb-v3-controller API") + historicalSyncCmd.Flags().StringSliceVar(&customerIds, "customer-ids", []string{}, "The list of customer IDs to sync") + historicalSyncCmd.Flags().StringSliceVar(&addresses, "addresses", []string{}, "The list of addresses to sync") + historicalSyncCmd.Flags().BoolVar(&auto, "auto", false, "Set this flag to sync all unfinished historical crawl from the database (default: false)") + historicalSyncCmd.Flags().IntVar(&threads, "threads", 5, "Number of go-routines for concurrent crawling (default: 5)") + + return historicalSyncCmd +} + func CreateStarknetParseCommand() *cobra.Command { var infile string var rawABI []byte diff --git a/crawler/crawler.go b/crawler/crawler.go index 8d98cd2..bdd1a13 100644 --- a/crawler/crawler.go +++ b/crawler/crawler.go @@ -108,7 +108,7 @@ func NewCrawler(blockchain string, startBlock, finalBlock, confirmations, batchS panic(err) } - client, err := seer_blockchain.NewClient(blockchain, BlockchainURLs[blockchain], timeout) + client, err := seer_blockchain.NewClient(blockchain, seer_blockchain.BlockchainURLs[blockchain], timeout) if err != nil { log.Fatal(err) } diff --git a/deploy/deploy.bash b/deploy/deploy.bash index f9cf474..97591f6 100755 --- a/deploy/deploy.bash +++ b/deploy/deploy.bash @@ -54,6 +54,37 @@ SEER_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_SERVICE_FILE="seer-synchronizer-imx-zkevm-se SEER_SYNCHRONIZER_B3_SERVICE_FILE="seer-synchronizer-b3.service" SEER_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE="seer-synchronizer-b3-sepolia.service" +# Historical Synchronizer (timers) +SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_SERVICE_FILE="seer-historical-synchronizer-ethereum.service" +SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_TIMER_FILE="seer-historical-synchronizer-ethereum.timer" +SEER_HISTORICAL_SYNCHRONIZER_POLYGON_SERVICE_FILE="seer-historical-synchronizer-polygon.service" +SEER_HISTORICAL_SYNCHRONIZER_POLYGON_TIMER_FILE="seer-historical-synchronizer-polygon.timer" +SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_SERVICE_FILE="seer-historical-synchronizer-arbitrum-one.service" +SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_TIMER_FILE="seer-historical-synchronizer-arbitrum-one.timer" +SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_SERVICE_FILE="seer-historical-synchronizer-arbitrum-sepolia.service" +SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_TIMER_FILE="seer-historical-synchronizer-arbitrum-sepolia.timer" +SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE="seer-historical-synchronizer-game7-testnet.service" +SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE="seer-historical-synchronizer-game7-testnet.timer" +SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SEPOLIA_SERVICE_FILE="seer-historical-synchronizer-mantle-sepolia.service" +SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE="seer-historical-synchronizer-mantle.timer" +SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SERVICE_FILE="seer-historical-synchronizer-mantle.service" +SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE="seer-historical-synchronizer-mantle.timer" +SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_SERVICE_FILE="seer-historical-synchronizer-xai-sepolia.service" +SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_TIMER_FILE="seer-historical-synchronizer-xai-sepolia.timer" +SEER_HISTORICAL_SYNCHRONIZER_XAI_SERVICE_FILE="seer-historical-synchronizer-xai.service" +SEER_HISTORICAL_SYNCHRONIZER_XAI_TIMER_FILE="seer-historical-synchronizer-xai.timer" +SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_SERVICE_FILE="seer-historical-synchronizer-sepolia.service" +SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_TIMER_FILE="seer-historical-synchronizer-sepolia.timer" +SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SERVICE_FILE="seer-historical-synchronizer-imx-zkevm.service" +SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_TIMER_FILE="seer-historical-synchronizer-imx-zkevm.timer" +SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_SERVICE_FILE="seer-historical-synchronizer-imx-zkevm-sepolia.service" +SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_TIMER_FILE="seer-historical-synchronizer-imx-zkevm-sepolia.timer" +SEER_HISTORICAL_SYNCHRONIZER_B3_SERVICE_FILE="seer-historical-synchronizer-b3.service" +SEER_HISTORICAL_SYNCHRONIZER_B3_TIMER_FILE="seer-historical-synchronizer-b3.timer" +SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE="seer-historical-synchronizer-b3-sepolia.service" +SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_TIMER_FILE="seer-historical-synchronizer-b3-sepolia.timer" + + set -eu if [ ! -d "${SECRETS_DIR}" ]; then @@ -372,4 +403,142 @@ echo -e "${PREFIX_INFO} Replacing existing seer synchronizer for B3 Sepolia bloc chmod 644 "${SCRIPT_DIR}/${SEER_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}" cp "${SCRIPT_DIR}/${SEER_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}" XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}" \ No newline at end of file +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}" + + +# Historical Synchronizers + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Ethereum blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Polygon blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Arbitrum One blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Arbitrum Sepolia blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Game7 Testnet blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Mantle Sepolia blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SEPOLIA_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SEPOLIA_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SEPOLIA_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SEPOLIA_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Mantle blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Xai Sepolia blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Xai blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_XAI_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_XAI_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_XAI_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Sepolia blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Immutable zkEvm blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Immutable zkEvm Sepolia blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Game7 Testnet blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for B3 blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_B3_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_B3_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_B3_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for B3 Sepolia blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_TIMER_FILE}" \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-arbitrum-one.service b/deploy/seer-historical-synchronizer-arbitrum-one.service new file mode 100644 index 0000000..bcb2c84 --- /dev/null +++ b/deploy/seer-historical-synchronizer-arbitrum-one.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for arbitrum one blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain arbitrum_one +SyslogIdentifier=seer-historical-synchronizer-arbitrum-one + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-arbitrum-one.timer b/deploy/seer-historical-synchronizer-arbitrum-one.timer new file mode 100644 index 0000000..ff74d15 --- /dev/null +++ b/deploy/seer-historical-synchronizer-arbitrum-one.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on Arbitrum One + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-arbitrum-sepolia.service b/deploy/seer-historical-synchronizer-arbitrum-sepolia.service new file mode 100644 index 0000000..7b283e2 --- /dev/null +++ b/deploy/seer-historical-synchronizer-arbitrum-sepolia.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for arbitrum sepolia blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain arbitrum_sepolia +SyslogIdentifier=seer-historical-synchronizer-arbitrum-sepolia + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-arbitrum-sepolia.timer b/deploy/seer-historical-synchronizer-arbitrum-sepolia.timer new file mode 100644 index 0000000..4bf23f1 --- /dev/null +++ b/deploy/seer-historical-synchronizer-arbitrum-sepolia.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on Arbitrum sepolia + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-b3-sepolia.service b/deploy/seer-historical-synchronizer-b3-sepolia.service new file mode 100644 index 0000000..b3d9cb7 --- /dev/null +++ b/deploy/seer-historical-synchronizer-b3-sepolia.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for b3 sepolia blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain b3_sepolia +SyslogIdentifier=seer-historical-synchronizer-ab3-sepolia + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-b3-sepolia.timer b/deploy/seer-historical-synchronizer-b3-sepolia.timer new file mode 100644 index 0000000..793ee70 --- /dev/null +++ b/deploy/seer-historical-synchronizer-b3-sepolia.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on b3 sepolia + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-b3.service b/deploy/seer-historical-synchronizer-b3.service new file mode 100644 index 0000000..e4f9313 --- /dev/null +++ b/deploy/seer-historical-synchronizer-b3.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for b3 blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain b3 +SyslogIdentifier=seer-historical-synchronizer-b3 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-b3.timer b/deploy/seer-historical-synchronizer-b3.timer new file mode 100644 index 0000000..1b05711 --- /dev/null +++ b/deploy/seer-historical-synchronizer-b3.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on b3 + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-ethereum.service b/deploy/seer-historical-synchronizer-ethereum.service new file mode 100644 index 0000000..accb275 --- /dev/null +++ b/deploy/seer-historical-synchronizer-ethereum.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for Ethereum blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain ethereum +SyslogIdentifier=seer-historical-synchronizer-ethereum + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-ethereum.timer b/deploy/seer-historical-synchronizer-ethereum.timer new file mode 100644 index 0000000..e91c854 --- /dev/null +++ b/deploy/seer-historical-synchronizer-ethereum.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on ethereum + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-game7-testnet.service b/deploy/seer-historical-synchronizer-game7-testnet.service new file mode 100644 index 0000000..293766e --- /dev/null +++ b/deploy/seer-historical-synchronizer-game7-testnet.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for game7 testnet blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain game7_testnet +SyslogIdentifier=seer-historical-synchronizer-game7-testnet + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-game7-testnet.timer b/deploy/seer-historical-synchronizer-game7-testnet.timer new file mode 100644 index 0000000..aa19912 --- /dev/null +++ b/deploy/seer-historical-synchronizer-game7-testnet.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on game7 testnet + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-imx-zkevm-sepolia.service b/deploy/seer-historical-synchronizer-imx-zkevm-sepolia.service new file mode 100644 index 0000000..b85f72d --- /dev/null +++ b/deploy/seer-historical-synchronizer-imx-zkevm-sepolia.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for Immutable zkEvm Sepolia blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain imx_zkevm_sepolia +SyslogIdentifier=seer-historical-synchronizer-imx-zkevm-sepolia + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-imx-zkevm-sepolia.timer b/deploy/seer-historical-synchronizer-imx-zkevm-sepolia.timer new file mode 100644 index 0000000..71804c8 --- /dev/null +++ b/deploy/seer-historical-synchronizer-imx-zkevm-sepolia.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on imx zkevm sepolia + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-imx-zkevm.service b/deploy/seer-historical-synchronizer-imx-zkevm.service new file mode 100644 index 0000000..796d668 --- /dev/null +++ b/deploy/seer-historical-synchronizer-imx-zkevm.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for Immutable zkEvm blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain imx_zkevm +SyslogIdentifier=seer-historical-synchronizer-imx-zkevm + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-imx-zkevm.timer b/deploy/seer-historical-synchronizer-imx-zkevm.timer new file mode 100644 index 0000000..57fc0a7 --- /dev/null +++ b/deploy/seer-historical-synchronizer-imx-zkevm.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on imx zkevm + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-mantle-sepolia.service b/deploy/seer-historical-synchronizer-mantle-sepolia.service new file mode 100644 index 0000000..23eeb48 --- /dev/null +++ b/deploy/seer-historical-synchronizer-mantle-sepolia.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for mantle sepolia blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain mantle_sepolia +SyslogIdentifier=seer-historical-synchronizer-mantle-sepolia + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-mantle-sepolia.timer b/deploy/seer-historical-synchronizer-mantle-sepolia.timer new file mode 100644 index 0000000..277bd9a --- /dev/null +++ b/deploy/seer-historical-synchronizer-mantle-sepolia.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on mantle sepolia + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-mantle.service b/deploy/seer-historical-synchronizer-mantle.service new file mode 100644 index 0000000..44d8a69 --- /dev/null +++ b/deploy/seer-historical-synchronizer-mantle.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for mantle blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain mantle +SyslogIdentifier=seer-historical-synchronizer-mantle + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-mantle.timer b/deploy/seer-historical-synchronizer-mantle.timer new file mode 100644 index 0000000..45da398 --- /dev/null +++ b/deploy/seer-historical-synchronizer-mantle.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on mantle + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-polygon.service b/deploy/seer-historical-synchronizer-polygon.service new file mode 100644 index 0000000..3561857 --- /dev/null +++ b/deploy/seer-historical-synchronizer-polygon.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for Polygon blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain polygon +SyslogIdentifier=seer-historical-synchronizer-polygon + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-polygon.timer b/deploy/seer-historical-synchronizer-polygon.timer new file mode 100644 index 0000000..0653765 --- /dev/null +++ b/deploy/seer-historical-synchronizer-polygon.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on polygon + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-sepolia.service b/deploy/seer-historical-synchronizer-sepolia.service new file mode 100644 index 0000000..96f2cd7 --- /dev/null +++ b/deploy/seer-historical-synchronizer-sepolia.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for Sepolia blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain sepolia +SyslogIdentifier=seer-historical-synchronizer-sepolia + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-sepolia.timer b/deploy/seer-historical-synchronizer-sepolia.timer new file mode 100644 index 0000000..f45fab4 --- /dev/null +++ b/deploy/seer-historical-synchronizer-sepolia.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on Sepolia + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-xai-sepolia.service b/deploy/seer-historical-synchronizer-xai-sepolia.service new file mode 100644 index 0000000..67a80e0 --- /dev/null +++ b/deploy/seer-historical-synchronizer-xai-sepolia.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for xai sepolia blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain xai_sepolia +SyslogIdentifier=seer-historical-synchronizer-xai-sepolia + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-xai-sepolia.timer b/deploy/seer-historical-synchronizer-xai-sepolia.timer new file mode 100644 index 0000000..ffc1c2c --- /dev/null +++ b/deploy/seer-historical-synchronizer-xai-sepolia.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on xai sepolia + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-xai.service b/deploy/seer-historical-synchronizer-xai.service new file mode 100644 index 0000000..0ee4ddb --- /dev/null +++ b/deploy/seer-historical-synchronizer-xai.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for xai blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain xai +SyslogIdentifier=seer-historical-synchronizer-xai + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-xai.timer b/deploy/seer-historical-synchronizer-xai.timer new file mode 100644 index 0000000..d5a8709 --- /dev/null +++ b/deploy/seer-historical-synchronizer-xai.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on xai + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/go.mod b/go.mod index 8c72727..0c7c138 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/jackc/pgx/v5 v5.5.3 github.com/spf13/cobra v1.8.0 golang.org/x/crypto v0.20.0 - golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa golang.org/x/term v0.17.0 golang.org/x/tools v0.15.0 google.golang.org/api v0.167.0 @@ -62,6 +61,7 @@ require ( go.opentelemetry.io/otel v1.23.0 // indirect go.opentelemetry.io/otel/metric v1.23.0 // indirect go.opentelemetry.io/otel/trace v1.23.0 // indirect + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.21.0 // indirect golang.org/x/oauth2 v0.17.0 // indirect diff --git a/indexer/db.go b/indexer/db.go index 88d6807..9acdcc6 100644 --- a/indexer/db.go +++ b/indexer/db.go @@ -4,7 +4,9 @@ import ( "bufio" "context" "encoding/hex" + "encoding/json" "fmt" + "io/ioutil" "log" "os" "strconv" @@ -81,6 +83,20 @@ func IsBlockchainWithL1Chain(blockchain string) bool { } } +func FilterABIJobs(abiJobs []AbiJob, ids []string) []AbiJob { + var filteredABIJobs []AbiJob + + for _, abiJob := range abiJobs { + for _, id := range ids { + if abiJob.ID == id { + filteredABIJobs = append(filteredABIJobs, abiJob) + } + } + } + + return filteredABIJobs +} + type PostgreSQLpgx struct { pool *pgxpool.Pool } @@ -240,7 +256,7 @@ func (p *PostgreSQLpgx) WriteIndexes(blockchain string, blocksIndexPack []BlockI pool := p.GetPool() conn, err := pool.Acquire(ctx) if err != nil { - fmt.Println("Connection error", err) + log.Println("Connection error", err) return err } defer conn.Release() @@ -294,7 +310,7 @@ func (p *PostgreSQLpgx) executeBatchInsert(tx pgx.Tx, ctx context.Context, table // track execution time if _, err := tx.Exec(ctx, query, valuesSlice...); err != nil { - fmt.Println("Error executing bulk insert", err) + log.Println("Error executing bulk insert", err) return fmt.Errorf("error executing bulk insert for batch: %w", err) } @@ -464,7 +480,7 @@ func (p *PostgreSQLpgx) ReadABIJobs(blockchain string) ([]AbiJob, error) { defer conn.Release() - rows, err := conn.Query(context.Background(), "SELECT id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, moonworm_task_pickedup, abi, (abi::jsonb)->>'type' as abiType, created_at, updated_at FROM abi_jobs where chain=$1 and (abi::jsonb)->>'type' is not null", blockchain) + rows, err := conn.Query(context.Background(), "SELECT id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, moonworm_task_pickedup, '[' || abi || ']' as abi, (abi::jsonb)->>'type' as abiType, created_at, updated_at, deployment_block_number FROM abi_jobs where chain=$1 and (abi::jsonb)->>'type' is not null", blockchain) if err != nil { return nil, err @@ -481,7 +497,7 @@ func (p *PostgreSQLpgx) ReadABIJobs(blockchain string) ([]AbiJob, error) { return nil, nil // or return an appropriate error if this is considered an error state } - log.Println("Parsed abiJobs:", len(abiJobs), "for blockchain:", blockchain) + //log.Println("Parsed abiJobs:", len(abiJobs), "for blockchain:", blockchain) // If you need to process or log the first ABI job separately, do it here return abiJobs, nil @@ -523,14 +539,14 @@ func (p *PostgreSQLpgx) GetCustomersIDs(blockchain string) ([]string, error) { return customerIds, nil } -func (p *PostgreSQLpgx) ReadUpdates(blockchain string, fromBlock uint64, customerIds []string) (uint64, string, []CustomerUpdates, error) { +func (p *PostgreSQLpgx) ReadUpdates(blockchain string, fromBlock uint64, customerIds []string) (uint64, uint64, string, []CustomerUpdates, error) { pool := p.GetPool() conn, err := pool.Acquire(context.Background()) if err != nil { - return 0, "", nil, err + return 0, 0, "", nil, err } defer conn.Release() @@ -578,10 +594,9 @@ func (p *PostgreSQLpgx) ReadUpdates(blockchain string, fromBlock uint64, custome json_object_agg( abi_selector, json_build_object( - 'abi', - '[' || abi || ']', - 'abi_name', - abi_name + 'abi', '[' || abi || ']', + 'abi_name', abi_name, + 'abi_type', abi_type ) ) AS abis_per_address FROM @@ -611,18 +626,18 @@ func (p *PostgreSQLpgx) ReadUpdates(blockchain string, fromBlock uint64, custome if err != nil { log.Println("Error querying abi jobs from database", err) - return 0, "", nil, err + return 0, 0, "", nil, err } - var customers []map[string]map[string]map[string]map[string]string + var customers []map[string]map[string]map[string]*AbiEntry var path string - var lastNumber uint64 + var firstBlockNumber, lastBlockNumber uint64 for rows.Next() { - err = rows.Scan(&lastNumber, &path, &customers) + err = rows.Scan(&lastBlockNumber, &path, &customers) if err != nil { log.Println("Error scanning row:", err) - return 0, "", nil, err + return 0, 0, "", nil, err } } @@ -641,11 +656,11 @@ func (p *PostgreSQLpgx) ReadUpdates(blockchain string, fromBlock uint64, custome } - return lastNumber, path, customerUpdates, nil + return firstBlockNumber, lastBlockNumber, path, customerUpdates, nil } -func (p *PostgreSQLpgx) EnsureCorrectSelectors(blockchain string, WriteToDB bool, outputFilePath string) error { +func (p *PostgreSQLpgx) EnsureCorrectSelectors(blockchain string, WriteToDB bool, outputFilePath string, ids []string) error { pool := p.GetPool() @@ -665,26 +680,35 @@ func (p *PostgreSQLpgx) EnsureCorrectSelectors(blockchain string, WriteToDB bool return err } - log.Println("Found", len(abiJobs), "ABI jobs for blockchain:", blockchain) + if len(ids) > 0 { + abiJobs = FilterABIJobs(abiJobs, ids) + } else { + log.Println("Found", len(abiJobs), "ABI jobs for blockchain:", blockchain) + } + var writer *bufio.Writer + var f *os.File // for each ABI job, check if the selector is correct - f, err := os.OpenFile(outputFilePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) + if outputFilePath != "" { - if err != nil { - log.Println("Error opening file:", err) - return err - } + f, err := os.OpenFile(outputFilePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) - writer := bufio.NewWriter(f) + if err != nil { + log.Println("Error opening file:", err) + return err + } - writer.WriteString(fmt.Sprintf("ABI jobs for blockchain: %s runned as WriteToDB: %v recorded at %s\n", blockchain, WriteToDB, time.Now().String())) + writer := bufio.NewWriter(f) - for _, abiJob := range abiJobs { + writer.WriteString(fmt.Sprintf("ABI jobs for blockchain: %s runned as WriteToDB: %v recorded at %s\n", blockchain, WriteToDB, time.Now().String())) - // Get the correct selector for the ABI - abiObj, err := abi.JSON(strings.NewReader("[" + abiJob.Abi + "]")) + } + for _, abiJob := range abiJobs { + + // Now you can use abiJSONStr as a string + abiObj, err := abi.JSON(strings.NewReader(abiJob.Abi)) if err != nil { log.Println("Error parsing ABI for ABI job:", abiJob.ID, err) return err @@ -722,18 +746,25 @@ func (p *PostgreSQLpgx) EnsureCorrectSelectors(blockchain string, WriteToDB bool } - _, err = writer.WriteString(fmt.Sprintf("ABI job ID: %s, Name: %s, Address: %x, Selector: %s, Correct Selector: %s\n", abiJob.ID, abiJob.AbiName, abiJob.Address, abiJob.AbiSelector, selector)) - if err != nil { - log.Println("Error writing to file:", err) - continue + if outputFilePath != "" { + + _, err = writer.WriteString(fmt.Sprintf("ABI job ID: %s, Name: %s, Address: %x, Selector: %s, Correct Selector: %s\n", abiJob.ID, abiJob.AbiName, abiJob.Address, abiJob.AbiSelector, selector)) + if err != nil { + log.Println("Error writing to file:", err) + continue + } + } } } - writer.Flush() - f.Close() + if outputFilePath != "" { + writer.Flush() + + f.Close() + } return nil } @@ -1069,3 +1100,425 @@ func (p *PostgreSQLpgx) CleanIndexes(blockchain string, batchLimit uint64, sleep return nil } + +func (p *PostgreSQLpgx) UpdateAbiJobsStatus(blockchain string) error { + pool := p.GetPool() + + conn, err := pool.Acquire(context.Background()) + if err != nil { + return err + } + defer conn.Release() + + query := ` + UPDATE abi_jobs + SET historical_crawl_status = 'in_progress', moonworm_task_pickedup = true + WHERE chain = @chain + AND historical_crawl_status = 'pending' + AND status = 'active' + AND deployment_block_number IS NOT NULL + ` + + queryArgs := pgx.NamedArgs{ + "chain": blockchain, + } + + _, err = conn.Exec(context.Background(), query, queryArgs) + if err != nil { + return err + } + + return nil +} + +func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, customersIds []string, autoJobs bool) ([]CustomerUpdates, map[string]AbiJobsDeployInfo, error) { + pool := p.GetPool() + + conn, err := pool.Acquire(context.Background()) + if err != nil { + return nil, nil, err + } + defer conn.Release() + + var queryBuilder strings.Builder + + queryArgs := make(pgx.NamedArgs) + + queryArgs["chain"] = blockchain + + queryBuilder.WriteString(` + SELECT id, address, user_id, customer_id, abi_selector, chain, abi_name, status, + historical_crawl_status, progress, moonworm_task_pickedup, '[' || abi || ']' as abi, + (abi::jsonb)->>'type' AS abiType, created_at, updated_at, deployment_block_number + FROM abi_jobs + WHERE chain = @chain AND ((abi::jsonb)->>'type' = 'function' or (abi::jsonb)->>'type' = 'event') and deployment_block_number is not null + `) + + if autoJobs { + queryBuilder.WriteString(" AND historical_crawl_status != 'done' ") + } + + if len(addresses) > 0 { + queryBuilder.WriteString(" AND address = ANY(@addresses) ") + + // decode addresses + addressesBytes := make([][]byte, len(addresses)) + for i, address := range addresses { + addressBytes, err := decodeAddress(address) + if err != nil { + return nil, nil, err + } + addressesBytes[i] = addressBytes // Assign directly to the index + } + + queryArgs["addresses"] = addressesBytes + } + + if len(customersIds) > 0 { + queryBuilder.WriteString(" AND customer_id = ANY(@customer_ids) ") + queryArgs["customer_ids"] = customersIds + } + + rows, err := conn.Query(context.Background(), queryBuilder.String(), queryArgs) + if err != nil { + log.Println("Error querying ABI jobs from database", err) + return nil, nil, err + } + + abiJobs, err := pgx.CollectRows(rows, pgx.RowToStructByName[AbiJob]) + if err != nil { + log.Println("Error collecting ABI jobs rows", err) + return nil, nil, err + } + + if len(abiJobs) == 0 { + return []CustomerUpdates{}, map[string]AbiJobsDeployInfo{}, nil + } + + customerUpdatesDict := make(map[string]CustomerUpdates) + addressDeployBlockDict := make(map[string]AbiJobsDeployInfo) + + for _, abiJob := range abiJobs { + address := fmt.Sprintf("0x%x", abiJob.Address) + + if _, exists := customerUpdatesDict[abiJob.CustomerID]; !exists { + customerUpdatesDict[abiJob.CustomerID] = CustomerUpdates{ + CustomerID: abiJob.CustomerID, + Abis: make(map[string]map[string]*AbiEntry), + } + } + + if _, exists := customerUpdatesDict[abiJob.CustomerID].Abis[address]; !exists { + customerUpdatesDict[abiJob.CustomerID].Abis[address] = make(map[string]*AbiEntry) + } + + customerUpdatesDict[abiJob.CustomerID].Abis[address][abiJob.AbiSelector] = &AbiEntry{ + AbiJSON: abiJob.Abi, + AbiName: abiJob.AbiName, + AbiType: abiJob.AbiType, + } + + if abiJob.DeploymentBlockNumber == nil { + value := uint64(1) + abiJob.DeploymentBlockNumber = &value + } + + // Retrieve the struct from the map + deployInfo, exists := addressDeployBlockDict[address] + + if !exists { + // Initialize the struct if it doesn't exist + deployInfo = AbiJobsDeployInfo{ + DeployedBlockNumber: *abiJob.DeploymentBlockNumber, + IDs: []string{}, + } + } + + // Modify the struct + deployInfo.IDs = append(deployInfo.IDs, abiJob.ID) + + // Store the modified struct back in the map + addressDeployBlockDict[address] = deployInfo + } + + var customerUpdates []CustomerUpdates + for _, customerUpdate := range customerUpdatesDict { + customerUpdates = append(customerUpdates, customerUpdate) + } + + return customerUpdates, addressDeployBlockDict, nil +} + +func (p *PostgreSQLpgx) UpdateAbisAsDone(ids []string) error { + pool := p.GetPool() + + conn, err := pool.Acquire(context.Background()) + if err != nil { + return err + } + defer conn.Release() + + query := ` + UPDATE abi_jobs + SET historical_crawl_status = 'done', progress = 100 + WHERE id = ANY($1) + ` + + _, err = conn.Exec(context.Background(), query, ids) + if err != nil { + return err + } + + return nil +} + +func (p *PostgreSQLpgx) FindBatchPath(blockchain string, blockNumber uint64) (string, uint64, uint64, error) { + pool := p.GetPool() + + conn, err := pool.Acquire(context.Background()) + + if err != nil { + return "", 0, 0, err + } + + defer conn.Release() + + var path string + + var minBlockNumber uint64 + + var maxBlockNumber uint64 + query := fmt.Sprintf(`WITH path as ( + SELECT + path + from + %s + WHERE + block_number = $1 + ) SELECT path, min(block_number), max(block_number) FROM %s WHERE path = (SELECT path from path) group by path`, BlocksTableName(blockchain), BlocksTableName(blockchain)) + + err = conn.QueryRow(context.Background(), query, blockNumber).Scan(&path, &minBlockNumber, &maxBlockNumber) + + if err != nil { + if err == pgx.ErrNoRows { + // Blocks not indexed yet + return "", 0, 0, nil + } + return "", + 0, + 0, + err + } + + return path, minBlockNumber, maxBlockNumber, nil + +} + +func (p *PostgreSQLpgx) GetAbiJobsWithoutDeployBlocks(blockchain string) (map[string]map[string][]string, error) { + pool := p.GetPool() + + conn, err := pool.Acquire(context.Background()) + + if err != nil { + return nil, err + } + + defer conn.Release() + + /// get all addresses that not have deploy block number + + rows, err := conn.Query(context.Background(), `SELECT + id, + chain, + address + FROM + abi_jobs + WHERE + deployment_block_number is null + and chain = $1 + and ( + (abi :: jsonb) ->> 'type' = 'event' + or ( + (abi :: jsonb) ->> 'type' = 'function' + and (abi :: jsonb) ->> 'stateMutability' != 'view' + ) + )`, blockchain) + if err != nil { + log.Println("Error querying abi jobs from database", err) + return nil, err + } + + // chain, address, ids + chainsAddresses := make(map[string]map[string][]string) + + for rows.Next() { + + var id string + var chain string + var raw_address []byte + var address string + + err = rows.Scan(&id, &chain, &raw_address) + + if err != nil { + return nil, err + } + + address = fmt.Sprintf("0x%x", raw_address) + + if _, exists := chainsAddresses[chain]; !exists { + chainsAddresses[chain] = make(map[string][]string) + } + + chainsAddresses[chain][address] = append(chainsAddresses[chain][address], id) + + } + + // Run ensure selector for each chain + + for chain, addressIds := range chainsAddresses { + + for address := range addressIds { + + err := p.EnsureCorrectSelectors(chain, true, "", addressIds[address]) + if err != nil { + + log.Println("Error ensuring correct selectors for chain:", chain, err) + return nil, err + } + } + + } + + return chainsAddresses, nil +} + +func (p *PostgreSQLpgx) UpdateAbisProgress(ids []string, process int) error { + pool := p.GetPool() + + conn, err := pool.Acquire(context.Background()) + + if err != nil { + return err + } + + defer conn.Release() + + // Transform the ids to a slice of UUIDs + idsUUID := make([]uuid.UUID, len(ids)) + for i, id := range ids { + idsUUID[i], err = uuid.Parse(id) + if err != nil { + return err + } + } + + _, err = conn.Exec(context.Background(), "UPDATE abi_jobs SET progress=$1 WHERE id=ANY($2)", process, idsUUID) + + if err != nil { + return err + } + + return nil + +} + +func (p *PostgreSQLpgx) UpdateAbiJobsDeployBlock(blockNumber uint64, ids []string) error { + pool := p.GetPool() + + conn, err := pool.Acquire(context.Background()) + + if err != nil { + return err + } + + defer conn.Release() + + // Transform the ids to a slice of UUIDs + idsUUID := make([]uuid.UUID, len(ids)) + for i, id := range ids { + idsUUID[i], err = uuid.Parse(id) + if err != nil { + return err + } + } + + _, err = conn.Exec(context.Background(), "UPDATE abi_jobs SET deployment_block_number=$1 WHERE id=ANY($2)", blockNumber, idsUUID) + + if err != nil { + return err + } + + return nil + +} + +func (p *PostgreSQLpgx) CreateJobsFromAbi(chain string, address string, abiFile string, customerID string, userID string, deployBlock uint64) error { + pool := p.GetPool() + + conn, err := pool.Acquire(context.Background()) + if err != nil { + return err + } + defer conn.Release() + + abiData, err := ioutil.ReadFile(abiFile) + if err != nil { + return err + } + + var abiJson []map[string]interface{} + err = json.Unmarshal(abiData, &abiJson) + if err != nil { + return err + } + + for _, abiJob := range abiJson { + + // Generate a new UUID for the id column + jobID := uuid.New() + + abiJobJson, err := json.Marshal(abiJob) + if err != nil { + log.Println("Error marshalling ABI job to JSON:", abiJob, err) + return err + } + + // Wrap the JSON string in an array + abiJsonArray := "[" + string(abiJobJson) + "]" + + // Get the correct selector for the ABI + abiObj, err := abi.JSON(strings.NewReader(abiJsonArray)) + if err != nil { + log.Println("Error parsing ABI for ABI job:", abiJsonArray, err) + return err + } + var selector string + + if abiJob["type"] == "event" { + selector = abiObj.Events[abiJob["name"].(string)].ID.String() + } else if abiJob["type"] == "function" { + selectorRaw := abiObj.Methods[abiJob["name"].(string)].ID + selector = fmt.Sprintf("0x%x", selectorRaw) + } else { + log.Println("ABI type not supported:", abiJob["type"]) + continue + } + + addressBytes, err := decodeAddress(address) + + if err != nil { + log.Println("Error decoding address:", err, address) + continue + } + _, err = conn.Exec(context.Background(), "INSERT INTO abi_jobs (id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, moonworm_task_pickedup, abi, deployment_block_number, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), now())", jobID, addressBytes, userID, customerID, selector, chain, abiJob["name"], "true", "pending", 0, false, abiJobJson, deployBlock) + + if err != nil { + return err + } + + } + + return nil + +} diff --git a/indexer/types.go b/indexer/types.go index 8438d2d..0c730c1 100644 --- a/indexer/types.go +++ b/indexer/types.go @@ -1,6 +1,11 @@ package indexer -import "time" +import ( + "sync" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi" +) // gorm is a Go ORM library for working with databases @@ -65,36 +70,26 @@ type AbiJob struct { AbiType string CreatedAt time.Time UpdatedAt time.Time + DeploymentBlockNumber *uint64 } type CustomerUpdates struct { - CustomerID string `json:"customer_id"` - Abis map[string]map[string]map[string]string `json:"abis"` - LastBlock uint64 `json:"last_block"` - Path string `json:"path"` + CustomerID string `json:"customer_id"` + Abis map[string]map[string]*AbiEntry `json:"abis"` + LastBlock uint64 `json:"last_block"` + Path string `json:"path"` } type TaskForTransaction struct { - Hash string `json:"hash"` Address string `json:"address"` Selector string `json:"selector"` ABI string `json:"abi"` - RowID uint64 `json:"row_id"` - Path string `json:"path"` } type TaskForLog struct { // Assuming structure similar to TransactionIndex - Hash string `json:"hash"` Address string `json:"address"` Selector string `json:"selector"` ABI string `json:"abi"` - RowID uint64 `json:"row_id"` - Path string `json:"path"` -} - -type RawChainData struct { - Transactions []TaskForTransaction `json:"transactions"` - Events []TaskForLog `json:"events"` } type EventLabel struct { @@ -125,3 +120,16 @@ type TransactionLabel struct { LabelData string BlockTimestamp uint64 } + +type AbiJobsDeployInfo struct { + DeployedBlockNumber uint64 + IDs []string +} + +type AbiEntry struct { + AbiJSON string `json:"abi"` + Abi *abi.ABI + AbiName string `json:"abi_name"` + AbiType string `json:"abi_type"` + Once sync.Once +} diff --git a/synchronizer/synchronizer.go b/synchronizer/synchronizer.go index 3c47a3c..3e9d9da 100644 --- a/synchronizer/synchronizer.go +++ b/synchronizer/synchronizer.go @@ -14,10 +14,10 @@ import ( "time" seer_blockchain "github.com/moonstream-to/seer/blockchain" + "github.com/moonstream-to/seer/blockchain/common" "github.com/moonstream-to/seer/crawler" "github.com/moonstream-to/seer/indexer" "github.com/moonstream-to/seer/storage" - "golang.org/x/exp/slices" ) type Synchronizer struct { @@ -30,10 +30,11 @@ type Synchronizer struct { batchSize uint64 baseDir string basePath string + threads int } // NewSynchronizer creates a new synchronizer instance with the given blockchain handler. -func NewSynchronizer(blockchain, baseDir string, startBlock, endBlock, batchSize uint64, timeout int) (*Synchronizer, error) { +func NewSynchronizer(blockchain, baseDir string, startBlock, endBlock, batchSize uint64, timeout int, threads int) (*Synchronizer, error) { var synchronizer Synchronizer basePath := filepath.Join(baseDir, crawler.SeerCrawlerStoragePrefix, "data", blockchain) @@ -51,6 +52,10 @@ func NewSynchronizer(blockchain, baseDir string, startBlock, endBlock, batchSize log.Printf("Initialized new synchronizer at blockchain: %s, startBlock: %d, endBlock: %d", blockchain, startBlock, endBlock) + if threads <= 0 { + threads = 1 + } + synchronizer = Synchronizer{ Client: client, StorageInstance: storageInstance, @@ -61,6 +66,7 @@ func NewSynchronizer(blockchain, baseDir string, startBlock, endBlock, batchSize batchSize: batchSize, baseDir: baseDir, basePath: basePath, + threads: threads, } return &synchronizer, nil @@ -155,30 +161,92 @@ type CustomerDBConnection struct { Pgx *indexer.PostgreSQLpgx } -// getCustomers fetch ABI jobs, customer IDs and database URLs -func (d *Synchronizer) getCustomers(customerDbUriFlag string) (map[string]CustomerDBConnection, []string, error) { - customerDBConnections := make(map[string]CustomerDBConnection) - var customerIds []string +func CustomersLatestBlocks(customerDBConnections map[string]CustomerDBConnection, blockchain string) (map[string]uint64, error) { + latestBlocks := make(map[string]uint64) + for id, customer := range customerDBConnections { + pool := customer.Pgx.GetPool() + conn, err := pool.Acquire(context.Background()) + if err != nil { + log.Println("Error acquiring pool connection: ", err) + return nil, err + } + defer conn.Release() - // Read ABI jobs from database - abiJobs, err := d.ReadAbiJobsFromDatabase(d.blockchain) + latestLabelBlock, err := customer.Pgx.ReadLastLabel(blockchain) + if err != nil { + log.Println("Error reading latest block: ", err) + return nil, err + } + latestBlocks[id] = latestLabelBlock + log.Printf("Latest block for customer %s is: %d\n", id, latestLabelBlock) + } + return latestBlocks, nil +} + +func (d *Synchronizer) StartBlockLookUp(customerDBConnections map[string]CustomerDBConnection, blockchain string, blockShift int64) (uint64, error) { + var startBlock uint64 + latestCustomerBlocks, err := CustomersLatestBlocks(customerDBConnections, d.blockchain) if err != nil { - return customerDBConnections, customerIds, err + return startBlock, fmt.Errorf("error getting latest blocks for customers: %w", err) } - // Create a set of customer IDs from ABI jobs to remove duplicates - customerIdsSet := make(map[string]struct{}) - for _, job := range abiJobs { - customerIdsSet[job.CustomerID] = struct{}{} + // Determine the start block as the maximum of the latest blocks of all customers + var maxCustomerLatestBlock uint64 + if len(latestCustomerBlocks) > 0 { + for _, block := range latestCustomerBlocks { + if block > maxCustomerLatestBlock { + maxCustomerLatestBlock = block + } + } + } + + if maxCustomerLatestBlock != 0 { + startBlock = maxCustomerLatestBlock + } else { + // In case start block is still 0, get the latest block from the blockchain minus shift + latestBlockNumber, latestErr := d.Client.GetLatestBlockNumber() + if latestErr != nil { + return startBlock, fmt.Errorf("error getting latest block number: %w", latestErr) + } + startBlock = uint64(latestBlockNumber.Int64() - blockShift) + } + + return startBlock, nil + +} + +// getCustomers fetches ABI jobs, extracts customer IDs, and establishes database connections. +func (d *Synchronizer) getCustomers(customerDbUriFlag string, customerIds []string) (map[string]CustomerDBConnection, []string, error) { + customerDBConnections := make(map[string]CustomerDBConnection) + customerIdSet := make(map[string]struct{}) + + // If no customer IDs are provided as arguments, fetch them from ABI jobs. + if len(customerIds) == 0 { + abiJobs, err := d.ReadAbiJobsFromDatabase(d.blockchain) + if err != nil { + return nil, nil, fmt.Errorf("failed to read ABI jobs: %w", err) + } + + // Extract unique customer IDs from the ABI jobs. + for _, job := range abiJobs { + customerIdSet[job.CustomerID] = struct{}{} + } + } else { + // Otherwise, use the provided customer IDs directly. + for _, id := range customerIds { + customerIdSet[id] = struct{}{} + } } - for id := range customerIdsSet { + var finalCustomerIds []string + for id := range customerIdSet { var connectionString string var dbConnErr error + if customerDbUriFlag == "" { connectionString, dbConnErr = GetDBConnection(id) if dbConnErr != nil { - log.Printf("Unable to get connection database URI for %s customer, err: %v", id, dbConnErr) + log.Printf("Unable to get connection database URI for customer %s, err: %v", id, dbConnErr) continue } } else { @@ -187,7 +255,7 @@ func (d *Synchronizer) getCustomers(customerDbUriFlag string) (map[string]Custom pgx, pgxErr := indexer.NewPostgreSQLpgxWithCustomURI(connectionString) if pgxErr != nil { - log.Printf("Error creating RDS connection for %s customer, err: %v", id, pgxErr) + log.Printf("Error creating RDS connection for customer %s, err: %v", id, pgxErr) continue } @@ -195,12 +263,12 @@ func (d *Synchronizer) getCustomers(customerDbUriFlag string) (map[string]Custom Uri: connectionString, Pgx: pgx, } - customerIds = append(customerIds, id) - + finalCustomerIds = append(finalCustomerIds, id) } - log.Println("Customer IDs to sync:", customerIds) - return customerDBConnections, customerIds, nil + log.Println("Customer IDs to sync:", finalCustomerIds) + + return customerDBConnections, finalCustomerIds, nil } func (d *Synchronizer) Start(customerDbUriFlag string) { @@ -234,48 +302,18 @@ func (d *Synchronizer) Start(customerDbUriFlag string) { func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { var isEnd bool - customerDBConnections, customerIds, customersErr := d.getCustomers(customerDbUriFlag) + customerDBConnections, customerIds, customersErr := d.getCustomers(customerDbUriFlag, nil) if customersErr != nil { return isEnd, customersErr } - // Set startBlocks as latest labeled block from customers or from the blockchain if customers are not indexed yet + // Set start block if 0 if d.startBlock == 0 { - var latestCustomerBlocks []uint64 - for id, customer := range customerDBConnections { - - pool := customer.Pgx.GetPool() - conn, err := pool.Acquire(context.Background()) - if err != nil { - log.Println("Error acquiring pool connection: ", err) - return isEnd, err - } - defer conn.Release() - - latestLabelBlock, err := customer.Pgx.ReadLastLabel(d.blockchain) - if err != nil { - log.Println("Error reading latest block: ", err) - return isEnd, err - } - latestCustomerBlocks = append(latestCustomerBlocks, latestLabelBlock) - log.Printf("Latest block for customer %s is: %d\n", id, latestLabelBlock) - } - - // Determine the start block as the maximum of the latest blocks of all customers - var maxCustomerLatestBlock uint64 - if len(latestCustomerBlocks) != 0 { - maxCustomerLatestBlock = slices.Max(latestCustomerBlocks) - } - if maxCustomerLatestBlock != 0 { - d.startBlock = maxCustomerLatestBlock - 100 - } else { - // In case start block is still 0, get the latest block from the blockchain minus shift - latestBlockNumber, latestErr := d.Client.GetLatestBlockNumber() - if latestErr != nil { - return isEnd, fmt.Errorf("failed to get latest block number: %v", latestErr) - } - d.startBlock = uint64(latestBlockNumber.Int64() - crawler.SeerDefaultBlockShift) + startBlock, startErr := d.StartBlockLookUp(customerDBConnections, d.blockchain, crawler.SeerDefaultBlockShift) + if startErr != nil { + return isEnd, fmt.Errorf("error determining start block: %w", startErr) } + d.startBlock = startBlock } // Get the latest block from the indexer db @@ -284,7 +322,7 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { return isEnd, idxLatestErr } - if d.endBlock != 0 && indexedLatestBlock > d.endBlock { + if d.endBlock > 0 && indexedLatestBlock > d.endBlock { indexedLatestBlock = d.endBlock } @@ -301,20 +339,24 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { //var noUpdatesFoundErr *indexer.NoUpdatesFoundError var isCycleFinished bool for { - if d.endBlock != 0 { - if d.startBlock >= d.endBlock { - isEnd = true - isCycleFinished = true - log.Printf("End block %d almost reached", d.endBlock) - } + // Check if end block is reached or start block exceeds end block + if d.endBlock > 0 && d.startBlock >= d.endBlock { + isEnd = true + isCycleFinished = true + log.Printf("End block %d almost reached", d.endBlock) + break } + if d.endBlock >= indexedLatestBlock { isCycleFinished = true } // Read updates from the indexer db // This function will return a list of customer updates 1 update is 1 customer - lastBlockOfChank, path, updates, err := indexer.DBConnection.ReadUpdates(d.blockchain, d.startBlock, customerIds) + _, lastBlockOfChank, path, updates, err := indexer.DBConnection.ReadUpdates(d.blockchain, d.startBlock, customerIds) + if err != nil { + return isEnd, fmt.Errorf("error reading updates: %w", err) + } if len(updates) == 0 { log.Printf("No updates found for block %d\n", d.startBlock) @@ -324,7 +366,6 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { if crawler.SEER_CRAWLER_DEBUG { log.Printf("Read batch key: %s", path) } - log.Println("Last block of current chank: ", lastBlockOfChank) // Read the raw data from the storage for current path @@ -333,10 +374,6 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { return isEnd, fmt.Errorf("error reading events for customers %s: %w", readErr) } - if err != nil { - return isEnd, fmt.Errorf("error reading updates: %w", err) - } - log.Printf("Read %d users updates from the indexer db in range of blocks %d-%d\n", len(updates), d.startBlock, lastBlockOfChank) var wg sync.WaitGroup @@ -345,63 +382,357 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { for _, update := range updates { wg.Add(1) - go func(update indexer.CustomerUpdates, rawData bytes.Buffer) { - defer wg.Done() + go d.processProtoCustomerUpdate(update, rawData, customerDBConnections, sem, errChan, &wg) + } - sem <- struct{}{} // Acquire semaphore + wg.Wait() - // Get the RDS connection for the customer - customer, customerExists := customerDBConnections[update.CustomerID] - if !customerExists { - errChan <- fmt.Errorf("no DB connection for customer %s", update.CustomerID) - return - } + close(sem) + close(errChan) // Close the channel to signal that all goroutines have finished + + // Check for errors from goroutines + if err := <-errChan; err != nil { + return isEnd, err + } + + d.startBlock = lastBlockOfChank + 1 + + if isCycleFinished { + break + } + } + + return isEnd, nil +} + +func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []string, customerIds []string, batchSize uint64, autoJobs bool) error { + var useRPC bool + var isCycleFinished bool + var updateDeadline time.Time + var initialStartBlock uint64 + + // Initialize start block if 0 + if d.startBlock == 0 { + // Get the latest block from the indexer db + indexedLatestBlock, err := indexer.DBConnection.GetLatestDBBlockNumber(d.blockchain) + if err != nil { + return fmt.Errorf("error getting latest block number: %w", err) + } + d.startBlock = indexedLatestBlock + fmt.Printf("Start block is %d\n", d.startBlock) + } + + // Automatically update ABI jobs as active if auto mode is enabled + if autoJobs { + if err := indexer.DBConnection.UpdateAbiJobsStatus(d.blockchain); err != nil { + return fmt.Errorf("error updating ABI: %w", err) + } + } + + // Retrieve customer updates and deployment blocks + customerUpdates, addressesAbisInfo, err := indexer.DBConnection.SelectAbiJobs(d.blockchain, addresses, customerIds, autoJobs) + if err != nil { + return fmt.Errorf("error selecting ABI jobs: %w", err) + } + + fmt.Printf("Found %d customer updates\n", len(customerUpdates)) + + // Filter out blocks more + // TODO: Maybe autoJobs only + for address, abisInfo := range addressesAbisInfo { + log.Printf("Address %s has deployed block %d\n", address, abisInfo.DeployedBlockNumber) + if abisInfo.DeployedBlockNumber > d.startBlock { + log.Printf("Finished crawling for address %s at block %d\n", address, abisInfo.DeployedBlockNumber) + delete(addressesAbisInfo, address) + } + } + + // Get customer database connections + customerIdsMap := make(map[string]bool) + for _, update := range customerUpdates { + customerIdsMap[update.CustomerID] = true + } + var customerIdsList []string + for id := range customerIdsMap { + customerIdsList = append(customerIdsList, id) + } + + customerDBConnections, _, err := d.getCustomers(customerDbUriFlag, customerIdsList) + if err != nil { + return fmt.Errorf("error getting customers: %w", err) + } + + updateDeadline = time.Now() + + // Main processing loop + for { + + if autoJobs { - // Create a connection to the user RDS - pool := customer.Pgx.GetPool() - conn, err := pool.Acquire(context.Background()) - if err != nil { - errChan <- fmt.Errorf("error acquiring connection for customer %s: %w", update.CustomerID, err) - return + for address, abisInfo := range addressesAbisInfo { + if abisInfo.DeployedBlockNumber > d.startBlock { + + log.Printf("Finished crawling for address %s at block %d\n", address, abisInfo.DeployedBlockNumber) + + // update the status of the address for the customer to done + err := indexer.DBConnection.UpdateAbisAsDone(abisInfo.IDs) + if err != nil { + return err + } + + // drop the address + delete(addressesAbisInfo, address) } - defer conn.Release() - var decodedEventsPack []indexer.EventLabel - var decodedTransactionsPack []indexer.TransactionLabel + // Check if the deadline for the update has passed + if updateDeadline.Add(1 * time.Minute).Before(time.Now()) { + for address, abisInfo := range addressesAbisInfo { + ids := abisInfo.IDs + + // calculate progress as a percentage between 0 and 100 + + progress := 100 - int(100*(d.startBlock-abisInfo.DeployedBlockNumber)/(d.startBlock-initialStartBlock)) + + err := indexer.DBConnection.UpdateAbisProgress(ids, progress) + if err != nil { + continue + } + + log.Printf("Updated progress for address %s to %d%%\n", address, progress) + + } + updateDeadline = time.Now() - // decodedEvents, decodedTransactions, decErr - decodedEvents, decodedTransactions, decErr := d.Client.DecodeProtoEntireBlockToLabels(&rawData, update.Abis) - if decErr != nil { - log.Println("Error decoding events: ", decErr) - errChan <- fmt.Errorf("error decoding events for customer %s: %w", update.CustomerID, decErr) - return } + } + + } - decodedEventsPack = append(decodedEventsPack, decodedEvents...) - decodedTransactionsPack = append(decodedTransactionsPack, decodedTransactions...) + if len(addressesAbisInfo) == 0 { + break + } - customer.Pgx.WriteLabes(d.blockchain, decodedTransactionsPack, decodedEventsPack) + // Determine the processing strategy (RPC or storage) + var path string + var firstBlockOfChunk uint64 + if !useRPC { - <-sem - }(update, rawData) + path, firstBlockOfChunk, _, err = indexer.DBConnection.FindBatchPath(d.blockchain, d.startBlock) + if err != nil { + return fmt.Errorf("error finding batch path: %w", err) + } + + if path == "" { + log.Printf("No batch path found for block %d\n", d.startBlock) + log.Println("Switching to RPC mode") + useRPC = true + } else { + d.endBlock = firstBlockOfChunk + } } - wg.Wait() + if useRPC { + // Calculate the end block + if d.startBlock > batchSize { + d.endBlock = d.startBlock - batchSize + } else { + d.endBlock = 1 // Prevent underflow by setting endBlock to 1 if startBlock < batchSize + } + + fmt.Printf("Start block is %d, end block is %d\n", d.startBlock, d.endBlock) + } + + // Read raw data from storage or via RPC + var rawData bytes.Buffer + if !useRPC { + rawData, err = d.StorageInstance.Read(path) + if err != nil { + return fmt.Errorf("error reading events from storage: %w", err) + } + } + log.Printf("Processing %d customer updates for block range %d-%d", len(customerUpdates), d.startBlock, d.endBlock) + + // Process customer updates in parallel + var wg sync.WaitGroup + sem := make(chan struct{}, d.threads) // Semaphore to control concurrency + errChan := make(chan error, 1) // Buffered channel for error handling + + for _, update := range customerUpdates { + wg.Add(1) + if useRPC { + go d.processRPCCustomerUpdate(update, customerDBConnections, sem, errChan, &wg) + } else { + go d.processProtoCustomerUpdate(update, rawData, customerDBConnections, sem, errChan, &wg) + } + } + + wg.Wait() close(sem) - close(errChan) // Close the channel to signal that all goroutines have finished // Check for errors from goroutines - if err := <-errChan; err != nil { - return isEnd, err + select { + case err := <-errChan: + log.Printf("Error processing customer updates: %v", err) + return err + default: } - d.startBlock = lastBlockOfChank + 1 + d.startBlock = d.endBlock - 1 - if isCycleFinished { + if isCycleFinished || d.startBlock == 0 { + log.Println("Finished processing all customer updates") break } } - return isEnd, nil + return nil +} + +func (d *Synchronizer) processProtoCustomerUpdate( + update indexer.CustomerUpdates, + rawData bytes.Buffer, + customerDBConnections map[string]CustomerDBConnection, + sem chan struct{}, + errChan chan error, + wg *sync.WaitGroup, +) { + // Decode input raw proto data using ABIs + // Write decoded data to the user Database + + defer wg.Done() + sem <- struct{}{} // Acquire semaphore + + customer, exists := customerDBConnections[update.CustomerID] + if !exists { + errChan <- fmt.Errorf("no DB connection for customer %s", update.CustomerID) + <-sem // Release semaphore + return + } + + conn, err := customer.Pgx.GetPool().Acquire(context.Background()) + if err != nil { + errChan <- fmt.Errorf("error acquiring connection for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + } + defer conn.Release() + decodedEvents, decodedTransactions, err := d.Client.DecodeProtoEntireBlockToLabels(&rawData, update.Abis, d.threads) + if err != nil { + errChan <- fmt.Errorf("error %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + } + + err = customer.Pgx.WriteLabes(d.blockchain, decodedTransactions, decodedEvents) + + if err != nil { + errChan <- fmt.Errorf("error writing labels for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + } + + <-sem // Release semaphore +} + +func (d *Synchronizer) processRPCCustomerUpdate( + update indexer.CustomerUpdates, + customerDBConnections map[string]CustomerDBConnection, + sem chan struct{}, + errChan chan error, + wg *sync.WaitGroup, +) { + // Decode input raw proto data using ABIs + // Write decoded data to the user Database + + defer wg.Done() + sem <- struct{}{} // Acquire semaphore + + customer, exists := customerDBConnections[update.CustomerID] + if !exists { + errChan <- fmt.Errorf("no DB connection for customer %s", update.CustomerID) + <-sem // Release semaphore + return + } + + conn, err := customer.Pgx.GetPool().Acquire(context.Background()) + if err != nil { + errChan <- fmt.Errorf("error acquiring connection for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + } + defer conn.Release() + + // split abis by the type of the task + + eventAbis := make(map[string]map[string]*indexer.AbiEntry) + transactionAbis := make(map[string]map[string]*indexer.AbiEntry) + + for address, selectorMap := range update.Abis { + + for selector, abiInfo := range selectorMap { + + if abiInfo.AbiType == "event" { + if _, ok := eventAbis[address]; !ok { + eventAbis[address] = make(map[string]*indexer.AbiEntry) + } + eventAbis[address][selector] = abiInfo + } else { + if _, ok := transactionAbis[address]; !ok { + transactionAbis[address] = make(map[string]*indexer.AbiEntry) + } + transactionAbis[address][selector] = abiInfo + } + } + + } + + // Transactions + + var transactions []indexer.TransactionLabel + var blocksCache map[uint64]common.BlockWithTransactions + + if len(transactionAbis) != 0 { + transactions, blocksCache, err = d.Client.GetTransactionsLabels(d.endBlock, d.startBlock, transactionAbis, d.threads) + + if err != nil { + log.Println("Error getting transactions for customer", update.CustomerID, ":", err) + errChan <- fmt.Errorf("error getting transactions for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + } + } else { + transactions = make([]indexer.TransactionLabel, 0) + } + + // Events + + var events []indexer.EventLabel + + if len(eventAbis) != 0 { + + events, err = d.Client.GetEventsLabels(d.endBlock, d.startBlock, eventAbis, blocksCache) + + if err != nil { + + log.Println("Error getting events for customer", update.CustomerID, ":", err) + + errChan <- fmt.Errorf("error getting events for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + + } + } else { + events = make([]indexer.EventLabel, 0) + } + + err = customer.Pgx.WriteLabes(d.blockchain, transactions, events) + + if err != nil { + errChan <- fmt.Errorf("error writing labels for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + } + + <-sem // Release semaphore } diff --git a/version/version.go b/version/version.go index 6b597fe..6144365 100644 --- a/version/version.go +++ b/version/version.go @@ -1,3 +1,3 @@ package version -var SeerVersion string = "0.1.21" +var SeerVersion string = "0.1.22"