Optimism - indexer improvements (#301)

* wip get blocks range * fix async getBlockRange * log to sentry * udpate read block interval time * update blockUpdateIntervalSeconds * decrease blockUpdateIntervalSeconds * set blockUpdateIntervalSeconds to 4 sec * fix followChain tests
voteagora · Jul 21, 2023 · c0ef7df · c0ef7df
1 parent ee30903
commit c0ef7df
Show file tree

Hide file tree

Showing 5 changed files with 122 additions and 34 deletions.
diff --git a/packages/backend/src/indexer/blockProvider/blockProvider.ts b/packages/backend/src/indexer/blockProvider/blockProvider.ts
@@ -1,5 +1,6 @@
 import { ethers } from "ethers";
 import { BlockIdentifier } from "../storageHandle";
+import { compareBy } from "../utils/sortUtils";
 import { executeWithRetries } from "../utils/asyncUtils";
 import { Toucan } from "toucan-js";
 
@@ -13,6 +14,15 @@ export interface BlockProvider {
   getBlockByHash(hash: string): Promise<BlockProviderBlock>;
   getBlockByNumber(number: number): Promise<BlockProviderBlock | null>;
   getLatestBlock(): Promise<BlockProviderBlock>;
+
+  /**
+   * Returns a list of blocks, sorted in ascending order of
+   * {@link BlockProviderBlock#number}.
+   */
+  getBlockRange(
+    fromBlockInclusive: number,
+    toBlockInclusive: number
+  ): Promise<BlockProviderBlock[]>;
 }
 
 export const maxBlockRange = 1000;
@@ -35,6 +45,23 @@ export class EthersBlockProvider implements BlockProvider {
     return transformResponse(raw);
   }
 
+  async getBlockRange(
+    fromBlockInclusive: number,
+    toBlockInclusive: number
+  ): Promise<BlockProviderBlock[]> {
+    const blocks: Promise<any>[] = new Array(
+      toBlockInclusive - fromBlockInclusive + 1
+    )
+      .fill(null)
+      .map(async (_, index) =>
+        this.getBlockByNumber(fromBlockInclusive + index)
+      );
+
+    return Promise.all(blocks).then((resolvedBlocks) =>
+      resolvedBlocks.sort(compareBy((it) => it.number))
+    );
+  }
+
   async getBlockByNumber(
     number: number,
     sentry?: Toucan

diff --git a/packages/backend/src/indexer/blockProvider/fakeBlockProvider.ts b/packages/backend/src/indexer/blockProvider/fakeBlockProvider.ts
@@ -50,6 +50,17 @@ export class FakeBlockProvider implements BlockProvider {
     return foundBlock?.block ?? null;
   }
 
+  async getBlockRange(
+    fromBlockInclusive: number,
+    toBlockInclusive: number
+  ): Promise<BlockProviderBlock[]> {
+    return getBlocksByRange(
+      this.blocks,
+      fromBlockInclusive,
+      toBlockInclusive
+    ).map((it) => it.block);
+  }
+
   async getLatestBlock(): Promise<BlockProviderBlock> {
     if (!this.blocks.length) {
       throw new Error("no blocks");

diff --git a/packages/backend/src/indexer/followChain.test.ts b/packages/backend/src/indexer/followChain.test.ts
@@ -157,8 +157,8 @@ describe("followChain", () => {
 
       expect(result).toMatchInlineSnapshot(`
         {
-          "depth": 9,
-          "nextBlock": 3,
+          "depth": -1,
+          "nextBlock": 13,
           "type": "MORE",
         }
       `);
@@ -175,10 +175,50 @@ describe("followChain", () => {
               "blockNumber": 1,
               "hash": "0x1",
             },
+            "0x3" => {
+              "blockNumber": 2,
+              "hash": "0x2",
+            },
+            "0x4" => {
+              "blockNumber": 3,
+              "hash": "0x3",
+            },
+            "0x5" => {
+              "blockNumber": 4,
+              "hash": "0x4",
+            },
+            "0x6" => {
+              "blockNumber": 5,
+              "hash": "0x5",
+            },
+            "0x7" => {
+              "blockNumber": 6,
+              "hash": "0x6",
+            },
+            "0x8" => {
+              "blockNumber": 7,
+              "hash": "0x7",
+            },
+            "0x9" => {
+              "blockNumber": 8,
+              "hash": "0x8",
+            },
+            "0xa" => {
+              "blockNumber": 9,
+              "hash": "0x9",
+            },
+            "0xb" => {
+              "blockNumber": 10,
+              "hash": "0xa",
+            },
+            "0xc" => {
+              "blockNumber": 11,
+              "hash": "0xb",
+            },
           },
           "tipBlock": {
-            "blockNumber": 2,
-            "hash": "0x2",
+            "blockNumber": 12,
+            "hash": "0xc",
           },
         }
       `);
@@ -202,9 +242,7 @@ describe("followChain", () => {
 
       expect(result).toMatchInlineSnapshot(`
         {
-          "depth": 8,
-          "nextBlock": 4,
-          "type": "MORE",
+          "type": "TIP",
         }
       `);
     }

diff --git a/packages/backend/src/indexer/followChain.ts b/packages/backend/src/indexer/followChain.ts
@@ -10,6 +10,7 @@ import {
   blockIdentifierFromParentBlock,
   BlockProvider,
   BlockProviderBlock,
+  maxBlockRange,
 } from "./blockProvider/blockProvider";
 import { LogProvider, topicFilterForIndexers } from "./logProvider/logProvider";
 import {
@@ -147,7 +148,7 @@ export function followChain(
 
   let nextBlockNumber = storageArea.finalizedBlock.blockNumber + 1;
 
-  return async () => {
+  return async (maxStepSize: number = maxBlockRange) => {
     const latestBlock = await blockProvider.getLatestBlock();
 
     if (nextBlockNumber > latestBlock.number) {
@@ -156,43 +157,54 @@ export function followChain(
       };
     }
 
-    const nextBlock = await blockProvider.getBlockByNumber(nextBlockNumber);
-    if (!nextBlock) {
-      if (sentry) {
-        sentry.captureException("block not found");
-      }
-      throw new Error("block not found");
-    }
+    // stepSize will be 1 <= stepSize <= maxBlockRange
+    const stepSize = Math.max(1, Math.min(maxStepSize, maxBlockRange));
+
+    const fetchTill = Math.min(latestBlock.number, nextBlockNumber + stepSize);
+
+    const blocks = await blockProvider.getBlockRange(
+      nextBlockNumber,
+      fetchTill
+    );
 
     const logs = await logProvider.getLogs({
       ...filter,
       fromBlock: nextBlockNumber,
-      toBlock: nextBlockNumber,
+      toBlock: fetchTill,
     });
 
-    const logsCache = await makeLogsCache(logs, [nextBlock]);
+    const logsCache = await makeLogsCache(logs, blocks);
+
+    for (const nextBlock of blocks) {
+      if (!nextBlock) {
+        if (sentry) {
+          sentry.captureException("block not found");
+        }
+        throw new Error("block not found");
+      }
 
-    await ensureParentsAvailable(blockIdentifierFromParentBlock(nextBlock));
+      await ensureParentsAvailable(blockIdentifierFromParentBlock(nextBlock));
 
-    addToParents(storageArea.parents, nextBlock);
+      addToParents(storageArea.parents, nextBlock);
 
-    await processBlock(nextBlock, logsCache);
+      await processBlock(nextBlock, logsCache);
 
-    await promoteFinalizedBlocks(
-      latestBlock.number,
-      blockIdentifierFromBlock(nextBlock),
-      storageArea.finalizedBlock
-    );
+      await promoteFinalizedBlocks(
+        latestBlock.number,
+        blockIdentifierFromBlock(nextBlock),
+        storageArea.finalizedBlock
+      );
 
-    // update storageArea.tipBlock
-    if (
-      !storageArea.tipBlock ||
-      nextBlock.number > storageArea.tipBlock.blockNumber
-    ) {
-      storageArea.tipBlock = blockIdentifierFromBlock(nextBlock);
-    }
+      // update storageArea.tipBlock
+      if (
+        !storageArea.tipBlock ||
+        nextBlock.number > storageArea.tipBlock.blockNumber
+      ) {
+        storageArea.tipBlock = blockIdentifierFromBlock(nextBlock);
+      }
 
-    nextBlockNumber = nextBlock.number + 1;
+      nextBlockNumber = nextBlock.number + 1;
+    }
 
     return {
       type: "MORE" as const,

diff --git a/packages/backend/src/worker/durableObject.ts b/packages/backend/src/worker/durableObject.ts
@@ -19,7 +19,7 @@ import {
 import { EthersBlockProvider } from "../indexer/blockProvider/blockProvider";
 import { EthersLogProvider } from "../indexer/logProvider/logProvider";
 
-export const blockUpdateIntervalSeconds = 10;
+export const blockUpdateIntervalSeconds = 4;
 
 export class StorageDurableObjectV1 {
   private readonly state: DurableObjectState;