From 9cd89fbd4cd1371e5fcf2d1ee04a8e61d2b2a192 Mon Sep 17 00:00:00 2001 From: Rob Moore Date: Mon, 4 Dec 2023 17:35:09 +0800 Subject: [PATCH] fix: Rename `skip-to-newest` to `skip-sync-newest` for consistency with `skip-oldest` test: Added test coverage for `skip-to-newest` --- README.md | 2 +- .../types_subscription.SubscriptionConfig.md | 32 +++++----- ...scription.TransactionSubscriptionParams.md | 39 ++++++------ docs/subscriber.md | 4 +- docs/subscriptions.md | 8 +-- examples/usdc/index.ts | 2 +- src/subscriptions.ts | 5 +- src/types/subscription.ts | 10 +-- tests/scenarios/skip-sync-newest.spec.ts | 62 +++++++++++++++++++ tests/scenarios/skip-to-newest.spec.ts | 47 -------------- tests/transactions.ts | 47 ++++++++++++++ 11 files changed, 162 insertions(+), 96 deletions(-) create mode 100644 tests/scenarios/skip-sync-newest.spec.ts delete mode 100644 tests/scenarios/skip-to-newest.spec.ts create mode 100644 tests/transactions.ts diff --git a/README.md b/README.md index 1894a92..70a86b2 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ const subscriber = new AlgorandSubscriber( ], frequencyInSeconds: 1, maxRoundsToSync: 100, - syncBehaviour: 'skip-to-newest', + syncBehaviour: 'skip-sync-newest', watermarkPersistence: { get: async () => watermark, set: async (newWatermark) => { diff --git a/docs/code/interfaces/types_subscription.SubscriptionConfig.md b/docs/code/interfaces/types_subscription.SubscriptionConfig.md index 1b7bf53..e6b606c 100644 --- a/docs/code/interfaces/types_subscription.SubscriptionConfig.md +++ b/docs/code/interfaces/types_subscription.SubscriptionConfig.md @@ -28,7 +28,7 @@ The set of events to subscribe to / emit [types/subscription.ts:106](https://github.com/algorandfoundation/algokit-subscriber-ts/blob/main/src/types/subscription.ts#L106) -___ +--- ### frequencyInSeconds @@ -40,7 +40,7 @@ The frequency to poll for new blocks in seconds [types/subscription.ts:102](https://github.com/algorandfoundation/algokit-subscriber-ts/blob/main/src/types/subscription.ts#L102) -___ +--- ### maxRoundsToSync @@ -52,27 +52,29 @@ The maximum number of rounds to sync at a time. [types/subscription.ts:104](https://github.com/algorandfoundation/algokit-subscriber-ts/blob/main/src/types/subscription.ts#L104) -___ +--- ### syncBehaviour -• **syncBehaviour**: ``"skip-to-newest"`` \| ``"sync-oldest"`` \| ``"sync-oldest-start-now"`` \| ``"catchup-with-indexer"`` +• **syncBehaviour**: `"skip-sync-newest"` \| `"sync-oldest"` \| `"sync-oldest-start-now"` \| `"catchup-with-indexer"` The behaviour when the number of rounds to sync is greater than `maxRoundsToSync`: - * `skip-to-newest`: Discard old rounds - * `sync-oldest`: Sync from the oldest records up to `maxRoundsToSync` rounds. - **Note:** will be slow to catch up if sync is significantly behind the tip of the chain - * `sync-oldest-start-now`: Sync from the oldest records up to `maxRoundsToSync` rounds, unless - current watermark is `0` in which case it will start `maxRoundsToSync` back from the tip of the chain. - * `catchup-with-indexer`: Will catch up to `tipOfTheChain - maxRoundsToSync` using indexer (fast) and then - continue with algod. +- `skip-sync-newest`: Discard old rounds +- `sync-oldest`: Sync from the oldest records up to `maxRoundsToSync` rounds. + + **Note:** will be slow to catch up if sync is significantly behind the tip of the chain + +- `sync-oldest-start-now`: Sync from the oldest records up to `maxRoundsToSync` rounds, unless + current watermark is `0` in which case it will start `maxRoundsToSync` back from the tip of the chain. +- `catchup-with-indexer`: Will catch up to `tipOfTheChain - maxRoundsToSync` using indexer (fast) and then + continue with algod. #### Defined in [types/subscription.ts:117](https://github.com/algorandfoundation/algokit-subscriber-ts/blob/main/src/types/subscription.ts#L117) -___ +--- ### watermarkPersistence @@ -83,9 +85,9 @@ its position in the chain. #### Type declaration -| Name | Type | -| :------ | :------ | -| `get` | () => `Promise`\<`number`\> | +| Name | Type | +| :---- | :------------------------------------------------ | +| `get` | () => `Promise`\<`number`\> | | `set` | (`newWatermark`: `number`) => `Promise`\<`void`\> | #### Defined in diff --git a/docs/code/interfaces/types_subscription.TransactionSubscriptionParams.md b/docs/code/interfaces/types_subscription.TransactionSubscriptionParams.md index 6d2dc03..17ffb6b 100644 --- a/docs/code/interfaces/types_subscription.TransactionSubscriptionParams.md +++ b/docs/code/interfaces/types_subscription.TransactionSubscriptionParams.md @@ -27,7 +27,7 @@ The filter to apply to find transactions of interest. [types/subscription.ts:7](https://github.com/algorandfoundation/algokit-subscriber-ts/blob/main/src/types/subscription.ts#L7) -___ +--- ### maxRoundsToSync @@ -36,41 +36,42 @@ ___ The maximum number of rounds to sync for each subscription pull/poll. This gives you control over how many rounds you wait for at a time, -your staleness tolerance when using `skip-to-newest` or `fail`, and +your staleness tolerance when using `skip-sync-newest` or `fail`, and your catchup speed when using `sync-oldest`. #### Defined in [types/subscription.ts:25](https://github.com/algorandfoundation/algokit-subscriber-ts/blob/main/src/types/subscription.ts#L25) -___ +--- ### syncBehaviour -• **syncBehaviour**: ``"skip-to-newest"`` \| ``"sync-oldest"`` \| ``"sync-oldest-start-now"`` \| ``"catchup-with-indexer"`` \| ``"fail"`` +• **syncBehaviour**: `"skip-sync-newest"` \| `"sync-oldest"` \| `"sync-oldest-start-now"` \| `"catchup-with-indexer"` \| `"fail"` If the current tip of the configured Algorand blockchain is more than `maxRoundsToSync` past `watermark` then how should that be handled: - * `skip-to-newest`: Discard old blocks/transactions and sync the newest; useful - for real-time notification scenarios where you don't care about history and - are happy to lose old transactions. - * `sync-oldest`: Sync from the oldest rounds forward `maxRoundsToSync` rounds - using algod; note: this will be slow if you are starting from 0 and requires - an archival node. - * `sync-oldest-start-now`: Same as `sync-oldest`, but if the `watermark` is `0` - then start at the current round i.e. don't sync historical records, but once - subscribing starts sync everything; note: if it falls behind it requires an - archival node. - * `catchup-with-indexer`: Sync to round `currentRound - maxRoundsToSync + 1` - using indexer (much faster than using algod for long time periods) and then - use algod from there. - * `fail`: Throw an error. + +- `skip-sync-newest`: Discard old blocks/transactions and sync the newest; useful + for real-time notification scenarios where you don't care about history and + are happy to lose old transactions. +- `sync-oldest`: Sync from the oldest rounds forward `maxRoundsToSync` rounds + using algod; note: this will be slow if you are starting from 0 and requires + an archival node. +- `sync-oldest-start-now`: Same as `sync-oldest`, but if the `watermark` is `0` + then start at the current round i.e. don't sync historical records, but once + subscribing starts sync everything; note: if it falls behind it requires an + archival node. +- `catchup-with-indexer`: Sync to round `currentRound - maxRoundsToSync + 1` + using indexer (much faster than using algod for long time periods) and then + use algod from there. +- `fail`: Throw an error. #### Defined in [types/subscription.ts:43](https://github.com/algorandfoundation/algokit-subscriber-ts/blob/main/src/types/subscription.ts#L43) -___ +--- ### watermark diff --git a/docs/subscriber.md b/docs/subscriber.md index 841fa6b..fcc7ce6 100644 --- a/docs/subscriber.md +++ b/docs/subscriber.md @@ -28,7 +28,7 @@ export interface SubscriptionConfig { /** The set of events to subscribe to / emit */ events: SubscriptionConfigEvent[] /** The behaviour when the number of rounds to sync is greater than `maxRoundsToSync`: - * * `skip-to-newest`: Discard old rounds + * * `skip-sync-newest`: Discard old rounds * * `sync-oldest`: Sync from the oldest records up to `maxRoundsToSync` rounds. * * **Note:** will be slow to catch up if sync is significantly behind the tip of the chain @@ -37,7 +37,7 @@ export interface SubscriptionConfig { * * `catchup-with-indexer`: Will catch up to `tipOfTheChain - maxRoundsToSync` using indexer (fast) and then * continue with algod. */ - syncBehaviour: 'skip-to-newest' | 'sync-oldest' | 'sync-oldest-start-now' | 'catchup-with-indexer' + syncBehaviour: 'skip-sync-newest' | 'sync-oldest' | 'sync-oldest-start-now' | 'catchup-with-indexer' /** Methods to retrieve and persist the current watermark so syncing is resilient and maintains * its position in the chain. */ watermarkPersistence: { diff --git a/docs/subscriptions.md b/docs/subscriptions.md index 513617a..e2f25ed 100644 --- a/docs/subscriptions.md +++ b/docs/subscriptions.md @@ -43,13 +43,13 @@ export interface TransactionSubscriptionParams { /** The maximum number of rounds to sync for each subscription pull/poll. * * This gives you control over how many rounds you wait for at a time, - * your staleness tolerance when using `skip-to-newest` or `fail`, and + * your staleness tolerance when using `skip-sync-newest` or `fail`, and * your catchup speed when using `sync-oldest`. **/ maxRoundsToSync: number /** If the current tip of the configured Algorand blockchain is more than `maxRoundsToSync` * past `watermark` then how should that be handled: - * * `skip-to-newest`: Discard old blocks/transactions and sync the newest; useful + * * `skip-sync-newest`: Discard old blocks/transactions and sync the newest; useful * for real-time notification scenarios where you don't care about history and * are happy to lose old transactions. * * `sync-oldest`: Sync from the oldest rounds forward `maxRoundsToSync` rounds @@ -64,7 +64,7 @@ export interface TransactionSubscriptionParams { * use algod from there. * * `fail`: Throw an error. **/ - syncBehaviour: 'skip-to-newest' | 'sync-oldest' | 'sync-oldest-start-now' | 'catchup-with-indexer' | 'fail' + syncBehaviour: 'skip-sync-newest' | 'sync-oldest' | 'sync-oldest-start-now' | 'catchup-with-indexer' | 'fail' } /** Specify a filter to apply to find transactions of interest. */ @@ -145,7 +145,7 @@ const subscription = await getSubscribedTransactions( }, watermark, maxRoundsToSync: 100, - onMaxRounds: 'skip-to-newest', + onMaxRounds: 'skip-sync-newest', }, algod, ) diff --git a/examples/usdc/index.ts b/examples/usdc/index.ts index 6e269ee..a8af324 100644 --- a/examples/usdc/index.ts +++ b/examples/usdc/index.ts @@ -31,7 +31,7 @@ process.on('uncaughtException', (e) => console.error(e)) ], frequencyInSeconds: 1, maxRoundsToSync: 100, - syncBehaviour: 'skip-to-newest', + syncBehaviour: 'skip-sync-newest', watermarkPersistence: { get: async () => watermark, set: async (newWatermark) => { diff --git a/src/subscriptions.ts b/src/subscriptions.ts index 685b67d..2239f1c 100644 --- a/src/subscriptions.ts +++ b/src/subscriptions.ts @@ -43,13 +43,14 @@ export async function getSubscribedTransactions( const catchupTransactions: TransactionResult[] = [] let start = +new Date() - if (currentRound - algodSyncFromRoundNumber > maxRoundsToSync) { + if (currentRound - watermark > maxRoundsToSync) { switch (onMaxRounds) { case 'fail': throw new Error(`Invalid round number to subscribe from ${algodSyncFromRoundNumber}; current round number is ${currentRound}`) - case 'skip-to-newest': + case 'skip-sync-newest': algodSyncFromRoundNumber = currentRound - maxRoundsToSync + 1 startRound = algodSyncFromRoundNumber + console.log('YO', algodSyncFromRoundNumber, startRound, endRound) break case 'sync-oldest': endRound = algodSyncFromRoundNumber + maxRoundsToSync - 1 diff --git a/src/types/subscription.ts b/src/types/subscription.ts index 048a2b5..05295f5 100644 --- a/src/types/subscription.ts +++ b/src/types/subscription.ts @@ -19,13 +19,13 @@ export interface TransactionSubscriptionParams { /** The maximum number of rounds to sync for each subscription pull/poll. * * This gives you control over how many rounds you wait for at a time, - * your staleness tolerance when using `skip-to-newest` or `fail`, and + * your staleness tolerance when using `skip-sync-newest` or `fail`, and * your catchup speed when using `sync-oldest`. **/ maxRoundsToSync: number /** If the current tip of the configured Algorand blockchain is more than `maxRoundsToSync` * past `watermark` then how should that be handled: - * * `skip-to-newest`: Discard old blocks/transactions and sync the newest; useful + * * `skip-sync-newest`: Discard old blocks/transactions and sync the newest; useful * for real-time notification scenarios where you don't care about history and * are happy to lose old transactions. * * `sync-oldest`: Sync from the oldest rounds forward `maxRoundsToSync` rounds @@ -40,7 +40,7 @@ export interface TransactionSubscriptionParams { * use algod from there. * * `fail`: Throw an error. **/ - syncBehaviour: 'skip-to-newest' | 'sync-oldest' | 'sync-oldest-start-now' | 'catchup-with-indexer' | 'fail' + syncBehaviour: 'skip-sync-newest' | 'sync-oldest' | 'sync-oldest-start-now' | 'catchup-with-indexer' | 'fail' } /** Specify a filter to apply to find transactions of interest. */ @@ -105,7 +105,7 @@ export interface SubscriptionConfig { /** The set of events to subscribe to / emit */ events: SubscriptionConfigEvent[] /** The behaviour when the number of rounds to sync is greater than `maxRoundsToSync`: - * * `skip-to-newest`: Discard old rounds + * * `skip-sync-newest`: Discard old rounds * * `sync-oldest`: Sync from the oldest records up to `maxRoundsToSync` rounds. * * **Note:** will be slow to catch up if sync is significantly behind the tip of the chain @@ -114,7 +114,7 @@ export interface SubscriptionConfig { * * `catchup-with-indexer`: Will catch up to `tipOfTheChain - maxRoundsToSync` using indexer (fast) and then * continue with algod. */ - syncBehaviour: 'skip-to-newest' | 'sync-oldest' | 'sync-oldest-start-now' | 'catchup-with-indexer' + syncBehaviour: 'skip-sync-newest' | 'sync-oldest' | 'sync-oldest-start-now' | 'catchup-with-indexer' /** Methods to retrieve and persist the current watermark so syncing is resilient and maintains * its position in the chain. */ watermarkPersistence: { diff --git a/tests/scenarios/skip-sync-newest.spec.ts b/tests/scenarios/skip-sync-newest.spec.ts new file mode 100644 index 0000000..8781ec6 --- /dev/null +++ b/tests/scenarios/skip-sync-newest.spec.ts @@ -0,0 +1,62 @@ +import { algorandFixture } from '@algorandfoundation/algokit-utils/testing' +import { beforeEach, describe, test } from '@jest/globals' +import { GetSubscribedTransactionsFromSender, SendXTransactions } from '../transactions' + +describe('skip-sync-newest', () => { + const localnet = algorandFixture() + + beforeEach(localnet.beforeEach, 10e6) + + test('Only processes the latest transaction when starting from beginning of chain', async () => { + const { algod, testAccount } = localnet.context + const { txns, lastTxnRound } = await SendXTransactions(2, testAccount, algod) + + const subscribed = await GetSubscribedTransactionsFromSender( + { roundsToSync: 1, syncBehaviour: 'skip-sync-newest', watermark: 0 }, + testAccount, + algod, + ) + + expect(subscribed.currentRound).toBe(lastTxnRound) + expect(subscribed.newWatermark).toBe(lastTxnRound) + expect(subscribed.syncedRoundRange).toEqual([lastTxnRound, lastTxnRound]) + expect(subscribed.subscribedTransactions.length).toBe(1) + expect(subscribed.subscribedTransactions[0].id).toBe(txns[1].transaction.txID()) + }) + + test('Only processes the latest transaction when starting from an earlier roudn with other transactions', async () => { + const { algod, testAccount } = localnet.context + const { lastTxnRound: olderTxnRound } = await SendXTransactions(2, testAccount, algod) + const { txns, lastTxnRound: currentRound } = await SendXTransactions(1, testAccount, algod) + + const subscribed = await GetSubscribedTransactionsFromSender( + { roundsToSync: 1, syncBehaviour: 'skip-sync-newest', watermark: olderTxnRound - 1 }, + testAccount, + algod, + ) + + expect(subscribed.currentRound).toBe(currentRound) + expect(subscribed.newWatermark).toBe(currentRound) + expect(subscribed.syncedRoundRange).toEqual([currentRound, currentRound]) + expect(subscribed.subscribedTransactions.length).toBe(1) + expect(subscribed.subscribedTransactions[0].id).toBe(txns[0].transaction.txID()) + }) + + test('Process multiple transactions', async () => { + const { algod, testAccount } = localnet.context + const { txns, lastTxnRound } = await SendXTransactions(3, testAccount, algod) + + const subscribed = await GetSubscribedTransactionsFromSender( + { roundsToSync: 2, syncBehaviour: 'skip-sync-newest', watermark: 0 }, + testAccount, + algod, + ) + + expect(subscribed.currentRound).toBe(lastTxnRound) + expect(subscribed.newWatermark).toBe(lastTxnRound) + expect(subscribed.syncedRoundRange).toEqual([lastTxnRound - 1, lastTxnRound]) + expect(subscribed.subscribedTransactions.length).toBe(2) + expect(subscribed.subscribedTransactions[0].id).toBe(txns[1].transaction.txID()) + expect(subscribed.subscribedTransactions[1].id).toBe(txns[2].transaction.txID()) + }) +}) diff --git a/tests/scenarios/skip-to-newest.spec.ts b/tests/scenarios/skip-to-newest.spec.ts deleted file mode 100644 index bb8257e..0000000 --- a/tests/scenarios/skip-to-newest.spec.ts +++ /dev/null @@ -1,47 +0,0 @@ -import * as algokit from '@algorandfoundation/algokit-utils' -import { algorandFixture } from '@algorandfoundation/algokit-utils/testing' -import { SendTransactionResult } from '@algorandfoundation/algokit-utils/types/transaction' -import { beforeEach, describe, test } from '@jest/globals' -import { getSubscribedTransactions } from '../../src' - -describe('skip-to-newest', () => { - const localnet = algorandFixture() - - beforeEach(localnet.beforeEach, 10e6) - - test('Only processes the last x transactions when starting from beginning of chain', async () => { - const { algod, testAccount } = localnet.context - const txns: SendTransactionResult[] = [] - for (let i = 0; i < 2; i++) { - txns.push( - await algokit.transferAlgos( - { - amount: (1).algos(), - from: testAccount, - to: testAccount, - }, - algod, - ), - ) - } - const lastTxnRound = Number(txns[1].confirmation?.confirmedRound) - - const subscribed = await getSubscribedTransactions( - { - filter: { - sender: testAccount.addr, - }, - maxRoundsToSync: 1, - syncBehaviour: 'skip-to-newest', - watermark: 0, - }, - algod, - ) - - expect(subscribed.currentRound).toBe(lastTxnRound) - expect(subscribed.newWatermark).toBe(lastTxnRound) - expect(subscribed.syncedRoundRange).toEqual([lastTxnRound, lastTxnRound]) - expect(subscribed.subscribedTransactions.length).toBe(1) - expect(subscribed.subscribedTransactions[0].id).toBe(txns[1].transaction.txID()) - }) -}) diff --git a/tests/transactions.ts b/tests/transactions.ts new file mode 100644 index 0000000..a3fcb0d --- /dev/null +++ b/tests/transactions.ts @@ -0,0 +1,47 @@ +import * as algokit from '@algorandfoundation/algokit-utils' +import { SendTransactionFrom, SendTransactionResult } from '@algorandfoundation/algokit-utils/types/transaction' +import { Algodv2 } from 'algosdk' +import { getSubscribedTransactions } from '../src' +import { TransactionSubscriptionParams } from '../src/types/subscription' + +export const SendXTransactions = async (x: number, account: SendTransactionFrom, algod: Algodv2) => { + const txns: SendTransactionResult[] = [] + for (let i = 0; i < x; i++) { + txns.push( + await algokit.transferAlgos( + { + amount: (1).algos(), + from: account, + to: account, + }, + algod, + ), + ) + } + const lastTxnRound = Number(txns[x - 1].confirmation?.confirmedRound) + + return { + txns, + lastTxnRound, + } +} + +export const GetSubscribedTransactionsFromSender = ( + subscription: { syncBehaviour: TransactionSubscriptionParams['syncBehaviour']; roundsToSync: number; watermark?: number }, + account: SendTransactionFrom, + algod: Algodv2, +) => { + const { roundsToSync, syncBehaviour, watermark } = subscription + + return getSubscribedTransactions( + { + filter: { + sender: algokit.getSenderAddress(account), + }, + maxRoundsToSync: roundsToSync, + syncBehaviour: syncBehaviour, + watermark: watermark ?? 0, + }, + algod, + ) +}