From 6ae0eaa59a87b8dd8ef4a04878e324b167b743cf Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Fri, 31 Mar 2023 12:33:52 -0500 Subject: [PATCH 01/21] implement `EventLog` --- .gitignore | 2 + README.md | 2 +- package-lock.json | 31 +++++ package.json | 4 +- src/dwn.ts | 22 ++- src/event-log/event-log-level.ts | 80 +++++++++++ src/event-log/event-log.ts | 32 +++++ .../protocols/handlers/protocols-configure.ts | 5 +- .../records/handlers/records-delete.ts | 3 +- .../records/handlers/records-write.ts | 5 +- src/store/storage-controller.ts | 5 + tests/dwn.spec.ts | 17 ++- tests/event-log/event-log-level.spec.ts | 130 ++++++++++++++++++ .../handlers/protocols-configure.spec.ts | 8 +- .../handlers/protocols-query.spec.ts | 9 +- .../records/handlers/records-delete.spec.ts | 13 +- .../records/handlers/records-query.spec.ts | 17 ++- .../records/handlers/records-read.spec.ts | 9 +- .../records/handlers/records-write.spec.ts | 27 ++-- 19 files changed, 385 insertions(+), 36 deletions(-) create mode 100644 src/event-log/event-log-level.ts create mode 100644 src/event-log/event-log.ts create mode 100644 tests/event-log/event-log-level.spec.ts diff --git a/.gitignore b/.gitignore index c1f179396..fa4e870d9 100644 --- a/.gitignore +++ b/.gitignore @@ -13,9 +13,11 @@ try.js # default location for levelDB data storage in a non-browser env MESSAGESTORE DATASTORE +EVENTLOG # location for levelDB data storage for non-browser tests TEST-DATASTORE TEST-MESSAGESTORE +TEST-EVENTLOG # default location for index specific levelDB data storage in a non-browser env INDEX diff --git a/README.md b/README.md index 711d704d0..e6798bd8d 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # Decentralized Web Node (DWN) SDK Code Coverage -![Statements](https://img.shields.io/badge/statements-93.73%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.9%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.55%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-93.73%25-brightgreen.svg?style=flat) +![Statements](https://img.shields.io/badge/statements-93.25%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.9%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.45%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-93.25%25-brightgreen.svg?style=flat) ## Introduction diff --git a/package-lock.json b/package-lock.json index e902f3837..485c97e56 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@js-temporal/polyfill": "0.4.3", "@noble/ed25519": "1.7.1", "@noble/secp256k1": "1.7.1", + "@scure/base": "^1.1.1", "@swc/helpers": "0.3.8", "@types/ms": "0.7.31", "@types/node": "^18.13.0", @@ -34,6 +35,7 @@ "multiformats": "11.0.2", "randombytes": "2.1.0", "readable-stream": "4.3.0", + "ulid": "^2.3.0", "uuid": "8.3.2", "varint": "6.0.0" }, @@ -750,6 +752,17 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, + "node_modules/@scure/base": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.1.1.tgz", + "integrity": "sha512-ZxOhsSyxYwLJj3pLZCefNitxsj093tb2vq90mp2txoYeBqbcjDjqFhyM8eUjq/uFm6zJ+mUuqxlS2FkuSY1MTA==", + "funding": [ + { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + ] + }, "node_modules/@sindresorhus/is": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.3.0.tgz", @@ -7320,6 +7333,14 @@ "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.9.0.tgz", "integrity": "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==" }, + "node_modules/ulid": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/ulid/-/ulid-2.3.0.tgz", + "integrity": "sha512-keqHubrlpvT6G2wH0OEfSW4mquYRcbe/J8NMmveoQOjUqmo+hXtO+ORCpWhdbZ7k72UtY61BL7haGxW6enBnjw==", + "bin": { + "ulid": "bin/cli.js" + } + }, "node_modules/unbox-primitive": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", @@ -8108,6 +8129,11 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, + "@scure/base": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.1.1.tgz", + "integrity": "sha512-ZxOhsSyxYwLJj3pLZCefNitxsj093tb2vq90mp2txoYeBqbcjDjqFhyM8eUjq/uFm6zJ+mUuqxlS2FkuSY1MTA==" + }, "@sindresorhus/is": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.3.0.tgz", @@ -13087,6 +13113,11 @@ } } }, + "ulid": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/ulid/-/ulid-2.3.0.tgz", + "integrity": "sha512-keqHubrlpvT6G2wH0OEfSW4mquYRcbe/J8NMmveoQOjUqmo+hXtO+ORCpWhdbZ7k72UtY61BL7haGxW6enBnjw==" + }, "unbox-primitive": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", diff --git a/package.json b/package.json index 9fc9efa77..c59f56a9d 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "@js-temporal/polyfill": "0.4.3", "@noble/ed25519": "1.7.1", "@noble/secp256k1": "1.7.1", + "@scure/base": "1.1.1", "@swc/helpers": "0.3.8", "@types/ms": "0.7.31", "@types/node": "^18.13.0", @@ -73,6 +74,7 @@ "multiformats": "11.0.2", "randombytes": "2.1.0", "readable-stream": "4.3.0", + "ulid": "2.3.0", "uuid": "8.3.2", "varint": "6.0.0" }, @@ -134,4 +136,4 @@ "url": "https://github.com/TBD54566975/dwn-sdk-js/issues" }, "homepage": "https://github.com/TBD54566975/dwn-sdk-js#readme" -} +} \ No newline at end of file diff --git a/src/dwn.ts b/src/dwn.ts index 7e22d5ce0..7c380324c 100644 --- a/src/dwn.ts +++ b/src/dwn.ts @@ -1,5 +1,6 @@ import type { BaseMessage } from './core/types.js'; import type { DataStore } from './store/data-store.js'; +import type { EventLog } from './event-log/event-log.js'; import type { MessageStore } from './store/message-store.js'; import type { MethodHandler } from './interfaces/types.js'; import type { Readable } from 'readable-stream'; @@ -8,6 +9,7 @@ import type { TenantGate } from './core/tenant-gate.js'; import { AllowAllTenantGate } from './core/tenant-gate.js'; import { DataStoreLevel } from './store/data-store-level.js'; import { DidResolver } from './did/did-resolver.js'; +import { EventLogLevel } from './event-log/event-log-level.js'; import { MessageReply } from './core/message-reply.js'; import { MessageStoreLevel } from './store/message-store-level.js'; import { PermissionsRequestHandler } from './interfaces/permissions/handlers/permissions-request.js'; @@ -24,22 +26,26 @@ export class Dwn { private didResolver: DidResolver; private messageStore: MessageStore; private dataStore: DataStore; + private eventLog: EventLog; private tenantGate: TenantGate; private constructor(config: DwnConfig) { this.didResolver = config.didResolver; this.messageStore = config.messageStore; this.dataStore = config.dataStore; + this.eventLog = config.eventLog; this.tenantGate = config.tenantGate; this.methodHandlers = { [DwnInterfaceName.Permissions + DwnMethodName.Request] : new PermissionsRequestHandler(this.didResolver, this.messageStore, this.dataStore), - [DwnInterfaceName.Protocols + DwnMethodName.Configure] : new ProtocolsConfigureHandler(this.didResolver, this.messageStore, this.dataStore), - [DwnInterfaceName.Protocols + DwnMethodName.Query] : new ProtocolsQueryHandler(this.didResolver, this.messageStore, this.dataStore), - [DwnInterfaceName.Records + DwnMethodName.Delete] : new RecordsDeleteHandler(this.didResolver, this.messageStore, this.dataStore), - [DwnInterfaceName.Records + DwnMethodName.Query] : new RecordsQueryHandler(this.didResolver, this.messageStore, this.dataStore), - [DwnInterfaceName.Records + DwnMethodName.Read] : new RecordsReadHandler(this.didResolver, this.messageStore, this.dataStore), - [DwnInterfaceName.Records + DwnMethodName.Write] : new RecordsWriteHandler(this.didResolver, this.messageStore, this.dataStore), + [DwnInterfaceName.Protocols + DwnMethodName.Configure] : new ProtocolsConfigureHandler( + this.didResolver, this.messageStore, this.dataStore, this.eventLog), + [DwnInterfaceName.Protocols + DwnMethodName.Query] : new ProtocolsQueryHandler(this.didResolver, this.messageStore, this.dataStore), + [DwnInterfaceName.Records + DwnMethodName.Delete] : new RecordsDeleteHandler( + this.didResolver, this.messageStore, this.dataStore, this.eventLog), + [DwnInterfaceName.Records + DwnMethodName.Query] : new RecordsQueryHandler(this.didResolver, this.messageStore, this.dataStore), + [DwnInterfaceName.Records + DwnMethodName.Read] : new RecordsReadHandler(this.didResolver, this.messageStore, this.dataStore), + [DwnInterfaceName.Records + DwnMethodName.Write] : new RecordsWriteHandler(this.didResolver, this.messageStore, this.dataStore, this.eventLog), }; } @@ -52,6 +58,7 @@ export class Dwn { config.tenantGate ??= new AllowAllTenantGate(); config.messageStore ??= new MessageStoreLevel(); config.dataStore ??= new DataStoreLevel(); + config.eventLog ??= new EventLogLevel(); const dwn = new Dwn(config); await dwn.open(); @@ -62,11 +69,13 @@ export class Dwn { private async open(): Promise { await this.messageStore.open(); await this.dataStore.open(); + await this.eventLog.open(); } public async close(): Promise { this.messageStore.close(); this.dataStore.close(); + this.eventLog.close(); } /** @@ -127,4 +136,5 @@ export type DwnConfig = { messageStore?: MessageStore; dataStore?: DataStore; tenantGate?: TenantGate; + eventLog?: EventLog }; diff --git a/src/event-log/event-log-level.ts b/src/event-log/event-log-level.ts new file mode 100644 index 000000000..bc92b2969 --- /dev/null +++ b/src/event-log/event-log-level.ts @@ -0,0 +1,80 @@ +import type { ULID } from 'ulid'; +import type { Event, EventLog } from './event-log.js'; + +import { base32crockford } from '@scure/base'; +import { Encoder } from '../utils/encoder.js'; +import { monotonicFactory } from 'ulid'; +import { createLevelDatabase, LevelWrapper } from '../store/level-wrapper.js'; + + +type EventLogLevelConfig = { + /** + * must be a directory path (relative or absolute) where + * LevelDB will store its files, or in browsers, the name of the + * {@link https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase IDBDatabase} to be opened. + */ + location: string, + createLevelDatabase?: typeof createLevelDatabase, +}; + +export class EventLogLevel implements EventLog { + config: EventLogLevelConfig; + level: LevelWrapper; + ulid: ULID; + + constructor(config: EventLogLevelConfig = { location: 'EVENTLOG' }) { + this.config = { + createLevelDatabase, + ...config, + }; + + this.level = new LevelWrapper({ ...this.config, valueEncoding: 'utf8' }); + this.ulid = monotonicFactory(); + } + + async open(): Promise { + // `db.open()` is automatically called by the database constructor. We're calling it explicitly + // in order to explicitly catch an error that would otherwise not surface until another method + // like `db.get()` is called. Once `db.open()` has then been called, any read & write + // operations will again be queued internally until opening has finished. + return this.level.open(); + } + + close(): Promise { + return this.level.close(); + } + + clear(): Promise { + return this.level.clear(); + } + + async append(tenant: string, messageCid: string): Promise { + const hashedTenant = this.hashTenant(tenant); + const tenantEventLog = await this.level.partition(hashedTenant); + + const watermark = this.ulid(); + await tenantEventLog.put(watermark, messageCid); + + return watermark; + } + async getEventsAfter(tenant: string, watermark?: string | undefined): Promise { + const hashedTenant = this.hashTenant(tenant); + const tenantEventLog = await this.level.partition(hashedTenant); + + const opts = watermark ? { gt: watermark } : {}; + const events: Array = []; + + for await (const [key, value] of tenantEventLog.iterator(opts)) { + const event = { watermark: key, messageCid: value }; + events.push(event); + } + + return events; + } + + hashTenant(tenant: string): string { + const tenantBytes = Encoder.stringToBytes(tenant); + + return base32crockford.encode(tenantBytes); + } +} \ No newline at end of file diff --git a/src/event-log/event-log.ts b/src/event-log/event-log.ts new file mode 100644 index 000000000..3c563d4e8 --- /dev/null +++ b/src/event-log/event-log.ts @@ -0,0 +1,32 @@ +export type Event = { + watermark: string, + messageCid: string +}; + +export interface EventLog { + /** + * opens a connection to the underlying store + */ + open(): Promise; + + /** + * closes the connection to the underlying store + */ + close(): Promise; + + /** + * adds an event to a tenant's event log + * @param tenant - the tenant's DID + * @param messageCid - the CID of the message + */ + append(tenant: string, messageCid: string): Promise + + /** + * retrieves all of a tenant's events that occurred after the watermark provided. + * If no watermark is provided, all events for a given tenant will be returned. + * + * @param tenant + * @param watermark + */ + getEventsAfter(tenant: string, watermark?: string): Promise> +} \ No newline at end of file diff --git a/src/interfaces/protocols/handlers/protocols-configure.ts b/src/interfaces/protocols/handlers/protocols-configure.ts index e0ae68c12..8b32b9fdc 100644 --- a/src/interfaces/protocols/handlers/protocols-configure.ts +++ b/src/interfaces/protocols/handlers/protocols-configure.ts @@ -1,3 +1,4 @@ +import type { EventLog } from '../../../event-log/event-log.js'; import type { MethodHandler } from '../../types.js'; import type { ProtocolsConfigureMessage } from '../types.js'; import type { DataStore, DidResolver, MessageStore } from '../../../index.js'; @@ -11,7 +12,7 @@ import { DwnInterfaceName, DwnMethodName, Message } from '../../../core/message. export class ProtocolsConfigureHandler implements MethodHandler { - constructor(private didResolver: DidResolver, private messageStore: MessageStore,private dataStore: DataStore) { } + constructor(private didResolver: DidResolver, private messageStore: MessageStore, private dataStore: DataStore, private eventLog: EventLog) { } public async handle({ tenant, @@ -62,7 +63,7 @@ export class ProtocolsConfigureHandler implements MethodHandler { author, ... message.descriptor }; - await StorageController.put(this.messageStore, this.dataStore, tenant, incomingMessage, indexes, dataStream); + await StorageController.put(this.messageStore, this.dataStore, this.eventLog, tenant, incomingMessage, indexes, dataStream); messageReply = new MessageReply({ status: { code: 202, detail: 'Accepted' } diff --git a/src/interfaces/records/handlers/records-delete.ts b/src/interfaces/records/handlers/records-delete.ts index 4fdf4f0c7..6a1b68060 100644 --- a/src/interfaces/records/handlers/records-delete.ts +++ b/src/interfaces/records/handlers/records-delete.ts @@ -1,3 +1,4 @@ +import type { EventLog } from '../../../event-log/event-log.js'; import type { MethodHandler } from '../../types.js'; import type { RecordsDeleteMessage } from '../types.js'; import type { TimestampedMessage } from '../../../core/types.js'; @@ -12,7 +13,7 @@ import { RecordsWrite } from '../messages/records-write.js'; export class RecordsDeleteHandler implements MethodHandler { - constructor(private didResolver: DidResolver, private messageStore: MessageStore, private dataStore: DataStore) { } + constructor(private didResolver: DidResolver, private messageStore: MessageStore, private dataStore: DataStore, private eventLog: EventLog) { } public async handle({ tenant, diff --git a/src/interfaces/records/handlers/records-write.ts b/src/interfaces/records/handlers/records-write.ts index af7f75255..78e7dbaf5 100644 --- a/src/interfaces/records/handlers/records-write.ts +++ b/src/interfaces/records/handlers/records-write.ts @@ -1,3 +1,4 @@ +import type { EventLog } from '../../../event-log/event-log.js'; import type { MethodHandler } from '../../types.js'; import type { RecordsWriteMessage } from '../types.js'; import type { TimestampedMessage } from '../../../core/types.js'; @@ -13,7 +14,7 @@ import { StorageController } from '../../../store/storage-controller.js'; export class RecordsWriteHandler implements MethodHandler { - constructor(private didResolver: DidResolver, private messageStore: MessageStore,private dataStore: DataStore) { } + constructor(private didResolver: DidResolver, private messageStore: MessageStore,private dataStore: DataStore, private eventLog: EventLog) { } public async handle({ tenant, @@ -81,7 +82,7 @@ export class RecordsWriteHandler implements MethodHandler { const indexes = await constructRecordsWriteIndexes(recordsWrite, isLatestBaseState); try { - await StorageController.put(this.messageStore, this.dataStore, tenant, incomingMessage, indexes, dataStream); + await StorageController.put(this.messageStore, this.dataStore, this.eventLog, tenant, incomingMessage, indexes, dataStream); } catch (error) { if (error.code === DwnErrorCode.MessageStoreDataCidMismatch || error.code === DwnErrorCode.MessageStoreDataNotFound || diff --git a/src/store/storage-controller.ts b/src/store/storage-controller.ts index 5b57e22bd..f4c94d006 100644 --- a/src/store/storage-controller.ts +++ b/src/store/storage-controller.ts @@ -1,4 +1,5 @@ import type { DataStore } from './data-store.js'; +import type { EventLog } from '../event-log/event-log.js'; import type { MessageStore } from './message-store.js'; import type { Readable } from 'readable-stream'; import type { BaseMessage, Filter } from '../core/types.js'; @@ -24,6 +25,7 @@ export class StorageController { public static async put( messageStore: MessageStore, dataStore: DataStore, + eventLog: EventLog, tenant: string, message: BaseMessage, indexes: { [key: string]: string }, @@ -78,6 +80,9 @@ export class StorageController { } await messageStore.put(tenant, message, indexes); + + const messageCid = await Message.getCid(message); + await eventLog.append(tenant, messageCid.toString()); } public static async query( diff --git a/tests/dwn.spec.ts b/tests/dwn.spec.ts index da7b569e6..73f37fca9 100644 --- a/tests/dwn.spec.ts +++ b/tests/dwn.spec.ts @@ -7,6 +7,7 @@ import chai, { expect } from 'chai'; import { DataStoreLevel } from '../src/store/data-store-level.js'; import { DidKeyResolver } from '../src/did/did-key-resolver.js'; import { Dwn } from '../src/dwn.js'; +import { EventLogLevel } from '../src/event-log/event-log-level.js'; import { Message } from '../src/core/message.js'; import { MessageStoreLevel } from '../src/store/message-store-level.js'; import { TestDataGenerator } from './utils/test-data-generator.js'; @@ -16,6 +17,7 @@ chai.use(chaiAsPromised); describe('DWN', () => { let messageStore: MessageStoreLevel; let dataStore: DataStoreLevel; + let eventLog: EventLogLevel; let dwn: Dwn; before(async () => { @@ -30,7 +32,11 @@ describe('DWN', () => { blockstoreLocation: 'TEST-DATASTORE' }); - dwn = await Dwn.create({ messageStore, dataStore }); + eventLog = new EventLogLevel({ + location: 'TEST-EVENTLOG' + }); + + dwn = await Dwn.create({ messageStore, dataStore, eventLog }); }); beforeEach(async () => { @@ -125,8 +131,15 @@ describe('DWN', () => { const messageStoreStub = sinon.createStubInstance(MessageStoreLevel); const dataStoreStub = sinon.createStubInstance(DataStoreLevel); + const eventLogStub = sinon.createStubInstance(EventLogLevel); + + const dwnWithConfig = await Dwn.create({ + tenantGate : blockAllTenantGate, + messageStore : messageStoreStub, + dataStore : dataStoreStub, + eventLog : eventLogStub + }); - const dwnWithConfig = await Dwn.create({ tenantGate: blockAllTenantGate, messageStore: messageStoreStub, dataStore: dataStoreStub }); const alice = await DidKeyResolver.generate(); const { requester, message } = await TestDataGenerator.generateRecordsQuery({ requester: alice }); diff --git a/tests/event-log/event-log-level.spec.ts b/tests/event-log/event-log-level.spec.ts new file mode 100644 index 000000000..5e9e53b38 --- /dev/null +++ b/tests/event-log/event-log-level.spec.ts @@ -0,0 +1,130 @@ +import chaiAsPromised from 'chai-as-promised'; +import { computeCid } from '../../src/utils/cid.js'; +import { EventLogLevel } from '../../src/event-log/event-log-level.js'; +import { TestDataGenerator as TDG } from '../utils/test-data-generator.js'; +import chai, { expect } from 'chai'; + +chai.use(chaiAsPromised); + +let eventLog: EventLogLevel; + +describe('EventLogLevel Tests', () => { + describe('append tests', () => { + before(async () => { + eventLog = new EventLogLevel({ location: 'TEST-EVENTLOG' }); + await eventLog.open(); + }); + + beforeEach(async () => { + await eventLog.clear(); + }); + + after(async () => { + await eventLog.close(); + }); + + it('appends a tenant namespaced entry into leveldb', async () => { + const { requester, message } = await TDG.generateRecordsWrite(); + const messageCid = await computeCid(message); + + const watermark = await eventLog.append(requester.did, messageCid.toString()); + + for await (const [key, value] of eventLog.level.iterator({})) { + expect(key).to.include(watermark); + expect(value).to.equal(messageCid.toString()); + } + }); + + it('maintains order in which events were appended', async () => { + const { requester, message } = await TDG.generateRecordsWrite(); + const messageCid = await computeCid(message); + + await eventLog.append(requester.did, messageCid.toString()); + + const { message: message2 } = await TDG.generateRecordsWrite({ requester }); + const messageCid2 = await computeCid(message2); + + await eventLog.append(requester.did, messageCid2.toString()); + + const storedValues = []; + for await (const [_, cid] of eventLog.level.iterator({})) { + storedValues.push(cid); + } + + expect(storedValues[0]).to.equal(messageCid.toString()); + expect(storedValues[1]).to.equal(messageCid2.toString()); + }); + }); + + describe('getEventsAfter', () => { + before(async () => { + eventLog = new EventLogLevel({ location: 'TEST-EVENTLOG' }); + await eventLog.open(); + }); + + beforeEach(async () => { + await eventLog.clear(); + }); + + after(async () => { + await eventLog.close(); + }); + + it('gets all events for a tenant if watermark is not provided', async () => { + const messageCids = []; + + const { requester, message } = await TDG.generateRecordsWrite(); + const messageCid = await computeCid(message); + messageCids.push(messageCid); + + await eventLog.append(requester.did, messageCid.toString()); + + for (let i = 0; i < 9; i += 1) { + const { message } = await TDG.generateRecordsWrite({ requester }); + const messageCid = await computeCid(message); + messageCids.push(messageCid); + + await eventLog.append(requester.did, messageCid.toString()); + } + + const events = await eventLog.getEventsAfter(requester.did); + expect(events.length).to.equal(10); + + for (let i = 0; i < events.length; i += 1) { + expect(events[i].messageCid).to.equal(messageCids[i]); + } + }); + + it('gets all events that occured after the watermark provided', async () => { + const { requester, message } = await TDG.generateRecordsWrite(); + const messageCid = await computeCid(message); + + await eventLog.append(requester.did, messageCid.toString()); + + const messageCids = []; + let testWatermark; + + for (let i = 0; i < 9; i += 1) { + const { message } = await TDG.generateRecordsWrite({ requester }); + const messageCid = await computeCid(message); + + const watermark = await eventLog.append(requester.did, messageCid.toString()); + + if (i === 4) { + testWatermark = watermark; + } + + if (i > 4) { + messageCids.push(messageCid.toString()); + } + } + + const events = await eventLog.getEventsAfter(requester.did, testWatermark); + expect(events.length).to.equal(4); + + for (let i = 0; i < events.length; i += 1) { + expect(events[i].messageCid).to.equal(messageCids[i], `${i}`); + } + }); + }); +}); \ No newline at end of file diff --git a/tests/interfaces/protocols/handlers/protocols-configure.spec.ts b/tests/interfaces/protocols/handlers/protocols-configure.spec.ts index 5a4f8bf37..1904a4d32 100644 --- a/tests/interfaces/protocols/handlers/protocols-configure.spec.ts +++ b/tests/interfaces/protocols/handlers/protocols-configure.spec.ts @@ -6,6 +6,7 @@ import chai, { expect } from 'chai'; import { DataStoreLevel } from '../../../../src/store/data-store-level.js'; import { DidKeyResolver } from '../../../../src/did/did-key-resolver.js'; +import { EventLogLevel } from '../../../../src/event-log/event-log-level.js'; import { GeneralJwsSigner } from '../../../../src/jose/jws/general/signer.js'; import { lexicographicalCompare } from '../../../../src/utils/string.js'; import { Message } from '../../../../src/core/message.js'; @@ -21,6 +22,7 @@ describe('ProtocolsConfigureHandler.handle()', () => { let didResolver: DidResolver; let messageStore: MessageStoreLevel; let dataStore: DataStoreLevel; + let eventLog: EventLogLevel; let dwn: Dwn; describe('functional tests', () => { @@ -38,7 +40,11 @@ describe('ProtocolsConfigureHandler.handle()', () => { blockstoreLocation: 'TEST-DATASTORE' }); - dwn = await Dwn.create({ didResolver, messageStore, dataStore }); + eventLog = new EventLogLevel({ + location: 'TEST-EVENTLOG' + }); + + dwn = await Dwn.create({ didResolver, messageStore, dataStore, eventLog }); }); beforeEach(async () => { diff --git a/tests/interfaces/protocols/handlers/protocols-query.spec.ts b/tests/interfaces/protocols/handlers/protocols-query.spec.ts index db1e5e0eb..30dc2f592 100644 --- a/tests/interfaces/protocols/handlers/protocols-query.spec.ts +++ b/tests/interfaces/protocols/handlers/protocols-query.spec.ts @@ -4,6 +4,7 @@ import chai, { expect } from 'chai'; import { DataStoreLevel } from '../../../../src/store/data-store-level.js'; import { DidKeyResolver } from '../../../../src/did/did-key-resolver.js'; +import { EventLogLevel } from '../../../../src/event-log/event-log-level.js'; import { GeneralJwsSigner } from '../../../../src/jose/jws/general/signer.js'; import { MessageStoreLevel } from '../../../../src/store/message-store-level.js'; import { TestDataGenerator } from '../../../utils/test-data-generator.js'; @@ -17,6 +18,7 @@ describe('ProtocolsQueryHandler.handle()', () => { let didResolver: DidResolver; let messageStore: MessageStoreLevel; let dataStore: DataStoreLevel; + let eventLog: EventLogLevel; let dwn: Dwn; describe('functional tests', () => { @@ -34,7 +36,11 @@ describe('ProtocolsQueryHandler.handle()', () => { blockstoreLocation: 'TEST-DATASTORE' }); - dwn = await Dwn.create({ didResolver, messageStore, dataStore }); + eventLog = new EventLogLevel({ + location: 'TEST-EVENTLOG' + }); + + dwn = await Dwn.create({ didResolver, messageStore, dataStore, eventLog }); }); beforeEach(async () => { @@ -43,6 +49,7 @@ describe('ProtocolsQueryHandler.handle()', () => { // clean up before each test rather than after so that a test does not depend on other tests to do the clean up await messageStore.clear(); await dataStore.clear(); + await eventLog.clear(); }); after(async () => { diff --git a/tests/interfaces/records/handlers/records-delete.spec.ts b/tests/interfaces/records/handlers/records-delete.spec.ts index 7d194dbdc..238b2a1a0 100644 --- a/tests/interfaces/records/handlers/records-delete.spec.ts +++ b/tests/interfaces/records/handlers/records-delete.spec.ts @@ -7,6 +7,7 @@ import { Cid } from '../../../../src/utils/cid.js'; import { DataStoreLevel } from '../../../../src/store/data-store-level.js'; import { DidKeyResolver } from '../../../../src/did/did-key-resolver.js'; import { DwnErrorCode } from '../../../../src/core/dwn-error.js'; +import { EventLogLevel } from '../../../../src/event-log/event-log-level.js'; import { MessageStoreLevel } from '../../../../src/store/message-store-level.js'; import { RecordsDeleteHandler } from '../../../../src/interfaces/records/handlers/records-delete.js'; import { TestDataGenerator } from '../../../utils/test-data-generator.js'; @@ -19,6 +20,7 @@ describe('RecordsDeleteHandler.handle()', () => { let didResolver: DidResolver; let messageStore: MessageStoreLevel; let dataStore: DataStoreLevel; + let eventLog: EventLogLevel; let dwn: Dwn; describe('functional tests', () => { @@ -36,7 +38,11 @@ describe('RecordsDeleteHandler.handle()', () => { blockstoreLocation: 'TEST-DATASTORE' }); - dwn = await Dwn.create({ didResolver, messageStore, dataStore }); + eventLog = new EventLogLevel({ + location: 'TEST-EVENTLOG' + }); + + dwn = await Dwn.create({ didResolver, messageStore, dataStore, eventLog }); }); beforeEach(async () => { @@ -45,6 +51,7 @@ describe('RecordsDeleteHandler.handle()', () => { // clean up before each test rather than after so that a test does not depend on other tests to do the clean up await messageStore.clear(); await dataStore.clear(); + await eventLog.clear(); }); after(async () => { @@ -512,7 +519,7 @@ describe('RecordsDeleteHandler.handle()', () => { const messageStore = sinon.createStubInstance(MessageStoreLevel); const dataStore = sinon.createStubInstance(DataStoreLevel); - const recordsDeleteHandler = new RecordsDeleteHandler(didResolver, messageStore, dataStore); + const recordsDeleteHandler = new RecordsDeleteHandler(didResolver, messageStore, dataStore, eventLog); const reply = await recordsDeleteHandler.handle({ tenant, message }); expect(reply.status.code).to.equal(401); }); @@ -525,7 +532,7 @@ describe('RecordsDeleteHandler.handle()', () => { const messageStore = sinon.createStubInstance(MessageStoreLevel); const dataStore = sinon.createStubInstance(DataStoreLevel); - const recordsDeleteHandler = new RecordsDeleteHandler(didResolver, messageStore, dataStore); + const recordsDeleteHandler = new RecordsDeleteHandler(didResolver, messageStore, dataStore, eventLog); // stub the `parse()` function to throw an error sinon.stub(RecordsDelete, 'parse').throws('anyError'); diff --git a/tests/interfaces/records/handlers/records-query.spec.ts b/tests/interfaces/records/handlers/records-query.spec.ts index 319873882..5b133ca49 100644 --- a/tests/interfaces/records/handlers/records-query.spec.ts +++ b/tests/interfaces/records/handlers/records-query.spec.ts @@ -8,6 +8,7 @@ import { DataStoreLevel } from '../../../../src/store/data-store-level.js'; import { DidKeyResolver } from '../../../../src/did/did-key-resolver.js'; import { DwnConstant } from '../../../../src/core/dwn-constant.js'; import { Encoder } from '../../../../src/utils/encoder.js'; +import { EventLogLevel } from '../../../../src/event-log/event-log-level.js'; import { Jws } from '../../../../src/utils/jws.js'; import { MessageStoreLevel } from '../../../../src/store/message-store-level.js'; import { RecordsQueryHandler } from '../../../../src/interfaces/records/handlers/records-query.js'; @@ -27,6 +28,7 @@ describe('RecordsQueryHandler.handle()', () => { let didResolver: DidResolver; let messageStore: MessageStoreLevel; let dataStore: DataStoreLevel; + let eventLog: EventLogLevel; let dwn: Dwn; before(async () => { @@ -43,7 +45,11 @@ describe('RecordsQueryHandler.handle()', () => { blockstoreLocation: 'TEST-DATASTORE' }); - dwn = await Dwn.create({ didResolver, messageStore, dataStore }); + eventLog = new EventLogLevel({ + location: 'TEST-EVENTLOG' + }); + + dwn = await Dwn.create({ didResolver, messageStore, dataStore, eventLog }); }); beforeEach(async () => { @@ -52,6 +58,7 @@ describe('RecordsQueryHandler.handle()', () => { // clean up before each test rather than after so that a test does not depend on other tests to do the clean up await messageStore.clear(); await dataStore.clear(); + await eventLog.clear(); }); after(async () => { @@ -460,10 +467,10 @@ describe('RecordsQueryHandler.handle()', () => { const additionalIndexes2 = await constructRecordsWriteIndexes(record2Data.recordsWrite, true); const additionalIndexes3 = await constructRecordsWriteIndexes(record3Data.recordsWrite, true); const additionalIndexes4 = await constructRecordsWriteIndexes(record4Data.recordsWrite, true); - await StorageController.put(messageStore, dataStore, alice.did, record1Data.message, additionalIndexes1, record1Data.dataStream); - await StorageController.put(messageStore, dataStore, alice.did, record2Data.message, additionalIndexes2, record2Data.dataStream); - await StorageController.put(messageStore, dataStore, alice.did, record3Data.message, additionalIndexes3, record3Data.dataStream); - await StorageController.put(messageStore, dataStore, alice.did, record4Data.message, additionalIndexes4, record4Data.dataStream); + await StorageController.put(messageStore, dataStore, eventLog, alice.did, record1Data.message, additionalIndexes1, record1Data.dataStream); + await StorageController.put(messageStore, dataStore, eventLog, alice.did, record2Data.message, additionalIndexes2, record2Data.dataStream); + await StorageController.put(messageStore, dataStore, eventLog, alice.did, record3Data.message, additionalIndexes3, record3Data.dataStream); + await StorageController.put(messageStore, dataStore, eventLog, alice.did, record4Data.message, additionalIndexes4, record4Data.dataStream); // test correctness for Bob's query const bobQueryMessageData = await TestDataGenerator.generateRecordsQuery({ diff --git a/tests/interfaces/records/handlers/records-read.spec.ts b/tests/interfaces/records/handlers/records-read.spec.ts index ad08fb219..223d0edd2 100644 --- a/tests/interfaces/records/handlers/records-read.spec.ts +++ b/tests/interfaces/records/handlers/records-read.spec.ts @@ -5,6 +5,7 @@ import chai, { expect } from 'chai'; import { Comparer } from '../../../utils/comparer.js'; import { DataStoreLevel } from '../../../../src/store/data-store-level.js'; import { DidKeyResolver } from '../../../../src/did/did-key-resolver.js'; +import { EventLogLevel } from '../../../../src/event-log/event-log-level.js'; import { MessageStoreLevel } from '../../../../src/store/message-store-level.js'; import { RecordsReadHandler } from '../../../../src/interfaces/records/handlers/records-read.js'; import { TestDataGenerator } from '../../../utils/test-data-generator.js'; @@ -18,6 +19,7 @@ describe('RecordsReadHandler.handle()', () => { let didResolver: DidResolver; let messageStore: MessageStoreLevel; let dataStore: DataStoreLevel; + let eventLog: EventLogLevel; let dwn: Dwn; describe('functional tests', () => { @@ -35,7 +37,11 @@ describe('RecordsReadHandler.handle()', () => { blockstoreLocation: 'TEST-DATASTORE' }); - dwn = await Dwn.create({ didResolver, messageStore, dataStore }); + eventLog = new EventLogLevel({ + location: 'TEST-EVENTLOG' + }); + + dwn = await Dwn.create({ didResolver, messageStore, dataStore, eventLog }); }); beforeEach(async () => { @@ -44,6 +50,7 @@ describe('RecordsReadHandler.handle()', () => { // clean up before each test rather than after so that a test does not depend on other tests to do the clean up await messageStore.clear(); await dataStore.clear(); + await eventLog.clear(); }); after(async () => { diff --git a/tests/interfaces/records/handlers/records-write.spec.ts b/tests/interfaces/records/handlers/records-write.spec.ts index 64b2ae6c6..7ed709557 100644 --- a/tests/interfaces/records/handlers/records-write.spec.ts +++ b/tests/interfaces/records/handlers/records-write.spec.ts @@ -17,6 +17,7 @@ import { DidKeyResolver } from '../../../../src/did/did-key-resolver.js'; import { DidResolver } from '../../../../src/did/did-resolver.js'; import { DwnErrorCode } from '../../../../src/core/dwn-error.js'; import { Encoder } from '../../../../src/utils/encoder.js'; +import { EventLogLevel } from '../../../../src/event-log/event-log-level.js'; import { GeneralJwsSigner } from '../../../../src/jose/jws/general/signer.js'; import { getCurrentTimeInHighPrecision } from '../../../../src/utils/time.js'; import { Message } from '../../../../src/core/message.js'; @@ -35,6 +36,7 @@ describe('RecordsWriteHandler.handle()', () => { let didResolver: DidResolver; let messageStore: MessageStoreLevel; let dataStore: DataStoreLevel; + let eventLog: EventLogLevel; let dwn: Dwn; describe('functional tests', () => { @@ -52,7 +54,11 @@ describe('RecordsWriteHandler.handle()', () => { blockstoreLocation: 'TEST-DATASTORE' }); - dwn = await Dwn.create({ didResolver, messageStore, dataStore }); + eventLog = new EventLogLevel({ + location: 'TEST-EVENTLOG' + }); + + dwn = await Dwn.create({ didResolver, messageStore, dataStore, eventLog }); }); beforeEach(async () => { @@ -61,6 +67,7 @@ describe('RecordsWriteHandler.handle()', () => { // clean up before each test rather than after so that a test does not depend on other tests to do the clean up await messageStore.clear(); await dataStore.clear(); + await eventLog.clear(); }); after(async () => { @@ -1293,7 +1300,7 @@ describe('RecordsWriteHandler.handle()', () => { const messageStore = sinon.createStubInstance(MessageStoreLevel); const dataStore = sinon.createStubInstance(DataStoreLevel); - const recordsWriteHandler = new RecordsWriteHandler(didResolver, messageStore, dataStore); + const recordsWriteHandler = new RecordsWriteHandler(didResolver, messageStore, dataStore, eventLog); const reply = await recordsWriteHandler.handle({ tenant, message, dataStream }); expect(reply.status.code).to.equal(400); @@ -1317,7 +1324,7 @@ describe('RecordsWriteHandler.handle()', () => { const messageStore = sinon.createStubInstance(MessageStoreLevel); const dataStore = sinon.createStubInstance(DataStoreLevel); - const recordsWriteHandler = new RecordsWriteHandler(didResolver, messageStore, dataStore); + const recordsWriteHandler = new RecordsWriteHandler(didResolver, messageStore, dataStore, eventLog); const reply = await recordsWriteHandler.handle({ tenant, message, dataStream }); expect(reply.status.code).to.equal(400); @@ -1335,7 +1342,7 @@ describe('RecordsWriteHandler.handle()', () => { const messageStore = sinon.createStubInstance(MessageStoreLevel); const dataStore = sinon.createStubInstance(DataStoreLevel); - const recordsWriteHandler = new RecordsWriteHandler(didResolver, messageStore, dataStore); + const recordsWriteHandler = new RecordsWriteHandler(didResolver, messageStore, dataStore, eventLog); const reply = await recordsWriteHandler.handle({ tenant, message, dataStream }); expect(reply.status.code).to.equal(401); @@ -1350,7 +1357,7 @@ describe('RecordsWriteHandler.handle()', () => { const messageStore = sinon.createStubInstance(MessageStoreLevel); const dataStore = sinon.createStubInstance(DataStoreLevel); - const recordsWriteHandler = new RecordsWriteHandler(didResolver, messageStore, dataStore); + const recordsWriteHandler = new RecordsWriteHandler(didResolver, messageStore, dataStore, eventLog); const tenant = await (await TestDataGenerator.generatePersona()).did; // unauthorized tenant const reply = await recordsWriteHandler.handle({ tenant, message, dataStream }); @@ -1384,7 +1391,7 @@ describe('RecordsWriteHandler.handle()', () => { const messageStore = sinon.createStubInstance(MessageStoreLevel); const dataStore = sinon.createStubInstance(DataStoreLevel); - const recordsWriteHandler = new RecordsWriteHandler(didResolver, messageStore, dataStore); + const recordsWriteHandler = new RecordsWriteHandler(didResolver, messageStore, dataStore, eventLog); const reply = await recordsWriteHandler.handle({ tenant, message, dataStream }); expect(reply.status.code).to.equal(400); @@ -1396,7 +1403,7 @@ describe('RecordsWriteHandler.handle()', () => { const bob = await DidKeyResolver.generate(); const { message, dataStream } = await TestDataGenerator.generateRecordsWrite({ requester: alice, attesters: [alice, bob] }); - const recordsWriteHandler = new RecordsWriteHandler(didResolver, messageStore, dataStore); + const recordsWriteHandler = new RecordsWriteHandler(didResolver, messageStore, dataStore, eventLog); const writeReply = await recordsWriteHandler.handle({ tenant: alice.did, message, dataStream }); expect(writeReply.status.code).to.equal(400); @@ -1411,7 +1418,7 @@ describe('RecordsWriteHandler.handle()', () => { const anotherWrite = await TestDataGenerator.generateRecordsWrite({ attesters: [alice] }); message.attestation = anotherWrite.message.attestation; - const recordsWriteHandler = new RecordsWriteHandler(didResolver, messageStore, dataStore); + const recordsWriteHandler = new RecordsWriteHandler(didResolver, messageStore, dataStore, eventLog); const writeReply = await recordsWriteHandler.handle({ tenant: alice.did, message, dataStream }); expect(writeReply.status.code).to.equal(400); @@ -1428,7 +1435,7 @@ describe('RecordsWriteHandler.handle()', () => { const attestationNotReferencedByAuthorization = await RecordsWrite['createAttestation'](descriptorCid, Jws.createSignatureInputs([bob])); message.attestation = attestationNotReferencedByAuthorization; - const recordsWriteHandler = new RecordsWriteHandler(didResolver, messageStore, dataStore); + const recordsWriteHandler = new RecordsWriteHandler(didResolver, messageStore, dataStore, eventLog); const writeReply = await recordsWriteHandler.handle({ tenant: alice.did, message, dataStream }); expect(writeReply.status.code).to.equal(400); @@ -1450,7 +1457,7 @@ describe('RecordsWriteHandler.handle()', () => { const dataStoreStub = sinon.createStubInstance(DataStoreLevel); - const recordsWriteHandler = new RecordsWriteHandler(didResolverStub, messageStoreStub, dataStoreStub); + const recordsWriteHandler = new RecordsWriteHandler(didResolverStub, messageStoreStub, dataStoreStub, eventLog); const handlerPromise = recordsWriteHandler.handle({ tenant, message, dataStream }); await expect(handlerPromise).to.be.rejectedWith('an unknown error in messageStore.put()'); From 085e0c9ae27866b910a8300e6335e13779386f2c Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Fri, 31 Mar 2023 16:01:56 -0500 Subject: [PATCH 02/21] make iterator options optional. this matches levels api for --- README.md | 2 +- src/store/level-wrapper.ts | 2 +- src/store/storage-controller.ts | 2 +- tests/event-log/event-log-level.spec.ts | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e6798bd8d..92663a6a8 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # Decentralized Web Node (DWN) SDK Code Coverage -![Statements](https://img.shields.io/badge/statements-93.25%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.9%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.45%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-93.25%25-brightgreen.svg?style=flat) +![Statements](https://img.shields.io/badge/statements-93.25%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.91%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.45%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-93.25%25-brightgreen.svg?style=flat) ## Introduction diff --git a/src/store/level-wrapper.ts b/src/store/level-wrapper.ts index 06eb96360..bc72ac1d8 100644 --- a/src/store/level-wrapper.ts +++ b/src/store/level-wrapper.ts @@ -121,7 +121,7 @@ export class LevelWrapper { } } - async * iterator(iteratorOptions: LevelWrapperIteratorOptions, options?: LevelWrapperOptions): AsyncGenerator<[string, V]> { + async * iterator(iteratorOptions?: LevelWrapperIteratorOptions, options?: LevelWrapperOptions): AsyncGenerator<[string, V]> { options?.signal?.throwIfAborted(); await abortOr(options?.signal, this.createLevelDatabase()); diff --git a/src/store/storage-controller.ts b/src/store/storage-controller.ts index f4c94d006..e441544fb 100644 --- a/src/store/storage-controller.ts +++ b/src/store/storage-controller.ts @@ -124,4 +124,4 @@ export class StorageController { await messageStore.delete(tenant, messageCid); } -} +} \ No newline at end of file diff --git a/tests/event-log/event-log-level.spec.ts b/tests/event-log/event-log-level.spec.ts index 5e9e53b38..926cb68a9 100644 --- a/tests/event-log/event-log-level.spec.ts +++ b/tests/event-log/event-log-level.spec.ts @@ -29,7 +29,7 @@ describe('EventLogLevel Tests', () => { const watermark = await eventLog.append(requester.did, messageCid.toString()); - for await (const [key, value] of eventLog.level.iterator({})) { + for await (const [key, value] of eventLog.level.iterator()) { expect(key).to.include(watermark); expect(value).to.equal(messageCid.toString()); } @@ -47,7 +47,7 @@ describe('EventLogLevel Tests', () => { await eventLog.append(requester.did, messageCid2.toString()); const storedValues = []; - for await (const [_, cid] of eventLog.level.iterator({})) { + for await (const [_, cid] of eventLog.level.iterator()) { storedValues.push(cid); } From f57a1613e23a2b3bb19afe19fb2ed2e3c91a0577 Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Mon, 3 Apr 2023 01:20:03 -0500 Subject: [PATCH 03/21] address PR comments --- src/event-log/event-log-level.ts | 27 +++++++++++-------------- tests/event-log/event-log-level.spec.ts | 20 +++++++++--------- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/src/event-log/event-log-level.ts b/src/event-log/event-log-level.ts index bc92b2969..45bd60e61 100644 --- a/src/event-log/event-log-level.ts +++ b/src/event-log/event-log-level.ts @@ -19,47 +19,44 @@ type EventLogLevelConfig = { export class EventLogLevel implements EventLog { config: EventLogLevelConfig; - level: LevelWrapper; + db: LevelWrapper; ulid: ULID; - constructor(config: EventLogLevelConfig = { location: 'EVENTLOG' }) { + constructor(config?: EventLogLevelConfig) { this.config = { + location: 'EVENTLOG', createLevelDatabase, ...config, }; - this.level = new LevelWrapper({ ...this.config, valueEncoding: 'utf8' }); + this.db = new LevelWrapper({ ...this.config, valueEncoding: 'utf8' }); this.ulid = monotonicFactory(); } async open(): Promise { - // `db.open()` is automatically called by the database constructor. We're calling it explicitly - // in order to explicitly catch an error that would otherwise not surface until another method - // like `db.get()` is called. Once `db.open()` has then been called, any read & write - // operations will again be queued internally until opening has finished. - return this.level.open(); + return this.db.open(); } - close(): Promise { - return this.level.close(); + async close(): Promise { + return this.db.close(); } - clear(): Promise { - return this.level.clear(); + async clear(): Promise { + return this.db.clear(); } async append(tenant: string, messageCid: string): Promise { const hashedTenant = this.hashTenant(tenant); - const tenantEventLog = await this.level.partition(hashedTenant); + const tenantEventLog = await this.db.partition(hashedTenant); const watermark = this.ulid(); await tenantEventLog.put(watermark, messageCid); return watermark; } - async getEventsAfter(tenant: string, watermark?: string | undefined): Promise { + async getEventsAfter(tenant: string, watermark?: string): Promise { const hashedTenant = this.hashTenant(tenant); - const tenantEventLog = await this.level.partition(hashedTenant); + const tenantEventLog = await this.db.partition(hashedTenant); const opts = watermark ? { gt: watermark } : {}; const events: Array = []; diff --git a/tests/event-log/event-log-level.spec.ts b/tests/event-log/event-log-level.spec.ts index 926cb68a9..a2fd6386f 100644 --- a/tests/event-log/event-log-level.spec.ts +++ b/tests/event-log/event-log-level.spec.ts @@ -1,7 +1,7 @@ import chaiAsPromised from 'chai-as-promised'; import { computeCid } from '../../src/utils/cid.js'; import { EventLogLevel } from '../../src/event-log/event-log-level.js'; -import { TestDataGenerator as TDG } from '../utils/test-data-generator.js'; +import { TestDataGenerator } from '../utils/test-data-generator.js'; import chai, { expect } from 'chai'; chai.use(chaiAsPromised); @@ -24,30 +24,30 @@ describe('EventLogLevel Tests', () => { }); it('appends a tenant namespaced entry into leveldb', async () => { - const { requester, message } = await TDG.generateRecordsWrite(); + const { requester, message } = await TestDataGenerator.generateRecordsWrite(); const messageCid = await computeCid(message); const watermark = await eventLog.append(requester.did, messageCid.toString()); - for await (const [key, value] of eventLog.level.iterator()) { + for await (const [key, value] of eventLog.db.iterator()) { expect(key).to.include(watermark); expect(value).to.equal(messageCid.toString()); } }); it('maintains order in which events were appended', async () => { - const { requester, message } = await TDG.generateRecordsWrite(); + const { requester, message } = await TestDataGenerator.generateRecordsWrite(); const messageCid = await computeCid(message); await eventLog.append(requester.did, messageCid.toString()); - const { message: message2 } = await TDG.generateRecordsWrite({ requester }); + const { message: message2 } = await TestDataGenerator.generateRecordsWrite({ requester }); const messageCid2 = await computeCid(message2); await eventLog.append(requester.did, messageCid2.toString()); const storedValues = []; - for await (const [_, cid] of eventLog.level.iterator()) { + for await (const [_, cid] of eventLog.db.iterator()) { storedValues.push(cid); } @@ -73,14 +73,14 @@ describe('EventLogLevel Tests', () => { it('gets all events for a tenant if watermark is not provided', async () => { const messageCids = []; - const { requester, message } = await TDG.generateRecordsWrite(); + const { requester, message } = await TestDataGenerator.generateRecordsWrite(); const messageCid = await computeCid(message); messageCids.push(messageCid); await eventLog.append(requester.did, messageCid.toString()); for (let i = 0; i < 9; i += 1) { - const { message } = await TDG.generateRecordsWrite({ requester }); + const { message } = await TestDataGenerator.generateRecordsWrite({ requester }); const messageCid = await computeCid(message); messageCids.push(messageCid); @@ -96,7 +96,7 @@ describe('EventLogLevel Tests', () => { }); it('gets all events that occured after the watermark provided', async () => { - const { requester, message } = await TDG.generateRecordsWrite(); + const { requester, message } = await TestDataGenerator.generateRecordsWrite(); const messageCid = await computeCid(message); await eventLog.append(requester.did, messageCid.toString()); @@ -105,7 +105,7 @@ describe('EventLogLevel Tests', () => { let testWatermark; for (let i = 0; i < 9; i += 1) { - const { message } = await TDG.generateRecordsWrite({ requester }); + const { message } = await TestDataGenerator.generateRecordsWrite({ requester }); const messageCid = await computeCid(message); const watermark = await eventLog.append(requester.did, messageCid.toString()); From 9ad3c30af44fc8f38a8347c19eb3b5d41be38264 Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Mon, 3 Apr 2023 15:33:38 -0500 Subject: [PATCH 04/21] address PR comments --- src/event-log/event-log.ts | 1 + .../records/handlers/records-delete.ts | 4 ++++ src/store/storage-controller.ts | 2 +- tests/event-log/event-log-level.spec.ts | 22 +++++++++---------- .../handlers/protocols-configure.spec.ts | 1 + 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/event-log/event-log.ts b/src/event-log/event-log.ts index 3c563d4e8..b070205dc 100644 --- a/src/event-log/event-log.ts +++ b/src/event-log/event-log.ts @@ -18,6 +18,7 @@ export interface EventLog { * adds an event to a tenant's event log * @param tenant - the tenant's DID * @param messageCid - the CID of the message + * @returns {Promise} watermark */ append(tenant: string, messageCid: string): Promise diff --git a/src/interfaces/records/handlers/records-delete.ts b/src/interfaces/records/handlers/records-delete.ts index 6a1b68060..4f57dadc5 100644 --- a/src/interfaces/records/handlers/records-delete.ts +++ b/src/interfaces/records/handlers/records-delete.ts @@ -5,6 +5,7 @@ import type { TimestampedMessage } from '../../../core/types.js'; import type { DataStore, DidResolver, MessageStore } from '../../../index.js'; import { authenticate } from '../../../core/auth.js'; +import { computeCid } from '../../../utils/cid.js'; import { deleteAllOlderMessagesButKeepInitialWrite } from '../records-interface.js'; import { DwnInterfaceName } from '../../../core/message.js'; import { MessageReply } from '../../../core/message-reply.js'; @@ -66,6 +67,9 @@ export class RecordsDeleteHandler implements MethodHandler { await this.messageStore.put(tenant, incomingMessage, indexes); + const messageCid = await computeCid(incomingMessage); + await this.eventLog.append(tenant, messageCid); + messageReply = new MessageReply({ status: { code: 202, detail: 'Accepted' } }); diff --git a/src/store/storage-controller.ts b/src/store/storage-controller.ts index e441544fb..1c653af02 100644 --- a/src/store/storage-controller.ts +++ b/src/store/storage-controller.ts @@ -82,7 +82,7 @@ export class StorageController { await messageStore.put(tenant, message, indexes); const messageCid = await Message.getCid(message); - await eventLog.append(tenant, messageCid.toString()); + await eventLog.append(tenant, messageCid); } public static async query( diff --git a/tests/event-log/event-log-level.spec.ts b/tests/event-log/event-log-level.spec.ts index a2fd6386f..93cfffab0 100644 --- a/tests/event-log/event-log-level.spec.ts +++ b/tests/event-log/event-log-level.spec.ts @@ -27,11 +27,11 @@ describe('EventLogLevel Tests', () => { const { requester, message } = await TestDataGenerator.generateRecordsWrite(); const messageCid = await computeCid(message); - const watermark = await eventLog.append(requester.did, messageCid.toString()); + const watermark = await eventLog.append(requester.did, messageCid); for await (const [key, value] of eventLog.db.iterator()) { expect(key).to.include(watermark); - expect(value).to.equal(messageCid.toString()); + expect(value).to.equal(messageCid); } }); @@ -39,20 +39,20 @@ describe('EventLogLevel Tests', () => { const { requester, message } = await TestDataGenerator.generateRecordsWrite(); const messageCid = await computeCid(message); - await eventLog.append(requester.did, messageCid.toString()); + await eventLog.append(requester.did, messageCid); const { message: message2 } = await TestDataGenerator.generateRecordsWrite({ requester }); const messageCid2 = await computeCid(message2); - await eventLog.append(requester.did, messageCid2.toString()); + await eventLog.append(requester.did, messageCid2); const storedValues = []; for await (const [_, cid] of eventLog.db.iterator()) { storedValues.push(cid); } - expect(storedValues[0]).to.equal(messageCid.toString()); - expect(storedValues[1]).to.equal(messageCid2.toString()); + expect(storedValues[0]).to.equal(messageCid); + expect(storedValues[1]).to.equal(messageCid2); }); }); @@ -77,14 +77,14 @@ describe('EventLogLevel Tests', () => { const messageCid = await computeCid(message); messageCids.push(messageCid); - await eventLog.append(requester.did, messageCid.toString()); + await eventLog.append(requester.did, messageCid); for (let i = 0; i < 9; i += 1) { const { message } = await TestDataGenerator.generateRecordsWrite({ requester }); const messageCid = await computeCid(message); messageCids.push(messageCid); - await eventLog.append(requester.did, messageCid.toString()); + await eventLog.append(requester.did, messageCid); } const events = await eventLog.getEventsAfter(requester.did); @@ -99,7 +99,7 @@ describe('EventLogLevel Tests', () => { const { requester, message } = await TestDataGenerator.generateRecordsWrite(); const messageCid = await computeCid(message); - await eventLog.append(requester.did, messageCid.toString()); + await eventLog.append(requester.did, messageCid); const messageCids = []; let testWatermark; @@ -108,14 +108,14 @@ describe('EventLogLevel Tests', () => { const { message } = await TestDataGenerator.generateRecordsWrite({ requester }); const messageCid = await computeCid(message); - const watermark = await eventLog.append(requester.did, messageCid.toString()); + const watermark = await eventLog.append(requester.did, messageCid); if (i === 4) { testWatermark = watermark; } if (i > 4) { - messageCids.push(messageCid.toString()); + messageCids.push(messageCid); } } diff --git a/tests/interfaces/protocols/handlers/protocols-configure.spec.ts b/tests/interfaces/protocols/handlers/protocols-configure.spec.ts index 1904a4d32..c22df7575 100644 --- a/tests/interfaces/protocols/handlers/protocols-configure.spec.ts +++ b/tests/interfaces/protocols/handlers/protocols-configure.spec.ts @@ -53,6 +53,7 @@ describe('ProtocolsConfigureHandler.handle()', () => { // clean up before each test rather than after so that a test does not depend on other tests to do the clean up await messageStore.clear(); await dataStore.clear(); + await eventLog.clear(); }); after(async () => { From d76748edde23194b17406c8d707b870edf9566e1 Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Mon, 3 Apr 2023 17:59:30 -0500 Subject: [PATCH 05/21] prevent computing message cid twice --- src/store/storage-controller.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/store/storage-controller.ts b/src/store/storage-controller.ts index 1c653af02..91cf84256 100644 --- a/src/store/storage-controller.ts +++ b/src/store/storage-controller.ts @@ -35,9 +35,9 @@ export class StorageController { // but NOTE: it is possible that a data stream is not given in such case, for instance, // a subsequent RecordsWrite that changes the `published` property, but the data hasn't changed, // in this case requiring re-uploading of the data is extremely inefficient so we take care allow omission of data stream - if (message.descriptor.dataCid !== undefined) { - const messageCid = await Message.getCid(message); + const messageCid = await Message.getCid(message); + if (message.descriptor.dataCid !== undefined) { let result; if (dataStream === undefined) { @@ -80,8 +80,6 @@ export class StorageController { } await messageStore.put(tenant, message, indexes); - - const messageCid = await Message.getCid(message); await eventLog.append(tenant, messageCid); } From 66f562395473a4bbc069cc7f9303b176d60157dc Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Mon, 3 Apr 2023 18:10:59 -0500 Subject: [PATCH 06/21] adjust test based on PR comment --- README.md | 2 +- tests/event-log/event-log-level.spec.ts | 62 ++++++++++++------------- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 92663a6a8..1228d2397 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # Decentralized Web Node (DWN) SDK Code Coverage -![Statements](https://img.shields.io/badge/statements-93.25%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.91%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.45%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-93.25%25-brightgreen.svg?style=flat) +![Statements](https://img.shields.io/badge/statements-93.23%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.89%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.45%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-93.23%25-brightgreen.svg?style=flat) ## Introduction diff --git a/tests/event-log/event-log-level.spec.ts b/tests/event-log/event-log-level.spec.ts index 93cfffab0..d8f213f80 100644 --- a/tests/event-log/event-log-level.spec.ts +++ b/tests/event-log/event-log-level.spec.ts @@ -9,32 +9,41 @@ chai.use(chaiAsPromised); let eventLog: EventLogLevel; describe('EventLogLevel Tests', () => { - describe('append tests', () => { - before(async () => { - eventLog = new EventLogLevel({ location: 'TEST-EVENTLOG' }); - await eventLog.open(); - }); + before(async () => { + eventLog = new EventLogLevel({ location: 'TEST-EVENTLOG' }); + await eventLog.open(); + }); - beforeEach(async () => { - await eventLog.clear(); - }); + beforeEach(async () => { + await eventLog.clear(); + }); - after(async () => { - await eventLog.close(); - }); + after(async () => { + await eventLog.close(); + }); - it('appends a tenant namespaced entry into leveldb', async () => { - const { requester, message } = await TestDataGenerator.generateRecordsWrite(); - const messageCid = await computeCid(message); + it('separates events by tenant', async () => { + const { requester, message } = await TestDataGenerator.generateRecordsWrite(); + const messageCid = await computeCid(message); + const watermark = await eventLog.append(requester.did, messageCid); - const watermark = await eventLog.append(requester.did, messageCid); + const { requester: requester2, message: message2 } = await TestDataGenerator.generateRecordsWrite(); + const messageCid2 = await computeCid(message2); + const watermark2 = await eventLog.append(requester2.did, messageCid2); - for await (const [key, value] of eventLog.db.iterator()) { - expect(key).to.include(watermark); - expect(value).to.equal(messageCid); - } - }); + let events = await eventLog.getEventsAfter(requester.did); + expect(events.length).to.equal(1); + expect(events[0].watermark).to.equal(watermark); + expect(events[0].messageCid).to.equal(messageCid); + events = await eventLog.getEventsAfter(requester2.did); + expect(events.length).to.equal(1); + expect(events[0].watermark).to.equal(watermark2); + expect(events[0].messageCid).to.equal(messageCid2); + + + }); + describe('append tests', () => { it('maintains order in which events were appended', async () => { const { requester, message } = await TestDataGenerator.generateRecordsWrite(); const messageCid = await computeCid(message); @@ -57,19 +66,6 @@ describe('EventLogLevel Tests', () => { }); describe('getEventsAfter', () => { - before(async () => { - eventLog = new EventLogLevel({ location: 'TEST-EVENTLOG' }); - await eventLog.open(); - }); - - beforeEach(async () => { - await eventLog.clear(); - }); - - after(async () => { - await eventLog.close(); - }); - it('gets all events for a tenant if watermark is not provided', async () => { const messageCids = []; From 1f9f08a5388a1f096df158998b47e4b7fc639840 Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Mon, 3 Apr 2023 18:18:58 -0500 Subject: [PATCH 07/21] remove tenant encoding --- README.md | 2 +- src/event-log/event-log-level.ts | 14 ++------------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 1228d2397..633dea9b8 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # Decentralized Web Node (DWN) SDK Code Coverage -![Statements](https://img.shields.io/badge/statements-93.23%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.89%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.45%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-93.23%25-brightgreen.svg?style=flat) +![Statements](https://img.shields.io/badge/statements-93.22%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.9%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.42%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-93.22%25-brightgreen.svg?style=flat) ## Introduction diff --git a/src/event-log/event-log-level.ts b/src/event-log/event-log-level.ts index 45bd60e61..7fba65d3f 100644 --- a/src/event-log/event-log-level.ts +++ b/src/event-log/event-log-level.ts @@ -1,8 +1,6 @@ import type { ULID } from 'ulid'; import type { Event, EventLog } from './event-log.js'; -import { base32crockford } from '@scure/base'; -import { Encoder } from '../utils/encoder.js'; import { monotonicFactory } from 'ulid'; import { createLevelDatabase, LevelWrapper } from '../store/level-wrapper.js'; @@ -46,8 +44,7 @@ export class EventLogLevel implements EventLog { } async append(tenant: string, messageCid: string): Promise { - const hashedTenant = this.hashTenant(tenant); - const tenantEventLog = await this.db.partition(hashedTenant); + const tenantEventLog = await this.db.partition(tenant); const watermark = this.ulid(); await tenantEventLog.put(watermark, messageCid); @@ -55,8 +52,7 @@ export class EventLogLevel implements EventLog { return watermark; } async getEventsAfter(tenant: string, watermark?: string): Promise { - const hashedTenant = this.hashTenant(tenant); - const tenantEventLog = await this.db.partition(hashedTenant); + const tenantEventLog = await this.db.partition(tenant); const opts = watermark ? { gt: watermark } : {}; const events: Array = []; @@ -68,10 +64,4 @@ export class EventLogLevel implements EventLog { return events; } - - hashTenant(tenant: string): string { - const tenantBytes = Encoder.stringToBytes(tenant); - - return base32crockford.encode(tenantBytes); - } } \ No newline at end of file From d0c1e26b3e7decdc430bc75c1051674d64fb506f Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Mon, 3 Apr 2023 19:32:48 -0500 Subject: [PATCH 08/21] improve test based on PR comment --- tests/event-log/event-log-level.spec.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/event-log/event-log-level.spec.ts b/tests/event-log/event-log-level.spec.ts index d8f213f80..ce0bbe4af 100644 --- a/tests/event-log/event-log-level.spec.ts +++ b/tests/event-log/event-log-level.spec.ts @@ -67,27 +67,28 @@ describe('EventLogLevel Tests', () => { describe('getEventsAfter', () => { it('gets all events for a tenant if watermark is not provided', async () => { - const messageCids = []; + const expectedEvents = []; const { requester, message } = await TestDataGenerator.generateRecordsWrite(); const messageCid = await computeCid(message); - messageCids.push(messageCid); - await eventLog.append(requester.did, messageCid); + const watermark = await eventLog.append(requester.did, messageCid); + expectedEvents.push({ messageCid, watermark }); for (let i = 0; i < 9; i += 1) { const { message } = await TestDataGenerator.generateRecordsWrite({ requester }); const messageCid = await computeCid(message); - messageCids.push(messageCid); - await eventLog.append(requester.did, messageCid); + const watermark = await eventLog.append(requester.did, messageCid); + expectedEvents.push({ messageCid, watermark }); } const events = await eventLog.getEventsAfter(requester.did); expect(events.length).to.equal(10); for (let i = 0; i < events.length; i += 1) { - expect(events[i].messageCid).to.equal(messageCids[i]); + expect(events[i].messageCid).to.equal(expectedEvents[i].messageCid); + expect(events[i].watermark).to.equal(expectedEvents[i].watermark); } }); From a267bac68e44548231f8845d000d735c585331bb Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Mon, 3 Apr 2023 22:51:46 -0500 Subject: [PATCH 09/21] add `dump` to `eventLogLevel` --- src/dwn.ts | 4 ++++ src/event-log/event-log-level.ts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/dwn.ts b/src/dwn.ts index 7c380324c..012d31ff2 100644 --- a/src/dwn.ts +++ b/src/dwn.ts @@ -128,6 +128,10 @@ export class Dwn { console.group('dataStore'); await this.dataStore['dump']?.(); console.groupEnd(); + + console.group('eventLog'); + await this.eventLog['dump']?.(); + console.groupEnd(); } }; diff --git a/src/event-log/event-log-level.ts b/src/event-log/event-log-level.ts index 7fba65d3f..a2294efee 100644 --- a/src/event-log/event-log-level.ts +++ b/src/event-log/event-log-level.ts @@ -64,4 +64,8 @@ export class EventLogLevel implements EventLog { return events; } + + async dump(): Promise { + await this.db['dump']?.(); + } } \ No newline at end of file From 902d7babcfcc2cfa3cc34d1bc944e7e3448741a1 Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Wed, 5 Apr 2023 00:22:01 -0500 Subject: [PATCH 10/21] add `deleteEventsByCid`. refactor `getEventsAfter` method signature --- README.md | 2 +- src/event-log/event-log-level.ts | 34 +++++++++++++++++--- src/event-log/event-log.ts | 7 ++++- tests/event-log/event-log-level.spec.ts | 42 ++++++++++++++++++++++--- 4 files changed, 74 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 633dea9b8..05fca1358 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # Decentralized Web Node (DWN) SDK Code Coverage -![Statements](https://img.shields.io/badge/statements-93.22%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.9%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.42%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-93.22%25-brightgreen.svg?style=flat) +![Statements](https://img.shields.io/badge/statements-92.97%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.8%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.11%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-92.97%25-brightgreen.svg?style=flat) ## Introduction diff --git a/src/event-log/event-log-level.ts b/src/event-log/event-log-level.ts index a2294efee..4a30e0ae6 100644 --- a/src/event-log/event-log-level.ts +++ b/src/event-log/event-log-level.ts @@ -1,5 +1,5 @@ import type { ULID } from 'ulid'; -import type { Event, EventLog } from './event-log.js'; +import type { Event, EventLog, GetEventsOptions } from './event-log.js'; import { monotonicFactory } from 'ulid'; import { createLevelDatabase, LevelWrapper } from '../store/level-wrapper.js'; @@ -51,13 +51,12 @@ export class EventLogLevel implements EventLog { return watermark; } - async getEventsAfter(tenant: string, watermark?: string): Promise { - const tenantEventLog = await this.db.partition(tenant); - const opts = watermark ? { gt: watermark } : {}; + async getEvents(tenant: string, options?: GetEventsOptions): Promise { + const tenantEventLog = await this.db.partition(tenant); const events: Array = []; - for await (const [key, value] of tenantEventLog.iterator(opts)) { + for await (const [key, value] of tenantEventLog.iterator(options)) { const event = { watermark: key, messageCid: value }; events.push(event); } @@ -65,7 +64,32 @@ export class EventLogLevel implements EventLog { return events; } + async deleteEventsByCid(tenant, cids): Promise { + if (cids.length === 0) { + return 0; + } + + const cidSet = new Set(cids); + const tenantEventLog = await this.db.partition(tenant); + const ops = []; + + let numEventsDeleted = 0; + + for await (const [key, value] of tenantEventLog.iterator()) { + if (cidSet.has(value)) { + ops.push({ type: 'del', key }); + numEventsDeleted += 1; + } + } + + await tenantEventLog.batch(ops); + + return numEventsDeleted; + } + async dump(): Promise { + console.group('db'); await this.db['dump']?.(); + console.groupEnd(); } } \ No newline at end of file diff --git a/src/event-log/event-log.ts b/src/event-log/event-log.ts index b070205dc..a1a9b249a 100644 --- a/src/event-log/event-log.ts +++ b/src/event-log/event-log.ts @@ -3,6 +3,11 @@ export type Event = { messageCid: string }; + +export type GetEventsOptions = { + gt: string +}; + export interface EventLog { /** * opens a connection to the underlying store @@ -29,5 +34,5 @@ export interface EventLog { * @param tenant * @param watermark */ - getEventsAfter(tenant: string, watermark?: string): Promise> + getEvents(tenant: string, options?: GetEventsOptions): Promise> } \ No newline at end of file diff --git a/tests/event-log/event-log-level.spec.ts b/tests/event-log/event-log-level.spec.ts index ce0bbe4af..4c4f314fb 100644 --- a/tests/event-log/event-log-level.spec.ts +++ b/tests/event-log/event-log-level.spec.ts @@ -31,18 +31,19 @@ describe('EventLogLevel Tests', () => { const messageCid2 = await computeCid(message2); const watermark2 = await eventLog.append(requester2.did, messageCid2); - let events = await eventLog.getEventsAfter(requester.did); + let events = await eventLog.getEvents(requester.did); expect(events.length).to.equal(1); expect(events[0].watermark).to.equal(watermark); expect(events[0].messageCid).to.equal(messageCid); - events = await eventLog.getEventsAfter(requester2.did); + events = await eventLog.getEvents(requester2.did); expect(events.length).to.equal(1); expect(events[0].watermark).to.equal(watermark2); expect(events[0].messageCid).to.equal(messageCid2); }); + describe('append tests', () => { it('maintains order in which events were appended', async () => { const { requester, message } = await TestDataGenerator.generateRecordsWrite(); @@ -83,7 +84,7 @@ describe('EventLogLevel Tests', () => { expectedEvents.push({ messageCid, watermark }); } - const events = await eventLog.getEventsAfter(requester.did); + const events = await eventLog.getEvents(requester.did); expect(events.length).to.equal(10); for (let i = 0; i < events.length; i += 1) { @@ -116,7 +117,7 @@ describe('EventLogLevel Tests', () => { } } - const events = await eventLog.getEventsAfter(requester.did, testWatermark); + const events = await eventLog.getEvents(requester.did, { gt: testWatermark }); expect(events.length).to.equal(4); for (let i = 0; i < events.length; i += 1) { @@ -124,4 +125,37 @@ describe('EventLogLevel Tests', () => { } }); }); + + describe('deleteEventsByCid', () => { + it('finds and deletes events that whose values match the cids provided', async () => { + const cids = []; + const { requester, message } = await TestDataGenerator.generateRecordsWrite(); + const messageCid = await computeCid(message); + + await eventLog.append(requester.did, messageCid); + + for (let i = 0; i < 9; i += 1) { + const { message } = await TestDataGenerator.generateRecordsWrite({ requester }); + const messageCid = await computeCid(message); + + await eventLog.append(requester.did, messageCid); + if (i % 2 === 0) { + cids.push(messageCid); + } + } + + const numEventsDeleted = await eventLog.deleteEventsByCid(requester.did, cids); + expect(numEventsDeleted).to.equal(cids.length); + + const remainingEvents = await eventLog.getEvents(requester.did); + expect(remainingEvents.length).to.equal(10 - cids.length); + + const cidSet = new Set(cids); + for (const event of remainingEvents) { + if (cidSet.has(event.messageCid)) { + expect.fail(`${event.messageCid} should not exist`); + } + } + }); + }); }); \ No newline at end of file From 513ab06bc3bd93ca8124bf7bc2c4b2f43b439741 Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Wed, 5 Apr 2023 00:32:56 -0500 Subject: [PATCH 11/21] add missing types to method signature and include method in interface --- src/event-log/event-log-level.ts | 2 +- src/event-log/event-log.ts | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/event-log/event-log-level.ts b/src/event-log/event-log-level.ts index 4a30e0ae6..726ceaf9f 100644 --- a/src/event-log/event-log-level.ts +++ b/src/event-log/event-log-level.ts @@ -64,7 +64,7 @@ export class EventLogLevel implements EventLog { return events; } - async deleteEventsByCid(tenant, cids): Promise { + async deleteEventsByCid(tenant: string, cids: Array): Promise { if (cids.length === 0) { return 0; } diff --git a/src/event-log/event-log.ts b/src/event-log/event-log.ts index a1a9b249a..4c15161fe 100644 --- a/src/event-log/event-log.ts +++ b/src/event-log/event-log.ts @@ -35,4 +35,11 @@ export interface EventLog { * @param watermark */ getEvents(tenant: string, options?: GetEventsOptions): Promise> + + /** + * deletes any events that have any of the cids provided + * @param tenant + * @param cids + */ + deleteEventsByCid(tenant: string, cids: Array): Promise } \ No newline at end of file From 74a91abd8c17a2c60395f87ccd22b5ac71432071 Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Wed, 5 Apr 2023 00:37:00 -0500 Subject: [PATCH 12/21] delete events whenever intermediary `RecordsWrite`s are deleted [relevant discussion](https://github.com/TBD54566975/dwn-sdk-js/pull/290#discussion_r1154950268). good catch @dcrousso --- README.md | 2 +- src/interfaces/records/handlers/records-delete.ts | 2 +- src/interfaces/records/handlers/records-write.ts | 2 +- src/interfaces/records/records-interface.ts | 11 ++++++++++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 05fca1358..1370885ce 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # Decentralized Web Node (DWN) SDK Code Coverage -![Statements](https://img.shields.io/badge/statements-92.97%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.8%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.11%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-92.97%25-brightgreen.svg?style=flat) +![Statements](https://img.shields.io/badge/statements-92.89%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.94%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.11%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-92.89%25-brightgreen.svg?style=flat) ## Introduction diff --git a/src/interfaces/records/handlers/records-delete.ts b/src/interfaces/records/handlers/records-delete.ts index 4f57dadc5..679dd54b9 100644 --- a/src/interfaces/records/handlers/records-delete.ts +++ b/src/interfaces/records/handlers/records-delete.ts @@ -80,7 +80,7 @@ export class RecordsDeleteHandler implements MethodHandler { } // delete all existing messages that are not newest, except for the initial write - await deleteAllOlderMessagesButKeepInitialWrite(tenant, existingMessages, newestMessage, this.messageStore, this.dataStore); + await deleteAllOlderMessagesButKeepInitialWrite(tenant, existingMessages, newestMessage, this.messageStore, this.dataStore, this.eventLog); return messageReply; }; diff --git a/src/interfaces/records/handlers/records-write.ts b/src/interfaces/records/handlers/records-write.ts index 78e7dbaf5..26dce892f 100644 --- a/src/interfaces/records/handlers/records-write.ts +++ b/src/interfaces/records/handlers/records-write.ts @@ -106,7 +106,7 @@ export class RecordsWriteHandler implements MethodHandler { } // delete all existing messages that are not newest, except for the initial write - await deleteAllOlderMessagesButKeepInitialWrite(tenant, existingMessages, newestMessage, this.messageStore, this.dataStore); + await deleteAllOlderMessagesButKeepInitialWrite(tenant, existingMessages, newestMessage, this.messageStore, this.dataStore, this.eventLog); return messageReply; }; diff --git a/src/interfaces/records/records-interface.ts b/src/interfaces/records/records-interface.ts index aea93b0a6..35a16b536 100644 --- a/src/interfaces/records/records-interface.ts +++ b/src/interfaces/records/records-interface.ts @@ -2,6 +2,7 @@ import type { DataStore } from '../../store/data-store.js'; import type { MessageStore } from '../../store/message-store.js'; import type { RecordsWriteMessage } from '../../interfaces/records/types.js'; import type { TimestampedMessage } from '../../core/types.js'; +import type { EventLog } from '../../event-log/event-log.js'; import { constructRecordsWriteIndexes } from './handlers/records-write.js'; import { Message } from '../../core/message.js'; @@ -17,8 +18,11 @@ export async function deleteAllOlderMessagesButKeepInitialWrite( existingMessages: TimestampedMessage[], comparedToMessage: TimestampedMessage, messageStore: MessageStore, - dataStore: DataStore + dataStore: DataStore, + eventLog: EventLog ): Promise { + const deletedMessageCids = []; + // NOTE: under normal operation, there should only be at most two existing records per `recordId` (initial + a potential subsequent write/delete), // but the DWN may crash before `delete()` is called below, so we use a loop as a tactic to clean up lingering data as needed for (const message of existingMessages) { @@ -37,7 +41,12 @@ export async function deleteAllOlderMessagesButKeepInitialWrite( const isLatestBaseState = false; const indexes = await constructRecordsWriteIndexes(existingRecordsWrite, isLatestBaseState); await messageStore.put(tenant, message, indexes); + } else { + const messageCid = await Message.getCid(message); + deletedMessageCids.push(messageCid); } } + + await eventLog.deleteEventsByCid(tenant, deletedMessageCids); } } From eb7f6972a281b8bc063719fdccedb9ea6bb8d235 Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Wed, 5 Apr 2023 00:38:11 -0500 Subject: [PATCH 13/21] fix comment placement --- src/store/storage-controller.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/store/storage-controller.ts b/src/store/storage-controller.ts index 91cf84256..5cebe4dfc 100644 --- a/src/store/storage-controller.ts +++ b/src/store/storage-controller.ts @@ -31,12 +31,12 @@ export class StorageController { indexes: { [key: string]: string }, dataStream?: Readable ): Promise { + const messageCid = await Message.getCid(message); + // if `dataCid` is given, it means there is corresponding data associated with this message // but NOTE: it is possible that a data stream is not given in such case, for instance, // a subsequent RecordsWrite that changes the `published` property, but the data hasn't changed, // in this case requiring re-uploading of the data is extremely inefficient so we take care allow omission of data stream - const messageCid = await Message.getCid(message); - if (message.descriptor.dataCid !== undefined) { let result; From 946b57a8335d847d3086452606d7aa0234fdeb35 Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Wed, 5 Apr 2023 01:36:45 -0500 Subject: [PATCH 14/21] fix merge related issues --- README.md | 2 +- package-lock.json | 4 ++-- src/interfaces/records/handlers/records-delete.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1370885ce..766232071 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # Decentralized Web Node (DWN) SDK Code Coverage -![Statements](https://img.shields.io/badge/statements-92.89%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.94%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.11%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-92.89%25-brightgreen.svg?style=flat) +![Statements](https://img.shields.io/badge/statements-92.86%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.34%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.34%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-92.86%25-brightgreen.svg?style=flat) ## Introduction diff --git a/package-lock.json b/package-lock.json index 4d694f2d3..fccee41c4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "@js-temporal/polyfill": "0.4.3", "@noble/ed25519": "1.7.1", "@noble/secp256k1": "1.7.1", - "@scure/base": "^1.1.1", + "@scure/base": "1.1.1", "@swc/helpers": "0.3.8", "@types/eccrypto": "1.1.3", "@types/ms": "0.7.31", @@ -37,7 +37,7 @@ "multiformats": "11.0.2", "randombytes": "2.1.0", "readable-stream": "4.3.0", - "ulid": "^2.3.0", + "ulid": "2.3.0", "uuid": "8.3.2", "varint": "6.0.0" }, diff --git a/src/interfaces/records/handlers/records-delete.ts b/src/interfaces/records/handlers/records-delete.ts index 6d9648a49..f67a2e2a4 100644 --- a/src/interfaces/records/handlers/records-delete.ts +++ b/src/interfaces/records/handlers/records-delete.ts @@ -62,7 +62,7 @@ export class RecordsDeleteHandler implements MethodHandler { await this.messageStore.put(tenant, message, indexes); - const messageCid = await computeCid(incomingMessage); + const messageCid = await computeCid(message); await this.eventLog.append(tenant, messageCid); messageReply = new MessageReply({ From 8d1897109dd0194544014ba5ceaf126a3a4c36d0 Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Wed, 5 Apr 2023 01:40:24 -0500 Subject: [PATCH 15/21] fix import order --- src/interfaces/records/records-interface.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interfaces/records/records-interface.ts b/src/interfaces/records/records-interface.ts index 35a16b536..cb0c3591e 100644 --- a/src/interfaces/records/records-interface.ts +++ b/src/interfaces/records/records-interface.ts @@ -1,8 +1,8 @@ import type { DataStore } from '../../store/data-store.js'; +import type { EventLog } from '../../event-log/event-log.js'; import type { MessageStore } from '../../store/message-store.js'; import type { RecordsWriteMessage } from '../../interfaces/records/types.js'; import type { TimestampedMessage } from '../../core/types.js'; -import type { EventLog } from '../../event-log/event-log.js'; import { constructRecordsWriteIndexes } from './handlers/records-write.js'; import { Message } from '../../core/message.js'; From f44f90d89772f634a8aacfd98020ab72758518c8 Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Wed, 5 Apr 2023 23:41:52 -0500 Subject: [PATCH 16/21] add functional tests --- .../handlers/protocols-configure.spec.ts | 17 +++++ .../records/handlers/records-delete.spec.ts | 70 ++++++++++++++++++- .../records/handlers/records-write.spec.ts | 53 ++++++++++++++ 3 files changed, 139 insertions(+), 1 deletion(-) diff --git a/tests/interfaces/protocols/handlers/protocols-configure.spec.ts b/tests/interfaces/protocols/handlers/protocols-configure.spec.ts index c22df7575..7d8ba9aa8 100644 --- a/tests/interfaces/protocols/handlers/protocols-configure.spec.ts +++ b/tests/interfaces/protocols/handlers/protocols-configure.spec.ts @@ -139,5 +139,22 @@ describe('ProtocolsConfigureHandler.handle()', () => { expect(actualDefinition).to.not.equal(initialDefinition); expect(actualDefinition).to.equal(expectedDefinition); }); + + describe('event log', () => { + it('should add event for ProtocolsConfigure', async () => { + const alice = await DidKeyResolver.generate(); + const protocol = 'exampleProtocol'; + const { message, dataStream } = await TestDataGenerator.generateProtocolsConfigure({ requester: alice, protocol }); + + const reply = await dwn.processMessage(alice.did, message, dataStream); + expect(reply.status.code).to.equal(202); + + const events = await eventLog.getEvents(alice.did); + expect(events.length).to.equal(1); + + const messageCid = await Message.getCid(message); + expect(events[0].messageCid).to.equal(messageCid); + }); + }); }); }); diff --git a/tests/interfaces/records/handlers/records-delete.spec.ts b/tests/interfaces/records/handlers/records-delete.spec.ts index 238b2a1a0..d70b99165 100644 --- a/tests/interfaces/records/handlers/records-delete.spec.ts +++ b/tests/interfaces/records/handlers/records-delete.spec.ts @@ -8,11 +8,12 @@ import { DataStoreLevel } from '../../../../src/store/data-store-level.js'; import { DidKeyResolver } from '../../../../src/did/did-key-resolver.js'; import { DwnErrorCode } from '../../../../src/core/dwn-error.js'; import { EventLogLevel } from '../../../../src/event-log/event-log-level.js'; +import { Message } from '../../../../src/core/message.js'; import { MessageStoreLevel } from '../../../../src/store/message-store-level.js'; import { RecordsDeleteHandler } from '../../../../src/interfaces/records/handlers/records-delete.js'; import { TestDataGenerator } from '../../../utils/test-data-generator.js'; import { TestStubGenerator } from '../../../utils/test-stub-generator.js'; -import { DidResolver, Dwn, Encoder, Jws, RecordsDelete } from '../../../../src/index.js'; +import { DidResolver, Dwn, Encoder, Jws, RecordsDelete, RecordsWrite } from '../../../../src/index.js'; chai.use(chaiAsPromised); @@ -506,6 +507,73 @@ describe('RecordsDeleteHandler.handle()', () => { await expect(asyncGeneratorToArray(blocks.db.keys())).to.eventually.eql([ ]); }); + + describe('event log', () => { + it('should include RecordsDelete event and keep initial RecordsWrite event', async () => { + const alice = await DidKeyResolver.generate(); + + const { message, dataStream } = await TestDataGenerator.generateRecordsWrite({ requester: alice }); + const writeReply = await dwn.processMessage(alice.did, message, dataStream); + expect(writeReply.status.code).to.equal(202); + + const recordsDelete = await RecordsDelete.create({ + recordId : message.recordId, + authorizationSignatureInput : Jws.createSignatureInput(alice) + }); + + const deleteReply = await dwn.processMessage(alice.did, recordsDelete.message); + expect(deleteReply.status.code).to.equal(202); + + const events = await eventLog.getEvents(alice.did); + expect(events.length).to.equal(2); + + const writeMessageCid = await Message.getCid(message); + const deleteMessageCid = await Message.getCid(recordsDelete.message); + const expectedMessageCids = new Set([writeMessageCid, deleteMessageCid]); + + for (const { messageCid } of events) { + expectedMessageCids.delete(messageCid); + } + + expect(expectedMessageCids.size).to.equal(0); + }); + + it('should only keep first write and delete when subsequent writes happen', async () => { + const { message, requester, dataStream, recordsWrite } = await TestDataGenerator.generateRecordsWrite(); + TestStubGenerator.stubDidResolver(didResolver, [requester]); + + const reply = await dwn.processMessage(requester.did, message, dataStream); + expect(reply.status.code).to.equal(202); + + const newWrite = await RecordsWrite.createFrom({ + unsignedRecordsWriteMessage : recordsWrite.message, + published : true, + authorizationSignatureInput : Jws.createSignatureInput(requester) + }); + + const newWriteReply = await dwn.processMessage(requester.did, newWrite.message); + expect(newWriteReply.status.code).to.equal(202); + + const recordsDelete = await RecordsDelete.create({ + recordId : message.recordId, + authorizationSignatureInput : Jws.createSignatureInput(requester) + }); + + const deleteReply = await dwn.processMessage(requester.did, recordsDelete.message); + expect(deleteReply.status.code).to.equal(202); + + const events = await eventLog.getEvents(requester.did); + expect(events.length).to.equal(2); + + const deletedMessageCid = await Message.getCid(newWrite.message); + + for (const { messageCid } of events) { + if (messageCid === deletedMessageCid ) { + expect.fail(`${messageCid} should not exist`); + } + } + }); + }); }); it('should return 401 if signature check fails', async () => { diff --git a/tests/interfaces/records/handlers/records-write.spec.ts b/tests/interfaces/records/handlers/records-write.spec.ts index 36547c949..a6636950f 100644 --- a/tests/interfaces/records/handlers/records-write.spec.ts +++ b/tests/interfaces/records/handlers/records-write.spec.ts @@ -447,6 +447,59 @@ describe('RecordsWriteHandler.handle()', () => { expect(reply.status.code).to.equal(400); expect(reply.status.detail).to.contain('does not match deterministic contextId'); }); + + describe('event log', () => { + it('should add an event to the eventlog on initial write', async () => { + const { message, requester, dataStream } = await TestDataGenerator.generateRecordsWrite(); + TestStubGenerator.stubDidResolver(didResolver, [requester]); + + const reply = await dwn.processMessage(requester.did, message, dataStream); + expect(reply.status.code).to.equal(202); + + const events = await eventLog.getEvents(requester.did); + expect(events.length).to.equal(1); + + const messageCid = await Message.getCid(message); + expect(events[0].messageCid).to.equal(messageCid); + }); + + it('should only keep first write and latest write when subsequent writes happen', async () => { + const { message, requester, dataStream, recordsWrite } = await TestDataGenerator.generateRecordsWrite(); + TestStubGenerator.stubDidResolver(didResolver, [requester]); + + const reply = await dwn.processMessage(requester.did, message, dataStream); + expect(reply.status.code).to.equal(202); + + const newWrite = await RecordsWrite.createFrom({ + unsignedRecordsWriteMessage : recordsWrite.message, + published : true, + authorizationSignatureInput : Jws.createSignatureInput(requester) + }); + + const newWriteReply = await dwn.processMessage(requester.did, newWrite.message); + expect(newWriteReply.status.code).to.equal(202); + + const newestWrite = await RecordsWrite.createFrom({ + unsignedRecordsWriteMessage : recordsWrite.message, + published : true, + authorizationSignatureInput : Jws.createSignatureInput(requester) + }); + + const newestWriteReply = await dwn.processMessage(requester.did, newestWrite.message); + expect(newestWriteReply.status.code).to.equal(202); + + const events = await eventLog.getEvents(requester.did); + expect(events.length).to.equal(2); + + const deletedMessageCid = await Message.getCid(newWrite.message); + + for (const { messageCid } of events) { + if (messageCid === deletedMessageCid ) { + expect.fail(`${messageCid} should not exist`); + } + } + }); + }); }); describe('protocol based writes', () => { From 45273550478a08baf931ad2175000c519b994d90 Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Wed, 5 Apr 2023 23:44:11 -0500 Subject: [PATCH 17/21] remove duplicate code --- src/dwn.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/dwn.ts b/src/dwn.ts index 774763e39..f9cc546aa 100644 --- a/src/dwn.ts +++ b/src/dwn.ts @@ -35,10 +35,6 @@ export class Dwn { this.dataStore = config.dataStore!; this.eventLog = config.eventLog!; this.tenantGate = config.tenantGate!; - this.didResolver = config.didResolver!; - this.messageStore = config.messageStore!; - this.dataStore = config.dataStore!; - this.tenantGate = config.tenantGate!; this.methodHandlers = { [DwnInterfaceName.Permissions + DwnMethodName.Request] : new PermissionsRequestHandler(this.didResolver, this.messageStore, this.dataStore), From c17f6d56e0d26cb093522b180dfd90f81e2c8de8 Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Wed, 5 Apr 2023 23:44:59 -0500 Subject: [PATCH 18/21] add jsdoc --- src/event-log/event-log.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/event-log/event-log.ts b/src/event-log/event-log.ts index 4c15161fe..444223243 100644 --- a/src/event-log/event-log.ts +++ b/src/event-log/event-log.ts @@ -40,6 +40,7 @@ export interface EventLog { * deletes any events that have any of the cids provided * @param tenant * @param cids + * @returns {Promise} the number of events deleted */ deleteEventsByCid(tenant: string, cids: Array): Promise } \ No newline at end of file From e1643a0243252fa8845f6d388c70e1eca03956ce Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Wed, 5 Apr 2023 23:58:42 -0500 Subject: [PATCH 19/21] delete older ProtocolsConfigure event when one is overwritten --- README.md | 2 +- .../protocols/handlers/protocols-configure.ts | 6 ++++ .../handlers/protocols-configure.spec.ts | 31 +++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 766232071..f049a0c21 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # Decentralized Web Node (DWN) SDK Code Coverage -![Statements](https://img.shields.io/badge/statements-92.86%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.34%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.34%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-92.86%25-brightgreen.svg?style=flat) +![Statements](https://img.shields.io/badge/statements-92.84%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.35%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.34%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-92.84%25-brightgreen.svg?style=flat) ## Introduction diff --git a/src/interfaces/protocols/handlers/protocols-configure.ts b/src/interfaces/protocols/handlers/protocols-configure.ts index d6cd442a0..e2142937c 100644 --- a/src/interfaces/protocols/handlers/protocols-configure.ts +++ b/src/interfaces/protocols/handlers/protocols-configure.ts @@ -73,12 +73,18 @@ export class ProtocolsConfigureHandler implements MethodHandler { } // delete all existing records that are smaller + const deletedMessageCids = []; for (const message of existingMessages) { if (await Message.isCidLarger(newestMessage, message)) { + const messageCid = await Message.getCid(message); + deletedMessageCids.push(messageCid); + await StorageController.delete(this.messageStore, this.dataStore, tenant, message); } } + await this.eventLog.deleteEventsByCid(tenant, deletedMessageCids); + return messageReply; }; } \ No newline at end of file diff --git a/tests/interfaces/protocols/handlers/protocols-configure.spec.ts b/tests/interfaces/protocols/handlers/protocols-configure.spec.ts index 7d8ba9aa8..6e998ef69 100644 --- a/tests/interfaces/protocols/handlers/protocols-configure.spec.ts +++ b/tests/interfaces/protocols/handlers/protocols-configure.spec.ts @@ -155,6 +155,37 @@ describe('ProtocolsConfigureHandler.handle()', () => { const messageCid = await Message.getCid(message); expect(events[0].messageCid).to.equal(messageCid); }); + + it('should delete older ProtocolsConfigure event when one overwritten', async () => { + const alice = await DidKeyResolver.generate(); + const protocol = 'exampleProtocol'; + const messageData1 = await TestDataGenerator.generateProtocolsConfigure({ requester: alice, protocol }); + const messageData2 = await TestDataGenerator.generateProtocolsConfigure({ requester: alice, protocol }); + + const messageDataWithCid = []; + for (const messageData of [messageData1, messageData2]) { + const cid = await Message.getCid(messageData.message); + messageDataWithCid.push({ cid, ...messageData }); + } + + // sort the message in lexicographic order + const [oldestWrite, newestWrite]: GenerateProtocolsConfigureOutput[] + = messageDataWithCid.sort((messageDataA, messageDataB) => { return lexicographicalCompare(messageDataA.cid, messageDataB.cid); }); + + // write the protocol with the middle lexicographic value + let reply = await dwn.processMessage(alice.did, oldestWrite.message, oldestWrite.dataStream); + expect(reply.status.code).to.equal(202); + + // test that the protocol with the largest lexicographic value can be written + reply = await dwn.processMessage(alice.did, newestWrite.message, newestWrite.dataStream); + expect(reply.status.code).to.equal(202); + + const events = await eventLog.getEvents(alice.did); + expect(events.length).to.equal(1); + + const newestMessageCid = await Message.getCid(newestWrite.message); + expect(events[0].messageCid).to.equal(newestMessageCid); + }); }); }); }); From 6625b71fbda9624fc255092afea2ac23878295e6 Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Thu, 6 Apr 2023 00:12:16 -0500 Subject: [PATCH 20/21] strict mode :sob: --- README.md | 2 +- src/event-log/event-log-level.ts | 3 ++- .../protocols/handlers/protocols-configure.ts | 2 +- src/interfaces/records/records-interface.ts | 2 +- src/store/level-wrapper.ts | 2 +- tests/event-log/event-log-level.spec.ts | 10 ++++++---- .../protocols/handlers/protocols-configure.spec.ts | 2 +- 7 files changed, 13 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f049a0c21..d541cd000 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # Decentralized Web Node (DWN) SDK Code Coverage -![Statements](https://img.shields.io/badge/statements-92.84%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.35%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.34%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-92.84%25-brightgreen.svg?style=flat) +![Statements](https://img.shields.io/badge/statements-92.85%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.35%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.34%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-92.85%25-brightgreen.svg?style=flat) ## Introduction diff --git a/src/event-log/event-log-level.ts b/src/event-log/event-log-level.ts index 726ceaf9f..a0549fd06 100644 --- a/src/event-log/event-log-level.ts +++ b/src/event-log/event-log-level.ts @@ -1,3 +1,4 @@ +import type { LevelWrapperBatchOperation } from '../store/level-wrapper.js'; import type { ULID } from 'ulid'; import type { Event, EventLog, GetEventsOptions } from './event-log.js'; @@ -71,7 +72,7 @@ export class EventLogLevel implements EventLog { const cidSet = new Set(cids); const tenantEventLog = await this.db.partition(tenant); - const ops = []; + const ops: LevelWrapperBatchOperation[] = []; let numEventsDeleted = 0; diff --git a/src/interfaces/protocols/handlers/protocols-configure.ts b/src/interfaces/protocols/handlers/protocols-configure.ts index e2142937c..55970431d 100644 --- a/src/interfaces/protocols/handlers/protocols-configure.ts +++ b/src/interfaces/protocols/handlers/protocols-configure.ts @@ -73,7 +73,7 @@ export class ProtocolsConfigureHandler implements MethodHandler { } // delete all existing records that are smaller - const deletedMessageCids = []; + const deletedMessageCids: string[] = []; for (const message of existingMessages) { if (await Message.isCidLarger(newestMessage, message)) { const messageCid = await Message.getCid(message); diff --git a/src/interfaces/records/records-interface.ts b/src/interfaces/records/records-interface.ts index cb0c3591e..c0cd53d27 100644 --- a/src/interfaces/records/records-interface.ts +++ b/src/interfaces/records/records-interface.ts @@ -21,7 +21,7 @@ export async function deleteAllOlderMessagesButKeepInitialWrite( dataStore: DataStore, eventLog: EventLog ): Promise { - const deletedMessageCids = []; + const deletedMessageCids: string[] = []; // NOTE: under normal operation, there should only be at most two existing records per `recordId` (initial + a potential subsequent write/delete), // but the DWN may crash before `delete()` is called below, so we use a loop as a tactic to clean up lingering data as needed diff --git a/src/store/level-wrapper.ts b/src/store/level-wrapper.ts index e81e10c22..c679510af 100644 --- a/src/store/level-wrapper.ts +++ b/src/store/level-wrapper.ts @@ -127,7 +127,7 @@ export class LevelWrapper { await abortOr(options?.signal, this.createLevelDatabase()); - for await (const entry of this.db.iterator(iteratorOptions)) { + for await (const entry of this.db.iterator(iteratorOptions!)) { options?.signal?.throwIfAborted(); yield entry; diff --git a/tests/event-log/event-log-level.spec.ts b/tests/event-log/event-log-level.spec.ts index 4c4f314fb..f48c80b67 100644 --- a/tests/event-log/event-log-level.spec.ts +++ b/tests/event-log/event-log-level.spec.ts @@ -1,3 +1,5 @@ +import type { Event } from '../../src/event-log/event-log.js'; + import chaiAsPromised from 'chai-as-promised'; import { computeCid } from '../../src/utils/cid.js'; import { EventLogLevel } from '../../src/event-log/event-log-level.js'; @@ -56,7 +58,7 @@ describe('EventLogLevel Tests', () => { await eventLog.append(requester.did, messageCid2); - const storedValues = []; + const storedValues: string[] = []; for await (const [_, cid] of eventLog.db.iterator()) { storedValues.push(cid); } @@ -68,7 +70,7 @@ describe('EventLogLevel Tests', () => { describe('getEventsAfter', () => { it('gets all events for a tenant if watermark is not provided', async () => { - const expectedEvents = []; + const expectedEvents: Event[] = []; const { requester, message } = await TestDataGenerator.generateRecordsWrite(); const messageCid = await computeCid(message); @@ -99,7 +101,7 @@ describe('EventLogLevel Tests', () => { await eventLog.append(requester.did, messageCid); - const messageCids = []; + const messageCids: string[] = []; let testWatermark; for (let i = 0; i < 9; i += 1) { @@ -128,7 +130,7 @@ describe('EventLogLevel Tests', () => { describe('deleteEventsByCid', () => { it('finds and deletes events that whose values match the cids provided', async () => { - const cids = []; + const cids: string[] = []; const { requester, message } = await TestDataGenerator.generateRecordsWrite(); const messageCid = await computeCid(message); diff --git a/tests/interfaces/protocols/handlers/protocols-configure.spec.ts b/tests/interfaces/protocols/handlers/protocols-configure.spec.ts index b4c58f206..9064d8a7f 100644 --- a/tests/interfaces/protocols/handlers/protocols-configure.spec.ts +++ b/tests/interfaces/protocols/handlers/protocols-configure.spec.ts @@ -162,7 +162,7 @@ describe('ProtocolsConfigureHandler.handle()', () => { const messageData1 = await TestDataGenerator.generateProtocolsConfigure({ requester: alice, protocol }); const messageData2 = await TestDataGenerator.generateProtocolsConfigure({ requester: alice, protocol }); - const messageDataWithCid = []; + const messageDataWithCid: (GenerateProtocolsConfigureOutput & { cid: string })[] = []; for (const messageData of [messageData1, messageData2]) { const cid = await Message.getCid(messageData.message); messageDataWithCid.push({ cid, ...messageData }); From 7b5be7292147ff04113a45f55904d8dc01117234 Mon Sep 17 00:00:00 2001 From: Moe Jangda Date: Fri, 7 Apr 2023 19:12:06 -0500 Subject: [PATCH 21/21] store events in watermark order and cid order --- README.md | 2 +- src/event-log/event-log-level.ts | 39 +++++++++++++++++++------ tests/event-log/event-log-level.spec.ts | 34 +++++++++++---------- 3 files changed, 50 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index d541cd000..200429710 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # Decentralized Web Node (DWN) SDK Code Coverage -![Statements](https://img.shields.io/badge/statements-92.85%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.35%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.34%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-92.85%25-brightgreen.svg?style=flat) +![Statements](https://img.shields.io/badge/statements-92.88%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.37%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-90.34%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-92.88%25-brightgreen.svg?style=flat) ## Introduction diff --git a/src/event-log/event-log-level.ts b/src/event-log/event-log-level.ts index a0549fd06..cdc39c4fe 100644 --- a/src/event-log/event-log-level.ts +++ b/src/event-log/event-log-level.ts @@ -5,7 +5,6 @@ import type { Event, EventLog, GetEventsOptions } from './event-log.js'; import { monotonicFactory } from 'ulid'; import { createLevelDatabase, LevelWrapper } from '../store/level-wrapper.js'; - type EventLogLevelConfig = { /** * must be a directory path (relative or absolute) where @@ -16,6 +15,9 @@ type EventLogLevelConfig = { createLevelDatabase?: typeof createLevelDatabase, }; +const WATERMARKS_SUBLEVEL_NAME = 'watermarks'; +const CIDS_SUBLEVEL_NAME = 'cids'; + export class EventLogLevel implements EventLog { config: EventLogLevelConfig; db: LevelWrapper; @@ -46,18 +48,23 @@ export class EventLogLevel implements EventLog { async append(tenant: string, messageCid: string): Promise { const tenantEventLog = await this.db.partition(tenant); + const watermarkLog = await tenantEventLog.partition(WATERMARKS_SUBLEVEL_NAME); + const cidLog = await tenantEventLog.partition(CIDS_SUBLEVEL_NAME); const watermark = this.ulid(); - await tenantEventLog.put(watermark, messageCid); + + await watermarkLog.put(watermark, messageCid); + await cidLog.put(messageCid, watermark); return watermark; } async getEvents(tenant: string, options?: GetEventsOptions): Promise { const tenantEventLog = await this.db.partition(tenant); + const watermarkLog = await tenantEventLog.partition(WATERMARKS_SUBLEVEL_NAME); const events: Array = []; - for await (const [key, value] of tenantEventLog.iterator(options)) { + for await (const [key, value] of watermarkLog.iterator(options)) { const event = { watermark: key, messageCid: value }; events.push(event); } @@ -70,20 +77,34 @@ export class EventLogLevel implements EventLog { return 0; } - const cidSet = new Set(cids); const tenantEventLog = await this.db.partition(tenant); - const ops: LevelWrapperBatchOperation[] = []; + const cidLog = await tenantEventLog.partition(CIDS_SUBLEVEL_NAME); + + let ops: LevelWrapperBatchOperation[] = []; + const promises: Array> = []; + + for (const cid of cids) { + ops.push({ type: 'del', key: cid }); + + const promise = cidLog.get(cid).catch(e => e); + promises.push(promise); + } + + await cidLog.batch(ops); + ops = []; let numEventsDeleted = 0; - for await (const [key, value] of tenantEventLog.iterator()) { - if (cidSet.has(value)) { - ops.push({ type: 'del', key }); + const watermarks: Array = await Promise.all(promises); + for (const watermark of watermarks) { + if (watermark) { + ops.push({ type: 'del', key: watermark }); numEventsDeleted += 1; } } - await tenantEventLog.batch(ops); + const watermarkLog = await tenantEventLog.partition('watermarks'); + await watermarkLog.batch(ops); return numEventsDeleted; } diff --git a/tests/event-log/event-log-level.spec.ts b/tests/event-log/event-log-level.spec.ts index f48c80b67..db480b364 100644 --- a/tests/event-log/event-log-level.spec.ts +++ b/tests/event-log/event-log-level.spec.ts @@ -46,26 +46,30 @@ describe('EventLogLevel Tests', () => { }); - describe('append tests', () => { - it('maintains order in which events were appended', async () => { - const { requester, message } = await TestDataGenerator.generateRecordsWrite(); - const messageCid = await computeCid(message); + it('returns events in the order that they were appended', async () => { + const expectedEvents: Array = []; - await eventLog.append(requester.did, messageCid); + const { requester, message } = await TestDataGenerator.generateRecordsWrite(); + const messageCid = await computeCid(message); + const watermark = await eventLog.append(requester.did, messageCid); - const { message: message2 } = await TestDataGenerator.generateRecordsWrite({ requester }); - const messageCid2 = await computeCid(message2); + expectedEvents.push({ watermark, messageCid }); - await eventLog.append(requester.did, messageCid2); + for (let i = 0; i < 9; i += 1) { + const { message } = await TestDataGenerator.generateRecordsWrite({ requester }); + const messageCid = await computeCid(message); + const watermark = await eventLog.append(requester.did, messageCid); - const storedValues: string[] = []; - for await (const [_, cid] of eventLog.db.iterator()) { - storedValues.push(cid); - } + expectedEvents.push({ watermark, messageCid }); + } - expect(storedValues[0]).to.equal(messageCid); - expect(storedValues[1]).to.equal(messageCid2); - }); + const events = await eventLog.getEvents(requester.did); + expect(events.length).to.equal(expectedEvents.length); + + for (let i = 0; i < 10; i += 1) { + expect(events[i].watermark).to.equal(expectedEvents[i].watermark); + expect(events[i].messageCid).to.equal(expectedEvents[i].messageCid); + } }); describe('getEventsAfter', () => {