From 78ded6c832a28e7a071449addc772a3428362313 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 00:52:28 +0000 Subject: [PATCH 01/18] Spelling fix --- src/plugins/liveobjects/livecounter.ts | 2 +- src/plugins/liveobjects/livemap.ts | 4 ++-- src/plugins/liveobjects/statemessage.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 10542f067..843203583 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -185,7 +185,7 @@ export class LiveCounter extends LiveObject this._siteTimeserials = stateObject.siteTimeserials ?? {}; if (this.isTombstoned()) { - // this object is tombstoned. this is a terminal state which can't be overriden. skip the rest of state object message processing + // this object is tombstoned. this is a terminal state which can't be overridden. skip the rest of state object message processing return { noop: true }; } diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 79b291d3d..e12c9a1c4 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -370,7 +370,7 @@ export class LiveMap extends LiveObject extends LiveObject Date: Wed, 15 Jan 2025 23:21:37 +0000 Subject: [PATCH 02/18] Add `IBufferUtils.sha256` method --- src/common/types/IBufferUtils.ts | 1 + src/platform/nodejs/lib/util/bufferutils.ts | 11 +++++++---- src/platform/web/lib/util/bufferutils.ts | 7 ++++++- src/platform/web/lib/util/hmac-sha256.ts | 4 ++-- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/common/types/IBufferUtils.ts b/src/common/types/IBufferUtils.ts index be573a44f..0ba48892e 100644 --- a/src/common/types/IBufferUtils.ts +++ b/src/common/types/IBufferUtils.ts @@ -19,5 +19,6 @@ export default interface IBufferUtils { * Returns ArrayBuffer on browser and Buffer on Node.js */ arrayBufferViewToBuffer: (arrayBufferView: ArrayBufferView) => Bufferlike; + sha256(message: Bufferlike): Output; hmacSha256(message: Bufferlike, key: Bufferlike): Output; } diff --git a/src/platform/nodejs/lib/util/bufferutils.ts b/src/platform/nodejs/lib/util/bufferutils.ts index 52b46ba2b..86baa4632 100644 --- a/src/platform/nodejs/lib/util/bufferutils.ts +++ b/src/platform/nodejs/lib/util/bufferutils.ts @@ -70,14 +70,17 @@ class BufferUtils implements IBufferUtils { return Buffer.from(string, 'utf8'); } + sha256(message: Bufferlike): Output { + const messageBuffer = this.toBuffer(message); + + return crypto.createHash('SHA256').update(messageBuffer).digest(); + } + hmacSha256(message: Bufferlike, key: Bufferlike): Output { const messageBuffer = this.toBuffer(message); const keyBuffer = this.toBuffer(key); - const hmac = crypto.createHmac('SHA256', keyBuffer); - hmac.update(messageBuffer); - - return hmac.digest(); + return crypto.createHmac('SHA256', keyBuffer).update(messageBuffer).digest(); } } diff --git a/src/platform/web/lib/util/bufferutils.ts b/src/platform/web/lib/util/bufferutils.ts index 062e66351..bc42a273e 100644 --- a/src/platform/web/lib/util/bufferutils.ts +++ b/src/platform/web/lib/util/bufferutils.ts @@ -1,6 +1,6 @@ import Platform from 'common/platform'; import IBufferUtils from 'common/types/IBufferUtils'; -import { hmac as hmacSha256 } from './hmac-sha256'; +import { hmac as hmacSha256, sha256 } from './hmac-sha256'; /* Most BufferUtils methods that return a binary object return an ArrayBuffer * The exception is toBuffer, which returns a Uint8Array */ @@ -195,6 +195,11 @@ class BufferUtils implements IBufferUtils { return this.toArrayBuffer(arrayBufferView); } + sha256(message: Bufferlike): Output { + const hash = sha256(this.toBuffer(message)); + return this.toArrayBuffer(hash); + } + hmacSha256(message: Bufferlike, key: Bufferlike): Output { const hash = hmacSha256(this.toBuffer(key), this.toBuffer(message)); return this.toArrayBuffer(hash); diff --git a/src/platform/web/lib/util/hmac-sha256.ts b/src/platform/web/lib/util/hmac-sha256.ts index dd2ac7671..ac69b6ee1 100644 --- a/src/platform/web/lib/util/hmac-sha256.ts +++ b/src/platform/web/lib/util/hmac-sha256.ts @@ -102,7 +102,7 @@ function rightRotate(word: number, bits: number) { return (word >>> bits) | (word << (32 - bits)); } -function sha256(data: Uint8Array) { +export function sha256(data: Uint8Array): Uint8Array { // Copy default state var STATE = DEFAULT_STATE.slice(); @@ -185,7 +185,7 @@ function sha256(data: Uint8Array) { ); } -export function hmac(key: Uint8Array, data: Uint8Array) { +export function hmac(key: Uint8Array, data: Uint8Array): Uint8Array { if (key.length > 64) key = sha256(key); if (key.length < 64) { From 3707c6786b5dfb5b24f5cd9c539b575c769d214f Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 15 Jan 2025 23:32:20 +0000 Subject: [PATCH 03/18] Add `IBufferUtils.base64UrlEncode` method --- src/common/types/IBufferUtils.ts | 1 + src/platform/nodejs/lib/util/bufferutils.ts | 4 ++++ src/platform/web/lib/util/bufferutils.ts | 5 +++++ 3 files changed, 10 insertions(+) diff --git a/src/common/types/IBufferUtils.ts b/src/common/types/IBufferUtils.ts index 0ba48892e..8cd1d39aa 100644 --- a/src/common/types/IBufferUtils.ts +++ b/src/common/types/IBufferUtils.ts @@ -8,6 +8,7 @@ export default interface IBufferUtils { toBuffer: (buffer: Bufferlike) => ToBufferOutput; toArrayBuffer: (buffer: Bufferlike) => ArrayBuffer; base64Encode: (buffer: Bufferlike) => string; + base64UrlEncode: (buffer: Bufferlike) => string; base64Decode: (string: string) => Output; hexEncode: (buffer: Bufferlike) => string; hexDecode: (string: string) => Output; diff --git a/src/platform/nodejs/lib/util/bufferutils.ts b/src/platform/nodejs/lib/util/bufferutils.ts index 86baa4632..8c93f4ef3 100644 --- a/src/platform/nodejs/lib/util/bufferutils.ts +++ b/src/platform/nodejs/lib/util/bufferutils.ts @@ -17,6 +17,10 @@ class BufferUtils implements IBufferUtils { return this.toBuffer(buffer).toString('base64'); } + base64UrlEncode(buffer: Bufferlike): string { + return this.toBuffer(buffer).toString('base64url'); + } + areBuffersEqual(buffer1: Bufferlike, buffer2: Bufferlike): boolean { if (!buffer1 || !buffer2) return false; return this.toBuffer(buffer1).compare(this.toBuffer(buffer2)) == 0; diff --git a/src/platform/web/lib/util/bufferutils.ts b/src/platform/web/lib/util/bufferutils.ts index bc42a273e..1d7af7d69 100644 --- a/src/platform/web/lib/util/bufferutils.ts +++ b/src/platform/web/lib/util/bufferutils.ts @@ -116,6 +116,11 @@ class BufferUtils implements IBufferUtils { return this.uint8ViewToBase64(this.toBuffer(buffer)); } + base64UrlEncode(buffer: Bufferlike): string { + // base64url encoding is based on regular base64 with following changes: https://base64.guru/standards/base64url + return this.base64Encode(buffer).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); + } + base64Decode(str: string): Output { if (ArrayBuffer && Platform.Config.atob) { return this.base64ToArrayBuffer(str); From ea3fd7ac7931e04ad1950d27ec7b5501466ef13e Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 15 Jan 2025 23:17:55 +0000 Subject: [PATCH 04/18] Add `ObjectId.msTimestamp` property --- src/plugins/liveobjects/objectid.ts | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/plugins/liveobjects/objectid.ts b/src/plugins/liveobjects/objectid.ts index c968eefd6..e92db1dba 100644 --- a/src/plugins/liveobjects/objectid.ts +++ b/src/plugins/liveobjects/objectid.ts @@ -11,6 +11,7 @@ export class ObjectId { private constructor( readonly type: LiveObjectType, readonly hash: string, + readonly msTimestamp: number, ) {} /** @@ -21,8 +22,8 @@ export class ObjectId { throw new client.ErrorInfo('Invalid object id string', 50000, 500); } - const [type, hash] = objectId.split(':'); - if (!type || !hash) { + const [type, rest] = objectId.split(':'); + if (!type || !rest) { throw new client.ErrorInfo('Invalid object id string', 50000, 500); } @@ -30,6 +31,19 @@ export class ObjectId { throw new client.ErrorInfo(`Invalid object type in object id: ${objectId}`, 50000, 500); } - return new ObjectId(type as LiveObjectType, hash); + const [hash, msTimestamp] = rest.split('@'); + if (!hash || !msTimestamp) { + throw new client.ErrorInfo('Invalid object id string', 50000, 500); + } + + if (!Number.isInteger(Number.parseInt(msTimestamp))) { + throw new client.ErrorInfo('Invalid object id string', 50000, 500); + } + + return new ObjectId(type as LiveObjectType, hash, Number.parseInt(msTimestamp)); + } + + toString(): string { + return `${this.type}:${this.hash}@${this.msTimestamp}`; } } From 21f1c26d0d453956c6ecad1f0aa0a4a90ca53cf2 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 15 Jan 2025 23:33:57 +0000 Subject: [PATCH 05/18] Add `ObjectId.fromInitialValue` method --- src/plugins/liveobjects/objectid.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/plugins/liveobjects/objectid.ts b/src/plugins/liveobjects/objectid.ts index e92db1dba..eb10b3f1f 100644 --- a/src/plugins/liveobjects/objectid.ts +++ b/src/plugins/liveobjects/objectid.ts @@ -1,4 +1,5 @@ import type BaseClient from 'common/lib/client/baseclient'; +import type Platform from 'common/platform'; export type LiveObjectType = 'map' | 'counter'; @@ -14,6 +15,21 @@ export class ObjectId { readonly msTimestamp: number, ) {} + static fromInitialValue( + platform: typeof Platform, + objectType: LiveObjectType, + encodedInitialValue: string, + nonce: string, + msTimestamp: number, + ): ObjectId { + const valueForHashBuffer = platform.BufferUtils.utf8Encode(`${encodedInitialValue}:${nonce}`); + const hashBuffer = platform.BufferUtils.sha256(valueForHashBuffer); + + const hash = platform.BufferUtils.base64UrlEncode(hashBuffer); + + return new ObjectId(objectType, hash, msTimestamp); + } + /** * Create ObjectId instance from hashed object id string. */ From c8784a6482e69df5993a7c72adee916df2a94bd2 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 05:11:48 +0000 Subject: [PATCH 06/18] Fix incorrect buffer type for `StateValue` --- src/plugins/liveobjects/statemessage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 1a1ca9fb1..573bba72b 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -21,7 +21,7 @@ export enum MapSemantics { } /** A StateValue represents a concrete leaf value in a state object graph. */ -export type StateValue = string | number | boolean | Buffer | Uint8Array; +export type StateValue = string | number | boolean | Buffer | ArrayBuffer; /** StateData captures a value in a state object. */ export interface StateData { From 6f5b171e8e15c9c276b41b02739cac82b30652ac Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 00:52:47 +0000 Subject: [PATCH 07/18] Add `initialValue` and `initialValueEncoding` to `StateMessage` --- src/plugins/liveobjects/statemessage.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 573bba72b..4bf40c8e9 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -104,6 +104,15 @@ export interface StateOperation { * that has been hashed with the type and initial value to create the object ID. */ nonce?: string; + /** + * The initial value bytes for the object. These bytes should be used along with the nonce + * and timestamp to create the object ID. Frontdoor will use this to verify the object ID. + * After verification the bytes will be decoded into the Map or Counter objects and + * the initialValue, nonce, and initialValueEncoding will be removed. + */ + initialValue?: Buffer | ArrayBuffer; + /** The initial value encoding defines how the initialValue should be interpreted. */ + initialValueEncoding?: string; } /** A StateObject describes the instantaneous state of an object. */ From 7fdbe7379e0f758016f6cd27c9c10b9e0c7d2e3d Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 00:53:05 +0000 Subject: [PATCH 08/18] Add `StateMessage.encodeInitialValue` method --- src/plugins/liveobjects/statemessage.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 4bf40c8e9..f2564d597 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -241,6 +241,19 @@ export class StateMessage { return result; } + static encodeInitialValue( + utils: typeof Utils, + initialValue: Partial, + ): { + encodedInitialValue: string; + format: Utils.Format; + } { + return { + encodedInitialValue: JSON.stringify(initialValue), + format: utils.Format.json, + }; + } + private static async _decodeMapEntries( mapEntries: Record, inputContext: ChannelOptions, From d9d8c385eeed3e68f89f0f9f014c976733fa5ee6 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 01:00:50 +0000 Subject: [PATCH 09/18] Update `LiveObjectsPool.createZeroValueObjectIfNotExists` to also return created object --- src/plugins/liveobjects/liveobjectspool.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index 94f667fdb..428bdf867 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -48,9 +48,10 @@ export class LiveObjectsPool { this._pool = this._getInitialPool(); } - createZeroValueObjectIfNotExists(objectId: string): void { - if (this.get(objectId)) { - return; + createZeroValueObjectIfNotExists(objectId: string): LiveObject { + const existingObject = this.get(objectId); + if (existingObject) { + return existingObject; } const parsedObjectId = ObjectId.fromString(this._client, objectId); @@ -67,6 +68,7 @@ export class LiveObjectsPool { } this.set(objectId, zeroValueObject); + return zeroValueObject; } private _getInitialPool(): Map { From ee0ff57d0d1e3f7d45a444d32abec604f664dcc6 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 01:03:36 +0000 Subject: [PATCH 10/18] Refactor `StateMessage` creation in Live Objects --- src/plugins/liveobjects/livecounter.ts | 48 ++++---- src/plugins/liveobjects/livemap.ts | 158 ++++++++++++++----------- 2 files changed, 117 insertions(+), 89 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 843203583..a681f6087 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -32,47 +32,49 @@ export class LiveCounter extends LiveObject return obj; } - value(): number { - return this._dataRef.data; - } - - /** - * Send a COUNTER_INC operation to the realtime system to increment a value on this LiveCounter object. - * - * This does not modify the underlying data of this LiveCounter object. Instead, the change will be applied when - * the published COUNTER_INC operation is echoed back to the client and applied to the object following the regular - * operation application procedure. - * - * @returns A promise which resolves upon receiving the ACK message for the published operation message. - */ - async increment(amount: number): Promise { - const stateMessage = this.createCounterIncMessage(amount); - return this._liveObjects.publish([stateMessage]); - } - /** * @internal */ - createCounterIncMessage(amount: number): StateMessage { + static createCounterIncMessage(liveObjects: LiveObjects, objectId: string, amount: number): StateMessage { + const client = liveObjects.getClient(); + if (typeof amount !== 'number' || !isFinite(amount)) { - throw new this._client.ErrorInfo('Counter value increment should be a valid number', 40013, 400); + throw new client.ErrorInfo('Counter value increment should be a valid number', 40013, 400); } const stateMessage = StateMessage.fromValues( { operation: { action: StateOperationAction.COUNTER_INC, - objectId: this.getObjectId(), + objectId, counterOp: { amount }, }, }, - this._client.Utils, - this._client.MessageEncoding, + client.Utils, + client.MessageEncoding, ); return stateMessage; } + value(): number { + return this._dataRef.data; + } + + /** + * Send a COUNTER_INC operation to the realtime system to increment a value on this LiveCounter object. + * + * This does not modify the underlying data of this LiveCounter object. Instead, the change will be applied when + * the published COUNTER_INC operation is echoed back to the client and applied to the object following the regular + * operation application procedure. + * + * @returns A promise which resolves upon receiving the ACK message for the published operation message. + */ + async increment(amount: number): Promise { + const stateMessage = LiveCounter.createCounterIncMessage(this._liveObjects, this.getObjectId(), amount); + return this._liveObjects.publish([stateMessage]); + } + /** * Alias for calling {@link LiveCounter.increment | LiveCounter.increment(-amount)} */ diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index e12c9a1c4..3a6efe90f 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -80,6 +80,96 @@ export class LiveMap extends LiveObject( + liveObjects: LiveObjects, + objectId: string, + key: TKey, + value: API.LiveMapType[TKey], + ): StateMessage { + const client = liveObjects.getClient(); + + LiveMap.validateKeyValue(liveObjects, key, value); + + const stateData: StateData = + value instanceof LiveObject + ? ({ objectId: value.getObjectId() } as ObjectIdStateData) + : ({ value } as ValueStateData); + + const stateMessage = StateMessage.fromValues( + { + operation: { + action: StateOperationAction.MAP_SET, + objectId, + mapOp: { + key, + data: stateData, + }, + }, + }, + client.Utils, + client.MessageEncoding, + ); + + return stateMessage; + } + + /** + * @internal + */ + static createMapRemoveMessage( + liveObjects: LiveObjects, + objectId: string, + key: TKey, + ): StateMessage { + const client = liveObjects.getClient(); + + if (typeof key !== 'string') { + throw new client.ErrorInfo('Map key should be string', 40013, 400); + } + + const stateMessage = StateMessage.fromValues( + { + operation: { + action: StateOperationAction.MAP_REMOVE, + objectId, + mapOp: { key }, + }, + }, + client.Utils, + client.MessageEncoding, + ); + + return stateMessage; + } + + /** + * @internal + */ + static validateKeyValue( + liveObjects: LiveObjects, + key: TKey, + value: API.LiveMapType[TKey], + ): void { + const client = liveObjects.getClient(); + + if (typeof key !== 'string') { + throw new client.ErrorInfo('Map key should be string', 40013, 400); + } + + if ( + typeof value !== 'string' && + typeof value !== 'number' && + typeof value !== 'boolean' && + !client.Platform.BufferUtils.isBuffer(value) && + !(value instanceof LiveObject) + ) { + throw new client.ErrorInfo('Map value data type is unsupported', 40013, 400); + } + } + /** * Returns the value associated with the specified key in the underlying Map object. * @@ -163,51 +253,10 @@ export class LiveMap extends LiveObject(key: TKey, value: T[TKey]): Promise { - const stateMessage = this.createMapSetMessage(key, value); + const stateMessage = LiveMap.createMapSetMessage(this._liveObjects, this.getObjectId(), key, value); return this._liveObjects.publish([stateMessage]); } - /** - * @internal - */ - createMapSetMessage(key: TKey, value: T[TKey]): StateMessage { - if (typeof key !== 'string') { - throw new this._client.ErrorInfo('Map key should be string', 40013, 400); - } - - if ( - typeof value !== 'string' && - typeof value !== 'number' && - typeof value !== 'boolean' && - !this._client.Platform.BufferUtils.isBuffer(value) && - !(value instanceof LiveObject) - ) { - throw new this._client.ErrorInfo('Map value data type is unsupported', 40013, 400); - } - - const stateData: StateData = - value instanceof LiveObject - ? ({ objectId: value.getObjectId() } as ObjectIdStateData) - : ({ value } as ValueStateData); - - const stateMessage = StateMessage.fromValues( - { - operation: { - action: StateOperationAction.MAP_SET, - objectId: this.getObjectId(), - mapOp: { - key, - data: stateData, - }, - }, - }, - this._client.Utils, - this._client.MessageEncoding, - ); - - return stateMessage; - } - /** * Send a MAP_REMOVE operation to the realtime system to tombstone a key on this LiveMap object. * @@ -218,33 +267,10 @@ export class LiveMap extends LiveObject(key: TKey): Promise { - const stateMessage = this.createMapRemoveMessage(key); + const stateMessage = LiveMap.createMapRemoveMessage(this._liveObjects, this.getObjectId(), key); return this._liveObjects.publish([stateMessage]); } - /** - * @internal - */ - createMapRemoveMessage(key: TKey): StateMessage { - if (typeof key !== 'string') { - throw new this._client.ErrorInfo('Map key should be string', 40013, 400); - } - - const stateMessage = StateMessage.fromValues( - { - operation: { - action: StateOperationAction.MAP_REMOVE, - objectId: this.getObjectId(), - mapOp: { key }, - }, - }, - this._client.Utils, - this._client.MessageEncoding, - ); - - return stateMessage; - } - /** * @internal */ From 5d5874dc6cd1a94a0a922c062447873f80d394fc Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 01:28:14 +0000 Subject: [PATCH 11/18] Move `getTimestamp` and related methods from `Auth` to `BaseClient` --- src/common/lib/client/auth.ts | 33 ++++++++--------------------- src/common/lib/client/baseclient.ts | 22 +++++++++++++++++++ 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/src/common/lib/client/auth.ts b/src/common/lib/client/auth.ts index 67e7704f2..d457ce10c 100644 --- a/src/common/lib/client/auth.ts +++ b/src/common/lib/client/auth.ts @@ -775,7 +775,7 @@ class Auth { capability = tokenParams.capability || ''; if (!request.timestamp) { - request.timestamp = await this.getTimestamp(authOptions && authOptions.queryTime); + request.timestamp = await this._getTimestamp(authOptions && authOptions.queryTime); } /* nonce */ @@ -832,28 +832,6 @@ class Auth { } } - /** - * Get the current time based on the local clock, - * or if the option queryTime is true, return the server time. - * The server time offset from the local time is stored so that - * only one request to the server to get the time is ever needed - */ - async getTimestamp(queryTime: boolean): Promise { - if (!this.isTimeOffsetSet() && (queryTime || this.authOptions.queryTime)) { - return this.client.time(); - } else { - return this.getTimestampUsingOffset(); - } - } - - getTimestampUsingOffset() { - return Date.now() + (this.client.serverTimeOffset || 0); - } - - isTimeOffsetSet() { - return this.client.serverTimeOffset !== null; - } - _saveBasicOptions(authOptions: AuthOptions) { this.method = 'basic'; this.key = authOptions.key; @@ -913,7 +891,7 @@ class Auth { /* RSA4b1 -- if we have a server time offset set already, we can * automatically remove expired tokens. Else just use the cached token. If it is * expired Ably will tell us and we'll discard it then. */ - if (!this.isTimeOffsetSet() || !token.expires || token.expires >= this.getTimestampUsingOffset()) { + if (!this.client.isTimeOffsetSet() || !token.expires || token.expires >= this.client.getTimestampUsingOffset()) { Logger.logAction( this.logger, Logger.LOG_MINOR, @@ -1020,6 +998,13 @@ class Auth { ): Promise { return this.client.rest.revokeTokens(specifiers, options); } + + /** + * Same as {@link BaseClient.getTimestamp} but also takes into account {@link Auth.authOptions} + */ + private async _getTimestamp(queryTime: boolean): Promise { + return this.client.getTimestamp(queryTime || !!this.authOptions.queryTime); + } } export default Auth; diff --git a/src/common/lib/client/baseclient.ts b/src/common/lib/client/baseclient.ts index c79677b65..b9fa36da4 100644 --- a/src/common/lib/client/baseclient.ts +++ b/src/common/lib/client/baseclient.ts @@ -171,6 +171,28 @@ class BaseClient { this.logger.setLog(logOptions.level, logOptions.handler); } + /** + * Get the current time based on the local clock, + * or if the option queryTime is true, return the server time. + * The server time offset from the local time is stored so that + * only one request to the server to get the time is ever needed + */ + async getTimestamp(queryTime: boolean): Promise { + if (!this.isTimeOffsetSet() && queryTime) { + return this.time(); + } + + return this.getTimestampUsingOffset(); + } + + getTimestampUsingOffset(): number { + return Date.now() + (this.serverTimeOffset || 0); + } + + isTimeOffsetSet(): boolean { + return this.serverTimeOffset !== null; + } + static Platform = Platform; /** From 5b70e79db2a4d512164b4ea898c40dbee9251a1e Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 01:06:05 +0000 Subject: [PATCH 12/18] Add object-level write API for LiveMap and LiveCounter creation Resolves DTP-1138 --- src/plugins/liveobjects/livecounter.ts | 65 ++++++++++++++++++++ src/plugins/liveobjects/livemap.ts | 84 ++++++++++++++++++++++++++ src/plugins/liveobjects/liveobject.ts | 5 -- src/plugins/liveobjects/liveobjects.ts | 60 ++++++++++++++++++ 4 files changed, 209 insertions(+), 5 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index a681f6087..6c08f3d88 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -1,5 +1,6 @@ import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { LiveObjects } from './liveobjects'; +import { ObjectId } from './objectid'; import { StateCounterOp, StateMessage, StateObject, StateOperation, StateOperationAction } from './statemessage'; export interface LiveCounterData extends LiveObjectData { @@ -32,6 +33,18 @@ export class LiveCounter extends LiveObject return obj; } + /** + * Returns a {@link LiveCounter} instance based on the provided state operation. + * The provided state operation must hold a valid counter object data. + * + * @internal + */ + static fromStateOperation(liveobjects: LiveObjects, stateOperation: StateOperation): LiveCounter { + const obj = new LiveCounter(liveobjects, stateOperation.objectId); + obj._mergeInitialDataFromCreateOperation(stateOperation); + return obj; + } + /** * @internal */ @@ -57,6 +70,58 @@ export class LiveCounter extends LiveObject return stateMessage; } + /** + * @internal + */ + static async createCounterCreateMessage(liveObjects: LiveObjects, count?: number): Promise { + const client = liveObjects.getClient(); + + if (count !== undefined && (typeof count !== 'number' || !Number.isFinite(count))) { + throw new client.ErrorInfo('Counter value should be a valid number', 40013, 400); + } + + const initialValueObj = LiveCounter.createInitialValueObject(count); + const { encodedInitialValue, format } = StateMessage.encodeInitialValue(client.Utils, initialValueObj); + const nonce = client.Utils.cheapRandStr(); + const msTimestamp = await client.getTimestamp(true); + + const objectId = ObjectId.fromInitialValue( + client.Platform, + 'counter', + encodedInitialValue, + nonce, + msTimestamp, + ).toString(); + + const stateMessage = StateMessage.fromValues( + { + operation: { + ...initialValueObj, + action: StateOperationAction.COUNTER_CREATE, + objectId, + nonce, + initialValue: encodedInitialValue, + initialValueEncoding: format, + }, + }, + client.Utils, + client.MessageEncoding, + ); + + return stateMessage; + } + + /** + * @internal + */ + static createInitialValueObject(count?: number): Pick { + return { + counter: { + count: count ?? 0, + }, + }; + } + value(): number { return this._dataRef.data; } diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 3a6efe90f..8b3fc41ef 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -4,6 +4,7 @@ import type * as API from '../../../ably'; import { DEFAULTS } from './defaults'; import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { LiveObjects } from './liveobjects'; +import { ObjectId } from './objectid'; import { MapSemantics, StateMapEntry, @@ -80,6 +81,21 @@ export class LiveMap extends LiveObject( + liveobjects: LiveObjects, + stateOperation: StateOperation, + ): LiveMap { + const obj = new LiveMap(liveobjects, stateOperation.map?.semantics!, stateOperation.objectId); + obj._mergeInitialDataFromCreateOperation(stateOperation); + return obj; + } + /** * @internal */ @@ -170,6 +186,74 @@ export class LiveMap extends LiveObject { + const client = liveObjects.getClient(); + + if (entries !== undefined && (entries === null || typeof entries !== 'object')) { + throw new client.ErrorInfo('Map entries should be a key/value object', 40013, 400); + } + + Object.entries(entries ?? {}).forEach(([key, value]) => LiveMap.validateKeyValue(liveObjects, key, value)); + + const initialValueObj = LiveMap.createInitialValueObject(entries); + const { encodedInitialValue, format } = StateMessage.encodeInitialValue(client.Utils, initialValueObj); + const nonce = client.Utils.cheapRandStr(); + const msTimestamp = await client.getTimestamp(true); + + const objectId = ObjectId.fromInitialValue( + client.Platform, + 'map', + encodedInitialValue, + nonce, + msTimestamp, + ).toString(); + + const stateMessage = StateMessage.fromValues( + { + operation: { + ...initialValueObj, + action: StateOperationAction.MAP_CREATE, + objectId, + nonce, + initialValue: encodedInitialValue, + initialValueEncoding: format, + }, + }, + client.Utils, + client.MessageEncoding, + ); + + return stateMessage; + } + + /** + * @internal + */ + static createInitialValueObject(entries?: API.LiveMapType): Pick { + const stateMapEntries: Record = {}; + + Object.entries(entries ?? {}).forEach(([key, value]) => { + const stateData: StateData = + value instanceof LiveObject + ? ({ objectId: value.getObjectId() } as ObjectIdStateData) + : ({ value } as ValueStateData); + + stateMapEntries[key] = { + data: stateData, + }; + }); + + return { + map: { + semantics: MapSemantics.LWW, + entries: stateMapEntries, + }, + }; + } + /** * Returns the value associated with the specified key in the underlying Map object. * diff --git a/src/plugins/liveobjects/liveobject.ts b/src/plugins/liveobjects/liveobject.ts index 1d5f84bf1..2fe39596f 100644 --- a/src/plugins/liveobjects/liveobject.ts +++ b/src/plugins/liveobjects/liveobject.ts @@ -161,11 +161,6 @@ export abstract class LiveObject< return this._updateFromDataDiff(previousDataRef, this._dataRef); } - private _createObjectId(): string { - // TODO: implement object id generation based on live object type and initial value - return Math.random().toString().substring(2); - } - /** * Apply state operation message on live object. * diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index c3c003c72..0854a38ea 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -54,6 +54,66 @@ export class LiveObjects { return this._liveObjectsPool.get(ROOT_OBJECT_ID) as LiveMap; } + /** + * Send a MAP_CREATE operation to the realtime system to create a new map object in the pool. + * + * Locally on the client it creates a zero-value object with the corresponding id and returns it. + * The object initialization with the initial value is expected to happen when the corresponding MAP_CREATE operation is echoed + * back to the client and applied to the object following the regular operation application procedure. + * + * @returns A promise which resolves upon receiving the ACK message for the published operation message. A promise is resolved with a zero-value object created in the local pool. + */ + async createMap(entries?: T): Promise> { + const stateMessage = await LiveMap.createMapCreateMessage(this, entries); + const objectId = stateMessage.operation?.objectId!; + + await this.publish([stateMessage]); + + // we might have received CREATE operation already at this point (it might arrive before the ACK message for our publish message), + // so the object could exist in the local pool as it was added there during regular CREATE operation application. + // check if the object is there and return it in case it is. otherwise create a new object client-side + if (this._liveObjectsPool.get(objectId)) { + return this._liveObjectsPool.get(objectId) as LiveMap; + } + + // new map object can be created using locally constructed state operation, even though it is missing timeserials for map entries. + // CREATE operation is only applied once, and all map entries will have an "earliest possible" timeserial so that any subsequent operation can be applied to them. + const map = LiveMap.fromStateOperation(this, stateMessage.operation!); + this._liveObjectsPool.set(objectId, map); + + return map; + } + + /** + * Send a COUNTER_CREATE operation to the realtime system to create a new counter object in the pool. + * + * Locally on the client it creates a zero-value object with the corresponding id and returns it. + * The object initialization with the initial value is expected to happen when the corresponding COUNTER_CREATE operation is echoed + * back to the client and applied to the object following the regular operation application procedure. + * + * @returns A promise which resolves upon receiving the ACK message for the published operation message. A promise is resolved with a zero-value object created in the local pool. + */ + async createCounter(count?: number): Promise { + const stateMessage = await LiveCounter.createCounterCreateMessage(this, count); + const objectId = stateMessage.operation?.objectId!; + + await this.publish([stateMessage]); + + // we might have received CREATE operation already at this point (it might arrive before the ACK message for our publish message), + // so the object could exist in the local pool as it was added there during regular CREATE operation application. + // check if the object is there and return it in case it is. otherwise create a new object client-side + if (this._liveObjectsPool.get(objectId)) { + return this._liveObjectsPool.get(objectId) as LiveCounter; + } + + // new counter object can be created using locally constructed state operation. + // CREATE operation is only applied once, so the initial counter value won't be double counted when we eventually receive an echoed CREATE operation + const counter = LiveCounter.fromStateOperation(this, stateMessage.operation!); + this._liveObjectsPool.set(objectId, counter); + + return counter; + } + /** * @internal */ From 77e0c3f75e37af248ebf7bcd72cb6e823a33462e Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 05:02:48 +0000 Subject: [PATCH 13/18] Add tests for LiveMap/LiveCounter creation --- test/common/modules/private_api_recorder.js | 1 + test/realtime/live_objects.test.js | 444 +++++++++++++++++++- 2 files changed, 437 insertions(+), 8 deletions(-) diff --git a/test/common/modules/private_api_recorder.js b/test/common/modules/private_api_recorder.js index 74158c910..13ce8f9c9 100644 --- a/test/common/modules/private_api_recorder.js +++ b/test/common/modules/private_api_recorder.js @@ -117,6 +117,7 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'read.transport.recvRequest.recvUri', 'read.transport.uri', 'replace.LiveObjects._liveObjectsPool._onGCInterval', + 'replace.LiveObjects.publish', 'replace.channel.attachImpl', 'replace.channel.processMessage', 'replace.channel.sendMessage', diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 0d60a6c19..b21acf116 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -714,29 +714,29 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // check created maps primitiveMapsFixtures.forEach((fixture) => { - const key = fixture.name; - const mapObj = root.get(key); + const mapKey = fixture.name; + const mapObj = root.get(mapKey); // check all maps exist on root - expect(mapObj, `Check map at "${key}" key in root exists`).to.exist; - expectInstanceOf(mapObj, 'LiveMap', `Check map at "${key}" key in root is of type LiveMap`); + expect(mapObj, `Check map at "${mapKey}" key in root exists`).to.exist; + expectInstanceOf(mapObj, 'LiveMap', `Check map at "${mapKey}" key in root is of type LiveMap`); // check primitive maps have correct values expect(mapObj.size()).to.equal( Object.keys(fixture.entries ?? {}).length, - `Check map "${key}" has correct number of keys`, + `Check map "${mapKey}" has correct number of keys`, ); Object.entries(fixture.entries ?? {}).forEach(([key, keyData]) => { if (keyData.data.encoding) { expect( BufferUtils.areBuffersEqual(mapObj.get(key), BufferUtils.base64Decode(keyData.data.value)), - `Check map "${key}" has correct value for "${key}" key`, + `Check map "${mapKey}" has correct value for "${key}" key`, ).to.be.true; } else { expect(mapObj.get(key)).to.equal( keyData.data.value, - `Check map "${key}" has correct value for "${key}" key`, + `Check map "${mapKey}" has correct value for "${key}" key`, ); } }); @@ -2425,6 +2425,434 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await expectRejectedWith(async () => map.remove(map), 'Map key should be string'); }, }, + + { + description: 'LiveObjects.createCounter sends COUNTER_CREATE operation', + action: async (ctx) => { + const { liveObjects } = ctx; + + const counters = await Promise.all(countersFixtures.map(async (x) => liveObjects.createCounter(x.count))); + + for (let i = 0; i < counters.length; i++) { + const counter = counters[i]; + const fixture = countersFixtures[i]; + + expect(counter, `Check counter #${i + 1} exists`).to.exist; + expectInstanceOf(counter, 'LiveCounter', `Check counter instance #${i + 1} is of an expected class`); + expect(counter.value()).to.equal( + fixture.count ?? 0, + `Check counter #${i + 1} has expected initial value`, + ); + } + }, + }, + + { + description: 'LiveCounter created with LiveObjects.createCounter can be assigned to the state tree', + action: async (ctx) => { + const { root, liveObjects } = ctx; + + const counter = await liveObjects.createCounter(1); + await root.set('counter', counter); + + expectInstanceOf(counter, 'LiveCounter', `Check counter instance is of an expected class`); + expectInstanceOf( + root.get('counter'), + 'LiveCounter', + `Check counter instance on root is of an expected class`, + ); + expect(root.get('counter')).to.equal( + counter, + 'Check counter object on root is the same as from create method', + ); + expect(root.get('counter').value()).to.equal( + 1, + 'Check counter assigned to the state tree has the expected value', + ); + }, + }, + + { + description: + 'LiveObjects.createCounter can return LiveCounter with initial value without applying CREATE operation', + action: async (ctx) => { + const { liveObjects, helper } = ctx; + + // prevent publishing of ops to realtime so we guarantee that the initial value doesn't come from a CREATE op + helper.recordPrivateApi('replace.LiveObjects.publish'); + liveObjects.publish = () => {}; + + const counter = await liveObjects.createCounter(1); + expect(counter.value()).to.equal(1, `Check counter has expected initial value`); + }, + }, + + { + description: + 'LiveObjects.createCounter can return LiveCounter with initial value from applied CREATE operation', + action: async (ctx) => { + const { liveObjects, liveObjectsHelper, helper, channel } = ctx; + + // instead of sending CREATE op to the realtime, echo it immediately to the client + // with forged initial value so we can check that counter gets initialized with a value from a CREATE op + helper.recordPrivateApi('replace.LiveObjects.publish'); + liveObjects.publish = async (stateMessages) => { + const counterId = stateMessages[0].operation.objectId; + // this should result in liveobjects' operation application procedure and create a object in the pool with forged initial value + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 1), + siteCode: 'aaa', + state: [liveObjectsHelper.counterCreateOp({ objectId: counterId, count: 10 })], + }); + }; + + const counter = await liveObjects.createCounter(1); + + // counter should be created with forged initial value instead of the actual one + expect(counter.value()).to.equal( + 10, + 'Check counter value has the expected initial value from a CREATE operation', + ); + }, + }, + + { + description: + 'Initial value is not double counted for LiveCounter from LiveObjects.createCounter when CREATE op is received', + action: async (ctx) => { + const { liveObjects, liveObjectsHelper, helper, channel } = ctx; + + // prevent publishing of ops to realtime so we can guarantee order of operations + helper.recordPrivateApi('replace.LiveObjects.publish'); + liveObjects.publish = () => {}; + + // create counter locally, should have an initial value set + const counter = await liveObjects.createCounter(1); + helper.recordPrivateApi('call.LiveObject.getObjectId'); + const counterId = counter.getObjectId(); + + // now inject CREATE op for a counter with a forged value. it should not be applied + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 1), + siteCode: 'aaa', + state: [liveObjectsHelper.counterCreateOp({ objectId: counterId, count: 10 })], + }); + + expect(counter.value()).to.equal( + 1, + `Check counter initial value is not double counted after being created and receiving CREATE operation`, + ); + }, + }, + + { + description: 'LiveObjects.createCounter throws on invalid input', + action: async (ctx) => { + const { root, liveObjects } = ctx; + + await expectRejectedWith( + async () => liveObjects.createCounter(null), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter(Number.NaN), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter(Number.POSITIVE_INFINITY), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter(Number.NEGATIVE_INFINITY), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter('foo'), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter(BigInt(1)), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter(true), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter(Symbol()), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter({}), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter([]), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter(root), + 'Counter value should be a valid number', + ); + }, + }, + + { + description: 'LiveObjects.createMap sends MAP_CREATE operation with primitive values', + action: async (ctx) => { + const { liveObjects } = ctx; + + const maps = await Promise.all( + primitiveMapsFixtures.map(async (mapFixture) => { + const entries = mapFixture.entries + ? Object.entries(mapFixture.entries).reduce((acc, [key, keyData]) => { + const value = keyData.data.encoding + ? BufferUtils.base64Decode(keyData.data.value) + : keyData.data.value; + acc[key] = value; + return acc; + }, {}) + : undefined; + + return liveObjects.createMap(entries); + }), + ); + + for (let i = 0; i < maps.length; i++) { + const map = maps[i]; + const fixture = primitiveMapsFixtures[i]; + + expect(map, `Check map #${i + 1} exists`).to.exist; + expectInstanceOf(map, 'LiveMap', `Check map instance #${i + 1} is of an expected class`); + + expect(map.size()).to.equal( + Object.keys(fixture.entries ?? {}).length, + `Check map #${i + 1} has correct number of keys`, + ); + + Object.entries(fixture.entries ?? {}).forEach(([key, keyData]) => { + if (keyData.data.encoding) { + expect( + BufferUtils.areBuffersEqual(map.get(key), BufferUtils.base64Decode(keyData.data.value)), + `Check map #${i + 1} has correct value for "${key}" key`, + ).to.be.true; + } else { + expect(map.get(key)).to.equal( + keyData.data.value, + `Check map #${i + 1} has correct value for "${key}" key`, + ); + } + }); + } + }, + }, + + { + description: 'LiveObjects.createMap sends MAP_CREATE operation with reference to another LiveObject', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, liveObjects } = ctx; + + await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: liveObjectsHelper.counterCreateOp(), + }); + await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map', + createOp: liveObjectsHelper.mapCreateOp(), + }); + + const counter = root.get('counter'); + const map = root.get('map'); + + const newMap = await liveObjects.createMap({ counter, map }); + + expect(newMap, 'Check map exists').to.exist; + expectInstanceOf(newMap, 'LiveMap', 'Check map instance is of an expected class'); + + expect(newMap.get('counter')).to.equal( + counter, + 'Check can set a reference to a LiveCounter object on a new map via a MAP_CREATE operation', + ); + expect(newMap.get('map')).to.equal( + map, + 'Check can set a reference to a LiveMap object on a new map via a MAP_CREATE operation', + ); + }, + }, + + { + description: 'LiveMap created with LiveObjects.createMap can be assigned to the state tree', + action: async (ctx) => { + const { root, liveObjects } = ctx; + + const counter = await liveObjects.createCounter(); + const map = await liveObjects.createMap({ foo: 'bar', baz: counter }); + await root.set('map', map); + + expectInstanceOf(map, 'LiveMap', `Check map instance is of an expected class`); + expectInstanceOf(root.get('map'), 'LiveMap', `Check map instance on root is of an expected class`); + expect(root.get('map')).to.equal(map, 'Check map object on root is the same as from create method'); + expect(root.get('map').size()).to.equal( + 2, + 'Check map assigned to the state tree has the expected number of keys', + ); + expect(root.get('map').get('foo')).to.equal( + 'bar', + 'Check map assigned to the state tree has the expected value for its string key', + ); + expect(root.get('map').get('baz')).to.equal( + counter, + 'Check map assigned to the state tree has the expected value for its LiveCounter key', + ); + }, + }, + + { + description: 'LiveObjects.createMap can return LiveMap with initial value without applying CREATE operation', + action: async (ctx) => { + const { liveObjects, helper } = ctx; + + // prevent publishing of ops to realtime so we guarantee that the initial value doesn't come from a CREATE op + helper.recordPrivateApi('replace.LiveObjects.publish'); + liveObjects.publish = () => {}; + + const map = await liveObjects.createMap({ foo: 'bar' }); + expect(map.get('foo')).to.equal('bar', `Check map has expected initial value`); + }, + }, + + { + description: 'LiveObjects.createMap can return LiveMap with initial value from applied CREATE operation', + action: async (ctx) => { + const { liveObjects, liveObjectsHelper, helper, channel } = ctx; + + // instead of sending CREATE op to the realtime, echo it immediately to the client + // with forged initial value so we can check that map gets initialized with a value from a CREATE op + helper.recordPrivateApi('replace.LiveObjects.publish'); + liveObjects.publish = async (stateMessages) => { + const mapId = stateMessages[0].operation.objectId; + // this should result in liveobjects' operation application procedure and create a object in the pool with forged initial value + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 1), + siteCode: 'aaa', + state: [ + liveObjectsHelper.mapCreateOp({ + objectId: mapId, + entries: { baz: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { value: 'qux' } } }, + }), + ], + }); + }; + + const map = await liveObjects.createMap({ foo: 'bar' }); + + // map should be created with forged initial value instead of the actual one + expect(map.get('foo'), `Check key "foo" was not set on a map client-side`).to.not.exist; + expect(map.get('baz')).to.equal( + 'qux', + `Check key "baz" was set on a map from a CREATE operation after object creation`, + ); + }, + }, + + { + description: + 'Initial value is not double counted for LiveMap from LiveObjects.createMap when CREATE op is received', + action: async (ctx) => { + const { liveObjects, liveObjectsHelper, helper, channel } = ctx; + + // prevent publishing of ops to realtime so we can guarantee order of operations + helper.recordPrivateApi('replace.LiveObjects.publish'); + liveObjects.publish = () => {}; + + // create map locally, should have an initial value set + const map = await liveObjects.createMap({ foo: 'bar' }); + helper.recordPrivateApi('call.LiveObject.getObjectId'); + const mapId = map.getObjectId(); + + // now inject CREATE op for a map with a forged value. it should not be applied + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 1), + siteCode: 'aaa', + state: [ + liveObjectsHelper.mapCreateOp({ + objectId: mapId, + entries: { + foo: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { value: 'qux' } }, + baz: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { value: 'qux' } }, + }, + }), + ], + }); + + expect(map.get('foo')).to.equal( + 'bar', + `Check key "foo" was not overridden by a CREATE operation after creating a map locally`, + ); + expect(map.get('baz'), `Check key "baz" was not set by a CREATE operation after creating a map locally`).to + .not.exist; + }, + }, + + { + description: 'LiveObjects.createMap throws on invalid input', + action: async (ctx) => { + const { root, liveObjects } = ctx; + + await expectRejectedWith( + async () => liveObjects.createMap(null), + 'Map entries should be a key/value object', + ); + await expectRejectedWith( + async () => liveObjects.createMap('foo'), + 'Map entries should be a key/value object', + ); + await expectRejectedWith(async () => liveObjects.createMap(1), 'Map entries should be a key/value object'); + await expectRejectedWith( + async () => liveObjects.createMap(BigInt(1)), + 'Map entries should be a key/value object', + ); + await expectRejectedWith( + async () => liveObjects.createMap(true), + 'Map entries should be a key/value object', + ); + await expectRejectedWith( + async () => liveObjects.createMap(Symbol()), + 'Map entries should be a key/value object', + ); + + await expectRejectedWith( + async () => liveObjects.createMap({ key: undefined }), + 'Map value data type is unsupported', + ); + await expectRejectedWith( + async () => liveObjects.createMap({ key: null }), + 'Map value data type is unsupported', + ); + await expectRejectedWith( + async () => liveObjects.createMap({ key: BigInt(1) }), + 'Map value data type is unsupported', + ); + await expectRejectedWith( + async () => liveObjects.createMap({ key: Symbol() }), + 'Map value data type is unsupported', + ); + await expectRejectedWith( + async () => liveObjects.createMap({ key: {} }), + 'Map value data type is unsupported', + ); + await expectRejectedWith( + async () => liveObjects.createMap({ key: [] }), + 'Map value data type is unsupported', + ); + }, + }, ]; /** @nospec */ @@ -2447,7 +2875,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await channel.attach(); const root = await liveObjects.getRoot(); - await scenario.action({ root, liveObjectsHelper, channelName, channel }); + await scenario.action({ liveObjects, root, liveObjectsHelper, channelName, channel, client, helper }); }, client); }, ); From 4a33d69fc3dfdb9b8461a603657f6bed5d560669 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 06:33:47 +0000 Subject: [PATCH 14/18] Update minimal bundle size --- scripts/moduleReport.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/moduleReport.ts b/scripts/moduleReport.ts index 8d1921789..e1b67a8b2 100644 --- a/scripts/moduleReport.ts +++ b/scripts/moduleReport.ts @@ -6,7 +6,7 @@ import { gzip } from 'zlib'; import Table from 'cli-table'; // The maximum size we allow for a minimal useful Realtime bundle (i.e. one that can subscribe to a channel) -const minimalUsefulRealtimeBundleSizeThresholdsKiB = { raw: 101, gzip: 31 }; +const minimalUsefulRealtimeBundleSizeThresholdsKiB = { raw: 102, gzip: 31 }; const baseClientNames = ['BaseRest', 'BaseRealtime']; From ae655f18a4669ff44e8e79e6028059d99fd63c02 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 30 Jan 2025 01:23:28 +0000 Subject: [PATCH 15/18] Add `IBufferUtils.concat` method This is required for following LiveObjects changes to be able to encode the hash for object IDs based on multiple buffers. Browser implementation is based on example from [1]. [1] https://stackoverflow.com/questions/10786128/appending-arraybuffers --- src/common/types/IBufferUtils.ts | 1 + src/platform/nodejs/lib/util/bufferutils.ts | 4 ++++ src/platform/web/lib/util/bufferutils.ts | 15 +++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/src/common/types/IBufferUtils.ts b/src/common/types/IBufferUtils.ts index 8cd1d39aa..8f9b8010b 100644 --- a/src/common/types/IBufferUtils.ts +++ b/src/common/types/IBufferUtils.ts @@ -20,6 +20,7 @@ export default interface IBufferUtils { * Returns ArrayBuffer on browser and Buffer on Node.js */ arrayBufferViewToBuffer: (arrayBufferView: ArrayBufferView) => Bufferlike; + concat(buffers: Bufferlike[]): Output; sha256(message: Bufferlike): Output; hmacSha256(message: Bufferlike, key: Bufferlike): Output; } diff --git a/src/platform/nodejs/lib/util/bufferutils.ts b/src/platform/nodejs/lib/util/bufferutils.ts index 8c93f4ef3..82ba2f287 100644 --- a/src/platform/nodejs/lib/util/bufferutils.ts +++ b/src/platform/nodejs/lib/util/bufferutils.ts @@ -74,6 +74,10 @@ class BufferUtils implements IBufferUtils { return Buffer.from(string, 'utf8'); } + concat(buffers: Bufferlike[]): Output { + return Buffer.concat(buffers.map((x) => this.toBuffer(x))); + } + sha256(message: Bufferlike): Output { const messageBuffer = this.toBuffer(message); diff --git a/src/platform/web/lib/util/bufferutils.ts b/src/platform/web/lib/util/bufferutils.ts index 1d7af7d69..cccd538c7 100644 --- a/src/platform/web/lib/util/bufferutils.ts +++ b/src/platform/web/lib/util/bufferutils.ts @@ -200,6 +200,21 @@ class BufferUtils implements IBufferUtils { return this.toArrayBuffer(arrayBufferView); } + concat(buffers: Bufferlike[]): Output { + const sumLength = buffers.reduce((acc, v) => acc + v.byteLength, 0); + const result = new Uint8Array(sumLength); + let offset = 0; + + for (const buffer of buffers) { + const uint8Array = this.toBuffer(buffer); + // see TypedArray.set for TypedArray argument https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypedArray/set#typedarray + result.set(uint8Array, offset); + offset += uint8Array.byteLength; + } + + return result.buffer; + } + sha256(message: Bufferlike): Output { const hash = sha256(this.toBuffer(message)); return this.toArrayBuffer(hash); From 109fefb4e7eefe4bbc084703afc1f62030fc66c7 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 30 Jan 2025 01:36:05 +0000 Subject: [PATCH 16/18] Encode `initialValue` for StateMessage based on the `useBinaryProtocol` client option This adds support for encoding `initialValue` for `json` and `msgpack` encodings. Also improves types in StateMessage, notably use `Bufferlike` instead of `Buffer | ArrayBuffer`. --- src/plugins/liveobjects/livecounter.ts | 6 +-- src/plugins/liveobjects/livemap.ts | 8 +-- src/plugins/liveobjects/objectid.ts | 9 +++- src/plugins/liveobjects/statemessage.ts | 71 +++++++++++++++---------- 4 files changed, 56 insertions(+), 38 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 6c08f3d88..8d8d8e932 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -61,7 +61,7 @@ export class LiveCounter extends LiveObject action: StateOperationAction.COUNTER_INC, objectId, counterOp: { amount }, - }, + } as StateOperation, }, client.Utils, client.MessageEncoding, @@ -81,7 +81,7 @@ export class LiveCounter extends LiveObject } const initialValueObj = LiveCounter.createInitialValueObject(count); - const { encodedInitialValue, format } = StateMessage.encodeInitialValue(client.Utils, initialValueObj); + const { encodedInitialValue, format } = StateMessage.encodeInitialValue(initialValueObj, client); const nonce = client.Utils.cheapRandStr(); const msTimestamp = await client.getTimestamp(true); @@ -102,7 +102,7 @@ export class LiveCounter extends LiveObject nonce, initialValue: encodedInitialValue, initialValueEncoding: format, - }, + } as StateOperation, }, client.Utils, client.MessageEncoding, diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 8b3fc41ef..9d35b868b 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -123,7 +123,7 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject LiveMap.validateKeyValue(liveObjects, key, value)); const initialValueObj = LiveMap.createInitialValueObject(entries); - const { encodedInitialValue, format } = StateMessage.encodeInitialValue(client.Utils, initialValueObj); + const { encodedInitialValue, format } = StateMessage.encodeInitialValue(initialValueObj, client); const nonce = client.Utils.cheapRandStr(); const msTimestamp = await client.getTimestamp(true); @@ -220,7 +220,7 @@ export class LiveMap extends LiveObject { value: StateValue | undefined; encoding: string | undefined }; +export type EncodeFunction = (data: any, encoding?: string | null) => { data: any; encoding?: string | null }; export enum StateOperationAction { MAP_CREATE = 0, @@ -21,7 +20,7 @@ export enum MapSemantics { } /** A StateValue represents a concrete leaf value in a state object graph. */ -export type StateValue = string | number | boolean | Buffer | ArrayBuffer; +export type StateValue = string | number | boolean | Bufferlike; /** StateData captures a value in a state object. */ export interface StateData { @@ -110,9 +109,9 @@ export interface StateOperation { * After verification the bytes will be decoded into the Map or Counter objects and * the initialValue, nonce, and initialValueEncoding will be removed. */ - initialValue?: Buffer | ArrayBuffer; + initialValue?: Bufferlike; /** The initial value encoding defines how the initialValue should be interpreted. */ - initialValueEncoding?: string; + initialValueEncoding?: Utils.Format; } /** A StateObject describes the instantaneous state of an object. */ @@ -172,12 +171,12 @@ export class StateMessage { * Uses encoding functions from regular `Message` processing. */ static async encode(message: StateMessage, messageEncoding: typeof MessageEncoding): Promise { - const encodeFn: StateDataEncodeFunction = (value, encoding) => { - const { data: newValue, encoding: newEncoding } = messageEncoding.encodeData(value, encoding); + const encodeFn: EncodeFunction = (data, encoding) => { + const { data: encodedData, encoding: newEncoding } = messageEncoding.encodeData(data, encoding); return { - value: newValue, - encoding: newEncoding!, + data: encodedData, + encoding: newEncoding, }; }; @@ -242,15 +241,26 @@ export class StateMessage { } static encodeInitialValue( - utils: typeof Utils, initialValue: Partial, + client: BaseClient, ): { - encodedInitialValue: string; + encodedInitialValue: Bufferlike; format: Utils.Format; } { + const format = client.options.useBinaryProtocol ? client.Utils.Format.msgpack : client.Utils.Format.json; + const encodedInitialValue = client.Utils.encodeBody(initialValue, client._MsgPack, format); + + // if we've got string result (for example, json format was used), we need to additionally convert it to bytes array with utf8 encoding + if (typeof encodedInitialValue === 'string') { + return { + encodedInitialValue: client.Platform.BufferUtils.utf8Encode(encodedInitialValue), + format, + }; + } + return { - encodedInitialValue: JSON.stringify(initialValue), - format: utils.Format.json, + encodedInitialValue, + format, }; } @@ -282,10 +292,7 @@ export class StateMessage { } } - private static _encodeStateOperation( - stateOperation: StateOperation, - encodeFn: StateDataEncodeFunction, - ): StateOperation { + private static _encodeStateOperation(stateOperation: StateOperation, encodeFn: EncodeFunction): StateOperation { // deep copy "stateOperation" object so we can modify the copy here. // buffer values won't be correctly copied, so we will need to set them again explicitly. const stateOperationCopy = JSON.parse(JSON.stringify(stateOperation)) as StateOperation; @@ -302,10 +309,16 @@ export class StateMessage { }); } + if (stateOperation.initialValue) { + // use original "stateOperation" object so we have access to the original buffer value + const { data: encodedInitialValue } = encodeFn(stateOperation.initialValue); + stateOperationCopy.initialValue = encodedInitialValue; + } + return stateOperationCopy; } - private static _encodeStateObject(stateObject: StateObject, encodeFn: StateDataEncodeFunction): StateObject { + private static _encodeStateObject(stateObject: StateObject, encodeFn: EncodeFunction): StateObject { // deep copy "stateObject" object so we can modify the copy here. // buffer values won't be correctly copied, so we will need to set them again explicitly. const stateObjectCopy = JSON.parse(JSON.stringify(stateObject)) as StateObject; @@ -325,13 +338,13 @@ export class StateMessage { return stateObjectCopy; } - private static _encodeStateData(data: StateData, encodeFn: StateDataEncodeFunction): StateData { - const { value: newValue, encoding: newEncoding } = encodeFn(data?.value, data?.encoding); + private static _encodeStateData(data: StateData, encodeFn: EncodeFunction): StateData { + const { data: encodedValue, encoding: newEncoding } = encodeFn(data?.value, data?.encoding); return { ...data, - value: newValue, - encoding: newEncoding!, + value: encodedValue, + encoding: newEncoding ?? undefined, }; } @@ -353,15 +366,15 @@ export class StateMessage { // if JSON protocol is being used, the JSON.stringify() will be called and this toJSON() method will have a non-empty arguments list. // MSGPack protocol implementation also calls toJSON(), but with an empty arguments list. const format = arguments.length > 0 ? this._utils.Format.json : this._utils.Format.msgpack; - const encodeFn: StateDataEncodeFunction = (value, encoding) => { - const { data: newValue, encoding: newEncoding } = this._messageEncoding.encodeDataForWireProtocol( - value, + const encodeFn: EncodeFunction = (data, encoding) => { + const { data: encodedData, encoding: newEncoding } = this._messageEncoding.encodeDataForWireProtocol( + data, encoding, format, ); return { - value: newValue, - encoding: newEncoding!, + data: encodedData, + encoding: newEncoding, }; }; From 6ea31dacf4fea78ffd0f175352e14e1a3ffff33f Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 30 Jan 2025 02:35:22 +0000 Subject: [PATCH 17/18] Fix `StateMessage.encodeInitialValue` incorrectly encoded initial value with `json` encoding when it contained buffers for LiveMap keys --- src/plugins/liveobjects/statemessage.ts | 71 ++++++++++++++++++------- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 0bf2e148e..77fdc5cba 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -248,9 +248,21 @@ export class StateMessage { format: Utils.Format; } { const format = client.options.useBinaryProtocol ? client.Utils.Format.msgpack : client.Utils.Format.json; - const encodedInitialValue = client.Utils.encodeBody(initialValue, client._MsgPack, format); - // if we've got string result (for example, json format was used), we need to additionally convert it to bytes array with utf8 encoding + // initial value object may contain user provided data that requires an additional encoding (for example buffers as map keys). + // so we need to encode that data first as if we were sending it over the wire. we can use a StateMessage methods for this + const stateMessage = StateMessage.fromValues({ operation: initialValue }, client.Utils, client.MessageEncoding); + StateMessage.encode(stateMessage, client.MessageEncoding); + const { operation: initialValueWithDataEncoding } = StateMessage._encodeForWireProtocol( + stateMessage, + client.MessageEncoding, + format, + ); + + // initial value field should be represented as an array of bytes over the wire. so we encode the whole object based on the client encoding format + const encodedInitialValue = client.Utils.encodeBody(initialValueWithDataEncoding, client._MsgPack, format); + + // if we've got string result (for example, json encoding was used), we need to additionally convert it to bytes array with utf8 encoding if (typeof encodedInitialValue === 'string') { return { encodedInitialValue: client.Platform.BufferUtils.utf8Encode(encodedInitialValue), @@ -348,6 +360,42 @@ export class StateMessage { }; } + /** + * Encodes operation and object fields of the StateMessage. Does not mutate the provided StateMessage. + * + * Uses encoding functions from regular `Message` processing. + */ + private static _encodeForWireProtocol( + message: StateMessage, + messageEncoding: typeof MessageEncoding, + format: Utils.Format, + ): { + operation?: StateOperation; + object?: StateObject; + } { + const encodeFn: EncodeFunction = (data, encoding) => { + const { data: encodedData, encoding: newEncoding } = messageEncoding.encodeDataForWireProtocol( + data, + encoding, + format, + ); + return { + data: encodedData, + encoding: newEncoding, + }; + }; + + const encodedOperation = message.operation + ? StateMessage._encodeStateOperation(message.operation, encodeFn) + : undefined; + const encodedObject = message.object ? StateMessage._encodeStateObject(message.object, encodeFn) : undefined; + + return { + operation: encodedOperation, + object: encodedObject, + }; + } + /** * Overload toJSON() to intercept JSON.stringify(). * @@ -366,26 +414,13 @@ export class StateMessage { // if JSON protocol is being used, the JSON.stringify() will be called and this toJSON() method will have a non-empty arguments list. // MSGPack protocol implementation also calls toJSON(), but with an empty arguments list. const format = arguments.length > 0 ? this._utils.Format.json : this._utils.Format.msgpack; - const encodeFn: EncodeFunction = (data, encoding) => { - const { data: encodedData, encoding: newEncoding } = this._messageEncoding.encodeDataForWireProtocol( - data, - encoding, - format, - ); - return { - data: encodedData, - encoding: newEncoding, - }; - }; - - const encodedOperation = this.operation ? StateMessage._encodeStateOperation(this.operation, encodeFn) : undefined; - const encodedObject = this.object ? StateMessage._encodeStateObject(this.object, encodeFn) : undefined; + const { operation, object } = StateMessage._encodeForWireProtocol(this, this._messageEncoding, format); return { id: this.id, clientId: this.clientId, - operation: encodedOperation, - object: encodedObject, + operation, + object, extras: this.extras, }; } From 5cfbc9b130eb35e3ef7fc77008afc8bd6243edbd Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 31 Jan 2025 01:57:23 +0000 Subject: [PATCH 18/18] Improve comments about counter/map creation --- src/plugins/liveobjects/livecounter.ts | 2 +- src/plugins/liveobjects/livemap.ts | 2 +- src/plugins/liveobjects/liveobjects.ts | 21 +++++++++++---------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 8d8d8e932..05425ff62 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -34,7 +34,7 @@ export class LiveCounter extends LiveObject } /** - * Returns a {@link LiveCounter} instance based on the provided state operation. + * Returns a {@link LiveCounter} instance based on the provided COUNTER_CREATE state operation. * The provided state operation must hold a valid counter object data. * * @internal diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 9d35b868b..f983c52e3 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -82,7 +82,7 @@ export class LiveMap extends LiveObject; } - // new map object can be created using locally constructed state operation, even though it is missing timeserials for map entries. - // CREATE operation is only applied once, and all map entries will have an "earliest possible" timeserial so that any subsequent operation can be applied to them. + // we haven't received the CREATE operation yet, so we can create a new map object using the locally constructed state operation. + // we don't know the timeserials for map entries, so we assign an "earliest possible" timeserial to each entry, so that any subsequent operation can be applied to them. + // we mark the CREATE operation as merged for the object, guaranteeing its idempotency and preventing it from being applied again when the operation arrives. const map = LiveMap.fromStateOperation(this, stateMessage.operation!); this._liveObjectsPool.set(objectId, map); @@ -99,15 +100,15 @@ export class LiveObjects { await this.publish([stateMessage]); - // we might have received CREATE operation already at this point (it might arrive before the ACK message for our publish message), - // so the object could exist in the local pool as it was added there during regular CREATE operation application. - // check if the object is there and return it in case it is. otherwise create a new object client-side + // we may have already received the CREATE operation at this point, as it could arrive before the ACK for our publish message. + // this means the object might already exist in the local pool, having been added during the usual CREATE operation process. + // here we check if the object is present, and return it if found; otherwise, create a new object on the client side. if (this._liveObjectsPool.get(objectId)) { return this._liveObjectsPool.get(objectId) as LiveCounter; } - // new counter object can be created using locally constructed state operation. - // CREATE operation is only applied once, so the initial counter value won't be double counted when we eventually receive an echoed CREATE operation + // we haven't received the CREATE operation yet, so we can create a new counter object using the locally constructed state operation. + // we mark the CREATE operation as merged for the object, guaranteeing its idempotency. this ensures we don't double count the initial counter value when the operation arrives. const counter = LiveCounter.fromStateOperation(this, stateMessage.operation!); this._liveObjectsPool.set(objectId, counter);