Implement new extensible events parsing interface and intercept non-extensible messages in supported rooms

package.json

@@ -60,7 +60,7 @@
         "bs58": "^5.0.0",
         "content-type": "^1.0.4",
         "loglevel": "^1.7.1",
-        "matrix-events-sdk": "0.0.1",
+        "matrix-events-sdk": "^2.0.0",
         "matrix-widget-api": "^1.0.0",
         "p-retry": "4",
         "sdp-transform": "^2.14.1",

spec/unit/matrix-client.spec.ts

@@ -32,7 +32,7 @@ import {
 } from "../../src/@types/event";
 import { MEGOLM_ALGORITHM } from "../../src/crypto/olmlib";
 import { Crypto } from "../../src/crypto";
-import { EventStatus, MatrixEvent } from "../../src/models/event";
+import { EventStatus, IContent, MatrixEvent } from "../../src/models/event";
 import { Preset } from "../../src/@types/partials";
 import { ReceiptType } from "../../src/@types/read_receipts";
 import * as testUtils from "../test-utils/test-utils";
@@ -2134,6 +2134,68 @@ describe("MatrixClient", function () {
         });
     });
 
+    describe("sendMessage", () => {
+        describe("roomId,content,txnId signature", () => {
+            it("should translate these values to the correct request", async () => {
+                const roomId = "!room:example.org";
+                const content: IContent = { hello: "world" };
+                const txnId = "m.1234";
+                const response = { event_id: "$example" };
+
+                const sendSpy = jest.spyOn(client, "sendEvent").mockResolvedValue(response);
+
+                const result = await client.sendMessage(roomId, content, txnId);
+                expect(result).toEqual(response);
+                expect(sendSpy).toHaveBeenCalledWith(roomId, null, "m.room.message", content, txnId);
+            });
+        });
+
+        describe("roomId,threadId,content,txnId signature", () => {
+            it("should translate these values to the correct request", async () => {
+                const roomId = "!room:example.org";
+                const threadId = "$thread";
+                const content: IContent = { hello: "world" };
+                const txnId = "m.1234";
+                const response = { event_id: "$example" };
+
+                const sendSpy = jest.spyOn(client, "sendEvent").mockResolvedValue(response);
+
+                const result = await client.sendMessage(roomId, threadId, content, txnId);
+                expect(result).toEqual(response);
+                expect(sendSpy).toHaveBeenCalledWith(roomId, threadId, "m.room.message", content, txnId);
+            });
+        });
+
+        it.each(["m.text", "m.emote"])("should override %s msgtype events", async (msgtype) => {
+            const roomId = "!room:example.org";
+            const content: IContent = {
+                msgtype: msgtype,
+                body: "**test**",
+                format: "org.matrix.custom.html",
+                formatted_body: "<b>test</b>",
+            };
+            const expectedContent: IContent = {
+                "org.matrix.msc1767.markup": [{ body: "**test**" }, { body: "<b>test</b>", mimetype: "text/html" }],
+            };
+            const expectedEventType = msgtype === "m.emote" ? "org.matrix.msc1767.emote" : "org.matrix.msc1767.message";
+            const txnId = "m.1234";
+            const response = { event_id: "$example" };
+
+            client.getRoom = (rid) => {
+                if (rid === roomId) {
+                    // XXX: This is a really bad mock.
+                    return { unstableRequiresExtensibleEvents: () => true } as unknown as Room;
+                }
+                return null;
+            };
+            const sendSpy = jest.spyOn(client, "sendEvent").mockResolvedValue(response);
+
+            const result = await client.sendMessage(roomId, content, txnId);
+            expect(result).toEqual(response);
+            expect(sendSpy).toHaveBeenCalledWith(roomId, null, expectedEventType, expectedContent, txnId);
+        });
+    });
+
     describe("delete account data", () => {
         afterEach(() => {
             jest.spyOn(featureUtils, "buildFeatureSupportMap").mockRestore();

spec/unit/room.spec.ts

@@ -193,6 +193,22 @@ describe("Room", function () {
         });
     });
 
+    describe("unstableRequiresExtensibleEvents", () => {
+        it("should return true for MSC1767 room version prefixes", () => {
+            room.getVersion = () => "org.matrix.msc1767.10";
+            expect(room.unstableRequiresExtensibleEvents()).toBe(true);
+
+            room.getVersion = () => "org.matrix.msc1767.9";
+            expect(room.unstableRequiresExtensibleEvents()).toBe(true);
+
+            room.getVersion = () => "not.extensible.10";
+            expect(room.unstableRequiresExtensibleEvents()).toBe(false);
+
+            room.getVersion = () => "10";
+            expect(room.unstableRequiresExtensibleEvents()).toBe(false);
+        });
+    });
+
     describe("getAvatarUrl", function () {
         const hsUrl = "https://my.home.server";
 

src/client.ts

@@ -18,7 +18,7 @@ limitations under the License.
  * This is an internal module. See {@link MatrixClient} for the public class.
  */
 
-import { Optional } from "matrix-events-sdk";
+import { EmoteEvent, MarkupBlock, Optional, WireMessageEvent, MessageEvent } from "matrix-events-sdk";
 
 import type { IMegolmSessionData } from "./@types/crypto";
 import { ISyncStateData, SyncApi, SyncApiOptions, SyncState } from "./sync";
@@ -4545,8 +4545,63 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             threadId = null;
         }
 
-        const eventType: string = EventType.RoomMessage;
-        const sendContent: IContent = content as IContent;
+        let eventType: string = EventType.RoomMessage;
+        let sendContent: IContent = content as IContent;
+
+        // Quickly check if we're about to send the event into an MSC1767-compatible room
+        const room = this.getRoom(roomId);
+        if (room?.unstableRequiresExtensibleEvents()) {
+            // This room supports extensible events! Override the details of the event and hope for the best.
+            const makeContentExtensible = (
+                content: IContent = {},
+                recurse = true,
+            ): [string | undefined, IContent | undefined] => {
+                let newContent: IContent | undefined = undefined;
+                let newEventType: string | undefined = undefined;
+
+                // Dev note: We exclude notices from this for now, at least while we figure out what to do with
+                // them at the spec level.
+                const textTypes: (string | undefined)[] = [MsgType.Text, MsgType.Emote];
+                if (textTypes.includes(content.msgtype)) {
+                    newContent = {
+                        [MarkupBlock.type.name]: [{ body: content.body }],
+                    } satisfies WireMessageEvent.ContentValue;
+                    if (content.format === "org.matrix.custom.html" && content.formatted_body) {
+                        newContent[MarkupBlock.type.name].push({ body: content.formatted_body, mimetype: "text/html" });
+                    }
+                    newEventType = content.msgtype === MsgType.Emote ? EmoteEvent.type.name : MessageEvent.type.name;
+
+                    // Remove the fields we processed, so we don't clone them in a later stage.
+                    delete content.body;
+                    delete content.format;
+                    delete content.formatted_body;
+                    delete content.msgtype;
+                }
+
+                // Deal with edits too: just a bit of recursion.
+                if (newContent && !!content["m.new_content"] && recurse) {
+                    const editContent = makeContentExtensible(content["m.new_content"], false);
+                    if (editContent) {
+                        newContent["m.new_content"] = editContent;
+                    }
+                }
+
+                if (newContent) {
+                    // copy over all other fields we don't know about (for safety)
+                    for (const [k, v] of Object.entries(content)) {
+                        if (!newContent.hasOwnProperty(k)) {
+                            newContent[k] = v;
+                        }
+                    }
+                }
+
+                return [newEventType, newContent];
+            };
+
+            const [newEventType, newContent] = makeContentExtensible(sendContent);
+            if (newEventType !== undefined) eventType = newEventType;
+            if (newContent !== undefined) sendContent = newContent;
+        }
 
         return this.sendEvent(roomId, threadId as string | null, eventType, sendContent, txnId);
     }

src/models/event.ts

@@ -19,7 +19,7 @@ limitations under the License.
  * the public classes.
  */
 
-import { ExtensibleEvent, ExtensibleEvents, Optional } from "matrix-events-sdk";
+import { EventParser, Optional, RoomEvent } from "matrix-events-sdk";
 
 import type { IEventDecryptionResult } from "../@types/crypto";
 import { logger } from "../logger";
@@ -212,6 +212,12 @@ export type MatrixEventHandlerMap = {
 } & Pick<ThreadEventHandlerMap, ThreadEvent.Update>;
 
 export class MatrixEvent extends TypedEventEmitter<MatrixEventEmittedEvents, MatrixEventHandlerMap> {
+    /**
+     * The default shared event parser for Extensible Events. To register custom event types or content blocks
+     * with the parser, do so here before constructing a new MatrixEvent object (implicitly or explicitly).
+     */
+    public static readonly defaultExtensibleEventParser = new EventParser();
+
     private pushActions: IActionsObject | null = null;
     private _replacingEvent: MatrixEvent | null = null;
     private _localRedactionEvent: MatrixEvent | null = null;
@@ -228,7 +234,7 @@ export class MatrixEvent extends TypedEventEmitter<MatrixEventEmittedEvents, Mat
     // addition to a falsy cached event value. We check the flag later on in
     // a public getter to decide if the cache is valid.
     private _hasCachedExtEv = false;
-    private _cachedExtEv: Optional<ExtensibleEvent> = undefined;
+    private _cachedExtEv: Optional<RoomEvent> = undefined;
 
     /* curve25519 key which we believe belongs to the sender of the event. See
      * getSenderKey()
@@ -334,8 +340,15 @@ export class MatrixEvent extends TypedEventEmitter<MatrixEventEmittedEvents, Mat
      * this property</b> directly unless you absolutely have to. Prefer the getter
      * methods defined on this class. Using the getter methods shields your app
      * from changes to event JSON between Matrix versions.
-     */
-    public constructor(public event: Partial<IEvent> = {}) {
+     * @param extensibleEventParser - The event parser to use when attempting to
+     * handle this event as an extensible event. <b>This field is experimental
+     * and unstable - it may be removed without warning at any time.</b> Defaults
+     * to the static parser from MatrixEvent.
+     */
+    public constructor(
+        public event: Partial<IEvent> = {},
+        private extensibleEventParser: EventParser = MatrixEvent.defaultExtensibleEventParser,
+    ) {
         super();
 
         // intern the values of matrix events to force share strings and reduce the
@@ -371,9 +384,15 @@ export class MatrixEvent extends TypedEventEmitter<MatrixEventEmittedEvents, Mat
      *
      * @deprecated Use stable functions where possible.
      */
-    public get unstableExtensibleEvent(): Optional<ExtensibleEvent> {
+    public get unstableExtensibleEvent(): Optional<RoomEvent> {
         if (!this._hasCachedExtEv) {
-            this._cachedExtEv = ExtensibleEvents.parse(this.getEffectiveEvent());
+            const effectiveEvent = this.getEffectiveEvent();
+            this._cachedExtEv = this.extensibleEventParser.parse({
+                ...effectiveEvent,
+                // XXX: This case should never happen, but MatrixEvent in the js-sdk
+                // covers room-less events, unlike the events-sdk.
+                room_id: effectiveEvent.room_id ?? "!unknown:localhost",
+            });
         }
         return this._cachedExtEv;
     }

src/models/room.ts

@@ -547,6 +547,18 @@ export class Room extends ReadReceipt<RoomEmittedEvents, RoomEventHandlerMap> {
         return createEvent?.getContent()["creator"] ?? null;
     }
 
+    /**
+     * Unstable/experimental function to determine if the room requires MSC1767-style
+     * events or not.
+     *
+     * @deprecated Use stable functions where possible. This function may be removed or
+     * changed without notice.
+     */
+    public unstableRequiresExtensibleEvents(): boolean {
+        // TODO(TR): We need a better check for this that doesn't involve hardcoding supported room versions
+        return this.getVersion().startsWith("org.matrix.msc1767.");
+    }
+
     /**
      * Gets the version of the room
      * @returns The version of the room, or null if it could not be determined

yarn.lock

@@ -1944,6 +1944,11 @@ agent-base@6:
   dependencies:
     debug "4"
 
+ajv-errors@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/ajv-errors/-/ajv-errors-3.0.0.tgz#e54f299f3a3d30fe144161e5f0d8d51196c527bc"
+  integrity sha512-V3wD15YHfHz6y0KdhYFjyy9vWtEVALT9UrxfN3zqlI6dMioHnJrqOYfyPKol3oqrnCM9uwkcdCwkJ0WUcbLMTQ==
+
 ajv@^6.10.0, ajv@^6.12.4, ajv@~6.12.6:
   version "6.12.6"
   resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4"
@@ -1954,6 +1959,16 @@ ajv@^6.10.0, ajv@^6.12.4, ajv@~6.12.6:
     json-schema-traverse "^0.4.1"
     uri-js "^4.2.2"
 
+ajv@^8.11.2:
+  version "8.12.0"
+  resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.12.0.tgz#d1a0527323e22f53562c567c00991577dfbe19d1"
+  integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==
+  dependencies:
+    fast-deep-equal "^3.1.1"
+    json-schema-traverse "^1.0.0"
+    require-from-string "^2.0.2"
+    uri-js "^4.2.2"
+
 align-text@^0.1.1, align-text@^0.1.3:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117"
@@ -4929,6 +4944,11 @@ json-schema-traverse@^0.4.1:
   resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660"
   integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==
 
+json-schema-traverse@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz#ae7bcb3656ab77a73ba5c49bf654f38e6b6860e2"
+  integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==
+
 json-stable-stringify-without-jsonify@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651"
@@ -5150,10 +5170,13 @@ marked@^4.2.5:
   resolved "https://registry.yarnpkg.com/marked/-/marked-4.2.5.tgz#979813dfc1252cc123a79b71b095759a32f42a5d"
   integrity sha512-jPueVhumq7idETHkb203WDD4fMA3yV9emQ5vLwop58lu8bTclMghBWcYAavlDqIEMaisADinV1TooIFCfqOsYQ==
 
-[email protected]:
-  version "0.0.1"
-  resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd"
-  integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==
+matrix-events-sdk@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-2.0.0.tgz#f5f8dafbe4eae07fdbb628627f430ca5b1fd8c7a"
+  integrity sha512-UZbifYIO2o9+sNn4YuGjhMof/88TG68PyecKnH/pt8V3MFq0RZsbBUe+3/k5ZeVcEtr0pQLmcKB7d8aQVsVO/w==
+  dependencies:
+    ajv "^8.11.2"
+    ajv-errors "^3.0.0"
 
 matrix-mock-request@^2.5.0:
   version "2.6.0"
@@ -6121,6 +6144,11 @@ require-directory@^2.1.1:
   resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
   integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==
 
+require-from-string@^2.0.2:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909"
+  integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==
+
 requires-port@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff"