From 0903c63cce6cfae3ce8c179613d6307ced1f1a03 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 19 Nov 2024 11:17:55 +0000 Subject: [PATCH 01/21] Deprecate legacy sled store --- src/config/Config.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/config/Config.ts b/src/config/Config.ts index 2a3aa502d..76c55757a 100644 --- a/src/config/Config.ts +++ b/src/config/Config.ts @@ -378,7 +378,10 @@ interface BridgeConfigBot { } interface BridgeConfigEncryption { storagePath: string; - useLegacySledStore: boolean; + /** + * @deprecated This is no longer supported. + */ + useLegacySledStore?: boolean; } export interface BridgeConfigServiceBot { @@ -548,6 +551,7 @@ export class BridgeConfig { } } + this.encryption = configData.experimentalEncryption; From c00fb904cd3ec8a08a5cfc733f0f87067f981ad1 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 19 Nov 2024 11:18:00 +0000 Subject: [PATCH 02/21] Add e2ee test --- spec/e2ee.spec.ts | 57 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 spec/e2ee.spec.ts diff --git a/spec/e2ee.spec.ts b/spec/e2ee.spec.ts new file mode 100644 index 000000000..88293207a --- /dev/null +++ b/spec/e2ee.spec.ts @@ -0,0 +1,57 @@ +import { MessageEventContent } from "matrix-bot-sdk"; +import { E2ESetupTestTimeout, E2ETestEnv } from "./util/e2e-test"; +import { describe, it, beforeEach, afterEach } from "@jest/globals"; + +const CryptoRoomState = [{ + content: { + "algorithm": "m.megolm.v1.aes-sha2" + }, + state_key: "", + type: "m.room.encryption" +}]; + +describe('End-2-End Encryption support', () => { + let testEnv: E2ETestEnv; + + beforeEach(async () => { + testEnv = await E2ETestEnv.createTestEnv({ matrixLocalparts: ['user'], enableE2EE: true }); + await testEnv.setUp(); + }, E2ESetupTestTimeout); + + afterEach(() => { + return testEnv?.tearDown(); + }); + + it('should be able to send the help command', async () => { + const user = testEnv.getUser('user'); + const testRoomId = await user.createRoom({ name: 'Test room', invite:[testEnv.botMxid], initial_state: CryptoRoomState}); + await user.setUserPowerLevel(testEnv.botMxid, testRoomId, 50); + await user.waitForRoomJoin({sender: testEnv.botMxid, roomId: testRoomId }); + await new Promise((r) => setTimeout(r, 5000)); + await user.sendText(testRoomId, "!hookshot help"); + // const inviteResponse = await user.waitForRoomInvite({sender: testEnv.botMxid}); + await user.waitForRoomEvent({ + eventType: 'm.room.message', sender: testEnv.botMxid, roomId: testRoomId, + // body: 'Room configured to bridge webhooks. See admin room for secret url.' + }); + // const webhookUrlMessage = user.waitForRoomEvent({ + // eventType: 'm.room.message', sender: testEnv.botMxid, roomId: inviteResponse.roomId + // }); + // await user.joinRoom(inviteResponse.roomId); + // const msgData = (await webhookUrlMessage).data.content.body; + // const webhookUrl = msgData.split('\n')[2]; + // const webhookNotice = user.waitForRoomEvent({ + // eventType: 'm.room.message', sender: testEnv.botMxid, roomId: testRoomId, body: 'Hello world!' + // }); + + // // Send a webhook + // await fetch(webhookUrl, { + // method: 'POST', + // headers: { 'Content-Type': 'application/json' }, + // body: JSON.stringify({text: 'Hello world!'}) + // }); + + // // And await the notice. + // await webhookNotice; + }); +}); From 494866774e6ccaecdfb989c3fd7539b05ac45e32 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 19 Nov 2024 11:18:12 +0000 Subject: [PATCH 03/21] Add support for e2ee testing in e2e environment --- spec/util/e2e-test.ts | 24 ++++++++++++++++++++---- spec/util/homerunner.ts | 23 ++++++++++++++--------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/spec/util/e2e-test.ts b/spec/util/e2e-test.ts index 28631077a..af402329e 100644 --- a/spec/util/e2e-test.ts +++ b/spec/util/e2e-test.ts @@ -5,6 +5,7 @@ import { BridgeConfig, BridgeConfigRoot } from "../../src/config/Config"; import { start } from "../../src/App/BridgeApp"; import { RSAKeyPairOptions, generateKeyPair } from "node:crypto"; import path from "node:path"; +import Redis from "ioredis"; const WAIT_EVENT_TIMEOUT = 10000; export const E2ESetupTestTimeout = 60000; @@ -12,6 +13,7 @@ export const E2ESetupTestTimeout = 60000; interface Opts { matrixLocalparts?: string[]; config?: Partial, + enableE2EE?: boolean, } export class E2ETestMatrixClient extends MatrixClient { @@ -173,10 +175,11 @@ export class E2ETestEnv { if (err) { reject(err) } else { resolve(privateKey) } })); + const dir = await mkdtemp('hookshot-int-test'); + // Configure homeserver and bots - const [homeserver, dir, privateKey] = await Promise.all([ - createHS([...matrixLocalparts || []], workerID), - mkdtemp('hookshot-int-test'), + const [homeserver, privateKey] = await Promise.all([ + createHS([...matrixLocalparts || []], workerID, opts.enableE2EE ? path.join(dir, 'client-crypto') : undefined), keyPromise, ]); const keyPath = path.join(dir, 'key.pem'); @@ -193,6 +196,10 @@ export class E2ETestEnv { providedConfig.github.auth.privateKeyFile = keyPath; } + // Clear away the existing DB. + const redisUri = `redis://localhost/99` + await new Redis(redisUri).flushdb(); + const config = new BridgeConfig({ bridge: { domain: homeserver.domain, @@ -201,7 +208,7 @@ export class E2ETestEnv { bindAddress: '0.0.0.0', }, logging: { - level: 'info', + level: 'debug', }, // Always enable webhooks so that hookshot starts. generic: { @@ -214,6 +221,15 @@ export class E2ETestEnv { resources: ['webhooks'], }], passFile: keyPath, + ...(opts.enableE2EE ? { + experimentalEncryption: { + storagePath: path.join(dir, 'crypto-store'), + useLegacySledStore: false, + } + } : undefined), + cache: { + redisUri, + }, ...providedConfig, }); const registration: IAppserviceRegistration = { diff --git a/spec/util/homerunner.ts b/spec/util/homerunner.ts index a33bbc96a..52b71c432 100644 --- a/spec/util/homerunner.ts +++ b/spec/util/homerunner.ts @@ -1,7 +1,8 @@ -import { MatrixClient } from "matrix-bot-sdk"; +import { MatrixClient, MemoryStorageProvider, RustSdkCryptoStorageProvider, RustSdkCryptoStoreType } from "matrix-bot-sdk"; import { createHash, createHmac, randomUUID } from "crypto"; import { Homerunner } from "homerunner-client"; import { E2ETestMatrixClient } from "./e2e-test"; +import path from "node:path"; const HOMERUNNER_IMAGE = process.env.HOMERUNNER_IMAGE || 'ghcr.io/element-hq/synapse/complement-synapse:latest'; export const DEFAULT_REGISTRATION_SHARED_SECRET = ( @@ -41,7 +42,7 @@ async function waitForHomerunner() { } } -export async function createHS(localparts: string[] = [], workerId: number): Promise { +export async function createHS(localparts: string[] = [], workerId: number, cryptoRootPath?: string): Promise { await waitForHomerunner(); const appPort = 49600 + workerId; @@ -70,16 +71,20 @@ export async function createHS(localparts: string[] = [], workerId: number): Pro // Skip AS user. const users = Object.entries(homeserver.AccessTokens) .filter(([_uId, accessToken]) => accessToken !== asToken) - .map(([userId, accessToken]) => ({ - userId: userId, - accessToken, - deviceId: homeserver.DeviceIDs[userId], - client: new E2ETestMatrixClient(homeserver.BaseURL, accessToken), - }) + .map(([userId, accessToken]) => { + const cryptoStore = cryptoRootPath ? new RustSdkCryptoStorageProvider(path.join(cryptoRootPath, userId), RustSdkCryptoStoreType.Sqlite) : undefined; + return { + userId: userId, + accessToken, + deviceId: homeserver.DeviceIDs[userId], + client: new E2ETestMatrixClient(homeserver.BaseURL, accessToken, new MemoryStorageProvider(), cryptoStore), + } + } ); + // Start syncing proactively. - await Promise.all(users.map(u => u.client.start())); + await Promise.all(users.map(u => void u.client.start())); return { users, id: blueprint, From fb6cec6621736417b7cb70e27f23dc108b3d8d79 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 19 Nov 2024 11:23:17 +0000 Subject: [PATCH 04/21] Tidy up redis support --- .github/workflows/main.yml | 6 ++++++ spec/util/e2e-test.ts | 23 +++++++++++++++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 978e61b8d..d52495ff2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -125,6 +125,11 @@ jobs: needs: - test - build-homerunner + services: + redis: + image: redis + ports: + - 6379:6379 steps: - name: Install Complement Dependencies run: | @@ -154,6 +159,7 @@ jobs: HOMERUNNER_SPAWN_HS_TIMEOUT_SECS: 100 HOMERUNNER_IMAGE: ghcr.io/element-hq/synapse/complement-synapse:latest NODE_OPTIONS: --dns-result-order ipv4first + REDIS_DATABASE_URI: "redis://localhost:6379" run: | docker pull $HOMERUNNER_IMAGE cd matrix-hookshot diff --git a/spec/util/e2e-test.ts b/spec/util/e2e-test.ts index af402329e..067f2393c 100644 --- a/spec/util/e2e-test.ts +++ b/spec/util/e2e-test.ts @@ -9,11 +9,13 @@ import Redis from "ioredis"; const WAIT_EVENT_TIMEOUT = 10000; export const E2ESetupTestTimeout = 60000; +const REDIS_DATABASE_URI = process.env.HOOKSHOT_E2E_REDIS_DB_URI ?? "redis://localhost:6379"; interface Opts { matrixLocalparts?: string[]; config?: Partial, enableE2EE?: boolean, + useRedis?: boolean, } export class E2ETestMatrixClient extends MatrixClient { @@ -196,9 +198,14 @@ export class E2ETestEnv { providedConfig.github.auth.privateKeyFile = keyPath; } - // Clear away the existing DB. - const redisUri = `redis://localhost/99` - await new Redis(redisUri).flushdb(); + opts.useRedis = opts.enableE2EE || opts.useRedis; + + let cacheConfig: BridgeConfigRoot["cache"]|undefined; + if (opts.useRedis) { + cacheConfig = { + redisUri: `${REDIS_DATABASE_URI}/${Math.ceil(Math.random() * 99)}`, + } + } const config = new BridgeConfig({ bridge: { @@ -227,9 +234,7 @@ export class E2ETestEnv { useLegacySledStore: false, } } : undefined), - cache: { - redisUri, - }, + cache: cacheConfig, ...providedConfig, }); const registration: IAppserviceRegistration = { @@ -271,6 +276,12 @@ export class E2ETestEnv { await this.app.bridgeApp.stop(); await this.app.listener.stop(); await this.app.storage.disconnect?.(); + + // Clear the redis DB. + if (this.config.cache?.redisUri) { + await new Redis(this.config.cache.redisUri).flushdb(); + } + this.homeserver.users.forEach(u => u.client.stop()); await destroyHS(this.homeserver.id); await rm(this.dir, { recursive: true }); From 1e8680ed601058772bafa2bc4c6c8351941b989f Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 19 Nov 2024 14:50:55 +0000 Subject: [PATCH 05/21] Attempt to get test working --- .github/workflows/main.yml | 11 ++++++----- spec/util/e2e-test.ts | 5 +++-- spec/util/homerunner.ts | 20 +++++++++++++------- src/Stores/RedisStorageProvider.ts | 4 ++-- src/appservice.ts | 2 +- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d52495ff2..631078f12 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -92,15 +92,16 @@ jobs: homerunnersha: ${{ steps.gitsha.outputs.sha }} steps: - name: Checkout matrix-org/complement - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: matrix-org/complement + ref: e1b98b67a3ca19551fe8640d10bd3611c0995288 - name: Get complement git sha id: gitsha run: echo sha=`git rev-parse --short HEAD` >> "$GITHUB_OUTPUT" - name: Cache homerunner id: cached - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: homerunner key: ${{ runner.os }}-homerunner-${{ steps.gitsha.outputs.sha }} @@ -135,18 +136,18 @@ jobs: run: | sudo apt-get update && sudo apt-get install -y libolm3 - name: Load cached homerunner bin - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: homerunner key: ${{ runner.os }}-homerunner-${{ needs.build-synapse.outputs.homerunnersha }} fail-on-cache-miss: true # Shouldn't happen, we build this in the needs step. - name: Checkout matrix-hookshot - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: matrix-hookshot # Setup node & run tests - name: Use Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version-file: matrix-hookshot/.node-version - uses: Swatinem/rust-cache@v2 diff --git a/spec/util/e2e-test.ts b/spec/util/e2e-test.ts index 067f2393c..cd6bd5165 100644 --- a/spec/util/e2e-test.ts +++ b/spec/util/e2e-test.ts @@ -203,7 +203,7 @@ export class E2ETestEnv { let cacheConfig: BridgeConfigRoot["cache"]|undefined; if (opts.useRedis) { cacheConfig = { - redisUri: `${REDIS_DATABASE_URI}/${Math.ceil(Math.random() * 99)}`, + redisUri: `${REDIS_DATABASE_URI}/${workerID}`, } } @@ -248,7 +248,8 @@ export class E2ETestEnv { }], rooms: [], aliases: [], - } + }, + "de.sorunome.msc2409.push_ephemeral": true }; const app = await start(config, registration); app.listener.finaliseListeners(); diff --git a/spec/util/homerunner.ts b/spec/util/homerunner.ts index 52b71c432..dfe378c7d 100644 --- a/spec/util/homerunner.ts +++ b/spec/util/homerunner.ts @@ -62,29 +62,35 @@ export async function createHS(localparts: string[] = [], workerId: number, cryp SenderLocalpart: 'hookshot', RateLimited: false, ...{ASToken: asToken, - HSToken: hsToken}, + HSToken: hsToken, + SendEphemeral: true, + EnableEncryption: true}, }] }], } }); const [homeserverName, homeserver] = Object.entries(blueprintResponse.homeservers)[0]; // Skip AS user. - const users = Object.entries(homeserver.AccessTokens) + const users = await Promise.all(Object.entries(homeserver.AccessTokens) .filter(([_uId, accessToken]) => accessToken !== asToken) - .map(([userId, accessToken]) => { + .map(async ([userId, accessToken]) => { const cryptoStore = cryptoRootPath ? new RustSdkCryptoStorageProvider(path.join(cryptoRootPath, userId), RustSdkCryptoStoreType.Sqlite) : undefined; + const client = new E2ETestMatrixClient(homeserver.BaseURL, accessToken, new MemoryStorageProvider(), cryptoStore); + if (cryptoStore) { + await client.crypto.prepare(); + } + // Start syncing proactively. + await client.start(); return { userId: userId, accessToken, deviceId: homeserver.DeviceIDs[userId], - client: new E2ETestMatrixClient(homeserver.BaseURL, accessToken, new MemoryStorageProvider(), cryptoStore), + client, } } - ); + )); - // Start syncing proactively. - await Promise.all(users.map(u => void u.client.start())); return { users, id: blueprint, diff --git a/src/Stores/RedisStorageProvider.ts b/src/Stores/RedisStorageProvider.ts index b4ed8ac22..54be0a1d6 100644 --- a/src/Stores/RedisStorageProvider.ts +++ b/src/Stores/RedisStorageProvider.ts @@ -104,7 +104,7 @@ export class RedisStorageProvider extends RedisStorageContextualProvider impleme } public async addRegisteredUser(userId: string) { - this.redis.sadd(REGISTERED_USERS_KEY, [userId]); + await this.redis.sadd(REGISTERED_USERS_KEY, [userId]); } public async isUserRegistered(userId: string): Promise { @@ -112,7 +112,7 @@ export class RedisStorageProvider extends RedisStorageContextualProvider impleme } public async setTransactionCompleted(transactionId: string) { - this.redis.sadd(COMPLETED_TRANSACTIONS_KEY, [transactionId]); + await this.redis.sadd(COMPLETED_TRANSACTIONS_KEY, [transactionId]); } public async isTransactionCompleted(transactionId: string): Promise { diff --git a/src/appservice.ts b/src/appservice.ts index 9fbd5f346..7d380b1a2 100644 --- a/src/appservice.ts +++ b/src/appservice.ts @@ -45,7 +45,7 @@ export function getAppservice(config: BridgeConfig, registration: IAppserviceReg }, storage: storage, intentOptions: { - encryption: !!config.encryption, + encryption: !!cryptoStorage, }, cryptoStorage: cryptoStorage, }); From c533bf8a7b6e35a0dac0f0fd78aa103bce134654 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 19 Nov 2024 15:49:52 +0000 Subject: [PATCH 06/21] cleanup test --- spec/basic.spec.ts | 33 ---------------------------- spec/e2ee.spec.ts | 51 +++++++++++++++++++++++++------------------ spec/util/e2e-test.ts | 41 ++++++++++++++++++++++++---------- 3 files changed, 60 insertions(+), 65 deletions(-) diff --git a/spec/basic.spec.ts b/spec/basic.spec.ts index e375a18ab..2400487bb 100644 --- a/spec/basic.spec.ts +++ b/spec/basic.spec.ts @@ -26,37 +26,4 @@ describe('Basic test setup', () => { // Expect help text. expect((await msg).data.content.body).to.include('!hookshot help` - This help text\n'); }); - - // TODO: Move test to it's own generic connections file. - it('should be able to setup a webhook', async () => { - const user = testEnv.getUser('user'); - const testRoomId = await user.createRoom({ name: 'Test room', invite:[testEnv.botMxid] }); - await user.waitForRoomJoin({sender: testEnv.botMxid, roomId: testRoomId }); - await user.setUserPowerLevel(testEnv.botMxid, testRoomId, 50); - await user.sendText(testRoomId, "!hookshot webhook test-webhook"); - const inviteResponse = await user.waitForRoomInvite({sender: testEnv.botMxid}); - await user.waitForRoomEvent({ - eventType: 'm.room.message', sender: testEnv.botMxid, roomId: testRoomId, - body: 'Room configured to bridge webhooks. See admin room for secret url.' - }); - const webhookUrlMessage = user.waitForRoomEvent({ - eventType: 'm.room.message', sender: testEnv.botMxid, roomId: inviteResponse.roomId - }); - await user.joinRoom(inviteResponse.roomId); - const msgData = (await webhookUrlMessage).data.content.body; - const webhookUrl = msgData.split('\n')[2]; - const webhookNotice = user.waitForRoomEvent({ - eventType: 'm.room.message', sender: testEnv.botMxid, roomId: testRoomId, body: 'Hello world!' - }); - - // Send a webhook - await fetch(webhookUrl, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({text: 'Hello world!'}) - }); - - // And await the notice. - await webhookNotice; - }); }); diff --git a/spec/e2ee.spec.ts b/spec/e2ee.spec.ts index 88293207a..974bcf927 100644 --- a/spec/e2ee.spec.ts +++ b/spec/e2ee.spec.ts @@ -27,31 +27,40 @@ describe('End-2-End Encryption support', () => { const testRoomId = await user.createRoom({ name: 'Test room', invite:[testEnv.botMxid], initial_state: CryptoRoomState}); await user.setUserPowerLevel(testEnv.botMxid, testRoomId, 50); await user.waitForRoomJoin({sender: testEnv.botMxid, roomId: testRoomId }); - await new Promise((r) => setTimeout(r, 5000)); await user.sendText(testRoomId, "!hookshot help"); - // const inviteResponse = await user.waitForRoomInvite({sender: testEnv.botMxid}); await user.waitForRoomEvent({ eventType: 'm.room.message', sender: testEnv.botMxid, roomId: testRoomId, - // body: 'Room configured to bridge webhooks. See admin room for secret url.' }); - // const webhookUrlMessage = user.waitForRoomEvent({ - // eventType: 'm.room.message', sender: testEnv.botMxid, roomId: inviteResponse.roomId - // }); - // await user.joinRoom(inviteResponse.roomId); - // const msgData = (await webhookUrlMessage).data.content.body; - // const webhookUrl = msgData.split('\n')[2]; - // const webhookNotice = user.waitForRoomEvent({ - // eventType: 'm.room.message', sender: testEnv.botMxid, roomId: testRoomId, body: 'Hello world!' - // }); + }); + it('should send notices in an encrypted format', async () => { + const user = testEnv.getUser('user'); + const testRoomId = await user.createRoom({ name: 'Test room', invite:[testEnv.botMxid], initial_state: CryptoRoomState}); + await user.setUserPowerLevel(testEnv.botMxid, testRoomId, 50); + await user.waitForRoomJoin({sender: testEnv.botMxid, roomId: testRoomId }); + await user.sendText(testRoomId, "!hookshot webhook test-webhook"); + const inviteResponse = await user.waitForRoomInvite({sender: testEnv.botMxid}); + await user.waitForEncryptedEvent({ + eventType: 'm.room.message', sender: testEnv.botMxid, roomId: testRoomId, + body: 'Room configured to bridge webhooks. See admin room for secret url.' + }); + const webhookUrlMessage = user.waitForEncryptedEvent({ + eventType: 'm.room.message', sender: testEnv.botMxid, roomId: inviteResponse.roomId + }); + await user.joinRoom(inviteResponse.roomId); + const msgData = (await webhookUrlMessage).data.content.body; + const webhookUrl = msgData.split('\n')[2]; + const webhookNotice = user.waitForEncryptedEvent({ + eventType: 'm.room.message', sender: testEnv.botMxid, roomId: testRoomId, body: 'Hello world!' + }); - // // Send a webhook - // await fetch(webhookUrl, { - // method: 'POST', - // headers: { 'Content-Type': 'application/json' }, - // body: JSON.stringify({text: 'Hello world!'}) - // }); + // Send a webhook + await fetch(webhookUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({text: 'Hello world!'}) + }); - // // And await the notice. - // await webhookNotice; - }); + // And await the notice. + console.log(await webhookNotice); + }, 90000); }); diff --git a/spec/util/e2e-test.ts b/spec/util/e2e-test.ts index cd6bd5165..45d7f8552 100644 --- a/spec/util/e2e-test.ts +++ b/spec/util/e2e-test.ts @@ -7,7 +7,7 @@ import { RSAKeyPairOptions, generateKeyPair } from "node:crypto"; import path from "node:path"; import Redis from "ioredis"; -const WAIT_EVENT_TIMEOUT = 10000; +const WAIT_EVENT_TIMEOUT = 20000; export const E2ESetupTestTimeout = 60000; const REDIS_DATABASE_URI = process.env.HOOKSHOT_E2E_REDIS_DB_URI ?? "redis://localhost:6379"; @@ -18,6 +18,13 @@ interface Opts { useRedis?: boolean, } +interface WaitForEventResponse> { + roomId: string, + data: { + sender: string, type: string, state_key?: string, content: T, event_id: string, + } +} + export class E2ETestMatrixClient extends MatrixClient { public async waitForPowerLevel( @@ -59,13 +66,10 @@ export class E2ETestMatrixClient extends MatrixClient { }, `Timed out waiting for powerlevel from in ${roomId}`) } - public async waitForRoomEvent>( - opts: {eventType: string, sender: string, roomId?: string, stateKey?: string, body?: string} - ): Promise<{roomId: string, data: { - sender: string, type: string, state_key?: string, content: T, event_id: string, - }}> { - const {eventType, sender, roomId, stateKey} = opts; - return this.waitForEvent('room.event', (eventRoomId: string, eventData: { + private async innerWaitForRoomEvent>( + {eventType, sender, roomId, stateKey, eventId, body}: {eventType: string, sender: string, roomId?: string, stateKey?: string, body?: string, eventId?: string}, expectEncrypted: boolean, + ): Promise> { + return this.waitForEvent(expectEncrypted ? 'room.decrypted_event' : 'room.event', (eventRoomId: string, eventData: { sender: string, type: string, state_key?: string, content: T, event_id: string, }) => { if (eventData.sender !== sender) { @@ -77,21 +81,36 @@ export class E2ETestMatrixClient extends MatrixClient { if (roomId && eventRoomId !== roomId) { return undefined; } + if (eventId && eventData.event_id !== eventId) { + return undefined; + } if (stateKey !== undefined && eventData.state_key !== stateKey) { return undefined; } - const body = 'body' in eventData.content && eventData.content.body; - if (opts.body && body !== opts.body) { + const evtBody = 'body' in eventData.content && eventData.content.body; + if (body && body !== evtBody) { return undefined; } console.info( // eslint-disable-next-line max-len - `${eventRoomId} ${eventData.event_id} ${eventData.type} ${eventData.sender} ${eventData.state_key ?? body ?? ''}` + `${eventRoomId} ${eventData.event_id} ${eventData.type} ${eventData.sender} ${eventData.state_key ?? evtBody ?? ''}` ); return {roomId: eventRoomId, data: eventData}; }, `Timed out waiting for ${eventType} from ${sender} in ${roomId || "any room"}`) } + public async waitForRoomEvent>( + opts: Parameters[0] + ): Promise> { + return this.innerWaitForRoomEvent(opts, false); + } + + public async waitForEncryptedEvent>( + opts: Parameters[0] + ): Promise> { + return this.innerWaitForRoomEvent(opts, true); + } + public async waitForRoomJoin( opts: {sender: string, roomId?: string} ): Promise<{roomId: string, data: unknown}> { From 76d99c3f9b0257aed7470d82d05d4863022e8b46 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 19 Nov 2024 15:49:57 +0000 Subject: [PATCH 07/21] opportunistic lint --- src/feeds/parser.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/feeds/parser.rs b/src/feeds/parser.rs index 631caee81..6b7ffadc3 100644 --- a/src/feeds/parser.rs +++ b/src/feeds/parser.rs @@ -60,7 +60,7 @@ fn parse_channel_to_js_result(channel: &Channel) -> JsRssChannel { .and_then(|i| i.permalink.then(|| i.value.to_string())) }), id: item.guid().map(|f| f.value().to_string()), - id_is_permalink: item.guid().map_or(false, |f| f.is_permalink()), + id_is_permalink: item.guid().is_some_and(|f| f.is_permalink()), pubdate: item.pub_date().map(String::from), summary: item.description().map(String::from), author: item.author().map(String::from), @@ -106,7 +106,7 @@ fn parse_feed_to_js_result(feed: &Feed) -> JsRssChannel { link: item .links() .iter() - .find(|l| l.mime_type.as_ref().map_or(false, |t| t == "text/html")) + .find(|l| l.mime_type.as_ref().is_some_and(|t| t == "text/html")) .or_else(|| item.links().first()) .map(|f| f.href.clone()), id: Some(item.id.clone()), From 3ddf0c3884fddca16be40f9f3255ea55639ef9c1 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 19 Nov 2024 16:26:18 +0000 Subject: [PATCH 08/21] tiny bit of cleanup --- spec/e2ee.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/e2ee.spec.ts b/spec/e2ee.spec.ts index 974bcf927..a5c1d8219 100644 --- a/spec/e2ee.spec.ts +++ b/spec/e2ee.spec.ts @@ -61,6 +61,6 @@ describe('End-2-End Encryption support', () => { }); // And await the notice. - console.log(await webhookNotice); - }, 90000); + await webhookNotice; + }); }); From 9821a2e75a371dbd39ba6749b5492ba7b19cdbdf Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 26 Nov 2024 09:13:04 +0000 Subject: [PATCH 09/21] remove ref --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 631078f12..6cd9ba48d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -95,7 +95,6 @@ jobs: uses: actions/checkout@v4 with: repository: matrix-org/complement - ref: e1b98b67a3ca19551fe8640d10bd3611c0995288 - name: Get complement git sha id: gitsha run: echo sha=`git rev-parse --short HEAD` >> "$GITHUB_OUTPUT" From 754aa090f3f2515eb3e64d81e5ec84a4a93411c7 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 26 Nov 2024 09:18:51 +0000 Subject: [PATCH 10/21] tweak to homerunner --- spec/util/homerunner.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/spec/util/homerunner.ts b/spec/util/homerunner.ts index dfe378c7d..0a0962632 100644 --- a/spec/util/homerunner.ts +++ b/spec/util/homerunner.ts @@ -130,7 +130,7 @@ export async function registerUser( .update(password).update("\x00") .update(user.admin ? 'admin' : 'notadmin') .digest('hex'); - return await fetch(registerUrl, { method: "POST", body: JSON.stringify( + const req = await fetch(registerUrl, { method: "POST", body: JSON.stringify( { nonce, username: user.username, @@ -138,8 +138,10 @@ export async function registerUser( admin: user.admin, mac: hmac, } - )}).then(res => res.json()).then(res => ({ - mxid: (res as {user_id: string}).user_id, - client: new MatrixClient(homeserverUrl, (res as {access_token: string}).access_token), - })).catch(err => { console.log(err.response.body); throw new Error(`Failed to register user: ${err}`); }); + )}); + const res = await req.json() as {user_id: string, access_token: string}; + return { + mxid: res.user_id, + client: new MatrixClient(homeserverUrl, res.access_token), + }; } From 88be70c7e2c332b150a37736191a40936c2286af Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 26 Nov 2024 09:45:50 +0000 Subject: [PATCH 11/21] switch to nightly images for Synapse (to test E2EE) --- spec/util/homerunner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/util/homerunner.ts b/spec/util/homerunner.ts index 0a0962632..16e4a1750 100644 --- a/spec/util/homerunner.ts +++ b/spec/util/homerunner.ts @@ -4,7 +4,7 @@ import { Homerunner } from "homerunner-client"; import { E2ETestMatrixClient } from "./e2e-test"; import path from "node:path"; -const HOMERUNNER_IMAGE = process.env.HOMERUNNER_IMAGE || 'ghcr.io/element-hq/synapse/complement-synapse:latest'; +const HOMERUNNER_IMAGE = process.env.HOMERUNNER_IMAGE || 'ghcr.io/element-hq/synapse/complement-synapse:nightly'; export const DEFAULT_REGISTRATION_SHARED_SECRET = ( process.env.REGISTRATION_SHARED_SECRET || 'complement' ); From 1695b8218f329da2615c4e09c56a6defb7f30c33 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 26 Nov 2024 10:03:59 +0000 Subject: [PATCH 12/21] use nightly --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6cd9ba48d..9b8528415 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -157,7 +157,7 @@ jobs: timeout-minutes: 10 env: HOMERUNNER_SPAWN_HS_TIMEOUT_SECS: 100 - HOMERUNNER_IMAGE: ghcr.io/element-hq/synapse/complement-synapse:latest + HOMERUNNER_IMAGE: ghcr.io/element-hq/synapse/complement-synapse:nightly NODE_OPTIONS: --dns-result-order ipv4first REDIS_DATABASE_URI: "redis://localhost:6379" run: | From 9428fa255cc2ecf490da20f567ab89139393d8ac Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 26 Nov 2024 10:07:12 +0000 Subject: [PATCH 13/21] newsfile. --- changelog.d/989.feature | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog.d/989.feature diff --git a/changelog.d/989.feature b/changelog.d/989.feature new file mode 100644 index 000000000..c396d2c08 --- /dev/null +++ b/changelog.d/989.feature @@ -0,0 +1,2 @@ +Support for E2E Encrypted rooms is now considered stable and can be enabled in production. Please see the [documentation](https://matrix-org.github.io/matrix-hookshot/latest/advanced/encryption.html) +on the requirements for enabling support. \ No newline at end of file From 7c92e5100564e450a14c69a6ed3dac41d935608e Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 26 Nov 2024 14:58:24 +0000 Subject: [PATCH 14/21] Update bot sdk to support authenticated media (now that Synapse requires it) --- package.json | 2 +- yarn.lock | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 9183f72ad..530f45113 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "jira-client": "^8.2.2", "markdown-it": "^14.0.0", "matrix-appservice-bridge": "^9.0.1", - "matrix-bot-sdk": "npm:@vector-im/matrix-bot-sdk@^0.7.0-specific-device-2", + "matrix-bot-sdk": "npm:@vector-im/matrix-bot-sdk@v0.7.1-element.6", "matrix-widget-api": "^1.6.0", "micromatch": "^4.0.8", "mime": "^4.0.1", diff --git a/yarn.lock b/yarn.lock index 73e043521..1624591dd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1027,14 +1027,6 @@ dependencies: "@lezer/common" "^1.0.0" -"@matrix-org/matrix-sdk-crypto-nodejs@0.1.0-beta.11": - version "0.1.0-beta.11" - resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-nodejs/-/matrix-sdk-crypto-nodejs-0.1.0-beta.11.tgz#537cd7a7bbce1d9745b812a5a7ffa9a5944e146c" - integrity sha512-z5adcQo4o0UAry4zs6JHGxbTDlYTUMKUfpOpigmso65ETBDumbeTSQCWRw8UeUV7aCAyVoHARqDTol9SrauEFA== - dependencies: - https-proxy-agent "^5.0.1" - node-downloader-helper "^2.1.5" - "@matrix-org/matrix-sdk-crypto-nodejs@0.1.0-beta.6": version "0.1.0-beta.6" resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-nodejs/-/matrix-sdk-crypto-nodejs-0.1.0-beta.6.tgz#0ecae51103ee3c107af0d6d0738f33eb7cc9857e" @@ -1043,6 +1035,14 @@ https-proxy-agent "^5.0.1" node-downloader-helper "^2.1.5" +"@matrix-org/matrix-sdk-crypto-nodejs@0.2.0-beta.1": + version "0.2.0-beta.1" + resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-nodejs/-/matrix-sdk-crypto-nodejs-0.2.0-beta.1.tgz#b696707ccfa944cfed3c96cf7e54799b0f1e3329" + integrity sha512-CgbOKORfD6dvYgQTPhfN73H1RbQknrFkMnRRwCIJMt15iL2AF1gEowgbrlGhkbG6gNng4CgPnKs1iHKCRrhvmA== + dependencies: + https-proxy-agent "^5.0.1" + node-downloader-helper "^2.1.5" + "@mdn/browser-compat-data@^5.2.34", "@mdn/browser-compat-data@^5.3.13": version "5.5.1" resolved "https://registry.yarnpkg.com/@mdn/browser-compat-data/-/browser-compat-data-5.5.1.tgz#cd480874a4ebb97010f488feb8204ac035a86332" @@ -1933,7 +1933,7 @@ "@types/range-parser" "*" "@types/send" "*" -"@types/express@^4.17.13", "@types/express@^4.17.14", "@types/express@^4.17.20": +"@types/express@^4.17.13", "@types/express@^4.17.14", "@types/express@^4.17.21": version "4.17.21" resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.21.tgz#c26d4a151e60efe0084b23dc3369ebc631ed192d" integrity sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ== @@ -5949,13 +5949,13 @@ matrix-appservice@^2.0.0: request-promise "^4.2.6" sanitize-html "^2.8.0" -"matrix-bot-sdk@npm:@vector-im/matrix-bot-sdk@^0.7.0-specific-device-2": - version "0.7.0-specific-device-2" - resolved "https://registry.yarnpkg.com/@vector-im/matrix-bot-sdk/-/matrix-bot-sdk-0.7.0-specific-device-2.tgz#3682e14708979a6f24cc19f3103e3292ed08bbea" - integrity sha512-97h2tIlcK6/3wEuLN3x6/LM9TITVISnnbjUo/9nVbqkDvSQ2TNFURxfAqjUfFmgQwo0o3KnhvaS6dZITrBvj6A== +"matrix-bot-sdk@npm:@vector-im/matrix-bot-sdk@v0.7.1-element.6": + version "0.7.1-element.6" + resolved "https://registry.yarnpkg.com/@vector-im/matrix-bot-sdk/-/matrix-bot-sdk-0.7.1-element.6.tgz#d1f8a86d3bd60084d92d150f42a48b25199871e1" + integrity sha512-0KfyTpQV5eyY4vPUZW89t7EZf1YF0UyFkyYqpsxL/6S7XIlbTMC4onod7vx/QpKC0lSREmwIiXx2JSjExP6CIw== dependencies: - "@matrix-org/matrix-sdk-crypto-nodejs" "0.1.0-beta.11" - "@types/express" "^4.17.20" + "@matrix-org/matrix-sdk-crypto-nodejs" "0.2.0-beta.1" + "@types/express" "^4.17.21" another-json "^0.2.0" async-lock "^1.4.0" chalk "4" From 36e650b3f10b51ecb3cbf15c07da450f433c6826 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 26 Nov 2024 15:07:08 +0000 Subject: [PATCH 15/21] fix typings --- src/Managers/BotUsersManager.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/Managers/BotUsersManager.ts b/src/Managers/BotUsersManager.ts index ab7ab871c..5927cd117 100644 --- a/src/Managers/BotUsersManager.ts +++ b/src/Managers/BotUsersManager.ts @@ -171,13 +171,10 @@ export default class BotUsersManager { // Determine if an avatar update is needed if (profile.avatar_url) { try { - const res = await axios.get( - botUser.intent.underlyingClient.mxcToHttp(profile.avatar_url), - { responseType: "arraybuffer" }, - ); + const res = await botUser.intent.underlyingClient.downloadContent(profile.avatar_url); const currentAvatarImage = { - image: Buffer.from(res.data), - contentType: res.headers["content-type"], + image: res.data, + contentType: res.contentType, }; if ( currentAvatarImage.image.equals(avatarImage.image) From 0f88b4d0a8818852cab65e20e5c6f8d5fceed498 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 26 Nov 2024 15:11:31 +0000 Subject: [PATCH 16/21] MatrixError --- tests/IntentUtilsTest.ts | 2 +- tests/connections/GenericHookTest.ts | 4 ++-- tests/utils/IntentMock.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/IntentUtilsTest.ts b/tests/IntentUtilsTest.ts index 1fcae634f..7628c09c0 100644 --- a/tests/IntentUtilsTest.ts +++ b/tests/IntentUtilsTest.ts @@ -46,7 +46,7 @@ describe("IntentUtils", () => { // This should fail the first time, then pass once we've tried to invite the user targetIntent.ensureJoined = () => { - throw new MatrixError({ errcode: "FORCED_FAILURE", error: "Test forced error"}, 500) + throw new MatrixError({ errcode: "FORCED_FAILURE", error: "Test forced error"}, 500, { }) }; try { ensureUserIsInRoom(targetIntent, matrixClient, ROOM_ID); diff --git a/tests/connections/GenericHookTest.ts b/tests/connections/GenericHookTest.ts index b683d6a6f..ef4ed5619 100644 --- a/tests/connections/GenericHookTest.ts +++ b/tests/connections/GenericHookTest.ts @@ -310,7 +310,7 @@ describe("GenericHookConnection", () => { return roomId; } expect(roomId).to.equal(ROOM_ID); - throw new MatrixError({ errcode: "M_FORBIDDEN", error: "Test forced error"}, 401) + throw new MatrixError({ errcode: "M_FORBIDDEN", error: "Test forced error"}, 401, { }) }; // This should invite the puppet user. @@ -333,7 +333,7 @@ describe("GenericHookConnection", () => { // This should fail the first time, then pass once we've tried to invite the user intent.ensureJoined = () => { - throw new MatrixError({ errcode: "FORCED_FAILURE", error: "Test forced error"}, 500) + throw new MatrixError({ errcode: "FORCED_FAILURE", error: "Test forced error"}, 500, { }) }; try { // regression test covering https://github.com/matrix-org/matrix-hookshot/issues/625 diff --git a/tests/utils/IntentMock.ts b/tests/utils/IntentMock.ts index 5877055b0..078091b17 100644 --- a/tests/utils/IntentMock.ts +++ b/tests/utils/IntentMock.ts @@ -39,7 +39,7 @@ export class MatrixClientMock { throw new MatrixError({ errcode: 'M_NOT_FOUND', error: 'Test error: No account data', - }, 404); + }, 404, { }); } async setRoomAccountData(key: string, roomId: string, value: string): Promise { From c44649ce1eac3a5851a1875da7f65843ec02f492 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 26 Nov 2024 15:14:03 +0000 Subject: [PATCH 17/21] one more --- tests/IntentUtilsTest.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/IntentUtilsTest.ts b/tests/IntentUtilsTest.ts index 7628c09c0..38bcd6dc0 100644 --- a/tests/IntentUtilsTest.ts +++ b/tests/IntentUtilsTest.ts @@ -25,7 +25,7 @@ describe("IntentUtils", () => { return; } expect(roomId).to.equal(ROOM_ID); - throw new MatrixError({ errcode: "M_FORBIDDEN", error: "Test forced error"}, 401) + throw new MatrixError({ errcode: "M_FORBIDDEN", error: "Test forced error"}, 401, { }) }; // This should invite the puppet user. From 1e3c4b9405d793b90f8e4f50ca3fa85246dd449e Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 26 Nov 2024 15:39:34 +0000 Subject: [PATCH 18/21] Graduate the encryption property to stable. --- config.sample.yml | 22 ++++++----- src/config/Config.ts | 64 +++++++------------------------ src/config/Defaults.ts | 6 +-- src/config/sections/encryption.ts | 27 +++++++++++++ 4 files changed, 56 insertions(+), 63 deletions(-) create mode 100644 src/config/sections/encryption.ts diff --git a/config.sample.yml b/config.sample.yml index b351530ab..9a54339b1 100644 --- a/config.sample.yml +++ b/config.sample.yml @@ -1,5 +1,11 @@ # This is an example configuration file +logging: + # Logging settings. You can have a severity debug,info,warn,error + level: info + colorize: true + json: false + timestampFormat: HH:mm:ss:SSS bridge: # Basic homeserver configuration domain: example.com @@ -11,12 +17,6 @@ passFile: # A passkey used to encrypt tokens stored inside the bridge. # Run openssl genpkey -out passkey.pem -outform PEM -algorithm RSA -pkeyopt rsa_keygen_bits:4096 to generate ./passkey.pem -logging: - # Logging settings. You can have a severity debug,info,warn,error - level: info - colorize: true - json: false - timestampFormat: HH:mm:ss:SSS listeners: # HTTP Listener configuration. # Bind resource endpoints to ports and addresses. @@ -143,10 +143,12 @@ listeners: # # For encryption to work, this must be configured. # redisUri: redis://localhost:6379 -#queue: -# # (Optional) Message queue configuration options for large scale deployments. -# # For encryption to work, this must not be configured. -# redisUri: redis://localhost:6379 +#encryption: +# # (Optional) Configuration for encryption support in the bridge. +# # If omitted, encryption support will be disabled. +# storagePath: +# # Path to the directory used to store encryption files. These files must be persist between restarts of the service. +# ./cryptostore #widgets: # # (Optional) EXPERIMENTAL support for complimentary widgets diff --git a/src/config/Config.ts b/src/config/Config.ts index 76c55757a..8c2061b8f 100644 --- a/src/config/Config.ts +++ b/src/config/Config.ts @@ -13,6 +13,7 @@ import { Logger } from "matrix-appservice-bridge"; import { BridgeConfigCache } from "./sections/cache"; import { BridgeConfigGenericWebhooks, BridgeConfigQueue, BridgeGenericWebhooksConfigYAML } from "./sections"; import { GenericHookServiceConfig } from "../Connections"; +import { BridgeConfigEncryption } from "./sections/encryption"; const log = new Logger("Config"); @@ -356,8 +357,6 @@ interface BridgeConfigBridge { mediaUrl?: string; port: number; bindAddress: string; - // Removed - pantalaimon?: never; } interface BridgeConfigWebhook { @@ -376,13 +375,7 @@ interface BridgeConfigBot { displayname?: string; avatar?: string; } -interface BridgeConfigEncryption { - storagePath: string; - /** - * @deprecated This is no longer supported. - */ - useLegacySledStore?: boolean; -} + export interface BridgeConfigServiceBot { localpart: string; @@ -418,7 +411,11 @@ export interface BridgeConfigRoot { bot?: BridgeConfigBot; bridge: BridgeConfigBridge; cache?: BridgeConfigCache; - experimentalEncryption?: BridgeConfigEncryption; + /** + * @deprecated Old, unsupported encryption propety. + */ + experimentalEncryption?: never; + encryption?: BridgeConfigEncryption; feeds?: BridgeConfigFeedsYAML; figma?: BridgeConfigFigma; generic?: BridgeGenericWebhooksConfigYAML; @@ -446,9 +443,7 @@ export class BridgeConfig { For encryption to work, this must be configured.`, true) public readonly cache?: BridgeConfigCache; @configKey(`Configuration for encryption support in the bridge. - If omitted, encryption support will be disabled. - This feature is HIGHLY EXPERIMENTAL AND SUBJECT TO CHANGE. - For more details, see https://github.com/matrix-org/matrix-hookshot/issues/594.`, true) + If omitted, encryption support will be disabled.`, true) public readonly encryption?: BridgeConfigEncryption; @configKey(`Message queue configuration options for large scale deployments. For encryption to work, this must not be configured.`, true) @@ -503,6 +498,9 @@ export class BridgeConfig { constructor(configData: BridgeConfigRoot, env?: {[key: string]: string|undefined}) { + this.logging = configData.logging || { + level: "info", + } this.bridge = configData.bridge; assert.ok(this.bridge); this.github = configData.github && new BridgeConfigGitHub(configData.github); @@ -551,14 +549,11 @@ export class BridgeConfig { } } - - this.encryption = configData.experimentalEncryption; - - - this.logging = configData.logging || { - level: "info", + if (configData.experimentalEncryption) { + throw new ConfigError("experimentalEncryption", `This key is now called 'encryption'. Please adjust your config file.`) } + this.encryption = configData.encryption && new BridgeConfigEncryption(configData.encryption, this.cache, this.queue); this.widgets = configData.widgets && new BridgeWidgetConfig(configData.widgets); this.sentry = configData.sentry; @@ -644,37 +639,6 @@ export class BridgeConfig { log.warn("The `widgets.openIdOverrides` config value SHOULD NOT be used in a production environment.") } - if (this.bridge.pantalaimon) { - throw new ConfigError("bridge.pantalaimon", "Pantalaimon support has been removed. Encrypted bridges should now use the `experimentalEncryption` config option"); - } - - if (this.encryption) { - log.warn(` -You have enabled encryption support in the bridge. This feature is HIGHLY EXPERIMENTAL AND SUBJECT TO CHANGE. -For more details, see https://github.com/matrix-org/matrix-hookshot/issues/594. - `); - - if (!this.encryption.storagePath) { - throw new ConfigError("experimentalEncryption.storagePath", "The crypto storage path must not be empty."); - } - - if (this.encryption.useLegacySledStore) { - throw new ConfigError( - "experimentalEncryption.useLegacySledStore", ` -The Sled crypto store format is no longer supported. -Please back up your crypto store at ${this.encryption.storagePath}, -remove "useLegacySledStore" from your configuration file, and restart Hookshot. - `); - } - if (!this.cache) { - throw new ConfigError("cache", "Encryption requires the Redis cache to be enabled."); - } - - if (this.queue) { - throw new ConfigError("queue", "Encryption does not support message queues."); - } - } - if (this.figma?.overrideUserId) { log.warn("The `figma.overrideUserId` config value is deprecated. A service bot should be configured instead."); } diff --git a/src/config/Defaults.ts b/src/config/Defaults.ts index 876870ea2..8a110c36e 100644 --- a/src/config/Defaults.ts +++ b/src/config/Defaults.ts @@ -15,9 +15,6 @@ export const DefaultConfigRoot: BridgeConfigRoot = { port: 9993, bindAddress: "127.0.0.1", }, - queue: { - redisUri: "redis://localhost:6379", - }, cache: { redisUri: "redis://localhost:6379", }, @@ -154,6 +151,9 @@ export const DefaultConfigRoot: BridgeConfigRoot = { sentry: { dsn: "https://examplePublicKey@o0.ingest.sentry.io/0", environment: "production" + }, + encryption: { + storagePath: "./cryptostore" } }; diff --git a/src/config/sections/encryption.ts b/src/config/sections/encryption.ts new file mode 100644 index 000000000..1dc96d167 --- /dev/null +++ b/src/config/sections/encryption.ts @@ -0,0 +1,27 @@ +import { ConfigError } from "../../errors"; +import { configKey } from "../Decorators"; +import { BridgeConfigCache } from "./cache"; +import { BridgeConfigQueue } from "./queue"; + +interface BridgeConfigEncryptionYAML { + storagePath: string; +} + +export class BridgeConfigEncryption { + @configKey("Path to the directory used to store encryption files. These files must be persist between restarts of the service.") + public readonly storagePath: string; + + constructor(config: BridgeConfigEncryptionYAML, cache: unknown|undefined, queue: unknown|undefined) { + if (typeof config.storagePath !== "string" || !config.storagePath) { + throw new ConfigError("encryption.storagePath", "The crypto storage path must not be empty."); + } + this.storagePath = config.storagePath; + + if (!cache) { + throw new ConfigError("cache", "Encryption requires the Redis cache to be enabled."); + } + if (queue) { + throw new ConfigError("queue", "Encryption does not support message queues."); + } + } +} From 3dd5d587917cf4f1d1bc31ec5c38246b9e01ceff Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 26 Nov 2024 15:40:52 +0000 Subject: [PATCH 19/21] update test config --- spec/util/e2e-test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spec/util/e2e-test.ts b/spec/util/e2e-test.ts index 45d7f8552..7d842c7b3 100644 --- a/spec/util/e2e-test.ts +++ b/spec/util/e2e-test.ts @@ -248,9 +248,8 @@ export class E2ETestEnv { }], passFile: keyPath, ...(opts.enableE2EE ? { - experimentalEncryption: { + encryption: { storagePath: path.join(dir, 'crypto-store'), - useLegacySledStore: false, } } : undefined), cache: cacheConfig, From f682987ea7dda76212fe8a601a519b81421ae953 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 26 Nov 2024 15:50:32 +0000 Subject: [PATCH 20/21] Update encryption docs. --- docs/advanced/encryption.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/advanced/encryption.md b/docs/advanced/encryption.md index 4133f6be9..6d74c989e 100644 --- a/docs/advanced/encryption.md +++ b/docs/advanced/encryption.md @@ -1,27 +1,32 @@ Encryption ========== -
-Encryption support is HIGHLY EXPERIMENTAL AND SUBJECT TO CHANGE. It should not be enabled for production workloads. -For more details, see issue 594. +
+Support for encryption is considered stable, but the underlying specification changes are not yet. + +Hookshot supports end-to-bridge encryption via [MSC3202](https://github.com/matrix-org/matrix-spec-proposals/pull/3202), and [MSC4203](https://github.com/matrix-org/matrix-spec-proposals/pull/4203). Hookshot needs to be configured against a a homeserver that supports these features, such as [Synapse](#running-with-synapse). + +Please check with your homeserver implementation before reporting bugs against matrix-hookshot.
-Hookshot supports end-to-bridge encryption via [MSC3202](https://github.com/matrix-org/matrix-spec-proposals/pull/3202). As such, encryption requires Hookshot to be connected to a homeserver that supports that MSC, such as [Synapse](#running-with-synapse). + ## Enabling encryption in Hookshot In order for Hookshot to use encryption, it must be configured as follows: -- The `experimentalEncryption.storagePath` setting must point to a directory that Hookshot has permissions to write files into. If running with Docker, this path should be within a volume (for persistency). Hookshot uses this directory for its crypto store (i.e. long-lived state relating to its encryption keys). +- The `encryption.storagePath` setting must point to a directory that Hookshot has permissions to write files into. If running with Docker, this path should be within a volume (for persistency). Hookshot uses this directory for its crypto store (i.e. long-lived state relating to its encryption keys). - Once a crypto store has been initialized, its files must not be modified, and Hookshot cannot be configured to use another crypto store of the same type as one it has used before. If a crypto store's files get lost or corrupted, Hookshot may fail to start up, or may be unable to decrypt command messages. To fix such issues, stop Hookshot, then reset its crypto store by running `yarn start:resetcrypto`. - [Redis](./workers.md) must be enabled. Note that worker mode is not yet supported with encryption, so `queue` MUST **NOT be configured**. -If you ever reset your homeserver's state, ensure you also reset Hookshot's encryption state. This includes clearing the `experimentalEncryption.storagePath` directory and all worker state stored in your redis instance. Otherwise, Hookshot may fail on start up with registration errors. +If you ever reset your homeserver's state, ensure you also reset Hookshot's encryption state. This includes clearing the `storagePath` directory and all worker state stored in your redis instance. Otherwise, Hookshot may fail on start up with registration errors. Also ensure that Hookshot's appservice registration file contains every line from `registration.sample.yml` that appears after the `If enabling encryption` comment. Note that changing the registration file may require restarting the homeserver that Hookshot is connected to. ## Running with Synapse -[Synapse](https://github.com/matrix-org/synapse/) has functional support for MSC3202 as of [v1.63.0](https://github.com/matrix-org/synapse/releases/tag/v1.63.0). To enable it, add the following section to Synapse's configuration file (typically named `homeserver.yaml`): +[Synapse](https://github.com/matrix-org/synapse/) has functional support for MSC3202 and MSC4203 as of [v1.63.0](https://github.com/matrix-org/synapse/releases/tag/v1.63.0). To enable it, add the following section to Synapse's configuration file (typically named `homeserver.yaml`): + +You may notice that MSC2409 is not listed above. Due to the changes being split out from MSC2409, `msc2409_to_device_messages_enabled` refers to MSC4203. ```yaml experimental_features: @@ -29,3 +34,4 @@ experimental_features: msc3202_transaction_extensions: true msc2409_to_device_messages_enabled: true ``` + From 8c37824181fa510f8d3e3e8d9fbeff94ea4678fd Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 26 Nov 2024 15:58:52 +0000 Subject: [PATCH 21/21] fix some old config bits --- tests/config/config.ts | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/config/config.ts b/tests/config/config.ts index 22a1a7528..0ac75e0a9 100644 --- a/tests/config/config.ts +++ b/tests/config/config.ts @@ -38,9 +38,13 @@ describe("Config/BridgeConfig", () => { expect(config.cache?.redisUri).to.equal("redis://bark:6379"); }); it("with monolithic disabled", () => { - const config = new BridgeConfig({ ...DefaultConfigRoot, queue: { - monolithic: false - }}); + const config = new BridgeConfig({ + ...DefaultConfigRoot, + encryption: undefined, + queue: { + monolithic: false + } + }); expect(config.queue).to.deep.equal({ monolithic: false, }); @@ -49,9 +53,13 @@ describe("Config/BridgeConfig", () => { }); describe("will handle the queue option", () => { it("with redisUri", () => { - const config = new BridgeConfig({ ...DefaultConfigRoot, queue: { - redisUri: "redis://localhost:6379" - }, cache: undefined}); + const config = new BridgeConfig({ ...DefaultConfigRoot, + encryption: undefined, + queue: { + redisUri: "redis://localhost:6379" + }, + cache: undefined + }); expect(config.queue).to.deep.equal({ redisUri: "redis://localhost:6379" });