From 786cdb1d88347f4cb55e166c2aa3fe5df5e8bd38 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 15 Jan 2025 13:14:05 -0500 Subject: [PATCH 1/4] Distinguish room state and timeline events in embedded clients (#4574) * Distinguish room state and timeline events in embedded clients This change enables room widget clients to take advantage of the more reliable method of communicating room state over the widget API provided by a recent update to MSC2762. * Add missing awaits * Upgrade matrix-widget-api --- spec/unit/embedded.spec.ts | 56 +++++++++++++++++------- src/embedded.ts | 89 ++++++++++++++++++-------------------- yarn.lock | 6 +-- 3 files changed, 84 insertions(+), 67 deletions(-) diff --git a/spec/unit/embedded.spec.ts b/spec/unit/embedded.spec.ts index c5ef3a6a2c6..5e35c02c969 100644 --- a/spec/unit/embedded.spec.ts +++ b/spec/unit/embedded.spec.ts @@ -28,7 +28,6 @@ import { WidgetApiToWidgetAction, MatrixCapabilities, ITurnServer, - IRoomEvent, IOpenIDCredentials, ISendEventFromWidgetResponseData, WidgetApiResponseError, @@ -635,12 +634,20 @@ describe("RoomWidgetClient", () => { }); it("receives", async () => { - await makeClient({ receiveState: [{ eventType: "org.example.foo", stateKey: "bar" }] }); + const init = makeClient({ receiveState: [{ eventType: "org.example.foo", stateKey: "bar" }] }); expect(widgetApi.requestCapabilityForRoomTimeline).toHaveBeenCalledWith("!1:example.org"); expect(widgetApi.requestCapabilityToReceiveState).toHaveBeenCalledWith("org.example.foo", "bar"); + // Client needs to be told that the room state is loaded + widgetApi.emit( + `action:${WidgetApiToWidgetAction.UpdateState}`, + new CustomEvent(`action:${WidgetApiToWidgetAction.UpdateState}`, { detail: { data: { state: [] } } }), + ); + await init; const emittedEvent = new Promise((resolve) => client.once(ClientEvent.Event, resolve)); const emittedSync = new Promise((resolve) => client.once(ClientEvent.Sync, resolve)); + // Let's assume that a state event comes in but it doesn't actually + // update the state of the room just yet (maybe it's unauthorized) widgetApi.emit( `action:${WidgetApiToWidgetAction.SendEvent}`, new CustomEvent(`action:${WidgetApiToWidgetAction.SendEvent}`, { detail: { data: event } }), @@ -649,26 +656,43 @@ describe("RoomWidgetClient", () => { // The client should've emitted about the received event expect((await emittedEvent).getEffectiveEvent()).toEqual(event); expect(await emittedSync).toEqual(SyncState.Syncing); - // It should've also inserted the event into the room object + // However it should not have changed the room state const room = client.getRoom("!1:example.org"); - expect(room).not.toBeNull(); + expect(room!.currentState.getStateEvents("org.example.foo", "bar")).toBe(null); + + // Now assume that the state event becomes favored by state + // resolution for whatever reason and enters into the current state + // of the room + widgetApi.emit( + `action:${WidgetApiToWidgetAction.UpdateState}`, + new CustomEvent(`action:${WidgetApiToWidgetAction.UpdateState}`, { + detail: { data: { state: [event] } }, + }), + ); + // It should now have changed the room state expect(room!.currentState.getStateEvents("org.example.foo", "bar")?.getEffectiveEvent()).toEqual(event); }); - it("backfills", async () => { - widgetApi.readStateEvents.mockImplementation(async (eventType, limit, stateKey) => - eventType === "org.example.foo" && (limit ?? Infinity) > 0 && stateKey === "bar" - ? [event as IRoomEvent] - : [], + it("ignores state updates for other rooms", async () => { + const init = makeClient({ receiveState: [{ eventType: "org.example.foo", stateKey: "bar" }] }); + // Client needs to be told that the room state is loaded + widgetApi.emit( + `action:${WidgetApiToWidgetAction.UpdateState}`, + new CustomEvent(`action:${WidgetApiToWidgetAction.UpdateState}`, { detail: { data: { state: [] } } }), ); + await init; - await makeClient({ receiveState: [{ eventType: "org.example.foo", stateKey: "bar" }] }); - expect(widgetApi.requestCapabilityForRoomTimeline).toHaveBeenCalledWith("!1:example.org"); - expect(widgetApi.requestCapabilityToReceiveState).toHaveBeenCalledWith("org.example.foo", "bar"); - - const room = client.getRoom("!1:example.org"); - expect(room).not.toBeNull(); - expect(room!.currentState.getStateEvents("org.example.foo", "bar")?.getEffectiveEvent()).toEqual(event); + // Now a room we're not interested in receives a state update + widgetApi.emit( + `action:${WidgetApiToWidgetAction.UpdateState}`, + new CustomEvent(`action:${WidgetApiToWidgetAction.UpdateState}`, { + detail: { data: { state: [{ ...event, room_id: "!other-room:example.org" }] } }, + }), + ); + // No change to the room state + for (const room of client.getRooms()) { + expect(room.currentState.getStateEvents("org.example.foo", "bar")).toBe(null); + } }); }); diff --git a/src/embedded.ts b/src/embedded.ts index b0cc4c158e8..53154e40e62 100644 --- a/src/embedded.ts +++ b/src/embedded.ts @@ -28,6 +28,7 @@ import { WidgetApiAction, IWidgetApiResponse, IWidgetApiResponseData, + IUpdateStateToWidgetActionRequest, } from "matrix-widget-api"; import { MatrixEvent, IEvent, IContent, EventStatus } from "./models/event.ts"; @@ -136,6 +137,7 @@ export type EventHandlerMap = { [RoomWidgetClientEvent.PendingEventsChanged]: () export class RoomWidgetClient extends MatrixClient { private room?: Room; private readonly widgetApiReady: Promise; + private readonly roomStateSynced: Promise; private lifecycle?: AbortController; private syncState: SyncState | null = null; @@ -189,6 +191,11 @@ export class RoomWidgetClient extends MatrixClient { }; this.widgetApiReady = new Promise((resolve) => this.widgetApi.once("ready", resolve)); + this.roomStateSynced = capabilities.receiveState?.length + ? new Promise((resolve) => + this.widgetApi.once(`action:${WidgetApiToWidgetAction.UpdateState}`, resolve), + ) + : Promise.resolve(); // Request capabilities for the functionality this client needs to support if ( @@ -241,6 +248,7 @@ export class RoomWidgetClient extends MatrixClient { widgetApi.on(`action:${WidgetApiToWidgetAction.SendEvent}`, this.onEvent); widgetApi.on(`action:${WidgetApiToWidgetAction.SendToDevice}`, this.onToDevice); + widgetApi.on(`action:${WidgetApiToWidgetAction.UpdateState}`, this.onStateUpdate); // Open communication with the host widgetApi.start(); @@ -276,28 +284,6 @@ export class RoomWidgetClient extends MatrixClient { await this.widgetApiReady; - // Backfill the requested events - // We only get the most recent event for every type + state key combo, - // so it doesn't really matter what order we inject them in - await Promise.all( - this.capabilities.receiveState?.map(async ({ eventType, stateKey }) => { - const rawEvents = await this.widgetApi.readStateEvents(eventType, undefined, stateKey, [this.roomId]); - const events = rawEvents.map((rawEvent) => new MatrixEvent(rawEvent as Partial)); - - if (this.syncApi instanceof SyncApi) { - // Passing undefined for `stateAfterEventList` allows will make `injectRoomEvents` run in legacy mode - // -> state events in `timelineEventList` will update the state. - await this.syncApi.injectRoomEvents(this.room!, undefined, events); - } else { - await this.syncApi!.injectRoomEvents(this.room!, events); // Sliding Sync - } - events.forEach((event) => { - this.emit(ClientEvent.Event, event); - logger.info(`Backfilled event ${event.getId()} ${event.getType()} ${event.getStateKey()}`); - }); - }) ?? [], - ); - if (opts.clientWellKnownPollPeriod !== undefined) { this.clientWellKnownIntervalID = setInterval(() => { this.fetchClientWellKnown(); @@ -305,8 +291,9 @@ export class RoomWidgetClient extends MatrixClient { this.fetchClientWellKnown(); } + await this.roomStateSynced; this.setSyncState(SyncState.Syncing); - logger.info("Finished backfilling events"); + logger.info("Finished initial sync"); this.matrixRTC.start(); @@ -317,6 +304,7 @@ export class RoomWidgetClient extends MatrixClient { public stopClient(): void { this.widgetApi.off(`action:${WidgetApiToWidgetAction.SendEvent}`, this.onEvent); this.widgetApi.off(`action:${WidgetApiToWidgetAction.SendToDevice}`, this.onToDevice); + this.widgetApi.off(`action:${WidgetApiToWidgetAction.UpdateState}`, this.onStateUpdate); super.stopClient(); this.lifecycle!.abort(); // Signal to other async tasks that the client has stopped @@ -574,36 +562,15 @@ export class RoomWidgetClient extends MatrixClient { // Only inject once we have update the txId await this.updateTxId(event); - // The widget API does not tell us whether a state event came from `state_after` or not so we assume legacy behaviour for now. if (this.syncApi instanceof SyncApi) { - // The code will want to be something like: - // ``` - // if (!params.addToTimeline && !params.addToState) { - // // Passing undefined for `stateAfterEventList` makes `injectRoomEvents` run in "legacy mode" - // // -> state events part of the `timelineEventList` parameter will update the state. - // this.injectRoomEvents(this.room!, [], undefined, [event]); - // } else { - // this.injectRoomEvents(this.room!, undefined, params.addToState ? [event] : [], params.addToTimeline ? [event] : []); - // } - // ``` - - // Passing undefined for `stateAfterEventList` allows will make `injectRoomEvents` run in legacy mode - // -> state events in `timelineEventList` will update the state. - await this.syncApi.injectRoomEvents(this.room!, [], undefined, [event]); + await this.syncApi.injectRoomEvents(this.room!, undefined, [], [event]); } else { - // The code will want to be something like: - // ``` - // if (!params.addToTimeline && !params.addToState) { - // this.injectRoomEvents(this.room!, [], [event]); - // } else { - // this.injectRoomEvents(this.room!, params.addToState ? [event] : [], params.addToTimeline ? [event] : []); - // } - // ``` - await this.syncApi!.injectRoomEvents(this.room!, [], [event]); // Sliding Sync + // Sliding Sync + await this.syncApi!.injectRoomEvents(this.room!, [], [event]); } this.emit(ClientEvent.Event, event); this.setSyncState(SyncState.Syncing); - logger.info(`Received event ${event.getId()} ${event.getType()} ${event.getStateKey()}`); + logger.info(`Received event ${event.getId()} ${event.getType()}`); } else { const { event_id: eventId, room_id: roomId } = ev.detail.data; logger.info(`Received event ${eventId} for a different room ${roomId}; discarding`); @@ -628,6 +595,32 @@ export class RoomWidgetClient extends MatrixClient { await this.ack(ev); }; + private onStateUpdate = async (ev: CustomEvent): Promise => { + ev.preventDefault(); + + for (const rawEvent of ev.detail.data.state) { + // Verify the room ID matches, since it's possible for the client to + // send us state updates from other rooms if this widget is always + // on screen + if (rawEvent.room_id === this.roomId) { + const event = new MatrixEvent(rawEvent as Partial); + + if (this.syncApi instanceof SyncApi) { + await this.syncApi.injectRoomEvents(this.room!, undefined, [event]); + } else { + // Sliding Sync + await this.syncApi!.injectRoomEvents(this.room!, [event]); + } + logger.info(`Updated state entry ${event.getType()} ${event.getStateKey()} to ${event.getId()}`); + } else { + const { event_id: eventId, room_id: roomId } = ev.detail.data; + logger.info(`Received state entry ${eventId} for a different room ${roomId}; discarding`); + } + } + + await this.ack(ev); + }; + private async watchTurnServers(): Promise { const servers = this.widgetApi.getTurnServers(); const onClientStopped = (): void => { diff --git a/yarn.lock b/yarn.lock index c02d9f3744c..6df971ed325 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4875,9 +4875,9 @@ matrix-mock-request@^2.5.0: expect "^28.1.0" matrix-widget-api@^1.10.0: - version "1.10.0" - resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.10.0.tgz#d31ea073a5871a1fb1a511ef900b0c125a37bf55" - integrity sha512-rkAJ29briYV7TJnfBVLVSKtpeBrBju15JZFSDP6wj8YdbCu1bdmlplJayQ+vYaw1x4fzI49Q+Nz3E85s46sRDw== + version "1.12.0" + resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.12.0.tgz#b3d22bab1670051c8eeee66bb96d08b33148bc99" + integrity sha512-6JRd9fJGGvuBRhcTg9wX+Skn/Q1wox3jdp5yYQKJ6pPw4urW9bkTR90APBKVDB1vorJKT44jml+lCzkDMRBjww== dependencies: "@types/events" "^3.0.0" events "^3.2.0" From ba58a1119ce8b246cc3980b2b18324f481088ed0 Mon Sep 17 00:00:00 2001 From: Timo Date: Thu, 23 Jan 2025 15:25:22 +0100 Subject: [PATCH 2/4] Allow the embedded client to work without UpdateState version --- src/embedded.ts | 53 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/src/embedded.ts b/src/embedded.ts index 53154e40e62..6eea0e23c87 100644 --- a/src/embedded.ts +++ b/src/embedded.ts @@ -29,6 +29,7 @@ import { IWidgetApiResponse, IWidgetApiResponseData, IUpdateStateToWidgetActionRequest, + UnstableApiVersion, } from "matrix-widget-api"; import { MatrixEvent, IEvent, IContent, EventStatus } from "./models/event.ts"; @@ -260,6 +261,10 @@ export class RoomWidgetClient extends MatrixClient { if (sendContentLoaded) widgetApi.sendContentLoaded(); } + public async supportUpdateState(): Promise { + return (await this.widgetApi.getClientVersions())?.includes(UnstableApiVersion.MSC2762_UPDATE_STATE); + } + public async startClient(opts: IStartClientOpts = {}): Promise { this.lifecycle = new AbortController(); @@ -284,6 +289,31 @@ export class RoomWidgetClient extends MatrixClient { await this.widgetApiReady; + if (!(await this.supportUpdateState())) { + // Backfill the requested events + // We only get the most recent event for every type + state key combo, + // so it doesn't really matter what order we inject them in + await Promise.all( + this.capabilities.receiveState?.map(async ({ eventType, stateKey }) => { + const rawEvents = await this.widgetApi.readStateEvents(eventType, undefined, stateKey, [ + this.roomId, + ]); + const events = rawEvents.map((rawEvent) => new MatrixEvent(rawEvent as Partial)); + + if (this.syncApi instanceof SyncApi) { + // Passing events as `stateAfterEventList` will update the state. + await this.syncApi.injectRoomEvents(this.room!, undefined, events); + } else { + await this.syncApi!.injectRoomEvents(this.room!, events); // Sliding Sync + } + events.forEach((event) => { + this.emit(ClientEvent.Event, event); + logger.info(`Backfilled event ${event.getId()} ${event.getType()} ${event.getStateKey()}`); + }); + }) ?? [], + ); + } + if (opts.clientWellKnownPollPeriod !== undefined) { this.clientWellKnownIntervalID = setInterval(() => { this.fetchClientWellKnown(); @@ -563,11 +593,24 @@ export class RoomWidgetClient extends MatrixClient { await this.updateTxId(event); if (this.syncApi instanceof SyncApi) { - await this.syncApi.injectRoomEvents(this.room!, undefined, [], [event]); + if (await this.supportUpdateState()) { + await this.syncApi.injectRoomEvents(this.room!, undefined, [], [event]); + } else { + // Passing undefined for `stateAfterEventList` will make `injectRoomEvents` run in legacy mode + // -> state events in `timelineEventList` will update the state. + await this.syncApi.injectRoomEvents(this.room!, [], undefined, [event]); + } } else { // Sliding Sync - await this.syncApi!.injectRoomEvents(this.room!, [], [event]); + if (await this.supportUpdateState()) { + await this.syncApi!.injectRoomEvents(this.room!, [], [event]); + } else { + logger.error( + "slididng sync cannot be used in widget mode if the client widget driver does not support the version: 'org.matrix.msc2762_update_state'", + ); + } } + this.emit(ClientEvent.Event, event); this.setSyncState(SyncState.Syncing); logger.info(`Received event ${event.getId()} ${event.getType()}`); @@ -597,7 +640,11 @@ export class RoomWidgetClient extends MatrixClient { private onStateUpdate = async (ev: CustomEvent): Promise => { ev.preventDefault(); - + if (!(await this.supportUpdateState())) { + logger.warn( + "received update_state widget action but the widget driver did not claim to support 'org.matrix.msc2762_update_state'", + ); + } for (const rawEvent of ev.detail.data.state) { // Verify the room ID matches, since it's possible for the client to // send us state updates from other rooms if this widget is always From 8f0a0aec4ed714a1fae520fd6c99b0c4e41408d7 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 24 Jan 2025 09:47:22 +0100 Subject: [PATCH 3/4] bump matrix-widget-api --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 6df971ed325..6c7d7ac1878 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4875,9 +4875,9 @@ matrix-mock-request@^2.5.0: expect "^28.1.0" matrix-widget-api@^1.10.0: - version "1.12.0" - resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.12.0.tgz#b3d22bab1670051c8eeee66bb96d08b33148bc99" - integrity sha512-6JRd9fJGGvuBRhcTg9wX+Skn/Q1wox3jdp5yYQKJ6pPw4urW9bkTR90APBKVDB1vorJKT44jml+lCzkDMRBjww== + version "1.13.0" + resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.13.0.tgz#40344b264b08d6d98ab9d547a41eb74dd6d8c3f7" + integrity sha512-+LrvwkR1izL4h2euX8PDrvG/3PZZDEd6As+lmnR3jAVwbFJtU5iTnwmZGnCca9ddngCvXvAHkcpJBEPyPTZneQ== dependencies: "@types/events" "^3.0.0" events "^3.2.0" From bd587b4a50bc603cf9e717dd445df781feb7f018 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 24 Jan 2025 14:20:40 +0100 Subject: [PATCH 4/4] fix awaiting for room state with and without new UpdateState method. --- src/embedded.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/embedded.ts b/src/embedded.ts index 6eea0e23c87..d885e7ac82c 100644 --- a/src/embedded.ts +++ b/src/embedded.ts @@ -289,7 +289,11 @@ export class RoomWidgetClient extends MatrixClient { await this.widgetApiReady; - if (!(await this.supportUpdateState())) { + // sync room state: + if (await this.supportUpdateState()) { + // This will resolve once the client driver has sent us all the allowed room state. + await this.roomStateSynced; + } else { // Backfill the requested events // We only get the most recent event for every type + state key combo, // so it doesn't really matter what order we inject them in @@ -320,8 +324,6 @@ export class RoomWidgetClient extends MatrixClient { }, 1000 * opts.clientWellKnownPollPeriod); this.fetchClientWellKnown(); } - - await this.roomStateSynced; this.setSyncState(SyncState.Syncing); logger.info("Finished initial sync");