Get open id token in embedded

src/client.ts

@@ -209,6 +209,7 @@ import {
     ServerSideSecretStorage,
     ServerSideSecretStorageImpl,
 } from "./secret-storage";
+import { FocusInfo } from "./webrtc/callEventTypes";
 
 export type Store = IStore;
 
@@ -367,6 +368,8 @@ export interface ICreateClientOpts {
      */
     useE2eForGroupCall?: boolean;
 
+    foci?: FocusInfo[];
+
     cryptoCallbacks?: ICryptoCallbacks;
 
     /**
@@ -381,6 +384,12 @@ export interface ICreateClientOpts {
      * Default: false.
      */
     isVoipWithNoMediaAllowed?: boolean;
+
+    /**
+     * If true, group calls will not establish media connectivity and only create the signaling events,
+     * so that livekit media can be used in the application layert (js-sdk contains no livekit code).
+     */
+    useLivekitForGroupCalls?: boolean;
 }
 
 export interface IMatrixClientCreateOpts extends ICreateClientOpts {
@@ -1205,6 +1214,8 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     public baseUrl: string;
     public readonly isVoipWithNoMediaAllowed;
 
+    public useLivekitForGroupCalls: boolean;
+
     // Note: these are all `protected` to let downstream consumers make mistakes if they want to.
     // We don't technically support this usage, but have reasons to do this.
 
@@ -1252,6 +1263,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
     private useE2eForGroupCall = true;
     private toDeviceMessageQueue: ToDeviceMessageQueue;
+    private foci: FocusInfo[] = [];
 
     private _secretStorage: ServerSideSecretStorageImpl;
 
@@ -1311,6 +1323,8 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             this.pickleKey = opts.pickleKey;
         }
 
+        this.useLivekitForGroupCalls = Boolean(opts.useLivekitForGroupCalls);
+
         this.scheduler = opts.scheduler;
         if (this.scheduler) {
             this.scheduler.setProcessFunction(async (eventToSend: MatrixEvent) => {
@@ -1354,6 +1368,8 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
 
         if (opts.useE2eForGroupCall !== undefined) this.useE2eForGroupCall = opts.useE2eForGroupCall;
 
+        this.foci = opts.foci ?? [];
+
         // List of which rooms have encryption enabled: separate from crypto because
         // we still want to know which rooms are encrypted even if crypto is disabled:
         // we don't want to start sending unencrypted events to them.
@@ -1931,9 +1947,15 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             dataChannelsEnabled || this.isVoipWithNoMediaAllowed,
             dataChannelOptions,
             this.isVoipWithNoMediaAllowed,
+            this.useLivekitForGroupCalls,
+            this.foci,
         ).create();
     }
 
+    public getFoci(): FocusInfo[] {
+        return this.foci;
+    }
+
     /**
      * Wait until an initial state for the given room has been processed by the
      * client and the client is aware of any ongoing group calls. Awaiting on

src/embedded.ts

@@ -29,7 +29,14 @@ import { MatrixEvent, IEvent, IContent, EventStatus } from "./models/event";
 import { ISendEventResponse } from "./@types/requests";
 import { EventType } from "./@types/event";
 import { logger } from "./logger";
-import { MatrixClient, ClientEvent, IMatrixClientCreateOpts, IStartClientOpts, SendToDeviceContentMap } from "./client";
+import {
+    MatrixClient,
+    ClientEvent,
+    IMatrixClientCreateOpts,
+    IStartClientOpts,
+    SendToDeviceContentMap,
+    IOpenIDToken,
+} from "./client";
 import { SyncApi, SyncState } from "./sync";
 import { SlidingSyncSdk } from "./sliding-sync-sdk";
 import { User } from "./models/user";
@@ -262,6 +269,18 @@ export class RoomWidgetClient extends MatrixClient {
         await this.widgetApi.sendToDevice((payload as { type: string }).type, true, recursiveMapToObject(contentMap));
     }
 
+    public async getOpenIdToken(): Promise<IOpenIDToken> {
+        const token = await this.widgetApi.requestOpenIDConnectToken();
+        // the IOpenIDCredentials from the widget-api and IOpenIDToken form the matrix-js-sdk are compatible.
+        // we still recreate the token to make this transparent and cathcable by the linter in case the types change in the future.
+        return <IOpenIDToken>{
+            access_token: token.access_token,
+            expires_in: token.expires_in,
+            matrix_server_name: token.matrix_server_name,
+            token_type: token.token_type,
+        };
+    }
+
     // Overridden since we get TURN servers automatically over the widget API,
     // and this method would otherwise complain about missing an access token
     public async checkTurnServers(): Promise<boolean> {

src/webrtc/callEventTypes.ts

@@ -89,4 +89,8 @@ export interface MCallHangupReject extends MCallBase {
     reason?: CallErrorCode;
 }
 
+export interface FocusInfo {
+    livekitServiceUrl: string;
+}
+
 /* eslint-enable camelcase */

src/webrtc/groupCall.ts

@@ -17,7 +17,7 @@ import { Room } from "../models/room";
 import { RoomStateEvent } from "../models/room-state";
 import { logger } from "../logger";
 import { ReEmitter } from "../ReEmitter";
-import { SDPStreamMetadataPurpose } from "./callEventTypes";
+import { FocusInfo, SDPStreamMetadataPurpose } from "./callEventTypes";
 import { MatrixEvent } from "../models/event";
 import { EventType } from "../@types/event";
 import { CallEventHandlerEvent } from "./callEventHandler";
@@ -170,6 +170,8 @@ export interface IGroupCallRoomState {
     // TODO: Specify data-channels
     "dataChannelsEnabled"?: boolean;
     "dataChannelOptions"?: IGroupCallDataChannelOptions;
+
+    "io.element.livekit_service_url"?: string;
 }
 
 export interface IGroupCallRoomMemberFeed {
@@ -268,10 +270,16 @@ export class GroupCall extends TypedEventEmitter<
         private dataChannelsEnabled?: boolean,
         private dataChannelOptions?: IGroupCallDataChannelOptions,
         isCallWithoutVideoAndAudio?: boolean,
+        // this tells the js-sdk not to actually establish any calls to exchange media and just to
+        // create the group call signaling events, with the intention that the actual media will be
+        // handled using livekit. The js-sdk doesn't contain any code to do the actual livekit call though.
+        private useLivekit = false,
+        foci?: FocusInfo[],
     ) {
         super();
         this.reEmitter = new ReEmitter(this);
         this.groupCallId = groupCallId ?? genCallID();
+        this._foci = foci ?? [];
         this.creationTs =
             room.currentState.getStateEvents(EventType.GroupCallPrefix, this.groupCallId)?.getTs() ?? null;
         this.updateParticipants();
@@ -320,6 +328,8 @@ export class GroupCall extends TypedEventEmitter<
         this.client.groupCallEventHandler!.groupCalls.set(this.room.roomId, this);
         this.client.emit(GroupCallEventHandlerEvent.Outgoing, this);
 
+        const focus = this._foci[0];
+
         const groupCallState: IGroupCallRoomState = {
             "m.intent": this.intent,
             "m.type": this.type,
@@ -328,12 +338,21 @@ export class GroupCall extends TypedEventEmitter<
             "dataChannelsEnabled": this.dataChannelsEnabled,
             "dataChannelOptions": this.dataChannelsEnabled ? this.dataChannelOptions : undefined,
         };
+        if (focus) {
+            groupCallState["io.element.livekit_service_url"] = focus.livekitServiceUrl;
+        }
 
         await this.client.sendStateEvent(this.room.roomId, EventType.GroupCallPrefix, groupCallState, this.groupCallId);
 
         return this;
     }
 
+    private _foci: FocusInfo[] = [];
+
+    public get foci(): FocusInfo[] {
+        return this._foci;
+    }
+
     private _state = GroupCallState.LocalCallFeedUninitialized;
 
     /**
@@ -442,6 +461,11 @@ export class GroupCall extends TypedEventEmitter<
     }
 
     public async initLocalCallFeed(): Promise<void> {
+        if (this.useLivekit) {
+            logger.info("Livekit group call: not starting local call feed.");
+            return;
+        }
+
         if (this.state !== GroupCallState.LocalCallFeedUninitialized) {
             throw new Error(`Cannot initialize local call feed in the "${this.state}" state.`);
         }
@@ -537,11 +561,13 @@ export class GroupCall extends TypedEventEmitter<
             this.onIncomingCall(call);
         }
 
-        this.retryCallLoopInterval = setInterval(this.onRetryCallLoop, this.retryCallInterval);
+        if (!this.useLivekit) {
+            this.retryCallLoopInterval = setInterval(this.onRetryCallLoop, this.retryCallInterval);
 
-        this.activeSpeaker = undefined;
-        this.onActiveSpeakerLoop();
-        this.activeSpeakerLoopInterval = setInterval(this.onActiveSpeakerLoop, this.activeSpeakerInterval);
+            this.activeSpeaker = undefined;
+            this.onActiveSpeakerLoop();
+            this.activeSpeakerLoopInterval = setInterval(this.onActiveSpeakerLoop, this.activeSpeakerInterval);
+        }
     }
 
     private dispose(): void {
@@ -923,6 +949,11 @@ export class GroupCall extends TypedEventEmitter<
             return;
         }
 
+        if (this.useLivekit) {
+            logger.info("Received incoming call whilst in signaling-only mode! Ignoring.");
+            return;
+        }
+
         const deviceMap = this.calls.get(opponentUserId) ?? new Map<string, MatrixCall>();
         const prevCall = deviceMap.get(newCall.getOpponentDeviceId()!);
 
@@ -1629,7 +1660,7 @@ export class GroupCall extends TypedEventEmitter<
             }
         });
 
-        if (this.state === GroupCallState.Entered) this.placeOutgoingCalls();
+        if (this.state === GroupCallState.Entered && !this.useLivekit) this.placeOutgoingCalls();
 
         // Update the participants stored in the stats object
     };

src/webrtc/groupCallEventHandler.ts

@@ -23,6 +23,7 @@
 import { logger } from "../logger";
 import { EventType } from "../@types/event";
 import { SyncState } from "../sync";
+import { FocusInfo } from "./callEventTypes";
 
 export enum GroupCallEventHandlerEvent {
     Incoming = "GroupCall.incoming",
@@ -177,6 +178,19 @@
             dataChannelOptions = { ordered, maxPacketLifeTime, maxRetransmits, protocol };
         }
 
+        const livekitServiceUrl = content["io.element.livekit_service_url"];
+
+        let focus: FocusInfo | undefined;
+        if (livekitServiceUrl) {
+            focus = {
+                livekitServiceUrl: livekitServiceUrl,
+            };
+        } else {
+            focus = this.client.getFoci()[0]!;
+            content["io.element.livekit_service_url"] = focus.livekitServiceUrl;
+            this.client.sendStateEvent(room.roomId, EventType.GroupCallPrefix, content, groupCallId);
+        }
+
         const groupCall = new GroupCall(
             this.client,
             room,
@@ -189,6 +203,8 @@
             content?.dataChannelsEnabled || this.client.isVoipWithNoMediaAllowed,
             dataChannelOptions,
             this.client.isVoipWithNoMediaAllowed,
+            this.client.useLivekitForGroupCalls,
+            focus ? [focus] : [],
         );
 
         this.groupCalls.set(room.roomId, groupCall);