diff --git a/src/client.ts b/src/client.ts index bf7b5adfcc5..109ab91e714 100644 --- a/src/client.ts +++ b/src/client.ts @@ -209,6 +209,7 @@ import { ServerSideSecretStorage, ServerSideSecretStorageImpl, } from "./secret-storage"; +import { FocusInfo } from "./webrtc/callEventTypes"; export type Store = IStore; @@ -367,6 +368,8 @@ export interface ICreateClientOpts { */ useE2eForGroupCall?: boolean; + foci?: FocusInfo[]; + cryptoCallbacks?: ICryptoCallbacks; /** @@ -381,6 +384,12 @@ export interface ICreateClientOpts { * Default: false. */ isVoipWithNoMediaAllowed?: boolean; + + /** + * If true, group calls will not establish media connectivity and only create the signaling events, + * so that livekit media can be used in the application layert (js-sdk contains no livekit code). + */ + useLivekitForGroupCalls?: boolean; } export interface IMatrixClientCreateOpts extends ICreateClientOpts { @@ -1205,6 +1214,8 @@ export class MatrixClient extends TypedEventEmitter { @@ -1354,6 +1368,8 @@ export class MatrixClient extends TypedEventEmitter { + const token = await this.widgetApi.requestOpenIDConnectToken(); + // the IOpenIDCredentials from the widget-api and IOpenIDToken form the matrix-js-sdk are compatible. + // we still recreate the token to make this transparent and cathcable by the linter in case the types change in the future. + return { + access_token: token.access_token, + expires_in: token.expires_in, + matrix_server_name: token.matrix_server_name, + token_type: token.token_type, + }; + } + // Overridden since we get TURN servers automatically over the widget API, // and this method would otherwise complain about missing an access token public async checkTurnServers(): Promise { diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index f06ed5b0db7..c6636ed1948 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -89,4 +89,8 @@ export interface MCallHangupReject extends MCallBase { reason?: CallErrorCode; } +export interface FocusInfo { + livekitServiceUrl: string; +} + /* eslint-enable camelcase */ diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 4c3224435cd..34d70ba3828 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -17,7 +17,7 @@ import { Room } from "../models/room"; import { RoomStateEvent } from "../models/room-state"; import { logger } from "../logger"; import { ReEmitter } from "../ReEmitter"; -import { SDPStreamMetadataPurpose } from "./callEventTypes"; +import { FocusInfo, SDPStreamMetadataPurpose } from "./callEventTypes"; import { MatrixEvent } from "../models/event"; import { EventType } from "../@types/event"; import { CallEventHandlerEvent } from "./callEventHandler"; @@ -170,6 +170,8 @@ export interface IGroupCallRoomState { // TODO: Specify data-channels "dataChannelsEnabled"?: boolean; "dataChannelOptions"?: IGroupCallDataChannelOptions; + + "io.element.livekit_service_url"?: string; } export interface IGroupCallRoomMemberFeed { @@ -268,10 +270,16 @@ export class GroupCall extends TypedEventEmitter< private dataChannelsEnabled?: boolean, private dataChannelOptions?: IGroupCallDataChannelOptions, isCallWithoutVideoAndAudio?: boolean, + // this tells the js-sdk not to actually establish any calls to exchange media and just to + // create the group call signaling events, with the intention that the actual media will be + // handled using livekit. The js-sdk doesn't contain any code to do the actual livekit call though. + private useLivekit = false, + foci?: FocusInfo[], ) { super(); this.reEmitter = new ReEmitter(this); this.groupCallId = groupCallId ?? genCallID(); + this._foci = foci ?? []; this.creationTs = room.currentState.getStateEvents(EventType.GroupCallPrefix, this.groupCallId)?.getTs() ?? null; this.updateParticipants(); @@ -320,6 +328,8 @@ export class GroupCall extends TypedEventEmitter< this.client.groupCallEventHandler!.groupCalls.set(this.room.roomId, this); this.client.emit(GroupCallEventHandlerEvent.Outgoing, this); + const focus = this._foci[0]; + const groupCallState: IGroupCallRoomState = { "m.intent": this.intent, "m.type": this.type, @@ -328,12 +338,21 @@ export class GroupCall extends TypedEventEmitter< "dataChannelsEnabled": this.dataChannelsEnabled, "dataChannelOptions": this.dataChannelsEnabled ? this.dataChannelOptions : undefined, }; + if (focus) { + groupCallState["io.element.livekit_service_url"] = focus.livekitServiceUrl; + } await this.client.sendStateEvent(this.room.roomId, EventType.GroupCallPrefix, groupCallState, this.groupCallId); return this; } + private _foci: FocusInfo[] = []; + + public get foci(): FocusInfo[] { + return this._foci; + } + private _state = GroupCallState.LocalCallFeedUninitialized; /** @@ -442,6 +461,11 @@ export class GroupCall extends TypedEventEmitter< } public async initLocalCallFeed(): Promise { + if (this.useLivekit) { + logger.info("Livekit group call: not starting local call feed."); + return; + } + if (this.state !== GroupCallState.LocalCallFeedUninitialized) { throw new Error(`Cannot initialize local call feed in the "${this.state}" state.`); } @@ -537,11 +561,13 @@ export class GroupCall extends TypedEventEmitter< this.onIncomingCall(call); } - this.retryCallLoopInterval = setInterval(this.onRetryCallLoop, this.retryCallInterval); + if (!this.useLivekit) { + this.retryCallLoopInterval = setInterval(this.onRetryCallLoop, this.retryCallInterval); - this.activeSpeaker = undefined; - this.onActiveSpeakerLoop(); - this.activeSpeakerLoopInterval = setInterval(this.onActiveSpeakerLoop, this.activeSpeakerInterval); + this.activeSpeaker = undefined; + this.onActiveSpeakerLoop(); + this.activeSpeakerLoopInterval = setInterval(this.onActiveSpeakerLoop, this.activeSpeakerInterval); + } } private dispose(): void { @@ -923,6 +949,11 @@ export class GroupCall extends TypedEventEmitter< return; } + if (this.useLivekit) { + logger.info("Received incoming call whilst in signaling-only mode! Ignoring."); + return; + } + const deviceMap = this.calls.get(opponentUserId) ?? new Map(); const prevCall = deviceMap.get(newCall.getOpponentDeviceId()!); @@ -1629,7 +1660,7 @@ export class GroupCall extends TypedEventEmitter< } }); - if (this.state === GroupCallState.Entered) this.placeOutgoingCalls(); + if (this.state === GroupCallState.Entered && !this.useLivekit) this.placeOutgoingCalls(); // Update the participants stored in the stats object }; diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 08487bdd234..e52302c0c83 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -23,6 +23,7 @@ import { RoomMember } from "../models/room-member"; import { logger } from "../logger"; import { EventType } from "../@types/event"; import { SyncState } from "../sync"; +import { FocusInfo } from "./callEventTypes"; export enum GroupCallEventHandlerEvent { Incoming = "GroupCall.incoming", @@ -177,6 +178,19 @@ export class GroupCallEventHandler { dataChannelOptions = { ordered, maxPacketLifeTime, maxRetransmits, protocol }; } + const livekitServiceUrl = content["io.element.livekit_service_url"]; + + let focus: FocusInfo | undefined; + if (livekitServiceUrl) { + focus = { + livekitServiceUrl: livekitServiceUrl, + }; + } else { + focus = this.client.getFoci()[0]!; + content["io.element.livekit_service_url"] = focus.livekitServiceUrl; + this.client.sendStateEvent(room.roomId, EventType.GroupCallPrefix, content, groupCallId); + } + const groupCall = new GroupCall( this.client, room, @@ -189,6 +203,8 @@ export class GroupCallEventHandler { content?.dataChannelsEnabled || this.client.isVoipWithNoMediaAllowed, dataChannelOptions, this.client.isVoipWithNoMediaAllowed, + this.client.useLivekitForGroupCalls, + focus ? [focus] : [], ); this.groupCalls.set(room.roomId, groupCall);