diff --git a/.gitignore b/.gitignore index 6bc9edb40a3..ccc8e35a6d8 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ out .vscode .vscode/ +.idea diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index ba93a6c1c97..7121e2f3b12 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -22,6 +22,7 @@ import { GroupCallIntent, GroupCallType, IContent, + IFocusInfo, ISendEventResponse, MatrixClient, MatrixEvent, @@ -32,10 +33,12 @@ import { RoomStateEventHandlerMap, } from "../../src"; import { TypedEventEmitter } from "../../src/models/typed-event-emitter"; +import { randomString } from "../../src/randomstring"; import { ReEmitter } from "../../src/ReEmitter"; import { SyncState } from "../../src/sync"; import { CallEvent, CallEventHandlerMap, CallState, MatrixCall } from "../../src/webrtc/call"; import { CallEventHandlerEvent, CallEventHandlerEventHandlerMap } from "../../src/webrtc/callEventHandler"; +import { SDPStreamMetadataPurpose } from "../../src/webrtc/callEventTypes"; import { CallFeed } from "../../src/webrtc/callFeed"; import { GroupCallEventHandlerMap } from "../../src/webrtc/groupCall"; import { GroupCallEventHandlerEvent } from "../../src/webrtc/groupCallEventHandler"; @@ -94,6 +97,30 @@ export const FAKE_DEVICE_ID_2 = "@BBBBBB"; export const FAKE_SESSION_ID_2 = "bob1"; export const FAKE_USER_ID_3 = "@charlie:test.dummy"; +export const runOnTrackForStream = (call: MatrixCall, stream: MediaStream) => { + let sdp = "v=0\n" + "o=- 7135465365607179083 2 IN IP4 127.0.0.1\n" + "s=-\n" + "t=0 0\n" + "a=group:BUNDLE 0 1 2\n"; + + stream.getTracks().forEach((track, index) => { + sdp += `m=${track.kind}\n`; + sdp += `a=mid:${index}\n`; + sdp += `a=msid:${stream.id} ${track.id}\n`; + }); + + // @ts-ignore + call.peerConn.remoteDescription = { + sdp: sdp, + }; + + stream.getTracks().forEach((track, index) => { + // @ts-ignore + call.onTrack({ + track, + streams: [stream], + transceiver: { mid: `${index}`, receiver: { track } }, + } as TrackEvent); + }); +}; + class MockMediaStreamAudioSourceNode { public connect() {} } @@ -115,6 +142,22 @@ export class MockAudioContext { public close() {} } +export class MockRTCDataChannel { + constructor(public readonly label: string, public readonly id?: number) {} + + public typed(): RTCDataChannel { + return this as unknown as RTCDataChannel; + } + + addEventListener() {} +} + +export interface MockRTCSessionDescription { + sdp: string; + type: RTCSdpType; + toJSON: () => any; +} + export class MockRTCPeerConnection { private static instances: MockRTCPeerConnection[] = []; @@ -125,7 +168,7 @@ export class MockRTCPeerConnection { public needsNegotiation = false; public readyToNegotiate: Promise; private onReadyToNegotiate?: () => void; - public localDescription: RTCSessionDescription; + public localDescription: MockRTCSessionDescription; public signalingState: RTCSignalingState = "stable"; public iceConnectionState: RTCIceConnectionState = "connected"; public transceivers: MockRTCRtpTransceiver[] = []; @@ -148,7 +191,7 @@ export class MockRTCPeerConnection { this.localDescription = { sdp: DUMMY_SDP, type: "offer", - toJSON: function () {}, + toJSON: () => {}, }; this.readyToNegotiate = new Promise((resolve) => { @@ -169,8 +212,8 @@ export class MockRTCPeerConnection { this.onTrackListener = listener; } } - public createDataChannel(label: string, opts: RTCDataChannelInit) { - return { label, ...opts }; + public createDataChannel(label: string, opts: RTCDataChannelInit): MockRTCDataChannel { + return new MockRTCDataChannel(label, opts.id); } public createOffer() { return Promise.resolve({ @@ -194,28 +237,40 @@ export class MockRTCPeerConnection { public getStats() { return []; } - public addTransceiver(track: MockMediaStreamTrack): MockRTCRtpTransceiver { + public addTransceiver(track: MockMediaStreamTrack, init?: RTCRtpTransceiverInit): MockRTCRtpTransceiver { this.needsNegotiation = true; if (this.onReadyToNegotiate) this.onReadyToNegotiate(); const newSender = new MockRTCRtpSender(track); const newReceiver = new MockRTCRtpReceiver(track); - const newTransceiver = new MockRTCRtpTransceiver(this); + const newTransceiver = new MockRTCRtpTransceiver(this, (this.transceivers.length + 1).toString()); newTransceiver.sender = newSender as unknown as RTCRtpSender; newTransceiver.receiver = newReceiver as unknown as RTCRtpReceiver; this.transceivers.push(newTransceiver); + let newSDP = this.localDescription.sdp; + init?.streams?.forEach((stream) => { + newSDP += `m=${track.kind === "audio" ? "audio 50609 UDP 126" : "video 9 UDP 114"} \r\n`; + newSDP += `a=sendrecv\r\n`; + newSDP += `a=mid:${this.transceivers.length}\r\n`; + newSDP += `a=msid:${stream.id} ${track.id}\r\n`; + }); + this.localDescription.sdp = newSDP; + return newTransceiver; } - public addTrack(track: MockMediaStreamTrack): MockRTCRtpSender { - return this.addTransceiver(track).sender as unknown as MockRTCRtpSender; + public addTrack(track: MockMediaStreamTrack, ...streams: MediaStream[]): MockRTCRtpSender { + return this.addTransceiver(track, { streams }).sender as unknown as MockRTCRtpSender; } - public removeTrack() { - this.needsNegotiation = true; - if (this.onReadyToNegotiate) this.onReadyToNegotiate(); + public removeTrack(sender: MockRTCRtpSender) { + const transceiver = this.transceivers.find((transceiver) => transceiver.sender === sender.typed()); + transceiver?.sender?.replaceTrack(null); + if (transceiver) { + transceiver.direction = transceiver?.direction === "sendrecv" ? "recvonly" : "inactive"; + } } public getTransceivers(): MockRTCRtpTransceiver[] { @@ -239,6 +294,13 @@ export class MockRTCRtpSender { public replaceTrack(track: MockMediaStreamTrack) { this.track = track; } + + public getParameters() {} + public setParameters() {} + + public typed(): RTCRtpSender { + return this as unknown as RTCRtpSender; + } } export class MockRTCRtpReceiver { @@ -246,12 +308,20 @@ export class MockRTCRtpReceiver { } export class MockRTCRtpTransceiver { - constructor(private peerConn: MockRTCPeerConnection) {} + constructor(private peerConn: MockRTCPeerConnection, public readonly mid: string) {} public sender?: RTCRtpSender; public receiver?: RTCRtpReceiver; - public set direction(_: string) { + private _direction = "sendrecv"; + + public get direction() { + return this._direction; + } + + public set direction(newDirection: string) { + if (this._direction === newDirection) return; + this._direction = newDirection; this.peerConn.needsNegotiation = true; } @@ -265,7 +335,7 @@ export class MockMediaStreamTrack { public listeners: [string, (...args: any[]) => any][] = []; public isStopped = false; - public settings?: MediaTrackSettings; + public settings: MediaTrackSettings = {}; public getSettings(): MediaTrackSettings { return this.settings!; @@ -296,7 +366,7 @@ export class MockMediaStreamTrack { // XXX: Using EventTarget in jest doesn't seem to work, so we write our own // implementation export class MockMediaStream { - constructor(public id: string, private tracks: MockMediaStreamTrack[] = []) {} + constructor(public id: string = randomString(32), private tracks: MockMediaStreamTrack[] = []) {} public listeners: [string, (...args: any[]) => any][] = []; public isStopped = false; @@ -492,6 +562,10 @@ export class MockCallMatrixClient extends TypedEventEmitter { @@ -508,15 +582,23 @@ export class MockMatrixCall extends TypedEventEmitter(), stream: new MockMediaStream("stream"), }; - public remoteUsermediaFeed?: CallFeed; - public remoteScreensharingFeed?: CallFeed; + public feeds: CallFeed[] = []; public reject = jest.fn(); public answerWithCallFeeds = jest.fn(); public hangup = jest.fn(); + public opponentSupportsSDPStreamMetadata = jest.fn(); public sendMetadataUpdate = jest.fn(); + public get remoteUsermediaFeed(): CallFeed | undefined { + return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + public get remoteScreensharingFeed(): CallFeed | undefined { + return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + } + public getOpponentMember(): Partial { return this.opponentMember; } @@ -525,16 +607,32 @@ export class MockMatrixCall extends TypedEventEmitter !feed.isLocal); + } + public typed(): MatrixCall { return this as unknown as MatrixCall; } + + public initStats = jest.fn(); } export class MockCallFeed { - constructor(public userId: string, public deviceId: string | undefined, public stream: MockMediaStream) {} + constructor( + public userId: string, + public deviceId: string | undefined, + public stream: MockMediaStream, + public isLocal?: boolean, + public purpose?: SDPStreamMetadataPurpose, + ) {} + + public disposed = false; public measureVolumeActivity(val: boolean) {} - public dispose() {} + public dispose() { + this.disposed = true; + } public typed(): CallFeed { return this as unknown as CallFeed; @@ -544,9 +642,12 @@ export class MockCallFeed { export function installWebRTCMocks() { global.navigator = { mediaDevices: new MockMediaDevices().typed(), + userAgent: "This is definitely a user agent string", } as unknown as Navigator; global.window = { + // @ts-ignore Mock + MediaStream: MockMediaStream, // @ts-ignore Mock RTCPeerConnection: MockRTCPeerConnection, // @ts-ignore Mock @@ -604,3 +705,124 @@ export function makeMockGroupCallMemberStateEvent(roomId: string, groupCallId: s getStateKey: jest.fn().mockReturnValue(groupCallId), } as unknown as MatrixEvent; } + +export const REMOTE_SFU_DESCRIPTION = + "v=0\n" + + "o=- 3242942315779688438 1678878001 IN IP4 0.0.0.0\n" + + "s=-\n" + + "t=0 0\n" + + "a=fingerprint:sha-256 EA:30:B2:7F:49:B5:46:D6:40:72:BF:79:95:C1:65:08:6E:35:09:FB:90:89:DA:EF:6B:82:D1:38:8C:25:39:B2\n" + + "a=group:BUNDLE 0 1 2\n" + + "m=audio 9 UDP/TLS/RTP/SAVPF 111 9 0 8\n" + + "c=IN IP4 0.0.0.0\n" + + "a=setup:actpass\n" + + "a=mid:0\n" + + "a=ice-ufrag:obZwzAcRtxwuozPZ\n" + + "a=ice-pwd:TWXNaPeyKTTvRLyIQhWHfHlZHJjtcoKs\n" + + "a=rtcp-mux\n" + + "a=rtcp-rsize\n" + + "a=rtpmap:111 opus/48000/2\n" + + "a=fmtp:111 minptime=10;usedtx=1;useinbandfec=1\n" + + "a=rtcp-fb:111 transport-cc \n" + + "a=rtpmap:9 G722/8000\n" + + "a=rtpmap:0 PCMU/8000\n" + + "a=rtpmap:8 PCMA/8000\n" + + "a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\n" + + "a=ssrc:2963372119 cname:dcc3a6d5-37a1-42e7-94a9-d520f20d0c90\n" + + "a=ssrc:2963372119 msid:dcc3a6d5-37a1-42e7-94a9-d520f20d0c90 4b811ab6-6926-473d-8ca5-ac45f268c507\n" + + "a=ssrc:2963372119 mslabel:dcc3a6d5-37a1-42e7-94a9-d520f20d0c90\n" + + "a=ssrc:2963372119 label:4b811ab6-6926-473d-8ca5-ac45f268c507\n" + + "a=msid:dcc3a6d5-37a1-42e7-94a9-d520f20d0c90 4b811ab6-6926-473d-8ca5-ac45f268c507\n" + + "a=sendrecv\n" + + "a=candidate:1155505470 1 udp 2130706431 13.41.173.213 41385 typ host\n" + + "a=candidate:1155505470 2 udp 2130706431 13.41.173.213 41385 typ host\n" + + "a=candidate:1155505470 1 udp 2130706431 13.41.173.213 40026 typ host\n" + + "a=candidate:1155505470 2 udp 2130706431 13.41.173.213 40026 typ host\n" + + "a=end-of-candidates\n" + + "m=video 9 UDP/TLS/RTP/SAVPF 96 97 102 103 104 106 108 109 98 99 112 116\n" + + "c=IN IP4 0.0.0.0\n" + + "a=setup:actpass\n" + + "a=mid:1\n" + + "a=ice-ufrag:obZwzAcRtxwuozPZ\n" + + "a=ice-pwd:TWXNaPeyKTTvRLyIQhWHfHlZHJjtcoKs\n" + + "a=rtcp-mux\n" + + "a=rtcp-rsize\n" + + "a=rtpmap:96 VP8/90000\n" + + "a=rtcp-fb:96 goog-remb \n" + + "a=rtcp-fb:96 transport-cc \n" + + "a=rtcp-fb:96 ccm fir\n" + + "a=rtcp-fb:96 nack \n" + + "a=rtcp-fb:96 nack pli\n" + + "a=rtpmap:97 rtx/90000\n" + + "a=fmtp:97 apt=96\n" + + "a=rtpmap:102 H264/90000\n" + + "a=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f\n" + + "a=rtcp-fb:102 goog-remb \n" + + "a=rtcp-fb:102 transport-cc \n" + + "a=rtcp-fb:102 ccm fir\n" + + "a=rtcp-fb:102 nack \n" + + "a=rtcp-fb:102 nack pli\n" + + "a=rtpmap:103 rtx/90000\n" + + "a=fmtp:103 apt=102\n" + + "a=rtpmap:104 H264/90000\n" + + "a=fmtp:104 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f\n" + + "a=rtcp-fb:104 goog-remb \n" + + "a=rtcp-fb:104 transport-cc \n" + + "a=rtcp-fb:104 ccm fir\n" + + "a=rtcp-fb:104 nack \n" + + "a=rtcp-fb:104 nack pli\n" + + "a=rtpmap:106 H264/90000\n" + + "a=fmtp:106 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f\n" + + "a=rtcp-fb:106 goog-remb \n" + + "a=rtcp-fb:106 transport-cc \n" + + "a=rtcp-fb:106 ccm fir\n" + + "a=rtcp-fb:106 nack \n" + + "a=rtcp-fb:106 nack pli\n" + + "a=rtpmap:108 H264/90000\n" + + "a=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f\n" + + "a=rtcp-fb:108 goog-remb \n" + + "a=rtcp-fb:108 transport-cc \n" + + "a=rtcp-fb:108 ccm fir\n" + + "a=rtcp-fb:108 nack \n" + + "a=rtcp-fb:108 nack pli\n" + + "a=rtpmap:109 rtx/90000\n" + + "a=fmtp:109 apt=108\n" + + "a=rtpmap:98 VP9/90000\n" + + "a=fmtp:98 profile-id=0\n" + + "a=rtcp-fb:98 goog-remb \n" + + "a=rtcp-fb:98 transport-cc \n" + + "a=rtcp-fb:98 ccm fir\n" + + "a=rtcp-fb:98 nack \n" + + "a=rtcp-fb:98 nack pli\n" + + "a=rtpmap:99 rtx/90000\n" + + "a=fmtp:99 apt=98\n" + + "a=rtpmap:112 H264/90000\n" + + "a=fmtp:112 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f\n" + + "a=rtcp-fb:112 goog-remb \n" + + "a=rtcp-fb:112 transport-cc \n" + + "a=rtcp-fb:112 ccm fir\n" + + "a=rtcp-fb:112 nack \n" + + "a=rtcp-fb:112 nack pli\n" + + "a=rtpmap:116 ulpfec/90000\n" + + "a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\n" + + "a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\n" + + "a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\n" + + "a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\n" + + "a=rid:f recv\n" + + "a=rid:h recv\n" + + "a=rid:q recv\n" + + "a=simulcast:recv f;h;q\n" + + "a=ssrc:1212931603 cname:dcc3a6d5-37a1-42e7-94a9-d520f20d0c90\n" + + "a=ssrc:1212931603 msid:dcc3a6d5-37a1-42e7-94a9-d520f20d0c90 12905f48-75b9-499f-ba50-fc00f56a86c6\n" + + "a=ssrc:1212931603 mslabel:dcc3a6d5-37a1-42e7-94a9-d520f20d0c90\n" + + "a=ssrc:1212931603 label:12905f48-75b9-499f-ba50-fc00f56a86c6\n" + + "a=msid:dcc3a6d5-37a1-42e7-94a9-d520f20d0c90 12905f48-75b9-499f-ba50-fc00f56a86c6\n" + + "a=sendrecv\n" + + "m=application 9 UDP/DTLS/SCTP webrtc-datachannel\n" + + "c=IN IP4 0.0.0.0\n" + + "a=setup:actpass\n" + + "a=mid:2\n" + + "a=sendrecv\n" + + "a=sctp-port:5000\n" + + "a=ice-ufrag:obZwzAcRtxwuozPZ\n" + + "a=ice-pwd:TWXNaPeyKTTvRLyIQhWHfHlZHJjtcoKs"; diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 6ec2dcb5d4d..9f7678af7e5 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -29,7 +29,6 @@ import { import { MCallAnswer, MCallHangupReject, - SDPStreamMetadata, SDPStreamMetadataKey, SDPStreamMetadataPurpose, } from "../../../src/webrtc/callEventTypes"; @@ -40,12 +39,12 @@ import { MockMediaStreamTrack, installWebRTCMocks, MockRTCPeerConnection, - MockRTCRtpTransceiver, SCREENSHARE_STREAM_ID, - MockRTCRtpSender, + runOnTrackForStream, } from "../../test-utils/webrtc"; -import { CallFeed } from "../../../src/webrtc/callFeed"; import { EventType, IContent, ISendEventResponse, MatrixEvent, Room } from "../../../src"; +import { RemoteCallFeed } from "../../../src/webrtc/remoteCallFeed"; +import { LocalCallFeed } from "../../../src/webrtc/localCallFeed"; const FAKE_ROOM_ID = "!foo:bar"; const CALL_LIFETIME = 60000; @@ -81,16 +80,14 @@ const fakeIncomingCall = async (client: TestClient, call: MatrixCall, version: s getLocalAge: () => 1, } as unknown as MatrixEvent); call.getFeeds().push( - new CallFeed({ + new RemoteCallFeed({ client: client.client, - userId: "remote_user_id", - deviceId: undefined, + call: call, + streamId: "remote_stream_id", stream: new MockMediaStream("remote_stream_id", [ new MockMediaStreamTrack("remote_tack_id", "audio"), ]) as unknown as MediaStream, - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: false, - videoMuted: false, + metadata: { purpose: SDPStreamMetadataPurpose.Usermedia }, }), ); await callPromise; @@ -178,6 +175,7 @@ describe("Call", function () { }), ); + // @ts-ignore const mockAddIceCandidate = (call.peerConn!.addIceCandidate = jest.fn()); call.onRemoteIceCandidatesReceived( makeMockEvent("@test:foo", { @@ -212,6 +210,7 @@ describe("Call", function () { it("should add candidates received before answer if party ID is correct", async function () { await startVoiceCall(client, call); + // @ts-ignore const mockAddIceCandidate = (call.peerConn!.addIceCandidate = jest.fn()); call.onRemoteIceCandidatesReceived( @@ -318,16 +317,20 @@ describe("Call", function () { }), ); - (call as any).pushRemoteFeed( + runOnTrackForStream( + call, new MockMediaStream("remote_stream", [ - new MockMediaStreamTrack("remote_audio_track", "audio"), - new MockMediaStreamTrack("remote_video_track", "video"), - ]), + new MockMediaStreamTrack("audio", "audio"), + new MockMediaStreamTrack("video", "video"), + ]).typed(), ); - const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream"); + + const feed = call.getRemoteFeeds().find((feed) => feed.streamId === "remote_stream"); expect(feed?.purpose).toBe(SDPStreamMetadataPurpose.Usermedia); - expect(feed?.isAudioMuted()).toBeTruthy(); - expect(feed?.isVideoMuted()).not.toBeTruthy(); + // @ts-ignore + expect(feed?.audioMuted).toBeTruthy(); + // @ts-ignore + expect(feed?.videoMuted).not.toBeTruthy(); }); it("should fallback to replaceTrack() if the other side doesn't support SPDStreamMetadata", async () => { @@ -385,22 +388,24 @@ describe("Call", function () { }), ); + const audioTrack = new MockMediaStreamTrack("new_audio_track", "audio"); + const videoTrack = new MockMediaStreamTrack("video_track", "video"); await call.updateLocalUsermediaStream( - new MockMediaStream("replacement_stream", [ - new MockMediaStreamTrack("new_audio_track", "audio"), - new MockMediaStreamTrack("video_track", "video"), - ]).typed(), + new MockMediaStream("replacement_stream", [audioTrack, videoTrack]).typed(), ); - // XXX: Lots of inspecting the prvate state of the call object here - const transceivers: Map = (call as any).transceivers; - - expect(call.localUsermediaStream!.id).toBe("stream"); expect(call.localUsermediaStream!.getAudioTracks()[0].id).toBe("new_audio_track"); expect(call.localUsermediaStream!.getVideoTracks()[0].id).toBe("video_track"); - // call has a function for generating these but we hardcode here to avoid exporting it - expect(transceivers.get("m.usermedia:audio")!.sender.track!.id).toBe("new_audio_track"); - expect(transceivers.get("m.usermedia:video")!.sender.track!.id).toBe("video_track"); + expect( + // @ts-ignore + call.peerConn?.getTransceivers().find((transceiver) => transceiver.sender.track?.kind === "audio")?.sender + .track, + ).toBe(audioTrack); + expect( + // @ts-ignore + call.peerConn?.getTransceivers().find((transceiver) => transceiver.sender.track?.kind === "video")?.sender + .track, + ).toBe(videoTrack); }); it("should handle upgrade to video call", async () => { @@ -422,27 +427,32 @@ describe("Call", function () { // setLocalVideoMuted probably? await (call as any).upgradeCall(false, true); - // XXX: More inspecting private state of the call object - const transceivers: Map = (call as any).transceivers; - expect(call.localUsermediaStream!.getAudioTracks()[0].id).toBe("usermedia_audio_track"); expect(call.localUsermediaStream!.getVideoTracks()[0].id).toBe("usermedia_video_track"); - expect(transceivers.get("m.usermedia:audio")!.sender.track!.id).toBe("usermedia_audio_track"); - expect(transceivers.get("m.usermedia:video")!.sender.track!.id).toBe("usermedia_video_track"); + expect( + // @ts-ignore + call.peerConn?.getTransceivers().find((transceiver) => transceiver.sender.track?.kind === "audio")?.sender + .track?.id, + ).toBe("usermedia_audio_track"); + expect( + // @ts-ignore + call.peerConn?.getTransceivers().find((transceiver) => transceiver.sender.track?.kind === "video")?.sender + .track?.id, + ).toBe("usermedia_video_track"); }); it("should handle SDPStreamMetadata changes", async () => { await startVoiceCall(client, call); - (call as any).updateRemoteSDPStreamMetadata({ + (call as any).onMetadata({ remote_stream: { purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: false, video_muted: false, }, }); - (call as any).pushRemoteFeed(new MockMediaStream("remote_stream", [])); - const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream"); + //(call as any).pushRemoteStream(new MockMediaStream("remote_stream", [])); + const feed = call.getRemoteFeeds().find((feed) => feed.streamId === "remote_stream"); call.onSDPStreamMetadataChangedReceived( makeMockEvent("@test:foo", { @@ -522,14 +532,14 @@ describe("Call", function () { it("if no video", async () => { call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - (call as any).pushRemoteFeed(new MockMediaStream("remote_stream1", [])); + (call as any).addRemoteFeedWithoutMetadata(new MockMediaStream("remote_stream1", [])); expect(call.type).toBe(CallType.Voice); }); it("if remote video", async () => { call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - (call as any).pushRemoteFeed( + (call as any).addRemoteFeedWithoutMetadata( new MockMediaStream("remote_stream1", [new MockMediaStreamTrack("track_id", "video")]), ); expect(call.type).toBe(CallType.Video); @@ -541,39 +551,33 @@ describe("Call", function () { // since this is testing for the presence of a local sender, we need to add a transciever // rather than just a source track const mockTrack = new MockMediaStreamTrack("track_id", "video"); - const mockTransceiver = new MockRTCRtpTransceiver(call.peerConn as unknown as MockRTCPeerConnection); - mockTransceiver.sender = new MockRTCRtpSender(mockTrack) as unknown as RTCRtpSender; - (call as any).transceivers.set("m.usermedia:video", mockTransceiver); - (call as any).pushNewLocalFeed( + (call as any).addLocalFeedFromStream( new MockMediaStream("remote_stream1", [mockTrack]), SDPStreamMetadataPurpose.Usermedia, false, ); + call.getLocalFeeds()[0].publish(call); expect(call.type).toBe(CallType.Video); }); }); it("should correctly generate local SDPStreamMetadata", async () => { const callPromise = call.placeCallWithCallFeeds([ - new CallFeed({ + new LocalCallFeed({ client: client.client, stream: new MockMediaStream("local_stream1", [ new MockMediaStreamTrack("track_id", "audio"), ]) as unknown as MediaStream, roomId: call.roomId, - userId: client.getUserId(), - deviceId: undefined, purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: false, - videoMuted: false, }), ]); await client.httpBackend!.flush(""); await callPromise; call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - (call as any).pushNewLocalFeed( + (call as any).addLocalFeedFromStream( new MockMediaStream("local_stream2", [ new MockMediaStreamTrack("track_id", "video"), ]) as unknown as MediaStream, @@ -581,17 +585,17 @@ describe("Call", function () { ); await call.setMicrophoneMuted(true); - expect((call as any).getLocalSDPStreamMetadata()).toStrictEqual({ - local_stream1: { + expect((call as any).metadata).toStrictEqual({ + local_stream1: expect.objectContaining({ purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: true, video_muted: true, - }, - local_stream2: { + }), + local_stream2: expect.objectContaining({ purpose: SDPStreamMetadataPurpose.Screenshare, audio_muted: true, video_muted: false, - }, + }), }); }); @@ -602,30 +606,22 @@ describe("Call", function () { const remoteScreensharingStream = new MockMediaStream("remote_screensharing_stream_id", []); const callPromise = call.placeCallWithCallFeeds([ - new CallFeed({ + new LocalCallFeed({ client: client.client, - userId: client.getUserId(), - deviceId: undefined, stream: localUsermediaStream as unknown as MediaStream, purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: false, - videoMuted: false, }), - new CallFeed({ + new LocalCallFeed({ client: client.client, - userId: client.getUserId(), - deviceId: undefined, stream: localScreensharingStream as unknown as MediaStream, purpose: SDPStreamMetadataPurpose.Screenshare, - audioMuted: false, - videoMuted: false, }), ]); await client.httpBackend!.flush(""); await callPromise; call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - (call as any).updateRemoteSDPStreamMetadata({ + (call as any).onMetadata({ remote_usermedia_stream_id: { purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: false, @@ -638,17 +634,18 @@ describe("Call", function () { video_muted: false, }, }); - (call as any).pushRemoteFeed(remoteUsermediaStream); - (call as any).pushRemoteFeed(remoteScreensharingStream); + + runOnTrackForStream(call, remoteUsermediaStream.typed()); + runOnTrackForStream(call, remoteScreensharingStream.typed()); expect(call.localUsermediaFeed!.stream).toBe(localUsermediaStream); expect(call.localUsermediaStream).toBe(localUsermediaStream); expect(call.localScreensharingFeed!.stream).toBe(localScreensharingStream); expect(call.localScreensharingStream).toBe(localScreensharingStream); - expect(call.remoteUsermediaFeed!.stream).toBe(remoteUsermediaStream); - expect(call.remoteUsermediaStream).toBe(remoteUsermediaStream); - expect(call.remoteScreensharingFeed!.stream).toBe(remoteScreensharingStream); - expect(call.remoteScreensharingStream).toBe(remoteScreensharingStream); + expect(call.remoteUsermediaFeed?.stream?.getTracks()).toStrictEqual(remoteUsermediaStream.getTracks()); + expect(call.remoteUsermediaStream?.getTracks()).toStrictEqual(remoteUsermediaStream.getTracks()); + expect(call.remoteScreensharingFeed?.stream?.getTracks()).toStrictEqual(remoteScreensharingStream.getTracks()); + expect(call.remoteScreensharingStream?.getTracks()).toStrictEqual(remoteScreensharingStream.getTracks()); expect(call.hasRemoteUserMediaAudioTrack).toBe(false); }); @@ -771,41 +768,21 @@ describe("Call", function () { call.off(CallEvent.FeedsChanged, FEEDS_CHANGED_CALLBACK); }); - it("should ignore stream passed to pushRemoteFeed()", async () => { - await call.onAnswerReceived( - makeMockEvent("@test:foo", { - version: 1, - call_id: call.callId, - party_id: "party_id", - answer: { - sdp: DUMMY_SDP, - }, - [SDPStreamMetadataKey]: { - [STREAM_ID]: { - purpose: SDPStreamMetadataPurpose.Usermedia, - }, - }, - }), - ); - - (call as any).pushRemoteFeed(new MockMediaStream(STREAM_ID)); - (call as any).pushRemoteFeed(new MockMediaStream(STREAM_ID)); - - expect(call.getRemoteFeeds().length).toBe(1); - expect(FEEDS_CHANGED_CALLBACK).toHaveBeenCalledTimes(1); - }); - - it("should ignore stream passed to pushRemoteFeedWithoutMetadata()", async () => { - (call as any).pushRemoteFeedWithoutMetadata(new MockMediaStream(STREAM_ID)); - (call as any).pushRemoteFeedWithoutMetadata(new MockMediaStream(STREAM_ID)); + it("should ignore stream passed to addRemoteFeedWithoutMetadata()", async () => { + const stream = new MockMediaStream(STREAM_ID); + try { + (call as any).addRemoteFeedWithoutMetadata(stream); + (call as any).addRemoteFeedWithoutMetadata(stream); + } catch (e) {} expect(call.getRemoteFeeds().length).toBe(1); expect(FEEDS_CHANGED_CALLBACK).toHaveBeenCalledTimes(1); }); - it("should ignore stream passed to pushNewLocalFeed()", async () => { - (call as any).pushNewLocalFeed(new MockMediaStream(STREAM_ID), SDPStreamMetadataPurpose.Screenshare); - (call as any).pushNewLocalFeed(new MockMediaStream(STREAM_ID), SDPStreamMetadataPurpose.Screenshare); + it("should ignore stream passed to addLocalFeedFromStream()", async () => { + const stream = new MockMediaStream(STREAM_ID); + (call as any).addLocalFeedFromStream(stream, SDPStreamMetadataPurpose.Screenshare); + (call as any).addLocalFeedFromStream(stream, SDPStreamMetadataPurpose.Screenshare); // We already have one local feed from placeVoiceCall() expect(call.getLocalFeeds().length).toBe(2); @@ -892,11 +869,11 @@ describe("Call", function () { await call.setMicrophoneMuted(true); expect(mockSendVoipEvent).toHaveBeenCalledWith(EventType.CallSDPStreamMetadataChangedPrefix, { [SDPStreamMetadataKey]: { - mock_stream_from_media_handler: { + mock_stream_from_media_handler: expect.objectContaining({ purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: true, video_muted: false, - }, + }), }, }); }); @@ -905,49 +882,48 @@ describe("Call", function () { await call.setLocalVideoMuted(true); expect(mockSendVoipEvent).toHaveBeenCalledWith(EventType.CallSDPStreamMetadataChangedPrefix, { [SDPStreamMetadataKey]: { - mock_stream_from_media_handler: { + mock_stream_from_media_handler: expect.objectContaining({ purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: false, video_muted: true, - }, + }), }, }); }); }); describe("receiving sdp_stream_metadata_changed events", () => { - const setupCall = (audio: boolean, video: boolean): SDPStreamMetadata => { - const metadata = { - stream: { - purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: audio, - video_muted: video, - }, - }; - (call as any).pushRemoteFeed( - new MockMediaStream("stream", [ - new MockMediaStreamTrack("track1", "audio"), - new MockMediaStreamTrack("track1", "video"), - ]), - ); + const setupCall = (audio: boolean, video: boolean): void => { call.onSDPStreamMetadataChangedReceived({ getContent: () => ({ - [SDPStreamMetadataKey]: metadata, + [SDPStreamMetadataKey]: { + stream: { + user_id: "user", + device_id: "device", + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: audio, + video_muted: video, + }, + }, }), } as MatrixEvent); - return metadata; + const stream = new MockMediaStream("stream", [ + new MockMediaStreamTrack("track1", "audio"), + new MockMediaStreamTrack("track2", "video"), + ]); + runOnTrackForStream(call, stream.typed()); }; it("should handle incoming sdp_stream_metadata_changed with audio muted", async () => { - const metadata = setupCall(true, false); - expect((call as any).remoteSDPStreamMetadata).toStrictEqual(metadata); + setupCall(true, false); + expect(call.opponentSupportsSDPStreamMetadata()).toBe(true); expect(call.getRemoteFeeds()[0].isAudioMuted()).toBe(true); expect(call.getRemoteFeeds()[0].isVideoMuted()).toBe(false); }); it("should handle incoming sdp_stream_metadata_changed with video muted", async () => { - const metadata = setupCall(false, true); - expect((call as any).remoteSDPStreamMetadata).toStrictEqual(metadata); + setupCall(false, true); + expect(call.opponentSupportsSDPStreamMetadata()).toBe(true); expect(call.getRemoteFeeds()[0].isAudioMuted()).toBe(false); expect(call.getRemoteFeeds()[0].isVideoMuted()).toBe(true); }); @@ -1081,6 +1057,7 @@ describe("Call", function () { beforeEach(async () => { await call.answer(); await untilEventSent(FAKE_ROOM_ID, EventType.CallAnswer, expect.objectContaining({})); + // @ts-ignore mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; }); @@ -1263,6 +1240,7 @@ describe("Call", function () { await sendNegotiatePromise; + // @ts-ignore const mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; expect( mockPeerConn.transceivers[mockPeerConn.transceivers.length - 1].setCodecPreferences, @@ -1270,6 +1248,7 @@ describe("Call", function () { }); it("re-uses transceiver when screen sharing is re-enabled", async () => { + // @ts-ignore const mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; // sanity check: we should start with one transciever (user media audio) @@ -1320,8 +1299,10 @@ describe("Call", function () { MockRTCPeerConnection.triggerAllNegotiations(); - const mockVideoSender = call.peerConn!.getSenders().find((s) => s.track!.kind === "video"); - const mockReplaceTrack = (mockVideoSender!.replaceTrack = jest.fn()); + // @ts-ignore + const mockVideoSender = call.peerConn.getSenders().find((s) => s.track!.kind === "video")!; + jest.spyOn(mockVideoSender, "replaceTrack"); + const mockReplaceTrack = mocked(mockVideoSender?.replaceTrack); await call.setScreensharingEnabled(true); @@ -1449,23 +1430,24 @@ describe("Call", function () { describe("onTrack", () => { it("ignores streamless track", async () => { - // @ts-ignore Mock pushRemoteFeed() is private - jest.spyOn(call, "pushRemoteFeed"); + // @ts-ignore Mock addRemoteFeedWithoutMetadata() is private + jest.spyOn(call, "addRemoteFeedWithoutMetadata"); await call.placeVoiceCall(); + // @ts-ignore (call.peerConn as unknown as MockRTCPeerConnection).onTrackListener!({ streams: [], track: new MockMediaStreamTrack("track_ev", "audio"), } as unknown as RTCTrackEvent); - // @ts-ignore Mock pushRemoteFeed() is private - expect(call.pushRemoteFeed).not.toHaveBeenCalled(); + // @ts-ignore Mock addRemoteFeedWithoutMetadata() is private + expect(call.addRemoteFeedWithoutMetadata).not.toHaveBeenCalled(); }); it("correctly pushes", async () => { - // @ts-ignore Mock pushRemoteFeed() is private - jest.spyOn(call, "pushRemoteFeed"); + // @ts-ignore Mock addRemoteFeedWithoutMetadata() is private + jest.spyOn(call, "addRemoteFeedWithoutMetadata"); await call.placeVoiceCall(); await call.onAnswerReceived( @@ -1480,14 +1462,16 @@ describe("Call", function () { ); const stream = new MockMediaStream("stream_ev", [new MockMediaStreamTrack("track_ev", "audio")]); + // @ts-ignore (call.peerConn as unknown as MockRTCPeerConnection).onTrackListener!({ streams: [stream], track: stream.getAudioTracks()[0], + transceiver: { mid: "0" }, } as unknown as RTCTrackEvent); - // @ts-ignore Mock pushRemoteFeed() is private - expect(call.pushRemoteFeed).toHaveBeenCalledWith(stream); - // @ts-ignore Mock pushRemoteFeed() is private + // @ts-ignore Mock addRemoteFeedWithoutMetadata() is private + expect(call.addRemoteFeedWithoutMetadata).toHaveBeenCalledWith(stream); + // @ts-ignore Mock removeTrackListeners() is private expect(call.removeTrackListeners.has(stream)).toBe(true); }); }); @@ -1571,6 +1555,7 @@ describe("Call", function () { jest.useFakeTimers(); call.addListener(CallEvent.LengthChanged, lengthChangedListener); await fakeIncomingCall(client, call, "1"); + // @ts-ignore (call.peerConn as unknown as MockRTCPeerConnection).iceConnectionStateChangeListener!(); let hasAdvancedBy = 0; @@ -1592,6 +1577,7 @@ describe("Call", function () { await fakeIncomingCall(client, call, "1"); + // @ts-ignore mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; mockPeerConn.iceConnectionState = "disconnected"; mockPeerConn.iceConnectionStateChangeListener!(); diff --git a/spec/unit/webrtc/callFeed.spec.ts b/spec/unit/webrtc/callFeed.spec.ts index 803e4648ed6..ea00553912a 100644 --- a/spec/unit/webrtc/callFeed.spec.ts +++ b/spec/unit/webrtc/callFeed.spec.ts @@ -15,22 +15,25 @@ limitations under the License. */ import { SDPStreamMetadataPurpose } from "../../../src/webrtc/callEventTypes"; -import { CallFeed } from "../../../src/webrtc/callFeed"; import { TestClient } from "../../TestClient"; -import { MockMatrixCall, MockMediaStream, MockMediaStreamTrack } from "../../test-utils/webrtc"; +import { installWebRTCMocks, MockMatrixCall, MockMediaStream, MockMediaStreamTrack } from "../../test-utils/webrtc"; import { CallEvent, CallState } from "../../../src/webrtc/call"; +import { LocalCallFeed } from "../../../src/webrtc/localCallFeed"; +import { RemoteCallFeed } from "../../../src/webrtc/remoteCallFeed"; describe("CallFeed", () => { const roomId = "room1"; let client: TestClient; let call: MockMatrixCall; - let feed: CallFeed; + let feed: LocalCallFeed; beforeEach(() => { + installWebRTCMocks(); + client = new TestClient("@alice:foo", "somedevice", "token", undefined, {}); call = new MockMatrixCall(roomId); - feed = new CallFeed({ + feed = new LocalCallFeed({ client: client.client, call: call.typed(), roomId, @@ -50,50 +53,49 @@ describe("CallFeed", () => { describe("muting", () => { describe("muting by default", () => { it("should mute audio by default", () => { - expect(feed.isAudioMuted()).toBeTruthy(); + expect(feed.audioMuted).toBeTruthy(); }); it("should mute video by default", () => { - expect(feed.isVideoMuted()).toBeTruthy(); + expect(feed.videoMuted).toBeTruthy(); }); }); describe("muting after adding a track", () => { it("should un-mute audio", () => { // @ts-ignore Mock - feed.stream.addTrack(new MockMediaStreamTrack("track", "audio", true)); - expect(feed.isAudioMuted()).toBeFalsy(); + feed.setNewStream(new MockMediaStream("stream", [new MockMediaStreamTrack("track", "audio", true)])); + expect(feed.audioMuted).toBeFalsy(); }); it("should un-mute video", () => { // @ts-ignore Mock - feed.stream.addTrack(new MockMediaStreamTrack("track", "video", true)); - expect(feed.isVideoMuted()).toBeFalsy(); + feed.setNewStream(new MockMediaStream("stream", [new MockMediaStreamTrack("track", "video", true)])); + expect(feed.videoMuted).toBeFalsy(); }); }); describe("muting after calling setAudioVideoMuted()", () => { it("should mute audio by default", () => { // @ts-ignore Mock - feed.stream.addTrack(new MockMediaStreamTrack("track", "audio", true)); + feed.setNewStream(new MockMediaStream("stream", [new MockMediaStreamTrack("track", "audio", true)])); feed.setAudioVideoMuted(true, false); - expect(feed.isAudioMuted()).toBeTruthy(); + expect(feed.audioMuted).toBeTruthy(); }); it("should mute video by default", () => { // @ts-ignore Mock - feed.stream.addTrack(new MockMediaStreamTrack("track", "video", true)); + feed.setNewStream(new MockMediaStream("stream", [new MockMediaStreamTrack("track", "video", true)])); feed.setAudioVideoMuted(false, true); - expect(feed.isVideoMuted()).toBeTruthy(); + expect(feed.videoMuted).toBeTruthy(); }); }); }); describe("connected", () => { - it.each([true, false])("should always be connected, if isLocal()", (val: boolean) => { + it.each([true, false])("should always be connected, if isLocal", (val: boolean) => { // @ts-ignore feed._connected = val; - jest.spyOn(feed, "isLocal").mockReturnValue(true); expect(feed.connected).toBeTruthy(); }); @@ -102,9 +104,19 @@ describe("CallFeed", () => { [CallState.Connected, true], [CallState.Connecting, false], ])("should react to call state, when !isLocal()", (state: CallState, expected: Boolean) => { + jest.spyOn(call, "opponentSupportsSDPStreamMetadata").mockReturnValue(false); + + const remoteFeed = new RemoteCallFeed({ + client: client.client, + call: call.typed(), + streamId: "id", + }); + + remoteFeed.stream?.addTrack(new MockMediaStreamTrack("track1", "video").typed()); + call.state = state; call.emit(CallEvent.State, state); - expect(feed.connected).toBe(expected); + expect(remoteFeed.connected).toBe(expected); }); }); }); diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 4b68100113e..ba6c3396e60 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -34,7 +34,7 @@ import { FAKE_USER_ID_2, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1, - FAKE_USER_ID_3, + runOnTrackForStream, } from "../../test-utils/webrtc"; import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from "../../../src/webrtc/callEventTypes"; import { sleep } from "../../../src/utils"; @@ -42,6 +42,7 @@ import { CallEventHandlerEvent } from "../../../src/webrtc/callEventHandler"; import { CallFeed } from "../../../src/webrtc/callFeed"; import { CallEvent, CallState } from "../../../src/webrtc/call"; import { flushPromises } from "../../test-utils/flushPromises"; +import { RemoteCallFeed } from "../../../src/webrtc/remoteCallFeed"; const FAKE_STATE_EVENTS = [ { @@ -168,13 +169,13 @@ describe("Group Call", function () { groupCall.leave(); }); - it("does not start initializing local call feed twice", () => { + it("does not start initializing local call feed twice", async () => { const promise1 = groupCall.initLocalCallFeed(); // @ts-ignore Mock groupCall.state = GroupCallState.LocalCallFeedUninitialized; const promise2 = groupCall.initLocalCallFeed(); - expect(promise1).toEqual(promise2); + expect(await promise1).toEqual(await promise2); }); it("sets state to local call feed uninitialized when getUserMedia() fails", async () => { @@ -384,17 +385,14 @@ describe("Group Call", function () { await groupCall.create(); }); - it("ignores changes, if we can't get user id of opponent", async () => { - const call = new MockMatrixCall(room.roomId, groupCall.groupCallId); - jest.spyOn(call, "getOpponentMember").mockReturnValue({ userId: undefined }); - - // @ts-ignore Mock - expect(() => groupCall.onCallFeedsChanged(call)).toThrow(); - }); - describe("usermedia feeds", () => { + beforeEach(() => { + currentFeed.purpose = SDPStreamMetadataPurpose.Usermedia; + newFeed.purpose = SDPStreamMetadataPurpose.Usermedia; + }); + it("adds new usermedia feed", async () => { - call.remoteUsermediaFeed = newFeed.typed(); + call.feeds = [newFeed.typed()]; // @ts-ignore Mock groupCall.onCallFeedsChanged(call); @@ -404,7 +402,8 @@ describe("Group Call", function () { it("replaces usermedia feed", async () => { groupCall.userMediaFeeds.push(currentFeed.typed()); - call.remoteUsermediaFeed = newFeed.typed(); + call.feeds = [newFeed.typed()]; + // @ts-ignore Mock groupCall.onCallFeedsChanged(call); @@ -412,6 +411,7 @@ describe("Group Call", function () { }); it("removes usermedia feed", async () => { + currentFeed.dispose(); groupCall.userMediaFeeds.push(currentFeed.typed()); // @ts-ignore Mock @@ -422,8 +422,14 @@ describe("Group Call", function () { }); describe("screenshare feeds", () => { + beforeEach(() => { + currentFeed.purpose = SDPStreamMetadataPurpose.Screenshare; + newFeed.purpose = SDPStreamMetadataPurpose.Screenshare; + }); + it("adds new screenshare feed", async () => { - call.remoteScreensharingFeed = newFeed.typed(); + call.feeds = [newFeed.typed()]; + // @ts-ignore Mock groupCall.onCallFeedsChanged(call); @@ -433,7 +439,8 @@ describe("Group Call", function () { it("replaces screenshare feed", async () => { groupCall.screenshareFeeds.push(currentFeed.typed()); - call.remoteScreensharingFeed = newFeed.typed(); + call.feeds = [newFeed.typed()]; + // @ts-ignore Mock groupCall.onCallFeedsChanged(call); @@ -441,6 +448,7 @@ describe("Group Call", function () { }); it("removes screenshare feed", async () => { + currentFeed.dispose(); groupCall.screenshareFeeds.push(currentFeed.typed()); // @ts-ignore Mock @@ -744,11 +752,13 @@ describe("Group Call", function () { while ( // @ts-ignore (newCall = groupCall1.calls.get(client2.userId)?.get(client2.deviceId)) === undefined || + // @ts-ignore newCall.peerConn === undefined || newCall.callId == oldCall.callId ) { await flushPromises(); } + // @ts-ignore const mockPc = newCall.peerConn as unknown as MockRTCPeerConnection; // ...then wait for it to be ready to negotiate @@ -838,7 +848,7 @@ describe("Group Call", function () { await groupCall.setMicrophoneMuted(true); - groupCall.localCallFeed!.stream.getAudioTracks().forEach((track) => expect(track.enabled).toBe(false)); + groupCall.localCallFeed!.stream!.getAudioTracks().forEach((track) => expect(track.enabled).toBe(false)); expect(groupCall.localCallFeed!.setAudioVideoMuted).toHaveBeenCalledWith(true, null); setAVMutedArray.forEach((f) => expect(f).toHaveBeenCalledWith(true, null)); tracksArray.forEach((track) => expect(track.enabled).toBe(false)); @@ -866,9 +876,8 @@ describe("Group Call", function () { await groupCall.setLocalVideoMuted(true); - groupCall.localCallFeed!.stream.getVideoTracks().forEach((track) => expect(track.enabled).toBe(false)); - expect(mockClient.getMediaHandler().getUserMediaStream).toHaveBeenCalledWith(true, false); - expect(groupCall.updateLocalUsermediaStream).toHaveBeenCalled(); + groupCall.localCallFeed!.stream!.getVideoTracks().forEach((track) => expect(track.enabled).toBe(false)); + expect(groupCall.localCallFeed!.setAudioVideoMuted).toHaveBeenCalledWith(null, true); setAVMutedArray.forEach((f) => expect(f).toHaveBeenCalledWith(null, true)); tracksArray.forEach((track) => expect(track.enabled).toBe(false)); sendMetadataUpdateArray.forEach((f) => expect(f).toHaveBeenCalled()); @@ -913,15 +922,15 @@ describe("Group Call", function () { // @ts-ignore const call = groupCall.calls.get(FAKE_USER_ID_2)!.get(FAKE_DEVICE_ID_2)!; call.getOpponentMember = () => ({ userId: call.invitee } as RoomMember); - // @ts-ignore Mock - call.pushRemoteFeed( - // @ts-ignore Mock + call.onSDPStreamMetadataChangedReceived(metadataEvent); + + runOnTrackForStream( + call, new MockMediaStream("stream", [ new MockMediaStreamTrack("audio_track", "audio"), new MockMediaStreamTrack("video_track", "video"), - ]), + ]).typed(), ); - call.onSDPStreamMetadataChangedReceived(metadataEvent); const feed = groupCall.getUserMediaFeed(call.invitee!, call.getOpponentDeviceId()!); expect(feed!.isAudioMuted()).toBe(true); @@ -931,6 +940,10 @@ describe("Group Call", function () { }); it("should mute remote feed's video after receiving metadata with video muted", async () => { + const stream = new MockMediaStream("stream", [ + new MockMediaStreamTrack("track1", "audio"), + new MockMediaStreamTrack("track2", "video"), + ]); const metadataEvent = getMetadataEvent(false, true); const groupCall = await createAndEnterGroupCall(mockClient, room); @@ -940,16 +953,10 @@ describe("Group Call", function () { // @ts-ignore const call = groupCall.calls.get(FAKE_USER_ID_2).get(FAKE_DEVICE_ID_2)!; call.getOpponentMember = () => ({ userId: call.invitee } as RoomMember); - // @ts-ignore Mock - call.pushRemoteFeed( - // @ts-ignore Mock - new MockMediaStream("stream", [ - new MockMediaStreamTrack("audio_track", "audio"), - new MockMediaStreamTrack("video_track", "video"), - ]), - ); call.onSDPStreamMetadataChangedReceived(metadataEvent); + runOnTrackForStream(call, stream.typed()); + const feed = groupCall.getUserMediaFeed(call.invitee!, call.getOpponentDeviceId()!); expect(feed!.isAudioMuted()).toBe(false); expect(feed!.isVideoMuted()).toBe(true); @@ -1199,11 +1206,6 @@ describe("Group Call", function () { }, }), } as MatrixEvent); - // @ts-ignore Mock - call.pushRemoteFeed( - // @ts-ignore Mock - new MockMediaStream("screensharing_stream", [new MockMediaStreamTrack("video_track", "video")]), - ); expect(groupCall.screenshareFeeds).toHaveLength(1); expect(groupCall.getScreenshareFeed(call.invitee!, call.getOpponentDeviceId()!)).toBeDefined(); @@ -1242,6 +1244,7 @@ describe("Group Call", function () { jest.useFakeTimers(); const mockClient = new MockCallMatrixClient(FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1); + const mockCall = new MockMatrixCall(FAKE_ROOM_ID).typed(); room = new Room(FAKE_ROOM_ID, mockClient.typed(), FAKE_USER_ID_1); room.currentState.members[FAKE_USER_ID_1] = { @@ -1249,27 +1252,19 @@ describe("Group Call", function () { } as unknown as RoomMember; groupCall = await createAndEnterGroupCall(mockClient.typed(), room); - mediaFeed1 = new CallFeed({ + mediaFeed1 = new RemoteCallFeed({ client: mockClient.typed(), roomId: FAKE_ROOM_ID, - userId: FAKE_USER_ID_2, - deviceId: FAKE_DEVICE_ID_1, - stream: new MockMediaStream("foo", []).typed(), - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: false, - videoMuted: true, + streamId: "stream1", + call: mockCall, }); groupCall.userMediaFeeds.push(mediaFeed1); - mediaFeed2 = new CallFeed({ + mediaFeed2 = new RemoteCallFeed({ client: mockClient.typed(), roomId: FAKE_ROOM_ID, - userId: FAKE_USER_ID_3, - deviceId: FAKE_DEVICE_ID_1, - stream: new MockMediaStream("foo", []).typed(), - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: false, - videoMuted: true, + streamId: "stream2", + call: mockCall, }); groupCall.userMediaFeeds.push(mediaFeed2); diff --git a/spec/unit/webrtc/mediaHandler.spec.ts b/spec/unit/webrtc/mediaHandler.spec.ts index 1b3a815b00a..3fe064e48ae 100644 --- a/spec/unit/webrtc/mediaHandler.spec.ts +++ b/spec/unit/webrtc/mediaHandler.spec.ts @@ -350,12 +350,12 @@ describe("Media Handler", function () { expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith( expect.objectContaining({ - video: { + video: expect.objectContaining({ mandatory: expect.objectContaining({ chromeMediaSource: "desktop", chromeMediaSourceId: FAKE_DESKTOP_SOURCE_ID, }), - }, + }), }), ); }); diff --git a/spec/unit/webrtc/stats/connectionStatsReporter.spec.ts b/spec/unit/webrtc/stats/connectionStatsReporter.spec.ts new file mode 100644 index 00000000000..1c9b2123319 --- /dev/null +++ b/spec/unit/webrtc/stats/connectionStatsReporter.spec.ts @@ -0,0 +1,46 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { ConnectionStatsReporter } from "../../../../src/webrtc/stats/connectionStatsReporter"; + +describe("ConnectionStatsReporter", () => { + describe("should on bandwidth stats", () => { + it("build bandwidth report if chromium starts attributes available", () => { + const stats = { + availableIncomingBitrate: 1000, + availableOutgoingBitrate: 2000, + } as RTCIceCandidatePairStats; + expect(ConnectionStatsReporter.buildBandwidthReport(stats)).toEqual({ download: 1, upload: 2 }); + }); + it("build empty bandwidth report if chromium starts attributes not available", () => { + const stats = {} as RTCIceCandidatePairStats; + expect(ConnectionStatsReporter.buildBandwidthReport(stats)).toEqual({ download: 0, upload: 0 }); + }); + }); + + describe("should on connection stats", () => { + it("build bandwidth report if chromium starts attributes available", () => { + const stats = { + availableIncomingBitrate: 1000, + availableOutgoingBitrate: 2000, + } as RTCIceCandidatePairStats; + expect(ConnectionStatsReporter.buildBandwidthReport(stats)).toEqual({ download: 1, upload: 2 }); + }); + it("build empty bandwidth report if chromium starts attributes not available", () => { + const stats = {} as RTCIceCandidatePairStats; + expect(ConnectionStatsReporter.buildBandwidthReport(stats)).toEqual({ download: 0, upload: 0 }); + }); + }); +}); diff --git a/spec/unit/webrtc/stats/groupCallStats.spec.ts b/spec/unit/webrtc/stats/groupCallStats.spec.ts new file mode 100644 index 00000000000..14a6d806622 --- /dev/null +++ b/spec/unit/webrtc/stats/groupCallStats.spec.ts @@ -0,0 +1,135 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { GroupCallStats } from "../../../../src/webrtc/stats/groupCallStats"; + +const GROUP_CALL_ID = "GROUP_ID"; +const LOCAL_USER_ID = "LOCAL_USER_ID"; +const TIME_INTERVAL = 10000; + +describe("GroupCallStats", () => { + let stats: GroupCallStats; + beforeEach(() => { + stats = new GroupCallStats(GROUP_CALL_ID, LOCAL_USER_ID, TIME_INTERVAL); + }); + + describe("should on adding a stats collector", () => { + it("creating a new one if not existing.", async () => { + expect(stats.addStatsCollector("CALL_ID", "USER_ID", mockRTCPeerConnection())).toBeTruthy(); + }); + + it("creating only one when trying add the same collector multiple times.", async () => { + expect(stats.addStatsCollector("CALL_ID", "USER_ID", mockRTCPeerConnection())).toBeTruthy(); + expect(stats.addStatsCollector("CALL_ID", "USER_ID", mockRTCPeerConnection())).toBeFalsy(); + // The User ID is not relevant! Because for stats the call is needed and the user id is for monitoring + expect(stats.addStatsCollector("CALL_ID", "SOME_OTHER_USER_ID", mockRTCPeerConnection())).toBeFalsy(); + }); + }); + + describe("should on removing a stats collector", () => { + it("returning `true` if the collector exists", async () => { + expect(stats.addStatsCollector("CALL_ID", "USER_ID", mockRTCPeerConnection())).toBeTruthy(); + expect(stats.removeStatsCollector("CALL_ID")).toBeTruthy(); + }); + it("returning false if the collector not exists", async () => { + expect(stats.removeStatsCollector("CALL_ID_NOT_EXIST")).toBeFalsy(); + }); + }); + + describe("should on get stats collector", () => { + it("returning `undefined` if collector not existing", async () => { + expect(stats.getStatsCollector("CALL_ID")).toBeUndefined(); + }); + + it("returning Collector if collector existing", async () => { + expect(stats.addStatsCollector("CALL_ID", "USER_ID", mockRTCPeerConnection())).toBeTruthy(); + expect(stats.getStatsCollector("CALL_ID")).toBeDefined(); + }); + }); + + describe("should on start", () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + afterEach(() => { + jest.useRealTimers(); + }); + + it("starting processing as well without stats collectors", async () => { + // @ts-ignore + stats.processStats = jest.fn(); + stats.start(); + jest.advanceTimersByTime(TIME_INTERVAL); + // @ts-ignore + expect(stats.processStats).toHaveBeenCalled(); + }); + + it("starting processing and calling the collectors", async () => { + stats.addStatsCollector("CALL_ID", "USER_ID", mockRTCPeerConnection()); + const collector = stats.getStatsCollector("CALL_ID"); + if (collector) { + const processStatsSpy = jest.spyOn(collector, "processStats"); + stats.start(); + jest.advanceTimersByTime(TIME_INTERVAL); + expect(processStatsSpy).toHaveBeenCalledWith(GROUP_CALL_ID, LOCAL_USER_ID); + } else { + throw new Error("Test failed, because no Collector found!"); + } + }); + + it("doing nothing if process already running", async () => { + // @ts-ignore + jest.spyOn(global, "setInterval").mockReturnValue(22); + stats.start(); + expect(setInterval).toHaveBeenCalledTimes(1); + stats.start(); + stats.start(); + stats.start(); + stats.start(); + expect(setInterval).toHaveBeenCalledTimes(1); + }); + }); + + describe("should on stop", () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + afterEach(() => { + jest.useRealTimers(); + }); + it("finish stats process if was started", async () => { + // @ts-ignore + jest.spyOn(global, "setInterval").mockReturnValue(22); + jest.spyOn(global, "clearInterval"); + stats.start(); + expect(setInterval).toHaveBeenCalledTimes(1); + stats.stop(); + expect(clearInterval).toHaveBeenCalledWith(22); + }); + + it("do nothing if stats process was not started", async () => { + jest.spyOn(global, "clearInterval"); + stats.stop(); + expect(clearInterval).not.toHaveBeenCalled(); + }); + }); +}); + +const mockRTCPeerConnection = (): RTCPeerConnection => { + const pc = {} as RTCPeerConnection; + pc.addEventListener = jest.fn(); + pc.getStats = jest.fn().mockResolvedValue(null); + return pc; +}; diff --git a/spec/unit/webrtc/stats/media/mediaSsrcHandler.spec.ts b/spec/unit/webrtc/stats/media/mediaSsrcHandler.spec.ts new file mode 100644 index 00000000000..4b6e93179a4 --- /dev/null +++ b/spec/unit/webrtc/stats/media/mediaSsrcHandler.spec.ts @@ -0,0 +1,41 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { Mid, Ssrc, MediaSsrcHandler } from "../../../../../src/webrtc/stats/media/mediaSsrcHandler"; +import { REMOTE_SFU_DESCRIPTION } from "../../../../test-utils/webrtc"; + +describe("MediaSsrcHandler", () => { + const remoteMap = new Map([ + ["0", ["2963372119"]], + ["1", ["1212931603"]], + ]); + let handler: MediaSsrcHandler; + beforeEach(() => { + handler = new MediaSsrcHandler(); + }); + describe("should parse description", () => { + it("and build mid ssrc map", () => { + handler.parse(REMOTE_SFU_DESCRIPTION, "remote"); + expect(handler.getSsrcToMidMap("remote")).toEqual(remoteMap); + }); + }); + + describe("should on find mid by ssrc", () => { + it("and return mid if mapping exists.", () => { + handler.parse(REMOTE_SFU_DESCRIPTION, "remote"); + expect(handler.findMidBySsrc("2963372119", "remote")).toEqual("0"); + }); + }); +}); diff --git a/spec/unit/webrtc/stats/media/mediaTrackHandler.spec.ts b/spec/unit/webrtc/stats/media/mediaTrackHandler.spec.ts new file mode 100644 index 00000000000..66dcc5ebf03 --- /dev/null +++ b/spec/unit/webrtc/stats/media/mediaTrackHandler.spec.ts @@ -0,0 +1,113 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { MediaTrackHandler } from "../../../../../src/webrtc/stats/media/mediaTrackHandler"; + +describe("TrackHandler", () => { + let pc: RTCPeerConnection; + let handler: MediaTrackHandler; + beforeEach(() => { + pc = { + getTransceivers: (): RTCRtpTransceiver[] => [mockTransceiver("1", "audio"), mockTransceiver("2", "video")], + } as RTCPeerConnection; + handler = new MediaTrackHandler(pc); + }); + describe("should get local tracks", () => { + it("returns video track", () => { + expect(handler.getLocalTracks("video")).toEqual([ + { + id: `sender-track-2`, + kind: "video", + } as MediaStreamTrack, + ]); + }); + + it("returns audio track", () => { + expect(handler.getLocalTracks("audio")).toEqual([ + { + id: `sender-track-1`, + kind: "audio", + } as MediaStreamTrack, + ]); + }); + }); + + describe("should get local track by mid", () => { + it("returns video track", () => { + expect(handler.getLocalTrackIdByMid("2")).toEqual("sender-track-2"); + }); + + it("returns audio track", () => { + expect(handler.getLocalTrackIdByMid("1")).toEqual("sender-track-1"); + }); + + it("returns undefined if not exists", () => { + expect(handler.getLocalTrackIdByMid("3")).toBeUndefined(); + }); + }); + + describe("should get remote track by mid", () => { + it("returns video track", () => { + expect(handler.getRemoteTrackIdByMid("2")).toEqual("receiver-track-2"); + }); + + it("returns audio track", () => { + expect(handler.getRemoteTrackIdByMid("1")).toEqual("receiver-track-1"); + }); + + it("returns undefined if not exists", () => { + expect(handler.getRemoteTrackIdByMid("3")).toBeUndefined(); + }); + }); + + describe("should get track by id", () => { + it("returns remote track", () => { + expect(handler.getTackById("receiver-track-2")).toEqual({ + id: `receiver-track-2`, + kind: "video", + } as MediaStreamTrack); + }); + + it("returns local track", () => { + expect(handler.getTackById("sender-track-1")).toEqual({ + id: `sender-track-1`, + kind: "audio", + } as MediaStreamTrack); + }); + + it("returns undefined if not exists", () => { + expect(handler.getTackById("sender-track-3")).toBeUndefined(); + }); + }); + + describe("should get simulcast track count", () => { + it("returns 2", () => { + expect(handler.getActiveSimulcastStreams()).toEqual(3); + }); + }); +}); + +const mockTransceiver = (mid: string, kind: "video" | "audio"): RTCRtpTransceiver => { + return { + mid, + currentDirection: "sendrecv", + sender: { + track: { id: `sender-track-${mid}`, kind } as MediaStreamTrack, + } as RTCRtpSender, + receiver: { + track: { id: `receiver-track-${mid}`, kind } as MediaStreamTrack, + } as RTCRtpReceiver, + } as RTCRtpTransceiver; +}; diff --git a/spec/unit/webrtc/stats/media/mediaTrackStatsHandler.spec.ts b/spec/unit/webrtc/stats/media/mediaTrackStatsHandler.spec.ts new file mode 100644 index 00000000000..d263786fda2 --- /dev/null +++ b/spec/unit/webrtc/stats/media/mediaTrackStatsHandler.spec.ts @@ -0,0 +1,83 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { MediaTrackHandler } from "../../../../../src/webrtc/stats/media/mediaTrackHandler"; +import { MediaTrackStatsHandler } from "../../../../../src/webrtc/stats/media/mediaTrackStatsHandler"; +import { MediaSsrcHandler } from "../../../../../src/webrtc/stats/media/mediaSsrcHandler"; + +describe("MediaTrackStatsHandler", () => { + let statsHandler: MediaTrackStatsHandler; + let ssrcHandler: MediaSsrcHandler; + let trackHandler: MediaTrackHandler; + beforeEach(() => { + ssrcHandler = {} as MediaSsrcHandler; + trackHandler = {} as MediaTrackHandler; + trackHandler.getLocalTrackIdByMid = jest.fn().mockReturnValue("2222"); + trackHandler.getRemoteTrackIdByMid = jest.fn().mockReturnValue("5555"); + trackHandler.getLocalTracks = jest.fn().mockReturnValue([{ id: "2222" } as MediaStreamTrack]); + trackHandler.getTackById = jest.fn().mockReturnValue([{ id: "2222", kind: "audio" } as MediaStreamTrack]); + statsHandler = new MediaTrackStatsHandler(ssrcHandler, trackHandler); + }); + describe("should find track stats", () => { + it("and returns stats if `trackIdentifier` exists in report", () => { + const report = { trackIdentifier: "123" }; + expect(statsHandler.findTrack2Stats(report, "remote")?.trackId).toEqual("123"); + }); + it("and returns stats if `mid` exists in report", () => { + const reportIn = { mid: "1", type: "inbound-rtp" }; + expect(statsHandler.findTrack2Stats(reportIn, "remote")?.trackId).toEqual("5555"); + const reportOut = { mid: "1", type: "outbound-rtp" }; + expect(statsHandler.findTrack2Stats(reportOut, "local")?.trackId).toEqual("2222"); + }); + it("and returns undefined if `ssrc` exists in report but not on connection", () => { + const report = { ssrc: "142443", type: "inbound-rtp" }; + ssrcHandler.findMidBySsrc = jest.fn().mockReturnValue(undefined); + expect(statsHandler.findTrack2Stats(report, "local")?.trackId).toBeUndefined(); + }); + it("and returns undefined if `ssrc` exists in inbound-rtp report", () => { + const report = { ssrc: "142443", type: "inbound-rtp" }; + ssrcHandler.findMidBySsrc = jest.fn().mockReturnValue("2"); + expect(statsHandler.findTrack2Stats(report, "remote")?.trackId).toEqual("5555"); + }); + it("and returns undefined if `ssrc` exists in outbound-rtp report", () => { + const report = { ssrc: "142443", type: "outbound-rtp" }; + ssrcHandler.findMidBySsrc = jest.fn().mockReturnValue("2"); + expect(statsHandler.findTrack2Stats(report, "local")?.trackId).toEqual("2222"); + }); + it("and returns undefined if needed property not existing", () => { + const report = {}; + expect(statsHandler.findTrack2Stats(report, "remote")?.trackId).toBeUndefined(); + }); + }); + describe("should find local video track stats", () => { + it("and returns stats if `trackIdentifier` exists in report", () => { + const report = { trackIdentifier: "2222" }; + expect(statsHandler.findLocalVideoTrackStats(report)?.trackId).toEqual("2222"); + }); + it("and returns stats if `mid` exists in report", () => { + const report = { mid: "1" }; + expect(statsHandler.findLocalVideoTrackStats(report)?.trackId).toEqual("2222"); + }); + it("and returns undefined if `ssrc` exists", () => { + const report = { ssrc: "142443", type: "outbound-rtp" }; + ssrcHandler.findMidBySsrc = jest.fn().mockReturnValue("2"); + expect(statsHandler.findTrack2Stats(report, "local")?.trackId).toEqual("2222"); + }); + it("and returns undefined if needed property not existing", () => { + const report = {}; + expect(statsHandler.findTrack2Stats(report, "remote")?.trackId).toBeUndefined(); + }); + }); +}); diff --git a/spec/unit/webrtc/stats/statsCollector.spec.ts b/spec/unit/webrtc/stats/statsCollector.spec.ts new file mode 100644 index 00000000000..c7cc0408994 --- /dev/null +++ b/spec/unit/webrtc/stats/statsCollector.spec.ts @@ -0,0 +1,68 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { StatsCollector } from "../../../../src/webrtc/stats/statsCollector"; +import { StatsReportEmitter } from "../../../../src/webrtc/stats/statsReportEmitter"; + +const CALL_ID = "CALL_ID"; +const USER_ID = "USER_ID"; + +describe("StatsCollector", () => { + let collector: StatsCollector; + let rtcSpy: RTCPeerConnection; + let emitter: StatsReportEmitter; + beforeEach(() => { + rtcSpy = { getStats: () => new Promise(() => null) } as RTCPeerConnection; + rtcSpy.addEventListener = jest.fn(); + emitter = new StatsReportEmitter(); + collector = new StatsCollector(CALL_ID, USER_ID, rtcSpy, emitter); + }); + + describe("on process stats", () => { + it("if active calculate stats reports", async () => { + const getStats = jest.spyOn(rtcSpy, "getStats"); + getStats.mockResolvedValue({} as RTCStatsReport); + await collector.processStats("GROUP_CALL_ID", "LOCAL_USER_ID"); + expect(getStats).toHaveBeenCalled(); + }); + + it("if not active do not calculate stats reports", async () => { + collector.setActive(false); + const getStats = jest.spyOn(rtcSpy, "getStats"); + await collector.processStats("GROUP_CALL_ID", "LOCAL_USER_ID"); + expect(getStats).not.toHaveBeenCalled(); + }); + + it("if get reports fails, the collector becomes inactive", async () => { + expect(collector.getActive()).toBeTruthy(); + const getStats = jest.spyOn(rtcSpy, "getStats"); + getStats.mockRejectedValue(new Error("unknown")); + await collector.processStats("GROUP_CALL_ID", "LOCAL_USER_ID"); + expect(getStats).toHaveBeenCalled(); + expect(collector.getActive()).toBeFalsy(); + }); + + it("if active an RTCStatsReport not a promise the collector becomes inactive", async () => { + const getStats = jest.spyOn(rtcSpy, "getStats"); + // @ts-ignore + getStats.mockReturnValue({}); + const actual = await collector.processStats("GROUP_CALL_ID", "LOCAL_USER_ID"); + expect(actual).toBeFalsy(); + expect(getStats).toHaveBeenCalled(); + expect(collector.getActive()).toBeFalsy(); + }); + }); +}); diff --git a/spec/unit/webrtc/stats/statsReportBuilder.spec.ts b/spec/unit/webrtc/stats/statsReportBuilder.spec.ts new file mode 100644 index 00000000000..b1843bbfc9a --- /dev/null +++ b/spec/unit/webrtc/stats/statsReportBuilder.spec.ts @@ -0,0 +1,115 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { TrackID } from "../../../../src/webrtc/stats/statsReport"; +import { MediaTrackStats } from "../../../../src/webrtc/stats/media/mediaTrackStats"; +import { StatsReportBuilder } from "../../../../src/webrtc/stats/statsReportBuilder"; + +describe("StatsReportBuilder", () => { + const LOCAL_VIDEO_TRACK_ID = "LOCAL_VIDEO_TRACK_ID"; + const LOCAL_AUDIO_TRACK_ID = "LOCAL_AUDIO_TRACK_ID"; + const REMOTE_AUDIO_TRACK_ID = "REMOTE_AUDIO_TRACK_ID"; + const REMOTE_VIDEO_TRACK_ID = "REMOTE_VIDEO_TRACK_ID"; + const localAudioTrack = new MediaTrackStats(LOCAL_AUDIO_TRACK_ID, "local", "audio"); + const localVideoTrack = new MediaTrackStats(LOCAL_VIDEO_TRACK_ID, "local", "video"); + const remoteAudioTrack = new MediaTrackStats(REMOTE_AUDIO_TRACK_ID, "remote", "audio"); + const remoteVideoTrack = new MediaTrackStats(REMOTE_VIDEO_TRACK_ID, "remote", "video"); + const stats = new Map([ + [LOCAL_AUDIO_TRACK_ID, localAudioTrack], + [LOCAL_VIDEO_TRACK_ID, localVideoTrack], + [REMOTE_AUDIO_TRACK_ID, remoteAudioTrack], + [REMOTE_VIDEO_TRACK_ID, remoteVideoTrack], + ]); + beforeEach(() => { + buildData(); + }); + + describe("should build stats", () => { + it("by media track stats.", async () => { + expect(StatsReportBuilder.build(stats)).toEqual({ + bitrate: { + audio: { + download: 4000, + upload: 5000, + }, + download: 5004000, + upload: 3005000, + video: { + download: 5000000, + upload: 3000000, + }, + }, + codec: { + local: new Map([ + ["LOCAL_AUDIO_TRACK_ID", "opus"], + ["LOCAL_VIDEO_TRACK_ID", "v8"], + ]), + remote: new Map([ + ["REMOTE_AUDIO_TRACK_ID", "opus"], + ["REMOTE_VIDEO_TRACK_ID", "v9"], + ]), + }, + framerate: { + local: new Map([ + ["LOCAL_AUDIO_TRACK_ID", 0], + ["LOCAL_VIDEO_TRACK_ID", 30], + ]), + remote: new Map([ + ["REMOTE_AUDIO_TRACK_ID", 0], + ["REMOTE_VIDEO_TRACK_ID", 60], + ]), + }, + packetLoss: { + download: 7, + total: 15, + upload: 28, + }, + resolution: { + local: new Map([ + ["LOCAL_AUDIO_TRACK_ID", { height: -1, width: -1 }], + ["LOCAL_VIDEO_TRACK_ID", { height: 460, width: 780 }], + ]), + remote: new Map([ + ["REMOTE_AUDIO_TRACK_ID", { height: -1, width: -1 }], + ["REMOTE_VIDEO_TRACK_ID", { height: 960, width: 1080 }], + ]), + }, + }); + }); + }); + + const buildData = (): void => { + localAudioTrack.setCodec("opus"); + localAudioTrack.setLoss({ packetsTotal: 10, packetsLost: 5, isDownloadStream: false }); + localAudioTrack.setBitrate({ download: 0, upload: 5000 }); + + remoteAudioTrack.setCodec("opus"); + remoteAudioTrack.setLoss({ packetsTotal: 20, packetsLost: 0, isDownloadStream: true }); + remoteAudioTrack.setBitrate({ download: 4000, upload: 0 }); + + localVideoTrack.setCodec("v8"); + localVideoTrack.setLoss({ packetsTotal: 30, packetsLost: 6, isDownloadStream: false }); + localVideoTrack.setBitrate({ download: 0, upload: 3000000 }); + localVideoTrack.setFramerate(30); + localVideoTrack.setResolution({ width: 780, height: 460 }); + + remoteVideoTrack.setCodec("v9"); + remoteVideoTrack.setLoss({ packetsTotal: 40, packetsLost: 4, isDownloadStream: true }); + remoteVideoTrack.setBitrate({ download: 5000000, upload: 0 }); + remoteVideoTrack.setFramerate(60); + remoteVideoTrack.setResolution({ width: 1080, height: 960 }); + }; +}); diff --git a/spec/unit/webrtc/stats/statsReportEmitter.spec.ts b/spec/unit/webrtc/stats/statsReportEmitter.spec.ts new file mode 100644 index 00000000000..de75d44746d --- /dev/null +++ b/spec/unit/webrtc/stats/statsReportEmitter.spec.ts @@ -0,0 +1,48 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { StatsReportEmitter } from "../../../../src/webrtc/stats/statsReportEmitter"; +import { ByteSentStatsReport, ConnectionStatsReport, StatsReport } from "../../../../src/webrtc/stats/statsReport"; + +describe("StatsReportEmitter", () => { + let emitter: StatsReportEmitter; + beforeEach(() => { + emitter = new StatsReportEmitter(); + }); + + it("should emit and receive ByteSendStatsReport", async () => { + const report = {} as ByteSentStatsReport; + return new Promise((resolve, _) => { + emitter.on(StatsReport.BYTE_SENT_STATS, (r) => { + expect(r).toBe(report); + resolve(null); + return; + }); + emitter.emitByteSendReport(report); + }); + }); + + it("should emit and receive ConnectionStatsReport", async () => { + const report = {} as ConnectionStatsReport; + return new Promise((resolve, _) => { + emitter.on(StatsReport.CONNECTION_STATS, (r) => { + expect(r).toBe(report); + resolve(null); + return; + }); + emitter.emitConnectionStatsReport(report); + }); + }); +}); diff --git a/spec/unit/webrtc/stats/statsValueFormatter.spec.ts b/spec/unit/webrtc/stats/statsValueFormatter.spec.ts new file mode 100644 index 00000000000..1ce563e91d6 --- /dev/null +++ b/spec/unit/webrtc/stats/statsValueFormatter.spec.ts @@ -0,0 +1,28 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { StatsValueFormatter } from "../../../../src/webrtc/stats/statsValueFormatter"; + +describe("StatsValueFormatter", () => { + describe("on get non negative values", () => { + it("formatter shod return number", async () => { + expect(StatsValueFormatter.getNonNegativeValue("2")).toEqual(2); + expect(StatsValueFormatter.getNonNegativeValue(0)).toEqual(0); + expect(StatsValueFormatter.getNonNegativeValue("-2")).toEqual(0); + expect(StatsValueFormatter.getNonNegativeValue("")).toEqual(0); + expect(StatsValueFormatter.getNonNegativeValue(NaN)).toEqual(0); + }); + }); +}); diff --git a/spec/unit/webrtc/stats/trackStatsReporter.spec.ts b/spec/unit/webrtc/stats/trackStatsReporter.spec.ts new file mode 100644 index 00000000000..6a1bb5bf21a --- /dev/null +++ b/spec/unit/webrtc/stats/trackStatsReporter.spec.ts @@ -0,0 +1,132 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { TrackStatsReporter } from "../../../../src/webrtc/stats/trackStatsReporter"; +import { MediaTrackStats } from "../../../../src/webrtc/stats/media/mediaTrackStats"; + +describe("TrackStatsReporter", () => { + describe("should on frame and resolution stats", () => { + it("creating empty frame and resolution report, if no data available.", async () => { + const trackStats = new MediaTrackStats("1", "local", "video"); + TrackStatsReporter.buildFramerateResolution(trackStats, {}); + expect(trackStats.getFramerate()).toEqual(0); + expect(trackStats.getResolution()).toEqual({ width: -1, height: -1 }); + }); + it("creating empty frame and resolution report.", async () => { + const trackStats = new MediaTrackStats("1", "remote", "video"); + TrackStatsReporter.buildFramerateResolution(trackStats, { + framesPerSecond: 22.2, + frameHeight: 180, + frameWidth: 360, + }); + expect(trackStats.getFramerate()).toEqual(22); + expect(trackStats.getResolution()).toEqual({ width: 360, height: 180 }); + }); + }); + + describe("should on simulcast", () => { + it("creating simulcast framerate.", async () => { + const trackStats = new MediaTrackStats("1", "local", "video"); + TrackStatsReporter.calculateSimulcastFramerate( + trackStats, + { + framesSent: 100, + timestamp: 1678957001000, + }, + { + framesSent: 10, + timestamp: 1678957000000, + }, + 3, + ); + expect(trackStats.getFramerate()).toEqual(30); + }); + }); + + describe("should on bytes received stats", () => { + it("creating build bitrate received report.", async () => { + const trackStats = new MediaTrackStats("1", "remote", "video"); + TrackStatsReporter.buildBitrateReceived( + trackStats, + { + bytesReceived: 2001000, + timestamp: 1678957010, + }, + { bytesReceived: 2000000, timestamp: 1678957000 }, + ); + expect(trackStats.getBitrate()).toEqual({ download: 800, upload: 0 }); + }); + }); + + describe("should on bytes send stats", () => { + it("creating build bitrate send report.", async () => { + const trackStats = new MediaTrackStats("1", "local", "video"); + TrackStatsReporter.buildBitrateSend( + trackStats, + { + bytesSent: 2001000, + timestamp: 1678957010, + }, + { bytesSent: 2000000, timestamp: 1678957000 }, + ); + expect(trackStats.getBitrate()).toEqual({ download: 0, upload: 800 }); + }); + }); + + describe("should on codec stats", () => { + it("creating build bitrate send report.", async () => { + const trackStats = new MediaTrackStats("1", "remote", "video"); + const remote = {} as RTCStatsReport; + remote.get = jest.fn().mockReturnValue({ mimeType: "video/v8" }); + TrackStatsReporter.buildCodec(remote, trackStats, { codecId: "codecID" }); + expect(trackStats.getCodec()).toEqual("v8"); + }); + }); + + describe("should on package lost stats", () => { + it("creating build package lost on send report.", async () => { + const trackStats = new MediaTrackStats("1", "local", "video"); + TrackStatsReporter.buildPacketsLost( + trackStats, + { + type: "outbound-rtp", + packetsSent: 200, + packetsLost: 120, + }, + { + packetsSent: 100, + packetsLost: 30, + }, + ); + expect(trackStats.getLoss()).toEqual({ packetsTotal: 190, packetsLost: 90, isDownloadStream: false }); + }); + it("creating build package lost on received report.", async () => { + const trackStats = new MediaTrackStats("1", "remote", "video"); + TrackStatsReporter.buildPacketsLost( + trackStats, + { + type: "inbound-rtp", + packetsReceived: 300, + packetsLost: 100, + }, + { + packetsReceived: 100, + packetsLost: 20, + }, + ); + expect(trackStats.getLoss()).toEqual({ packetsTotal: 280, packetsLost: 80, isDownloadStream: true }); + }); + }); +}); diff --git a/spec/unit/webrtc/stats/transportStatsReporter.spec.ts b/spec/unit/webrtc/stats/transportStatsReporter.spec.ts new file mode 100644 index 00000000000..bd3288b15ae --- /dev/null +++ b/spec/unit/webrtc/stats/transportStatsReporter.spec.ts @@ -0,0 +1,126 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { TransportStatsReporter } from "../../../../src/webrtc/stats/transportStatsReporter"; +import { TransportStats } from "../../../../src/webrtc/stats/transportStats"; + +describe("TransportStatsReporter", () => { + describe("should on build report", () => { + const REMOTE_CANDIDATE_ID = "REMOTE_CANDIDATE_ID"; + const LOCAL_CANDIDATE_ID = "LOCAL_CANDIDATE_ID"; + const localIC = { ip: "88.88.99.1", port: 56670, protocol: "tcp", candidateType: "local", networkType: "lan" }; + const remoteIC = { + ip: "123.88.99.1", + port: 46670, + protocol: "udp", + candidateType: "srfx", + networkType: "wifi", + }; + const isFocus = false; + const rtt = 200000; + + it("build new transport stats if all properties there", () => { + const { report, stats } = mockStatsReport(isFocus, 0); + const conferenceStatsTransport: TransportStats[] = []; + const transportStats = TransportStatsReporter.buildReport(report, stats, conferenceStatsTransport, isFocus); + expect(transportStats).toEqual([ + { + ip: `${remoteIC.ip + 0}:${remoteIC.port}`, + type: remoteIC.protocol, + localIp: `${localIC.ip + 0}:${localIC.port}`, + isFocus, + localCandidateType: localIC.candidateType, + remoteCandidateType: remoteIC.candidateType, + networkType: localIC.networkType, + rtt, + }, + ]); + }); + + it("build next transport stats if candidates different", () => { + const mock1 = mockStatsReport(isFocus, 0); + const mock2 = mockStatsReport(isFocus, 1); + let transportStats: TransportStats[] = []; + transportStats = TransportStatsReporter.buildReport(mock1.report, mock1.stats, transportStats, isFocus); + transportStats = TransportStatsReporter.buildReport(mock2.report, mock2.stats, transportStats, isFocus); + expect(transportStats).toEqual([ + { + ip: `${remoteIC.ip + 0}:${remoteIC.port}`, + type: remoteIC.protocol, + localIp: `${localIC.ip + 0}:${localIC.port}`, + isFocus, + localCandidateType: localIC.candidateType, + remoteCandidateType: remoteIC.candidateType, + networkType: localIC.networkType, + rtt, + }, + { + ip: `${remoteIC.ip + 1}:${remoteIC.port}`, + type: remoteIC.protocol, + localIp: `${localIC.ip + 1}:${localIC.port}`, + isFocus, + localCandidateType: localIC.candidateType, + remoteCandidateType: remoteIC.candidateType, + networkType: localIC.networkType, + rtt, + }, + ]); + }); + + it("build not a second transport stats if candidates the same", () => { + const mock1 = mockStatsReport(isFocus, 0); + const mock2 = mockStatsReport(isFocus, 0); + let transportStats: TransportStats[] = []; + transportStats = TransportStatsReporter.buildReport(mock1.report, mock1.stats, transportStats, isFocus); + transportStats = TransportStatsReporter.buildReport(mock2.report, mock2.stats, transportStats, isFocus); + expect(transportStats).toEqual([ + { + ip: `${remoteIC.ip + 0}:${remoteIC.port}`, + type: remoteIC.protocol, + localIp: `${localIC.ip + 0}:${localIC.port}`, + isFocus, + localCandidateType: localIC.candidateType, + remoteCandidateType: remoteIC.candidateType, + networkType: localIC.networkType, + rtt, + }, + ]); + }); + + const mockStatsReport = ( + isFocus: boolean, + prifix: number, + ): { report: RTCStatsReport; stats: RTCIceCandidatePairStats } => { + const report = {} as RTCStatsReport; + report.get = (key: string) => { + if (key === LOCAL_CANDIDATE_ID) { + return { ...localIC, ip: localIC.ip + prifix }; + } + if (key === REMOTE_CANDIDATE_ID) { + return { ...remoteIC, ip: remoteIC.ip + prifix }; + } + // remote + return {}; + }; + const stats = { + remoteCandidateId: REMOTE_CANDIDATE_ID, + localCandidateId: LOCAL_CANDIDATE_ID, + currentRoundTripTime: 200, + } as RTCIceCandidatePairStats; + return { report, stats }; + }; + }); +}); diff --git a/src/@types/event.ts b/src/@types/event.ts index 17af8df0272..4c2a67ce810 100644 --- a/src/@types/event.ts +++ b/src/@types/event.ts @@ -43,6 +43,7 @@ export enum EventType { RoomMessage = "m.room.message", RoomMessageEncrypted = "m.room.encrypted", Sticker = "m.sticker", + CallInvite = "m.call.invite", CallCandidates = "m.call.candidates", CallAnswer = "m.call.answer", @@ -55,6 +56,10 @@ export enum EventType { CallReplaces = "m.call.replaces", CallAssertedIdentity = "m.call.asserted_identity", CallAssertedIdentityPrefix = "org.matrix.call.asserted_identity", + CallTrackSubscription = "m.call.track_subscription", + CallPing = "m.call.ping", + CallPong = "m.call.pong", + KeyVerificationRequest = "m.key.verification.request", KeyVerificationStart = "m.key.verification.start", KeyVerificationCancel = "m.key.verification.cancel", diff --git a/src/@types/global.d.ts b/src/@types/global.d.ts index 749eb7f417b..ec6a1722be6 100644 --- a/src/@types/global.d.ts +++ b/src/@types/global.d.ts @@ -44,29 +44,22 @@ declare global { } interface MediaDevices { - // This is experimental and types don't know about it yet - // https://github.com/microsoft/TypeScript/issues/33232 - getDisplayMedia(constraints: MediaStreamConstraints | DesktopCapturerConstraints): Promise; - getUserMedia(constraints: MediaStreamConstraints | DesktopCapturerConstraints): Promise; + getDisplayMedia(constraints: ExtendedMediaStreamConstraints): Promise; + getUserMedia(constraints: ExtendedMediaStreamConstraints): Promise; } - interface DesktopCapturerConstraints { - audio: - | boolean - | { - mandatory: { - chromeMediaSource: string; - chromeMediaSourceId: string; - }; - }; - video: - | boolean - | { - mandatory: { - chromeMediaSource: string; - chromeMediaSourceId: string; - }; - }; + interface ChromeMediaSourceConstraints { + chromeMediaSource: string; + chromeMediaSourceId: string; + } + + interface ExtendedMediaTrackConstraints extends MediaTrackConstraints { + mandatory?: ChromeMediaSourceConstraints; + } + + interface ExtendedMediaStreamConstraints { + audio: boolean | ExtendedMediaTrackConstraints; + video: boolean | ExtendedMediaTrackConstraints; } interface DummyInterfaceWeShouldntBeUsingThis {} diff --git a/src/client.ts b/src/client.ts index d46116675ee..8cfb449bc55 100644 --- a/src/client.ts +++ b/src/client.ts @@ -366,6 +366,16 @@ export interface ICreateClientOpts { cryptoCallbacks?: ICryptoCallbacks; + /** + * The user ID for the local SFU to use for group calling, if any + */ + localSfuUserId?: string; + + /** + * The device ID for the local SFU to use for group calling, if any + */ + localSfuDeviceId?: string; + /** * Method to generate room names for empty rooms and rooms names based on membership. * Defaults to a built-in English handler with basic pluralisation. @@ -834,6 +844,11 @@ interface ITimestampToEventResponse { origin_server_ts: string; } +export interface IFocusInfo { + user_id: string; + device_id: string; +} + interface IWhoamiResponse { user_id: string; device_id?: string; @@ -1215,6 +1230,8 @@ export class MatrixClient extends TypedEventEmitter>(); + private localSfuUserId?: string; + private localSfuDeviceId?: string; private useE2eForGroupCall = true; private toDeviceMessageQueue: ToDeviceMessageQueue; @@ -1291,6 +1308,9 @@ export class MatrixClient extends TypedEventEmitter; +} + export type CallEventHandlerMap = { [CallEvent.DataChannel]: (channel: RTCDataChannel) => void; [CallEvent.FeedsChanged]: (feeds: CallFeed[]) => void; @@ -300,19 +332,9 @@ export type CallEventHandlerMap = { [CallEvent.AssertedIdentityChanged]: () => void; /* @deprecated */ [CallEvent.HoldUnhold]: (onHold: boolean) => void; - [CallEvent.SendVoipEvent]: (event: Record) => void; + [CallEvent.SendVoipEvent]: (event: VoipEvent) => void; }; -// The key of the transceiver map (purpose + media type, separated by ':') -type TransceiverKey = string; - -// generates keys for the map of transceivers -// kind is unfortunately a string rather than MediaType as this is the type of -// track.kind -function getTransceiverKey(purpose: SDPStreamMetadataPurpose, kind: TransceiverKey): string { - return purpose + ":" + kind; -} - export class MatrixCall extends TypedEventEmitter { public roomId?: string; public callId: string; @@ -321,14 +343,15 @@ export class MatrixCall extends TypedEventEmitter; @@ -339,9 +362,8 @@ export class MatrixCall extends TypedEventEmitter = []; - - // our transceivers for each purpose and type of media - private transceivers = new Map(); + private trackPublications: TrackPublication[] = []; + private feedPublications: FeedPublication[] = []; private inviteOrAnswerSent = false; private waitForLocalAVStream = false; @@ -377,19 +399,25 @@ export class MatrixCall extends TypedEventEmitter(); private remoteAssertedIdentity?: AssertedIdentity; - private remoteSDPStreamMetadata?: SDPStreamMetadata; private callLengthInterval?: ReturnType; private callStartTime?: number; + private dataChannel?: RTCDataChannel; + private opponentDeviceId?: string; private opponentDeviceInfo?: DeviceInfo; private opponentSessionId?: string; public groupCallId?: string; + private subscribeToFocusTimeout?: ReturnType; + + private _opponentSupportsSDPStreamMetadata = false; + // Used to keep the timer for the delay before actually stopping our // video track after muting (see setLocalVideoMuted) private stopVideoTrackTimer?: ReturnType; + private stats: GroupCallStats | undefined; /** * Construct a new Matrix Call. @@ -409,6 +437,7 @@ export class MatrixCall extends TypedEventEmitter track.purpose === SDPStreamMetadataPurpose.Usermedia && track.isAudio, + ); } private get hasUserMediaVideoSender(): boolean { - return Boolean(this.transceivers.get(getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, "video"))?.sender); + return this.trackPublications.some( + ({ track }) => track.purpose === SDPStreamMetadataPurpose.Usermedia && track.isVideo, + ); } - public get localUsermediaFeed(): CallFeed | undefined { + public get localUsermediaFeed(): LocalCallFeed | undefined { return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); } - public get localScreensharingFeed(): CallFeed | undefined { + public get localScreensharingFeed(): LocalCallFeed | undefined { return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); } @@ -533,11 +582,11 @@ export class MatrixCall extends TypedEventEmitter feed.purpose === SDPStreamMetadataPurpose.Usermedia); } - public get remoteScreensharingFeed(): CallFeed | undefined { + public get remoteScreensharingFeed(): RemoteCallFeed | undefined { return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); } @@ -549,8 +598,68 @@ export class MatrixCall extends TypedEventEmitter feed.stream.id === streamId); + private getFreeTransceiverByKind(kind: string): RTCRtpTransceiver | undefined { + return this.peerConn?.getTransceivers().find((transceiver) => { + if (transceiver.sender.track) return false; + if (!transceiver.mid) return false; + if (this.getLocalMediaTypeByMid(transceiver.mid) !== kind) return false; + + return true; + }); + } + + public getLocalMediaLineByMid(mid: string): SessionDescription["media"][number] | undefined { + return this.localSDP?.media?.find((m) => m.mid == mid); + } + + public getLocalMediaTypeByMid(mid: string): string | undefined { + return this.getLocalMediaLineByMid(mid)?.type; + } + + public getLocalMSIDByMid(mid: string): string[] | undefined { + return this.getLocalMediaLineByMid(mid)?.msid?.split(" "); + } + + public getLocalStreamIdByMid(mid: string): string | undefined { + return this.getLocalMSIDByMid(mid)?.[0]; + } + + public getLocalTrackIdByMid(mid: string): string | undefined { + return this.getLocalMSIDByMid(mid)?.[1]; + } + + public getRemoteMediaLineByMid(mid: string): SessionDescription["media"][number] | undefined { + return this.remoteSDP?.media?.find((m) => m.mid == mid); + } + + public getRemoteMediaTypeByMid(mid: string): string | undefined { + return this.getRemoteMediaLineByMid(mid)?.type; + } + + public getRemoteMSIDByMid(mid: string): string[] | undefined { + return this.getRemoteMediaLineByMid(mid)?.msid?.split(" "); + } + + public getRemoteStreamIdByMid(mid: string): string | undefined { + return this.getRemoteMSIDByMid(mid)?.[0]; + } + + public getRemoteTrackIdByMid(mid: string): string | undefined { + return this.getRemoteMSIDByMid(mid)?.[1]; + } + + public getRemoteTrackInfoByMid(mid: string): string { + return `streamId=${this.getRemoteStreamIdByMid(mid)}, trackId=${this.getRemoteTrackIdByMid( + mid, + )}, mid=${mid}, kind=${this.getRemoteMediaTypeByMid(mid)}`; + } + + private getLocalFeedById(feedId: string): LocalCallFeed | undefined { + return this.getLocalFeeds().find((feed) => feed.id === feedId); + } + + private getLocalFeedByStream(stream: MediaStream): LocalCallFeed | undefined { + return this.getLocalFeeds().find((feed) => feed.stream === stream); } /** @@ -565,21 +674,24 @@ export class MatrixCall extends TypedEventEmitter { - return this.feeds.filter((feed) => feed.isLocal()); + public getLocalFeeds(): Array { + return this.feeds.filter((feed) => feed instanceof LocalCallFeed) as LocalCallFeed[]; } /** * Returns an array of all remote CallFeeds * @returns remote CallFeeds */ - public getRemoteFeeds(): Array { - return this.feeds.filter((feed) => !feed.isLocal()); + public getRemoteFeeds(): Array { + return this.feeds.filter((feed) => feed instanceof RemoteCallFeed) as RemoteCallFeed[]; } private async initOpponentCrypto(): Promise { if (!this.opponentDeviceId) return; if (!this.client.getUseE2eForGroupCall()) return; + // We (currently) don't speak e2e with foci. It's debateable whether there would be + // any benefit in doing so. + if (this.isFocus) return; // It's possible to want E2EE and yet not have the means to manage E2EE // ourselves (for example if the client is a RoomWidgetClient) if (!this.client.isCryptoEnabled()) { @@ -605,20 +717,13 @@ export class MatrixCall extends TypedEventEmitter { + if (!publication.streamId) return metadata; - metadata[localFeed.sdpMetadataStreamId] = { - purpose: localFeed.purpose, - audio_muted: localFeed.isAudioMuted(), - video_muted: localFeed.isVideoMuted(), - }; - } - return metadata; + metadata[publication.streamId] = publication.metadata; + return metadata; + }, {}); } /** @@ -627,230 +732,179 @@ export class MatrixCall extends TypedEventEmitter !feed.isLocal()); + return !this.feeds.some((feed) => !feed.isLocal); } - private pushRemoteFeed(stream: MediaStream): void { - // Fallback to old behavior if the other side doesn't support SDPStreamMetadata - if (!this.opponentSupportsSDPStreamMetadata()) { - this.pushRemoteFeedWithoutMetadata(stream); - return; - } - - const userId = this.getOpponentMember()!.userId; - const purpose = this.remoteSDPStreamMetadata![stream.id].purpose; - const audioMuted = this.remoteSDPStreamMetadata![stream.id].audio_muted; - const videoMuted = this.remoteSDPStreamMetadata![stream.id].video_muted; - - if (!purpose) { + private addLocalFeedFromStream( + stream: MediaStream, + purpose: SDPStreamMetadataPurpose, + addToPeerConnection = true, + ): void { + if (this.getLocalFeedByStream(stream)) { logger.warn( - `Call ${this.callId} pushRemoteFeed() ignoring stream because we didn't get any metadata about it (streamId=${stream.id})`, + `Call ${this.callId} addLocalFeedFromStream() ignoring stream for which we already have a feed (streamId=${stream.id})`, ); return; } - if (this.getFeedByStreamId(stream.id)) { - logger.warn( - `Call ${this.callId} pushRemoteFeed() ignoring stream because we already have a feed for it (streamId=${stream.id})`, - ); - return; - } + // Tracks don't always start off enabled, eg. chrome will give a disabled + // audio track if you ask for user media audio and already had one that + // you'd set to disabled (presumably because it clones them internally). + setTracksEnabled(stream.getAudioTracks(), true); + setTracksEnabled(stream.getVideoTracks(), true); - this.feeds.push( - new CallFeed({ + this.pushLocalFeed( + new LocalCallFeed({ client: this.client, - call: this, roomId: this.roomId, - userId, - deviceId: this.getOpponentDeviceId(), stream, purpose, - audioMuted, - videoMuted, }), - ); - - this.emit(CallEvent.FeedsChanged, this.feeds); - - logger.info( - `Call ${this.callId} pushRemoteFeed() pushed stream (streamId=${stream.id}, active=${stream.active}, purpose=${purpose})`, + addToPeerConnection, ); } /** - * This method is used ONLY if the other client doesn't support sending SDPStreamMetadata + * Pushes supplied feed to the call + * @param feed - to push + * @param addToPeerConnection - whether to add the tracks to the peer connection */ - private pushRemoteFeedWithoutMetadata(stream: MediaStream): void { - const userId = this.getOpponentMember()!.userId; - // We can guess the purpose here since the other client can only send one stream - const purpose = SDPStreamMetadataPurpose.Usermedia; - const oldRemoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; - - // Note that we check by ID and always set the remote stream: Chrome appears - // to make new stream objects when transceiver directionality is changed and the 'active' - // status of streams change - Dave - // If we already have a stream, check this stream has the same id - if (oldRemoteStream && stream.id !== oldRemoteStream.id) { - logger.warn( - `Call ${this.callId} pushRemoteFeedWithoutMetadata() ignoring new stream because we already have stream (streamId=${stream.id})`, - ); - return; - } - - if (this.getFeedByStreamId(stream.id)) { - logger.warn( - `Call ${this.callId} pushRemoteFeedWithoutMetadata() ignoring stream because we already have a feed for it (streamId=${stream.id})`, - ); + public pushLocalFeed(feed: LocalCallFeed, addToPeerConnection = true): void { + if (this.getLocalFeedById(feed.id)) { + logger.info(`Call ${this.callId} pushLocalFeed() ignoring duplicate local stream (feedId=${feed.id})`); return; } - this.feeds.push( - new CallFeed({ - client: this.client, - call: this, - roomId: this.roomId, - audioMuted: false, - videoMuted: false, - userId, - deviceId: this.getOpponentDeviceId(), - stream, - purpose, - }), - ); - + this.feeds.push(feed); this.emit(CallEvent.FeedsChanged, this.feeds); + logger.info(`Call ${this.callId} pushLocalFeed() succeeded (id=${feed.id} purpose=${feed.purpose})`); + if (addToPeerConnection) { + this.feedPublications.push(feed.publish(this)); + } + } + + /** + * @internal + */ + public publishTrackOnNewTransceiver(track: LocalCallTrack): RTCRtpTransceiver { + if (!this.peerConn) { + throw new Error("MatrixCall publish() called on call without a peer connection"); + } + if (this.trackPublications.some((publication) => publication.track === track)) { + throw new Error("Cannot publish a track that is already published"); + } logger.info( - `Call ${this.callId} pushRemoteFeedWithoutMetadata() pushed stream (streamId=${stream.id}, active=${stream.active})`, + `Call ${this.callId} publishTrackUsingNewTransceiver() running (id=${track.id}, kind=${track.kind})`, ); - } - private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { - const userId = this.client.getUserId()!; + const { stream, encodings } = track; - // Tracks don't always start off enabled, eg. chrome will give a disabled - // audio track if you ask for user media audio and already had one that - // you'd set to disabled (presumably because it clones them internally). - setTracksEnabled(stream.getAudioTracks(), true); - setTracksEnabled(stream.getVideoTracks(), true); + const transceiver = this.peerConn.addTransceiver(track.track, { + streams: stream ? [stream] : undefined, + // Chrome does not allow us to change the encodings + // later, so we have to use addTransceiver() to set them + // (It's fine to specify the parameter on Firefox too, + // it just won't work.) + sendEncodings: this.isFocus ? track.encodings : undefined, + direction: track.purpose === SDPStreamMetadataPurpose.Usermedia ? "sendrecv" : "sendonly", + }); - if (this.getFeedByStreamId(stream.id)) { - logger.warn( - `Call ${this.callId} pushNewLocalFeed() ignoring stream because we already have a feed for it (streamId=${stream.id})`, - ); - return; + if (this.isFocus && isFirefox()) { + const parameters = transceiver.sender.getParameters(); + transceiver.sender.setParameters({ + ...parameters, + // Firefox does not support the sendEncodings + // parameter on addTransceiver(), so we use + // setParameters() to set them + encodings: encodings, + }); } - this.pushLocalFeed( - new CallFeed({ - client: this.client, - roomId: this.roomId, - audioMuted: false, - videoMuted: false, - userId, - deviceId: this.getOpponentDeviceId(), - stream, - purpose, - }), - addToPeerConnection, - ); + return transceiver; } /** - * Pushes supplied feed to the call - * @param callFeed - to push - * @param addToPeerConnection - whether to add the tracks to the peer connection + * @internal */ - public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void { - if (this.feeds.some((feed) => callFeed.stream.id === feed.stream.id)) { - logger.info( - `Call ${this.callId} pushLocalFeed() ignoring duplicate local stream (streamId=${callFeed.stream.id})`, - ); - return; + public unpublishTrackOnTransceiver(publication: TrackPublication): void { + if (!this.peerConn) { + throw new Error("MatrixCall unpublishTrackOnTransceiver() called on call without a peer connection"); } + logger.info(`Call ${this.callId} unpublishTrackOnTransceiver() running (${publication.logInfo})`); - this.feeds.push(callFeed); + this.peerConn?.removeTrack(publication.transceiver.sender); + } - if (addToPeerConnection) { - for (const track of callFeed.stream.getTracks()) { - logger.info( - `Call ${this.callId} pushLocalFeed() adding track to peer connection (id=${track.id}, kind=${track.kind}, streamId=${callFeed.stream.id}, streamPurpose=${callFeed.purpose}, enabled=${track.enabled})`, - ); + public publishTrack(track: LocalCallTrack): TrackPublication { + if (!this.peerConn) { + throw new Error("MatrixCall publish() called on call without a peer connection"); + } + if (this.trackPublications.some((publication) => publication.track === track)) { + throw new Error("Cannot publish a track that is already published"); + } + logger.info(`Call ${this.callId} publishTrack() running (id=${track.id}, kind=${track.kind})`); - const tKey = getTransceiverKey(callFeed.purpose, track.kind); - if (this.transceivers.has(tKey)) { - // we already have a sender, so we re-use it. We try to re-use transceivers as much - // as possible because they can't be removed once added, so otherwise they just - // accumulate which makes the SDP very large very quickly: in fact it only takes - // about 6 video tracks to exceed the maximum size of an Olm-encrypted - // Matrix event. - const transceiver = this.transceivers.get(tKey)!; - - // this is what would allow us to use addTransceiver(), but it's not available - // on Firefox yet. We call it anyway if we have it. - if (transceiver.sender.setStreams) transceiver.sender.setStreams(callFeed.stream); - - transceiver.sender.replaceTrack(track); - // set the direction to indicate we're going to start sending again - // (this will trigger the re-negotiation) - transceiver.direction = transceiver.direction === "inactive" ? "sendonly" : "sendrecv"; - } else { - // create a new one. We need to use addTrack rather addTransceiver for this because firefox - // doesn't yet implement RTCRTPSender.setStreams() - // (https://bugzilla.mozilla.org/show_bug.cgi?id=1510802) so we'd have no way to group the - // two tracks together into a stream. - const newSender = this.peerConn!.addTrack(track, callFeed.stream); - - // now go & fish for the new transceiver - const newTransceiver = this.peerConn!.getTransceivers().find((t) => t.sender === newSender); - if (newTransceiver) { - this.transceivers.set(tKey, newTransceiver); - } else { - logger.warn( - `Call ${this.callId} pushLocalFeed() didn't find a matching transceiver after adding track!`, - ); - } - } - } + // XXX: We try to re-use transceivers here, but we don't re-use + // transceivers with the SFU: this is to work around + // https://github.com/matrix-org/waterfall/issues/98 - see the bug for + // more. Since we use WebRTC data channels to renegotiate with the SFU, + // we're not limited to the size of a Matrix event, so it's 'ok' if the + // SDP grows indefinitely (although presumably this would break if we + // tried to do an ICE restart over to-device messages after you'd turned + // screen sharing on & off too many times...) + let transceiver = this.getFreeTransceiverByKind(track.kind); + if (!transceiver || (this.isFocus && track.purpose === SDPStreamMetadataPurpose.Screenshare)) { + transceiver = this.publishTrackOnNewTransceiver(track); } - logger.info( - `Call ${this.callId} pushLocalFeed() pushed stream (id=${callFeed.stream.id}, active=${callFeed.stream.active}, purpose=${callFeed.purpose})`, - ); + const publication = new TrackPublication({ + call: this, + track, + transceiver, + }); - this.emit(CallEvent.FeedsChanged, this.feeds); + this.trackPublications.push(publication); + return publication; + } + + public unpublishTrack(publication: TrackPublication): void { + logger.info(`Call ${this.callId} unpublishTrack() running (${publication.logInfo})`); + + this.unpublishTrackOnTransceiver(publication); + this.trackPublications = this.trackPublications.splice(this.trackPublications.indexOf(publication), 1); } /** * Removes local call feed from the call and its tracks from the peer * connection - * @param callFeed - to remove + * @param feed - to remove */ - public removeLocalFeed(callFeed: CallFeed): void { - const audioTransceiverKey = getTransceiverKey(callFeed.purpose, "audio"); - const videoTransceiverKey = getTransceiverKey(callFeed.purpose, "video"); - - for (const transceiverKey of [audioTransceiverKey, videoTransceiverKey]) { - // this is slightly mixing the track and transceiver API but is basically just shorthand. - // There is no way to actually remove a transceiver, so this just sets it to inactive - // (or recvonly) and replaces the source with nothing. - if (this.transceivers.has(transceiverKey)) { - const transceiver = this.transceivers.get(transceiverKey)!; - if (transceiver.sender) this.peerConn!.removeTrack(transceiver.sender); + public removeLocalFeed(feed: LocalCallFeed): void { + feed.unpublish(this); + + if (feed.stream) { + switch (feed.purpose) { + case SDPStreamMetadataPurpose.Usermedia: + this.client.getMediaHandler().stopUserMediaStream(feed.stream); + break; + + case SDPStreamMetadataPurpose.Screenshare: + this.client.getMediaHandler().stopScreensharingStream(feed.stream); + break; } } - if (callFeed.purpose === SDPStreamMetadataPurpose.Screenshare) { - this.client.getMediaHandler().stopScreensharingStream(callFeed.stream); - } - - this.deleteFeed(callFeed); + this.removeFeed(feed); } private deleteAllFeeds(): void { for (const feed of this.feeds) { - if (!feed.isLocal() || !this.groupCallId) { - feed.dispose(); + if (!feed.isLocal) { + feed.removeListener(CallFeedEvent.SizeChanged, this.onCallFeedSizeChanged); + if (!this.groupCallId) { + feed.dispose(); + } } } @@ -859,20 +913,39 @@ export class MatrixCall extends TypedEventEmitter 0) { + throw new Error( + "MatrixCall addRemoteFeed() cannot add multiple remote feeds if opponent does not support sdp_stream_metadata", + ); + } + + this.feeds.push(feed); + feed.addListener(CallFeedEvent.SizeChanged, this.onCallFeedSizeChanged); + + if (emit) { + this.emit(CallEvent.FeedsChanged, this.feeds); + } } - private deleteFeed(feed: CallFeed): void { + private removeFeed(feed: CallFeed, emit = true): void { feed.dispose(); this.feeds.splice(this.feeds.indexOf(feed), 1); - this.emit(CallEvent.FeedsChanged, this.feeds); + feed.removeListener(CallFeedEvent.SizeChanged, this.onCallFeedSizeChanged); + + if (emit) { + this.emit(CallEvent.FeedsChanged, this.feeds); + } } // The typescript definitions have this type as 'any' :( @@ -917,13 +990,20 @@ export class MatrixCall extends TypedEventEmitter !feed.isLocal())?.stream; + const remoteStream = this.feeds.find((feed) => !feed.isLocal)?.stream; // According to previous comments in this file, firefox at some point did not // add streams until media started arriving on them. Testing latest firefox @@ -1033,15 +1113,11 @@ export class MatrixCall extends TypedEventEmitter track.kind === "video"); - - const sender = this.transceivers.get( - getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, "video"), - )?.sender; + const screensharingStream = await this.client.getMediaHandler().getScreensharingStream(opts); + if (!screensharingStream) return false; + const screensharingTrack = screensharingStream.getTracks().find((track) => track.kind === "video"); + if (!screensharingTrack) return false; + const usermediaTrack = this.localUsermediaFeed?.tracks?.find((track) => track.kind === "video"); + if (!usermediaTrack) return false; - sender?.replaceTrack(track ?? null); - - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); + usermediaTrack?.setNewTrack(screensharingTrack); + this.addLocalFeedFromStream(screensharingStream, SDPStreamMetadataPurpose.Screenshare, false); return true; } catch (err) { @@ -1284,12 +1346,13 @@ export class MatrixCall extends TypedEventEmitter track.kind === "video"); - const sender = this.transceivers.get( - getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, "video"), - )?.sender; - sender?.replaceTrack(track ?? null); + const usermediaTrack = this.localUsermediaStream?.getTracks().find((track) => track.kind === "video"); + if (!usermediaTrack) return true; + // We get the screensharing track here from the USERMEDIA feed + const screensharingTrack = this.localUsermediaFeed?.tracks?.find((track) => track.kind === "video"); + if (!screensharingTrack) return true; + screensharingTrack.setNewTrack(usermediaTrack); this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream!); this.deleteFeedByStream(this.localScreensharingStream!); @@ -1299,75 +1362,40 @@ export class MatrixCall extends TypedEventEmitter { - const callFeed = this.localUsermediaFeed!; - const audioEnabled = forceAudio || (!callFeed.isAudioMuted() && !this.remoteOnHold); - const videoEnabled = forceVideo || (!callFeed.isVideoMuted() && !this.remoteOnHold); - logger.log( - `Call ${this.callId} updateLocalUsermediaStream() running (streamId=${stream.id}, audio=${audioEnabled}, video=${videoEnabled})`, - ); - setTracksEnabled(stream.getAudioTracks(), audioEnabled); - setTracksEnabled(stream.getVideoTracks(), videoEnabled); - - // We want to keep the same stream id, so we replace the tracks rather - // than the whole stream. - - // Firstly, we replace the tracks in our localUsermediaStream. - for (const track of this.localUsermediaStream!.getTracks()) { - this.localUsermediaStream!.removeTrack(track); - track.stop(); + const feed = this.localUsermediaFeed; + if (!feed) { + logger.log(`Call ${this.callId} updateLocalUsermediaStream() failed: we do not have localUsermediaFeed`); + return; } - for (const track of stream.getTracks()) { - this.localUsermediaStream!.addTrack(track); + const oldStream = this.localUsermediaStream; + if (!oldStream) { + logger.log(`Call ${this.callId} updateLocalUsermediaStream() failed: we do not have localUsermediaStream`); + return; } - // Then replace the old tracks, if possible. - for (const track of stream.getTracks()) { - const tKey = getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, track.kind); - - const transceiver = this.transceivers.get(tKey); - const oldSender = transceiver?.sender; - let added = false; - if (oldSender) { - try { - logger.info( - `Call ${this.callId} updateLocalUsermediaStream() replacing track (id=${track.id}, kind=${track.kind}, streamId=${stream.id}, streamPurpose=${callFeed.purpose})`, - ); - await oldSender.replaceTrack(track); - // Set the direction to indicate we're going to be sending. - // This is only necessary in the cases where we're upgrading - // the call to video after downgrading it. - transceiver.direction = transceiver.direction === "inactive" ? "sendonly" : "sendrecv"; - added = true; - } catch (error) { - logger.warn( - `Call ${this.callId} updateLocalUsermediaStream() replaceTrack failed: adding new transceiver instead`, - error, - ); - } - } + const audioEnabled = forceAudio || (!feed.isAudioMuted() && !this.remoteOnHold); + const videoEnabled = forceVideo || (!feed.isVideoMuted() && !this.remoteOnHold); + logger.log( + `Call ${this.callId} updateLocalUsermediaStream() running (streamId=${newStream.id}, audio=${audioEnabled}, video=${videoEnabled})`, + ); - if (!added) { - logger.info( - `Call ${this.callId} updateLocalUsermediaStream() adding track to peer connection (id=${track.id}, kind=${track.kind}, streamId=${stream.id}, streamPurpose=${callFeed.purpose})`, - ); + this.localUsermediaFeed?.setNewStream(newStream); + // Firstly, make sure we keep the mute state + setTracksEnabled(newStream.getAudioTracks(), audioEnabled); + setTracksEnabled(newStream.getVideoTracks(), videoEnabled); - const newSender = this.peerConn!.addTrack(track, this.localUsermediaStream!); - const newTransceiver = this.peerConn!.getTransceivers().find((t) => t.sender === newSender); - if (newTransceiver) { - this.transceivers.set(tKey, newTransceiver); - } else { - logger.warn( - `Call ${this.callId} updateLocalUsermediaStream() couldn't find matching transceiver for newly added track!`, - ); - } - } + // Secondly, we replace the tracks in our oldStream with the tracks from + // the newStream + for (const track of oldStream.getTracks()) { + oldStream.removeTrack(track); + track.stop(); } } @@ -1377,7 +1405,7 @@ export class MatrixCall extends TypedEventEmitter { - logger.log(`Call ${this.callId} setLocalVideoMuted() running ${muted}`); + logger.log(`Call ${this.callId} setLocalVideoMuted() running (muted=${muted})`); // if we were still thinking about stopping and removing the video // track: don't, because we want it back. @@ -1538,9 +1566,9 @@ export class MatrixCall extends TypedEventEmitter { - await this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), - }); + if (this.isFocus) { + this.sendFocusEvent(EventType.CallSDPStreamMetadataChanged); + } else { + await this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { + [SDPStreamMetadataKey]: this.metadata, + }); + } } - private gotCallFeedsForInvite(callFeeds: CallFeed[], requestScreenshareFeed = false): void { + private gotCallFeedsForInvite(callFeeds: LocalCallFeed[], requestScreenshareFeed = false): void { if (this.successor) { this.successor.queueGotCallFeedsForAnswer(callFeeds); return; @@ -1587,7 +1619,7 @@ export class MatrixCall extends TypedEventEmitter this.gotCallFeedsForAnswer(callFeeds)); @@ -1699,7 +1731,7 @@ export class MatrixCall extends TypedEventEmitter { + private async gotCallFeedsForAnswer(callFeeds: LocalCallFeed[]): Promise { if (this.callHasEnded()) return; this.waitForLocalAVStream = false; @@ -1848,13 +1880,20 @@ export class MatrixCall extends TypedEventEmitter { const content = event.getContent(); - const description = content.description; + return this.onNegotiateContentReceived(content.description, content[SDPStreamMetadataKey]); + } + + private async onNegotiateContentReceived( + description: RTCSessionDescription, + sdpStreamMetadata: SDPStreamMetadata, + ): Promise { if (!description || !description.sdp || !description.type) { logger.info(`Call ${this.callId} onNegotiateReceived() ignoring invalid m.call.negotiate event`); return; @@ -1932,9 +1977,8 @@ export class MatrixCall extends TypedEventEmitter => { + // TODO: play nice with application layer DC listeners + + let json: FocusEvent; + try { + json = JSON.parse(event.data); + } catch (e) { + logger.warn(`Call ${this.callId} onDataChannelMessage() ignoring non-JSON message:"`, event.data); + return; + } + + logger.info(`Call ${this.callId} onDataChannelMessage() received event (type=${json.type})`, json); + + switch (json.type) { + case EventType.CallNegotiate: + { + const negotiate = json.content as FocusNegotiateEvent; + this.onNegotiateContentReceived(negotiate.description, negotiate[SDPStreamMetadataKeyStable]); + } + break; + case EventType.CallSDPStreamMetadataChanged: + { + const metadata = json.content as FocusSDPStreamMetadataChangedEvent; + this.onMetadata(metadata[SDPStreamMetadataKeyStable]!); + } + break; + case EventType.CallPing: + { + this.sendFocusEvent(EventType.CallPong); + } + break; + default: + logger.warn( + `Call ${this.callId} onDataChannelMessage() received event of unknown type (type=${json.type})`, + ); + + break; + } + }; + + private async waitForDatachannelToBeOpen(): Promise { + if (this.dataChannel?.readyState === "connecting") { + const p = new Promise((resolve) => { + this.dataChannel!.onopen = (): void => resolve(); + this.dataChannel!.onclose = (): void => resolve(); + }); + await p; + } + return; + } + + /** + * Send an m.call.track_subscription event to the focus. The method is + * throttled to avoid spamming the focus with events when scrolling or + * resizing the window. + * The m.call.track_subscription lets the focus know about the visibility + * and size of different tracks. + * @param force - whether or not to force the request to be sent immediately + */ + public subscribeToFocus(force = false): void { + // TODO: Can we throttle this better? Probably using lodash + if (force) { + this.sendSubscriptionFocusEvent(); + return; + } + + clearTimeout(this.subscribeToFocusTimeout); + this.subscribeToFocusTimeout = setTimeout(() => { + this.sendSubscriptionFocusEvent(); + }, SUBSCRIBE_TO_FOCUS_TIMEOUT); + } + + /** + * This method should only ever be called by MatrixCall::subscribeToFocus()! + */ + private sendSubscriptionFocusEvent(): void { + const subscribe: FocusTrackDescription[] = []; + const unsubscribe: FocusTrackDescription[] = []; + for (const { streamId, tracks, isVisible, width, height } of this.getRemoteFeeds()) { + if (!streamId) continue; + if (!tracks) continue; + if (!Object.keys(tracks).length) continue; + + for (const { trackId, isAudio } of tracks) { + if (!trackId) continue; + + const trackDescription: FocusTrackDescription = { + track_id: trackId, + stream_id: streamId, + }; + + if (isAudio) { + // We want audio from everyone + subscribe.push(trackDescription); + } else if (isVisible && width !== 0 && height !== 0) { + // Subscribe to visible videos + trackDescription.width = width; + trackDescription.height = height; + + subscribe.push(trackDescription); + } else { + // Unsubscribe from invisible videos + unsubscribe.push(trackDescription); + } + } + } + + // Return, if there is nothing to do + if (subscribe.length === 0 && unsubscribe.length === 0) return; + + // TODO: Is it ok to keep re-requesting tracks + this.sendFocusEvent(EventType.CallTrackSubscription, { + subscribe, + unsubscribe, + } as FocusTrackSubscriptionEvent); + } + + private onCallFeedSizeChanged = async (): Promise => { + this.subscribeToFocus(); + }; + + /** + * If the opponent + */ + private addRemoteFeedWithoutMetadata(stream: MediaStream): void { + if (this.opponentSupportsSDPStreamMetadata()) { + throw new Error("createRemoteFeedWithoutMetadata() called with sdp_stream_metadata support"); + } + + logger.info(`Call ${this.callId} addRemoteFeedWithoutMetadata() running`); + + this.addRemoteFeed( + new RemoteCallFeed({ + client: this.client, + call: this, + roomId: this.roomId, + stream: stream, + streamId: stream.id, + }), + ); + } + + private onMetadata(metadata: SDPStreamMetadata): void { + this._opponentSupportsSDPStreamMetadata = true; + + let feedsChanged = false; + + // Add new feeds and update existing ones + for (const [streamId, streamMetadata] of Object.entries(metadata)) { + const feed = this.getRemoteFeeds().find((f) => f.streamId === streamId); + if (feed) { + feed.metadata = streamMetadata; + continue; + } + + // We don't emit here and only emit at the end of onMetadata to + // avoid spam + this.addRemoteFeed( + new RemoteCallFeed({ + client: this.client, + call: this, + roomId: this.roomId, + streamId: streamId, + metadata: streamMetadata, + }), + false, + ); + feedsChanged = true; + } + + // Remove old feeds for (const feed of this.getRemoteFeeds()) { - const streamId = feed.stream.id; - const metadata = this.remoteSDPStreamMetadata![streamId]; + if (!Object.keys(metadata).includes(feed.streamId ?? "")) { + this.removeFeed(feed, false); + feedsChanged = true; + } + } - feed.setAudioVideoMuted(metadata?.audio_muted, metadata?.video_muted); - feed.purpose = this.remoteSDPStreamMetadata![streamId]?.purpose; + if (feedsChanged) { + this.emit(CallEvent.FeedsChanged, this.feeds); + } + if (this.isFocus) { + this.subscribeToFocus(); } } public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void { - const content = event.getContent(); - const metadata = content[SDPStreamMetadataKey]; - this.updateRemoteSDPStreamMetadata(metadata); + const metadata = event.getContent()?.[SDPStreamMetadataKey]; + if (metadata) { + this.onMetadata(metadata); + } } public async onAssertedIdentityReceived(event: MatrixEvent): Promise { @@ -2052,6 +2278,7 @@ export class MatrixCall extends TypedEventEmitter (track.metadataMuted = true)); } } }; @@ -2223,34 +2463,62 @@ export class MatrixCall extends TypedEventEmitter { - if (ev.streams.length === 0) { - logger.warn( - `Call ${this.callId} onTrack() called with streamless track streamless (kind=${ev.track.kind})`, - ); + const stream = ev.streams[0]; + const transceiver = ev.transceiver; + + if (!stream) { + logger.warn(`Call ${this.callId} onTrack() called with streamless track (kind=${ev.track.kind})`); + return; + } + if (!transceiver?.mid) { + logger.warn(`Call ${this.callId} onTrack() called with transceiver without an mid (kind=${ev.track.kind})`); + return; + } + if (!this.opponentSupportsSDPStreamMetadata()) { + this.addRemoteFeedWithoutMetadata(stream); + if (!this.removeTrackListeners.has(stream)) { + const onRemoveTrack = (): void => { + if (stream.getTracks().length === 0) { + logger.info(`Call ${this.callId} onTrack() removing track (streamId=${stream.id})`); + this.deleteFeedByStream(stream); + stream.removeEventListener("removetrack", onRemoveTrack); + this.removeTrackListeners.delete(stream); + } + }; + stream.addEventListener("removetrack", onRemoveTrack); + this.removeTrackListeners.set(stream, onRemoveTrack); + } return; } - const stream = ev.streams[0]; - this.pushRemoteFeed(stream); - - if (!this.removeTrackListeners.has(stream)) { - const onRemoveTrack = (): void => { - if (stream.getTracks().length === 0) { - logger.info(`Call ${this.callId} onTrack() removing track (streamId=${stream.id})`); - this.deleteFeedByStream(stream); - stream.removeEventListener("removetrack", onRemoveTrack); - this.removeTrackListeners.delete(stream); - } - }; - stream.addEventListener("removetrack", onRemoveTrack); - this.removeTrackListeners.set(stream, onRemoveTrack); + logger.log( + `Call ${this.callId} onTrack() running (mid=${transceiver.mid}, kind=${transceiver.receiver.track.kind})`, + ); + + const feed = this.getRemoteFeeds().find((feed) => feed.canAddTransceiver(transceiver)); + if (!feed) { + logger.warn( + `Call ${ + this.callId + } onTrack() did not find feed for transceiver (streamId=${this.getRemoteStreamIdByMid( + transceiver.mid, + )}, trackId=${this.getRemoteTrackIdByMid(transceiver.mid)} kind=${transceiver.receiver.track.kind})`, + ); + return; } + feed.addTransceiver(transceiver); }; private onDataChannel = (ev: RTCDataChannelEvent): void => { - this.emit(CallEvent.DataChannel, ev.channel); + this.setupDataChannel(ev.channel); }; + private setupDataChannel(dataChannel: RTCDataChannel): void { + this.dataChannel = dataChannel; + this.dataChannel.addEventListener("message", this.onDataChannelMessage); + this.emit(CallEvent.DataChannel, dataChannel); + } + /** * This method removes all video/rtx codecs from screensharing video * transceivers. This is necessary since they can cause problems. Without @@ -2279,10 +2547,10 @@ export class MatrixCall extends TypedEventEmitter track.purpose === SDPStreamMetadataPurpose.Screenshare && track.isVideo, + )?.transceiver; + if (screensharingVideoTransceiver) screensharingVideoTransceiver.setCodecPreferences(codecs); } private onNegotiationNeeded = async (): Promise => { @@ -2299,7 +2567,7 @@ export class MatrixCall extends TypedEventEmitter { - logger.debug(`Call ${this.callId} onHangupReceived() running`); + logger.debug(`Call ${this.callId} onHangupReceived() running`, msg); // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen // a partner yet but we're treating the hangup as a reject as per VoIP v0) @@ -2366,11 +2634,11 @@ export class MatrixCall extends TypedEventEmitter { + await this.waitForDatachannelToBeOpen(); + if (this.dataChannel?.readyState !== "open") { + logger.error("Failed to send focus event because data-channel is not open"); + return; + } + + const event: FocusEvent = { + type: type, + content: content, + }; + + if ([EventType.CallNegotiate, EventType.CallSDPStreamMetadataChanged].includes(type)) { + event.content[SDPStreamMetadataKeyStable] = this.metadata; + } + + // FIXME: RPC reliability over DC + this.dataChannel!.send(JSON.stringify(event)); + logger.info(`Call ${this.callId} sendFocusEvent() sent event (type=${event.type}):`, event); + } + /** * Queue a candidate to be sent * @param content - The candidate to queue up, or null if candidates have finished being generated @@ -2555,7 +2844,11 @@ export class MatrixCall extends TypedEventEmitter { + public async placeCallWithCallFeeds(callFeeds: LocalCallFeed[], requestScreenshareFeed = false): Promise { this.checkForErrorListener(); this.direction = CallDirection.Outbound; @@ -2752,6 +3042,7 @@ export class MatrixCall extends TypedEventEmitter, enabled: boolean): void { @@ -2878,7 +3174,10 @@ export function supportsMatrixCall(): boolean { export function createNewMatrixCall( client: MatrixClient, roomId: string, - options?: Pick, + options?: Pick< + CallOpts, + "forceTURN" | "invitee" | "opponentDeviceId" | "opponentSessionId" | "groupCallId" | "isFocus" + >, ): MatrixCall | null { if (!supportsMatrixCall()) return null; @@ -2894,6 +3193,7 @@ export function createNewMatrixCall( opponentDeviceId: options?.opponentDeviceId, opponentSessionId: options?.opponentSessionId, groupCallId: options?.groupCallId, + isFocus: options?.isFocus, }; const call = new MatrixCall(opts); diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index f06ed5b0db7..53a08ac1974 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -1,20 +1,35 @@ // allow non-camelcase as these are events type that go onto the wire /* eslint-disable camelcase */ +import { EventType } from "../matrix"; import { CallErrorCode } from "./call"; // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; +export const SDPStreamMetadataKeyStable = "sdp_stream_metadata"; export enum SDPStreamMetadataPurpose { Usermedia = "m.usermedia", Screenshare = "m.screenshare", } +export interface SDPStreamMetadataTrack { + kind: string; + width?: number; + height?: number; +} + +export interface SDPStreamMetadataTracks { + [key: string]: SDPStreamMetadataTrack; +} + export interface SDPStreamMetadataObject { + user_id?: string; + device_id?: string; purpose: SDPStreamMetadataPurpose; - audio_muted: boolean; - video_muted: boolean; + audio_muted?: boolean; + video_muted?: boolean; + tracks?: SDPStreamMetadataTracks; } export interface SDPStreamMetadata { @@ -89,4 +104,38 @@ export interface MCallHangupReject extends MCallBase { reason?: CallErrorCode; } +export interface FocusTrackDescription { + stream_id: string; + track_id: string; + width?: number; + height?: number; +} + +export interface FocusEvent { + type: EventType; + content: FocusEventBaseContent; +} + +export interface FocusEventBaseContent { + [SDPStreamMetadataKeyStable]?: SDPStreamMetadata; +} + +export interface FocusTrackSubscriptionEvent extends FocusEventBaseContent { + subscribe: FocusTrackDescription[]; + unsubscribe: FocusTrackDescription[]; +} + +export interface FocusNegotiateEvent extends FocusEventBaseContent { + description: RTCSessionDescription; + [SDPStreamMetadataKeyStable]: SDPStreamMetadata; +} + +export interface FocusSDPStreamMetadataChangedEvent extends FocusEventBaseContent { + [SDPStreamMetadataKeyStable]: SDPStreamMetadata; +} + +export interface FocusPingEvent extends FocusEventBaseContent {} + +export interface FocusPongEvent extends FocusEventBaseContent {} + /* eslint-enable camelcase */ diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 505cf56f843..37a19ce7fdd 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -1,5 +1,6 @@ /* -Copyright 2021 Šimon Brandner +Copyright 2021 - 2022 Šimon Brandner +Copyright 2021 - 2023 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,9 +19,9 @@ import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { acquireContext, releaseContext } from "./audioContext"; import { MatrixClient } from "../client"; import { RoomMember } from "../models/room-member"; -import { logger } from "../logger"; import { TypedEventEmitter } from "../models/typed-event-emitter"; -import { CallEvent, CallState, MatrixCall } from "./call"; +import { CallTrack } from "./callTrack"; +import { randomString } from "../randomstring"; const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB @@ -29,22 +30,6 @@ const SPEAKING_SAMPLE_COUNT = 8; // samples export interface ICallFeedOpts { client: MatrixClient; roomId?: string; - userId: string; - deviceId: string | undefined; - stream: MediaStream; - purpose: SDPStreamMetadataPurpose; - /** - * Whether or not the remote SDPStreamMetadata says audio is muted - */ - audioMuted: boolean; - /** - * Whether or not the remote SDPStreamMetadata says video is muted - */ - videoMuted: boolean; - /** - * The MatrixCall which is the source of this CallFeed - */ - call?: MatrixCall; } export enum CallFeedEvent { @@ -53,130 +38,196 @@ export enum CallFeedEvent { LocalVolumeChanged = "local_volume_changed", VolumeChanged = "volume_changed", ConnectedChanged = "connected_changed", + SizeChanged = "size_changed", Speaking = "speaking", Disposed = "disposed", } type EventHandlerMap = { - [CallFeedEvent.NewStream]: (stream: MediaStream) => void; + [CallFeedEvent.NewStream]: (stream?: MediaStream) => void; [CallFeedEvent.MuteStateChanged]: (audioMuted: boolean, videoMuted: boolean) => void; [CallFeedEvent.LocalVolumeChanged]: (localVolume: number) => void; [CallFeedEvent.VolumeChanged]: (volume: number) => void; [CallFeedEvent.ConnectedChanged]: (connected: boolean) => void; + [CallFeedEvent.SizeChanged]: () => void; [CallFeedEvent.Speaking]: (speaking: boolean) => void; [CallFeedEvent.Disposed]: () => void; }; -export class CallFeed extends TypedEventEmitter { - public stream: MediaStream; - public sdpMetadataStreamId: string; - public userId: string; - public readonly deviceId: string | undefined; - public purpose: SDPStreamMetadataPurpose; +/** + * CallFeed is a wrapper around a MediaStream. It includes useful information + * such as the userId and deviceId of the stream's sender, mute state, volume + * activity etc. This class would be usually used to display the video tiles in + * the UI. + */ +export abstract class CallFeed extends TypedEventEmitter { + public abstract get id(): string; + public abstract get purpose(): SDPStreamMetadataPurpose; + public abstract get connected(): boolean; + public abstract get userId(): string; + public abstract get deviceId(): string | undefined; + + public abstract isLocal: boolean; + public abstract isRemote: boolean; + public speakingVolumeSamples: number[]; - private client: MatrixClient; - private call?: MatrixCall; - private roomId?: string; - private audioMuted: boolean; - private videoMuted: boolean; + protected readonly _id: string; + protected _tracks: CallTrack[] = []; + protected _stream?: MediaStream; + protected roomId?: string; + protected client: MatrixClient; + private localVolume = 1; private measuringVolumeActivity = false; private audioContext?: AudioContext; private analyser?: AnalyserNode; + private audioSourceNode?: MediaStreamAudioSourceNode; private frequencyBinCount?: Float32Array; private speakingThreshold = SPEAKING_THRESHOLD; private speaking = false; private volumeLooperTimeout?: ReturnType; private _disposed = false; - private _connected = false; + private _width = 0; + private _height = 0; + private _isVisible = false; public constructor(opts: ICallFeedOpts) { super(); + this._id = randomString(32); this.client = opts.client; - this.call = opts.call; this.roomId = opts.roomId; - this.userId = opts.userId; - this.deviceId = opts.deviceId; - this.purpose = opts.purpose; - this.audioMuted = opts.audioMuted; - this.videoMuted = opts.videoMuted; this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); - this.sdpMetadataStreamId = opts.stream.id; - this.updateStream(null, opts.stream); - this.stream = opts.stream; // updateStream does this, but this makes TS happier + this.startMeasuringVolume(); + } - if (this.hasAudioTrack) { - this.initVolumeMeasuring(); - } + public get stream(): MediaStream | undefined { + return this._stream; + } - if (opts.call) { - opts.call.addListener(CallEvent.State, this.onCallState); - this.onCallState(opts.call.state); - } + public get isVisible(): boolean { + return this._isVisible; + } + + public get width(): number | undefined { + return this._width; + } + + public get height(): number | undefined { + return this._height; + } + + public get audioTracks(): CallTrack[] { + return this._tracks.filter((track) => track.isAudio); + } + + public get videoTracks(): CallTrack[] { + return this._tracks.filter((track) => track.isVideo); + } + + public get audioTrack(): CallTrack | undefined { + return this.audioTracks[0]; + } + + public get videoTrack(): CallTrack | undefined { + return this.videoTracks[0]; } - public get connected(): boolean { - // Local feeds are always considered connected - return this.isLocal() || this._connected; + public get audioMuted(): boolean { + return !this.audioTracks.some((track) => !track.muted); } - private set connected(connected: boolean) { - this._connected = connected; - this.emit(CallFeedEvent.ConnectedChanged, this.connected); + public get videoMuted(): boolean { + return !this.videoTracks.some((track) => !track.muted); } private get hasAudioTrack(): boolean { - return this.stream.getAudioTracks().length > 0; + return this.stream ? this.stream.getAudioTracks().length > 0 : false; + } + + protected get tracks(): CallTrack[] { + return [...this._tracks]; } - private updateStream(oldStream: MediaStream | null, newStream: MediaStream): void { + protected updateStream(oldStream?: MediaStream, newStream?: MediaStream): void { if (newStream === oldStream) return; if (oldStream) { - oldStream.removeEventListener("addtrack", this.onAddTrack); - this.measureVolumeActivity(false); + clearTimeout(this.volumeLooperTimeout); } - this.stream = newStream; - newStream.addEventListener("addtrack", this.onAddTrack); + this._stream = newStream; - if (this.hasAudioTrack) { - this.initVolumeMeasuring(); - } else { - this.measureVolumeActivity(false); - } + this.startMeasuringVolume(); this.emit(CallFeedEvent.NewStream, this.stream); } - private initVolumeMeasuring(): void { + /** + * Sets up the volume measuring and/or starts the measuring loop + */ + protected startMeasuringVolume(): void { + if (!this.stream) return; if (!this.hasAudioTrack) return; if (!this.audioContext) this.audioContext = acquireContext(); - this.analyser = this.audioContext.createAnalyser(); - this.analyser.fftSize = 512; - this.analyser.smoothingTimeConstant = 0.1; + // If streams changed, setup the things we need for measuring volume + if (this.audioSourceNode?.mediaStream !== this.stream) { + this.analyser = this.audioContext.createAnalyser(); + this.analyser.fftSize = 512; + this.analyser.smoothingTimeConstant = 0.1; - const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); - mediaStreamAudioSourceNode.connect(this.analyser); + this.audioSourceNode = this.audioContext.createMediaStreamSource(this.stream); + this.audioSourceNode.connect(this.analyser); - this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); - } + this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); + } - private onAddTrack = (): void => { - this.emit(CallFeedEvent.NewStream, this.stream); - }; + // If we should not be measuring volume activity atm, don't start the loop + if (!this.measuringVolumeActivity) return; - private onCallState = (state: CallState): void => { - if (state === CallState.Connected) { - this.connected = true; - } else if (state === CallState.Connecting) { - this.connected = false; - } - }; + const loop = (): void => { + if (!this.analyser || !this.frequencyBinCount) { + clearTimeout(this.volumeLooperTimeout); + this.volumeLooperTimeout = undefined; + return; + } + + this.analyser.getFloatFrequencyData(this.frequencyBinCount); + + let maxVolume = -Infinity; + for (const volume of this.frequencyBinCount!) { + if (volume > maxVolume) { + maxVolume = volume; + } + } + + this.speakingVolumeSamples.shift(); + this.speakingVolumeSamples.push(maxVolume); + + this.emit(CallFeedEvent.VolumeChanged, maxVolume); + + let newSpeaking = false; + + for (const volume of this.speakingVolumeSamples) { + if (volume > this.speakingThreshold) { + newSpeaking = true; + break; + } + } + + if (this.speaking !== newSpeaking) { + this.speaking = newSpeaking; + this.emit(CallFeedEvent.Speaking, this.speaking); + } + + this.volumeLooperTimeout = setTimeout(loop, POLLING_INTERVAL); + }; + + loop(); + } /** * Returns callRoom member @@ -187,78 +238,39 @@ export class CallFeed extends TypedEventEmitter return callRoom?.getMember(this.userId) ?? null; } - /** - * Returns true if CallFeed is local, otherwise returns false - * @returns is local? - */ - public isLocal(): boolean { - return ( - this.userId === this.client.getUserId() && - (this.deviceId === undefined || this.deviceId === this.client.getDeviceId()) - ); - } - /** * Returns true if audio is muted or if there are no audio * tracks, otherwise returns false + * @deprecated use audioMuted instead * @returns is audio muted? */ public isAudioMuted(): boolean { - return this.stream.getAudioTracks().length === 0 || this.audioMuted; + return this.audioMuted; } /** * Returns true video is muted or if there are no video * tracks, otherwise returns false + * @deprecated use videoMuted instead * @returns is video muted? */ public isVideoMuted(): boolean { - // We assume only one video track - return this.stream.getVideoTracks().length === 0 || this.videoMuted; + return this.videoMuted; } public isSpeaking(): boolean { return this.speaking; } - /** - * Replaces the current MediaStream with a new one. - * The stream will be different and new stream as remote parties are - * concerned, but this can be used for convenience locally to set up - * volume listeners automatically on the new stream etc. - * @param newStream - new stream with which to replace the current one - */ - public setNewStream(newStream: MediaStream): void { - this.updateStream(this.stream, newStream); - } - - /** - * Set one or both of feed's internal audio and video video mute state - * Either value may be null to leave it as-is - * @param audioMuted - is the feed's audio muted? - * @param videoMuted - is the feed's video muted? - */ - public setAudioVideoMuted(audioMuted: boolean | null, videoMuted: boolean | null): void { - if (audioMuted !== null) { - if (this.audioMuted !== audioMuted) { - this.speakingVolumeSamples.fill(-Infinity); - } - this.audioMuted = audioMuted; - } - if (videoMuted !== null) this.videoMuted = videoMuted; - this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); - } - /** * Starts emitting volume_changed events where the emitter value is in decibels * @param enabled - emit volume changes */ public measureVolumeActivity(enabled: boolean): void { if (enabled) { - if (!this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; - + clearTimeout(this.volumeLooperTimeout); this.measuringVolumeActivity = true; - this.volumeLooper(); + this.startMeasuringVolume(); } else { this.measuringVolumeActivity = false; this.speakingVolumeSamples.fill(-Infinity); @@ -270,69 +282,8 @@ export class CallFeed extends TypedEventEmitter this.speakingThreshold = threshold; } - private volumeLooper = (): void => { - if (!this.analyser) return; - - if (!this.measuringVolumeActivity) return; - - this.analyser.getFloatFrequencyData(this.frequencyBinCount!); - - let maxVolume = -Infinity; - for (const volume of this.frequencyBinCount!) { - if (volume > maxVolume) { - maxVolume = volume; - } - } - - this.speakingVolumeSamples.shift(); - this.speakingVolumeSamples.push(maxVolume); - - this.emit(CallFeedEvent.VolumeChanged, maxVolume); - - let newSpeaking = false; - - for (const volume of this.speakingVolumeSamples) { - if (volume > this.speakingThreshold) { - newSpeaking = true; - break; - } - } - - if (this.speaking !== newSpeaking) { - this.speaking = newSpeaking; - this.emit(CallFeedEvent.Speaking, this.speaking); - } - - this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); - }; - - public clone(): CallFeed { - const mediaHandler = this.client.getMediaHandler(); - const stream = this.stream.clone(); - logger.log(`CallFeed clone() cloning stream (originalStreamId=${this.stream.id}, newStreamId${stream.id})`); - - if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { - mediaHandler.userMediaStreams.push(stream); - } else { - mediaHandler.screensharingStreams.push(stream); - } - - return new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.userId, - deviceId: this.deviceId, - stream, - purpose: this.purpose, - audioMuted: this.audioMuted, - videoMuted: this.videoMuted, - }); - } - public dispose(): void { clearTimeout(this.volumeLooperTimeout); - this.stream?.removeEventListener("addtrack", this.onAddTrack); - this.call?.removeListener(CallEvent.State, this.onCallState); if (this.audioContext) { this.audioContext = undefined; this.analyser = undefined; @@ -358,4 +309,17 @@ export class CallFeed extends TypedEventEmitter this.localVolume = localVolume; this.emit(CallFeedEvent.LocalVolumeChanged, localVolume); } + + public setResolution(width: number, height: number): void { + this._width = Math.round(width); + this._height = Math.round(height); + + this.emit(CallFeedEvent.SizeChanged); + } + + public setIsVisible(isVisible: boolean): void { + this._isVisible = isVisible; + + this.emit(CallFeedEvent.SizeChanged); + } } diff --git a/src/webrtc/callTrack.ts b/src/webrtc/callTrack.ts new file mode 100644 index 00000000000..8c254931943 --- /dev/null +++ b/src/webrtc/callTrack.ts @@ -0,0 +1,44 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { randomString } from "../randomstring"; + +export interface CallTrackOpts {} + +/** + * CallTrack is a wrapper around MediaStreamTrack. It includes some additional + * useful information such as the mute state. + */ +export abstract class CallTrack { + public abstract get id(): string | undefined; + public abstract get track(): MediaStreamTrack | undefined; + public abstract get kind(): string | undefined; + public abstract get muted(): boolean; + + protected readonly _id: string; + + public constructor(opts: CallTrackOpts) { + this._id = randomString(32); + } + + public get isAudio(): boolean { + return this.kind === "audio"; + } + + public get isVideo(): boolean { + return this.kind === "video"; + } +} diff --git a/src/webrtc/feedPublication.ts b/src/webrtc/feedPublication.ts new file mode 100644 index 00000000000..2a11afeac8f --- /dev/null +++ b/src/webrtc/feedPublication.ts @@ -0,0 +1,73 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { MatrixCall } from "./call"; +import { SDPStreamMetadataObject, SDPStreamMetadataTracks } from "./callEventTypes"; +import { LocalCallFeed } from "./localCallFeed"; +import { TrackPublication } from "./trackPublication"; + +interface CallFeedPublicationOpts { + call: MatrixCall; + feed: LocalCallFeed; +} + +/** + * FeedPublication represents a LocalCallFeed being published to a specific peer + * connection. It stores an array of track publications. This class needs to + * exist, so that we are able elegantly retrieve feed's track publications on a + * given peer connection. + */ +export class FeedPublication { + public readonly call: MatrixCall; + public readonly feed: LocalCallFeed; + private trackPublications: TrackPublication[] = []; + + public constructor(opts: CallFeedPublicationOpts) { + this.call = opts.call; + this.feed = opts.feed; + } + + public get metadata(): SDPStreamMetadataObject { + return { + user_id: this.feed.userId, + device_id: this.feed.deviceId, + purpose: this.feed.purpose, + audio_muted: this.feed.audioMuted, + video_muted: this.feed.videoMuted, + tracks: this.trackPublications.reduce( + (metadata: SDPStreamMetadataTracks, publication: TrackPublication) => { + if (!publication.trackId) return metadata; + + metadata[publication.trackId] = publication.metadata; + return metadata; + }, + {}, + ), + }; + } + + public get streamId(): string | undefined { + return this.trackPublications[0]?.streamId; + } + + public addTrackPublication(publication: TrackPublication): void { + this.trackPublications.push(publication); + } + + public removeTrackPublication(publication: TrackPublication): void { + this.trackPublications.splice(this.trackPublications.indexOf(publication), 1); + } +} diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 4e9971f10ad..c0f74cc4fbd 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1,6 +1,6 @@ import { TypedEventEmitter } from "../models/typed-event-emitter"; import { CallFeed, SPEAKING_THRESHOLD } from "./callFeed"; -import { MatrixClient, IMyDevice } from "../client"; +import { IFocusInfo, MatrixClient, IMyDevice } from "../client"; import { CallErrorCode, CallEvent, @@ -24,6 +24,9 @@ import { CallEventHandlerEvent } from "./callEventHandler"; import { GroupCallEventHandlerEvent } from "./groupCallEventHandler"; import { IScreensharingOpts } from "./mediaHandler"; import { mapsEqual } from "../utils"; +import { LocalCallFeed } from "./localCallFeed"; +import { GroupCallStats } from "./stats/groupCallStats"; +import { ByteSentStatsReport, ConnectionStatsReport, StatsReport } from "./stats/statsReport"; export enum GroupCallIntent { Ring = "m.ring", @@ -83,12 +86,26 @@ export type GroupCallEventHandlerMap = { [GroupCallEvent.Error]: (error: GroupCallError) => void; }; +export enum GroupCallStatsReportEvent { + ConnectionStats = "GroupCall.connection_stats", + ByteSentStats = "GroupCall.byte_sent_stats", +} + +export type GroupCallStatsReportEventHandlerMap = { + [GroupCallStatsReportEvent.ConnectionStats]: (report: GroupCallStatsReport) => void; + [GroupCallStatsReportEvent.ByteSentStats]: (report: GroupCallStatsReport) => void; +}; + export enum GroupCallErrorCode { NoUserMedia = "no_user_media", UnknownDevice = "unknown_device", PlaceCallFailed = "place_call_failed", } +export interface GroupCallStatsReport { + report: T; +} + export class GroupCallError extends Error { public code: string; @@ -137,15 +154,16 @@ export interface IGroupCallRoomMemberFeed { } export interface IGroupCallRoomMemberDevice { - device_id: string; - session_id: string; - expires_ts: number; - feeds: IGroupCallRoomMemberFeed[]; + "device_id": string; + "session_id": string; + "expires_ts": number; + "feeds": IGroupCallRoomMemberFeed[]; + "org.matrix.msc3898.foci.active"?: IFocusInfo[]; + "org.matrix.msc3898.foci.preferred"?: IFocusInfo[]; } export interface IGroupCallRoomMemberCallState { "m.call_id": string; - "m.foci"?: string[]; "m.devices": IGroupCallRoomMemberDevice[]; } @@ -174,14 +192,15 @@ interface ICallHandlers { } const DEVICE_TIMEOUT = 1000 * 60 * 60; // 1 hour +const FOCUS_SESSION_ID = "sfu"; function getCallUserId(call: MatrixCall): string | null { return call.getOpponentMember()?.userId || call.invitee || null; } export class GroupCall extends TypedEventEmitter< - GroupCallEvent | CallEvent, - GroupCallEventHandlerMap & CallEventHandlerMap + GroupCallEvent | CallEvent | GroupCallStatsReportEvent, + GroupCallEventHandlerMap & CallEventHandlerMap & GroupCallStatsReportEventHandlerMap > { // Config public activeSpeakerInterval = 1000; @@ -190,12 +209,13 @@ export class GroupCall extends TypedEventEmitter< public pttMaxTransmitTime = 1000 * 20; public activeSpeaker?: CallFeed; - public localCallFeed?: CallFeed; - public localScreenshareFeed?: CallFeed; + public localCallFeed?: LocalCallFeed; + public localScreenshareFeed?: LocalCallFeed; public localDesktopCapturerSourceId?: string; public readonly userMediaFeeds: CallFeed[] = []; public readonly screenshareFeeds: CallFeed[] = []; public groupCallId: string; + public foci: IFocusInfo[] = []; private readonly calls = new Map>(); // user_id -> device_id -> MatrixCall private callHandlers = new Map>(); // user_id -> device_id -> ICallHandlers @@ -210,6 +230,8 @@ export class GroupCall extends TypedEventEmitter< private initWithVideoMuted = false; private initCallFeedPromise?: Promise; + private readonly stats: GroupCallStats; + public constructor( private client: MatrixClient, public room: Room, @@ -231,8 +253,23 @@ export class GroupCall extends TypedEventEmitter< this.on(GroupCallEvent.ParticipantsChanged, this.onParticipantsChanged); this.on(GroupCallEvent.GroupCallStateChanged, this.onStateChanged); this.on(GroupCallEvent.LocalScreenshareStateChanged, this.onLocalFeedsChanged); + + const userID = this.client.getUserId() || "unknown"; + this.stats = new GroupCallStats(this.groupCallId, userID); + this.stats.reports.on(StatsReport.CONNECTION_STATS, this.onConnectionStats); + this.stats.reports.on(StatsReport.BYTE_SENT_STATS, this.onByteSentStats); } + private onConnectionStats = (report: ConnectionStatsReport): void => { + // @TODO: Implement data argumentation + this.emit(GroupCallStatsReportEvent.ConnectionStats, { report }); + }; + + private onByteSentStats = (report: ByteSentStatsReport): void => { + // @TODO: Implement data argumentation + this.emit(GroupCallStatsReportEvent.ByteSentStats, { report }); + }; + public async create(): Promise { this.creationTs = Date.now(); this.client.groupCallEventHandler!.groupCalls.set(this.room.roomId, this); @@ -332,8 +369,19 @@ export class GroupCall extends TypedEventEmitter< } } - public getLocalFeeds(): CallFeed[] { - const feeds: CallFeed[] = []; + private getPreferredFoci(): IFocusInfo[] { + const preferredFoci = this.client.getFoci(); + const isUsingPreferredFocus = Boolean( + preferredFoci.find((pf) => + this.foci.find((f) => pf.user_id === f.user_id && pf.device_id === pf.device_id), + ), + ); + + return isUsingPreferredFocus ? [] : preferredFoci; + } + + public getLocalFeeds(): LocalCallFeed[] { + const feeds: LocalCallFeed[] = []; if (this.localCallFeed) feeds.push(this.localCallFeed); if (this.localScreenshareFeed) feeds.push(this.localScreenshareFeed); @@ -386,16 +434,16 @@ export class GroupCall extends TypedEventEmitter< throw new Error("Group call disposed while gathering media stream"); } - const callFeed = new CallFeed({ + const callFeed = new LocalCallFeed({ client: this.client, roomId: this.room.roomId, - userId: this.client.getUserId()!, - deviceId: this.client.getDeviceId()!, stream, purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: this.initWithAudioMuted || stream.getAudioTracks().length === 0 || this.isPtt, - videoMuted: this.initWithVideoMuted || stream.getVideoTracks().length === 0, }); + callFeed.setAudioVideoMuted( + this.initWithAudioMuted || stream.getAudioTracks().length === 0 || this.isPtt, + this.initWithVideoMuted || stream.getVideoTracks().length === 0, + ); setTracksEnabled(stream.getAudioTracks(), !callFeed.isAudioMuted()); setTracksEnabled(stream.getVideoTracks(), !callFeed.isVideoMuted()); @@ -409,15 +457,18 @@ export class GroupCall extends TypedEventEmitter< public async updateLocalUsermediaStream(stream: MediaStream): Promise { if (this.localCallFeed) { const oldStream = this.localCallFeed.stream; - this.localCallFeed.setNewStream(stream); - const micShouldBeMuted = this.localCallFeed.isAudioMuted(); - const vidShouldBeMuted = this.localCallFeed.isVideoMuted(); - logger.log( - `GroupCall ${this.groupCallId} updateLocalUsermediaStream() (oldStreamId=${oldStream.id}, newStreamId=${stream.id}, micShouldBeMuted=${micShouldBeMuted}, vidShouldBeMuted=${vidShouldBeMuted})`, - ); + const micShouldBeMuted = this.localCallFeed.audioMuted; + const vidShouldBeMuted = this.localCallFeed.videoMuted; setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); - this.client.getMediaHandler().stopUserMediaStream(oldStream); + this.localCallFeed.setNewStream(stream); + + if (oldStream) { + this.client.getMediaHandler().stopUserMediaStream(oldStream); + logger.log( + `GroupCall ${this.groupCallId} updateLocalUsermediaStream() (oldStreamId=${oldStream.id}, newStreamId=${stream.id}, micShouldBeMuted=${micShouldBeMuted}, vidShouldBeMuted=${vidShouldBeMuted})`, + ); + } } } @@ -428,6 +479,15 @@ export class GroupCall extends TypedEventEmitter< throw new Error(`Cannot enter call in the "${this.state}" state`); } + // TODO: Call preferred foci + + // This needs to be done before we set the state to entered. With the + // state set to entered, we'll start calling other participants full-mesh + // which we don't want, if we have a focus + this.chooseFocus(); + + await this.updateMemberState(); + logger.log(`GroupCall ${this.groupCallId} enter() running`); this.state = GroupCallState.Entered; @@ -444,6 +504,28 @@ export class GroupCall extends TypedEventEmitter< this.activeSpeakerLoopInterval = setInterval(this.onActiveSpeakerLoop, this.activeSpeakerInterval); } + private chooseFocus(): void { + // TODO: Go through all state and find best focus and try to use that + + // Try to find a focus of another user to use + let focusOfAnotherMember: IFocusInfo | undefined; + for (const event of this.getMemberStateEvents()) { + const focus = + event.getContent()?.["m.calls"]?.[0]?.["m.devices"]?.[0]?.[ + "org.matrix.msc3898.foci.active" + ]?.[0]; + if (focus) { + focusOfAnotherMember = focus; + break; + } + } + + const focus = focusOfAnotherMember ?? this.client.getFoci()[0]; + if (focus && !this.foci.some((f) => f.user_id === focus.user_id && f.device_id === focus.device_id)) { + this.foci.push(focus); + } + } + private dispose(): void { if (this.localCallFeed) { this.removeUserMediaFeed(this.localCallFeed); @@ -451,12 +533,17 @@ export class GroupCall extends TypedEventEmitter< } if (this.localScreenshareFeed) { - this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed.stream); + if (this.localScreenshareFeed.stream) { + this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed.stream); + } this.removeScreenshareFeed(this.localScreenshareFeed); this.localScreenshareFeed = undefined; this.localDesktopCapturerSourceId = undefined; } + this.userMediaFeeds.splice(0, this.userMediaFeeds.length); + this.screenshareFeeds.splice(0, this.screenshareFeeds.length); + this.client.getMediaHandler().stopAllStreams(); if (this.transmitTimer !== null) { @@ -522,7 +609,7 @@ export class GroupCall extends TypedEventEmitter< public isLocalVideoMuted(): boolean { if (this.localCallFeed) { - return this.localCallFeed.isVideoMuted(); + return this.localCallFeed.videoMuted; } return true; @@ -530,7 +617,7 @@ export class GroupCall extends TypedEventEmitter< public isMicrophoneMuted(): boolean { if (this.localCallFeed) { - return this.localCallFeed.isAudioMuted(); + return this.localCallFeed.audioMuted; } return true; @@ -582,20 +669,26 @@ export class GroupCall extends TypedEventEmitter< if (this.localCallFeed) { logger.log( - `GroupCall ${this.groupCallId} setMicrophoneMuted() (streamId=${this.localCallFeed.stream.id}, muted=${muted})`, + `GroupCall ${this.groupCallId} setMicrophoneMuted() (feedId=${this.localCallFeed.id}, muted=${muted})`, ); this.localCallFeed.setAudioVideoMuted(muted, null); // I don't believe its actually necessary to enable these tracks: they // are the one on the GroupCall's own CallFeed and are cloned before being // given to any of the actual calls, so these tracks don't actually go // anywhere. Let's do it anyway to avoid confusion. - setTracksEnabled(this.localCallFeed.stream.getAudioTracks(), !muted); + if (this.localCallFeed.stream) { + setTracksEnabled(this.localCallFeed.stream.getAudioTracks(), !muted); + } } else { logger.log(`GroupCall ${this.groupCallId} setMicrophoneMuted() no stream muted (muted=${muted})`); this.initWithAudioMuted = muted; } - this.forEachCall((call) => setTracksEnabled(call.localUsermediaFeed!.stream.getAudioTracks(), !muted)); + this.forEachCall((call) => { + if (call.localUsermediaStream) { + setTracksEnabled(call.localUsermediaStream.getAudioTracks(), !muted); + } + }); this.emit(GroupCallEvent.LocalMuteStateChanged, muted, this.isLocalVideoMuted()); if (!sendUpdatesBefore) await sendUpdates(); @@ -618,13 +711,15 @@ export class GroupCall extends TypedEventEmitter< if (this.localCallFeed) { logger.log( - `GroupCall ${this.groupCallId} setLocalVideoMuted() (stream=${this.localCallFeed.stream.id}, muted=${muted})`, + `GroupCall ${this.groupCallId} setLocalVideoMuted() running (feedId=${this.localCallFeed.id}, muted=${muted})`, ); const stream = await this.client.getMediaHandler().getUserMediaStream(true, !muted); await this.updateLocalUsermediaStream(stream); this.localCallFeed.setAudioVideoMuted(null, muted); - setTracksEnabled(this.localCallFeed.stream.getVideoTracks(), !muted); + if (this.localCallFeed.stream) { + setTracksEnabled(this.localCallFeed.stream.getVideoTracks(), !muted); + } } else { logger.log(`GroupCall ${this.groupCallId} setLocalVideoMuted() no stream muted (muted=${muted})`); this.initWithVideoMuted = muted; @@ -665,15 +760,11 @@ export class GroupCall extends TypedEventEmitter< ); this.localDesktopCapturerSourceId = opts.desktopCapturerSourceId; - this.localScreenshareFeed = new CallFeed({ + this.localScreenshareFeed = new LocalCallFeed({ client: this.client, roomId: this.room.roomId, - userId: this.client.getUserId()!, - deviceId: this.client.getDeviceId()!, stream, purpose: SDPStreamMetadataPurpose.Screenshare, - audioMuted: false, - videoMuted: false, }); this.addScreenshareFeed(this.localScreenshareFeed); @@ -708,7 +799,11 @@ export class GroupCall extends TypedEventEmitter< this.forEachCall((call) => { if (call.localScreensharingFeed) call.removeLocalFeed(call.localScreensharingFeed); }); - this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed!.stream); + if (this.localScreenshareFeed?.stream) { + this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed.stream); + } + // We have to remove the feed manually as MatrixCall has its clone, + // so it won't be removed automatically this.removeScreenshareFeed(this.localScreenshareFeed!); this.localScreenshareFeed = undefined; this.localDesktopCapturerSourceId = undefined; @@ -800,85 +895,126 @@ export class GroupCall extends TypedEventEmitter< private placeOutgoingCalls(): void { let callsChanged = false; - for (const [{ userId }, participantMap] of this.participants) { - const callMap = this.calls.get(userId) ?? new Map(); + const onError = ( + error: Error, + userId: string, + deviceId: string, + newCall: MatrixCall | null, + callMap: Map, + ): void => { + logger.error( + `GroupCall ${this.groupCallId} placeOutgoingCalls() failed to place call (userId=${userId}, device=${deviceId})`, + ); - for (const [deviceId, participant] of participantMap) { - const prevCall = callMap.get(deviceId); + if (error instanceof CallError && error.code === GroupCallErrorCode.UnknownDevice) { + this.emit(GroupCallEvent.Error, error); + } else { + this.emit(GroupCallEvent.Error, new GroupCallError(GroupCallErrorCode.PlaceCallFailed, error.message)); + } - if ( - prevCall?.getOpponentSessionId() !== participant.sessionId && - this.wantsOutgoingCall(userId, deviceId) - ) { - callsChanged = true; + if (newCall !== null) { + this.disposeCall(newCall, CallErrorCode.SignallingFailed); + if (callMap.get(deviceId) === newCall) callMap.delete(deviceId); + } + }; - if (prevCall !== undefined) { - logger.debug( - `GroupCall ${this.groupCallId} placeOutgoingCalls() replacing call (userId=${userId}, deviceId=${deviceId}, callId=${prevCall.callId})`, - ); - this.disposeCall(prevCall, CallErrorCode.NewSession); - } + const replaceSession = ( + userId: string, + deviceId: string, + prevCall: MatrixCall | undefined, + opponentSessionId: string, + opponentIsScreensharing: boolean, + callMap: Map, + ): void => { + callsChanged = true; + + if (prevCall) { + logger.debug( + `GroupCall ${this.groupCallId} placeOutgoingCalls() replacing call (userId=${userId}, deviceId=${deviceId}, callId=${prevCall.callId})`, + ); + this.disposeCall(prevCall, CallErrorCode.NewSession); + } - const newCall = createNewMatrixCall(this.client, this.room.roomId, { - invitee: userId, - opponentDeviceId: deviceId, - opponentSessionId: participant.sessionId, - groupCallId: this.groupCallId, - }); + const newCall = createNewMatrixCall(this.client, this.room.roomId, { + invitee: userId, + opponentDeviceId: deviceId, + opponentSessionId: opponentSessionId, + groupCallId: this.groupCallId, + isFocus: opponentSessionId === FOCUS_SESSION_ID, + }); - if (newCall === null) { - logger.error( - `GroupCall ${this.groupCallId} placeOutgoingCalls() failed to create call (userId=${userId}, device=${deviceId})`, - ); - callMap.delete(deviceId); - } else { - this.initCall(newCall); - callMap.set(deviceId, newCall); - - logger.debug( - `GroupCall ${this.groupCallId} placeOutgoingCalls() placing call (userId=${userId}, deviceId=${deviceId}, sessionId=${participant.sessionId})`, - ); - - newCall - .placeCallWithCallFeeds( - this.getLocalFeeds().map((feed) => feed.clone()), - participant.screensharing, - ) - .then(() => { - if (this.dataChannelsEnabled) { - newCall.createDataChannel("datachannel", this.dataChannelOptions); - } - }) - .catch((e) => { - logger.warn( - `GroupCall ${this.groupCallId} placeOutgoingCalls() failed to place call (userId=${userId})`, - e, - ); - - if (e instanceof CallError && e.code === GroupCallErrorCode.UnknownDevice) { - this.emit(GroupCallEvent.Error, e); - } else { - this.emit( - GroupCallEvent.Error, - new GroupCallError( - GroupCallErrorCode.PlaceCallFailed, - `Failed to place call to ${userId}`, - ), - ); - } - - this.disposeCall(newCall, CallErrorCode.SignallingFailed); - if (callMap.get(deviceId) === newCall) callMap.delete(deviceId); - }); - } - } + if (newCall === null) { + onError(new Error("Failed to create new call"), userId, deviceId, newCall, callMap); + return; } + this.initCall(newCall); + callMap.set(deviceId, newCall); + + logger.debug( + `GroupCall ${this.groupCallId} placeOutgoingCalls() placing call (userId=${userId}, deviceId=${deviceId}, sessionId=${opponentSessionId})`, + ); + + newCall + .placeCallWithCallFeeds( + this.getLocalFeeds().map((feed) => feed.clone()), + opponentIsScreensharing, + ) + .then(() => { + if (this.dataChannelsEnabled || opponentSessionId === FOCUS_SESSION_ID) { + newCall.createDataChannel("datachannel", this.dataChannelOptions); + } + }) + .catch((e) => { + onError(e, userId, deviceId, newCall, callMap); + }); + if (callMap.size > 0) { this.calls.set(userId, callMap); } else { this.calls.delete(userId); } + }; + + if (this.foci.length > 0) { + // We have a focus to call, so we call it + for (const { user_id: userId, device_id: deviceId } of this.foci) { + const callMap = this.calls.get(userId) ?? new Map(); + const prevCall = callMap.get(deviceId); + + if (prevCall && !prevCall.callHasEnded()) { + continue; + } + + callsChanged = true; + replaceSession(userId, deviceId, prevCall, FOCUS_SESSION_ID, false, callMap); + } + } else { + // There is no focus to call, so we connect full-mesh + for (const [{ userId }, participantMap] of this.participants) { + const callMap = this.calls.get(userId) ?? new Map(); + + for (const [deviceId, participant] of participantMap) { + const prevCall = callMap.get(deviceId); + + if ( + prevCall?.getOpponentSessionId() === participant.sessionId || + !this.wantsOutgoingCall(userId, deviceId) + ) { + continue; + } + + callsChanged = true; + replaceSession( + userId, + deviceId, + prevCall, + participant.sessionId, + participant.screensharing, + callMap, + ); + } + } } if (callsChanged) this.emit(GroupCallEvent.CallsChanged, this.calls); @@ -922,6 +1058,21 @@ export class GroupCall extends TypedEventEmitter< } } + for (const { user_id: userId, device_id: deviceId } of this.foci) { + const call = this.calls.get(userId)?.get(deviceId); + let retriesMap = this.retryCallCounts.get(userId); + const retries = retriesMap?.get(deviceId) ?? 0; + + if ((!call || call.callHasEnded()) && retries < 3) { + if (retriesMap === undefined) { + retriesMap = new Map(); + this.retryCallCounts.set(userId, retriesMap); + } + retriesMap.set(deviceId, retries + 1); + needsRetry = true; + } + } + if (needsRetry) this.placeOutgoingCalls(); }; @@ -960,6 +1111,8 @@ export class GroupCall extends TypedEventEmitter< this.reEmitter.reEmit(call, Object.values(CallEvent)); + call.initStats(this.stats); + onCallFeedsChanged(); } @@ -1005,40 +1158,33 @@ export class GroupCall extends TypedEventEmitter< } private onCallFeedsChanged = (call: MatrixCall): void => { - const opponentMemberId = getCallUserId(call); - const opponentDeviceId = call.getOpponentDeviceId()!; - - if (!opponentMemberId) { - throw new Error("Cannot change call feeds without user id"); - } - - const currentUserMediaFeed = this.getUserMediaFeed(opponentMemberId, opponentDeviceId); - const remoteUsermediaFeed = call.remoteUsermediaFeed; - const remoteFeedChanged = remoteUsermediaFeed !== currentUserMediaFeed; - - if (remoteFeedChanged) { - if (!currentUserMediaFeed && remoteUsermediaFeed) { - this.addUserMediaFeed(remoteUsermediaFeed); - } else if (currentUserMediaFeed && remoteUsermediaFeed) { - this.replaceUserMediaFeed(currentUserMediaFeed, remoteUsermediaFeed); - } else if (currentUserMediaFeed && !remoteUsermediaFeed) { - this.removeUserMediaFeed(currentUserMediaFeed); - } - } + // Find replaced feeds + call.getRemoteFeeds().filter((cf) => { + [...this.userMediaFeeds, ...this.screenshareFeeds].forEach((gf) => { + if (gf !== cf && gf.userId === cf.userId && gf.deviceId === cf.deviceId && gf.purpose === cf.purpose) { + if (cf.purpose === SDPStreamMetadataPurpose.Usermedia) this.replaceUserMediaFeed(gf, cf); + else if (cf.purpose === SDPStreamMetadataPurpose.Screenshare) this.replaceScreenshareFeed(gf, cf); + } + }); + }); - const currentScreenshareFeed = this.getScreenshareFeed(opponentMemberId, opponentDeviceId); - const remoteScreensharingFeed = call.remoteScreensharingFeed; - const remoteScreenshareFeedChanged = remoteScreensharingFeed !== currentScreenshareFeed; + // Find removed feeds + [...this.userMediaFeeds, ...this.screenshareFeeds] + .filter((gf) => gf.disposed) + .forEach((feed) => { + if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.removeUserMediaFeed(feed); + else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.removeScreenshareFeed(feed); + }); - if (remoteScreenshareFeedChanged) { - if (!currentScreenshareFeed && remoteScreensharingFeed) { - this.addScreenshareFeed(remoteScreensharingFeed); - } else if (currentScreenshareFeed && remoteScreensharingFeed) { - this.replaceScreenshareFeed(currentScreenshareFeed, remoteScreensharingFeed); - } else if (currentScreenshareFeed && !remoteScreensharingFeed) { - this.removeScreenshareFeed(currentScreenshareFeed); - } - } + // Find new feeds + call.getRemoteFeeds() + .filter((cf) => { + return ![...this.userMediaFeeds, ...this.screenshareFeeds].find((gf) => gf === cf); + }) + .forEach((feed) => { + if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.addUserMediaFeed(feed); + else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.addScreenshareFeed(feed); + }); }; private onCallStateChanged = (call: MatrixCall, state: CallState, _oldState: CallState | undefined): void => { @@ -1054,18 +1200,26 @@ export class GroupCall extends TypedEventEmitter< call.setLocalVideoMuted(videoMuted); } - const opponentUserId = call.getOpponentMember()?.userId; - if (state === CallState.Connected && opponentUserId) { - const retriesMap = this.retryCallCounts.get(opponentUserId); - retriesMap?.delete(call.getOpponentDeviceId()!); - if (retriesMap?.size === 0) this.retryCallCounts.delete(opponentUserId); + if (state === CallState.Connected) { + if (call.isFocus) { + call.subscribeToFocus(true); + } + + const opponentUserId = call.getOpponentMember()?.userId || call.invitee; + if (opponentUserId) { + const retriesMap = this.retryCallCounts.get(opponentUserId); + retriesMap?.delete(call.getOpponentDeviceId()!); + if (retriesMap?.size === 0) this.retryCallCounts.delete(opponentUserId); + } } }; private onCallHangup = (call: MatrixCall): void => { if (call.hangupReason === CallErrorCode.Replaced) return; - const opponentUserId = call.getOpponentMember()?.userId ?? this.room.getMember(call.invitee!)!.userId; + const opponentUserId = call.invitee ?? call.getOpponentMember()?.userId; + if (!opponentUserId) return; + const deviceMap = this.calls.get(opponentUserId); // Sanity check that this call is in fact in the map @@ -1147,7 +1301,7 @@ export class GroupCall extends TypedEventEmitter< let nextActiveSpeaker: CallFeed | undefined = undefined; for (const callFeed of this.userMediaFeeds) { - if (callFeed.isLocal() && this.userMediaFeeds.length > 1) continue; + if (callFeed.isLocal && this.userMediaFeeds.length > 1) continue; const total = callFeed.speakingVolumeSamples.reduce( (acc, volume) => acc + Math.max(volume, SPEAKING_THRESHOLD), @@ -1360,10 +1514,12 @@ export class GroupCall extends TypedEventEmitter< await this.updateDevices((devices) => [ ...devices.filter((d) => d.device_id !== this.client.getDeviceId()!), { - device_id: this.client.getDeviceId()!, - session_id: this.client.getSessionId(), - expires_ts: Date.now() + DEVICE_TIMEOUT, - feeds: this.getLocalFeeds().map((feed) => ({ purpose: feed.purpose })), + "device_id": this.client.getDeviceId()!, + "session_id": this.client.getSessionId(), + "expires_ts": Date.now() + DEVICE_TIMEOUT, + "feeds": this.getLocalFeeds().map((feed) => ({ purpose: feed.purpose })), + "org.matrix.msc3898.foci.active": this.foci, + "org.matrix.msc3898.foci.preferred": this.getPreferredFoci(), // TODO: Add data channels }, ]); diff --git a/src/webrtc/localCallFeed.ts b/src/webrtc/localCallFeed.ts new file mode 100644 index 00000000000..6726710e742 --- /dev/null +++ b/src/webrtc/localCallFeed.ts @@ -0,0 +1,198 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { logger } from "../logger"; +import { MatrixCall } from "./call"; +import { SDPStreamMetadataPurpose } from "./callEventTypes"; +import { CallFeed, CallFeedEvent, ICallFeedOpts } from "./callFeed"; +import { FeedPublication } from "./feedPublication"; +import { LocalCallTrack } from "./localCallTrack"; + +export interface LocalCallFeedOpts extends ICallFeedOpts { + purpose: SDPStreamMetadataPurpose; + stream: MediaStream; +} + +/** + * LocalCallFeed is a wrapper around MediaStream. It represents a stream that + * we've created locally by getting user/display media. N.B. that this is not + * linked to a specific peer connection, a FeedPublication is used for that + * purpose. + */ +export class LocalCallFeed extends CallFeed { + protected _tracks: LocalCallTrack[] = []; + protected publications: FeedPublication[] = []; + private _purpose: SDPStreamMetadataPurpose; + + protected _stream: MediaStream; + + public readonly connected = true; + public readonly isLocal = true; + public readonly isRemote = false; + + public constructor(opts: LocalCallFeedOpts) { + super(opts); + + this._purpose = opts.purpose; + + this.updateStream(undefined, opts.stream); + // updateStream() already did the job, but this shuts up typescript from + // complaining about it not being set in the constructor + this._stream = opts.stream; + } + + public get id(): string { + return this._id; + } + + public get tracks(): LocalCallTrack[] { + return super.tracks as LocalCallTrack[]; + } + + public get audioTracks(): LocalCallTrack[] { + return super.audioTracks as LocalCallTrack[]; + } + + public get videoTracks(): LocalCallTrack[] { + return super.videoTracks as LocalCallTrack[]; + } + + public get purpose(): SDPStreamMetadataPurpose { + return this._purpose; + } + + public get userId(): string { + return this.client.getUserId()!; + } + + public get deviceId(): string | undefined { + return this.client.getDeviceId() ?? undefined; + } + + /** + * Set one or both of feed's internal audio and video video mute state + * Either value may be null to leave it as-is + * @param audioMuted - is the feed's audio muted? + * @param videoMuted - is the feed's video muted? + */ + public setAudioVideoMuted(audioMuted: boolean | null, videoMuted: boolean | null): void { + logger.log(`CallFeed ${this.id} setAudioVideoMuted() running (audio=${audioMuted}, video=${videoMuted})`); + + if (audioMuted !== null) { + if (this.audioMuted !== audioMuted) { + this.speakingVolumeSamples.fill(-Infinity); + } + this.audioTracks.forEach((track) => (track.muted = audioMuted)); + } + if (videoMuted !== null) { + this.videoTracks.forEach((track) => (track.muted = videoMuted)); + } + + this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); + } + + public clone(): LocalCallFeed { + const mediaHandler = this.client.getMediaHandler(); + const stream = this._stream.clone(); + logger.log( + `CallFeed ${this.id} clone() cloning stream (originalStreamId=${this._stream.id}, newStreamId=${stream.id})`, + ); + + if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { + mediaHandler.userMediaStreams.push(stream); + } else if (this.purpose === SDPStreamMetadataPurpose.Screenshare) { + mediaHandler.screensharingStreams.push(stream); + } + + const feed = new LocalCallFeed({ + client: this.client, + roomId: this.roomId, + stream, + purpose: this.purpose, + }); + feed.setAudioVideoMuted(this.audioMuted, this.videoMuted); + return feed; + } + + public setNewStream(newStream: MediaStream): void { + this.updateStream(this.stream, newStream); + } + + protected updateStream(oldStream?: MediaStream, newStream?: MediaStream): void { + super.updateStream(oldStream, newStream); + + // First, remove tracks which won't be used anymore + for (const track of this._tracks) { + if (!newStream?.getTracks().some((streamTrack) => streamTrack.kind === track.kind)) { + this._tracks.splice(this._tracks.indexOf(track), 1); + this.publications.forEach((publication) => this.unpublishTrack(track, publication)); + } + } + + if (!newStream) return; + + // Then, replace old track where we can and add new tracks + for (const streamTrack of newStream.getTracks()) { + let track = this._tracks.find((track) => track.kind === streamTrack.kind); + if (track) { + track.setNewTrack(streamTrack); + continue; + } + + track = new LocalCallTrack({ + feed: this, + track: streamTrack, + }); + this._tracks.push(track); + this.publications.forEach((publication) => this.publishTrack(track!, publication)); + } + } + + public publish(call: MatrixCall): FeedPublication { + if (this.publications.some((publication) => publication.call === call)) { + throw new Error("Cannot publish a feed that is already published"); + } + + const feedPublication = new FeedPublication({ + feed: this, + call, + }); + this.tracks.forEach((track) => this.publishTrack(track, feedPublication)); + + this.publications.push(feedPublication); + return feedPublication; + } + + public unpublish(call: MatrixCall): void { + const feedPublication = this.publications.find((publication) => publication.call === call); + if (!feedPublication) return; + + this.publications.splice(this.publications.indexOf(feedPublication), 1); + this.tracks.forEach((track) => this.unpublishTrack(track, feedPublication)); + } + + private publishTrack(track: LocalCallTrack, feedPublication: FeedPublication): void { + const trackPublication = track.publish(feedPublication.call); + if (!trackPublication) return; + feedPublication.addTrackPublication(trackPublication); + } + + private unpublishTrack(track: LocalCallTrack, feedPublication: FeedPublication): void { + const trackPublication = track.unpublish(feedPublication.call); + if (!trackPublication) return; + feedPublication.removeTrackPublication(trackPublication); + } +} diff --git a/src/webrtc/localCallTrack.ts b/src/webrtc/localCallTrack.ts new file mode 100644 index 00000000000..2a97d36b094 --- /dev/null +++ b/src/webrtc/localCallTrack.ts @@ -0,0 +1,205 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { logger } from "../logger"; +import { MatrixCall } from "./call"; +import { SDPStreamMetadataPurpose } from "./callEventTypes"; +import { CallTrack, CallTrackOpts } from "./callTrack"; +import { LocalCallFeed } from "./localCallFeed"; +import { TrackPublication } from "./trackPublication"; + +export enum SimulcastResolution { + Full = "f", + Half = "h", + Quarter = "q", +} + +// Order is important here: some browsers (e.g. +// Chrome) will only send some of the encodings, if +// the track has a resolution to low for it to send +// all, in that case the encoding higher in the list +// has priority and therefore we put full as first +// as we always want to send the full resolution +const SIMULCAST_USERMEDIA_ENCODINGS: RTCRtpEncodingParameters[] = [ + { + // 720p (base) + maxFramerate: 30, + maxBitrate: 1_700_000, + rid: SimulcastResolution.Full, + }, + { + // 360p + maxFramerate: 20, + maxBitrate: 300_000, + rid: SimulcastResolution.Half, + scaleResolutionDownBy: 2.0, + }, + { + // 180p + maxFramerate: 15, + maxBitrate: 120_000, + + rid: SimulcastResolution.Quarter, + scaleResolutionDownBy: 4.0, + }, +]; + +const SIMULCAST_SCREENSHARING_ENCODINGS: RTCRtpEncodingParameters[] = [ + { + // 1080p (base) + maxFramerate: 30, + maxBitrate: 3_000_000, + rid: SimulcastResolution.Full, + }, + { + // 720p + maxFramerate: 15, + maxBitrate: 1_000_000, + rid: SimulcastResolution.Half, + scaleResolutionDownBy: 1.5, + }, + { + // 360p + maxFramerate: 3, + maxBitrate: 200_000, + rid: SimulcastResolution.Quarter, + scaleResolutionDownBy: 3, + }, +]; + +export const getSimulcastEncodings = (purpose: SDPStreamMetadataPurpose): RTCRtpEncodingParameters[] => { + if (purpose === SDPStreamMetadataPurpose.Usermedia) { + return SIMULCAST_USERMEDIA_ENCODINGS; + } + if (purpose === SDPStreamMetadataPurpose.Screenshare) { + return SIMULCAST_SCREENSHARING_ENCODINGS; + } + + // Fallback to usermedia encodings + return SIMULCAST_USERMEDIA_ENCODINGS; +}; + +export interface LocalCallTrackOpts extends CallTrackOpts { + feed: LocalCallFeed; + track: MediaStreamTrack; +} + +/** + * LocalCallTrack is a wrapper around a MediaStream. It represents a track of a + * stream which we retrieved using get user/display media. N.B. that this is not + * linked to a specific peer connection, a TrackPublication is used for that + * purpose. + */ +export class LocalCallTrack extends CallTrack { + private _track: MediaStreamTrack; + private feed: LocalCallFeed; + private publications: TrackPublication[] = []; + + public constructor(opts: LocalCallTrackOpts) { + super(opts); + + this._track = opts.track; + this.feed = opts.feed; + } + + private get logInfo(): string { + return `kind=${this.kind}`; + } + + public get id(): string | undefined { + return this._id; + } + + public get track(): MediaStreamTrack { + return this._track; + } + + public get kind(): string { + return this.track.kind; + } + + public get muted(): boolean { + return !this.track.enabled; + } + + public set muted(muted: boolean) { + this.track.enabled = !muted; + } + + public get purpose(): SDPStreamMetadataPurpose { + return this.feed.purpose; + } + + public get stream(): MediaStream | undefined { + return this.feed.stream; + } + + public get encodings(): RTCRtpEncodingParameters[] { + return getSimulcastEncodings(this.purpose); + } + + public publish(call: MatrixCall): TrackPublication | undefined { + if (this.publications.some((publication) => publication.call === call)) { + throw new Error("Cannot publish a track that is already published"); + } + + try { + const publication = call.publishTrack(this); + this.publications.push(publication); + return publication; + } catch (error) { + logger.error( + `LocalCallTrack ${this.id} publish() failed to publish track to call (callId=${call.callId}):`, + error, + ); + } + } + + public unpublish(call: MatrixCall): TrackPublication | undefined { + const publication = this.publications.find((publication) => publication.call === call); + if (!publication) return; + + try { + publication?.unpublish(); + this.publications.splice(this.publications.indexOf(publication), 1); + return publication; + } catch (error) { + logger.error( + `LocalCallTrack ${this.id} unpublish() failed to unpublish track to call (callId=${publication.call.callId})`, + error, + ); + } + } + + public setNewTrack(track: MediaStreamTrack): void { + logger.log(`LocalCallTrack ${this.id} setNewTrack() running (${this.logInfo})`); + this._track = track; + + for (const publication of this.publications) { + try { + publication.updateSenderTrack(); + logger.log( + `LocalCallTrack ${this.id} setNewTrack() updated published track (callId=${publication.call.callId}, ${this.logInfo})`, + ); + } catch (error) { + logger.log( + `LocalCallTrack ${this.id} setNewTrack() failed to update published track (callId=${publication.call.callId}, ${this.logInfo})`, + error, + ); + } + } + } +} diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index b472b7a4b3d..537d66b0bde 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -22,6 +22,22 @@ import { GroupCallType, GroupCallState } from "../webrtc/groupCall"; import { logger } from "../logger"; import { MatrixClient } from "../client"; +const isWebkit = (): boolean => Boolean(navigator.webkitGetUserMedia); +// Chrome will give it only if we ask exactly, FF refuses entirely if we ask +// exactly, so have to ask for ideal instead XXX: Is this still true? +const getIdealOrExact = (value: number): ConstrainULong => (isWebkit() ? { exact: value } : { ideal: value }); +const getIdealAndMax = (value: number): ConstrainULong => ({ ideal: value, max: value }); +const getDimensionConstraints = ( + func: (value: number) => ConstrainULong, + width: number, + height: number, + framerate: number, +): Pick => ({ + width: func(width), + height: func(height), + frameRate: func(framerate), +}); + export enum MediaHandlerEvent { LocalStreamsChanged = "local_streams_changed", } @@ -411,8 +427,6 @@ export class MediaHandler extends TypedEventEmitter< } private getUserMediaContraints(audio: boolean, video: boolean): MediaStreamConstraints { - const isWebkit = !!navigator.webkitGetUserMedia; - return { audio: audio ? { @@ -424,36 +438,27 @@ export class MediaHandler extends TypedEventEmitter< : false, video: video ? { + ...getDimensionConstraints(getIdealOrExact, 1280, 720, 30), deviceId: this.videoInput ? { ideal: this.videoInput } : undefined, - /* We want 640x360. Chrome will give it only if we ask exactly, - FF refuses entirely if we ask exactly, so have to ask for ideal - instead - XXX: Is this still true? - */ - width: isWebkit ? { exact: 640 } : { ideal: 640 }, - height: isWebkit ? { exact: 360 } : { ideal: 360 }, } : false, }; } - private getScreenshareContraints(opts: IScreensharingOpts): DesktopCapturerConstraints { + private getScreenshareContraints(opts: IScreensharingOpts): ExtendedMediaStreamConstraints { const { desktopCapturerSourceId, audio } = opts; - if (desktopCapturerSourceId) { - return { - audio: audio ?? false, - video: { - mandatory: { - chromeMediaSource: "desktop", - chromeMediaSourceId: desktopCapturerSourceId, - }, - }, - }; - } else { - return { - audio: audio ?? false, - video: true, - }; - } + + return { + audio: audio ?? false, + video: { + ...getDimensionConstraints(getIdealAndMax, 1920, 1080, 30), + mandatory: desktopCapturerSourceId + ? { + chromeMediaSource: "desktop", + chromeMediaSourceId: desktopCapturerSourceId, + } + : undefined, + }, + }; } } diff --git a/src/webrtc/remoteCallFeed.ts b/src/webrtc/remoteCallFeed.ts new file mode 100644 index 00000000000..316f76e9162 --- /dev/null +++ b/src/webrtc/remoteCallFeed.ts @@ -0,0 +1,252 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { logger } from "../logger"; +import { CallEvent, CallState, MatrixCall } from "./call"; +import { SDPStreamMetadataObject, SDPStreamMetadataPurpose } from "./callEventTypes"; +import { CallFeed, CallFeedEvent, ICallFeedOpts } from "./callFeed"; +import { RemoteCallTrack } from "./remoteCallTrack"; + +export interface RemoteCallFeedOpts extends ICallFeedOpts { + streamId: string; + metadata?: SDPStreamMetadataObject; + call: MatrixCall; + + /** + * @deprecated addTransceiver() should be used instead + */ + stream?: MediaStream; +} + +/** + * RemoteCallFeed is a wrapper around MediaStream. It represents an incoming + * stream. + */ +export class RemoteCallFeed extends CallFeed { + private _connected = false; + private _metadata?: SDPStreamMetadataObject; + + protected _tracks: RemoteCallTrack[] = []; + protected call: MatrixCall; + protected _stream: MediaStream; + + public readonly streamId: string; + public readonly isLocal = false; + public readonly isRemote = true; + + public constructor(opts: RemoteCallFeedOpts) { + super(opts); + + if (!opts.metadata && opts.call.opponentSupportsSDPStreamMetadata()) { + throw new Error( + "Cannot create RemoteCallFeed without metadata if the opponents supports sending sdp_stream_metadata", + ); + } + + this.streamId = opts.streamId; + this.call = opts.call; + this.metadata = opts.metadata; + + this._stream = opts.stream || new window.MediaStream(); + + if (opts.call) { + opts.call.addListener(CallEvent.State, this.onCallState); + } + this.updateConnected(); + } + + public get id(): string { + return this.streamId; + } + + public get metadata(): SDPStreamMetadataObject | undefined { + return this._metadata; + } + + public set metadata(metadata: SDPStreamMetadataObject | undefined) { + if (!metadata) return; + + this._metadata = metadata; + + this.audioTracks.forEach((track) => (track.metadataMuted = metadata.audio_muted ?? false)); + this.videoTracks.forEach((track) => (track.metadataMuted = metadata.video_muted ?? false)); + + if (!metadata.tracks) return; + for (const [metadataTrackId, metadataTrack] of Object.entries(metadata.tracks)) { + const track = this._tracks.find((track) => track.trackId === metadataTrackId); + if (track) { + track.metadata = metadataTrack; + continue; + } + + logger.info( + `RemoteCallFeed ${this.id} set metadata() adding track (streamId=${this.streamId} trackId=${metadataTrackId}, kind=${metadataTrack.kind})`, + ); + this._tracks.push( + new RemoteCallTrack({ + call: this.call, + trackId: metadataTrackId, + metadataMuted: + (metadataTrack.kind === "audio" ? metadata.audio_muted : metadata.video_muted) ?? false, + metadata: metadataTrack, + }), + ); + } + + for (const track of this._tracks) { + if (!track.trackId) continue; + if (!Object.keys(metadata.tracks).includes(track.trackId)) { + logger.info( + `RemoteCallFeed ${this.id} set metadata() removing track (streamId=${this.streamId} trackId=${track.trackId}, kind=${track.kind})`, + ); + this._tracks.splice(this._tracks.indexOf(track), 1); + if (track.track) { + this.stream?.removeTrack(track.track); + } + } + } + + this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); + } + + public get purpose(): SDPStreamMetadataPurpose { + // If the opponent did not send a purpose, they probably don't support + // sdp_stream_metadata, so we can assume they're only sending usermedia + return this._metadata?.purpose ?? SDPStreamMetadataPurpose.Usermedia; + } + + public get userId(): string { + const metadataUserId = this._metadata?.user_id; + return this.call.isFocus && metadataUserId + ? metadataUserId + : (this.call.invitee ?? this.call.getOpponentMember()?.userId)!; + } + + public get deviceId(): string | undefined { + return this.call.isFocus ? this._metadata?.device_id : this.call.getOpponentDeviceId(); + } + + public get tracks(): RemoteCallTrack[] { + return [...this._tracks]; + } + + public get audioTracks(): RemoteCallTrack[] { + return super.audioTracks as RemoteCallTrack[]; + } + + public get videoTracks(): RemoteCallTrack[] { + return super.videoTracks as RemoteCallTrack[]; + } + + public get connected(): boolean { + return this._connected; + } + + private set connected(connected: boolean) { + if (this._connected === connected) return; + this._connected = connected; + this.emit(CallFeedEvent.ConnectedChanged, this.connected); + } + + private onCallState = (): void => { + this.updateConnected(); + }; + + private updateConnected(): void { + if (this.call?.state === CallState.Connecting) { + this.connected = false; + } else if (!this.stream) { + this.connected = false; + } else if (this.stream.getTracks().length === 0) { + this.connected = false; + } else if (this.call?.state === CallState.Connected) { + this.connected = true; + } + } + + private streamIdMatches(transceiver: RTCRtpTransceiver): boolean { + if (!transceiver.mid) return false; + if (this.streamId !== this.call.getRemoteStreamIdByMid(transceiver.mid)) return false; + + return true; + } + + public canAddTransceiver(transceiver: RTCRtpTransceiver): boolean { + if (!transceiver.mid) return false; + + // If the opponent does not support sdp_stream_metadata at all, we + // always allow adding transceivers + if (!this._metadata) return true; + // If the opponent does not support tracks on sdp_stream_metadata, we + // just check the streamId + if (!this._metadata.tracks && this.streamIdMatches(transceiver)) return true; + + if (!this._tracks.some((track) => track.canSetTransceiver(transceiver))) return false; + if (!this.streamIdMatches(transceiver)) return false; + + return true; + } + + public addTransceiver(transceiver: RTCRtpTransceiver): void { + if (!transceiver.mid) { + throw new Error("RemoteCallFeed addTransceiver() called with transceiver without an mid"); + } + if (!transceiver.receiver?.track) { + throw new Error("RemoteCallFeed addTransceiver() called with transceiver without a receiver or track"); + } + if (!this.canAddTransceiver(transceiver)) { + throw new Error("RemoteCallFeed addTransceiver() called with wrong trackId or streamId"); + } + + const track = this._tracks.find((t) => t.canSetTransceiver(transceiver)); + const trackId = this.call.getRemoteTrackIdByMid(transceiver.mid); + + const trackInfo = `streamId=${this.streamId}, trackId=${trackId}, kind=${transceiver.receiver.track.kind}`; + logger.log(`RemoteCallFeed ${this.id} addTransceiver() running (${trackInfo})`); + + if (!track && !this._metadata?.tracks) { + // If the opponent does not support tracks on sdp_stream_metadata or + // it does not support sdp_stream_metadata at all, we simply create + // new tracks + logger.info(`RemoteCallFeed ${this.id} addTransceiver() adding track (${trackInfo})`); + const track = new RemoteCallTrack({ + call: this.call, + metadataMuted: + (transceiver.receiver.track.kind === "audio" + ? this._metadata?.audio_muted + : this._metadata?.video_muted) ?? false, + trackId, + }); + track.setTransceiver(transceiver); + this._tracks.push(track); + } else if (!track) { + logger.warn(`RemoteCallFeed ${this.id} addTransceiver() did not find track for transceiver (${trackInfo})`); + return; + } else { + track.setTransceiver(transceiver); + } + + this.stream?.addTrack(transceiver.receiver.track); + this.startMeasuringVolume(); + this.updateConnected(); + this.emit(CallFeedEvent.NewStream, this.stream); + } + + public dispose(): void { + super.dispose(); + this.call?.removeListener(CallEvent.State, this.onCallState); + } +} diff --git a/src/webrtc/remoteCallTrack.ts b/src/webrtc/remoteCallTrack.ts new file mode 100644 index 00000000000..39b16ad4aa7 --- /dev/null +++ b/src/webrtc/remoteCallTrack.ts @@ -0,0 +1,112 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { logger } from "../logger"; +import { MatrixCall } from "./call"; +import { SDPStreamMetadataTrack } from "./callEventTypes"; +import { CallTrack, CallTrackOpts } from "./callTrack"; + +export interface RemoteCallTrackOpts extends CallTrackOpts { + call: MatrixCall; + trackId?: string; + metadata?: SDPStreamMetadataTrack; + metadataMuted?: boolean; +} + +/** + * RemoteCallTrack is a wrapper around MediaStreamTrack. It represent an + * incoming track. + */ +export class RemoteCallTrack extends CallTrack { + private readonly _trackId?: string; + private _metadata?: SDPStreamMetadataTrack; + private _metadataMuted?: boolean; + private _transceiver?: RTCRtpTransceiver; + private call: MatrixCall; + + public constructor(opts: RemoteCallTrackOpts) { + super(opts); + + this.call = opts.call; + this._trackId = opts.trackId; + this.metadata = opts.metadata; + this.metadataMuted = opts.metadataMuted; + } + + public get id(): string | undefined { + return this._trackId; + } + + public get trackId(): string | undefined { + return this._trackId; + } + + public get metadata(): SDPStreamMetadataTrack | undefined { + return this._metadata; + } + + public set metadata(metadata: SDPStreamMetadataTrack | undefined) { + if (!metadata) return; + this._metadata = metadata; + } + + public get track(): MediaStreamTrack | undefined { + return this._transceiver?.receiver?.track; + } + + public get kind(): string | undefined { + return this.track?.kind ?? this._metadata?.kind; + } + + public get muted(): boolean { + if (!this.track) return true; + + return this._metadataMuted ?? false; + } + + public set metadataMuted(metadataMuted: boolean | undefined) { + this._metadataMuted = metadataMuted; + } + + public canSetTransceiver(transceiver: RTCRtpTransceiver): boolean { + if (!this._trackId) return true; + + if (!transceiver.mid) return false; + if (this.call.getRemoteTrackIdByMid(transceiver.mid) !== this._trackId) return false; + + return true; + } + + public setTransceiver(transceiver: RTCRtpTransceiver): void { + if (!this.canSetTransceiver(transceiver)) { + throw new Error("Wrong track_id"); + } + if (!transceiver.receiver.track) { + throw new Error("No receiver or track"); + } + if (!transceiver.mid) { + throw new Error("No mid"); + } + + logger.log( + `RemoteCallTrack ${this.id} setTransceiver() running (${this.call.getRemoteTrackInfoByMid( + transceiver.mid, + )})`, + ); + + this._transceiver = transceiver; + } +} diff --git a/src/webrtc/stats/connectionStats.ts b/src/webrtc/stats/connectionStats.ts new file mode 100644 index 00000000000..dbde6e50327 --- /dev/null +++ b/src/webrtc/stats/connectionStats.ts @@ -0,0 +1,47 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { TransportStats } from "./transportStats"; +import { Bitrate } from "./media/mediaTrackStats"; + +export interface ConnectionStatsBandwidth { + /** + * bytes per second + */ + download: number; + /** + * bytes per second + */ + upload: number; +} + +export interface ConnectionStatsBitrate extends Bitrate { + audio?: Bitrate; + video?: Bitrate; +} + +export interface PacketLoos { + total: number; + download: number; + upload: number; +} + +export class ConnectionStats { + public bandwidth: ConnectionStatsBitrate = {} as ConnectionStatsBitrate; + public bitrate: ConnectionStatsBitrate = {} as ConnectionStatsBitrate; + public packetLoss: PacketLoos = {} as PacketLoos; + public transport: TransportStats[] = []; +} diff --git a/src/webrtc/stats/connectionStatsReporter.ts b/src/webrtc/stats/connectionStatsReporter.ts new file mode 100644 index 00000000000..c43b9b40c19 --- /dev/null +++ b/src/webrtc/stats/connectionStatsReporter.ts @@ -0,0 +1,28 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { Bitrate } from "./media/mediaTrackStats"; + +export class ConnectionStatsReporter { + public static buildBandwidthReport(now: RTCIceCandidatePairStats): Bitrate { + const availableIncomingBitrate = now.availableIncomingBitrate; + const availableOutgoingBitrate = now.availableOutgoingBitrate; + + return { + download: availableIncomingBitrate ? Math.round(availableIncomingBitrate / 1000) : 0, + upload: availableOutgoingBitrate ? Math.round(availableOutgoingBitrate / 1000) : 0, + }; + } +} diff --git a/src/webrtc/stats/groupCallStats.ts b/src/webrtc/stats/groupCallStats.ts new file mode 100644 index 00000000000..27b89346431 --- /dev/null +++ b/src/webrtc/stats/groupCallStats.ts @@ -0,0 +1,64 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { StatsCollector } from "./statsCollector"; +import { StatsReportEmitter } from "./statsReportEmitter"; + +export class GroupCallStats { + private timer: undefined | ReturnType; + private readonly collectors: Map = new Map(); + public readonly reports = new StatsReportEmitter(); + + public constructor(private groupCallId: string, private userId: string, private interval: number = 10000) {} + + public start(): void { + if (this.timer === undefined) { + this.timer = setInterval(() => { + this.processStats(); + }, this.interval); + } + } + + public stop(): void { + if (this.timer !== undefined) { + clearInterval(this.timer); + this.collectors.forEach((c) => c.stopProcessingStats()); + } + } + + public hasStatsCollector(callId: string): boolean { + return this.collectors.has(callId); + } + + public addStatsCollector(callId: string, userId: string, peerConnection: RTCPeerConnection): boolean { + if (this.hasStatsCollector(callId)) { + return false; + } + this.collectors.set(callId, new StatsCollector(callId, userId, peerConnection, this.reports)); + return true; + } + + public removeStatsCollector(callId: string): boolean { + return this.collectors.delete(callId); + } + + public getStatsCollector(callId: string): StatsCollector | undefined { + return this.hasStatsCollector(callId) ? this.collectors.get(callId) : undefined; + } + + private processStats(): void { + this.collectors.forEach((c) => c.processStats(this.groupCallId, this.userId)); + } +} diff --git a/src/webrtc/stats/media/mediaSsrcHandler.ts b/src/webrtc/stats/media/mediaSsrcHandler.ts new file mode 100644 index 00000000000..e60605152c9 --- /dev/null +++ b/src/webrtc/stats/media/mediaSsrcHandler.ts @@ -0,0 +1,57 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { parse as parseSdp } from "sdp-transform"; + +export type Mid = string; +export type Ssrc = string; +export type MapType = "local" | "remote"; + +export class MediaSsrcHandler { + private readonly ssrcToMid = { local: new Map(), remote: new Map() }; + + public findMidBySsrc(ssrc: Ssrc, type: "local" | "remote"): Mid | undefined { + let mid: Mid | undefined; + this.ssrcToMid[type].forEach((ssrcs, m) => { + if (ssrcs.find((s) => s == ssrc)) { + mid = m; + return; + } + }); + return mid; + } + + public parse(description: string, type: MapType): void { + const sdp = parseSdp(description); + const ssrcToMid = new Map(); + sdp.media.forEach((m) => { + if ((!!m.mid && m.type === "video") || m.type === "audio") { + const ssrcs: Ssrc[] = []; + m.ssrcs?.forEach((ssrc) => { + if (ssrc.attribute === "cname") { + ssrcs.push(`${ssrc.id}`); + } + }); + ssrcToMid.set(`${m.mid}`, ssrcs); + } + }); + this.ssrcToMid[type] = ssrcToMid; + } + + public getSsrcToMidMap(type: MapType): Map { + return this.ssrcToMid[type]; + } +} diff --git a/src/webrtc/stats/media/mediaTrackHandler.ts b/src/webrtc/stats/media/mediaTrackHandler.ts new file mode 100644 index 00000000000..32580b1228a --- /dev/null +++ b/src/webrtc/stats/media/mediaTrackHandler.ts @@ -0,0 +1,71 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export type TrackId = string; + +export class MediaTrackHandler { + public constructor(private readonly pc: RTCPeerConnection) {} + + public getLocalTracks(kind: "audio" | "video"): MediaStreamTrack[] { + const isNotNullAndKind = (track: MediaStreamTrack | null): boolean => { + return track !== null && track.kind === kind; + }; + // @ts-ignore The linter don't get it + return this.pc + .getTransceivers() + .filter((t) => t.currentDirection === "sendonly" || t.currentDirection === "sendrecv") + .filter((t) => t.sender !== null) + .map((t) => t.sender) + .map((s) => s.track) + .filter(isNotNullAndKind); + } + + public getTackById(trackId: string): MediaStreamTrack | undefined { + return this.pc + .getTransceivers() + .map((t) => { + if (t?.sender.track !== null && t.sender.track.id === trackId) { + return t.sender.track; + } + if (t?.receiver.track !== null && t.receiver.track.id === trackId) { + return t.receiver.track; + } + return undefined; + }) + .find((t) => t !== undefined); + } + + public getLocalTrackIdByMid(mid: string): string | undefined { + const transceiver = this.pc.getTransceivers().find((t) => t.mid === mid); + if (transceiver !== undefined && !!transceiver.sender && !!transceiver.sender.track) { + return transceiver.sender.track.id; + } + return undefined; + } + + public getRemoteTrackIdByMid(mid: string): string | undefined { + const transceiver = this.pc.getTransceivers().find((t) => t.mid === mid); + if (transceiver !== undefined && !!transceiver.receiver && !!transceiver.receiver.track) { + return transceiver.receiver.track.id; + } + return undefined; + } + + public getActiveSimulcastStreams(): number { + //@TODO implement this right.. Check how many layer configured + return 3; + } +} diff --git a/src/webrtc/stats/media/mediaTrackStats.ts b/src/webrtc/stats/media/mediaTrackStats.ts new file mode 100644 index 00000000000..69ee9bdfadf --- /dev/null +++ b/src/webrtc/stats/media/mediaTrackStats.ts @@ -0,0 +1,104 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { TrackId } from "./mediaTrackHandler"; + +export interface PacketLoss { + packetsTotal: number; + packetsLost: number; + isDownloadStream: boolean; +} + +export interface Bitrate { + /** + * bytes per second + */ + download: number; + /** + * bytes per second + */ + upload: number; +} + +export interface Resolution { + width: number; + height: number; +} + +export type TrackStatsType = "local" | "remote"; + +export class MediaTrackStats { + private loss: PacketLoss = { packetsTotal: 0, packetsLost: 0, isDownloadStream: false }; + private bitrate: Bitrate = { download: 0, upload: 0 }; + private resolution: Resolution = { width: -1, height: -1 }; + private framerate = 0; + private codec = ""; + + public constructor( + public readonly trackId: TrackId, + public readonly type: TrackStatsType, + public readonly kind: "audio" | "video", + ) {} + + public getType(): TrackStatsType { + return this.type; + } + + public setLoss(loos: PacketLoss): void { + this.loss = loos; + } + + public getLoss(): PacketLoss { + return this.loss; + } + + public setResolution(resolution: Resolution): void { + this.resolution = resolution; + } + + public getResolution(): Resolution { + return this.resolution; + } + + public setFramerate(framerate: number): void { + this.framerate = framerate; + } + + public getFramerate(): number { + return this.framerate; + } + + public setBitrate(bitrate: Bitrate): void { + this.bitrate = bitrate; + } + + public getBitrate(): Bitrate { + return this.bitrate; + } + + public setCodec(codecShortType: string): boolean { + this.codec = codecShortType; + return true; + } + + public getCodec(): string { + return this.codec; + } + + public resetBitrate(): void { + this.bitrate = { download: 0, upload: 0 }; + } +} diff --git a/src/webrtc/stats/media/mediaTrackStatsHandler.ts b/src/webrtc/stats/media/mediaTrackStatsHandler.ts new file mode 100644 index 00000000000..6fb119c8a75 --- /dev/null +++ b/src/webrtc/stats/media/mediaTrackStatsHandler.ts @@ -0,0 +1,86 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { TrackID } from "../statsReport"; +import { MediaTrackStats } from "./mediaTrackStats"; +import { MediaTrackHandler } from "./mediaTrackHandler"; +import { MediaSsrcHandler } from "./mediaSsrcHandler"; + +export class MediaTrackStatsHandler { + private readonly track2stats = new Map(); + + public constructor( + public readonly mediaSsrcHandler: MediaSsrcHandler, + public readonly mediaTrackHandler: MediaTrackHandler, + ) {} + + /** + * Find tracks by rtc stats + * Argument report is any because the stats api is not consistent: + * For example `trackIdentifier`, `mid` not existing in every implementations + * https://www.w3.org/TR/webrtc-stats/#dom-rtcinboundrtpstreamstats + * https://developer.mozilla.org/en-US/docs/Web/API/RTCInboundRtpStreamStats + */ + public findTrack2Stats(report: any, type: "remote" | "local"): MediaTrackStats | undefined { + let trackID; + if (report.trackIdentifier) { + trackID = report.trackIdentifier; + } else if (report.mid) { + trackID = + type === "remote" + ? this.mediaTrackHandler.getRemoteTrackIdByMid(report.mid) + : this.mediaTrackHandler.getLocalTrackIdByMid(report.mid); + } else if (report.ssrc) { + const mid = this.mediaSsrcHandler.findMidBySsrc(report.ssrc, type); + if (!mid) { + return undefined; + } + trackID = + type === "remote" + ? this.mediaTrackHandler.getRemoteTrackIdByMid(report.mid) + : this.mediaTrackHandler.getLocalTrackIdByMid(report.mid); + } + + if (!trackID) { + return undefined; + } + + let trackStats = this.track2stats.get(trackID); + + if (!trackStats) { + const track = this.mediaTrackHandler.getTackById(trackID); + if (track !== undefined) { + const kind: "audio" | "video" = track.kind === "audio" ? track.kind : "video"; + trackStats = new MediaTrackStats(trackID, type, kind); + this.track2stats.set(trackID, trackStats); + } else { + return undefined; + } + } + return trackStats; + } + + public findLocalVideoTrackStats(report: any): MediaTrackStats | undefined { + const localVideoTracks = this.mediaTrackHandler.getLocalTracks("video"); + if (localVideoTracks.length === 0) { + return undefined; + } + return this.findTrack2Stats(report, "local"); + } + + public getTrack2stats(): Map { + return this.track2stats; + } +} diff --git a/src/webrtc/stats/statsCollector.ts b/src/webrtc/stats/statsCollector.ts new file mode 100644 index 00000000000..b58201183d7 --- /dev/null +++ b/src/webrtc/stats/statsCollector.ts @@ -0,0 +1,183 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { ConnectionStats } from "./connectionStats"; +import { StatsReportEmitter } from "./statsReportEmitter"; +import { ByteSend, ByteSentStatsReport, TrackID } from "./statsReport"; +import { ConnectionStatsReporter } from "./connectionStatsReporter"; +import { TransportStatsReporter } from "./transportStatsReporter"; +import { MediaSsrcHandler } from "./media/mediaSsrcHandler"; +import { MediaTrackHandler } from "./media/mediaTrackHandler"; +import { MediaTrackStatsHandler } from "./media/mediaTrackStatsHandler"; +import { TrackStatsReporter } from "./trackStatsReporter"; +import { StatsReportBuilder } from "./statsReportBuilder"; +import { StatsValueFormatter } from "./statsValueFormatter"; + +export class StatsCollector { + private isActive = true; + private previousStatsReport: RTCStatsReport | undefined; + private currentStatsReport: RTCStatsReport | undefined; + private readonly connectionStats = new ConnectionStats(); + + private readonly trackStats: MediaTrackStatsHandler; + + // private readonly ssrcToMid = { local: new Map(), remote: new Map() }; + + public constructor( + public readonly callId: string, + public readonly remoteUserId: string, + private readonly pc: RTCPeerConnection, + private readonly emitter: StatsReportEmitter, + private readonly isFocus = true, + ) { + pc.addEventListener("signalingstatechange", this.onSignalStateChange.bind(this)); + this.trackStats = new MediaTrackStatsHandler(new MediaSsrcHandler(), new MediaTrackHandler(pc)); + } + + public async processStats(groupCallId: string, localUserId: string): Promise { + if (this.isActive) { + const statsPromise = this.pc.getStats(); + if (typeof statsPromise?.then === "function") { + return statsPromise + .then((report) => { + // @ts-ignore + this.currentStatsReport = typeof report?.result === "function" ? report.result() : report; + try { + this.processStatsReport(groupCallId, localUserId); + } catch (error) { + this.isActive = false; + return false; + } + + this.previousStatsReport = this.currentStatsReport; + return true; + }) + .catch((error) => { + this.handleError(error); + return false; + }); + } + this.isActive = false; + } + return Promise.resolve(false); + } + + private processStatsReport(groupCallId: string, localUserId: string): void { + const byteSentStats: ByteSentStatsReport = new Map(); + + this.currentStatsReport?.forEach((now) => { + const before = this.previousStatsReport ? this.previousStatsReport.get(now.id) : null; + // RTCIceCandidatePairStats - https://w3c.github.io/webrtc-stats/#candidatepair-dict* + if (now.type === "candidate-pair" && now.nominated && now.state === "succeeded") { + this.connectionStats.bandwidth = ConnectionStatsReporter.buildBandwidthReport(now); + this.connectionStats.transport = TransportStatsReporter.buildReport( + this.currentStatsReport, + now, + this.connectionStats.transport, + this.isFocus, + ); + + // RTCReceivedRtpStreamStats + // https://w3c.github.io/webrtc-stats/#receivedrtpstats-dict* + // RTCSentRtpStreamStats + // https://w3c.github.io/webrtc-stats/#sentrtpstats-dict* + } else if (now.type === "inbound-rtp" || now.type === "outbound-rtp") { + const trackStats = this.trackStats.findTrack2Stats( + now, + now.type === "inbound-rtp" ? "remote" : "local", + ); + if (!trackStats) { + return; + } + + if (before) { + TrackStatsReporter.buildPacketsLost(trackStats, now, before); + } + + // Get the resolution and framerate for only remote video sources here. For the local video sources, + // 'track' stats will be used since they have the updated resolution based on the simulcast streams + // currently being sent. Promise based getStats reports three 'outbound-rtp' streams and there will be + // more calculations needed to determine what is the highest resolution stream sent by the client if the + // 'outbound-rtp' stats are used. + if (now.type === "inbound-rtp") { + TrackStatsReporter.buildFramerateResolution(trackStats, now); + if (before) { + TrackStatsReporter.buildBitrateReceived(trackStats, now, before); + } + } else if (before) { + byteSentStats.set(trackStats.trackId, StatsValueFormatter.getNonNegativeValue(now.bytesSent)); + TrackStatsReporter.buildBitrateSend(trackStats, now, before); + } + TrackStatsReporter.buildCodec(this.currentStatsReport, trackStats, now); + } else if (now.type === "track" && now.kind === "video" && !now.remoteSource) { + const trackStats = this.trackStats.findLocalVideoTrackStats(now); + if (!trackStats) { + return; + } + TrackStatsReporter.buildFramerateResolution(trackStats, now); + TrackStatsReporter.calculateSimulcastFramerate( + trackStats, + now, + before, + this.trackStats.mediaTrackHandler.getActiveSimulcastStreams(), + ); + } + }); + + this.emitter.emitByteSendReport(byteSentStats); + this.processAndEmitReport(); + } + + public setActive(isActive: boolean): void { + this.isActive = isActive; + } + + public getActive(): boolean { + return this.isActive; + } + + private handleError(_: any): void { + this.isActive = false; + } + + private processAndEmitReport(): void { + const report = StatsReportBuilder.build(this.trackStats.getTrack2stats()); + + this.connectionStats.bandwidth = report.bandwidth; + this.connectionStats.bitrate = report.bitrate; + this.connectionStats.packetLoss = report.packetLoss; + + this.emitter.emitConnectionStatsReport({ + ...report, + transport: this.connectionStats.transport, + }); + + this.connectionStats.transport = []; + } + + public stopProcessingStats(): void {} + + private onSignalStateChange(): void { + if (this.pc.signalingState === "stable") { + if (this.pc.currentRemoteDescription) { + this.trackStats.mediaSsrcHandler.parse(this.pc.currentRemoteDescription.sdp, "remote"); + } + if (this.pc.currentLocalDescription) { + this.trackStats.mediaSsrcHandler.parse(this.pc.currentLocalDescription.sdp, "local"); + } + } + } +} diff --git a/src/webrtc/stats/statsReport.ts b/src/webrtc/stats/statsReport.ts new file mode 100644 index 00000000000..56d6c4b2e48 --- /dev/null +++ b/src/webrtc/stats/statsReport.ts @@ -0,0 +1,56 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { ConnectionStatsBandwidth, ConnectionStatsBitrate, PacketLoos } from "./connectionStats"; +import { TransportStats } from "./transportStats"; +import { Resolution } from "./media/mediaTrackStats"; + +export enum StatsReport { + CONNECTION_STATS = "StatsReport.connection_stats", + BYTE_SENT_STATS = "StatsReport.byte_sent_stats", +} + +export type TrackID = string; +export type ByteSend = number; + +export interface ByteSentStatsReport extends Map { + // is a map: `local trackID` => byte send +} + +export interface ConnectionStatsReport { + bandwidth: ConnectionStatsBandwidth; + bitrate: ConnectionStatsBitrate; + packetLoss: PacketLoos; + resolution: ResolutionMap; + framerate: FramerateMap; + codec: CodecMap; + transport: TransportStats[]; +} + +export interface ResolutionMap { + local: Map; + remote: Map; +} + +export interface FramerateMap { + local: Map; + remote: Map; +} + +export interface CodecMap { + local: Map; + remote: Map; +} diff --git a/src/webrtc/stats/statsReportBuilder.ts b/src/webrtc/stats/statsReportBuilder.ts new file mode 100644 index 00000000000..c1af471ce30 --- /dev/null +++ b/src/webrtc/stats/statsReportBuilder.ts @@ -0,0 +1,110 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { CodecMap, ConnectionStatsReport, FramerateMap, ResolutionMap, TrackID } from "./statsReport"; +import { MediaTrackStats, Resolution } from "./media/mediaTrackStats"; + +export class StatsReportBuilder { + public static build(stats: Map): ConnectionStatsReport { + const report = {} as ConnectionStatsReport; + + // process stats + const totalPackets = { + download: 0, + upload: 0, + }; + const lostPackets = { + download: 0, + upload: 0, + }; + let bitrateDownload = 0; + let bitrateUpload = 0; + const resolutions: ResolutionMap = { + local: new Map(), + remote: new Map(), + }; + const framerates: FramerateMap = { local: new Map(), remote: new Map() }; + const codecs: CodecMap = { local: new Map(), remote: new Map() }; + + let audioBitrateDownload = 0; + let audioBitrateUpload = 0; + let videoBitrateDownload = 0; + let videoBitrateUpload = 0; + + for (const [trackId, trackStats] of stats) { + // process packet loss stats + const loss = trackStats.getLoss(); + const type = loss.isDownloadStream ? "download" : "upload"; + + totalPackets[type] += loss.packetsTotal; + lostPackets[type] += loss.packetsLost; + + // process bitrate stats + bitrateDownload += trackStats.getBitrate().download; + bitrateUpload += trackStats.getBitrate().upload; + + // collect resolutions and framerates + if (trackStats.kind === "audio") { + audioBitrateDownload += trackStats.getBitrate().download; + audioBitrateUpload += trackStats.getBitrate().upload; + } else { + videoBitrateDownload += trackStats.getBitrate().download; + videoBitrateUpload += trackStats.getBitrate().upload; + } + + resolutions[trackStats.getType()].set(trackId, trackStats.getResolution()); + framerates[trackStats.getType()].set(trackId, trackStats.getFramerate()); + codecs[trackStats.getType()].set(trackId, trackStats.getCodec()); + + trackStats.resetBitrate(); + } + + report.bitrate = { + upload: bitrateUpload, + download: bitrateDownload, + }; + + report.bitrate.audio = { + upload: audioBitrateUpload, + download: audioBitrateDownload, + }; + + report.bitrate.video = { + upload: videoBitrateUpload, + download: videoBitrateDownload, + }; + + report.packetLoss = { + total: StatsReportBuilder.calculatePacketLoss( + lostPackets.download + lostPackets.upload, + totalPackets.download + totalPackets.upload, + ), + download: StatsReportBuilder.calculatePacketLoss(lostPackets.download, totalPackets.download), + upload: StatsReportBuilder.calculatePacketLoss(lostPackets.upload, totalPackets.upload), + }; + report.framerate = framerates; + report.resolution = resolutions; + report.codec = codecs; + return report; + } + + private static calculatePacketLoss(lostPackets: number, totalPackets: number): number { + if (!totalPackets || totalPackets <= 0 || !lostPackets || lostPackets <= 0) { + return 0; + } + + return Math.round((lostPackets / totalPackets) * 100); + } +} diff --git a/src/webrtc/stats/statsReportEmitter.ts b/src/webrtc/stats/statsReportEmitter.ts new file mode 100644 index 00000000000..cf014708e89 --- /dev/null +++ b/src/webrtc/stats/statsReportEmitter.ts @@ -0,0 +1,33 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { TypedEventEmitter } from "../../models/typed-event-emitter"; +import { ByteSentStatsReport, ConnectionStatsReport, StatsReport } from "./statsReport"; + +export type StatsReportHandlerMap = { + [StatsReport.BYTE_SENT_STATS]: (report: ByteSentStatsReport) => void; + [StatsReport.CONNECTION_STATS]: (report: ConnectionStatsReport) => void; +}; + +export class StatsReportEmitter extends TypedEventEmitter { + public emitByteSendReport(byteSentStats: ByteSentStatsReport): void { + this.emit(StatsReport.BYTE_SENT_STATS, byteSentStats); + } + + public emitConnectionStatsReport(report: ConnectionStatsReport): void { + this.emit(StatsReport.CONNECTION_STATS, report); + } +} diff --git a/src/webrtc/stats/statsValueFormatter.ts b/src/webrtc/stats/statsValueFormatter.ts new file mode 100644 index 00000000000..b377a409b5b --- /dev/null +++ b/src/webrtc/stats/statsValueFormatter.ts @@ -0,0 +1,30 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +export class StatsValueFormatter { + public static getNonNegativeValue(imput: any): number { + let value = imput; + + if (typeof value !== "number") { + value = Number(value); + } + + if (isNaN(value)) { + return 0; + } + + return Math.max(0, value); + } +} diff --git a/src/webrtc/stats/trackStatsReporter.ts b/src/webrtc/stats/trackStatsReporter.ts new file mode 100644 index 00000000000..1f6fcd6d1ce --- /dev/null +++ b/src/webrtc/stats/trackStatsReporter.ts @@ -0,0 +1,117 @@ +import { MediaTrackStats } from "./media/mediaTrackStats"; +import { StatsValueFormatter } from "./statsValueFormatter"; + +export class TrackStatsReporter { + public static buildFramerateResolution(trackStats: MediaTrackStats, now: any): void { + const resolution = { + height: now.frameHeight, + width: now.frameWidth, + }; + const frameRate = now.framesPerSecond; + + if (resolution.height && resolution.width) { + trackStats.setResolution(resolution); + } + trackStats.setFramerate(Math.round(frameRate || 0)); + } + + public static calculateSimulcastFramerate(trackStats: MediaTrackStats, now: any, before: any, layer: number): void { + let frameRate = trackStats.getFramerate(); + if (!frameRate) { + if (before) { + const timeMs = now.timestamp - before.timestamp; + + if (timeMs > 0 && now.framesSent) { + const numberOfFramesSinceBefore = now.framesSent - before.framesSent; + + frameRate = (numberOfFramesSinceBefore / timeMs) * 1000; + } + } + + if (!frameRate) { + return; + } + } + + // Reset frame rate to 0 when video is suspended as a result of endpoint falling out of last-n. + frameRate = layer ? Math.round(frameRate / layer) : 0; + trackStats.setFramerate(frameRate); + } + + public static buildCodec(report: RTCStatsReport | undefined, trackStats: MediaTrackStats, now: any): void { + const codec = report?.get(now.codecId); + + if (codec) { + /** + * The mime type has the following form: video/VP8 or audio/ISAC, + * so we what to keep just the type after the '/', audio and video + * keys will be added on the processing side. + */ + const codecShortType = codec.mimeType.split("/")[1]; + + codecShortType && trackStats.setCodec(codecShortType); + } + } + + public static buildBitrateReceived(trackStats: MediaTrackStats, now: any, before: any): void { + trackStats.setBitrate({ + download: TrackStatsReporter.calculateBitrate( + now.bytesReceived, + before.bytesReceived, + now.timestamp, + before.timestamp, + ), + upload: 0, + }); + } + + public static buildBitrateSend(trackStats: MediaTrackStats, now: any, before: any): void { + trackStats.setBitrate({ + download: 0, + upload: this.calculateBitrate(now.bytesSent, before.bytesSent, now.timestamp, before.timestamp), + }); + } + + public static buildPacketsLost(trackStats: MediaTrackStats, now: any, before: any): void { + const key = now.type === "outbound-rtp" ? "packetsSent" : "packetsReceived"; + + let packetsNow = now[key]; + if (!packetsNow || packetsNow < 0) { + packetsNow = 0; + } + + const packetsBefore = StatsValueFormatter.getNonNegativeValue(before[key]); + const packetsDiff = Math.max(0, packetsNow - packetsBefore); + + const packetsLostNow = StatsValueFormatter.getNonNegativeValue(now.packetsLost); + const packetsLostBefore = StatsValueFormatter.getNonNegativeValue(before.packetsLost); + const packetsLostDiff = Math.max(0, packetsLostNow - packetsLostBefore); + + trackStats.setLoss({ + packetsTotal: packetsDiff + packetsLostDiff, + packetsLost: packetsLostDiff, + isDownloadStream: now.type !== "outbound-rtp", + }); + } + + private static calculateBitrate( + bytesNowAny: any, + bytesBeforeAny: any, + nowTimestamp: number, + beforeTimestamp: number, + ): number { + const bytesNow = StatsValueFormatter.getNonNegativeValue(bytesNowAny); + const bytesBefore = StatsValueFormatter.getNonNegativeValue(bytesBeforeAny); + const bytesProcessed = Math.max(0, bytesNow - bytesBefore); + + const timeMs = nowTimestamp - beforeTimestamp; + let bitrateKbps = 0; + + if (timeMs > 0) { + // TODO is there any reason to round here? + bitrateKbps = Math.round((bytesProcessed * 8) / timeMs); + } + + return bitrateKbps; + } +} diff --git a/src/webrtc/stats/transportStats.ts b/src/webrtc/stats/transportStats.ts new file mode 100644 index 00000000000..2b6e975484f --- /dev/null +++ b/src/webrtc/stats/transportStats.ts @@ -0,0 +1,26 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export interface TransportStats { + ip: string; + type: string; + localIp: string; + isFocus: boolean; + localCandidateType: string; + remoteCandidateType: string; + networkType: string; + rtt: number; +} diff --git a/src/webrtc/stats/transportStatsReporter.ts b/src/webrtc/stats/transportStatsReporter.ts new file mode 100644 index 00000000000..d419a73972b --- /dev/null +++ b/src/webrtc/stats/transportStatsReporter.ts @@ -0,0 +1,48 @@ +import { TransportStats } from "./transportStats"; + +export class TransportStatsReporter { + public static buildReport( + report: RTCStatsReport | undefined, + now: RTCIceCandidatePairStats, + conferenceStatsTransport: TransportStats[], + isFocus: boolean, + ): TransportStats[] { + const localUsedCandidate = report?.get(now.localCandidateId); + const remoteUsedCandidate = report?.get(now.remoteCandidateId); + + // RTCIceCandidateStats + // https://w3c.github.io/webrtc-stats/#icecandidate-dict* + if (remoteUsedCandidate && localUsedCandidate) { + const remoteIpAddress = + remoteUsedCandidate.ip !== undefined ? remoteUsedCandidate.ip : remoteUsedCandidate.address; + const remotePort = remoteUsedCandidate.port; + const ip = `${remoteIpAddress}:${remotePort}`; + + const localIpAddress = + localUsedCandidate.ip !== undefined ? localUsedCandidate.ip : localUsedCandidate.address; + const localPort = localUsedCandidate.port; + const localIp = `${localIpAddress}:${localPort}`; + + const type = remoteUsedCandidate.protocol; + + // Save the address unless it has been saved already. + if ( + !conferenceStatsTransport.some( + (t: TransportStats) => t.ip === ip && t.type === type && t.localIp === localIp, + ) + ) { + conferenceStatsTransport.push({ + ip, + type, + localIp, + isFocus, + localCandidateType: localUsedCandidate.candidateType, + remoteCandidateType: remoteUsedCandidate.candidateType, + networkType: localUsedCandidate.networkType, + rtt: now.currentRoundTripTime ? now.currentRoundTripTime * 1000 : NaN, + } as TransportStats); + } + } + return conferenceStatsTransport; + } +} diff --git a/src/webrtc/trackPublication.ts b/src/webrtc/trackPublication.ts new file mode 100644 index 00000000000..5a83185c9b5 --- /dev/null +++ b/src/webrtc/trackPublication.ts @@ -0,0 +1,123 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { logger } from "../logger"; +import { MatrixCall } from "./call"; +import { SDPStreamMetadataTrack } from "./callEventTypes"; +import { LocalCallTrack } from "./localCallTrack"; + +interface TrackPublicationOpts { + call: MatrixCall; + track: LocalCallTrack; + transceiver: RTCRtpTransceiver; +} + +/** + * TrackPublication represents a LocalCallTrack being published to a specific peer + * connection. + */ +export class TrackPublication { + public readonly call: MatrixCall; + public readonly track: LocalCallTrack; + private _transceiver: RTCRtpTransceiver; + + public constructor(opts: TrackPublicationOpts) { + this.call = opts.call; + this.track = opts.track; + this._transceiver = opts.transceiver; + + this.updateSenderTrack(); + } + + public get logInfo(): string { + return `streamId=${this.streamId}, trackId=${this.trackId}, mid=${this.mid} kind=${this.track.kind}`; + } + + public get metadata(): SDPStreamMetadataTrack { + const track = this.track; + const trackMetadata: SDPStreamMetadataTrack = { + kind: this.track.kind, + }; + + if (track.isVideo) { + trackMetadata.width = track.track.getSettings().width; + trackMetadata.height = track.track.getSettings().height; + } + + return trackMetadata; + } + + public get mid(): string | undefined { + return this.transceiver?.mid ?? undefined; + } + + public get trackId(): string | undefined { + const mid = this.transceiver?.mid; + return mid ? this.call?.getLocalTrackIdByMid(mid) : undefined; + } + + public get streamId(): string | undefined { + const mid = this.transceiver?.mid; + return mid ? this.call?.getLocalStreamIdByMid(mid) : undefined; + } + + public get transceiver(): RTCRtpTransceiver { + return this._transceiver; + } + + public unpublish(): void { + this.call.unpublishTrack(this); + } + + public updateSenderTrack(): void { + const { stream, track } = this.track; + const transceiver = this.transceiver; + const sender = this.transceiver.sender; + const parameters = sender.getParameters(); + + // No need to update the track + if (sender.track === track) return; + + // setStreams() is currently not supported by Firefox but we + // try to use it at least in other browsers (once we switch + // to using mids and throw away streamIds we will be able to + // throw this away) + if (sender.setStreams && stream) sender.setStreams(stream); + + try { + sender.replaceTrack(track); + + // Does this even work, where does it work? + transceiver.sender.setParameters({ + ...parameters, + encodings: this.track.encodings, + }); + + // Set the direction of the transceiver to indicate we're + // going to be sending. This may trigger re-negotiation, if + // we weren't sending until now + transceiver.direction = transceiver.direction === "inactive" ? "sendonly" : "sendrecv"; + } catch (error) { + logger.warn( + `TrackPublication ${this.trackId} updateSenderTrack() failed to replace track - publishing on new transceiver:`, + error, + ); + + this.call.unpublishTrackOnTransceiver(this); + this._transceiver = this.call.publishTrackOnNewTransceiver(this.track); + } + } +}