Add media hints for notification events. (#3493)

* Add media hints for notification events.

* Prevent showing calling view when disconnected from Livekit. (#3491)

* Refactor disconnection handling

* Use "unknown"

* Update signature

* Add tests

* Expose livekitConnectionState directly

* fix whoopsie

* Update dependency livekit-client to v2.15.7 (#3496)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

* Fix the interactivity of buttons while reconnecting or in earpiece mode (#3486)

* Fix the interactivity of buttons while reconnecting or in earpiece mode

When we're in one of these modes, we need to ensure that everything above the overlay (the header and footer buttons) is interactive, while everything obscured by the overlay (the media tiles) is non-interactive and removed from the accessibility tree. It's not a very easy task to trap focus *outside* an element, so the best solution I could come up with is to set tabindex="-1" manually on all interactive elements belonging to the media tiles.

* Write a Playwright test for reconnecting

* fix lints

Signed-off-by: Timo K <[email protected]>

* fix test

Signed-off-by: Timo K <[email protected]>

* enable http2 for matrx-rtc host to allow the jwt service to talk to the SFU

* remove rate limit for delayed events

* more time to connect to livekit SFU

* Due to a Firefox issue we set the start anchor for the tab test to the Mute microphone button

* adapt to most recent Element Web version

* Use the "End call" button as proofe for a started call

* Currrenty disabled due to recent Element Web
- not indicating the number of participants
- bypassing Lobby

* linting

* disable 'can only interact with header and footer while reconnecting' for firefox

---------

Signed-off-by: Timo K <[email protected]>
Co-authored-by: Timo <[email protected]>
Co-authored-by: Timo K <[email protected]>
Co-authored-by: fkwp <[email protected]>

* Log when a track is unpublished or runs into an error (#3495)

* default mute states (unmuted!) in widget mode (embedded + intent) (#3494)

* default mute states (unmuted!) in widget mode (embedded + intent)

Signed-off-by: Timo K <[email protected]>

* review

Signed-off-by: Timo K <[email protected]>

* introduce a cache for the url params.

Signed-off-by: Timo K <[email protected]>

* Add an option to skip the cache.

Signed-off-by: Timo K <[email protected]>

---------

Signed-off-by: Timo K <[email protected]>

* Apply new hint code

* missed a bit

* fix intent

* Automatically update intent on mute change

* update packages

* lint

* Fix tests

* fix merge fails

---------

Signed-off-by: Timo K <[email protected]>
Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: Robin <[email protected]>
Co-authored-by: Timo <[email protected]>
Co-authored-by: Timo K <[email protected]>
Co-authored-by: fkwp <[email protected]>

Author: 5 people

src/UrlParams.ts

@@ -8,7 +8,10 @@ Please see LICENSE in the repository root for full details.
 import { useMemo } from "react";
 import { useLocation } from "react-router-dom";
 import { logger } from "matrix-js-sdk/lib/logger";
-import { type RTCNotificationType } from "matrix-js-sdk/lib/matrixrtc";
+import {
+  type RTCCallIntent,
+  type RTCNotificationType,
+} from "matrix-js-sdk/lib/matrixrtc";
 import { pickBy } from "lodash-es";
 
 import { Config } from "./config/Config";
@@ -26,7 +29,9 @@ export enum UserIntent {
   StartNewCall = "start_call",
   JoinExistingCall = "join_existing",
   StartNewCallDM = "start_call_dm",
+  StartNewCallDMVoice = "start_call_dm_voice",
   JoinExistingCallDM = "join_existing_dm",
+  JoinExistingCallDMVoice = "join_existing_dm_voice",
   Unknown = "unknown",
 }
 
@@ -227,6 +232,12 @@ export interface UrlConfiguration {
    *  - auto-dismiss the call widget once the notification lifetime expires on the receivers side.
    */
   waitForCallPickup: boolean;
+
+  callIntent?: RTCCallIntent;
+}
+interface IntentAndPlatformDerivedConfiguration {
+  defaultAudioEnabled?: boolean;
+  defaultVideoEnabled?: boolean;
 }
 interface IntentAndPlatformDerivedConfiguration {
   defaultAudioEnabled?: boolean;
@@ -395,22 +406,31 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => {
   switch (intent) {
     case UserIntent.StartNewCall:
       intentPreset.skipLobby = false;
+      intentPreset.callIntent = "video";
       break;
     case UserIntent.JoinExistingCall:
       // On desktop this will be overridden based on which button was used to join the call
       intentPreset.skipLobby = false;
+      intentPreset.callIntent = "video";
       break;
+    case UserIntent.StartNewCallDMVoice:
+      intentPreset.callIntent = "audio";
+    // Fall through
     case UserIntent.StartNewCallDM:
       intentPreset.skipLobby = true;
       intentPreset.sendNotificationType = "ring";
       intentPreset.autoLeaveWhenOthersLeft = true;
       intentPreset.waitForCallPickup = true;
-
+      intentPreset.callIntent = intentPreset.callIntent ?? "video";
       break;
+    case UserIntent.JoinExistingCallDMVoice:
+      intentPreset.callIntent = "audio";
+    // Fall through
     case UserIntent.JoinExistingCallDM:
       // On desktop this will be overridden based on which button was used to join the call
       intentPreset.skipLobby = true;
       intentPreset.autoLeaveWhenOthersLeft = true;
+      intentPreset.callIntent = intentPreset.callIntent ?? "video";
       break;
     // Non widget usecase defaults
     default:
@@ -447,6 +467,11 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => {
           intentAndPlatformDerivedConfiguration.defaultAudioEnabled = true;
           intentAndPlatformDerivedConfiguration.defaultVideoEnabled = true;
           break;
+        case UserIntent.StartNewCallDMVoice:
+        case UserIntent.JoinExistingCallDMVoice:
+          intentAndPlatformDerivedConfiguration.defaultAudioEnabled = true;
+          intentAndPlatformDerivedConfiguration.defaultVideoEnabled = false;
+          break;
       }
   }
 

src/room/GroupCallView.tsx

@@ -128,6 +128,16 @@ export const GroupCallView: FC<Props> = ({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
+  // Update our member event when our mute state changes.
+  useEffect(() => {
+    if (!isJoined) {
+      return;
+    }
+    void rtcSession.updateCallIntent(
+      muteStates.video.enabled ? "video" : "audio",
+    );
+  }, [rtcSession, isJoined, muteStates.video.enabled]);
+
   useEffect(() => {
     logger.info("[Lifecycle] GroupCallView Component mounted");
     return (): void => {

src/rtcSessionHelpers.ts

@@ -120,11 +120,13 @@ export async function enterRTCSession(
   const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get();
   const useDeviceSessionMemberEvents =
     features?.feature_use_device_session_member_events;
+  const { sendNotificationType: notificationType, callIntent } = getUrlParams();
   rtcSession.joinRoomSession(
     await makePreferredLivekitFoci(rtcSession, livekitAlias),
     makeActiveFocus(),
     {
-      notificationType: getUrlParams().sendNotificationType,
+      notificationType,
+      callIntent,
       useNewMembershipManager,
       manageMediaKeys: encryptMedia,
       ...(useDeviceSessionMemberEvents !== undefined && {

src/utils/test.ts

@@ -360,6 +360,8 @@ export class MockRTCSession extends TypedEventEmitter<
     return this;
   }
 
+  public updateCallIntent = vitest.fn();
+
   private _membershipStatus = Status.Connected;
   public get membershipStatus(): Status {
     return this._membershipStatus;

yarn.lock

@@ -2793,10 +2793,10 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@matrix-org/matrix-sdk-crypto-wasm@npm:^15.1.0":
-  version: 15.1.0
-  resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:15.1.0"
-  checksum: 10c0/19edc6d0045ff49fad8d77b6e561cee994f7513f8c18a7176ae2d3f0116c1a91980e02d10300b09c2b72dea4da4a8c3392f2bf1752057f2d6b53030a056d76d8
+"@matrix-org/matrix-sdk-crypto-wasm@npm:^15.3.0":
+  version: 15.3.0
+  resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:15.3.0"
+  checksum: 10c0/45628f36b7b0e54a8777ae67a7233dbdf3e3cf14e0d95d21f62f89a7ea7e3f907232f1eb7b1262193b1e227759fad47af829dcccc103ded89011f13c66f01d76
   languageName: node
   linkType: hard
 
@@ -5290,13 +5290,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@types/retry@npm:0.12.0":
-  version: 0.12.0
-  resolution: "@types/retry@npm:0.12.0"
-  checksum: 10c0/7c5c9086369826f569b83a4683661557cab1361bac0897a1cefa1a915ff739acd10ca0d62b01071046fe3f5a3f7f2aec80785fe283b75602dc6726781ea3e328
-  languageName: node
-  linkType: hard
-
 "@types/sdp-transform@npm:^2.4.5":
   version: 2.4.10
   resolution: "@types/sdp-transform@npm:2.4.10"
@@ -9528,6 +9521,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"is-network-error@npm:^1.1.0":
+  version: 1.3.0
+  resolution: "is-network-error@npm:1.3.0"
+  checksum: 10c0/3e85a69e957988db66d5af5412efdd531a5a63e150d1bdd5647cfd4dc54fd89b1dbdd472621f8915233c3176ba1e6922afa8a51a9e363ba4693edf96a294f898
+  languageName: node
+  linkType: hard
+
 "is-number-object@npm:^1.1.1":
   version: 1.1.1
   resolution: "is-number-object@npm:1.1.1"
@@ -10298,11 +10298,11 @@ __metadata:
   linkType: hard
 
 "matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=develop":
-  version: 37.13.0
-  resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=2f1d654f14be8dd03896e9e76f12017b6f9eec1c"
+  version: 38.3.0
+  resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=41d70d0b5d3f0eba92686f8089cb329d875b26b5"
   dependencies:
     "@babel/runtime": "npm:^7.12.5"
-    "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.1.0"
+    "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0"
     another-json: "npm:^0.2.0"
     bs58: "npm:^6.0.0"
     content-type: "npm:^1.0.4"
@@ -10311,11 +10311,11 @@ __metadata:
     matrix-events-sdk: "npm:0.0.1"
     matrix-widget-api: "npm:^1.10.0"
     oidc-client-ts: "npm:^3.0.1"
-    p-retry: "npm:4"
+    p-retry: "npm:7"
     sdp-transform: "npm:^2.14.1"
     unhomoglyph: "npm:^1.0.6"
-    uuid: "npm:11"
-  checksum: 10c0/ecd019c677c272c5598617dcde407dbe4b1b11460863b2a577e33f3fd8732c9d9073ec0221b471ec1eb24e2839eec20728db7f92c9348be83126547286e50805
+    uuid: "npm:13"
+  checksum: 10c0/b48528fec573f3e14d1297f360a56d52d7f313da0d4cf82ab51e4c29798b86995b8a6bd72409779746e7bcf02949bc2788bffa9aba276bfb1a76dbcbe89900a0
   languageName: node
   linkType: hard
 
@@ -10922,13 +10922,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"p-retry@npm:4":
-  version: 4.6.2
-  resolution: "p-retry@npm:4.6.2"
+"p-retry@npm:7":
+  version: 7.0.0
+  resolution: "p-retry@npm:7.0.0"
   dependencies:
-    "@types/retry": "npm:0.12.0"
-    retry: "npm:^0.13.1"
-  checksum: 10c0/d58512f120f1590cfedb4c2e0c42cb3fa66f3cea8a4646632fcb834c56055bb7a6f138aa57b20cc236fb207c9d694e362e0b5c2b14d9b062f67e8925580c73b0
+    is-network-error: "npm:^1.1.0"
+  checksum: 10c0/3c090ac72bbe00fd2f062ee6178c44f7302f298936ab2290a458575e73650e7834b556beb2b09fa9fbebedab2ec3358cb474c09710cf828972b670c3c0cb89e4
   languageName: node
   linkType: hard
 
@@ -12230,13 +12229,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"retry@npm:^0.13.1":
-  version: 0.13.1
-  resolution: "retry@npm:0.13.1"
-  checksum: 10c0/9ae822ee19db2163497e074ea919780b1efa00431d197c7afdb950e42bf109196774b92a49fc9821f0b8b328a98eea6017410bfc5e8a0fc19c85c6d11adb3772
-  languageName: node
-  linkType: hard
-
 "reusify@npm:^1.0.4":
   version: 1.1.0
   resolution: "reusify@npm:1.1.0"
@@ -13766,12 +13758,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"uuid@npm:11":
-  version: 11.0.5
-  resolution: "uuid@npm:11.0.5"
+"uuid@npm:13":
+  version: 13.0.0
+  resolution: "uuid@npm:13.0.0"
   bin:
-    uuid: dist/esm/bin/uuid
-  checksum: 10c0/6f59f0c605e02c14515401084ca124b9cb462b4dcac866916a49862bcf831874508a308588c23a7718269226ad11a92da29b39d761ad2b86e736623e3a33b6e7
+    uuid: dist-node/bin/uuid
+  checksum: 10c0/950e4c18d57fef6c69675344f5700a08af21e26b9eff2bf2180427564297368c538ea11ac9fb2e6528b17fc3966a9fd2c5049361b0b63c7d654f3c550c9b3d67
   languageName: node
   linkType: hard