From 22c9ee8fff47ed5452303e39f998c79b22dd2dbc Mon Sep 17 00:00:00 2001 From: Erwan Leboucher Date: Sat, 1 Aug 2026 23:33:11 +0200 Subject: [PATCH 1/6] chore(call): vendor the native call plugin and its iOS frameworks --- .github/workflows/tauri-build.yml | 4 + .ignore | 9 + package.json | 3 + pnpm-lock.yaml | 161 ++++++++++++++++++ scripts/download-ios-frameworks.js | 128 ++++++++++++++ src-tauri/.gitignore | 3 + src-tauri/Cargo.lock | 15 ++ src-tauri/Cargo.toml | 1 + src-tauri/Info.plist | 5 + src-tauri/capabilities/mobile.json | 1 + src-tauri/gen/android/app/build.gradle.kts | 18 ++ .../android/app/src/main/AndroidManifest.xml | 1 + src-tauri/gen/android/build.gradle.kts | 3 + .../ios-entitlements/sable.debug.entitlements | 6 + .../sable.release.entitlements | 12 ++ .../sable.share-extension.debug.entitlements | 6 + ...sable.share-extension.release.entitlements | 10 ++ src-tauri/ios-project.yml | 9 +- src-tauri/src/lib.rs | 1 + src-tauri/tauri.conf.json | 8 +- src-tauri/tauri.local.ios.conf.json | 3 + src/app/generated/tauri/commands.ts | 2 +- src/app/generated/tauri/events.ts | 2 +- src/app/generated/tauri/index.ts | 2 +- src/app/generated/tauri/types.ts | 2 +- 25 files changed, 402 insertions(+), 13 deletions(-) create mode 100644 .ignore create mode 100644 scripts/download-ios-frameworks.js create mode 100644 src-tauri/ios-entitlements/sable.debug.entitlements create mode 100644 src-tauri/ios-entitlements/sable.release.entitlements create mode 100644 src-tauri/ios-entitlements/sable.share-extension.debug.entitlements create mode 100644 src-tauri/ios-entitlements/sable.share-extension.release.entitlements create mode 100644 src-tauri/tauri.local.ios.conf.json diff --git a/.github/workflows/tauri-build.yml b/.github/workflows/tauri-build.yml index 3d976d4623..95eebcde26 100644 --- a/.github/workflows/tauri-build.yml +++ b/.github/workflows/tauri-build.yml @@ -461,6 +461,10 @@ jobs: shell: bash run: mise -y run tauri:setup:ios + - name: Download iOS frameworks + shell: bash + run: pnpm ios:frameworks + - name: Symlink icons to the ios folder shell: bash run: mise run icons:symlink --write --force diff --git a/.ignore b/.ignore new file mode 100644 index 0000000000..b2f5e7e8af --- /dev/null +++ b/.ignore @@ -0,0 +1,9 @@ +!.slim/deepwork/ +!.slim/deepwork/** + +# BEGIN oh-my-opencode-slim worktrees +!.slim/ +!.slim/worktrees.json +!.slim/worktrees/ +!.slim/worktrees/** +# END oh-my-opencode-slim worktrees diff --git a/package.json b/package.json index 2660b32fbb..0a7fde8157 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "typecheck": "tsc", "tauri": "tauri", "tauri:cef": "node scripts/tauri.js cef", + "ios:frameworks": "node scripts/download-ios-frameworks.js", "tauri:wry": "node scripts/tauri.js wry", "test": "vitest", "test:ui": "vitest --ui", @@ -46,6 +47,7 @@ "@choochmeque/tauri-plugin-sharekit-api": "0.4.0-rc.5", "@fontsource-variable/nunito": "5.2.7", "@fontsource/space-mono": "5.2.9", + "@livekit/components-react": "2.9.23", "@lottiefiles/dotlottie-react": "^0.12.0", "@noble/hashes": "^2.2.0", "@phosphor-icons/react": "^2.1.10", @@ -96,6 +98,7 @@ "leaflet": "1.9.4", "linkify-react": "^4.3.3", "linkifyjs": "^4.3.3", + "livekit-client": "2.21.0", "marked": "^18.0.5", "matrix-js-sdk": "42.0.0", "matrix-widget-api": "^1.17.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 834f3a991d..c35a9028ea 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -43,6 +43,9 @@ importers: '@fontsource/space-mono': specifier: 5.2.9 version: 5.2.9 + '@livekit/components-react': + specifier: 2.9.23 + version: 2.9.23(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tslib@2.8.1) '@lottiefiles/dotlottie-react': specifier: ^0.12.0 version: 0.12.3(react@18.3.1) @@ -193,6 +196,9 @@ importers: linkifyjs: specifier: ^4.3.3 version: 4.3.3 + livekit-client: + specifier: 2.21.0 + version: 2.21.0(@types/dom-mediacapture-record@1.0.22) marked: specifier: ^18.0.5 version: 18.0.5 @@ -952,6 +958,9 @@ packages: resolution: {integrity: sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==} hasBin: true + '@bufbuild/protobuf@1.10.1': + resolution: {integrity: sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==} + '@choochmeque/tauri-plugin-notifications-api@0.5.0-rc.11': resolution: {integrity: sha512-TSuf8HQ4Ayp9IKKB5v2yTdgKMa/I0bqbcACj0IZ46CSpzn/HKner/BKG7rM8tGOarEcyBd/VXnDhK5cJA/YMtw==} @@ -1405,6 +1414,15 @@ packages: resolution: {integrity: sha512-bsxD8WLS5lIj7aaoCx1YJkktqYj5vlBUE6HWzu2Q51ksrGJ0H737ECCKlFU7Yf8Br45z9t99frBp/J7kzbMPAg==} engines: {node: ^20.19.0 || ^22.13.0 || ^23.5.0 || >=24.0.0, npm: '>=10'} + '@floating-ui/core@1.8.0': + resolution: {integrity: sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ==} + + '@floating-ui/dom@1.7.6': + resolution: {integrity: sha512-9gZSAI5XM36880PPMm//9dfiEngYoC6Am2izES1FF406YFsjvyBMmeJ2g4SAju3xWwtuynNRFL2s9hgxpLI5SQ==} + + '@floating-ui/utils@0.2.12': + resolution: {integrity: sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==} + '@fontsource-variable/nunito@5.2.7': resolution: {integrity: sha512-2N8QhatkyKgSUbAGZO2FYLioxA32+RyI1EplVLawbpkGjUeui9Qg9VMrpkCaik1ydjFjfLV+kzQ0cGEsMrMenQ==} @@ -1626,6 +1644,32 @@ packages: '@kwsites/file-exists@1.1.1': resolution: {integrity: sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==} + '@livekit/components-core@0.12.14': + resolution: {integrity: sha512-6OKP/1Ok2fCZewDLKd3SzaTb7KvfZl/6hjggI+TgUdhp0Y7HBg3+tHA7YmhZRc0BO8wIyVy4VgTPn7qkLHt2nQ==} + engines: {node: '>=18'} + peerDependencies: + livekit-client: ^2.20.1 + tslib: ^2.6.2 + + '@livekit/components-react@2.9.23': + resolution: {integrity: sha512-clO+0g/u3YBpuOvnAjUSAJBH/o7w+RpUAseswjiSML9rHrXlTUhwBNm8LPk+qN5GOU3A6lzsNnFYVpUHUBVOkg==} + engines: {node: '>=18'} + peerDependencies: + '@livekit/krisp-noise-filter': ^0.2.12 || ^0.3.0 || ^0.4.0 + livekit-client: ^2.20.1 + react: '>=18' + react-dom: '>=18' + tslib: ^2.6.2 + peerDependenciesMeta: + '@livekit/krisp-noise-filter': + optional: true + + '@livekit/mutex@1.1.1': + resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==} + + '@livekit/protocol@1.50.4': + resolution: {integrity: sha512-L1uggNQAqyY21smQY8AllyOYbcv9Me9TaxwuLytL1R8ck9nbYPmQLNwEDi3pOFGAMa5F8I2nUi2Jc59W5awxlA==} + '@lottiefiles/dotlottie-react@0.12.3': resolution: {integrity: sha512-b0k0Lakj2hmhyIkwJSpKySh6xoelpgWRaijyrCb6fraOCnzuitKglVTtYs0VWf72rk+2aSmC/0IK8j8wruVfOw==} peerDependencies: @@ -3105,6 +3149,9 @@ packages: '@types/dockerode@4.0.1': resolution: {integrity: sha512-cmUpB+dPN955PxBEuXE3f6lKO1hHiIGYJA46IVF3BJpNsZGvtBDcRnlrHYHtOH/B6vtDOyl2kZ2ShAu3mgc27Q==} + '@types/dom-mediacapture-record@1.0.22': + resolution: {integrity: sha512-mUMZLK3NvwRLcAAT9qmcK+9p7tpU2FHdDsntR3YI4+GY88XrgG4XiE7u1Q2LAN2/FZOz/tdMDC3GQCR4T8nFuw==} + '@types/estree@0.0.39': resolution: {integrity: sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw==} @@ -4336,6 +4383,9 @@ packages: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true + jose@6.2.4: + resolution: {integrity: sha512-N8acGzVsQy6M/fjFcxtysNc4Q379TcM5dM/qKkNtsHFji88yANnXTr7BLeP75iPnFwBfQzM/jg2BZ9+HZrHCZA==} + jotai@2.20.1: resolution: {integrity: sha512-dnuKfU/GLi8B28RRMjQ3AfoN7kfzP8o41+AX2FmITZqEMY8PHnjABq+VkEooomLwYaGjda+pgy0yFSjaHX/ZPg==} engines: {node: '>=12.20.0'} @@ -4506,6 +4556,11 @@ packages: linkifyjs@4.3.3: resolution: {integrity: sha512-P8aEP5U/D1/IlTY2OeYsErdwh9bGuLE30NcXtKEjgdHcahveQoQwM2yZNsioQHsWFz0P7KKudisbrzCgR0sDHg==} + livekit-client@2.21.0: + resolution: {integrity: sha512-RBUhPkV/sl1nzl8lokVlK5uATPwn0AlsudCBZXissw/kDl9yz8ac4pNJ43iPpMVoHOeBYH/BZ3vUC1adqa/zFQ==} + peerDependencies: + '@types/dom-mediacapture-record': ^1 + locate-path@6.0.0: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} @@ -4522,6 +4577,10 @@ packages: lodash@4.18.1: resolution: {integrity: sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==} + loglevel@1.9.1: + resolution: {integrity: sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==} + engines: {node: '>= 0.6.0'} + loglevel@1.9.2: resolution: {integrity: sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==} engines: {node: '>= 0.6.0'} @@ -5045,6 +5104,9 @@ packages: engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} + safe-array-concat@1.1.4: resolution: {integrity: sha512-wtZlHyOje6OZTGqAoaDKxFkgRtkF9CnHAVnCHKfuj200wAgL+bSJhdsCD2l0Qx/2ekEXjPWcyKkfGb5CPboslg==} engines: {node: '>=0.4'} @@ -5076,10 +5138,17 @@ packages: scroll-into-view-if-needed@3.1.0: resolution: {integrity: sha512-49oNpRjWRvnU8NyGVmUaYG4jtTkNonFZI86MmGRDqBphEK2EXT9gdEUoQPZhuBM8yWHxCWbobltqYO5M4XrUvQ==} + sdp-transform@2.15.0: + resolution: {integrity: sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==} + hasBin: true + sdp-transform@3.0.0: resolution: {integrity: sha512-gfYVRGxjHkGF2NPeUWHw5u6T/KGFtS5/drPms73gaSuMaVHKCY3lpLnGDfswVQO0kddeePoti09AwhYP4zA8dQ==} hasBin: true + sdp@3.2.2: + resolution: {integrity: sha512-xZocWwfyp4hkbN4hLWxMjmv2Q8aNa9MhmOZ7L9aCZPT+dZsgRr6wZRrSYE3HTdyk/2pZKPSgqI7ns7Een1xMSA==} + semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true @@ -5425,6 +5494,9 @@ packages: resolution: {integrity: sha512-phPGCwqr2+Qo0fwniCE8e4pKnGu/yFb5nD5Y8bf0EEeiI5GklnACYA9GFy/DrAeRrKHXvHn+1SUsOWgJp6RO+g==} engines: {node: '>= 0.4'} + typed-emitter@2.1.0: + resolution: {integrity: sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==} + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -5503,6 +5575,12 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + usehooks-ts@3.1.1: + resolution: {integrity: sha512-I4diPp9Cq6ieSUH2wu+fDAVQO43xwtulo+fKEidHUwZPnYImbtkTjzIJYcDcJqxgmX31GVqNFURodvcgHcW0pA==} + engines: {node: '>=16.15.0'} + peerDependencies: + react: ^16.8.0 || ^17 || ^18 || ^19 || ^19.0.0-rc + util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -5713,6 +5791,10 @@ packages: resolution: {integrity: sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==} engines: {node: '>=20'} + webrtc-adapter@9.0.6: + resolution: {integrity: sha512-CHbl2ZQbxx164IgWRgzJno4hWtM4tFbRam1QfI3Yxhs3w/DvqluVxVWeXs3oL5/fbGkSNLKo0Ty5MgUWceNhog==} + engines: {node: '>=6.0.0', npm: '>=3.10.0'} + whatwg-mimetype@5.0.0: resolution: {integrity: sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==} engines: {node: '>=20'} @@ -6632,6 +6714,8 @@ snapshots: dependencies: css-tree: 3.2.1 + '@bufbuild/protobuf@1.10.1': {} + '@choochmeque/tauri-plugin-notifications-api@0.5.0-rc.11': dependencies: '@tauri-apps/api': 2.11.1 @@ -6915,6 +6999,17 @@ snapshots: '@faker-js/faker@10.5.0': {} + '@floating-ui/core@1.8.0': + dependencies: + '@floating-ui/utils': 0.2.12 + + '@floating-ui/dom@1.7.6': + dependencies: + '@floating-ui/core': 1.8.0 + '@floating-ui/utils': 0.2.12 + + '@floating-ui/utils@0.2.12': {} + '@fontsource-variable/nunito@5.2.7': {} '@fontsource/space-mono@5.2.9': {} @@ -7096,6 +7191,32 @@ snapshots: transitivePeerDependencies: - supports-color + '@livekit/components-core@0.12.14(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1)': + dependencies: + '@floating-ui/dom': 1.7.6 + livekit-client: 2.21.0(@types/dom-mediacapture-record@1.0.22) + loglevel: 1.9.1 + rxjs: 7.8.2 + tslib: 2.8.1 + + '@livekit/components-react@2.9.23(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tslib@2.8.1)': + dependencies: + '@livekit/components-core': 0.12.14(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1) + clsx: 2.1.1 + events: 3.3.0 + jose: 6.2.4 + livekit-client: 2.21.0(@types/dom-mediacapture-record@1.0.22) + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + tslib: 2.8.1 + usehooks-ts: 3.1.1(react@18.3.1) + + '@livekit/mutex@1.1.1': {} + + '@livekit/protocol@1.50.4': + dependencies: + '@bufbuild/protobuf': 1.10.1 + '@lottiefiles/dotlottie-react@0.12.3(react@18.3.1)': dependencies: '@lottiefiles/dotlottie-web': 0.40.1 @@ -8151,6 +8272,8 @@ snapshots: '@types/node': 24.13.2 '@types/ssh2': 1.15.5 + '@types/dom-mediacapture-record@1.0.22': {} + '@types/estree@0.0.39': {} '@types/estree@1.0.9': {} @@ -9554,6 +9677,8 @@ snapshots: jiti@2.7.0: {} + jose@6.2.4: {} + jotai@2.20.1(@babel/core@7.29.7)(@babel/template@7.29.7)(@types/react@18.3.31)(react@18.3.1): optionalDependencies: '@babel/core': 7.29.7 @@ -9699,6 +9824,19 @@ snapshots: linkifyjs@4.3.3: {} + livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22): + dependencies: + '@livekit/mutex': 1.1.1 + '@livekit/protocol': 1.50.4 + '@types/dom-mediacapture-record': 1.0.22 + events: 3.3.0 + jose: 6.2.4 + loglevel: 1.9.2 + sdp-transform: 2.15.0 + tslib: 2.8.1 + typed-emitter: 2.1.0 + webrtc-adapter: 9.0.6 + locate-path@6.0.0: dependencies: p-locate: 5.0.0 @@ -9711,6 +9849,8 @@ snapshots: lodash@4.18.1: {} + loglevel@1.9.1: {} + loglevel@1.9.2: {} long@5.3.2: {} @@ -10360,6 +10500,10 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.62.0 fsevents: 2.3.3 + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + safe-array-concat@1.1.4: dependencies: call-bind: 1.0.9 @@ -10397,8 +10541,12 @@ snapshots: dependencies: compute-scroll-into-view: 3.1.1 + sdp-transform@2.15.0: {} + sdp-transform@3.0.0: {} + sdp@3.2.2: {} + semver@6.3.1: {} semver@7.8.4: {} @@ -10875,6 +11023,10 @@ snapshots: possible-typed-array-names: 1.1.0 reflect.getprototypeof: 1.0.10 + typed-emitter@2.1.0: + optionalDependencies: + rxjs: 7.8.2 + typescript@5.9.3: {} ua-is-frozen@0.1.2: {} @@ -10937,6 +11089,11 @@ snapshots: dependencies: react: 18.3.1 + usehooks-ts@3.1.1(react@18.3.1): + dependencies: + lodash.debounce: 4.0.8 + react: 18.3.1 + util-deprecate@1.0.2: {} uuid@10.0.0: {} @@ -11090,6 +11247,10 @@ snapshots: webidl-conversions@8.0.1: {} + webrtc-adapter@9.0.6: + dependencies: + sdp: 3.2.2 + whatwg-mimetype@5.0.0: {} whatwg-url@16.0.1(@noble/hashes@2.2.0): diff --git a/scripts/download-ios-frameworks.js b/scripts/download-ios-frameworks.js new file mode 100644 index 0000000000..5c334905ca --- /dev/null +++ b/scripts/download-ios-frameworks.js @@ -0,0 +1,128 @@ +#!/usr/bin/env node +//MISE description="Download and verify the iOS XCFrameworks used by tauri-plugin-livekit-mobile" +// Downloads pinned iOS frameworks into src-tauri/Frameworks. +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import process from 'node:process'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { PrefixedLogger } from './utils/console-style.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const log = new PrefixedLogger('[ios-frameworks]'); + +const FRAMEWORKS_DIR = resolve(__dirname, '../src-tauri/Frameworks'); +const MANIFEST_PATH = join(FRAMEWORKS_DIR, '.manifest.json'); + +const FRAMEWORKS = [ + { + name: 'LiveKitWebRTC', + url: 'https://github.com/livekit/webrtc-xcframework/releases/download/144.7559.11/LiveKitWebRTC.xcframework.zip', + sha256: '07c5caf718058af3c528dcabd257298c40e5a8527e4fb9f47c48336ba5899853', + }, + { + name: 'RustLiveKitUniFFI', + url: 'https://github.com/livekit/livekit-uniffi-xcframework/releases/download/0.0.6/RustLiveKitUniFFI.xcframework.zip', + sha256: '0d3f2ce159a224c728f8b131068d53bbf9b13d968cda0edc68a6a2290f2651ed', + }, +]; + +function sha256Hex(buffer) { + return createHash('sha256').update(buffer).digest('hex'); +} + +async function readManifest() { + try { + return JSON.parse(await readFile(MANIFEST_PATH, 'utf8')); + } catch { + return {}; + } +} + +async function writeManifest(manifest) { + await writeFile(MANIFEST_PATH, `${JSON.stringify(manifest, null, 2)}\n`); +} + +function findExtractor() { + for (const tool of ['ditto', 'unzip']) { + if (!spawnSync(tool, ['--help'], { stdio: 'ignore' }).error) return tool; + } + throw new Error('Neither ditto nor unzip is available to extract XCFramework archives'); +} + +function extractZip(zipPath, destination) { + const tool = findExtractor(); + const args = + tool === 'ditto' ? ['-x', '-k', zipPath, destination] : ['-q', zipPath, '-d', destination]; + const result = spawnSync(tool, args, { stdio: 'inherit' }); + if (result.status !== 0) { + throw new Error(`${tool} failed to extract ${zipPath} (exit ${result.status})`); + } +} + +async function provision(framework, manifest) { + const { name, url, sha256 } = framework; + const target = join(FRAMEWORKS_DIR, `${name}.xcframework`); + + if (manifest[name] === sha256 && existsSync(join(target, 'Info.plist'))) { + log.info(`${name}.xcframework already installed (verified), skipping`); + return; + } + + log.info(`Downloading ${name} from ${url}`); + const response = await fetch(url, { redirect: 'follow' }); + if (!response.ok) { + throw new Error(`Failed to download ${name}: HTTP ${response.status} ${response.statusText}`); + } + const zipBuffer = Buffer.from(await response.arrayBuffer()); + + const actual = sha256Hex(zipBuffer); + if (actual !== sha256) { + throw new Error( + `Checksum mismatch for ${name}: expected ${sha256}, got ${actual}. Refusing to install.` + ); + } + log.info(`Verified SHA-256 for ${name}`); + + // Verify and unpack before replacing an existing framework. + const tempDir = await mkdtemp(join(FRAMEWORKS_DIR, '.tmp-')); + try { + const zipPath = join(tempDir, `${name}.zip`); + await writeFile(zipPath, zipBuffer); + extractZip(zipPath, tempDir); + + const extracted = join(tempDir, `${name}.xcframework`); + if (!existsSync(join(extracted, 'Info.plist'))) { + throw new Error( + `Downloaded archive for ${name} did not contain ${name}.xcframework; refusing to install.` + ); + } + + if (existsSync(target)) { + await rename(target, join(tempDir, '__replaced__')); + } + await rename(extracted, target); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + + manifest[name] = sha256; + await writeManifest(manifest); + log.info(`Installed ${name}.xcframework`); +} + +async function main() { + await mkdir(FRAMEWORKS_DIR, { recursive: true }); + const manifest = await readManifest(); + for (const framework of FRAMEWORKS) { + // oxlint-disable-next-line no-await-in-loop + await provision(framework, manifest); + } +} + +main().catch((error) => { + log.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/src-tauri/.gitignore b/src-tauri/.gitignore index 489be3277b..dd5680a87d 100644 --- a/src-tauri/.gitignore +++ b/src-tauri/.gitignore @@ -2,3 +2,6 @@ # will have compiled files and executables /target/ /gen/schemas + +# Downloaded by scripts/download-ios-frameworks.js (pinned, checksummed binaries) +/Frameworks/ diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index f58607f692..094d207c79 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -5424,6 +5424,7 @@ dependencies = [ "tauri-plugin-fs", "tauri-plugin-global-shortcut", "tauri-plugin-http", + "tauri-plugin-livekit-mobile", "tauri-plugin-log", "tauri-plugin-notifications", "tauri-plugin-opener", @@ -6730,6 +6731,20 @@ dependencies = [ "urlpattern", ] +[[package]] +name = "tauri-plugin-livekit-mobile" +version = "0.2.0" +source = "git+https://github.com/SableClient/tauri-plugin-livekit-mobile.git?rev=b59bfb369a06c2404ffcaeb6f1c2ccb9c0b57ad2#b59bfb369a06c2404ffcaeb6f1c2ccb9c0b57ad2" +dependencies = [ + "base64 0.22.1", + "serde", + "serde_json", + "tauri", + "tauri-plugin", + "thiserror 2.0.19", + "tokio", +] + [[package]] name = "tauri-plugin-log" version = "2.9.0" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index c8c27dac89..24d6309129 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -124,6 +124,7 @@ tauri-plugin-notifications = { git = "https://github.com/SableClient/tauri-plugi ] } tauri-plugin-edge-to-edge = { git = "https://github.com/SableClient/tauri-plugin-edge-to-edge.git", rev = "33c6116c27be28c06df5a9d02231ecc5fdeb93c5" } tauri-plugin-sharekit = { git = "https://github.com/Choochmeque/tauri-plugin-sharekit", rev = "9f2b4c5d8a4f0ab910d900ba234ed8bae0ab854b" } +tauri-plugin-livekit-mobile = { git = "https://github.com/SableClient/tauri-plugin-livekit-mobile.git", rev = "b59bfb369a06c2404ffcaeb6f1c2ccb9c0b57ad2" } [target.'cfg(target_os = "ios")'.dependencies] objc2 = "0.6" diff --git a/src-tauri/Info.plist b/src-tauri/Info.plist index 99b9ecb70b..e1425ba122 100644 --- a/src-tauri/Info.plist +++ b/src-tauri/Info.plist @@ -10,5 +10,10 @@ Sable needs camera access for video calls. NSMicrophoneUsageDescription Sable needs microphone access for voice and video calls. + UIBackgroundModes + + audio + voip + diff --git a/src-tauri/capabilities/mobile.json b/src-tauri/capabilities/mobile.json index e5500a57d6..1cb676dbde 100644 --- a/src-tauri/capabilities/mobile.json +++ b/src-tauri/capabilities/mobile.json @@ -7,6 +7,7 @@ "core:window:allow-set-badge-count", "deep-link:default", "edge-to-edge:default", + "livekit-mobile:default", "notifications:default", "sharekit:default" ] diff --git a/src-tauri/gen/android/app/build.gradle.kts b/src-tauri/gen/android/app/build.gradle.kts index 10397a7805..90abfc67a0 100644 --- a/src-tauri/gen/android/app/build.gradle.kts +++ b/src-tauri/gen/android/app/build.gradle.kts @@ -61,6 +61,11 @@ android { ) } } + compileOptions { + // tauri-plugin-livekit-mobile requires it: LiveKit/WebRTC call java.time + // APIs that only exist from API 26, and minSdk here is 24. + isCoreLibraryDesugaringEnabled = true + } kotlinOptions { jvmTarget = "1.8" } @@ -73,11 +78,24 @@ rust { rootDirRel = "../../../" } +// tink (via the unifiedpush connector) pulls protobuf-java, which collides with +// the protobuf-javalite livekit-android is generated against: its Timestamp +// extends GeneratedMessage rather than GeneratedMessageLite, so livekit's +// classes fail dex verification at runtime. tink-android shades its own +// protobuf and needs none, which leaves javalite as the only protobuf. +configurations.all { + resolutionStrategy.dependencySubstitution { + substitute(module("com.google.crypto.tink:tink")) + .using(module("com.google.crypto.tink:tink-android:1.18.0")) + } +} + dependencies { implementation("androidx.webkit:webkit:1.14.0") implementation("androidx.appcompat:appcompat:1.7.1") implementation("androidx.activity:activity-ktx:1.10.1") implementation("com.google.android.material:material:1.12.0") + coreLibraryDesugaring("com.android.tools:desugar_jdk_libs:2.1.5") testImplementation("junit:junit:4.13.2") androidTestImplementation("androidx.test.ext:junit:1.1.4") androidTestImplementation("androidx.test.espresso:espresso-core:3.5.0") diff --git a/src-tauri/gen/android/app/src/main/AndroidManifest.xml b/src-tauri/gen/android/app/src/main/AndroidManifest.xml index 7ada59c8d5..32f238399e 100644 --- a/src-tauri/gen/android/app/src/main/AndroidManifest.xml +++ b/src-tauri/gen/android/app/src/main/AndroidManifest.xml @@ -47,6 +47,7 @@ android:launchMode="singleTask" android:label="@string/main_activity_title" android:name="moe.sable.client.MainActivity" + android:supportsPictureInPicture="true" android:exported="true"> diff --git a/src-tauri/gen/android/build.gradle.kts b/src-tauri/gen/android/build.gradle.kts index a91af1aefb..d5b85e453b 100644 --- a/src-tauri/gen/android/build.gradle.kts +++ b/src-tauri/gen/android/build.gradle.kts @@ -14,6 +14,9 @@ allprojects { repositories { google() mavenCentral() + // livekit-android pulls com.github.davidliu:audioswitch, published only + // on JitPack. + maven { setUrl("https://jitpack.io") } } } diff --git a/src-tauri/ios-entitlements/sable.debug.entitlements b/src-tauri/ios-entitlements/sable.debug.entitlements new file mode 100644 index 0000000000..6631ffa6f2 --- /dev/null +++ b/src-tauri/ios-entitlements/sable.debug.entitlements @@ -0,0 +1,6 @@ + + + + + + diff --git a/src-tauri/ios-entitlements/sable.release.entitlements b/src-tauri/ios-entitlements/sable.release.entitlements new file mode 100644 index 0000000000..723ff878de --- /dev/null +++ b/src-tauri/ios-entitlements/sable.release.entitlements @@ -0,0 +1,12 @@ + + + + + aps-environment + development + com.apple.security.application-groups + + group.moe.sable.client + + + diff --git a/src-tauri/ios-entitlements/sable.share-extension.debug.entitlements b/src-tauri/ios-entitlements/sable.share-extension.debug.entitlements new file mode 100644 index 0000000000..6631ffa6f2 --- /dev/null +++ b/src-tauri/ios-entitlements/sable.share-extension.debug.entitlements @@ -0,0 +1,6 @@ + + + + + + diff --git a/src-tauri/ios-entitlements/sable.share-extension.release.entitlements b/src-tauri/ios-entitlements/sable.share-extension.release.entitlements new file mode 100644 index 0000000000..6707d84a11 --- /dev/null +++ b/src-tauri/ios-entitlements/sable.share-extension.release.entitlements @@ -0,0 +1,10 @@ + + + + + com.apple.security.application-groups + + group.moe.sable.client + + + diff --git a/src-tauri/ios-project.yml b/src-tauri/ios-project.yml index 298ff0cb6d..b35f746d85 100644 --- a/src-tauri/ios-project.yml +++ b/src-tauri/ios-project.yml @@ -60,9 +60,7 @@ targets: # APPLICATION_EXTENSION_API_ONLY must stay unset: NO fails Xcode's # validation, YES breaks UIApplication.open in ShareViewController. SKIP_INSTALL: YES - {{#if apple.development-team}} - DEVELOPMENT_TEAM: {{apple.development-team}} - {{/if}} + DEVELOPMENT_TEAM: PK66NJM372 {{app.name}}_iOS: type: application platform: iOS @@ -107,7 +105,6 @@ targets: - UIInterfaceOrientationLandscapeRight CFBundleShortVersionString: {{apple.bundle-version-short}} CFBundleVersion: "{{apple.bundle-version}}" - UIBackgroundModes: [audio, voip] NSPhotoLibraryAddUsageDescription: Sable saves images you download to your photo library. # Mirrors plugins.deep-link in tauri.conf.json: the plugin only patches # these in when cargo reruns its build script, so a fresh `tauri ios @@ -121,10 +118,6 @@ targets: {{this.key}}: {{this.value}}{{/each}} entitlements: path: {{app.name}}_iOS/{{app.name}}_iOS.entitlements - properties: - aps-environment: development - com.apple.security.application-groups: - - group.{{app.identifier}} scheme: environmentVariables: RUST_BACKTRACE: full diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 7fcfe176c5..9468df0ed8 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -355,6 +355,7 @@ pub fn run() { #[cfg(mobile)] let builder = builder .plugin(tauri_plugin_edge_to_edge::init()) + .plugin(tauri_plugin_livekit_mobile::init()) .plugin(tauri_plugin_sharekit::init()); #[cfg(target_os = "android")] diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 7fb809737f..bccd176b93 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -24,7 +24,13 @@ }, "iOS": { "template": "src-tauri/ios-project.yml", - "frameworks": ["AudioToolbox", "Photos"] + "minimumSystemVersion": "16.0", + "frameworks": [ + "AudioToolbox", + "Photos", + "Frameworks/LiveKitWebRTC.xcframework", + "Frameworks/RustLiveKitUniFFI.xcframework" + ] }, "linux": { "deb": { diff --git a/src-tauri/tauri.local.ios.conf.json b/src-tauri/tauri.local.ios.conf.json new file mode 100644 index 0000000000..97948f54d3 --- /dev/null +++ b/src-tauri/tauri.local.ios.conf.json @@ -0,0 +1,3 @@ +{ + "identifier": "moe.sable.client.local" +} diff --git a/src/app/generated/tauri/commands.ts b/src/app/generated/tauri/commands.ts index 1dd132bbba..33b711ba19 100644 --- a/src/app/generated/tauri/commands.ts +++ b/src/app/generated/tauri/commands.ts @@ -1,7 +1,7 @@ /** * Auto-generated TypeScript bindings for Tauri commands * Generated by tauri-typegen v0.5.0 - * Generated at: 2026-07-31T11:14:58.599659+00:00 + * Generated at: 2026-07-30T16:36:15.050428+00:00 * Generator: none * * Do not edit manually - regenerate using: cargo tauri-typegen generate diff --git a/src/app/generated/tauri/events.ts b/src/app/generated/tauri/events.ts index 0260d39ed0..1d611b14e6 100644 --- a/src/app/generated/tauri/events.ts +++ b/src/app/generated/tauri/events.ts @@ -1,7 +1,7 @@ /** * Auto-generated TypeScript bindings for Tauri commands * Generated by tauri-typegen v0.5.0 - * Generated at: 2026-07-31T11:14:58.602356+00:00 + * Generated at: 2026-07-30T16:36:15.051835+00:00 * Generator: none * * Do not edit manually - regenerate using: cargo tauri-typegen generate diff --git a/src/app/generated/tauri/index.ts b/src/app/generated/tauri/index.ts index 727bb5d08e..e62a83936a 100644 --- a/src/app/generated/tauri/index.ts +++ b/src/app/generated/tauri/index.ts @@ -1,7 +1,7 @@ /** * Auto-generated TypeScript bindings for Tauri commands * Generated by tauri-typegen v0.5.0 - * Generated at: 2026-07-31T11:14:58.603222+00:00 + * Generated at: 2026-07-30T16:36:15.052085+00:00 * Generator: none * * Do not edit manually - regenerate using: cargo tauri-typegen generate diff --git a/src/app/generated/tauri/types.ts b/src/app/generated/tauri/types.ts index 5c41032aff..9a11ec402f 100644 --- a/src/app/generated/tauri/types.ts +++ b/src/app/generated/tauri/types.ts @@ -1,7 +1,7 @@ /** * Auto-generated TypeScript bindings for Tauri commands * Generated by tauri-typegen v0.5.0 - * Generated at: 2026-07-31T11:14:58.596290+00:00 + * Generated at: 2026-07-30T16:36:15.048780+00:00 * Generator: none * * Do not edit manually - regenerate using: cargo tauri-typegen generate From ed5ea68889ba5b835160e14f8bebc7e62c684607 Mon Sep 17 00:00:00 2001 From: Erwan Leboucher Date: Sat, 1 Aug 2026 23:33:19 +0200 Subject: [PATCH 2/6] feat(call): discover LiveKit transports and subscribe the call room --- src/app/cs-api.ts | 23 ++++++++++++++----- src/client/slidingSync.test.ts | 38 ++++++++++++++++++++++++++++++ src/client/slidingSync.ts | 42 ++++++++++++++++++++++++++++++---- src/types/matrix-sdk.ts | 3 +++ src/types/matrix/common.ts | 3 +-- 5 files changed, 96 insertions(+), 13 deletions(-) diff --git a/src/app/cs-api.ts b/src/app/cs-api.ts index ec985b97b4..386ca3d19b 100644 --- a/src/app/cs-api.ts +++ b/src/app/cs-api.ts @@ -1,4 +1,5 @@ import to from 'await-to-js'; +import type { LivekitTransportConfig } from '$types/matrix-sdk'; import { trimTrailingSlash } from './utils/common'; export enum AutoDiscoveryAction { @@ -22,12 +23,22 @@ export type AutoDiscoveryInfo = Record & { account?: string; issuer?: string; }; - 'org.matrix.msc4143.rtc_foci'?: [ - { - livekit_service_url: string; - type: 'livekit'; - }, - ]; + 'org.matrix.msc4143.rtc_foci'?: LivekitTransportConfig[]; +}; + +export const getLivekitTransports = ( + discovery: Pick | undefined +): LivekitTransportConfig[] => { + const foci = discovery?.['org.matrix.msc4143.rtc_foci']; + if (!Array.isArray(foci)) return []; + + return foci.filter( + (focus): focus is LivekitTransportConfig => + typeof focus === 'object' && + focus !== null && + focus.type === 'livekit' && + typeof focus.livekit_service_url === 'string' + ); }; export const autoDiscovery = async ( diff --git a/src/client/slidingSync.test.ts b/src/client/slidingSync.test.ts index 3324f4b002..463cc431b9 100644 --- a/src/client/slidingSync.test.ts +++ b/src/client/slidingSync.test.ts @@ -500,6 +500,39 @@ describe('SlidingSyncManager initial request', () => { }); describe('SlidingSyncManager room subscription coordination', () => { + it('uses the full-membership subscription for the lifetime of a call', () => { + const manager = makeManager(makeMockMx()); + const roomId = '!call:example.com'; + + const subscription = manager.subscribeToCallRoom(roomId); + + expect(mocks.slidingSyncInstance.useCustomSubscription).toHaveBeenLastCalledWith( + roomId, + 'call_room' + ); + expect(mocks.slidingSyncInstance.modifyRoomSubscriptions).toHaveBeenLastCalledWith( + new Set([roomId]) + ); + + subscription(); + + expect(mocks.slidingSyncInstance.modifyRoomSubscriptions).toHaveBeenLastCalledWith(new Set()); + }); + + it('restores an active room subscription after a call ends', () => { + const manager = makeManager(makeMockMx()); + const roomId = '!call:example.com'; + manager.subscribeToRoom(roomId); + + const subscription = manager.subscribeToCallRoom(roomId); + subscription(); + + expect(mocks.slidingSyncInstance.useCustomSubscription).toHaveBeenLastCalledWith( + roomId, + 'active_room' + ); + }); + it('replaces route subscriptions atomically without cycling the retained space', () => { const manager = makeManager(makeMockMx()); const spaceId = '!space:example.com'; @@ -701,12 +734,17 @@ describe('SlidingSyncManager room subscription coordination', () => { { timeline_limit: number; required_state: [string, string][] }, ][]; const activeRoom = calls.find(([name]) => name === 'active_room'); + const callRoom = calls.find(([name]) => name === 'call_room'); const sidebarRoom = calls.find(([name]) => name === 'sidebar_room'); expect(activeRoom).toBeDefined(); expect(activeRoom![1].timeline_limit).toBe(50); expect(activeRoom![1].required_state).toContainEqual([EventType.RoomMember, '$LAZY']); + expect(callRoom).toBeDefined(); + expect(callRoom![1].timeline_limit).toBe(50); + expect(callRoom![1].required_state).toContainEqual([EventType.RoomMember, '*']); + expect(sidebarRoom).toBeDefined(); expect(sidebarRoom![1].timeline_limit).toBe(1); }); diff --git a/src/client/slidingSync.ts b/src/client/slidingSync.ts index 3eec08431e..afb5701497 100644 --- a/src/client/slidingSync.ts +++ b/src/client/slidingSync.ts @@ -43,6 +43,7 @@ const DEFAULT_POLL_TIMEOUT_MS = 45000; const LIST_SORT_ORDER = ['by_recency', 'by_name']; const ACTIVE_ROOM_SUBSCRIPTION_KEY = 'active_room'; +const CALL_ROOM_SUBSCRIPTION_KEY = 'call_room'; const SIDEBAR_ROOM_SUBSCRIPTION_KEY = 'sidebar_room'; const SPACE_SUBSCRIPTION_KEY = 'space'; const IMAGE_PACK_SUBSCRIPTION_KEY = 'image_packs'; @@ -88,6 +89,8 @@ export type SlidingSyncDiagnostics = { export type HydrationProgress = { loadedRooms: number; totalRooms: number }; +export type CallRoomSubscription = () => void; + const clampPositive = (value: number | undefined, fallback: number): number => { if (typeof value !== 'number' || Number.isNaN(value) || value <= 0) return fallback; return Math.round(value); @@ -170,8 +173,8 @@ const SPACE_REQUIRED_STATE: MSC3575RoomSubscription['required_state'] = [ [EventType.RoomEncryption, ''], [EventType.RoomTombstone, ''], [CustomStateEvent.RoomBanner, ''], - [EventType.SpaceChild, MSC3575_WILDCARD], - [EventType.SpaceParent, MSC3575_WILDCARD], + ['m.space.child', MSC3575_WILDCARD], + ['m.space.parent', MSC3575_WILDCARD], ]; const ACTIVE_ROOM_REQUIRED_STATE: MSC3575RoomSubscription['required_state'] = [ @@ -192,8 +195,8 @@ const ACTIVE_ROOM_REQUIRED_STATE: MSC3575RoomSubscription['required_state'] = [ [EventType.RoomThirdPartyInvite, MSC3575_WILDCARD], [EventType.RoomMember, MSC3575_STATE_KEY_ME], [EventType.RoomMember, MSC3575_STATE_KEY_LAZY], - [EventType.SpaceChild, MSC3575_WILDCARD], - [EventType.SpaceParent, MSC3575_WILDCARD], + ['m.space.child', MSC3575_WILDCARD], + ['m.space.parent', MSC3575_WILDCARD], [EventType.GroupCallPrefix, ''], [EventType.GroupCallMemberPrefix, MSC3575_WILDCARD], ...Object.values(CustomStateEvent).map((type) => [type, MSC3575_WILDCARD] as [string, string]), @@ -209,6 +212,13 @@ const buildUnencryptedSubscription = (timelineLimit: number): MSC3575RoomSubscri required_state: ACTIVE_ROOM_REQUIRED_STATE, }); +const buildCallRoomSubscription = (timelineLimit: number): MSC3575RoomSubscription => ({ + timeline_limit: timelineLimit, + // MatrixRTC ignores memberships for users that are absent from the room + // roster. Unlike a timeline, calls need every membership continuously. + required_state: [...ACTIVE_ROOM_REQUIRED_STATE, [EventType.RoomMember, MSC3575_WILDCARD]], +}); + const IMAGE_PACK_REQUIRED_STATE: MSC3575RoomSubscription['required_state'] = [ [CustomStateEvent.ImagePack, MSC3575_WILDCARD], [CustomStateEvent.PoniesRoomEmotes, MSC3575_WILDCARD], @@ -302,6 +312,8 @@ export class SlidingSyncManager { private readonly activeRoomSubscriptions = new Set(); + private readonly callRoomSubscriptions = new Set(); + /** * Rooms joined locally via reconcileRoomMembership(Join) but not yet confirmed * joined by a sliding-sync response. The SDK reverts these to "invite" when the @@ -445,6 +457,10 @@ export class SlidingSyncManager { ACTIVE_ROOM_SUBSCRIPTION_KEY, buildUnencryptedSubscription(roomTimelineLimit) ); + this.slidingSync.addCustomSubscription( + CALL_ROOM_SUBSCRIPTION_KEY, + buildCallRoomSubscription(roomTimelineLimit) + ); this.slidingSync.addCustomSubscription( SIDEBAR_ROOM_SUBSCRIPTION_KEY, buildSidebarRoomSubscription() @@ -1309,6 +1325,7 @@ export class SlidingSyncManager { private syncRoomSubscriptions(): void { const desiredSubscriptions = new Set([ + ...this.callRoomSubscriptions, ...this.activeRoomSubscriptions, ...this.sidebarRoomSubscriptions, ...this.spaceSubscriptions, @@ -1316,7 +1333,9 @@ export class SlidingSyncManager { ]); desiredSubscriptions.forEach((roomId) => { - if (this.activeRoomSubscriptions.has(roomId)) { + if (this.callRoomSubscriptions.has(roomId)) { + this.slidingSync.useCustomSubscription(roomId, CALL_ROOM_SUBSCRIPTION_KEY); + } else if (this.activeRoomSubscriptions.has(roomId)) { this.slidingSync.useCustomSubscription(roomId, ACTIVE_ROOM_SUBSCRIPTION_KEY); } else if (this.sidebarRoomSubscriptions.has(roomId)) { this.slidingSync.useCustomSubscription(roomId, SIDEBAR_ROOM_SUBSCRIPTION_KEY); @@ -1517,6 +1536,19 @@ export class SlidingSyncManager { this.reportActiveSubscriptionCount(); } + public subscribeToCallRoom(roomId: string): CallRoomSubscription { + if (this.disposed) return () => undefined; + + this.callRoomSubscriptions.add(roomId); + this.syncRoomSubscriptions(); + return () => this.unsubscribeFromCallRoom(roomId); + } + + public unsubscribeFromCallRoom(roomId: string): void { + if (this.disposed || !this.callRoomSubscriptions.delete(roomId)) return; + this.syncRoomSubscriptions(); + } + public unsubscribeFromRoom(roomId: string): void { if (this.disposed || !this.removeActiveRoomSubscription(roomId)) return; this.syncRoomSubscriptions(); diff --git a/src/types/matrix-sdk.ts b/src/types/matrix-sdk.ts index f1bea4e091..7c2ec06531 100644 --- a/src/types/matrix-sdk.ts +++ b/src/types/matrix-sdk.ts @@ -55,5 +55,8 @@ export * from 'matrix-js-sdk/lib/common-crypto/CryptoBackend'; export * from 'matrix-js-sdk/lib/matrixrtc/CallMembership'; export * from 'matrix-js-sdk/lib/matrixrtc/MatrixRTCSession'; export * from 'matrix-js-sdk/lib/matrixrtc/MatrixRTCSessionManager'; +export { type CallMembershipIdentityParts } from 'matrix-js-sdk/lib/matrixrtc/EncryptionManager'; +export { type LivekitTransportConfig } from 'matrix-js-sdk/lib/matrixrtc/LivekitTransport'; +export { type Transport } from 'matrix-js-sdk/lib/matrixrtc/types'; export * from 'matrix-js-sdk/lib/models/thread'; diff --git a/src/types/matrix/common.ts b/src/types/matrix/common.ts index c900deea01..acef5ed083 100644 --- a/src/types/matrix/common.ts +++ b/src/types/matrix/common.ts @@ -2,7 +2,6 @@ import type { SerializableMap } from '$types/wrapper/SerializableMap'; import type { SerializableSet } from '$types/wrapper/SerializableSet'; import type { EncryptedAttachmentInfo } from 'browser-encrypt-attachment'; import type { MsgType } from '$types/matrix-sdk'; -import type { RelationType } from '$types/matrix-sdk'; import type * as prefix from '$unstable/prefixes'; export type IImageInfo = { @@ -14,7 +13,7 @@ export type IImageInfo = { }; export type MatrixRelatesTo = { - rel_type: RelationType.Annotation; + rel_type: 'm.annotation'; event_id: string; key?: string; }; From 750f298590b48bfc5da7c5d1db1916f73c2e25ce Mon Sep 17 00:00:00 2001 From: Erwan Leboucher Date: Sat, 1 Aug 2026 23:33:19 +0200 Subject: [PATCH 3/6] feat(call): join MatrixRTC and distribute media keys --- src/app/features/call/callKeyPipeline.test.ts | 204 +++++++ src/app/features/call/callKeyPipeline.ts | 138 +++++ src/app/features/call/callProtocol.test.ts | 58 ++ src/app/features/call/callProtocol.ts | 34 ++ src/app/features/call/callSession.test.ts | 100 ++++ src/app/features/call/callSession.ts | 84 +++ src/app/features/call/livekitCallIdentity.ts | 29 + .../call/livekitMatrixKeyProvider.test.ts | 227 ++++++++ .../features/call/livekitMatrixKeyProvider.ts | 159 ++++++ .../features/call/livekitProvisioning.test.ts | 197 +++++++ src/app/features/call/livekitProvisioning.ts | 114 ++++ .../call/matrixRtcCallLifecycle.test.ts | 508 ++++++++++++++++++ .../features/call/matrixRtcCallLifecycle.ts | 310 +++++++++++ 13 files changed, 2162 insertions(+) create mode 100644 src/app/features/call/callKeyPipeline.test.ts create mode 100644 src/app/features/call/callKeyPipeline.ts create mode 100644 src/app/features/call/callProtocol.test.ts create mode 100644 src/app/features/call/callProtocol.ts create mode 100644 src/app/features/call/callSession.test.ts create mode 100644 src/app/features/call/callSession.ts create mode 100644 src/app/features/call/livekitCallIdentity.ts create mode 100644 src/app/features/call/livekitMatrixKeyProvider.test.ts create mode 100644 src/app/features/call/livekitMatrixKeyProvider.ts create mode 100644 src/app/features/call/livekitProvisioning.test.ts create mode 100644 src/app/features/call/livekitProvisioning.ts create mode 100644 src/app/features/call/matrixRtcCallLifecycle.test.ts create mode 100644 src/app/features/call/matrixRtcCallLifecycle.ts diff --git a/src/app/features/call/callKeyPipeline.test.ts b/src/app/features/call/callKeyPipeline.test.ts new file mode 100644 index 0000000000..216d5bf897 --- /dev/null +++ b/src/app/features/call/callKeyPipeline.test.ts @@ -0,0 +1,204 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + MatrixRTCSessionEvent, + type CallMembershipIdentityParts, + type MatrixRTCSession, +} from '$types/matrix-sdk'; +import { + createCallKeyPipeline, + OWN_KEY_UNAVAILABLE_ERROR, + OWN_KEY_WAIT_CANCELLED_ERROR, + ownKeyWaitTimeoutMs, + type CallKeyListener, +} from './callKeyPipeline'; + +type EncryptionKeyHandler = ( + key: Uint8Array, + encryptionKeyIndex: number, + membership: CallMembershipIdentityParts, + rtcBackendIdentity: string +) => void; + +const localIdentity = { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }; +const localParts = localIdentity as CallMembershipIdentityParts; +const remoteParts = { + userId: '@bob:example.org', + deviceId: 'BOBDEVICE', +} as CallMembershipIdentityParts; + +const makeSession = () => { + const handlers = new Map(); + const session = { + on: vi.fn<(event: MatrixRTCSessionEvent, handler: EncryptionKeyHandler) => void>( + (event, handler) => { + handlers.set(event, handler); + } + ), + off: vi.fn<(event: MatrixRTCSessionEvent, handler: EncryptionKeyHandler) => void>( + (event, handler) => { + if (handlers.get(event) === handler) handlers.delete(event); + } + ), + reemitEncryptionKeys: vi.fn<() => void>(), + } as unknown as MatrixRTCSession; + return { + session, + handlers, + emitKey: ( + key: number[], + keyIndex: number, + identity: string, + parts: CallMembershipIdentityParts = remoteParts + ) => + handlers.get(MatrixRTCSessionEvent.EncryptionKeyChanged)?.( + new Uint8Array(key) as Uint8Array, + keyIndex, + parts, + identity + ), + }; +}; + +describe('call key pipeline', () => { + beforeEach(() => { + vi.useRealTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('subscribes to key changes and re-emits on attach, unsubscribes on detach', () => { + const { session, handlers } = makeSession(); + const pipeline = createCallKeyPipeline(); + + pipeline.attach(session, localIdentity); + + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.EncryptionKeyChanged, + expect.any(Function) + ); + expect(session.reemitEncryptionKeys).toHaveBeenCalled(); + + pipeline.detach(); + expect(session.off).toHaveBeenCalledWith( + MatrixRTCSessionEvent.EncryptionKeyChanged, + expect.any(Function) + ); + expect(handlers.has(MatrixRTCSessionEvent.EncryptionKeyChanged)).toBe(false); + }); + + it('caches the latest raw key per identity', () => { + const { session, emitKey } = makeSession(); + const pipeline = createCallKeyPipeline(); + pipeline.attach(session, localIdentity); + + emitKey([1, 2, 3, 4], 0, 'backend-a'); + emitKey([0, 1, 2], 2, 'backend-b'); + + expect(pipeline.getKeys()).toEqual([ + { identity: 'backend-a', keyIndex: 0, key: new Uint8Array([1, 2, 3, 4]) }, + { identity: 'backend-b', keyIndex: 2, key: new Uint8Array([0, 1, 2]) }, + ]); + }); + + it('keeps the latest key per identity, including after a peer restarts its indices', () => { + const { session, emitKey } = makeSession(); + const pipeline = createCallKeyPipeline(); + pipeline.attach(session, localIdentity); + + emitKey([1, 2, 3, 4], 2, 'backend-a'); + // A peer that rejoins starts a fresh outbound session back at a low index, + // so a lower index carries a genuinely new key rather than a stale one. + emitKey([9, 9], 1, 'backend-a'); + // The same index can also carry new material after a rotation. + emitKey([5, 6], 2, 'backend-a'); + + expect(pipeline.getKeys()).toEqual([ + { identity: 'backend-a', keyIndex: 2, key: new Uint8Array([5, 6]) }, + ]); + }); + + it('forwards every key it is given, since an index is a key-ring slot', () => { + const { session, emitKey } = makeSession(); + const pipeline = createCallKeyPipeline(); + pipeline.attach(session, localIdentity); + const onKey = vi.fn(); + pipeline.setOnKey(onKey); + + // LiveKit stores keys per participant at `cryptoKeyRing[index % keyringSize]` + // and reads the index out of the frame, so the receiver has to hold whatever + // the sender last wrote there. Filtering here would strand frames. + emitKey([1, 2, 3, 4], 0, 'backend-a'); + emitKey([1, 2, 3, 4], 0, 'backend-a'); + emitKey([7, 7, 7], 1, 'backend-a'); + + expect(onKey).toHaveBeenCalledTimes(3); + expect(onKey).toHaveBeenLastCalledWith( + { identity: 'backend-a', keyIndex: 1, key: new Uint8Array([7, 7, 7]) }, + false + ); + }); + + it('keeps a listener registered before attach, which is when keys are re-emitted', () => { + const { session, emitKey } = makeSession(); + const pipeline = createCallKeyPipeline(); + const onKey = vi.fn(); + + pipeline.setOnKey(onKey); + pipeline.attach(session, localIdentity); + emitKey([1], 0, 'own-backend', localParts); + + expect(onKey).toHaveBeenCalledWith( + { identity: 'own-backend', keyIndex: 0, key: new Uint8Array([1]) }, + true + ); + }); + + it('resolves the own-key wait on the key whose membership is ours', async () => { + const { session, emitKey } = makeSession(); + const pipeline = createCallKeyPipeline(); + pipeline.attach(session, localIdentity); + + const wait = pipeline.waitForOwnKey(); + emitKey([1], 0, 'other-backend'); + + emitKey([2], 0, 'own-backend', localParts); + await expect(wait).resolves.toBeUndefined(); + }); + + it('resolves the own-key wait for a key that arrived before the wait started', async () => { + const { session, emitKey } = makeSession(); + const pipeline = createCallKeyPipeline(); + pipeline.attach(session, localIdentity); + emitKey([2], 0, 'own-backend', localParts); + + await expect(pipeline.waitForOwnKey()).resolves.toBeUndefined(); + }); + + it('rejects the own-key wait after the timeout', async () => { + vi.useFakeTimers(); + const { session } = makeSession(); + const pipeline = createCallKeyPipeline(); + pipeline.attach(session, localIdentity); + + const waitError = pipeline.waitForOwnKey().catch((error: unknown) => error); + await vi.advanceTimersByTimeAsync(ownKeyWaitTimeoutMs); + const timeoutError = await waitError; + expect(timeoutError).toBeInstanceOf(Error); + expect((timeoutError as Error).message).toBe(OWN_KEY_UNAVAILABLE_ERROR); + }); + + it('rejects a pending own-key wait on detach', async () => { + const { session } = makeSession(); + const pipeline = createCallKeyPipeline(); + pipeline.attach(session, localIdentity); + + const waitError = pipeline.waitForOwnKey().catch((error: unknown) => error); + pipeline.detach(); + const detachError = await waitError; + expect(detachError).toBeInstanceOf(Error); + expect((detachError as Error).message).toBe(OWN_KEY_WAIT_CANCELLED_ERROR); + expect(pipeline.getKeys()).toEqual([]); + }); +}); diff --git a/src/app/features/call/callKeyPipeline.ts b/src/app/features/call/callKeyPipeline.ts new file mode 100644 index 0000000000..c43544156f --- /dev/null +++ b/src/app/features/call/callKeyPipeline.ts @@ -0,0 +1,138 @@ +import { + MatrixRTCSessionEvent, + type CallMembershipIdentityParts, + type MatrixRTCSession, +} from '$types/matrix-sdk'; +import type { CallEncryptionKey } from './callTransport'; +import type { LocalCallIdentity } from './livekitCallIdentity'; +import { createDebugLogger } from '$utils/debugLogger'; + +const debugLog = createDebugLogger('callKeyPipeline'); + +export const ownKeyWaitTimeoutMs = 10_000; + +export const OWN_KEY_UNAVAILABLE_ERROR = 'Call own encryption key unavailable'; +export const OWN_KEY_WAIT_CANCELLED_ERROR = 'Call own key wait cancelled'; + +/** `own` marks the key our own membership published, the one we encrypt with. */ +export type CallKeyListener = (key: CallEncryptionKey, own: boolean) => void; + +export type CallKeyPipeline = { + attach: (session: MatrixRTCSession, localIdentity: LocalCallIdentity) => void; + detach: () => void; + waitForOwnKey: () => Promise; + getKeys: () => CallEncryptionKey[]; + setOnKey: (listener: CallKeyListener | undefined) => void; +}; + +/** + * The one consumer of `EncryptionKeyChanged`, whatever transport carries the + * media. It only ever runs for an encrypted room: MSC4143 makes MatrixRTC + * encryption REQUIRED there and forbids it everywhere else. + */ +export const createCallKeyPipeline = (): CallKeyPipeline => { + let rtcSession: MatrixRTCSession | undefined; + const keys = new Map(); + let localIdentity: LocalCallIdentity | null = null; + let localOutboundIdentity: string | null = null; + let onKey: CallKeyListener | undefined; + + let waitResolve: (() => void) | undefined; + let waitReject: ((error: Error) => void) | undefined; + let waitTimeout: ReturnType | undefined; + + const settleWait = (settle: () => void): void => { + waitResolve = undefined; + waitReject = undefined; + if (waitTimeout !== undefined) clearTimeout(waitTimeout); + waitTimeout = undefined; + settle(); + }; + + const hasOwnKey = (): boolean => + localOutboundIdentity !== null && keys.has(localOutboundIdentity); + + const maybeResolveOwnKey = (): void => { + const resolve = waitResolve; + if (resolve && hasOwnKey()) settleWait(resolve); + }; + + const onEncryptionKeyChanged = ( + encryptionKey: Uint8Array, + encryptionKeyIndex: number, + membershipParts: CallMembershipIdentityParts, + rtcBackendIdentity: string + ): void => { + debugLog.debug( + 'call', + `key changed identity=${rtcBackendIdentity} index=${encryptionKeyIndex} ownIdentity=${localOutboundIdentity ?? 'unset'}` + ); + const own = + membershipParts.userId === localIdentity?.userId && + membershipParts.deviceId === localIdentity?.deviceId; + if (own) localOutboundIdentity = rtcBackendIdentity; + // Every key is forwarded as it arrives. A key index is a slot in LiveKit's + // per-participant key ring (`cryptoKeyRing[keyIndex % keyringSize]`) and the + // sender writes it into the frame, so the receiver must simply hold whatever + // the sender last put there. Filtering on the index would strand frames: the + // SDK reuses indices modulo 256 and restarts at 0 whenever a peer rejoins. + const entry: CallEncryptionKey = { + identity: rtcBackendIdentity, + keyIndex: encryptionKeyIndex, + key: encryptionKey, + }; + keys.set(rtcBackendIdentity, entry); + onKey?.(entry, own); + maybeResolveOwnKey(); + }; + + const reset = (): void => { + if (rtcSession) { + rtcSession.off(MatrixRTCSessionEvent.EncryptionKeyChanged, onEncryptionKeyChanged); + rtcSession = undefined; + } + keys.clear(); + localIdentity = null; + localOutboundIdentity = null; + const reject = waitReject; + if (reject) settleWait(() => reject(new Error(OWN_KEY_WAIT_CANCELLED_ERROR))); + }; + + const detach = (): void => { + reset(); + onKey = undefined; + }; + + // The listener survives a re-attach: `reemitEncryptionKeys` fires during the + // attach, so a caller that set it first must still receive those keys. + const attach = (session: MatrixRTCSession, identity: LocalCallIdentity): void => { + reset(); + rtcSession = session; + localIdentity = identity; + session.on(MatrixRTCSessionEvent.EncryptionKeyChanged, onEncryptionKeyChanged); + session.reemitEncryptionKeys(); + }; + + const waitForOwnKey = (): Promise => { + if (hasOwnKey()) return Promise.resolve(); + return new Promise((resolve, reject) => { + waitResolve = resolve; + waitReject = reject; + waitTimeout = setTimeout( + () => settleWait(() => reject(new Error(OWN_KEY_UNAVAILABLE_ERROR))), + ownKeyWaitTimeoutMs + ); + maybeResolveOwnKey(); + }); + }; + + return { + attach, + detach, + waitForOwnKey, + getKeys: () => [...keys.values()], + setOnKey: (listener) => { + onKey = listener; + }, + }; +}; diff --git a/src/app/features/call/callProtocol.test.ts b/src/app/features/call/callProtocol.test.ts new file mode 100644 index 0000000000..6e73130df7 --- /dev/null +++ b/src/app/features/call/callProtocol.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; +import type { IOpenIDToken } from '$types/matrix-sdk'; +import { advertiseCallTransport, buildProvisioningRequest, callMemberId } from './callProtocol'; + +const openidToken: IOpenIDToken = { + access_token: 'openid-secret', + token_type: 'Bearer', + matrix_server_name: 'example.org', + expires_in: 3600, +}; + +describe('callProtocol', () => { + it('derives the legacy LiveKit identity', () => { + expect(callMemberId('@alice:example.org', 'DEVICE')).toBe('@alice:example.org:DEVICE'); + }); + + it('builds the legacy provisioning request', () => { + const request = buildProvisioningRequest({ + serviceUrl: 'https://sfu.example', + roomId: '!room:example.org', + deviceId: 'DEVICE', + openidToken, + }); + + expect(request.url).toBe('https://sfu.example/sfu/get'); + expect(request.body).toEqual({ + room: '!room:example.org', + openid_token: openidToken, + device_id: 'DEVICE', + }); + }); + + it('advertises the transport with the alias legacy Element Call reads', () => { + expect( + advertiseCallTransport( + { type: 'livekit', livekit_service_url: 'https://sfu.example' }, + '!room:example.org' + ) + ).toEqual({ + type: 'livekit', + livekit_service_url: 'https://sfu.example', + livekit_alias: '!room:example.org', + }); + }); + + it('lets the transport keep an alias it already advertises', () => { + expect( + advertiseCallTransport( + { + type: 'livekit', + livekit_service_url: 'https://sfu.example', + livekit_alias: '!other:example.org', + }, + '!room:example.org' + ).livekit_alias + ).toBe('!other:example.org'); + }); +}); diff --git a/src/app/features/call/callProtocol.ts b/src/app/features/call/callProtocol.ts new file mode 100644 index 0000000000..633c50d53e --- /dev/null +++ b/src/app/features/call/callProtocol.ts @@ -0,0 +1,34 @@ +import type { IOpenIDToken, LivekitTransportConfig } from '$types/matrix-sdk'; + +export const callMemberId = (userId: string, deviceId: string): string => `${userId}:${deviceId}`; + +/** + * Element Call builds that predate Matrix 2.0 read `livekit_alias` off the + * advertised transport, so `legacy-livekit` keeps carrying it. + */ +export const advertiseCallTransport = ( + transport: LivekitTransportConfig, + roomId: string +): LivekitTransportConfig => ({ livekit_alias: roomId, ...transport }); + +export type CallProvisioningRequest = { + url: string; + body: { room: string; openid_token: IOpenIDToken; device_id: string }; +}; + +export type CallProvisioningInputs = { + serviceUrl: string; + roomId: string; + deviceId: string; + openidToken: IOpenIDToken; +}; + +export const buildProvisioningRequest = ({ + serviceUrl, + roomId, + deviceId, + openidToken, +}: CallProvisioningInputs): CallProvisioningRequest => ({ + url: `${serviceUrl}/sfu/get`, + body: { room: roomId, openid_token: openidToken, device_id: deviceId }, +}); diff --git a/src/app/features/call/callSession.test.ts b/src/app/features/call/callSession.test.ts new file mode 100644 index 0000000000..a3430ffb3a --- /dev/null +++ b/src/app/features/call/callSession.test.ts @@ -0,0 +1,100 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { MatrixClient, MatrixRTCSession, Room } from '$types/matrix-sdk'; +import type { + joinAndProvisionMatrixRTC as joinAndProvision, + leaveMatrixRTCOnPageHide as leaveOnPageHide, +} from './matrixRtcCallLifecycle'; +import { + callEncryptsMedia, + createCallSessionHandles, + joinCallSession, + type CallSessionJoinOptions, +} from './callSession'; + +const removePageHide = vi.fn<() => void>(); + +vi.mock('./matrixRtcCallLifecycle', () => ({ + joinAndProvisionMatrixRTC: vi.fn(async () => ({ + ownMembership: undefined, + provisioned: { url: 'wss://livekit.example', jwt: 'jwt' }, + })), + leaveMatrixRTCOnPageHide: vi.fn(() => removePageHide), +})); + +const { joinAndProvisionMatrixRTC } = await import('./matrixRtcCallLifecycle'); +const joinMock = vi.mocked(joinAndProvisionMatrixRTC); + +const mx = {} as MatrixClient; +const session = {} as MatrixRTCSession; + +const makeOptions = (overrides: Partial = {}): CallSessionJoinOptions => ({ + mx, + room: { roomId: '!room:example.org' } as Room, + session, + callIntent: 'audio', + dm: false, + ongoing: false, + encryptMedia: true, + isCancelled: () => false, + onStage: () => {}, + ...overrides, +}); + +describe('callEncryptsMedia', () => { + it('reads the room encryption state and nothing else', () => { + const encrypted = { hasEncryptionStateEvent: () => true } as Room; + const unencrypted = { hasEncryptionStateEvent: () => false } as Room; + + expect(callEncryptsMedia(encrypted)).toBe(true); + expect(callEncryptsMedia(unencrypted)).toBe(false); + }); +}); + +describe('joinCallSession', () => { + beforeEach(() => { + joinMock.mockClear(); + removePageHide.mockClear(); + }); + + // MSC4143 requires MatrixRTC encryption in encrypted rooms and forbids it in + // unencrypted ones, so this flag may only ever mirror the room. + it('manages media keys exactly when the room encrypts media', async () => { + await joinCallSession(makeOptions({ encryptMedia: true }), createCallSessionHandles()); + expect(joinMock.mock.calls[0]?.[0]).toMatchObject({ manageMediaKeys: true }); + + await joinCallSession(makeOptions({ encryptMedia: false }), createCallSessionHandles()); + expect(joinMock.mock.calls[1]?.[0]).toMatchObject({ manageMediaKeys: false }); + }); + + it('rings a DM, notifies a room, and stays silent for an ongoing call', async () => { + await joinCallSession(makeOptions({ dm: true }), createCallSessionHandles()); + expect(joinMock.mock.calls[0]?.[0].notificationType).toBe('ring'); + + await joinCallSession(makeOptions({ dm: false }), createCallSessionHandles()); + expect(joinMock.mock.calls[1]?.[0].notificationType).toBe('notification'); + + await joinCallSession(makeOptions({ dm: true, ongoing: true }), createCallSessionHandles()); + expect(joinMock.mock.calls[2]?.[0]).not.toHaveProperty('notificationType'); + }); + + it('records what the join installed so a teardown can undo it', async () => { + const handles = createCallSessionHandles(); + const cancel = vi.fn<() => void>(); + const unsubscribe = vi.fn<() => void>(); + joinMock.mockImplementationOnce(async (options) => { + options.onMembershipWait?.(cancel); + options.onCallRoomSubscribed?.(unsubscribe); + options.onJoinStarted?.(); + return { ownMembership: undefined, provisioned: { url: 'wss://x', jwt: 'jwt' } }; + }); + + expect(handles.joinStarted).toBe(false); + await joinCallSession(makeOptions(), handles); + + expect(handles.joinStarted).toBe(true); + expect(handles.cancelMembershipWait).toBe(cancel); + expect(handles.unsubscribeCallRoom).toBe(unsubscribe); + expect(handles.removePageHideListener).toBe(removePageHide); + }); +}); diff --git a/src/app/features/call/callSession.ts b/src/app/features/call/callSession.ts new file mode 100644 index 0000000000..5957a16992 --- /dev/null +++ b/src/app/features/call/callSession.ts @@ -0,0 +1,84 @@ +import type { MatrixClient, Room } from '$types/matrix-sdk'; +import { + joinAndProvisionMatrixRTC, + leaveMatrixRTCOnPageHide, + type MatrixRTCJoinProvisionOptions, + type MatrixRTCJoinProvisionResult, +} from './matrixRtcCallLifecycle'; + +/** + * Everything a join installs that a teardown has to undo. The engines fill one + * of these through `joinCallSession` and release them themselves: the web lane + * drops the page-hide listener and the room subscription only after LiveKit has + * disconnected, the native lane drops them up front, and that difference is + * real rather than incidental. + */ +export type CallSessionHandles = { + /** True once `joinRTCSession` was called, so a membership may need leaving. */ + joinStarted: boolean; + cancelMembershipWait?: () => void; + removePageHideListener?: () => void; + unsubscribeCallRoom?: () => void; +}; + +export const createCallSessionHandles = (): CallSessionHandles => ({ joinStarted: false }); + +/** + * The single gate on media encryption. MSC4143 makes MatrixRTC encryption + * REQUIRED in an encrypted room and forbids it in an unencrypted one, where a + * violating membership may be treated as left, so the room's own encryption + * state decides `manageMediaKeys`, the key pipeline and the frame cryptor + * together. Nothing else may be consulted. + */ +export const callEncryptsMedia = (room: Room): boolean => room.hasEncryptionStateEvent(); + +/** + * Whether the room already has a call running. Joining one must not ring or + * notify again: whoever started it already did. + */ +export const isCallOngoing = (mx: MatrixClient, room: Room): boolean => + mx.matrixRTC.getRoomSession(room).memberships.length > 0; + +/** + * The join, minus everything this module derives or wires up itself. The + * derived fields are the ones that were being written twice. + */ +export type CallSessionJoinOptions = Omit< + MatrixRTCJoinProvisionOptions, + | 'notificationType' + | 'manageMediaKeys' + | 'onMembershipWait' + | 'onCallRoomSubscribed' + | 'onJoinStarted' +> & { + /** A DM rings; a room only notifies. */ + dm: boolean; + /** A call that is already running was announced by whoever started it. */ + ongoing: boolean; + encryptMedia: boolean; +}; + +/** + * Join MatrixRTC and provision an SFU token, recording what the join installed + * in `handles`. Both engines run this identically; only what they do with the + * resulting token differs. + */ +export const joinCallSession = ( + { dm, ongoing, encryptMedia, ...join }: CallSessionJoinOptions, + handles: CallSessionHandles +): Promise => + joinAndProvisionMatrixRTC({ + ...join, + ...(ongoing ? {} : { notificationType: dm ? 'ring' : 'notification' }), + manageMediaKeys: encryptMedia, + onMembershipWait: (cancel) => { + handles.cancelMembershipWait = cancel; + }, + onCallRoomSubscribed: (unsubscribe) => { + handles.unsubscribeCallRoom = unsubscribe; + }, + onJoinStarted: () => { + handles.joinStarted = true; + handles.removePageHideListener = leaveMatrixRTCOnPageHide(join.session); + }, + }); diff --git a/src/app/features/call/livekitCallIdentity.ts b/src/app/features/call/livekitCallIdentity.ts new file mode 100644 index 0000000000..d9a5f37a81 --- /dev/null +++ b/src/app/features/call/livekitCallIdentity.ts @@ -0,0 +1,29 @@ +import type { CallMembership } from '$types/matrix-sdk'; + +/** Maps a LiveKit participant identity to the Matrix user behind it. */ +export type UserIdByRtcIdentity = ReadonlyMap; + +/** + * Our own user and device, used to recognise our key among the ones + * `EncryptionKeyChanged` reports. The LiveKit identity itself cannot be derived + * locally: it depends on the membership format the SFU was told to expect, so + * it is only ever read off the event. + */ +export type LocalCallIdentity = { userId: string; deviceId: string | null }; + +// The SFU decides what a LiveKit participant identity looks like: the +// anonymised SHA-256 for sticky-event RTC memberships, or `user:device` on the +// legacy path. Index every candidate so either shape resolves. +export const buildRtcIdentityMap = (members: CallMembership[]): UserIdByRtcIdentity => { + const identities = new Map(); + members.forEach((member) => { + const { userId, deviceId } = member; + if (!userId) return; + [member.rtcBackendIdentity, member.memberId, deviceId && `${userId}:${deviceId}`].forEach( + (candidate) => { + if (candidate) identities.set(candidate, userId); + } + ); + }); + return identities; +}; diff --git a/src/app/features/call/livekitMatrixKeyProvider.test.ts b/src/app/features/call/livekitMatrixKeyProvider.test.ts new file mode 100644 index 0000000000..daf2277e4d --- /dev/null +++ b/src/app/features/call/livekitMatrixKeyProvider.test.ts @@ -0,0 +1,227 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { LivekitMatrixKeyProvider, isLivekitE2EESupported } from './livekitMatrixKeyProvider'; +import type { CallEncryptionKey } from './callTransport'; + +const deferred = (): { + promise: Promise; + resolve: (value: T) => void; + reject: (reason?: unknown) => void; +} => { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((promiseResolve, promiseReject) => { + resolve = promiseResolve; + reject = promiseReject; + }); + return { promise, resolve, reject }; +}; + +const key = (identity: string, keyIndex: number, bytes: number[]): CallEncryptionKey => ({ + identity, + keyIndex, + key: new Uint8Array(bytes) as Uint8Array, +}); + +const spyOnSetEncryptionKey = (provider: LivekitMatrixKeyProvider) => + vi.spyOn( + provider as unknown as { + onSetEncryptionKey: (material: CryptoKey, identity: string, index: number) => void; + }, + 'onSetEncryptionKey' + ); + +describe('LivekitMatrixKeyProvider', () => { + const importedKey = { imported: true } as unknown as CryptoKey; + const importKey = vi.fn().mockResolvedValue(importedKey); + + beforeEach(() => { + importKey.mockClear(); + vi.stubGlobal('crypto', { subtle: { importKey } }); + }); + + it('forwards HKDF material with the identity and index the pipeline reported', async () => { + const provider = new LivekitMatrixKeyProvider(); + const onSetEncryptionKey = spyOnSetEncryptionKey(provider); + const own = key('hashed-member', 7, [1, 2, 3, 4]); + + // livekit-client stores at `cryptoKeyRing[keyIndex % keyringSize]` and caps + // the ring at 256, so only 256 keeps two live indices off one slot. + expect(provider.getOptions()).toMatchObject({ + ratchetWindowSize: 10, + keyringSize: 256, + sharedKey: false, + }); + + provider.setKey(own, true); + await vi.waitFor(() => expect(onSetEncryptionKey).toHaveBeenCalledOnce()); + + expect(importKey).toHaveBeenCalledWith('raw', own.key, 'HKDF', false, [ + 'deriveBits', + 'deriveKey', + ]); + expect(onSetEncryptionKey).toHaveBeenCalledWith(importedKey, 'hashed-member', 7); + expect(provider.getKeyState()).toEqual({ + ready: true, + localOutboundIdentity: 'hashed-member', + keyIndex: 7, + lastImportFailure: null, + }); + }); + + it('does not expose raw key material through the key ring', async () => { + const provider = new LivekitMatrixKeyProvider(); + const raw = key('member', 3, [9, 8, 7]); + + provider.setKey(raw, false); + await vi.waitFor(() => expect(provider.getKeys()).toHaveLength(1)); + + expect(provider.getKeys()[0]?.key).toBe(importedKey); + expect(provider.getKeys()[0]?.key).not.toBe(raw.key); + }); + + it('does not forward an import that completes after a reset', async () => { + const pendingImport = deferred(); + importKey.mockImplementationOnce(() => pendingImport.promise); + const provider = new LivekitMatrixKeyProvider(); + const onSetEncryptionKey = spyOnSetEncryptionKey(provider); + + provider.setKey(key('member', 1, [1]), true); + await vi.waitFor(() => expect(importKey).toHaveBeenCalledOnce()); + provider.reset(); + pendingImport.resolve(importedKey); + await Promise.resolve(); + await Promise.resolve(); + + expect(onSetEncryptionKey).not.toHaveBeenCalled(); + expect(provider.getKeyState()).toEqual({ + ready: false, + localOutboundIdentity: null, + keyIndex: null, + lastImportFailure: null, + }); + }); + + it('applies keys in the order they were reported when imports complete out of order', async () => { + const firstImport = deferred(); + const secondImport = deferred(); + importKey + .mockImplementationOnce(() => firstImport.promise) + .mockImplementationOnce(() => secondImport.promise); + const provider = new LivekitMatrixKeyProvider(); + const onSetEncryptionKey = spyOnSetEncryptionKey(provider); + + provider.setKey(key('member', 1, [1]), true); + provider.setKey(key('member', 2, [2]), true); + await vi.waitFor(() => expect(importKey).toHaveBeenCalledTimes(2)); + + secondImport.resolve(importedKey); + await Promise.resolve(); + expect(onSetEncryptionKey).not.toHaveBeenCalled(); + firstImport.resolve(importedKey); + await vi.waitFor(() => expect(onSetEncryptionKey).toHaveBeenCalledTimes(2)); + + expect(onSetEncryptionKey.mock.calls[0]).toEqual([importedKey, 'member', 1]); + expect(onSetEncryptionKey.mock.calls[1]).toEqual([importedKey, 'member', 2]); + expect(provider.getKeyState().keyIndex).toBe(2); + }); + + it('applies a lower index for the same identity, since a rejoin restarts the ring at 0', async () => { + // A key index is a ring slot, not a sequence. The sender writes it into + // every frame, so dropping a key for a non-increasing index leaves that + // peer's media undecryptable for the rest of the call. + const provider = new LivekitMatrixKeyProvider(); + const onSetEncryptionKey = spyOnSetEncryptionKey(provider); + + provider.setKey(key('remote-member', 5, [1]), false); + provider.setKey(key('remote-member', 0, [2]), false); + provider.setKey(key('remote-member', 0, [3]), false); + await vi.waitFor(() => expect(onSetEncryptionKey).toHaveBeenCalledTimes(3)); + + expect(onSetEncryptionKey.mock.calls.map((call) => call[2])).toEqual([5, 0, 0]); + }); + + it('applies a lower index for our own key too, so a rejoin can still publish', async () => { + const provider = new LivekitMatrixKeyProvider(); + const onSetEncryptionKey = spyOnSetEncryptionKey(provider); + + provider.setKey(key('local-member', 3, [1]), true); + await vi.waitFor(() => expect(provider.getKeyState().keyIndex).toBe(3)); + + provider.setKey(key('local-member', 0, [2]), true); + await vi.waitFor(() => expect(onSetEncryptionKey).toHaveBeenCalledTimes(2)); + + expect(onSetEncryptionKey).toHaveBeenLastCalledWith(importedKey, 'local-member', 0); + expect(provider.getKeyState()).toMatchObject({ ready: true, keyIndex: 0 }); + }); + + it('only lets our own key decide readiness', async () => { + const provider = new LivekitMatrixKeyProvider(); + const onSetEncryptionKey = spyOnSetEncryptionKey(provider); + + provider.setKey(key('remote-member', 10, [1]), false); + provider.setKey(key('local-member', 1, [2]), true); + await vi.waitFor(() => expect(onSetEncryptionKey).toHaveBeenCalledTimes(2)); + + expect(onSetEncryptionKey).toHaveBeenNthCalledWith(1, importedKey, 'remote-member', 10); + expect(onSetEncryptionKey).toHaveBeenNthCalledWith(2, importedKey, 'local-member', 1); + expect(provider.getKeyState()).toMatchObject({ + ready: true, + localOutboundIdentity: 'local-member', + keyIndex: 1, + lastImportFailure: null, + }); + }); + + it('records a safe failure when key import is rejected', async () => { + importKey.mockRejectedValueOnce(new Error('raw key internals')); + const provider = new LivekitMatrixKeyProvider(); + + provider.setKey(key('member', 1, [1]), true); + + await vi.waitFor(() => expect(provider.getKeyState().lastImportFailure).toBe('import-failed')); + expect(JSON.stringify(provider.getKeyState())).not.toContain('raw key internals'); + + provider.setKey(key('member', 2, [2]), true); + await vi.waitFor(() => expect(provider.getKeyState().ready).toBe(true)); + expect(provider.getKeyState().lastImportFailure).toBeNull(); + }); + + it('records a safe failure when WebCrypto import support is missing', async () => { + vi.stubGlobal('crypto', { subtle: {} }); + const provider = new LivekitMatrixKeyProvider(); + + provider.setKey(key('member', 1, [1]), false); + + await vi.waitFor(() => + expect(provider.getKeyState().lastImportFailure).toBe('webcrypto-unavailable') + ); + }); +}); + +describe('isLivekitE2EESupported', () => { + beforeEach(() => { + vi.stubGlobal('crypto', { subtle: { importKey: vi.fn() } }); + }); + + it('fails closed when the current LiveKit API reports unsupported E2EE', () => { + Object.defineProperty(window, 'RTCRtpScriptTransform', { + configurable: true, + value: undefined, + }); + Object.defineProperty(window, 'RTCRtpSender', { + configurable: true, + value: undefined, + }); + + expect(isLivekitE2EESupported()).toBe(false); + }); + + it('reports support when LiveKit and WebCrypto are available', () => { + Object.defineProperty(window, 'RTCRtpScriptTransform', { + configurable: true, + value: vi.fn<() => void>(), + }); + + expect(isLivekitE2EESupported()).toBe(true); + }); +}); diff --git a/src/app/features/call/livekitMatrixKeyProvider.ts b/src/app/features/call/livekitMatrixKeyProvider.ts new file mode 100644 index 0000000000..b9e8bd2662 --- /dev/null +++ b/src/app/features/call/livekitMatrixKeyProvider.ts @@ -0,0 +1,159 @@ +import { BaseKeyProvider, isE2EESupported } from 'livekit-client'; +import type { CallEncryptionKey } from './callTransport'; + +export const isLivekitE2EESupported = (): boolean => { + const subtle = globalThis.crypto?.subtle; + return typeof subtle?.importKey === 'function' && isE2EESupported(); +}; + +export type LivekitMatrixKeyImportFailure = 'webcrypto-unavailable' | 'import-failed'; + +export type LivekitMatrixKeyProviderState = { + ready: boolean; + localOutboundIdentity: string | null; + keyIndex: number | null; + lastImportFailure: LivekitMatrixKeyImportFailure | null; +}; + +export type LivekitMatrixKeyProviderStateListener = ( + state: Readonly +) => void; + +type KeyImportResult = + | { keyMaterial: CryptoKey; key: CallEncryptionKey; own: boolean } + | { failure: LivekitMatrixKeyImportFailure }; + +/** + * Turns the keys `callKeyPipeline` reports into LiveKit key-ring entries. It + * takes no part in the Matrix session: the pipeline is the only subscriber to + * `EncryptionKeyChanged`, so there is one place where a key can be lost. + */ +export class LivekitMatrixKeyProvider extends BaseKeyProvider { + private generation = 0; + private nextImportSequence = 0; + private nextUpdateSequence = 0; + private readonly pendingUpdates = new Map(); + private readonly stateListeners = new Set(); + private state: LivekitMatrixKeyProviderState = { + ready: false, + localOutboundIdentity: null, + keyIndex: null, + lastImportFailure: null, + }; + + public constructor() { + super({ + ratchetWindowSize: 10, + keyringSize: 256, + sharedKey: false, + }); + } + + /** Drops everything still importing so a new call starts from a clean ring. */ + public reset(): void { + this.generation += 1; + this.nextImportSequence = 0; + this.nextUpdateSequence = 0; + this.pendingUpdates.clear(); + this.updateState({ + ready: false, + localOutboundIdentity: null, + keyIndex: null, + lastImportFailure: null, + }); + } + + public getKeyState(): Readonly { + return { ...this.state }; + } + + public subscribe(listener: LivekitMatrixKeyProviderStateListener): () => void { + this.stateListeners.add(listener); + listener(this.getKeyState()); + return () => { + this.stateListeners.delete(listener); + }; + } + + /** + * Sequenced on entry so keys reach the ring in the order the session emitted + * them however long each WebCrypto import takes. + */ + public readonly setKey = (key: CallEncryptionKey, own: boolean): void => { + const generation = this.generation; + const sequence = this.nextImportSequence++; + const subtle = globalThis.crypto?.subtle; + if (!subtle || typeof subtle.importKey !== 'function') { + this.enqueueUpdate(generation, sequence, { failure: 'webcrypto-unavailable' }); + return; + } + + let importPromise: Promise; + try { + importPromise = subtle.importKey('raw', key.key, 'HKDF', false, ['deriveBits', 'deriveKey']); + } catch { + this.enqueueUpdate(generation, sequence, { failure: 'import-failed' }); + return; + } + + void importPromise.then( + (keyMaterial) => { + this.enqueueUpdate(generation, sequence, { keyMaterial, key, own }); + }, + () => { + this.enqueueUpdate(generation, sequence, { failure: 'import-failed' }); + } + ); + }; + + private enqueueUpdate(generation: number, sequence: number, result: KeyImportResult): void { + if (generation !== this.generation) return; + this.pendingUpdates.set(sequence, result); + + while (this.pendingUpdates.has(this.nextUpdateSequence)) { + const update = this.pendingUpdates.get(this.nextUpdateSequence); + this.pendingUpdates.delete(this.nextUpdateSequence); + this.nextUpdateSequence += 1; + if (!update) continue; + + if ('failure' in update) { + this.updateState({ lastImportFailure: update.failure }); + continue; + } + + // Updates are applied in the order the session emitted them, so the most + // recent key always wins. Skipping lower indices would be wrong: a peer + // that rejoins starts a fresh outbound session back at index 0, and + // treating that as stale leaves its media undecryptable for the call. + try { + this.onSetEncryptionKey(update.keyMaterial, update.key.identity, update.key.keyIndex); + } catch { + this.updateState({ lastImportFailure: 'import-failed' }); + continue; + } + + // Only the local key clears the failure flag: a remote participant's key + // succeeding says nothing about whether our own outbound key imported. + if (update.own) { + this.updateState({ + ready: true, + localOutboundIdentity: update.key.identity, + keyIndex: update.key.keyIndex, + lastImportFailure: null, + }); + } + } + } + + private updateState(changes: Partial): void { + this.state = { ...this.state, ...changes }; + const state = this.getKeyState(); + this.stateListeners.forEach((listener) => { + try { + listener(state); + } catch { + // A state observer must not interrupt key updates. + } + }); + } +} diff --git a/src/app/features/call/livekitProvisioning.test.ts b/src/app/features/call/livekitProvisioning.test.ts new file mode 100644 index 0000000000..6e53fb1a57 --- /dev/null +++ b/src/app/features/call/livekitProvisioning.test.ts @@ -0,0 +1,197 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { IOpenIDToken, MatrixClient } from '$types/matrix-sdk'; + +const fetchMock = vi.hoisted(() => vi.fn()); + +vi.mock('$utils/fetch', () => ({ fetch: fetchMock })); + +import { getPreferredLivekitTransport, provisionLivekitToken } from './livekitProvisioning'; + +const openidToken: IOpenIDToken = { + access_token: 'openid-secret', + token_type: 'Bearer', + matrix_server_name: 'example.org', + expires_in: 3600, +}; + +const response = (status: number, body: unknown): Response => + new Response(JSON.stringify(body), { status }); + +/** Everything a caller could read off a thrown error, cause chain included. */ +const flattenError = (value: unknown, seen = new Set()): string => { + if (value === null || value === undefined) return ''; + if (typeof value !== 'object') return JSON.stringify(value) ?? ''; + if (seen.has(value)) return ''; + seen.add(value); + const error = value as Error & { cause?: unknown }; + return [ + error.message ?? '', + JSON.stringify(error, Object.getOwnPropertyNames(error)), + flattenError(error.cause, seen), + ].join(' '); +}; + +type TestClient = Pick; + +const client = (overrides: Partial = {}): TestClient => ({ + getOpenIdToken: vi.fn().mockResolvedValue(openidToken), + _unstable_getRTCTransports: vi + .fn() + .mockResolvedValue([]), + ...overrides, +}); + +describe('getPreferredLivekitTransport', () => { + it('prefers the SDK LiveKit transport over discovery', async () => { + const mx = client({ + _unstable_getRTCTransports: vi + .fn() + .mockResolvedValue([{ type: 'livekit', livekit_service_url: 'https://sdk.example' }]), + }); + + await expect( + getPreferredLivekitTransport(mx, { + 'org.matrix.msc4143.rtc_foci': [ + { type: 'livekit', livekit_service_url: 'https://discovery.example' }, + ], + }) + ).resolves.toEqual({ + type: 'livekit', + livekit_service_url: 'https://sdk.example', + }); + }); + + it('falls back to discovery when SDK transport discovery fails', async () => { + const mx = client({ + _unstable_getRTCTransports: vi + .fn() + .mockRejectedValue(new Error('unsupported')), + }); + + await expect( + getPreferredLivekitTransport(mx, { + 'org.matrix.msc4143.rtc_foci': [ + { type: 'livekit', livekit_service_url: 'https://discovery.example' }, + ], + }) + ).resolves.toEqual({ + type: 'livekit', + livekit_service_url: 'https://discovery.example', + }); + }); +}); + +describe('provisionLivekitToken', () => { + beforeEach(() => { + fetchMock.mockReset(); + }); + + const options = { + mx: client(), + roomId: '!room:example.org', + slotId: 'm.call#real-slot', + deviceId: 'DEVICE', + serviceUrl: 'https://sfu.example///', + memberId: 'member-id', + userId: '@alice:example.org', + }; + + it('provisions through the endpoint that matches the advertised membership format', async () => { + fetchMock.mockResolvedValueOnce( + response(200, { url: 'wss://livekit.example', jwt: 'jwt-secret' }) + ); + + await expect(provisionLivekitToken(options)).resolves.toEqual({ + url: 'wss://livekit.example', + jwt: 'jwt-secret', + }); + + expect(fetchMock).toHaveBeenCalledOnce(); + const [url, request] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(url).toBe('https://sfu.example/sfu/get'); + expect(JSON.parse(request.body as string)).toEqual({ + room: '!room:example.org', + openid_token: openidToken, + device_id: 'DEVICE', + }); + }); + + it('never tries the other endpoint, whose identity convention would not match', async () => { + fetchMock.mockResolvedValueOnce(response(404, { error: 'not found' })); + + await expect(provisionLivekitToken(options)).rejects.toThrow( + 'LiveKit token provisioning failed' + ); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('rejects an invalid response without retrying', async () => { + fetchMock.mockResolvedValueOnce(response(200, { url: 'wss://livekit.example' })); + + await expect(provisionLivekitToken(options)).rejects.toThrow( + 'LiveKit token provisioning failed' + ); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('does not retry after a server error', async () => { + fetchMock.mockResolvedValueOnce(response(500, { error: 'boom' })); + + await expect(provisionLivekitToken(options)).rejects.toThrow( + 'LiveKit token provisioning failed' + ); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('does not expose token values in errors', async () => { + fetchMock.mockRejectedValue(new Error('request failed: jwt-secret openid-secret')); + const provisioning = provisionLivekitToken(options); + + await expect(provisioning).rejects.toThrow('LiveKit token provisioning failed'); + await expect(provisioning).rejects.not.toThrow('openid-secret'); + await expect(provisioning).rejects.not.toThrow('jwt-secret'); + }); + + it('keeps the tokens out of the cause chain too, not just the message', async () => { + // The request body carries the OpenID token and the response carries the + // JWT, so the caught error is unsafe to forward by any route. `toThrow` + // only reads the message, so `new Error(msg, { cause })` slips past it. + fetchMock.mockRejectedValue(new Error('request failed: jwt-secret openid-secret')); + + const thrown = await provisionLivekitToken(options).then( + () => undefined, + (error: unknown) => error + ); + + expect(thrown).toBeInstanceOf(Error); + expect(flattenError(thrown)).not.toContain('openid-secret'); + expect(flattenError(thrown)).not.toContain('jwt-secret'); + }); + + it('keeps the OpenID token out of the error when minting it fails', async () => { + const mx = client({ + getOpenIdToken: vi + .fn() + .mockRejectedValue(new Error('minting failed for openid-secret')), + }); + + const thrown = await provisionLivekitToken({ ...options, mx }).then( + () => undefined, + (error: unknown) => error + ); + + expect(thrown).toBeInstanceOf(Error); + expect(flattenError(thrown)).not.toContain('openid-secret'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('names the endpoint and status so a dead transport is distinguishable', async () => { + // Which SFU refused and with what status is the difference between a stale + // transport advertised by another participant and a rejected token. + fetchMock.mockResolvedValueOnce(response(404, { error: 'nope' })); + + await expect(provisionLivekitToken(options)).rejects.toThrow( + 'LiveKit token provisioning failed against https://sfu.example with status 404' + ); + }); +}); diff --git a/src/app/features/call/livekitProvisioning.ts b/src/app/features/call/livekitProvisioning.ts new file mode 100644 index 0000000000..0558d9d3fc --- /dev/null +++ b/src/app/features/call/livekitProvisioning.ts @@ -0,0 +1,114 @@ +import type { AutoDiscoveryInfo } from '../../cs-api'; +import { getLivekitTransports } from '../../cs-api'; +import { fetch as appFetch } from '$utils/fetch'; +import { trimTrailingSlash } from '$utils/common'; +import type { + IOpenIDToken, + LivekitTransportConfig, + MatrixClient, + Transport, +} from '$types/matrix-sdk'; +import { buildProvisioningRequest, type CallProvisioningRequest } from './callProtocol'; + +export type LivekitProvisioningOptions = { + mx: Pick; + roomId: string; + deviceId: string; + serviceUrl: string; +}; + +export type LivekitProvisioningResult = { + url: string; + jwt: string; +}; + +export const isLivekitTransportConfig = ( + transport: Transport +): transport is LivekitTransportConfig => + transport.type === 'livekit' && typeof transport.livekit_service_url === 'string'; + +const isProvisioningResult = (value: unknown): value is LivekitProvisioningResult => + typeof value === 'object' && + value !== null && + 'url' in value && + typeof value.url === 'string' && + value.url.length > 0 && + 'jwt' in value && + typeof value.jwt === 'string' && + value.jwt.length > 0; + +export const getPreferredLivekitTransport = async ( + mx: Pick, + discovery?: Pick +): Promise => { + const transports = await mx['_unstable_getRTCTransports']().catch(() => undefined); + const livekitTransport = transports?.find(isLivekitTransportConfig); + if (livekitTransport) return livekitTransport; + + return getLivekitTransports(discovery)[0]; +}; + +const requestLivekitToken = async ({ + url, + body, +}: CallProvisioningRequest): Promise => { + const response = await appFetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + + if (response.status < 200 || response.status >= 300) { + const error: Error & { status?: number } = new Error( + `LiveKit provisioning request failed with status ${response.status}` + ); + error.status = response.status; + throw error; + } + + const data = (await response.json()) as unknown; + + if (!isProvisioningResult(data)) { + throw new Error('LiveKit provisioning response was invalid'); + } + + return { url: data.url, jwt: data.jwt }; +}; + +export const provisionLivekitToken = async ({ + mx, + roomId, + deviceId, + serviceUrl, +}: LivekitProvisioningOptions): Promise => { + let openidToken: IOpenIDToken; + try { + openidToken = await mx.getOpenIdToken(); + } catch { + throw new Error('Unable to obtain an OpenID token for LiveKit provisioning'); + } + + const endpoint = trimTrailingSlash(serviceUrl); + const request = buildProvisioningRequest({ + serviceUrl: endpoint, + roomId, + deviceId, + openidToken, + }); + + try { + return await requestLivekitToken(request); + } catch (error) { + // Report which SFU refused us and with what status, since that separates a + // dead transport advertised by another participant from a rejected token. + // Never forward the underlying message or cause: the request body carries + // the OpenID token and the response carries the issued JWT. + const status = (error as { status?: number } | null)?.status; + // oxlint-disable-next-line preserve-caught-error -- attaching the cause leaks the tokens + throw new Error( + status === undefined + ? `LiveKit token provisioning failed against ${endpoint}` + : `LiveKit token provisioning failed against ${endpoint} with status ${status}` + ); + } +}; diff --git a/src/app/features/call/matrixRtcCallLifecycle.test.ts b/src/app/features/call/matrixRtcCallLifecycle.test.ts new file mode 100644 index 0000000000..a1d541f7d7 --- /dev/null +++ b/src/app/features/call/matrixRtcCallLifecycle.test.ts @@ -0,0 +1,508 @@ +import { beforeEach, describe, expect, it, vi, type Mock } from 'vitest'; +import type { + MatrixClient, + MatrixRTCSession, + CallMembership, + JoinSessionConfig, + Room, +} from '$types/matrix-sdk'; +import { MatrixRTCSessionEvent } from '$types/matrix-sdk'; +import { + joinAndProvisionMatrixRTC, + leaveMatrixRTCOnPageHide, + ROSTER_HYDRATION_ERROR, +} from './matrixRtcCallLifecycle'; +import type { + LivekitProvisioningResult, + getPreferredLivekitTransport, + provisionLivekitToken, +} from './livekitProvisioning'; + +const mocks = vi.hoisted(() => ({ getSlidingSyncManager: vi.fn<() => unknown>() })); + +vi.mock('$client/initMatrix', () => ({ getSlidingSyncManager: mocks.getSlidingSyncManager })); + +type SessionHandler = (...args: unknown[]) => void; + +type TestSession = MatrixRTCSession & { + handlers: Map; +}; + +const makeSession = (): TestSession => { + const handlers = new Map(); + const session = { + handlers, + memberships: [] as CallMembership[], + slotId: 'm.call#slot', + on: vi + .fn<(event: MatrixRTCSessionEvent, handler: SessionHandler) => void>() + .mockImplementation((event, handler) => { + handlers.set(event, handler); + }), + removeListener: vi + .fn<(event: MatrixRTCSessionEvent, handler: SessionHandler) => void>() + .mockImplementation((event, handler) => { + if (handlers.get(event) === handler) { + handlers.delete(event); + } + }), + joinRTCSession: vi.fn<(identity: unknown, transports: unknown[], ..._: unknown[]) => void>(), + getOldestMembership: vi.fn<() => CallMembership | undefined>().mockReturnValue(undefined), + leaveRoomSession: vi.fn().mockResolvedValue(true), + } as unknown as TestSession; + return session; +}; + +const emitMembershipManagerError = (session: TestSession): void => { + session.handlers.get(MatrixRTCSessionEvent.MembershipManagerError)?.(); +}; + +const makeClient = (overrides: Partial = {}): MatrixClient => + ({ + getDeviceId: () => 'ALICEDEVICE', + getSafeUserId: () => '@alice:example.org', + getStateEvent: vi + .fn<() => Promise>() + .mockResolvedValue(undefined) as unknown as MatrixClient['getStateEvent'], + ...overrides, + }) as unknown as MatrixClient; + +const provisioned: LivekitProvisioningResult = { url: 'wss://livekit.example', jwt: 'jwt' }; + +const makeTransport = () => ({ + type: 'livekit' as const, + livekit_service_url: 'https://sfu.example', +}); + +type RosterOverrides = { + loadMembersIfNeeded?: () => Promise; + clearLoadedMembersIfNeeded?: () => Promise; + membersLoaded?: () => boolean; + joinedInState?: number | (() => number); + joinedCount?: number; +}; + +const makeRoom = ({ + loadMembersIfNeeded = () => Promise.resolve(true), + clearLoadedMembersIfNeeded = () => Promise.resolve(), + membersLoaded = () => true, + joinedInState = 2, + joinedCount = 2, +}: RosterOverrides = {}): Room => { + const inState = typeof joinedInState === 'function' ? joinedInState : () => joinedInState; + return { + roomId: '!room:example.org', + loadMembersIfNeeded, + clearLoadedMembersIfNeeded, + membersLoaded, + getMembersWithMembership: () => Array.from({ length: inState() }, () => ({})), + getJoinedMemberCount: () => joinedCount, + } as unknown as Room; +}; + +describe('joinAndProvisionMatrixRTC', () => { + beforeEach(() => { + // vi.useFakeTimers() would interfere with setInterval; use real timers. + mocks.getSlidingSyncManager.mockReset().mockReturnValue(undefined); + }); + + const callOpts = (overrides: Record = {}) => ({ + mx: makeClient(), + room: makeRoom(), + session: makeSession(), + callIntent: 'audio' as const, + getPreferredTransport: vi + .fn() + .mockResolvedValue(makeTransport()), + provisionToken: vi.fn().mockResolvedValue(provisioned), + ...overrides, + }); + + it('uses the call-room subscription before publishing an RTC membership', async () => { + const session = makeSession(); + const unsubscribe = vi.fn<() => void>(); + const subscribeToCallRoom = vi.fn<() => () => void>().mockReturnValue(unsubscribe); + mocks.getSlidingSyncManager.mockReturnValue({ subscribeToCallRoom }); + const onCallRoomSubscribed = vi.fn<(unsubscribe: () => void) => void>(); + + const promise = joinAndProvisionMatrixRTC(callOpts({ session, onCallRoomSubscribed })); + await vi.waitFor(() => expect(session.joinRTCSession).toHaveBeenCalled()); + + session.memberships = [ + { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }, + ] as CallMembership[]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)!([], session.memberships); + await promise; + + expect(subscribeToCallRoom).toHaveBeenCalledWith('!room:example.org'); + expect(onCallRoomSubscribed).toHaveBeenCalledWith(unsubscribe); + }); + + it('resolves when MembershipsChanged fires with own membership', async () => { + const session = makeSession(); + const opts = callOpts({ session }); + const promise = joinAndProvisionMatrixRTC(opts); + + // wait for listener registration in the microtask/event loop + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + + // fire the membership event + session.memberships = [ + { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }, + ] as CallMembership[]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)!([], session.memberships); + + const result = await promise; + expect(result.provisioned).toEqual(provisioned); + expect(opts.getPreferredTransport).toHaveBeenCalledOnce(); + }); + + it('advertises the transport with a livekit alias and a bounded membership expiry', async () => { + const session = makeSession(); + const opts = callOpts({ session }); + const promise = joinAndProvisionMatrixRTC(opts); + + await vi.waitFor(() => expect(session.joinRTCSession).toHaveBeenCalled()); + session.memberships = [ + { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }, + ] as CallMembership[]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)!([], session.memberships); + await promise; + + const [, transports, , joinConfig] = (session.joinRTCSession as Mock).mock.calls[0] as [ + unknown, + unknown[], + unknown, + JoinSessionConfig, + ]; + expect(transports).toEqual([{ ...makeTransport(), livekit_alias: '!room:example.org' }]); + expect(joinConfig.membershipEventExpiryMs).toBe(30 * 60 * 1000); + expect(joinConfig.unstableSendStickyEvents).toBeUndefined(); + }); + + it('provisions against the oldest membership transport, not our own preference', async () => { + const session = makeSession(); + const oldestTransport = { type: 'livekit' as const, livekit_service_url: 'https://oldest.sfu' }; + const oldest = { + userId: '@bob:example.org', + deviceId: 'BOBDEVICE', + getTransport: () => oldestTransport, + } as unknown as CallMembership; + (session.getOldestMembership as Mock<() => CallMembership | undefined>).mockReturnValue(oldest); + const opts = callOpts({ session }); + const promise = joinAndProvisionMatrixRTC(opts); + + await vi.waitFor(() => expect(session.on).toHaveBeenCalled()); + session.memberships = [ + { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }, + ] as CallMembership[]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)!([], session.memberships); + await promise; + + expect(opts.provisionToken).toHaveBeenCalledWith( + expect.objectContaining({ serviceUrl: 'https://oldest.sfu' }) + ); + }); + + it('falls back to the preferred transport when the oldest membership has none', async () => { + const session = makeSession(); + const oldest = { + userId: '@bob:example.org', + deviceId: 'BOBDEVICE', + getTransport: () => undefined, + } as unknown as CallMembership; + (session.getOldestMembership as Mock<() => CallMembership | undefined>).mockReturnValue(oldest); + const opts = callOpts({ session }); + const promise = joinAndProvisionMatrixRTC(opts); + + await vi.waitFor(() => expect(session.on).toHaveBeenCalled()); + session.memberships = [ + { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }, + ] as CallMembership[]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)!([], session.memberships); + await promise; + + expect(opts.provisionToken).toHaveBeenCalledWith( + expect.objectContaining({ serviceUrl: 'https://sfu.example' }) + ); + }); + + it('rejects on MembershipManagerError', async () => { + const session = makeSession(); + const opts = callOpts({ session }); + const promise = joinAndProvisionMatrixRTC(opts); + + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipManagerError, + expect.any(Function) + ) + ); + + emitMembershipManagerError(session); + + await expect(promise).rejects.toThrow('MatrixRTC membership publication failed'); + }); + + it('rejects on timeout (30s) when no membership event fires', async () => { + vi.useFakeTimers(); + try { + const session = makeSession(); + const mx = makeClient({ + getStateEvent: vi.fn().mockResolvedValue(undefined as never), + }); + const opts = callOpts({ session, mx }); + const promise = joinAndProvisionMatrixRTC(opts); + // Handled by the assertion below; attached now so the rejection that + // lands during the timer flush is never seen as unhandled. + promise.catch(() => {}); + + await vi.runAllTimersAsync(); + await expect(promise).rejects.toThrow('MatrixRTC membership publication timed out'); + } finally { + vi.useRealTimers(); + } + }); + + it('rejects when the membership wait is cancelled', async () => { + const session = makeSession(); + let cancelMembership!: (() => void) | undefined; + const opts = callOpts({ + session, + onMembershipWait: (cancel: (() => void) | undefined) => { + cancelMembership = cancel; + }, + }); + const promise = joinAndProvisionMatrixRTC(opts); + promise.catch(() => {}); + + await vi.waitFor(() => expect(session.on).toHaveBeenCalled()); + expect(cancelMembership).toBeDefined(); + + cancelMembership!(); + await expect(promise).rejects.toThrow('MatrixRTC membership wait cancelled'); + }); + + it('fills the roster from the server when there is no sliding sync subscription', async () => { + const session = makeSession(); + const loadMembersIfNeeded = vi.fn<() => Promise>().mockResolvedValue(true); + // Without the roster the SDK discards the other participants' RTC + // memberships, which breaks key exchange in both directions. + const room = makeRoom({ loadMembersIfNeeded }); + + const promise = joinAndProvisionMatrixRTC(callOpts({ session, room })); + await vi.waitFor(() => expect(loadMembersIfNeeded).toHaveBeenCalled()); + + session.memberships = [ + { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }, + ] as CallMembership[]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)!([], session.memberships); + await promise; + }); + + it('aborts before joining when roster hydration rejects', async () => { + const session = makeSession(); + const room = makeRoom({ + loadMembersIfNeeded: () => Promise.reject(new Error('roster request failed')), + }); + + await expect(joinAndProvisionMatrixRTC(callOpts({ session, room }))).rejects.toThrow( + ROSTER_HYDRATION_ERROR + ); + expect(session.joinRTCSession).not.toHaveBeenCalled(); + }); + + it('aborts before joining when the hydrated roster is short of the joined count', async () => { + const session = makeSession(); + // `Room.loadMembers` resolves from the out-of-band store without touching + // the server, so a settled promise can still leave a partial roster. + const room = makeRoom({ joinedInState: 2, joinedCount: 5 }); + + await expect(joinAndProvisionMatrixRTC(callOpts({ session, room }))).rejects.toThrow( + ROSTER_HYDRATION_ERROR + ); + expect(session.joinRTCSession).not.toHaveBeenCalled(); + }); + + it('refetches from the server when the cached roster is stale, then joins', async () => { + const session = makeSession(); + let joined = 1; + const clearLoadedMembersIfNeeded = vi.fn<() => Promise>(() => Promise.resolve()); + // First load is answered by the out-of-band cache and is short; the second, + // after the cache is cleared, reaches the server and completes the roster. + const loadMembersIfNeeded = vi + .fn<() => Promise>() + .mockResolvedValueOnce(false) + .mockImplementation(() => { + joined = 2; + return Promise.resolve(true); + }); + const room = makeRoom({ + loadMembersIfNeeded, + clearLoadedMembersIfNeeded, + joinedInState: () => joined, + joinedCount: 2, + }); + + const promise = joinAndProvisionMatrixRTC(callOpts({ session, room })); + await vi.waitFor(() => expect(loadMembersIfNeeded).toHaveBeenCalledTimes(2)); + + session.memberships = [ + { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }, + ] as CallMembership[]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)!([], session.memberships); + await promise; + + expect(clearLoadedMembersIfNeeded).toHaveBeenCalledOnce(); + expect(session.joinRTCSession).toHaveBeenCalled(); + }); + + it('surfaces the failure that clearing the cached roster threw, with its cause', async () => { + const session = makeSession(); + const cause = new Error('out-of-band store unavailable'); + const room = makeRoom({ + loadMembersIfNeeded: () => Promise.resolve(false), + clearLoadedMembersIfNeeded: () => Promise.reject(cause), + joinedInState: 1, + joinedCount: 2, + }); + + const error = await joinAndProvisionMatrixRTC(callOpts({ session, room })).catch( + (reason: unknown) => reason + ); + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toBe(ROSTER_HYDRATION_ERROR); + expect((error as Error).cause).toBe(cause); + expect(session.joinRTCSession).not.toHaveBeenCalled(); + }); + + it('gives up after one refetch when clearing the cache changed nothing', async () => { + const session = makeSession(); + // `Room.clearLoadedMembersIfNeeded` is a no-op unless lazy loading is on + // and a members promise exists, so the memoized `loadMembersIfNeeded` + // answers the retry with the same short roster. + const loadMembersIfNeeded = vi.fn<() => Promise>().mockResolvedValue(false); + const clearLoadedMembersIfNeeded = vi.fn<() => Promise>(() => Promise.resolve()); + const room = makeRoom({ + loadMembersIfNeeded, + clearLoadedMembersIfNeeded, + joinedInState: 1, + joinedCount: 2, + }); + + await expect(joinAndProvisionMatrixRTC(callOpts({ session, room }))).rejects.toThrow( + ROSTER_HYDRATION_ERROR + ); + expect(clearLoadedMembersIfNeeded).toHaveBeenCalledOnce(); + expect(loadMembersIfNeeded).toHaveBeenCalledTimes(2); + expect(session.joinRTCSession).not.toHaveBeenCalled(); + }); + + it('does not refetch when a server-sourced roster is already short', async () => { + const session = makeSession(); + const clearLoadedMembersIfNeeded = vi.fn<() => Promise>(() => Promise.resolve()); + const room = makeRoom({ clearLoadedMembersIfNeeded, joinedInState: 2, joinedCount: 5 }); + + await expect(joinAndProvisionMatrixRTC(callOpts({ session, room }))).rejects.toThrow( + ROSTER_HYDRATION_ERROR + ); + expect(clearLoadedMembersIfNeeded).not.toHaveBeenCalled(); + }); + + it('aborts before joining when out-of-band members never finished loading', async () => { + const session = makeSession(); + const room = makeRoom({ membersLoaded: () => false }); + + await expect(joinAndProvisionMatrixRTC(callOpts({ session, room }))).rejects.toThrow( + ROSTER_HYDRATION_ERROR + ); + expect(session.joinRTCSession).not.toHaveBeenCalled(); + }); + + it('rejects on SDK error path even with fallback available', async () => { + // MembershipManagerError wins: membership on server does not matter + const session = makeSession(); + const mx = makeClient({ + getStateEvent: vi.fn().mockResolvedValue({}), + }); + const opts = callOpts({ session, mx }); + const promise = joinAndProvisionMatrixRTC(opts); + + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipManagerError, + expect.any(Function) + ) + ); + + emitMembershipManagerError(session); + + await expect(promise).rejects.toThrow('MatrixRTC membership publication failed'); + // fallback may have polled, but error settles first + }); + + it('cleans up listeners after resolution', async () => { + const session = makeSession(); + const opts = callOpts({ session }); + const promise = joinAndProvisionMatrixRTC(opts); + + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + + session.memberships = [ + { userId: '@alice:example.org', deviceId: 'ALICEDEVICE' }, + ] as CallMembership[]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)!([], session.memberships); + + await promise; + + expect(session.removeListener).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ); + expect(session.removeListener).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipManagerError, + expect.any(Function) + ); + }); +}); + +const firePageHide = (persisted: boolean): void => { + const event = new Event('pagehide') as PageTransitionEvent; + Object.defineProperty(event, 'persisted', { value: persisted }); + window.dispatchEvent(event); +}; + +describe('leaveMatrixRTCOnPageHide', () => { + it('leaves the session when the page is torn down for good', () => { + const session = makeSession(); + const remove = leaveMatrixRTCOnPageHide(session); + + firePageHide(false); + expect(session.leaveRoomSession).toHaveBeenCalledOnce(); + + remove(); + firePageHide(false); + expect(session.leaveRoomSession).toHaveBeenCalledOnce(); + }); + + it('stays in the call when the page is only frozen', () => { + const session = makeSession(); + const remove = leaveMatrixRTCOnPageHide(session); + + firePageHide(true); + + expect(session.leaveRoomSession).not.toHaveBeenCalled(); + remove(); + }); +}); diff --git a/src/app/features/call/matrixRtcCallLifecycle.ts b/src/app/features/call/matrixRtcCallLifecycle.ts new file mode 100644 index 0000000000..f9ade34257 --- /dev/null +++ b/src/app/features/call/matrixRtcCallLifecycle.ts @@ -0,0 +1,310 @@ +import type { AutoDiscoveryInfo } from '../../cs-api'; +import { + KnownMembership, + MatrixRTCSessionEvent, + type CallMembership, + type JoinSessionConfig, + type MatrixClient, + type MatrixRTCSession, + type Room, +} from '$types/matrix-sdk'; +import { + getPreferredLivekitTransport, + isLivekitTransportConfig, + provisionLivekitToken, +} from './livekitProvisioning'; +import { advertiseCallTransport, callMemberId } from './callProtocol'; +import type { LivekitProvisioningResult } from './livekitProvisioning'; +import { getSlidingSyncManager } from '$client/initMatrix'; +import { createDebugLogger } from '$utils/debugLogger'; + +const debugLog = createDebugLogger('matrixRtcCallLifecycle'); + +// Without delayed events nothing on the server retracts our membership when the +// app dies, so the expiry is what bounds a ghost participant. The SDK refreshes +// the event 5s before it lapses, so half an hour is far more headroom than a +// live call needs while cutting the stale window down from the 4h default. +const membershipEventExpiryMs = 30 * 60 * 1000; + +const membershipWaitTimeoutMs = 30_000; + +export type MatrixRTCJoinProvisionOptions = { + mx: MatrixClient; + room: Room; + session: MatrixRTCSession; + discovery?: Pick; + getPreferredTransport?: typeof getPreferredLivekitTransport; + provisionToken?: typeof provisionLivekitToken; + callIntent: JoinSessionConfig['callIntent']; + notificationType?: JoinSessionConfig['notificationType']; + manageMediaKeys?: boolean; + isCancelled?: () => boolean; + onStage?: (stage: 'joining-matrix' | 'provisioning') => void; + onMembershipWait?: (cancel: (() => void) | undefined) => void; + onCallRoomSubscribed?: (unsubscribe: () => void) => void; + onJoinStarted?: () => void; +}; + +export type MatrixRTCJoinProvisionResult = { + ownMembership: CallMembership | undefined; + provisioned: LivekitProvisioningResult; +}; + +type MembershipWait = { + promise: Promise; + cancel: () => void; +}; + +export const ROSTER_HYDRATION_ERROR = 'MatrixRTC roster hydration failed'; + +const loadRoster = async (room: Room): Promise => { + try { + return await room.loadMembersIfNeeded(); + } catch (error) { + debugLog.error('call', `roster hydration threw for ${room.roomId}`, error); + throw new Error(ROSTER_HYDRATION_ERROR, { cause: error }); + } +}; + +const rosterComplete = (room: Room): boolean => + room.membersLoaded() && + room.getMembersWithMembership(KnownMembership.Join).length >= room.getJoinedMemberCount(); + +const rosterState = (room: Room, fromServer: boolean): string => + `joined=${room.getMembersWithMembership(KnownMembership.Join).length}/${room.getJoinedMemberCount()} loaded=${room.membersLoaded()} fromServer=${fromServer}`; + +/** + * The SDK needs the full roster before we publish a membership: without it + * `isValidMembership` discards the other participants, which breaks key + * exchange both ways at once. + */ +const hydrateCallRoster = async (room: Room): Promise => { + let fromServer = await loadRoster(room); + + // `loadMembers` only asks the server when the out-of-band store is empty or + // the room is encrypted, so an unencrypted room can be answered from a stale + // cache. Clearing it forces the next load to fetch. + if (!rosterComplete(room) && !fromServer) { + debugLog.warn('call', `roster stale for ${room.roomId}, refetching: ${rosterState(room, false)}`); + try { + await room.clearLoadedMembersIfNeeded(); + } catch (error) { + debugLog.error('call', `clearing the cached roster failed for ${room.roomId}`, error); + throw new Error(ROSTER_HYDRATION_ERROR, { cause: error }); + } + fromServer = await loadRoster(room); + } + + if (!rosterComplete(room)) { + debugLog.error('call', `roster incomplete for ${room.roomId}: ${rosterState(room, fromServer)}`); + throw new Error(ROSTER_HYDRATION_ERROR); + } + debugLog.info('call', `roster hydrated for ${room.roomId}: ${rosterState(room, fromServer)}`); +}; + +const waitForOwnMembership = ( + session: MatrixRTCSession, + userId: string, + deviceId: string +): MembershipWait => { + let resolveWait!: () => void; + let rejectWait!: (reason?: unknown) => void; + let settled = false; + let timeout: ReturnType | undefined; + let membershipsListenerInstalled = false; + let membershipErrorListenerInstalled = false; + + const handleMembershipsChanged = ( + _oldMemberships: CallMembership[], + memberships: CallMembership[] + ): void => { + debugLog.info( + 'call', + `membership changed: n=${memberships.length} want=${userId}:${deviceId} have=${memberships.map((m) => `${m.userId}:${m.deviceId}`).join(',')}` + ); + + if ( + memberships.some( + (membership) => membership.userId === userId && membership.deviceId === deviceId + ) + ) { + settle(resolveWait); + } + }; + + const handleMembershipManagerError = (): void => { + settle(() => rejectWait(new Error('MatrixRTC membership publication failed'))); + }; + + const removeListeners = (): void => { + if (membershipsListenerInstalled) { + try { + session.removeListener(MatrixRTCSessionEvent.MembershipsChanged, handleMembershipsChanged); + } catch {} + membershipsListenerInstalled = false; + } + if (membershipErrorListenerInstalled) { + try { + session.removeListener( + MatrixRTCSessionEvent.MembershipManagerError, + handleMembershipManagerError + ); + } catch {} + membershipErrorListenerInstalled = false; + } + }; + + const settle = (settlePromise: () => void): void => { + if (settled) return; + settled = true; + if (timeout !== undefined) clearTimeout(timeout); + removeListeners(); + settlePromise(); + }; + + const promise = new Promise((resolve, reject) => { + resolveWait = resolve; + rejectWait = reject; + }); + // joinRTCSession can throw before the promise is awaited; keep the cancel + // rejection from surfacing as an unhandled rejection in that window. + promise.catch(() => {}); + + try { + session.on(MatrixRTCSessionEvent.MembershipsChanged, handleMembershipsChanged); + membershipsListenerInstalled = true; + session.on(MatrixRTCSessionEvent.MembershipManagerError, handleMembershipManagerError); + membershipErrorListenerInstalled = true; + timeout = setTimeout( + () => settle(() => rejectWait(new Error('MatrixRTC membership publication timed out'))), + membershipWaitTimeoutMs + ); + } catch { + settle(() => rejectWait(new Error('MatrixRTC membership listener setup failed'))); + } + + return { + promise, + cancel: () => settle(() => rejectWait(new Error('MatrixRTC membership wait cancelled'))), + }; +}; + +export const joinAndProvisionMatrixRTC = async ({ + mx, + room, + session, + discovery, + getPreferredTransport = getPreferredLivekitTransport, + provisionToken = provisionLivekitToken, + callIntent, + notificationType, + manageMediaKeys = false, + isCancelled, + onStage, + onMembershipWait, + onCallRoomSubscribed, + onJoinStarted, +}: MatrixRTCJoinProvisionOptions): Promise => { + const deviceId = mx.getDeviceId(); + if (!deviceId) throw new Error('MatrixRTC device unavailable'); + + const preferredTransport = await getPreferredTransport(mx, discovery); + if (!preferredTransport) { + debugLog.error('call', `no LiveKit transport advertised for ${room.roomId}`); + throw new Error('No LiveKit transport available'); + } + + const advertisedTransport = advertiseCallTransport(preferredTransport, room.roomId); + + const userId = mx.getSafeUserId(); + const identity = { userId, deviceId, memberId: callMemberId(userId, deviceId) }; + if (isCancelled?.()) throw new Error('MatrixRTC setup cancelled'); + + // The subscription keeps the roster fresh for the rest of the call, but it + // cannot be waited on: MSC4186 only returns a room when it has new data or + // has never been sent, and Synapse does not return expanded state until the + // room next changes. So fetch the roster outright rather than hoping sync + // volunteers it. `lazyLoadMembers` makes this necessary on both sync paths. + const callRoomSubscription = getSlidingSyncManager(mx)?.subscribeToCallRoom(room.roomId); + if (callRoomSubscription) onCallRoomSubscribed?.(callRoomSubscription); + await hydrateCallRoster(room); + if (isCancelled?.()) throw new Error('MatrixRTC setup cancelled'); + + const membershipWait = waitForOwnMembership(session, identity.userId, identity.deviceId); + onMembershipWait?.(membershipWait.cancel); + onStage?.('joining-matrix'); + + try { + const joinConfig: JoinSessionConfig = { + callIntent, + membershipEventExpiryMs, + ...(notificationType ? { notificationType } : {}), + ...(manageMediaKeys ? { manageMediaKeys: true } : {}), + }; + onJoinStarted?.(); + session.joinRTCSession(identity, [advertisedTransport], undefined, joinConfig); + await membershipWait.promise; + debugLog.info( + 'call', + `own membership published in ${room.roomId} as ${identity.userId}:${identity.deviceId}` + ); + } catch (error) { + membershipWait.cancel(); + debugLog.error('call', `own membership was not published in ${room.roomId}`, error); + throw error; + } finally { + onMembershipWait?.(undefined); + } + + if (isCancelled?.()) throw new Error('MatrixRTC setup cancelled'); + const slotId = session.slotId; + if (!slotId) throw new Error('MatrixRTC slot was not assigned'); + const ownMembership = session.memberships?.find( + (membership) => + membership.userId === identity.userId && membership.deviceId === identity.deviceId + ); + + // Use the oldest membership's transport so every participant uses the same SFU. + const oldestMembership = session.getOldestMembership(); + const oldestTransport = oldestMembership?.getTransport(oldestMembership); + const fromOldestMembership = !!oldestTransport && isLivekitTransportConfig(oldestTransport); + const callTransport = fromOldestMembership ? oldestTransport : preferredTransport; + + debugLog.info( + 'call', + `sfu selected for ${room.roomId}: source=${fromOldestMembership ? 'oldest-membership' : 'preferred'} url=${callTransport.livekit_service_url} slot=${slotId}` + ); + onStage?.('provisioning'); + const provisioned = await provisionToken({ + mx, + roomId: room.roomId, + deviceId, + serviceUrl: callTransport.livekit_service_url, + }); + if (isCancelled?.()) throw new Error('MatrixRTC setup cancelled'); + + return { ownMembership, provisioned }; +}; + +export const leaveMatrixRTCOnPageHide = (session: MatrixRTCSession): (() => void) => { + const handlePageHide = (event: PageTransitionEvent): void => { + // A persisted page is only frozen (mobile app switch, back/forward cache) + // and the call is still ours when it resumes; only a real teardown leaves. + if (event.persisted) return; + void session.leaveRoomSession().catch(() => undefined); + }; + window.addEventListener('pagehide', handlePageHide); + return () => window.removeEventListener('pagehide', handlePageHide); +}; + +export const disconnectLivekitThenLeaveMatrixRTC = async ( + disconnect: () => Promise, + session: MatrixRTCSession +): Promise => { + try { + await disconnect(); + } catch {} + try { + await session.leaveRoomSession(5000); + } catch {} +}; From 25ddfbd5713ace1dec395b5f5138b740373adc29 Mon Sep 17 00:00:00 2001 From: Erwan Leboucher Date: Sat, 1 Aug 2026 23:33:28 +0200 Subject: [PATCH 4/6] feat(call): put both call engines behind one transport seam --- src/app/features/call/callTransport.ts | 95 +++ .../features/call/livekitJsController.test.ts | 534 ++++++++++++++++ src/app/features/call/livekitJsController.ts | 402 +++++++++++++ src/app/features/call/livekitMobileBridge.ts | 275 +++++++++ .../call/nativeCallController.test.ts | 569 ++++++++++++++++++ src/app/features/call/nativeCallController.ts | 417 +++++++++++++ src/app/features/call/nativeCallProbe.test.ts | 67 +++ src/app/features/call/nativeCallProbe.ts | 19 + src/app/features/call/nativeTransport.test.ts | 377 ++++++++++++ src/app/features/call/nativeTransport.ts | 291 +++++++++ 10 files changed, 3046 insertions(+) create mode 100644 src/app/features/call/callTransport.ts create mode 100644 src/app/features/call/livekitJsController.test.ts create mode 100644 src/app/features/call/livekitJsController.ts create mode 100644 src/app/features/call/livekitMobileBridge.ts create mode 100644 src/app/features/call/nativeCallController.test.ts create mode 100644 src/app/features/call/nativeCallController.ts create mode 100644 src/app/features/call/nativeCallProbe.test.ts create mode 100644 src/app/features/call/nativeCallProbe.ts create mode 100644 src/app/features/call/nativeTransport.test.ts create mode 100644 src/app/features/call/nativeTransport.ts diff --git a/src/app/features/call/callTransport.ts b/src/app/features/call/callTransport.ts new file mode 100644 index 0000000000..5ed6f0eca2 --- /dev/null +++ b/src/app/features/call/callTransport.ts @@ -0,0 +1,95 @@ +/** + * The seam between the shared call lifecycle and the media stack that carries + * it: an in-page `livekit-client` Room on the web, a native LiveKit SDK behind + * IPC on mobile. Everything above this type is written once. + */ + +export type CallEncryptionKey = { + /** + * The identity the SFU knows, which is the one the JWT granted. It is only + * ever read off `EncryptionKeyChanged`: the shape depends on the membership + * format the SFU was told to expect, so it cannot be derived locally. + */ + identity: string; + /** + * A slot in LiveKit's per-participant key ring, not a sequence number. The + * sender writes it into every frame, slots are reused modulo 256, and a peer + * that rejoins restarts at 0. Carry it through untouched: filtering on it + * strands every frame encrypted with the key that was dropped. + */ + keyIndex: number; + /** Raw material, as `EncryptionKeyChanged` gives it. Transports encode. */ + key: Uint8Array; +}; + +export type CallTrack = { + id: string; + muted: boolean; + subscribed: boolean; +}; + +export type CallConnectionQuality = 'lost' | 'poor' | 'good' | 'excellent' | 'unknown'; + +export type CallParticipant = { + identity: string; + camera?: CallTrack; + screenShare?: CallTrack; + connectionQuality?: CallConnectionQuality; +}; + +export type CallTransportConnection = 'connecting' | 'connected' | 'reconnecting' | 'disconnected'; + +export type CallTransportState = { + connection: CallTransportConnection; + /** Remote peers only; the local participant is never listed. */ + participants: CallParticipant[]; + microphoneEnabled: boolean; + cameraEnabled: boolean; + /** User-facing copy, set only when the media stack failed. */ + error?: string; +}; + +export type CallTransportConnectOptions = { + url: string; + token: string; + microphoneEnabled: boolean; + cameraEnabled: boolean; + /** Keys known at connect time; `setEncryptionKey` keeps feeding the rest. */ + encryptionKeys: CallEncryptionKey[]; +}; + +/** `name` is display-ready; `type` is the bounded platform vocabulary. */ +export type CallAudioRoute = { + id: string; + name: string; + type: string; + current: boolean; +}; + +/** + * Extras only one platform has. Feature-detecting these keeps `CallTransport` + * free of methods half the implementations would have to throw on, and keeps + * the CallKit/Telecom surface out of the web path. + */ +export type CallTransportCapabilities = { + camera?: { switch: () => Promise }; + audioRoutes?: { + list: () => Promise; + select: (routeId: string) => Promise; + }; + pictureInPicture?: { setEnabled: (enabled: boolean) => Promise }; +}; + +export type CallTransport = { + connect: (options: CallTransportConnectOptions) => Promise; + disconnect: () => Promise; + + setMicrophoneEnabled: (enabled: boolean) => Promise; + setCameraEnabled: (enabled: boolean) => Promise; + /** Keys arriving before `connect` resolves are queued, never dropped. */ + setEncryptionKey: (key: CallEncryptionKey) => Promise; + + subscribe: (listener: (state: CallTransportState) => void) => () => void; + getState: () => CallTransportState; + capabilities: CallTransportCapabilities; +}; diff --git a/src/app/features/call/livekitJsController.test.ts b/src/app/features/call/livekitJsController.test.ts new file mode 100644 index 0000000000..b60044866b --- /dev/null +++ b/src/app/features/call/livekitJsController.test.ts @@ -0,0 +1,534 @@ +import { beforeEach, describe, expect, it, vi, type Mock } from 'vitest'; + +import { + MatrixRTCSessionEvent, + type CallMembership, + type MatrixRTCSession, +} from '$types/matrix-sdk'; +import type { MatrixClient, Room as MatrixRoom } from '$types/matrix-sdk'; +import type { RoomOptions } from 'livekit-client'; +import { + createLivekitJsController, + type LivekitJsControllerDependencies, +} from './livekitJsController'; +import type { + LivekitMatrixKeyProvider, + LivekitMatrixKeyProviderState, +} from './livekitMatrixKeyProvider'; +import type { CallEncryptionKey } from './callTransport'; +import { resetCallOwnerForTests } from '$state/callOwner'; + +const transport = { + type: 'livekit' as const, + livekit_service_url: 'https://sfu.example', +}; +const makeMatrixRoom = (encrypted: boolean): MatrixRoom => + ({ + roomId: '!room:example.org', + loadMembersIfNeeded: () => Promise.resolve(true), + membersLoaded: () => true, + getMembersWithMembership: () => [{}, {}], + getJoinedMemberCount: () => 2, + // MSC4143 ties media encryption to the room. + hasEncryptionStateEvent: () => encrypted, + }) as unknown as MatrixRoom; + +const room = makeMatrixRoom(true); +const unencryptedRoom = makeMatrixRoom(false); +const membership = { + userId: '@alice:example.org', + deviceId: 'DEVICE', + rtcBackendIdentity: 'local-backend-identity', +} as CallMembership; + +const deferred = (): { + promise: Promise; + resolve: (value: T) => void; +} => { + let resolve!: (value: T) => void; + const promise = new Promise((promiseResolve) => { + resolve = promiseResolve; + }); + return { promise, resolve }; +}; + +type SessionHandler = (...args: unknown[]) => void; + +type TestSession = MatrixRTCSession & { + handlers: Map; +}; + +const makeSession = (order: string[] = []): TestSession => { + const handlers = new Map(); + const session = { + handlers, + memberships: [] as CallMembership[], + slotId: 'm.call#real-slot', + on: vi.fn<(...args: unknown[]) => void>().mockImplementation((event, handler) => { + if (event === MatrixRTCSessionEvent.EncryptionKeyChanged) order.push('attach'); + handlers.set(event as MatrixRTCSessionEvent, handler as SessionHandler); + }), + off: vi.fn<(...args: unknown[]) => void>().mockImplementation((event, handler) => { + if (event === MatrixRTCSessionEvent.EncryptionKeyChanged) order.push('detach'); + if (handlers.get(event as MatrixRTCSessionEvent) === handler) { + handlers.delete(event as MatrixRTCSessionEvent); + } + }), + reemitEncryptionKeys: vi.fn<() => void>(), + removeListener: vi.fn<(...args: unknown[]) => void>().mockImplementation((event, handler) => { + if (handlers.get(event as MatrixRTCSessionEvent) === handler) { + handlers.delete(event as MatrixRTCSessionEvent); + } + }), + joinRTCSession: vi + .fn<(...args: unknown[]) => void>() + .mockImplementation(() => order.push('join')), + getOldestMembership: vi.fn<() => CallMembership | undefined>().mockReturnValue(undefined), + leaveRoomSession: vi.fn().mockImplementation(async () => { + order.push('leave'); + return true; + }), + } as unknown as TestSession; + return session; +}; + +const emitOwnMembership = (session: TestSession): void => { + session.memberships = [membership]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)?.([], [membership]); +}; + +type FakeProvider = { + state: LivekitMatrixKeyProviderState; + emit: (state: LivekitMatrixKeyProviderState) => void; + setKey: Mock<(key: CallEncryptionKey, own: boolean) => void>; + reset: Mock<() => void>; + getKeyState: Mock<() => LivekitMatrixKeyProviderState>; + subscribe: Mock< + (listener: (state: Readonly) => void) => () => void + >; +}; + +const makeProvider = (state: Partial = {}): FakeProvider => { + const listeners = new Set<(state: Readonly) => void>(); + const provider = { + emit: (next: LivekitMatrixKeyProviderState) => + listeners.forEach((listener) => listener(next)), + state: { + ready: false, + localOutboundIdentity: null, + keyIndex: null, + lastImportFailure: null, + ...state, + }, + setKey: vi.fn<(key: CallEncryptionKey, own: boolean) => void>(), + reset: vi.fn<() => void>(), + getKeyState: vi.fn<() => LivekitMatrixKeyProviderState>(), + subscribe: + vi.fn<(listener: (state: Readonly) => void) => () => void>(), + } as FakeProvider; + provider.getKeyState.mockImplementation(() => provider.state); + provider.subscribe.mockImplementation((listener) => { + listeners.add(listener); + listener(provider.state); + return () => listeners.delete(listener); + }); + return provider; +}; + +const makeClient = (session: MatrixRTCSession): MatrixClient => + ({ + getDeviceId: () => 'DEVICE', + getSafeUserId: () => '@alice:example.org', + matrixRTC: { getRoomSession: () => session }, + on: vi.fn<(event: string, cb: unknown) => void>(), + removeListener: vi.fn<(event: string, cb: unknown) => void>(), + }) as unknown as MatrixClient; + +const makeDependencies = ( + session: TestSession, + order: string[], + provider = makeProvider() +): { + dependencies: LivekitJsControllerDependencies; + provider: FakeProvider; + roomOptions: { value?: RoomOptions }; + livekitRoom: { + connect: Mock<(url: string, token: string, options?: unknown) => Promise>; + disconnect: Mock<() => Promise>; + setE2EEEnabled: Mock<(enabled: boolean) => Promise>; + }; +} => { + const worker = { terminate: vi.fn<() => void>() } as unknown as Worker; + const livekitRoom = { + connect: vi + .fn<(url: string, token: string, options?: unknown) => Promise>() + .mockImplementation(async () => { + order.push('connect'); + }), + disconnect: vi.fn<() => Promise>().mockImplementation(async () => { + order.push('disconnect'); + }), + setE2EEEnabled: vi.fn<(enabled: boolean) => Promise>().mockImplementation(async () => { + order.push('e2ee-enabled'); + }), + }; + const roomOptions: { value?: RoomOptions } = {}; + return { + provider, + roomOptions, + livekitRoom, + dependencies: { + createKeyProvider: () => provider as unknown as LivekitMatrixKeyProvider, + isE2EESupported: () => true, + createWorker: () => { + order.push('worker'); + return worker; + }, + createRoom: (options) => { + order.push('room'); + roomOptions.value = options; + return livekitRoom; + }, + getPreferredTransport: async () => { + order.push('transport'); + return transport; + }, + provisionToken: async () => { + order.push('provision'); + return { url: 'wss://livekit.example', jwt: 'jwt' }; + }, + }, + }; +}; + +const connectToActive = async ( + controller: ReturnType, + session: TestSession, + matrixRoom: MatrixRoom = room +): Promise => { + const connectPromise = controller.connect({ mx: makeClient(session), room: matrixRoom }); + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + emitOwnMembership(session); + await connectPromise; +}; + +describe('livekit JS controller', () => { + beforeEach(() => { + resetCallOwnerForTests(); + }); + + it('attaches E2EE before joining and provisions before connecting one Room', async () => { + const order: string[] = []; + const session = makeSession(order); + const { dependencies, provider, roomOptions, livekitRoom } = makeDependencies(session, order); + const controller = createLivekitJsController(dependencies); + + await connectToActive(controller, session); + + expect(order).toEqual([ + 'attach', + 'transport', + 'join', + 'provision', + 'worker', + 'room', + 'connect', + 'e2ee-enabled', + ]); + expect(session.reemitEncryptionKeys).toHaveBeenCalledBefore(session.joinRTCSession as Mock); + expect(session.joinRTCSession).toHaveBeenCalledWith( + { + userId: '@alice:example.org', + deviceId: 'DEVICE', + memberId: '@alice:example.org:DEVICE', + }, + [{ ...transport, livekit_alias: room.roomId }], + undefined, + { + callIntent: 'audio', + membershipEventExpiryMs: 30 * 60 * 1000, + notificationType: 'notification', + manageMediaKeys: true, + } + ); + expect(roomOptions.value?.encryption).toEqual({ + keyProvider: provider, + worker: expect.anything(), + }); + expect(roomOptions.value?.adaptiveStream).toBe(true); + expect(roomOptions.value?.dynacast).toBe(true); + // Without this the local participant stays on Encryption_Type.NONE and + // livekit-client passes outbound frames through unencrypted. + expect(livekitRoom.setE2EEEnabled).toHaveBeenCalledWith(true); + expect(controller.getState().room).toBe(livekitRoom); + expect(controller.getState().lifecycle).toBe('active'); + }); + + it('runs an unencrypted call with no worker, key provider or encryption option', async () => { + const order: string[] = []; + const session = makeSession(order); + const { dependencies, provider, roomOptions, livekitRoom } = makeDependencies(session, order); + const createKeyProvider = vi + .fn<() => LivekitMatrixKeyProvider>() + .mockReturnValue(provider as unknown as LivekitMatrixKeyProvider); + dependencies.createKeyProvider = createKeyProvider; + const controller = createLivekitJsController(dependencies); + + await connectToActive(controller, session, unencryptedRoom); + + // MSC4143 forbids MatrixRTC encryption in an unencrypted room, so nothing + // of the key pipeline may be built for one. + expect(createKeyProvider).not.toHaveBeenCalled(); + expect(order).toEqual(['transport', 'join', 'provision', 'room', 'connect']); + expect(roomOptions.value?.encryption).toBeUndefined(); + expect(roomOptions.value?.adaptiveStream).toBe(true); + expect(roomOptions.value?.dynacast).toBe(true); + expect(livekitRoom.setE2EEEnabled).not.toHaveBeenCalled(); + expect(session.joinRTCSession).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + undefined, + expect.not.objectContaining({ manageMediaKeys: expect.anything() }) + ); + expect(controller.getState().lifecycle).toBe('active'); + }); + + it('reports media ready at once when the room is unencrypted', async () => { + const session = makeSession(); + const { dependencies } = makeDependencies(session, []); + const controller = createLivekitJsController(dependencies); + + await connectToActive(controller, session, unencryptedRoom); + + expect(controller.getState().mediaReady).toBe(true); + }); + + it('holds media until the matrix key is imported when the room is encrypted', async () => { + const session = makeSession(); + const provider = makeProvider(); + const { dependencies } = makeDependencies(session, [], provider); + const controller = createLivekitJsController(dependencies); + + await connectToActive(controller, session); + expect(controller.getState().mediaReady).toBe(false); + + provider.state = { ...provider.state, ready: true }; + provider.emit(provider.state); + + expect(controller.getState().mediaReady).toBe(true); + }); + + it('does not run an unencrypted call through the e2ee-unsupported check', async () => { + const session = makeSession(); + const { dependencies } = makeDependencies(session, []); + dependencies.isE2EESupported = () => false; + const controller = createLivekitJsController(dependencies); + + await connectToActive(controller, session, unencryptedRoom); + + expect(controller.getState().lifecycle).toBe('active'); + expect(controller.getState().failure).toBeNull(); + }); + + it('refuses unsupported E2EE without joining or creating a Room', async () => { + const session = makeSession(); + const { dependencies } = makeDependencies(session, []); + dependencies.isE2EESupported = () => false; + const controller = createLivekitJsController(dependencies); + + await controller.connect({ mx: makeClient(session), room }); + + expect(session.joinRTCSession).not.toHaveBeenCalled(); + expect(controller.getState()).toMatchObject({ + lifecycle: 'failed', + failure: 'e2ee-unsupported', + }); + }); + + it('refuses a provider import failure without connecting LiveKit', async () => { + const session = makeSession(); + const provider = makeProvider({ lastImportFailure: 'import-failed' }); + const { dependencies } = makeDependencies(session, [], provider); + const controller = createLivekitJsController(dependencies); + + await controller.connect({ mx: makeClient(session), room }); + + expect(session.joinRTCSession).not.toHaveBeenCalled(); + expect(controller.getState()).toMatchObject({ + lifecycle: 'failed', + failure: 'e2ee-import-failed', + }); + }); + + it('disconnects LiveKit before leaving MatrixRTC and cleans up idempotently', async () => { + const order: string[] = []; + const session = makeSession(order); + const { dependencies, provider } = makeDependencies(session, order); + const controller = createLivekitJsController(dependencies); + await connectToActive(controller, session); + + await Promise.all([controller.disconnect(), controller.disconnect()]); + + expect(order.indexOf('disconnect')).toBeGreaterThan(-1); + expect(order.indexOf('detach')).toBeGreaterThan(order.indexOf('disconnect')); + expect(order.indexOf('disconnect')).toBeLessThan(order.indexOf('leave')); + expect(order.indexOf('detach')).toBeLessThan(order.indexOf('leave')); + expect(provider.reset).toHaveBeenCalledOnce(); + expect(controller.getState().room).toBeUndefined(); + expect(controller.getState().lifecycle).toBe('idle'); + }); + + it('detaches once when setup fails after MatrixRTC join', async () => { + const session = makeSession(); + const { dependencies, provider } = makeDependencies(session, []); + dependencies.provisionToken = vi + .fn>() + .mockRejectedValue(new Error('provision failed')); + const controller = createLivekitJsController(dependencies); + const connectPromise = controller.connect({ + mx: makeClient(session), + room, + }); + + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + emitOwnMembership(session); + await connectPromise; + + expect(provider.reset).toHaveBeenCalledOnce(); + expect(session.leaveRoomSession).toHaveBeenCalledOnce(); + expect(controller.getState().room).toBeUndefined(); + expect(controller.getState().lifecycle).toBe('failed'); + }); + + it('cancels during provisioning without creating a Room or becoming active', async () => { + const session = makeSession(); + const pendingProvision = deferred<{ url: string; jwt: string }>(); + const { dependencies, provider, roomOptions } = makeDependencies(session, []); + dependencies.provisionToken = vi + .fn>() + .mockImplementation(() => pendingProvision.promise); + const controller = createLivekitJsController(dependencies); + const connectPromise = controller.connect({ + mx: makeClient(session), + room, + }); + + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + emitOwnMembership(session); + await vi.waitFor(() => expect(dependencies.provisionToken).toHaveBeenCalledOnce()); + const disconnectPromise = controller.disconnect(); + pendingProvision.resolve({ url: 'wss://livekit.example', jwt: 'jwt' }); + await Promise.all([connectPromise, disconnectPromise]); + + expect(roomOptions.value).toBeUndefined(); + expect(provider.reset).toHaveBeenCalledOnce(); + expect(session.leaveRoomSession).toHaveBeenCalledOnce(); + expect(controller.getState().room).toBeUndefined(); + expect(controller.getState().lifecycle).toBe('idle'); + }); + + it('cancels during Room.connect and cleans up its stale room before leaving', async () => { + const session = makeSession(); + const pendingConnect = deferred(); + const { dependencies, provider } = makeDependencies(session, []); + const livekitRoom = { + connect: vi + .fn<(url: string, token: string, options?: unknown) => Promise>() + .mockImplementation(() => pendingConnect.promise), + disconnect: vi.fn<() => Promise>().mockResolvedValue(undefined), + setE2EEEnabled: vi.fn<(enabled: boolean) => Promise>().mockResolvedValue(undefined), + }; + dependencies.createRoom = () => livekitRoom; + const controller = createLivekitJsController(dependencies); + const connectPromise = controller.connect({ + mx: makeClient(session), + room, + }); + + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + emitOwnMembership(session); + await vi.waitFor(() => expect(livekitRoom.connect).toHaveBeenCalledOnce()); + const disconnectPromise = controller.disconnect(); + pendingConnect.resolve(); + await Promise.all([connectPromise, disconnectPromise]); + + expect(livekitRoom.disconnect).toHaveBeenCalledOnce(); + expect(provider.reset).toHaveBeenCalledOnce(); + expect(session.leaveRoomSession).toHaveBeenCalledOnce(); + expect(controller.getState().lifecycle).toBe('idle'); + expect(controller.getState().lifecycle).not.toBe('active'); + }); + + it('rejects duplicate setup and cancellation leaves started MatrixRTC membership', async () => { + const session = makeSession(); + const { dependencies, provider } = makeDependencies(session, []); + const controller = createLivekitJsController(dependencies); + const firstConnect = controller.connect({ mx: makeClient(session), room }); + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + + await expect(controller.connect({ mx: makeClient(session), room })).rejects.toThrow( + 'already in use' + ); + await controller.disconnect(); + await firstConnect; + + expect(session.leaveRoomSession).toHaveBeenCalledWith(5000); + expect(provider.reset).toHaveBeenCalledOnce(); + expect(controller.getState().lifecycle).toBe('idle'); + }); + + it('exposes only lifecycle and state methods', () => { + const session = makeSession(); + const { dependencies } = makeDependencies(session, []); + const controller = createLivekitJsController(dependencies); + + expect(Object.keys(controller).toSorted()).toEqual([ + 'connect', + 'disconnect', + 'getState', + 'subscribe', + ]); + }); + + it('disconnects the previous controller before a replacement connects', async () => { + const firstSession = makeSession(); + const first = makeDependencies(firstSession, []); + const firstController = createLivekitJsController(first.dependencies); + await connectToActive(firstController, firstSession); + + const replacementSession = makeSession(); + const replacement = makeDependencies(replacementSession, [], makeProvider({ ready: true })); + const replacementController = createLivekitJsController(replacement.dependencies); + + await firstController.disconnect(); + expect(firstController.getState().room).toBeUndefined(); + await connectToActive(replacementController, replacementSession); + + expect(replacementController.getState().lifecycle).toBe('active'); + }); +}); diff --git a/src/app/features/call/livekitJsController.ts b/src/app/features/call/livekitJsController.ts new file mode 100644 index 0000000000..127c8be4de --- /dev/null +++ b/src/app/features/call/livekitJsController.ts @@ -0,0 +1,402 @@ +import { Room as LivekitRoom, type RoomOptions } from 'livekit-client'; +import type { AutoDiscoveryInfo } from '../../cs-api'; +import type { MatrixClient, MatrixRTCSession, Room as MatrixRoom } from '$types/matrix-sdk'; +import { disconnectLivekitThenLeaveMatrixRTC } from './matrixRtcCallLifecycle'; +import { + callEncryptsMedia, + createCallSessionHandles, + joinCallSession, + type CallSessionHandles, +} from './callSession'; +import { + LivekitMatrixKeyProvider, + type LivekitMatrixKeyProviderState, + isLivekitE2EESupported, +} from './livekitMatrixKeyProvider'; +import { getPreferredLivekitTransport, provisionLivekitToken } from './livekitProvisioning'; +import { createCallKeyPipeline, type CallKeyPipeline } from './callKeyPipeline'; +import { acquireCallOwner, type CallOwnerLease } from '$state/callOwner'; +import { createDebugLogger } from '$utils/debugLogger'; + +const debugLog = createDebugLogger('livekitJsController'); + +export type LivekitJsControllerLifecycle = + | 'idle' + | 'joining-matrix' + | 'provisioning' + | 'connecting-livekit' + | 'active' + | 'stopping' + | 'failed'; + +export type LivekitJsControllerFailure = 'e2ee-unsupported' | 'e2ee-import-failed' | 'setup-failed'; + +export type LivekitJsControllerState = { + lifecycle: LivekitJsControllerLifecycle; + failure: LivekitJsControllerFailure | null; + room?: LivekitRoom; + /** + * Whether local media may be published. An unencrypted call is ready at once; + * an encrypted one has to hold until the Matrix key is imported, or the first + * frames would go out in the clear. + */ + mediaReady: boolean; + e2ee: Readonly; +}; + +type LivekitJsControllerStateListener = (state: Readonly) => void; + +type LivekitJsConnectOptions = { + mx: MatrixClient; + room: MatrixRoom; + discovery?: Pick; + callIntent?: 'audio' | 'video'; + dm?: boolean; + ongoing?: boolean; +}; + +type LivekitRoomLike = Pick; + +type LivekitJsControllerBase = { + connect: (options: LivekitJsConnectOptions) => Promise; + disconnect: () => Promise; + getState: () => Readonly; + subscribe: (listener: LivekitJsControllerStateListener) => () => void; +}; + +export type LivekitJsControllerDependencies = { + createRoom?: (options: RoomOptions) => LivekitRoomLike; + createWorker?: () => Worker; + createKeyProvider?: () => LivekitMatrixKeyProvider; + isE2EESupported?: () => boolean; + getPreferredTransport?: typeof getPreferredLivekitTransport; + provisionToken?: typeof provisionLivekitToken; +}; + +type ControllerRecord = { + session: MatrixRTCSession; + pipeline: CallKeyPipeline; + // MSC4143: encryption is REQUIRED in encrypted rooms and MUST NOT be used in + // unencrypted ones. An unencrypted call therefore has no provider, no worker + // and no `encryption` option on the Room. + encryptMedia: boolean; + provider?: LivekitMatrixKeyProvider; + worker?: Worker; + room?: LivekitRoomLike; + handles: CallSessionHandles; + providerDetached: boolean; + ownerLease: CallOwnerLease; + cancelled: boolean; + e2eeFailure: boolean; + removeKeyStateListener?: () => void; + cleanupPromise?: Promise; + resourcesReady: Promise; + resolveResources: () => void; +}; + +const initialE2EEState: LivekitMatrixKeyProviderState = { + ready: false, + localOutboundIdentity: null, + keyIndex: null, + lastImportFailure: null, +}; + +const defaultCreateRoom = (options: RoomOptions): LivekitRoomLike => new LivekitRoom(options); + +const defaultCreateWorker = (): Worker => + new Worker(new URL('livekit-client/e2ee-worker', import.meta.url), { + type: 'module', + }); + +export function createLivekitJsController(dependencies: LivekitJsControllerDependencies = {}) { + const createRoom = dependencies.createRoom ?? defaultCreateRoom; + const createWorker = dependencies.createWorker ?? defaultCreateWorker; + const createKeyProvider = + dependencies.createKeyProvider ?? (() => new LivekitMatrixKeyProvider()); + const supportsE2EE = dependencies.isE2EESupported ?? isLivekitE2EESupported; + const getPreferredTransport = dependencies.getPreferredTransport ?? getPreferredLivekitTransport; + const provisionToken = dependencies.provisionToken ?? provisionLivekitToken; + + let state: LivekitJsControllerState = { + lifecycle: 'idle', + failure: null, + room: undefined, + mediaReady: false, + e2ee: initialE2EEState, + }; + let record: ControllerRecord | undefined; + let operation: Promise | undefined; + const listeners = new Set(); + + const publish = (changes: Partial): void => { + state = { ...state, ...changes }; + const snapshot = { ...state, e2ee: { ...state.e2ee } }; + listeners.forEach((listener) => { + try { + listener(snapshot); + } catch { + // A state observer must not interrupt lifecycle cleanup. + } + }); + }; + + const cleanup = async ( + current: ControllerRecord, + result: 'idle' | 'failed', + failure: LivekitJsControllerFailure | null + ): Promise => { + if (current.cleanupPromise) { + await current.cleanupPromise; + return; + } + + current.cancelled = true; + current.handles.cancelMembershipWait?.(); + current.handles.cancelMembershipWait = undefined; + publish({ lifecycle: 'stopping' }); + const detachProvider = (): void => { + if (current.providerDetached) return; + current.providerDetached = true; + current.pipeline.detach(); + current.provider?.reset(); + }; + current.cleanupPromise = (async () => { + if (!current.room && !current.worker) await current.resourcesReady; + const stopRoom = async (): Promise => { + await current.room?.disconnect(); + }; + if (current.handles.joinStarted) { + await disconnectLivekitThenLeaveMatrixRTC(async () => { + try { + await stopRoom(); + } finally { + detachProvider(); + } + }, current.session); + } else { + try { + await stopRoom(); + } catch { + // Cleanup continues even when a setup room rejects disconnect. + } finally { + detachProvider(); + } + } + current.worker?.terminate(); + current.handles.removePageHideListener?.(); + current.handles.removePageHideListener = undefined; + current.handles.unsubscribeCallRoom?.(); + current.handles.unsubscribeCallRoom = undefined; + current.removeKeyStateListener?.(); + current.removeKeyStateListener = undefined; + current.ownerLease.release(); + if (record === current) record = undefined; + publish({ + lifecycle: result, + failure, + room: undefined, + mediaReady: false, + ...(result === 'idle' ? { e2ee: initialE2EEState } : {}), + }); + })(); + await current.cleanupPromise; + }; + + const setup = async ( + current: ControllerRecord, + connectOptions: LivekitJsConnectOptions + ): Promise => { + let failure: LivekitJsControllerFailure | null = null; + const { encryptMedia, provider } = current; + try { + publish({ lifecycle: 'joining-matrix', failure: null }); + debugLog.info( + 'call', + `setup started for ${connectOptions.room.roomId}: encryptMedia=${encryptMedia}` + ); + if (encryptMedia && !supportsE2EE()) { + failure = 'e2ee-unsupported'; + } else { + if (provider) { + current.pipeline.setOnKey(provider.setKey); + current.pipeline.attach(current.session, { + userId: connectOptions.mx.getSafeUserId(), + deviceId: connectOptions.mx.getDeviceId(), + }); + } + if (current.e2eeFailure || provider?.getKeyState().lastImportFailure) { + failure = 'e2ee-import-failed'; + } else { + const joined = await joinCallSession( + { + mx: connectOptions.mx, + room: connectOptions.room, + session: current.session, + discovery: connectOptions.discovery, + getPreferredTransport, + provisionToken, + callIntent: connectOptions.callIntent ?? 'audio', + dm: connectOptions.dm ?? false, + ongoing: connectOptions.ongoing ?? false, + encryptMedia, + isCancelled: () => current.cancelled, + onStage: (stage) => publish({ lifecycle: stage }), + }, + current.handles + ); + + if (current.e2eeFailure || provider?.getKeyState().lastImportFailure) { + failure = 'e2ee-import-failed'; + } else if (!current.cancelled) { + publish({ lifecycle: 'connecting-livekit' }); + if (provider) current.worker = createWorker(); + current.room = createRoom({ + // Both default to false in livekit-client. Without them a + // multi-party call receives every published layer at full quality + // and keeps sending video nobody is displaying. The native engine + // already enables both. + adaptiveStream: true, + dynacast: true, + ...(provider && current.worker + ? { encryption: { keyProvider: provider, worker: current.worker } } + : {}), + }); + await current.room.connect(joined.provisioned.url, joined.provisioned.jwt); + debugLog.info('call', `livekit connected for ${connectOptions.room.roomId}`); + // The Room only wires up the key provider; it publishes in the + // clear until this flips `LocalParticipant.encryptionType` to GCM, + // which is also what tells the SFU our tracks are encrypted. It has + // to happen before any track is published, so before 'active'. + if (encryptMedia) await current.room.setE2EEEnabled(true); + if (current.e2eeFailure) failure = 'e2ee-import-failed'; + } + } + } + } catch (error) { + failure = current.e2eeFailure ? 'e2ee-import-failed' : 'setup-failed'; + debugLog.error('call', `setup failed for ${connectOptions.room.roomId}: ${failure}`, error); + } finally { + current.resolveResources(); + } + + if (failure) { + debugLog.warn('call', `call did not start in ${connectOptions.room.roomId}: ${failure}`); + await cleanup(current, 'failed', failure); + } else if (current.cancelled) { + await cleanup(current, 'idle', null); + } else { + publish({ + lifecycle: 'active', + failure: null, + room: current.room as LivekitRoom, + }); + } + }; + + const connect = (connectOptions: LivekitJsConnectOptions): Promise => { + if (state.lifecycle === 'failed' && !record && !operation) { + publish({ + lifecycle: 'idle', + failure: null, + room: undefined, + mediaReady: false, + e2ee: initialE2EEState, + }); + } + if (record || operation || state.lifecycle !== 'idle') { + return Promise.reject(new Error('LiveKit JS call controller is already in use')); + } + + const ownerLease = acquireCallOwner('livekit-js', connectOptions.room.roomId); + if (!ownerLease) { + publish({ + lifecycle: 'failed', + failure: 'setup-failed', + room: undefined, + }); + return Promise.resolve(); + } + + let resolveResources!: () => void; + const resourcesReady = new Promise((resolve) => { + resolveResources = resolve; + }); + const encryptMedia = callEncryptsMedia(connectOptions.room); + let session: MatrixRTCSession; + let provider: LivekitMatrixKeyProvider | undefined; + try { + session = connectOptions.mx.matrixRTC.getRoomSession(connectOptions.room); + if (encryptMedia) provider = createKeyProvider(); + } catch { + ownerLease.release(); + publish({ + lifecycle: 'failed', + failure: 'setup-failed', + room: undefined, + }); + return Promise.resolve(); + } + const current: ControllerRecord = { + session, + pipeline: createCallKeyPipeline(), + encryptMedia, + provider, + handles: createCallSessionHandles(), + cancelled: false, + e2eeFailure: false, + providerDetached: false, + ownerLease, + resourcesReady, + resolveResources, + }; + record = current; + // An unencrypted call has no key to wait for, so its media is ready as soon + // as it starts. Gating it on the Matrix key would hold the controls and the + // prescreen choice forever, since no key is ever imported. + publish({ mediaReady: !encryptMedia }); + current.removeKeyStateListener = provider?.subscribe((e2ee) => { + publish({ e2ee, mediaReady: e2ee.ready }); + debugLog.info( + 'call', + `matrix key state for ${connectOptions.room.roomId}: ready=${e2ee.ready} index=${e2ee.keyIndex ?? 'none'} failure=${e2ee.lastImportFailure ?? 'none'}` + ); + if (e2ee.lastImportFailure) { + current.e2eeFailure = true; + current.handles.cancelMembershipWait?.(); + if (current.room) void cleanup(current, 'failed', 'e2ee-import-failed'); + } + }); + + operation = setup(current, connectOptions).finally(() => { + operation = undefined; + }); + return operation; + }; + + const disconnect = async (): Promise => { + if (!record) { + if (state.lifecycle === 'failed') { + publish({ lifecycle: 'idle', failure: null }); + } + return; + } + await cleanup(record, 'idle', null); + await operation; + }; + + const controller: LivekitJsControllerBase = { + connect, + disconnect, + getState: (): Readonly => ({ + ...state, + e2ee: { ...state.e2ee }, + }), + subscribe: (listener: LivekitJsControllerStateListener): (() => void) => { + listeners.add(listener); + listener({ ...state, e2ee: { ...state.e2ee } }); + return () => listeners.delete(listener); + }, + }; + + return controller; +} diff --git a/src/app/features/call/livekitMobileBridge.ts b/src/app/features/call/livekitMobileBridge.ts new file mode 100644 index 0000000000..cede91488a --- /dev/null +++ b/src/app/features/call/livekitMobileBridge.ts @@ -0,0 +1,275 @@ +import { addPluginListener, invoke, type PluginListener } from '@tauri-apps/api/core'; +import { listen, type UnlistenFn } from '@tauri-apps/api/event'; +import type { CallAudioRoute } from './callTransport'; + +export type NativeCallConnectionState = + | 'idle' + | 'connecting' + | 'connected' + | 'reconnecting' + | 'failed'; + +export type NativeCallFailureCode = + | 'invalid_request' + | 'busy' + | 'permission_denied' + | 'connect_failed' + | 'media_failed' + | 'disconnected' + | 'cancelled' + | 'unavailable' + | 'unexpected'; + +export type NativeCallCapabilities = { + supported: boolean; + microphone: boolean; + backgroundAudio: boolean; + nativeRoom: boolean; + camera: boolean; + nativeVideoOverlay: boolean; + callKit: boolean; +}; + +export type NativeCallEncryptionKeyPayload = { + identity: string; + keyIndex: number; + key: string; +}; + +export type NativeCallRemoteCamera = { + sid: string; + muted: boolean; + subscribed: boolean; +}; + +export type NativeCallRemoteParticipant = { + identity: string; + camera?: NativeCallRemoteCamera; + screenShare?: NativeCallRemoteCamera; + connectionQuality?: string; +}; + +export type NativeCallSnapshot = { + revision: number; + callId: string | null; + connectionState: NativeCallConnectionState; + microphoneEnabled: boolean; + cameraEnabled: boolean; + participantCount: number; + // Present on current native builds; optional so older payloads and test + // fixtures without the field remain valid. + remoteParticipants?: NativeCallRemoteParticipant[]; + lastError?: { code: NativeCallFailureCode; message: string }; +}; + +export type ConnectNativeCallRequest = { + callId: string; + url: string; + token: string; + microphoneEnabled: boolean; + encryptionKeys?: NativeCallEncryptionKeyPayload[]; +}; + +export type SetNativeCallEncryptionKeyRequest = { + callId: string; + identity: string; + keyIndex: number; + key: string; +}; + +/** + * Pins the single native-rendered remote camera view over a DOM tile. `x`, + * `y`, `width`, `height` are the tile's viewport-relative CSS rect; the + * native side maps them into view coordinates. A repeated call with the same + * track repositions the view; a new track rebinds it. + */ +export type SetNativeCallRemoteVideoOverlayRequest = { + callId: string; + participantIdentity: string; + trackId: string; + x: number; + y: number; + width: number; + height: number; + devicePixelRatio: number; +}; + +export type ClearNativeCallRemoteVideoOverlayRequest = { + callId: string; +}; + +export type SetNativeCallLocalVideoOverlayRequest = { + callId: string; + x: number; + y: number; + width: number; + height: number; + devicePixelRatio: number; +}; + +export type ClearNativeCallLocalVideoOverlayRequest = { + callId: string; +}; + +export type StartSystemCallRequest = { + callId: string; + uuid: string; + callerName: string; +}; + +export type EndSystemCallRequest = { + callId: string; + remoteEnded?: boolean; +}; + +export type SetSystemCallMutedRequest = { + callId: string; + muted: boolean; +}; + +export type GetAudioRoutesRequest = { + callId: string; +}; + +export type SetAudioRouteRequest = { + callId: string; + routeId: string; +}; + +export type UpdateCallDisplayRequest = { + callId: string; + callerName: string; + hasVideo?: boolean; +}; + +export type NativeCallAudioRoute = CallAudioRoute; + +export type GetAudioRoutesResponse = { + routes: NativeCallAudioRoute[]; + receiver: NativeCallSnapshot; +}; + +export type SystemCallActionKind = 'answer' | 'end' | 'mute'; + +export type SystemCallAction = { + action: SystemCallActionKind; + uuid: string; + muted?: boolean; +}; + +const NATIVE_CALL_EVENT = 'plugin:livekit-mobile://native-call-event'; + +export const getNativeCallCapabilities = (): Promise => + invoke('plugin:livekit-mobile|get_native_call_capabilities'); + +export const connectNativeCall = (request: ConnectNativeCallRequest): Promise => + invoke('plugin:livekit-mobile|connect_native_call', { payload: request }); + +export const disconnectNativeCall = (request: { callId: string }): Promise => + invoke('plugin:livekit-mobile|disconnect_native_call', { payload: request }); + +export const setNativeCallMicrophoneEnabled = (request: { + callId: string; + enabled: boolean; +}): Promise => + invoke('plugin:livekit-mobile|set_native_call_microphone_enabled', { + payload: request, + }); + +export const setNativeCallCameraEnabled = (request: { + callId: string; + enabled: boolean; +}): Promise => + invoke('plugin:livekit-mobile|set_native_call_camera_enabled', { + payload: request, + }); + +export const setNativeCallPiPEnabled = (request: { + callId: string; + enabled: boolean; +}): Promise => + invoke('plugin:livekit-mobile|set_native_call_pip_enabled', { + payload: request, + }); + +export const switchNativeCallCamera = (request: { callId: string }): Promise => + invoke('plugin:livekit-mobile|switch_native_call_camera', { + payload: request, + }); + +export const setNativeCallEncryptionKey = ( + request: SetNativeCallEncryptionKeyRequest +): Promise => + invoke('plugin:livekit-mobile|set_native_call_encryption_key', { + payload: request, + }); + +export const setNativeCallRemoteVideoOverlay = ( + request: SetNativeCallRemoteVideoOverlayRequest +): Promise => + invoke('plugin:livekit-mobile|set_native_call_remote_video_overlay', { + payload: request, + }); + +export const clearNativeCallRemoteVideoOverlay = ( + request: ClearNativeCallRemoteVideoOverlayRequest +): Promise => + invoke('plugin:livekit-mobile|clear_native_call_remote_video_overlay', { + payload: request, + }); + +export const setNativeCallLocalVideoOverlay = ( + request: SetNativeCallLocalVideoOverlayRequest +): Promise => + invoke('plugin:livekit-mobile|set_native_call_local_video_overlay', { + payload: request, + }); + +export const clearNativeCallLocalVideoOverlay = ( + request: ClearNativeCallLocalVideoOverlayRequest +): Promise => + invoke('plugin:livekit-mobile|clear_native_call_local_video_overlay', { + payload: request, + }); + +export const getNativeCallState = (): Promise => + invoke('plugin:livekit-mobile|get_native_call_state'); + +export const listenNativeCallSnapshot = ( + handler: (snapshot: NativeCallSnapshot) => void +): Promise => + listen(NATIVE_CALL_EVENT, ({ payload }) => handler(payload)); + +export const startSystemCall = (request: StartSystemCallRequest): Promise => + invoke('plugin:livekit-mobile|start_system_call', { payload: request }); + +export const endSystemCall = (request: EndSystemCallRequest): Promise => + invoke('plugin:livekit-mobile|end_system_call', { payload: request }); + +export const setSystemCallMuted = (request: SetSystemCallMutedRequest): Promise => + invoke('plugin:livekit-mobile|set_system_call_muted', { payload: request }); + +export const drainPendingSystemCallActions = (): Promise => + invoke('plugin:livekit-mobile|drain_pending_system_call_actions'); + +export const fulfillAnswerCall = (uuid: string): Promise => + invoke('plugin:livekit-mobile|fulfill_answer_call', { payload: { uuid } }); + +export const fulfillEndCall = (uuid: string): Promise => + invoke('plugin:livekit-mobile|fulfill_end_call', { payload: { uuid } }); + +export const reportSystemCallConnected = (uuid: string): Promise => + invoke('plugin:livekit-mobile|report_system_call_connected', { payload: { uuid } }); + +export const getAudioRoutes = (request: GetAudioRoutesRequest): Promise => + invoke('plugin:livekit-mobile|get_audio_routes', { payload: request }); + +export const setAudioRoute = (request: SetAudioRouteRequest): Promise => + invoke('plugin:livekit-mobile|set_audio_route', { payload: request }); + +export const updateCallDisplay = (request: UpdateCallDisplayRequest): Promise => + invoke('plugin:livekit-mobile|update_call_display', { payload: request }); + +export const onSystemCallAction = ( + handler: (action: SystemCallAction) => void +): Promise => addPluginListener('livekit-mobile', 'callkit_event', handler); diff --git a/src/app/features/call/nativeCallController.test.ts b/src/app/features/call/nativeCallController.test.ts new file mode 100644 index 0000000000..4e1cdb7dc9 --- /dev/null +++ b/src/app/features/call/nativeCallController.test.ts @@ -0,0 +1,569 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { + MatrixRTCSessionEvent, + type CallMembership, + type CallMembershipIdentityParts, + type MatrixClient, + type MatrixRTCSession, + type Room, +} from '$types/matrix-sdk'; +import { + createNativeCallController, + type NativeCallControllerDependencies, +} from './nativeCallController'; +import type { NativeCallSnapshot } from './livekitMobileBridge'; +import type { NativeCallSession } from '$state/nativeCall'; +import { + acquireCallOwner, + getActiveCallOwner, + resetCallOwnerForTests, + type CallOwnerLease, +} from '$state/callOwner'; + +const OWN_IDENTITY = '@alice:example.org:DEVICE'; +const OWN_PARTS = { + userId: '@alice:example.org', + deviceId: 'DEVICE', +} as CallMembershipIdentityParts; +const REMOTE_PARTS = { + userId: '@bob:example.org', + deviceId: 'BOBDEVICE', +} as CallMembershipIdentityParts; +const room = { + roomId: '!room:example.org', + loadMembersIfNeeded: () => Promise.resolve(true), + membersLoaded: () => true, + getMembersWithMembership: () => [{}, {}], + getJoinedMemberCount: () => 2, + // MSC4143 ties media encryption to the room; these tests cover the + // encrypted case, where the key pipeline runs. + hasEncryptionStateEvent: () => true, +} as unknown as Room; +const unencryptedRoom = Object.create(room, { + hasEncryptionStateEvent: { value: () => false }, +}) as Room; +const transport = { type: 'livekit' as const, livekit_service_url: 'https://sfu.example' }; +const ownMembership = { + userId: '@alice:example.org', + deviceId: 'DEVICE', + rtcBackendIdentity: OWN_IDENTITY, +} as CallMembership; + +const idleSnapshot = (callId: string | null = null): NativeCallSnapshot => ({ + revision: 1, + callId, + connectionState: 'idle', + microphoneEnabled: false, + cameraEnabled: false, + participantCount: 0, +}); + +const connectedSnapshot = (callId: string): NativeCallSnapshot => ({ + revision: 1, + callId, + connectionState: 'connected', + microphoneEnabled: true, + cameraEnabled: false, + participantCount: 1, +}); + +type SessionHandler = (...args: unknown[]) => void; + +type TestSession = MatrixRTCSession & { + handlers: Map; +}; + +const makeSession = (order: string[] = []): TestSession => { + const handlers = new Map(); + const session = { + handlers, + memberships: [] as CallMembership[], + slotId: 'm.call#real-slot', + on: vi.fn<(event: MatrixRTCSessionEvent, handler: SessionHandler) => void>((event, handler) => { + if (event === MatrixRTCSessionEvent.EncryptionKeyChanged) order.push('attach-keys'); + handlers.set(event, handler); + }), + off: vi.fn<(event: MatrixRTCSessionEvent, handler: SessionHandler) => void>( + (event, handler) => { + if (handlers.get(event) === handler) handlers.delete(event); + } + ), + removeListener: vi.fn<(event: MatrixRTCSessionEvent, handler: SessionHandler) => void>( + (event, handler) => { + if (handlers.get(event) === handler) handlers.delete(event); + } + ), + reemitEncryptionKeys: vi.fn<() => void>(), + joinRTCSession: vi.fn<() => void>(() => order.push('join')), + getOldestMembership: vi.fn<() => CallMembership | undefined>(() => undefined), + leaveRoomSession: vi.fn<() => Promise>(async () => { + order.push('leave'); + return true; + }), + } as unknown as TestSession; + return session; +}; + +const emitOwnMembership = (session: TestSession): void => { + session.memberships = [ownMembership]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)?.([], [ownMembership]); +}; + +const emitKey = ( + session: TestSession, + key: number[], + keyIndex: number, + identity: string, + parts: CallMembershipIdentityParts = OWN_PARTS +): void => { + session.handlers.get(MatrixRTCSessionEvent.EncryptionKeyChanged)?.( + new Uint8Array(key), + keyIndex, + parts, + identity + ); +}; + +const waitForMembershipListener = async (session: TestSession): Promise => { + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); +}; + +const flushMicrotasks = async (ticks = 20): Promise => { + for (let index = 0; index < ticks; index += 1) { + // oxlint-disable-next-line no-await-in-loop -- flushing chained microtasks one tick at a time + await Promise.resolve(); + } +}; + +const makeClient = (session: MatrixRTCSession): MatrixClient => + ({ + getDeviceId: () => 'DEVICE', + getSafeUserId: () => '@alice:example.org', + matrixRTC: { getRoomSession: () => session }, + on: vi.fn<(event: string, cb: unknown) => void>(), + removeListener: vi.fn<(event: string, cb: unknown) => void>(), + }) as unknown as MatrixClient; + +type Harness = { + dependencies: NativeCallControllerDependencies; + setSession: ReturnType void>>; + connectCall: ReturnType; + disconnectCall: ReturnType; + setCamera: ReturnType; + setEncryptionKey: ReturnType; + emitSnapshot: (snapshot: NativeCallSnapshot) => void; + unlisten: ReturnType void>>; +}; + +const makeDependencies = (order: string[]): Harness => { + const setSession = vi.fn<(session: NativeCallSession | undefined) => void>(); + const connectCall = vi.fn<() => Promise>(async () => { + order.push('connect'); + return connectedSnapshot('call-id'); + }); + const disconnectCall = vi.fn<() => Promise>(async () => { + order.push('native-disconnect'); + return idleSnapshot(); + }); + const setCamera = vi.fn<() => Promise>(async () => + connectedSnapshot('call-id') + ); + const setMicrophone = vi.fn<() => Promise>(async () => + connectedSnapshot('call-id') + ); + const setEncryptionKey = vi.fn<() => Promise>(async () => + connectedSnapshot('call-id') + ); + const unlisten = vi.fn<() => void>(); + let snapshotHandler: ((snapshot: NativeCallSnapshot) => void) | undefined; + + return { + setSession, + connectCall, + disconnectCall, + setCamera, + setEncryptionKey, + unlisten, + emitSnapshot: (snapshot) => snapshotHandler?.(snapshot), + dependencies: { + setSession, + connectCall, + disconnectCall, + setCamera, + setMicrophone, + setEncryptionKey, + listenSnapshot: async (handler) => { + snapshotHandler = handler; + return unlisten; + }, + getPreferredTransport: async () => transport, + provisionToken: async () => ({ url: 'wss://livekit.example', jwt: 'jwt' }), + createCallId: () => 'call-id', + }, + }; +}; + +const startOptions = ( + session: MatrixRTCSession, + video = false, + microphone = true, + callRoom: Room = room +) => ({ + mx: makeClient(session), + room: callRoom, + dm: false, + video, + microphone, + ongoing: false, +}); + +const lastSession = ( + setSession: ReturnType void>> +): NativeCallSession | undefined => setSession.mock.calls.at(-1)?.[0]; + +beforeEach(() => { + resetCallOwnerForTests(); +}); + +describe('native call controller', () => { + it('attaches the key forwarder before the Matrix join with managed media keys', async () => { + const order: string[] = []; + const session = makeSession(order); + const { dependencies, connectCall } = makeDependencies(order); + const controller = createNativeCallController(dependencies); + + const startPromise = controller.start(startOptions(session)); + await waitForMembershipListener(session); + expect(order).toEqual(['attach-keys', 'join']); + emitOwnMembership(session); + emitKey(session, [1], 0, OWN_IDENTITY); + await startPromise; + + expect(session.joinRTCSession).toHaveBeenCalledWith( + { userId: '@alice:example.org', deviceId: 'DEVICE', memberId: '@alice:example.org:DEVICE' }, + [{ ...transport, livekit_alias: room.roomId }], + undefined, + { + callIntent: 'audio', + membershipEventExpiryMs: 30 * 60 * 1000, + notificationType: 'notification', + manageMediaKeys: true, + } + ); + expect(connectCall).toHaveBeenCalled(); + }); + + it('runs an unencrypted call with no key pipeline and no managed media keys', async () => { + const order: string[] = []; + const session = makeSession(order); + const { dependencies, connectCall, setEncryptionKey } = makeDependencies(order); + const controller = createNativeCallController(dependencies); + + const startPromise = controller.start(startOptions(session, false, true, unencryptedRoom)); + await waitForMembershipListener(session); + emitOwnMembership(session); + // No own key is ever emitted: an unencrypted call must not wait for one. + await startPromise; + + // MSC4143 forbids MatrixRTC encryption in an unencrypted room, so nothing + // of the key pipeline may run for one. + expect(order).toEqual(['join', 'connect']); + expect(session.reemitEncryptionKeys).not.toHaveBeenCalled(); + expect(session.joinRTCSession).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + undefined, + expect.not.objectContaining({ manageMediaKeys: expect.anything() }) + ); + expect(connectCall).toHaveBeenCalledWith(expect.objectContaining({ encryptionKeys: [] })); + expect(setEncryptionKey).not.toHaveBeenCalled(); + }); + + it('gates the native connect on the own-identity key and caches other keys', async () => { + const order: string[] = []; + const session = makeSession(order); + const { dependencies, connectCall, setEncryptionKey } = makeDependencies(order); + const controller = createNativeCallController(dependencies); + + const startPromise = controller.start(startOptions(session)); + await waitForMembershipListener(session); + emitOwnMembership(session); + await vi.waitFor(() => expect(session.joinRTCSession).toHaveBeenCalled()); + emitKey(session, [9, 9], 0, 'remote-backend', REMOTE_PARTS); + await Promise.resolve(); + expect(connectCall).not.toHaveBeenCalled(); + + emitKey(session, [1, 2, 3, 4], 1, OWN_IDENTITY); + await startPromise; + + expect(connectCall).toHaveBeenCalledWith({ + callId: 'call-id', + url: 'wss://livekit.example', + token: 'jwt', + microphoneEnabled: true, + encryptionKeys: [ + { identity: 'remote-backend', keyIndex: 0, key: 'CQk=' }, + { identity: OWN_IDENTITY, keyIndex: 1, key: 'AQIDBA==' }, + ], + }); + expect(lastSession(dependencies.setSession as Harness['setSession'])).toMatchObject({ + backend: 'livekit-mobile', + roomId: room.roomId, + callId: 'call-id', + lifecycle: 'connected', + }); + // The known keys are replayed once the connection resolves so that anything + // arriving during the connect window still reaches the plugin. Re-sending a + // key into the same ring slot is a no-op for LiveKit. + expect(setEncryptionKey).toHaveBeenCalledTimes(2); + }); + + it('joins muted when the prescreen microphone was off', async () => { + const session = makeSession(); + const { dependencies, connectCall } = makeDependencies([]); + const controller = createNativeCallController(dependencies); + + const startPromise = controller.start(startOptions(session, false, false)); + await waitForMembershipListener(session); + emitOwnMembership(session); + await vi.waitFor(() => expect(session.joinRTCSession).toHaveBeenCalled()); + emitKey(session, [1, 2, 3, 4], 1, OWN_IDENTITY); + await startPromise; + + expect(connectCall).toHaveBeenCalledWith(expect.objectContaining({ microphoneEnabled: false })); + }); + + it('rotates keys through the set command only after connect resolves', async () => { + const order: string[] = []; + const session = makeSession(order); + const { dependencies, connectCall, setEncryptionKey } = makeDependencies(order); + const controller = createNativeCallController(dependencies); + + const startPromise = controller.start(startOptions(session)); + await waitForMembershipListener(session); + emitOwnMembership(session); + emitKey(session, [9, 9], 0, 'remote-backend', REMOTE_PARTS); + emitKey(session, [1], 0, OWN_IDENTITY); + expect(setEncryptionKey).not.toHaveBeenCalled(); + await startPromise; + // Known keys are replayed once connected, so nothing that arrived during + // the connect window is stranded. + expect(setEncryptionKey).toHaveBeenCalledTimes(2); + + setEncryptionKey.mockClear(); + emitKey(session, [5, 6, 7, 8], 4, 'remote-backend', REMOTE_PARTS); + expect(connectCall).toHaveBeenCalledTimes(1); + expect(setEncryptionKey).toHaveBeenCalledWith({ + callId: 'call-id', + identity: 'remote-backend', + keyIndex: 4, + key: 'BQYHCA==', + }); + + // An index is a slot in LiveKit's per-participant key ring, not a sequence, + // and a peer that rejoins restarts at 0. Every key is passed through: any + // filtering here strands the frames encrypted with the key we dropped. + setEncryptionKey.mockClear(); + emitKey(session, [8, 8], 2, 'remote-backend', REMOTE_PARTS); + expect(setEncryptionKey).toHaveBeenCalledWith({ + callId: 'call-id', + identity: 'remote-backend', + keyIndex: 2, + key: 'CAg=', + }); + }); + + it('disconnects natively before leaving Matrix, detaches, and releases the owner once', async () => { + const order: string[] = []; + const session = makeSession(order); + const harness = makeDependencies(order); + const release = vi.fn<() => void>(); + harness.dependencies.acquireOwner = (): CallOwnerLease => ({ + kind: 'livekit-mobile', + roomId: room.roomId, + release, + }); + const controller = createNativeCallController(harness.dependencies); + + const startPromise = controller.start(startOptions(session)); + await waitForMembershipListener(session); + emitOwnMembership(session); + emitKey(session, [1], 0, OWN_IDENTITY); + await startPromise; + order.length = 0; + + const connected = lastSession(harness.setSession); + expect(connected?.lifecycle).toBe('connected'); + await connected?.hangup(); + await connected?.hangup(); + + expect(order).toEqual(['native-disconnect', 'leave']); + expect(session.off).toHaveBeenCalledWith( + MatrixRTCSessionEvent.EncryptionKeyChanged, + expect.any(Function) + ); + expect(harness.unlisten).toHaveBeenCalledTimes(1); + expect(release).toHaveBeenCalledTimes(1); + expect(lastSession(harness.setSession)).toBeUndefined(); + }); + + it('enables the camera after connect only when video was requested', async () => { + const order: string[] = []; + const session = makeSession(order); + const { dependencies, setCamera } = makeDependencies(order); + const controller = createNativeCallController(dependencies); + + const startPromise = controller.start(startOptions(session, true)); + await waitForMembershipListener(session); + emitOwnMembership(session); + emitKey(session, [1], 0, OWN_IDENTITY); + await startPromise; + + expect(order).toEqual(['attach-keys', 'join', 'connect']); + expect(setCamera).toHaveBeenCalledWith({ callId: 'call-id', enabled: true }); + + // Release the owner lease so the audio-path controller can start. + await lastSession(dependencies.setSession as Harness['setSession'])?.hangup(); + + const audioOrder: string[] = []; + const audioSession = makeSession(audioOrder); + const audioHarness = makeDependencies(audioOrder); + const audioController = createNativeCallController(audioHarness.dependencies); + const audioStart = audioController.start(startOptions(audioSession)); + await waitForMembershipListener(audioSession); + emitOwnMembership(audioSession); + emitKey(audioSession, [1], 0, OWN_IDENTITY); + await audioStart; + expect(audioHarness.setCamera).not.toHaveBeenCalled(); + }); + + it('tracks lifecycle and media flags from snapshots and ends on a terminal snapshot', async () => { + const order: string[] = []; + const session = makeSession(order); + const harness = makeDependencies(order); + const controller = createNativeCallController(harness.dependencies); + + const startPromise = controller.start(startOptions(session)); + await waitForMembershipListener(session); + emitOwnMembership(session); + emitKey(session, [1], 0, OWN_IDENTITY); + await startPromise; + + harness.emitSnapshot({ + ...connectedSnapshot('call-id'), + connectionState: 'reconnecting', + microphoneEnabled: false, + cameraEnabled: true, + }); + expect(lastSession(harness.setSession)).toMatchObject({ + lifecycle: 'reconnecting', + microphoneEnabled: false, + cameraEnabled: true, + }); + + order.length = 0; + harness.emitSnapshot(idleSnapshot(null)); + await vi.waitFor(() => expect(session.leaveRoomSession).toHaveBeenCalled()); + expect(order).toEqual(['native-disconnect', 'leave']); + // A normal end clears the session; it is not an error the user must dismiss. + expect(lastSession(harness.setSession)).toBeUndefined(); + expect(getActiveCallOwner()).toBeUndefined(); + }); + + it('treats a failed snapshot for the current call as a call failure', async () => { + const order: string[] = []; + const session = makeSession(order); + const harness = makeDependencies(order); + const controller = createNativeCallController(harness.dependencies); + + const startPromise = controller.start(startOptions(session)); + await waitForMembershipListener(session); + emitOwnMembership(session); + emitKey(session, [1], 0, OWN_IDENTITY); + await startPromise; + + harness.emitSnapshot({ + ...idleSnapshot('call-id'), + connectionState: 'failed', + lastError: { code: 'disconnected', message: 'disconnected' }, + }); + await vi.waitFor(() => expect(harness.disconnectCall).toHaveBeenCalled()); + // The failure code is mapped to user-facing copy, not passed through raw. + expect(lastSession(harness.setSession)).toMatchObject({ + lifecycle: 'error', + error: 'The connection to the call was lost.', + }); + }); + + it('cleans up and hides secrets when the own-key wait times out', async () => { + vi.useFakeTimers(); + try { + const order: string[] = []; + const session = makeSession(order); + const harness = makeDependencies(order); + const controller = createNativeCallController(harness.dependencies); + + const startPromise = controller.start(startOptions(session)); + await flushMicrotasks(); + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ); + emitOwnMembership(session); + await flushMicrotasks(); + await vi.advanceTimersByTimeAsync(10_000); + await startPromise; + + expect(harness.connectCall).not.toHaveBeenCalled(); + expect(order).toEqual(['attach-keys', 'join', 'native-disconnect', 'leave']); + const failed = lastSession(harness.setSession); + expect(failed).toMatchObject({ lifecycle: 'error' }); + expect(failed?.error).toContain('Could not connect to the call.'); + // The surfaced message must never carry the provisioned token. + expect(failed?.error).not.toContain('jwt'); + expect(getActiveCallOwner()).toBeUndefined(); + } finally { + vi.useRealTimers(); + } + }); + + it('does not start while another call owns the owner lease', async () => { + const order: string[] = []; + const session = makeSession(order); + const harness = makeDependencies(order); + const onCleanup = vi.fn<() => void>(); + harness.dependencies.onCleanup = onCleanup; + const elementLease = acquireCallOwner('element', '!other:example.org'); + + const controller = createNativeCallController(harness.dependencies); + await controller.start(startOptions(session)); + + expect(session.joinRTCSession).not.toHaveBeenCalled(); + expect(harness.setSession).not.toHaveBeenCalled(); + expect(onCleanup).toHaveBeenCalled(); + elementLease?.release(); + }); + + it('ignores a second start while a native call is active', async () => { + const order: string[] = []; + const session = makeSession(order); + const harness = makeDependencies(order); + const controller = createNativeCallController(harness.dependencies); + + const startPromise = controller.start(startOptions(session)); + await waitForMembershipListener(session); + emitOwnMembership(session); + emitKey(session, [1], 0, OWN_IDENTITY); + await startPromise; + + const secondSession = makeSession(); + await controller.start(startOptions(secondSession)); + expect(secondSession.joinRTCSession).not.toHaveBeenCalled(); + expect(harness.connectCall).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/app/features/call/nativeCallController.ts b/src/app/features/call/nativeCallController.ts new file mode 100644 index 0000000000..5737fbc165 --- /dev/null +++ b/src/app/features/call/nativeCallController.ts @@ -0,0 +1,417 @@ +import type { AutoDiscoveryInfo } from '../../cs-api'; +import type { MatrixClient, MatrixRTCSession, Room } from '$types/matrix-sdk'; +import { + drainPendingSystemCallActions, + endSystemCall, + fulfillAnswerCall, + fulfillEndCall, + onSystemCallAction, + reportSystemCallConnected, + setSystemCallMuted, + startSystemCall, + updateCallDisplay, + type NativeCallAudioRoute, + type SystemCallAction, +} from './livekitMobileBridge'; +import { getPreferredLivekitTransport, provisionLivekitToken } from './livekitProvisioning'; +import { createCallKeyPipeline, type CallKeyPipeline } from './callKeyPipeline'; +import { disconnectLivekitThenLeaveMatrixRTC } from './matrixRtcCallLifecycle'; +import { + callEncryptsMedia, + createCallSessionHandles, + joinCallSession, + type CallSessionHandles, +} from './callSession'; +import { + createNativeTransport, + type NativeCallTransport, + type NativeTransportDependencies, +} from './nativeTransport'; +import type { CallTransportState } from './callTransport'; +import { acquireCallOwner, type CallOwnerLease } from '$state/callOwner'; +import type { NativeCallLifecycle, NativeCallSession } from '$state/nativeCall'; +import { createDebugLogger } from '$utils/debugLogger'; + +const debugLog = createDebugLogger('nativeCallController'); + +type SetupStage = 'joining the call' | 'authorizing' | 'connecting'; + +const setupErrorMessage = (stage: SetupStage): string => { + if (stage === 'joining the call') return 'Could not join the call.'; + if (stage === 'authorizing') return 'Could not get permission to join the call.'; + return 'Could not connect to the call.'; +}; + +export type NativeCallStartOptions = { + mx: MatrixClient; + room: Room; + discovery?: Pick; + dm: boolean; + video: boolean; + microphone: boolean; + ongoing: boolean; +}; + +type NativeCallRecord = { + roomId: string; + callId: string; + session: MatrixRTCSession; + keys: CallKeyPipeline; + transport: NativeCallTransport; + unsubscribeTransport: () => void; + ownerLease: CallOwnerLease; + cancelled: boolean; + handles: CallSessionHandles; + cleanupPromise?: Promise; +}; + +export type NativeCallController = { + start: (options: NativeCallStartOptions) => Promise; + setMicrophoneEnabled: (enabled: boolean) => Promise; + setCameraEnabled: (enabled: boolean) => Promise; + switchCamera: () => Promise; + listAudioRoutes: () => Promise; + selectAudioRoute: (routeId: string) => Promise; +}; + +export type NativeCallControllerDependencies = NativeTransportDependencies & { + setSession: (session: NativeCallSession | undefined) => void; + updateDisplay?: typeof updateCallDisplay; + createKeyPipeline?: () => CallKeyPipeline; + createTransport?: typeof createNativeTransport; + getPreferredTransport?: typeof getPreferredLivekitTransport; + provisionToken?: typeof provisionLivekitToken; + createCallId?: () => string; + acquireOwner?: typeof acquireCallOwner; + onCleanup?: () => void; +}; + +const noMediaControls = { + setMicrophoneEnabled: async (): Promise => {}, + setCameraEnabled: async (): Promise => {}, + switchCamera: async (): Promise => {}, + listAudioRoutes: async (): Promise => [], + selectAudioRoute: async (): Promise => {}, +}; + +export const createNativeCallController = ( + dependencies: NativeCallControllerDependencies +): NativeCallController => { + const deps = { + ...dependencies, + updateDisplay: dependencies.updateDisplay ?? updateCallDisplay, + createKeyPipeline: dependencies.createKeyPipeline ?? createCallKeyPipeline, + createTransport: dependencies.createTransport ?? createNativeTransport, + getPreferredTransport: dependencies.getPreferredTransport ?? getPreferredLivekitTransport, + provisionToken: dependencies.provisionToken ?? provisionLivekitToken, + createCallId: dependencies.createCallId ?? (() => crypto.randomUUID()), + acquireOwner: dependencies.acquireOwner ?? acquireCallOwner, + }; + + // Register the system-call (CallKit) listener once at controller creation. + // Events (answer, end, mute from lock-screen/system-UI) flow from the + // native side regardless of call state. + let pendingSystemUuid: string | undefined; + + void onSystemCallAction((action: SystemCallAction) => { + if (action.action === 'end') { + // System UI ended the call: hang up if active, then fulfill the + // pending CXEndCallAction so the system UI dismisses immediately. + if (activeRecord && !activeRecord.cancelled) { + void cleanup(activeRecord, undefined, true).finally(() => { + if (action.uuid) void fulfillEndCall(action.uuid).catch(() => undefined); + }); + } else if (action.uuid) { + void fulfillEndCall(action.uuid).catch(() => undefined); + } + void drainPendingSystemCallActions().catch(() => undefined); + } else if (action.action === 'answer') { + // System UI answered an incoming call: store the uuid for the + // start path so it can map back to the system call. + pendingSystemUuid = action.uuid; + } else if (action.action === 'mute') { + // System UI mute toggle: push to LiveKit. + void activeRecord?.transport.setMicrophoneEnabled(!action.muted); + } + }).catch(() => undefined); + + let activeRecord: NativeCallRecord | undefined; + let displayedRecord: NativeCallRecord | undefined; + + const isCurrent = (record: NativeCallRecord): boolean => + activeRecord === record && !record.cancelled; + + const publishError = ( + roomId: string, + callId: string, + error: string, + hangup: () => Promise + ): void => { + try { + deps.setSession({ + backend: 'livekit-mobile', + roomId, + callId, + lifecycle: 'error', + error, + participants: [], + microphoneEnabled: false, + cameraEnabled: false, + ...noMediaControls, + hangup, + }); + } catch {} + }; + + const publish = ( + record: NativeCallRecord, + lifecycle: NativeCallLifecycle, + media?: Pick + ): void => { + if (!isCurrent(record)) return; + deps.setSession({ + backend: 'livekit-mobile', + roomId: record.roomId, + callId: record.callId, + lifecycle, + participants: media?.participants ?? [], + microphoneEnabled: media?.microphoneEnabled ?? true, + cameraEnabled: media?.cameraEnabled ?? false, + setMicrophoneEnabled, + setCameraEnabled, + switchCamera, + listAudioRoutes, + selectAudioRoute, + hangup: () => cleanup(record, undefined, true), + }); + displayedRecord = record; + }; + + const cleanup = async ( + record: NativeCallRecord, + failure: string | undefined, + clear: boolean + ): Promise => { + if (record.cleanupPromise) { + await record.cleanupPromise; + if (clear && displayedRecord === record) { + displayedRecord = undefined; + try { + deps.setSession(undefined); + } catch {} + } + return; + } + + record.cancelled = true; + record.handles.cancelMembershipWait?.(); + record.handles.cancelMembershipWait = undefined; + record.handles.removePageHideListener?.(); + record.handles.removePageHideListener = undefined; + record.handles.unsubscribeCallRoom?.(); + record.handles.unsubscribeCallRoom = undefined; + record.unsubscribeTransport(); + record.keys.setOnKey(undefined); + if (failure && activeRecord === record) { + publishError(record.roomId, record.callId, failure, () => cleanup(record, undefined, true)); + displayedRecord = record; + } + + record.cleanupPromise = (async () => { + // End the system call so CallKit dismisses the active-call UI. + void endSystemCall({ callId: record.callId, remoteEnded: true }).catch(() => undefined); + void record.transport.capabilities.pictureInPicture?.setEnabled(false); + await disconnectLivekitThenLeaveMatrixRTC( + () => record.transport.disconnect(), + record.session + ); + record.keys.detach(); + record.ownerLease.release(); + if (activeRecord === record) activeRecord = undefined; + if (clear && displayedRecord === record) { + displayedRecord = undefined; + try { + deps.setSession(undefined); + } catch {} + } + deps.onCleanup?.(); + })(); + await record.cleanupPromise; + }; + + const handleTransportState = (record: NativeCallRecord, state: CallTransportState): void => { + if (!isCurrent(record)) return; + if (state.connection === 'disconnected') { + // A normal end clears the session; only a failure leaves an error the + // user has to dismiss. + void cleanup(record, state.error, state.error === undefined); + return; + } + publish(record, state.connection, state); + }; + + const start = async ({ + mx, + room, + discovery, + dm, + video, + microphone, + ongoing, + }: NativeCallStartOptions) => { + if (activeRecord) { + deps.onCleanup?.(); + return; + } + const ownerLease = deps.acquireOwner('livekit-mobile', room.roomId); + if (!ownerLease) { + deps.onCleanup?.(); + return; + } + + let record: NativeCallRecord | undefined; + let stage: SetupStage = 'joining the call'; + try { + const callId = deps.createCallId(); + const session = mx.matrixRTC.getRoomSession(room); + const keys = deps.createKeyPipeline(); + const transport = deps.createTransport(callId, dependencies); + record = { + roomId: room.roomId, + callId, + session, + keys, + transport, + unsubscribeTransport: () => {}, + ownerLease, + cancelled: false, + handles: createCallSessionHandles(), + }; + const currentRecord = record; + activeRecord = record; + publish(record, 'starting'); + + record.unsubscribeTransport = transport.subscribe((state) => + handleTransportState(currentRecord, state) + ); + await transport.ready; + + const encryptMedia = callEncryptsMedia(room); + if (encryptMedia) { + // The transport queues anything that arrives before it is connected, so + // a key is never dropped for being early. + keys.setOnKey((key) => void transport.setEncryptionKey(key)); + keys.attach(session, { userId: mx.getSafeUserId(), deviceId: mx.getDeviceId() }); + } + + const joined = await joinCallSession( + { + mx, + room, + session, + discovery, + getPreferredTransport: deps.getPreferredTransport, + provisionToken: deps.provisionToken, + callIntent: video ? 'video' : 'audio', + dm, + ongoing, + encryptMedia, + isCancelled: () => !isCurrent(currentRecord), + onStage: (joinStage) => { + stage = joinStage === 'joining-matrix' ? 'joining the call' : 'authorizing'; + }, + }, + currentRecord.handles + ); + if (!isCurrent(currentRecord)) return; + + stage = 'connecting'; + if (encryptMedia) { + await keys.waitForOwnKey(); + if (!isCurrent(currentRecord)) return; + } + + await transport.connect({ + url: joined.provisioned.url, + token: joined.provisioned.jwt, + microphoneEnabled: microphone, + cameraEnabled: video, + encryptionKeys: keys.getKeys(), + }); + + // Report the outgoing system call so CallKit shows the active-call UI. + // Use the pending uuid (from an answer action) if present; otherwise + // generate a new uuid. The native side maps callId to uuid internally. + const systemUuid = pendingSystemUuid ?? crypto.randomUUID(); + const isIncomingAnswer = pendingSystemUuid !== undefined; + pendingSystemUuid = undefined; + const callerName = room.name || room.roomId; + void startSystemCall({ callId, uuid: systemUuid, callerName }).catch(() => undefined); + // Update the system call display with the room name and video flag so + // CallKit shows the correct caller info. + void deps + .updateDisplay({ + callId, + callerName, + hasVideo: video, + }) + .catch(() => undefined); + // Report the call as connected to CallKit so the system UI updates. + void reportSystemCallConnected(systemUuid).catch(() => undefined); + void transport.capabilities.pictureInPicture?.setEnabled(true); + // For system-initiated incoming answers: fulfill the deferred answer action. + if (isIncomingAnswer) { + void fulfillAnswerCall(systemUuid).catch(() => undefined); + } + } catch (cause) { + const detail = cause instanceof Error ? `${cause.name}: ${cause.message}` : String(cause); + debugLog.error('call', `Native call setup failed during ${stage}. ${detail}`); + if (record) { + await cleanup(record, setupErrorMessage(stage), false); + } else { + ownerLease.release(); + publishError(room.roomId, '', setupErrorMessage(stage), async () => { + try { + deps.setSession(undefined); + } catch {} + }); + deps.onCleanup?.(); + } + } + }; + + const setMicrophoneEnabled = async (enabled: boolean): Promise => { + const record = activeRecord; + if (!record || record.cancelled) return; + await record.transport.setMicrophoneEnabled(enabled); + // Push mute state back to CallKit for UI consistency. + void setSystemCallMuted({ callId: record.callId, muted: !enabled }).catch(() => undefined); + }; + + const active = (): NativeCallTransport | undefined => + activeRecord && !activeRecord.cancelled ? activeRecord.transport : undefined; + + const setCameraEnabled = async (enabled: boolean): Promise => { + await active()?.setCameraEnabled(enabled); + }; + + const switchCamera = async (): Promise => { + await active()?.capabilities.camera?.switch(); + }; + + const listAudioRoutes = async (): Promise => + (await active()?.capabilities.audioRoutes?.list()) ?? []; + + const selectAudioRoute = async (routeId: string): Promise => { + await active()?.capabilities.audioRoutes?.select(routeId); + }; + + return { + start, + setMicrophoneEnabled, + setCameraEnabled, + switchCamera, + listAudioRoutes, + selectAudioRoute, + }; +}; diff --git a/src/app/features/call/nativeCallProbe.test.ts b/src/app/features/call/nativeCallProbe.test.ts new file mode 100644 index 0000000000..3c31ed89aa --- /dev/null +++ b/src/app/features/call/nativeCallProbe.test.ts @@ -0,0 +1,67 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { getNativeCallAvailability, resetNativeCallAvailabilityForTests } from './nativeCallProbe'; +import { isMobileTauri } from '$utils/platform'; +import { getNativeCallCapabilities, type NativeCallCapabilities } from './livekitMobileBridge'; + +vi.mock('$utils/platform', () => ({ + isMobileTauri: vi.fn<() => boolean>(), +})); + +vi.mock('./livekitMobileBridge', () => ({ + getNativeCallCapabilities: vi.fn<() => Promise>(), +})); + +const allCapabilities: NativeCallCapabilities = { + supported: true, + microphone: true, + backgroundAudio: true, + nativeRoom: true, + camera: true, + nativeVideoOverlay: false, + callKit: true, +}; + +beforeEach(() => { + resetNativeCallAvailabilityForTests(); + vi.mocked(isMobileTauri).mockReturnValue(false); + vi.mocked(getNativeCallCapabilities).mockResolvedValue(allCapabilities); +}); + +describe('getNativeCallAvailability', () => { + it('is unavailable outside Tauri mobile even with new calls enabled', async () => { + vi.mocked(isMobileTauri).mockReturnValue(false); + + await expect(getNativeCallAvailability(true)).resolves.toBe(false); + expect(getNativeCallCapabilities).not.toHaveBeenCalled(); + }); + + it('is unavailable with new calls disabled on Tauri mobile', async () => { + vi.mocked(isMobileTauri).mockReturnValue(true); + + await expect(getNativeCallAvailability(false)).resolves.toBe(false); + expect(getNativeCallCapabilities).not.toHaveBeenCalled(); + }); + + it('is available on Tauri mobile with new calls enabled and supporting capabilities', async () => { + vi.mocked(isMobileTauri).mockReturnValue(true); + + await expect(getNativeCallAvailability(true)).resolves.toBe(true); + }); + + it('is unavailable when the native plugin does not support calls', async () => { + vi.mocked(isMobileTauri).mockReturnValue(true); + vi.mocked(getNativeCallCapabilities).mockResolvedValue({ + ...allCapabilities, + supported: false, + }); + + await expect(getNativeCallAvailability(true)).resolves.toBe(false); + }); + + it('is unavailable when the capabilities request fails', async () => { + vi.mocked(isMobileTauri).mockReturnValue(true); + vi.mocked(getNativeCallCapabilities).mockRejectedValue(new Error('plugin missing')); + + await expect(getNativeCallAvailability(true)).resolves.toBe(false); + }); +}); diff --git a/src/app/features/call/nativeCallProbe.ts b/src/app/features/call/nativeCallProbe.ts new file mode 100644 index 0000000000..83cf6c2225 --- /dev/null +++ b/src/app/features/call/nativeCallProbe.ts @@ -0,0 +1,19 @@ +import { isMobileTauri } from '$utils/platform'; +import { getNativeCallCapabilities, type NativeCallCapabilities } from './livekitMobileBridge'; + +const supportsNativeCall = (capabilities: NativeCallCapabilities): boolean => + capabilities.supported && capabilities.microphone; + +let availabilityPromise: Promise | undefined; + +// Native calls are gated solely by the new-call setting; platform and +// capability checks still apply on top of it. +export const getNativeCallAvailability = (newCallsEnabled: boolean): Promise => { + if (!newCallsEnabled || !isMobileTauri()) return Promise.resolve(false); + availabilityPromise ??= getNativeCallCapabilities().then(supportsNativeCall, () => false); + return availabilityPromise; +}; + +export const resetNativeCallAvailabilityForTests = (): void => { + availabilityPromise = undefined; +}; diff --git a/src/app/features/call/nativeTransport.test.ts b/src/app/features/call/nativeTransport.test.ts new file mode 100644 index 0000000000..9b47c55a7c --- /dev/null +++ b/src/app/features/call/nativeTransport.test.ts @@ -0,0 +1,377 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { UnlistenFn } from '@tauri-apps/api/event'; +import { createNativeTransport } from './nativeTransport'; +import type { + connectNativeCall, + disconnectNativeCall, + getAudioRoutes, + getNativeCallState, + listenNativeCallSnapshot, + setAudioRoute, + setNativeCallCameraEnabled, + setNativeCallEncryptionKey, + setNativeCallMicrophoneEnabled, + setNativeCallPiPEnabled, + switchNativeCallCamera, + NativeCallSnapshot, +} from './livekitMobileBridge'; +import type { CallEncryptionKey, CallTransportState } from './callTransport'; + +const bridge = vi.hoisted(() => ({ + connectNativeCall: vi.fn(), + disconnectNativeCall: vi.fn(), + setNativeCallMicrophoneEnabled: vi.fn(), + setNativeCallCameraEnabled: vi.fn(), + setNativeCallEncryptionKey: vi.fn(), + setNativeCallPiPEnabled: vi.fn(), + switchNativeCallCamera: vi.fn(), + getAudioRoutes: vi.fn(), + setAudioRoute: vi.fn(), + getNativeCallState: vi.fn(), + listenNativeCallSnapshot: vi.fn(), +})); + +vi.mock('./livekitMobileBridge', () => bridge); + +const CALL_ID = 'call-1'; + +const snapshot = (overrides: Partial = {}): NativeCallSnapshot => ({ + revision: 1, + callId: CALL_ID, + connectionState: 'connected', + microphoneEnabled: true, + cameraEnabled: false, + participantCount: 0, + ...overrides, +}); + +const key = (identity: string, keyIndex: number, bytes: number[]): CallEncryptionKey => ({ + identity, + keyIndex, + key: new Uint8Array(bytes), +}); + +let emit!: (next: NativeCallSnapshot) => void; +let unlisten: ReturnType>; + +const connectOptions = (overrides: Record = {}) => ({ + url: 'wss://livekit.example', + token: 'jwt', + microphoneEnabled: true, + cameraEnabled: false, + encryptionKeys: [] as CallEncryptionKey[], + ...overrides, +}); + +const sentKeys = () => bridge.setNativeCallEncryptionKey.mock.calls.map(([request]) => request); + +beforeEach(() => { + vi.clearAllMocks(); + unlisten = vi.fn(); + bridge.listenNativeCallSnapshot.mockImplementation((handler) => { + emit = handler; + return Promise.resolve(unlisten); + }); + bridge.connectNativeCall.mockResolvedValue(snapshot()); + bridge.disconnectNativeCall.mockResolvedValue(snapshot({ connectionState: 'idle' })); + bridge.setNativeCallEncryptionKey.mockResolvedValue(snapshot()); + bridge.setNativeCallCameraEnabled.mockResolvedValue(snapshot()); + bridge.getNativeCallState.mockResolvedValue(snapshot()); +}); + +const connectedTransport = async () => { + const transport = createNativeTransport(CALL_ID); + await transport.ready; + await transport.connect(connectOptions()); + return transport; +}; + +describe('createNativeTransport encryption keys', () => { + it('forwards every key whatever the index, including a rejoin back to 0 and a decrease', async () => { + const transport = await connectedTransport(); + + await transport.setEncryptionKey(key('@bob:example.org:BOB', 3, [1, 2, 3])); + await transport.setEncryptionKey(key('@bob:example.org:BOB', 0, [4, 5])); + await transport.setEncryptionKey(key('@bob:example.org:BOB', 2, [9, 9])); + await transport.setEncryptionKey(key('@bob:example.org:BOB', 1, [7])); + + expect(sentKeys().map((request) => request.keyIndex)).toEqual([3, 0, 2, 1]); + expect(sentKeys().map((request) => request.key)).toEqual(['AQID', 'BAU=', 'CQk=', 'Bw==']); + }); + + it('forwards a repeat of an index already used by the same identity', async () => { + const transport = await connectedTransport(); + + await transport.setEncryptionKey(key('@bob:example.org:BOB', 4, [1, 2, 3])); + await transport.setEncryptionKey(key('@bob:example.org:BOB', 4, [4, 5])); + + expect(sentKeys()).toEqual([ + { callId: CALL_ID, identity: '@bob:example.org:BOB', keyIndex: 4, key: 'AQID' }, + { callId: CALL_ID, identity: '@bob:example.org:BOB', keyIndex: 4, key: 'BAU=' }, + ]); + }); + + it('keeps each identity on its own key ring', async () => { + const transport = await connectedTransport(); + + await transport.setEncryptionKey(key('@bob:example.org:BOB', 5, [1, 2, 3])); + await transport.setEncryptionKey(key('@carol:example.org:CAROL', 1, [4, 5])); + + expect(sentKeys().map((request) => [request.identity, request.keyIndex])).toEqual([ + ['@bob:example.org:BOB', 5], + ['@carol:example.org:CAROL', 1], + ]); + }); + + it('base64-encodes the raw key bytes for the bridge', async () => { + const transport = await connectedTransport(); + + await transport.setEncryptionKey(key('@bob:example.org:BOB', 0, [0, 1, 250, 255])); + + expect(sentKeys()[0]).toEqual({ + callId: CALL_ID, + identity: '@bob:example.org:BOB', + keyIndex: 0, + key: 'AAH6/w==', + }); + }); + + it('base64-encodes the keys handed to connect', async () => { + const transport = createNativeTransport(CALL_ID); + await transport.ready; + + await transport.connect( + connectOptions({ encryptionKeys: [key('@bob:example.org:BOB', 2, [200, 201, 202, 203])] }) + ); + + expect(bridge.connectNativeCall).toHaveBeenCalledWith( + expect.objectContaining({ + encryptionKeys: [{ identity: '@bob:example.org:BOB', keyIndex: 2, key: 'yMnKyw==' }], + }) + ); + }); + + it('replays the keys that arrived while connect was still in flight', async () => { + let resolveConnect!: (value: NativeCallSnapshot) => void; + bridge.connectNativeCall.mockReturnValue( + new Promise((resolve) => { + resolveConnect = resolve; + }) + ); + const transport = createNativeTransport(CALL_ID); + await transport.ready; + + const connecting = transport.connect(connectOptions()); + await transport.setEncryptionKey(key('@bob:example.org:BOB', 1, [1, 2, 3])); + await transport.setEncryptionKey(key('@carol:example.org:CAROL', 0, [4, 5])); + expect(bridge.setNativeCallEncryptionKey).not.toHaveBeenCalled(); + + resolveConnect(snapshot()); + await connecting; + + expect(sentKeys().map((request) => [request.identity, request.keyIndex])).toEqual([ + ['@bob:example.org:BOB', 1], + ['@carol:example.org:CAROL', 0], + ]); + }); + + it('drops keys once the transport is disposed', async () => { + const transport = await connectedTransport(); + await transport.disconnect(); + + await transport.setEncryptionKey(key('@bob:example.org:BOB', 1, [1, 2, 3])); + + expect(bridge.setNativeCallEncryptionKey).not.toHaveBeenCalled(); + }); + + it('swallows a bridge rejection so a bad key never breaks the call', async () => { + const transport = await connectedTransport(); + bridge.setNativeCallEncryptionKey.mockRejectedValue(new Error('key ring full')); + + await expect( + transport.setEncryptionKey(key('@bob:example.org:BOB', 1, [1, 2, 3])) + ).resolves.toBeUndefined(); + }); +}); + +describe('createNativeTransport snapshots', () => { + it('ignores snapshots belonging to another call', async () => { + const transport = await connectedTransport(); + + emit(snapshot({ callId: 'other-call', connectionState: 'reconnecting', participantCount: 4 })); + + expect(transport.getState().connection).toBe('connected'); + }); + + it('ignores an idle snapshot with no callId before connect resolves', async () => { + const transport = createNativeTransport(CALL_ID); + await transport.ready; + + // The plugin reports its own idle state on subscribe; that is not this + // call ending. + emit(snapshot({ callId: null, connectionState: 'idle', microphoneEnabled: false })); + + expect(transport.getState().connection).toBe('connecting'); + }); + + it('treats an idle snapshot with no callId after connect as this call ending', async () => { + const transport = await connectedTransport(); + + emit(snapshot({ callId: null, connectionState: 'idle', microphoneEnabled: false })); + + expect(transport.getState().connection).toBe('disconnected'); + }); + + it('ignores a terminal snapshot naming another call', async () => { + const transport = await connectedTransport(); + + emit( + snapshot({ + callId: 'other-call', + connectionState: 'failed', + lastError: { code: 'busy', message: 'busy' }, + }) + ); + + expect(transport.getState().connection).toBe('connected'); + }); + + it('surfaces the failure copy for a terminal snapshot naming this call', async () => { + const transport = await connectedTransport(); + + emit( + snapshot({ + callId: CALL_ID, + connectionState: 'failed', + lastError: { code: 'permission_denied', message: 'denied' }, + }) + ); + + expect(transport.getState()).toMatchObject({ + connection: 'disconnected', + error: 'Microphone or camera access was denied.', + participants: [], + }); + }); + + it('maps remote participants, tracks and connection quality', async () => { + const transport = await connectedTransport(); + + emit( + snapshot({ + remoteParticipants: [ + { + identity: '@bob:example.org:BOB', + camera: { sid: 'cam-1', muted: false, subscribed: true }, + screenShare: { sid: 'screen-1', muted: true, subscribed: false }, + connectionQuality: 'poor', + }, + { identity: '@carol:example.org:CAROL', connectionQuality: 'nonsense' }, + ], + }) + ); + + expect(transport.getState().participants).toEqual([ + { + identity: '@bob:example.org:BOB', + camera: { id: 'cam-1', muted: false, subscribed: true }, + screenShare: { id: 'screen-1', muted: true, subscribed: false }, + connectionQuality: 'poor', + }, + { identity: '@carol:example.org:CAROL', connectionQuality: 'unknown' }, + ]); + }); + + it('ignores snapshots arriving after disconnect', async () => { + const transport = await connectedTransport(); + await transport.disconnect(); + const before = transport.getState(); + + emit(snapshot({ connectionState: 'reconnecting' })); + + expect(transport.getState()).toBe(before); + }); +}); + +describe('createNativeTransport lifecycle', () => { + it('removes the visibilitychange listener and unlistens the snapshot stream on disconnect', async () => { + const added = vi.spyOn(document, 'addEventListener'); + const removed = vi.spyOn(document, 'removeEventListener'); + try { + const transport = await connectedTransport(); + const [, resync] = added.mock.calls.find(([type]) => type === 'visibilitychange')!; + + await transport.disconnect(); + + expect(removed).toHaveBeenCalledWith('visibilitychange', resync); + expect(bridge.disconnectNativeCall).toHaveBeenCalledWith({ callId: CALL_ID }); + expect(unlisten).toHaveBeenCalledOnce(); + } finally { + added.mockRestore(); + removed.mockRestore(); + } + }); + + it('still unlistens when the bridge disconnect rejects', async () => { + const transport = await connectedTransport(); + bridge.disconnectNativeCall.mockRejectedValue(new Error('no such call')); + + await expect(transport.disconnect()).resolves.toBeUndefined(); + expect(unlisten).toHaveBeenCalledOnce(); + }); + + it('does not let a throwing state listener stop the other listeners or the teardown', async () => { + const transport = await connectedTransport(); + const seen: CallTransportState[] = []; + transport.subscribe(() => { + throw new Error('listener blew up'); + }); + transport.subscribe((next) => seen.push(next)); + + emit(snapshot({ connectionState: 'reconnecting' })); + + expect(seen.map((state) => state.connection)).toEqual(['reconnecting']); + await expect(transport.disconnect()).resolves.toBeUndefined(); + expect(unlisten).toHaveBeenCalledOnce(); + }); + + it('stops delivering to an unsubscribed listener', async () => { + const transport = await connectedTransport(); + const listener = vi.fn<(next: CallTransportState) => void>(); + const unsubscribe = transport.subscribe(listener); + + unsubscribe(); + emit(snapshot({ connectionState: 'reconnecting' })); + + expect(listener).not.toHaveBeenCalled(); + }); + + it('scopes commands to the call and refuses them before connect', async () => { + const transport = createNativeTransport(CALL_ID); + await transport.ready; + + await transport.setMicrophoneEnabled(true); + expect(bridge.setNativeCallMicrophoneEnabled).not.toHaveBeenCalled(); + + await transport.connect(connectOptions()); + bridge.setNativeCallMicrophoneEnabled.mockResolvedValue(snapshot()); + await transport.setMicrophoneEnabled(false); + + expect(bridge.setNativeCallMicrophoneEnabled).toHaveBeenCalledWith({ + callId: CALL_ID, + enabled: false, + }); + }); + + it('turns the camera on after connecting when the call started with video', async () => { + const transport = createNativeTransport(CALL_ID); + await transport.ready; + + await transport.connect(connectOptions({ cameraEnabled: true })); + + expect(bridge.setNativeCallCameraEnabled).toHaveBeenCalledWith({ + callId: CALL_ID, + enabled: true, + }); + }); +}); diff --git a/src/app/features/call/nativeTransport.ts b/src/app/features/call/nativeTransport.ts new file mode 100644 index 0000000000..1958e4fd08 --- /dev/null +++ b/src/app/features/call/nativeTransport.ts @@ -0,0 +1,291 @@ +import { + connectNativeCall, + disconnectNativeCall, + getAudioRoutes, + getNativeCallState, + listenNativeCallSnapshot, + setAudioRoute, + setNativeCallCameraEnabled, + setNativeCallEncryptionKey, + setNativeCallMicrophoneEnabled, + setNativeCallPiPEnabled, + switchNativeCallCamera, + type NativeCallFailureCode, + type NativeCallRemoteCamera, + type NativeCallRemoteParticipant, + type NativeCallSnapshot, +} from './livekitMobileBridge'; +import type { + CallAudioRoute, + CallConnectionQuality, + CallEncryptionKey, + CallParticipant, + CallTrack, + CallTransport, + CallTransportConnectOptions, + CallTransportState, +} from './callTransport'; +import { createDebugLogger } from '$utils/debugLogger'; + +const debugLog = createDebugLogger('nativeTransport'); + +const logFailure = (what: string, cause: unknown): void => { + const detail = cause instanceof Error ? `${cause.name}: ${cause.message}` : String(cause); + debugLog.error('call', `${what}. ${detail}`); +}; + +const failureMessages: Record = { + invalid_request: 'Could not start the call.', + busy: 'Another call is already in progress.', + permission_denied: 'Microphone or camera access was denied.', + connect_failed: 'Could not connect to the call.', + media_failed: 'Your microphone or camera stopped working.', + disconnected: 'The connection to the call was lost.', + cancelled: 'The call was cancelled.', + unavailable: 'Calls are not available on this device.', + unexpected: 'The call ended unexpectedly.', +}; + +const failureMessage = (code: NativeCallFailureCode | undefined): string => + (code && failureMessages[code]) || 'The call ended unexpectedly.'; + +const toBase64 = (key: Uint8Array): string => { + let binary = ''; + key.forEach((byte) => { + binary += String.fromCharCode(byte); + }); + return btoa(binary); +}; + +const toTrack = (track: NativeCallRemoteCamera): CallTrack => ({ + id: track.sid, + muted: track.muted, + subscribed: track.subscribed, +}); + +const toQuality = (quality: string | undefined): CallConnectionQuality => + quality === 'lost' || quality === 'poor' || quality === 'good' || quality === 'excellent' + ? quality + : 'unknown'; + +const toParticipant = (participant: NativeCallRemoteParticipant): CallParticipant => ({ + identity: participant.identity, + ...(participant.camera ? { camera: toTrack(participant.camera) } : {}), + ...(participant.screenShare ? { screenShare: toTrack(participant.screenShare) } : {}), + connectionQuality: toQuality(participant.connectionQuality), +}); + +export type NativeTransportDependencies = { + connectCall?: typeof connectNativeCall; + disconnectCall?: typeof disconnectNativeCall; + setMicrophone?: typeof setNativeCallMicrophoneEnabled; + setCamera?: typeof setNativeCallCameraEnabled; + setEncryptionKey?: typeof setNativeCallEncryptionKey; + listenSnapshot?: typeof listenNativeCallSnapshot; +}; + +export type NativeCallTransport = CallTransport & { + /** + * Resolves once the snapshot subscription is live. The caller awaits it + * before joining so no state change between join and connect is missed. + */ + ready: Promise; +}; + +/** + * `CallTransport` over the native LiveKit SDK, reached through the + * `livekit-mobile` plugin. One transport per call: `callId` scopes every + * command and filters the snapshot stream, which is global to the plugin. + */ +export const createNativeTransport = ( + callId: string, + dependencies: NativeTransportDependencies = {} +): NativeCallTransport => { + const connectCall = dependencies.connectCall ?? connectNativeCall; + const disconnectCall = dependencies.disconnectCall ?? disconnectNativeCall; + const setMicrophone = dependencies.setMicrophone ?? setNativeCallMicrophoneEnabled; + const setCamera = dependencies.setCamera ?? setNativeCallCameraEnabled; + const setKey = dependencies.setEncryptionKey ?? setNativeCallEncryptionKey; + const listenSnapshot = dependencies.listenSnapshot ?? listenNativeCallSnapshot; + + let state: CallTransportState = { + connection: 'connecting', + participants: [], + microphoneEnabled: false, + cameraEnabled: false, + }; + let connected = false; + let disposed = false; + const pendingKeys: CallEncryptionKey[] = []; + const listeners = new Set<(next: CallTransportState) => void>(); + + const publish = (next: CallTransportState): void => { + state = next; + listeners.forEach((listener) => { + try { + listener(next); + } catch (cause) { + // A state observer must not interrupt lifecycle cleanup. + logFailure('Native call state listener threw', cause); + } + }); + }; + + const apply = (snapshot: NativeCallSnapshot): void => { + if (disposed) return; + if (snapshot.connectionState === 'idle' || snapshot.connectionState === 'failed') { + // A snapshot with no callId before our connect resolves reports the + // plugin's own idle state, not the end of this call. + if (snapshot.callId === null && !connected) return; + if (snapshot.callId !== null && snapshot.callId !== callId) return; + if (snapshot.connectionState === 'failed') { + debugLog.error('call', `Native call failed: ${snapshot.lastError?.code ?? 'unknown'}`); + } + publish({ + connection: 'disconnected', + participants: [], + microphoneEnabled: false, + cameraEnabled: false, + ...(snapshot.connectionState === 'failed' + ? { error: failureMessage(snapshot.lastError?.code) } + : {}), + }); + return; + } + if (snapshot.callId !== callId) return; + publish({ + connection: snapshot.connectionState, + participants: (snapshot.remoteParticipants ?? []).map(toParticipant), + microphoneEnabled: snapshot.microphoneEnabled, + cameraEnabled: snapshot.cameraEnabled, + }); + }; + + const unlistenPromise = listenSnapshot(apply); + // Mark the listen promise as handled immediately; `disconnect` re-attaches. + unlistenPromise.catch(() => undefined); + + // A suspended webview loses the snapshots emitted while it slept, so the + // state is stale on resume. Poll instead of trusting the event stream. + const resync = (): void => { + if (disposed || document.visibilityState !== 'visible') return; + void getNativeCallState() + .then(apply) + .catch((cause: unknown) => logFailure('Native call state resync failed', cause)); + }; + document.addEventListener('visibilitychange', resync); + + const send = (key: CallEncryptionKey): Promise => + setKey({ + callId, + identity: key.identity, + keyIndex: key.keyIndex, + key: toBase64(key.key), + }).then( + () => undefined, + (cause: unknown) => logFailure(`Native call rejected a key for ${key.identity}`, cause) + ); + + const whenConnected = async ( + what: string, + command: () => Promise + ): Promise => { + if (!connected || disposed) return; + await command().then( + () => undefined, + (cause: unknown) => logFailure(what, cause) + ); + }; + + const setCameraEnabled = (enabled: boolean): Promise => + whenConnected(`Native call camera ${enabled ? 'on' : 'off'} failed`, () => + setCamera({ callId, enabled }) + ); + + const connect = async (options: CallTransportConnectOptions): Promise => { + const snapshot = await connectCall({ + callId, + url: options.url, + token: options.token, + microphoneEnabled: options.microphoneEnabled, + encryptionKeys: options.encryptionKeys.map((key) => ({ + identity: key.identity, + keyIndex: key.keyIndex, + key: toBase64(key.key), + })), + }); + connected = true; + // Everything that arrived while the connection was being established is + // replayed rather than dropped. Re-writing a key ring slot is a no-op. + await Promise.all(pendingKeys.splice(0).map(send)); + apply(snapshot); + if (options.cameraEnabled) await setCameraEnabled(true); + }; + + const disconnect = async (): Promise => { + disposed = true; + document.removeEventListener('visibilitychange', resync); + await disconnectCall({ callId }).then( + () => undefined, + (cause: unknown) => logFailure('Native call disconnect failed', cause) + ); + await unlistenPromise.then( + (unlisten) => unlisten(), + (cause: unknown) => logFailure('Native call snapshot listener was never attached', cause) + ); + }; + + return { + ready: unlistenPromise.then(() => undefined), + connect, + disconnect, + setMicrophoneEnabled: (enabled) => + whenConnected(`Native call microphone ${enabled ? 'on' : 'off'} failed`, () => + setMicrophone({ callId, enabled }) + ), + setCameraEnabled, + setEncryptionKey: async (key) => { + if (disposed) return; + if (!connected) { + pendingKeys.push(key); + return; + } + await send(key); + }, + subscribe: (listener) => { + listeners.add(listener); + return () => listeners.delete(listener); + }, + getState: () => state, + capabilities: { + camera: { + switch: () => + whenConnected('Native call camera switch failed', () => + switchNativeCallCamera({ callId }) + ), + }, + audioRoutes: { + list: async (): Promise => { + if (!connected || disposed) return []; + return getAudioRoutes({ callId }).then( + (result) => result.routes, + (cause: unknown) => { + logFailure('Native call audio routes unavailable', cause); + return []; + } + ); + }, + select: (routeId) => + whenConnected('Native call audio route select failed', () => + setAudioRoute({ callId, routeId }) + ), + }, + pictureInPicture: { + setEnabled: (enabled) => + whenConnected(`Native call picture in picture ${enabled ? 'on' : 'off'} failed`, () => + setNativeCallPiPEnabled({ callId, enabled }) + ), + }, + }, + }; +}; From a604d27354cfa51243beb242ea00973f7433f3b8 Mon Sep 17 00:00:00 2001 From: Erwan Leboucher Date: Sat, 1 Aug 2026 23:33:28 +0200 Subject: [PATCH 5/6] feat(call): add the call UI, controls and settings gate --- .changeset/add-new-calls.md | 7 + src/app/components/CallEmbedProvider.tsx | 7 +- src/app/components/create-room/utils.ts | 2 +- .../components/event-history/EventHistory.tsx | 4 +- src/app/components/message/PollEvent.tsx | 7 +- .../message/modals/MessageForward.tsx | 3 +- src/app/features/call-status/CallControl.tsx | 8 +- .../features/call-status/CallStatusShell.tsx | 108 +++ .../call-status/LivekitCallStatus.tsx | 72 ++ .../features/call-status/NativeCallStatus.tsx | 50 ++ .../features/call/CallDevicePreview.css.ts | 51 ++ .../features/call/CallDevicePreview.test.tsx | 96 +++ src/app/features/call/CallDevicePreview.tsx | 141 ++++ src/app/features/call/CallView.test.tsx | 144 ++++ src/app/features/call/CallView.tsx | 69 +- src/app/features/call/Controls.tsx | 8 +- src/app/features/call/IncomingCallModal.tsx | 15 +- .../features/call/LivekitCallParticipant.tsx | 79 +++ .../features/call/LivekitJsCallSurface.css.ts | 288 ++++++++ .../call/LivekitJsCallSurface.test.tsx | 624 +++++++++++++++++ .../features/call/LivekitJsCallSurface.tsx | 477 +++++++++++++ .../features/call/NativeCallSurface.css.ts | 171 +++++ src/app/features/call/NativeCallSurface.tsx | 633 ++++++++++++++++++ src/app/features/call/PrescreenControls.tsx | 137 ++-- src/app/features/call/callChrome.css.ts | 82 +++ src/app/features/call/callChrome.tsx | 207 ++++++ src/app/features/call/callClient.ts | 89 +++ .../features/call/callStartCapabilities.ts | 9 +- .../features/call/callStartSelection.test.ts | 30 + src/app/features/call/callStartSelection.ts | 13 + .../features/call/livekitCallIdentity.test.ts | 46 ++ .../call/livekitJsCallManager.test.tsx | 313 +++++++++ .../features/call/livekitJsCallManager.tsx | 148 ++++ .../features/call/livekitJsController.test.ts | 3 +- .../features/call/matrixRtcCallLifecycle.ts | 10 +- .../features/call/nativeCallManager.test.ts | 145 ++++ src/app/features/call/nativeCallManager.ts | 63 ++ src/app/features/call/useCallSignaling.ts | 60 +- .../call/useCallStartCapabilities.test.tsx | 146 ++++ .../features/call/useCallStartCapabilities.ts | 24 +- .../experimental/Experimental.test.tsx | 94 +++ .../settings/experimental/Experimental.tsx | 18 + .../settings/notifications/AllMessages.tsx | 2 +- .../notifications/UnifiedPushNotifications.ts | 2 +- src/app/features/settings/settingsLink.ts | 1 + src/app/hooks/commands/fun.ts | 3 +- src/app/hooks/commands/misc.ts | 6 +- .../timeline/useProcessedTimeline.test.tsx | 16 - .../hooks/timeline/useProcessedTimeline.ts | 30 +- src/app/hooks/timeline/useTimelineActions.ts | 6 +- .../timeline/useTimelineRendererContext.ts | 7 +- .../hooks/timeline/useTimelineSync.test.tsx | 66 +- src/app/hooks/timeline/useTimelineSync.ts | 10 - src/app/hooks/useAutoJoinCall.ts | 4 + src/app/hooks/useCallEmbed.ts | 69 +- src/app/hooks/useLivekitSupport.test.tsx | 77 +++ src/app/hooks/useLivekitSupport.ts | 48 +- src/app/pages/CallStatusRenderer.test.tsx | 151 +++++ src/app/pages/CallStatusRenderer.tsx | 25 +- .../pages/client/BackgroundNotifications.tsx | 2 +- .../client/client-non-ui/notifications.tsx | 2 +- src/app/plugins/call/CallEmbed.ts | 12 +- src/app/state/callEmbed.ts | 2 + src/app/state/callOwner.test.ts | 41 ++ src/app/state/callOwner.ts | 40 ++ src/app/state/callPreferences.ts | 2 + src/app/state/hooks/callPreferences.ts | 24 +- src/app/state/livekitJsCall.test.ts | 53 ++ src/app/state/livekitJsCall.ts | 52 ++ src/app/state/nativeCall.test.ts | 61 ++ src/app/state/nativeCall.ts | 64 ++ src/app/state/settings.defaults.test.ts | 53 ++ src/app/state/settings.ts | 12 + src/app/utils/messageReaction.ts | 5 +- src/app/utils/notificationStyle.ts | 4 +- src/app/utils/room/relations.ts | 1 + src/app/utils/room/unread.ts | 23 +- src/app/utils/sendFeedbackToUser.ts | 6 +- 78 files changed, 5432 insertions(+), 251 deletions(-) create mode 100644 .changeset/add-new-calls.md create mode 100644 src/app/features/call-status/CallStatusShell.tsx create mode 100644 src/app/features/call-status/LivekitCallStatus.tsx create mode 100644 src/app/features/call-status/NativeCallStatus.tsx create mode 100644 src/app/features/call/CallDevicePreview.css.ts create mode 100644 src/app/features/call/CallDevicePreview.test.tsx create mode 100644 src/app/features/call/CallDevicePreview.tsx create mode 100644 src/app/features/call/CallView.test.tsx create mode 100644 src/app/features/call/LivekitCallParticipant.tsx create mode 100644 src/app/features/call/LivekitJsCallSurface.css.ts create mode 100644 src/app/features/call/LivekitJsCallSurface.test.tsx create mode 100644 src/app/features/call/LivekitJsCallSurface.tsx create mode 100644 src/app/features/call/NativeCallSurface.css.ts create mode 100644 src/app/features/call/NativeCallSurface.tsx create mode 100644 src/app/features/call/callChrome.css.ts create mode 100644 src/app/features/call/callChrome.tsx create mode 100644 src/app/features/call/callClient.ts create mode 100644 src/app/features/call/callStartSelection.test.ts create mode 100644 src/app/features/call/callStartSelection.ts create mode 100644 src/app/features/call/livekitCallIdentity.test.ts create mode 100644 src/app/features/call/livekitJsCallManager.test.tsx create mode 100644 src/app/features/call/livekitJsCallManager.tsx create mode 100644 src/app/features/call/nativeCallManager.test.ts create mode 100644 src/app/features/call/nativeCallManager.ts create mode 100644 src/app/features/call/useCallStartCapabilities.test.tsx create mode 100644 src/app/features/settings/experimental/Experimental.test.tsx create mode 100644 src/app/hooks/useLivekitSupport.test.tsx create mode 100644 src/app/pages/CallStatusRenderer.test.tsx create mode 100644 src/app/state/callOwner.test.ts create mode 100644 src/app/state/callOwner.ts create mode 100644 src/app/state/livekitJsCall.test.ts create mode 100644 src/app/state/livekitJsCall.ts create mode 100644 src/app/state/nativeCall.test.ts create mode 100644 src/app/state/nativeCall.ts diff --git a/.changeset/add-new-calls.md b/.changeset/add-new-calls.md new file mode 100644 index 0000000000..05125226ea --- /dev/null +++ b/.changeset/add-new-calls.md @@ -0,0 +1,7 @@ +--- +default: minor +--- + +# Add experimental new calls + +Introduce an opt-in new call system behind a single experimental setting. It uses LiveKit JS on web and desktop and native LiveKit on supported mobile devices, with Element Call remaining the fallback when the setting is off. diff --git a/src/app/components/CallEmbedProvider.tsx b/src/app/components/CallEmbedProvider.tsx index 78b2b5ee03..b5bbc05bad 100644 --- a/src/app/components/CallEmbedProvider.tsx +++ b/src/app/components/CallEmbedProvider.tsx @@ -17,6 +17,7 @@ import { useSelectedRoom } from '$hooks/router/useSelectedRoom'; import { ScreenSize, useScreenSizeContext } from '$hooks/useScreenSize'; import { IncomingCallModal } from '$features/call/IncomingCallModal'; import { toCallEmbedStartError } from '$plugins/call/callEmbedError'; +import { LivekitJsCallManagerProvider } from '$features/call/livekitJsCallManager'; function CallUtils({ embed }: { embed: CallEmbed }) { const setCallEmbed = useSetAtom(callEmbedAtom); @@ -80,8 +81,10 @@ export function CallEmbedProvider({ children }: CallEmbedProviderProps) { {callEmbed && } - - {children} + + + {children} +
({ }); export const createRoomEncryptionState = () => ({ - type: EventType.RoomEncryption, + type: 'm.room.encryption', state_key: '', content: { algorithm: 'm.megolm.v1.aes-sha2', diff --git a/src/app/components/event-history/EventHistory.tsx b/src/app/components/event-history/EventHistory.tsx index 810cbd9b0e..7fc6442629 100644 --- a/src/app/components/event-history/EventHistory.tsx +++ b/src/app/components/event-history/EventHistory.tsx @@ -48,7 +48,7 @@ import { usePowerLevelsContext } from '$hooks/usePowerLevels'; import { useSettingsLinkBaseUrl } from '$features/settings/useSettingsLinkBaseUrl'; import * as css from './EventHistory.css'; -import { EventType, RelationType } from '$types/matrix-sdk'; +import { EventType } from '$types/matrix-sdk'; type EventHistoryProps = { room: Room; @@ -104,7 +104,7 @@ export const EventHistory = as<'div', EventHistoryProps>( const formattedBody = content?.['m.new_content']?.formatted_body ?? content?.formatted_body ?? ''; const { 'm.relates_to': relation } = startThread - ? { 'm.relates_to': { rel_type: RelationType.Thread, event_id: replyId } } + ? { 'm.relates_to': { rel_type: 'm.thread', event_id: replyId } } : replyEvt.getWireContent(); const senderId = replyEvt.getSender(); if (senderId) { diff --git a/src/app/components/message/PollEvent.tsx b/src/app/components/message/PollEvent.tsx index dd29bf9691..7060af819c 100644 --- a/src/app/components/message/PollEvent.tsx +++ b/src/app/components/message/PollEvent.tsx @@ -13,7 +13,6 @@ import { } from 'matrix-js-sdk'; import * as css from './PollEvent.css'; import { useCallback, useEffect, useState } from 'react'; -import { MsgType, RelationType } from '$types/matrix-sdk'; import { PollResponsesViewer } from '$features/room/poll-modals'; import { ModalOverlay } from '$components/modal-overlay/ModalOverlay'; import { useMatrixEvent } from '$hooks/useMatrixEvent'; @@ -189,7 +188,7 @@ export function PollEvent({ content, mEvent, mx, room }: PollEventProps) { let newContent: PollResponse = { 'm.relates_to': { - rel_type: RelationType.Reference, + rel_type: 'm.reference', event_id: eventId, }, [M_POLL_RESPONSE.name]: { @@ -221,13 +220,13 @@ export function PollEvent({ content, mEvent, mx, room }: PollEventProps) { const endContent = { 'm.relates_to': { - rel_type: RelationType.Reference, + rel_type: 'm.reference', event_id: eventId, }, 'org.matrix.msc3381.poll.end': {}, [M_TEXT.name]: endText, body: endText, - msgtype: MsgType.Text, + msgtype: 'm.text', }; mx.sendEvent( roomId, diff --git a/src/app/components/message/modals/MessageForward.tsx b/src/app/components/message/modals/MessageForward.tsx index 01a6519951..9752dec200 100644 --- a/src/app/components/message/modals/MessageForward.tsx +++ b/src/app/components/message/modals/MessageForward.tsx @@ -6,7 +6,6 @@ import { MenuItem, Text, as } from 'folds'; import { ArrowRight, menuIcon } from '$components/icons/phosphor'; import { useSetAtom } from 'jotai'; import type { MatrixEvent, Room } from '$types/matrix-sdk'; -import { MsgType } from '$types/matrix-sdk'; import { useCallback, useEffect, useMemo, useState } from 'react'; import { useAllJoinedRoomsSet, useGetRoom } from '$hooks/useGetRoom'; import { useMessageTargetRooms } from '$hooks/useMessageTargetRooms'; @@ -130,7 +129,7 @@ export function MessageForwardInternal({ const eventType = mEvent.getType() as SendEventType; const originalContent = mEvent.getContent(); - const isTextMessage = originalContent.msgtype === MsgType.Text; + const isTextMessage = originalContent.msgtype === 'm.text'; const originalBody = typeof originalContent.body === 'string' ? originalContent.body : ''; const originalFormattedBody = diff --git a/src/app/features/call-status/CallControl.tsx b/src/app/features/call-status/CallControl.tsx index 5162b8b08d..c716c65bd5 100644 --- a/src/app/features/call-status/CallControl.tsx +++ b/src/app/features/call-status/CallControl.tsx @@ -23,7 +23,7 @@ type MicrophoneButtonProps = { onToggle: () => Promise; disabled?: boolean; }; -function MicrophoneButton({ enabled, onToggle, disabled }: MicrophoneButtonProps) { +export function MicrophoneButton({ enabled, onToggle, disabled }: MicrophoneButtonProps) { return ( void; disabled?: boolean; }; -function SoundButton({ enabled, onToggle, disabled }: SoundButtonProps) { +export function SoundButton({ enabled, onToggle, disabled }: SoundButtonProps) { return ( Promise; disabled?: boolean; }; -function VideoButton({ enabled, onToggle, disabled }: VideoButtonProps) { +export function VideoButton({ enabled, onToggle, disabled }: VideoButtonProps) { return ( void; disabled?: boolean; }; -function ScreenShareButton({ enabled, onToggle, disabled }: ScreenShareButtonProps) { +export function ScreenShareButton({ enabled, onToggle, disabled }: ScreenShareButtonProps) { return ( (); + +export function HangupChip({ + compact, + onHangup, +}: { + compact: boolean; + onHangup: () => Promise; +}) { + const [hangupState, hangup] = useAsyncCallback(useCallback(() => onHangup(), [onHangup])); + const exiting = + hangupState.status === AsyncStatus.Loading || hangupState.status === AsyncStatus.Success; + + return ( + + ) : ( + sizedIcon(PhoneDisconnect, '50', { filled: true }) + ) + } + disabled={exiting} + outlined + onClick={() => hangup()} + > + {!compact && ( + + End + + )} + + ); +} + +/** + * The persistent call bar both engines render into. Only the control cluster + * differs, so it is passed in. + */ +export function CallStatusShell({ + room, + compact, + connected, + controls, +}: { + room: Room; + compact: boolean; + connected: boolean; + controls: ReactNode; +}) { + const callSession = useCallSession(room); + const callMembers = useCallMembers(room, callSession); + const memberVisible = connected && callMembers.length > 0; + + return ( + + + {memberVisible ? ( + + + + ) : ( + + )} + + {!compact && } + + {memberVisible && ( + + + + )} + + {memberVisible && !compact && } + + {compact && ( + + + + )} + {controls} + + + ); +} diff --git a/src/app/features/call-status/LivekitCallStatus.tsx b/src/app/features/call-status/LivekitCallStatus.tsx new file mode 100644 index 0000000000..3afc8183e6 --- /dev/null +++ b/src/app/features/call-status/LivekitCallStatus.tsx @@ -0,0 +1,72 @@ +import { Box } from 'folds'; +import { useAtom } from 'jotai'; +import { RoomContext, useLocalParticipant } from '@livekit/components-react'; +import { useMatrixClient } from '$hooks/useMatrixClient'; +import { ScreenSize, useScreenSize } from '$hooks/useScreenSize'; +import { livekitJsCallSoundAtom, type LivekitJsCallSession } from '$state/livekitJsCall'; +import { MicrophoneButton, ScreenShareButton, SoundButton, VideoButton } from './CallControl'; +import { CallStatusShell, HangupChip } from './CallStatusShell'; +import { StatusDivider } from './components'; + +function LivekitCallControl({ + compact, + onHangup, +}: { + compact: boolean; + onHangup: () => Promise; +}) { + const { localParticipant, isMicrophoneEnabled, isCameraEnabled, isScreenShareEnabled } = + useLocalParticipant(); + const [sound, setSound] = useAtom(livekitJsCallSoundAtom); + + return ( + + + localParticipant.setMicrophoneEnabled(!isMicrophoneEnabled)} + /> + setSound(!sound)} /> + {!compact && } + localParticipant.setCameraEnabled(!isCameraEnabled)} + /> + {!compact && ( + void localParticipant.setScreenShareEnabled(!isScreenShareEnabled)} + /> + )} + + + + + ); +} + +export function LivekitCallStatus({ session }: { session: LivekitJsCallSession }) { + const mx = useMatrixClient(); + const screenSize = useScreenSize(); + const room = mx.getRoom(session.roomId); + const compact = screenSize === ScreenSize.Mobile; + + if (!room) return null; + + return ( + + + + ) : ( + + ) + } + /> + ); +} diff --git a/src/app/features/call-status/NativeCallStatus.tsx b/src/app/features/call-status/NativeCallStatus.tsx new file mode 100644 index 0000000000..2a8f44df69 --- /dev/null +++ b/src/app/features/call-status/NativeCallStatus.tsx @@ -0,0 +1,50 @@ +import { Box } from 'folds'; +import { useMatrixClient } from '$hooks/useMatrixClient'; +import { ScreenSize, useScreenSize } from '$hooks/useScreenSize'; +import type { NativeCallSession } from '$state/nativeCall'; +import { MicrophoneButton, VideoButton } from './CallControl'; +import { CallStatusShell, HangupChip } from './CallStatusShell'; +import { StatusDivider } from './components'; + +function NativeCallControl({ session, compact }: { session: NativeCallSession; compact: boolean }) { + // Media commands are rejected until the native room has connected. + const disabled = session.lifecycle !== 'connected'; + + return ( + + + session.setMicrophoneEnabled(!session.microphoneEnabled)} + disabled={disabled} + /> + {!compact && } + session.setCameraEnabled(!session.cameraEnabled)} + disabled={disabled} + /> + + + + + ); +} + +export function NativeCallStatus({ session }: { session: NativeCallSession }) { + const mx = useMatrixClient(); + const screenSize = useScreenSize(); + const room = mx.getRoom(session.roomId); + const compact = screenSize === ScreenSize.Mobile; + + if (!room) return null; + + return ( + } + /> + ); +} diff --git a/src/app/features/call/CallDevicePreview.css.ts b/src/app/features/call/CallDevicePreview.css.ts new file mode 100644 index 0000000000..90252a9110 --- /dev/null +++ b/src/app/features/call/CallDevicePreview.css.ts @@ -0,0 +1,51 @@ +import { style } from '@vanilla-extract/css'; +import { color, config, toRem } from 'folds'; + +export const PreviewSurface = style({ + position: 'relative', + width: '100%', + aspectRatio: '16 / 9', + borderRadius: config.radii.R400, + background: '#14171f', + color: color.Surface.OnContainer, + overflow: 'hidden', +}); + +export const PreviewVideo = style({ + position: 'absolute', + inset: 0, + width: '100%', + height: '100%', + objectFit: 'cover', + // Front cameras read as a mirror to the person looking at them. + transform: 'scaleX(-1)', +}); + +export const DeviceSelect = style({ + width: '100%', + minWidth: 0, + minHeight: toRem(36), + padding: `0 ${config.space.S200}`, + borderRadius: config.radii.R400, + border: `1px solid ${color.Surface.ContainerLine}`, + background: color.Surface.Container, + color: color.Surface.OnContainer, + font: 'inherit', + fontSize: toRem(14), +}); + +export const LevelTrack = style({ + width: '100%', + height: toRem(6), + borderRadius: config.radii.R400, + background: color.Surface.ContainerLine, + overflow: 'hidden', +}); + +export const LevelFill = style({ + width: '100%', + height: '100%', + transformOrigin: 'left center', + background: color.Success.Main, + transition: 'transform 80ms linear', +}); diff --git a/src/app/features/call/CallDevicePreview.test.tsx b/src/app/features/call/CallDevicePreview.test.tsx new file mode 100644 index 0000000000..24bcd4f1e5 --- /dev/null +++ b/src/app/features/call/CallDevicePreview.test.tsx @@ -0,0 +1,96 @@ +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { CallDevicePreview } from './CallDevicePreview'; + +const mocks = vi.hoisted(() => ({ + usePreviewTracks: vi.fn<() => unknown[] | undefined>(), + useMediaDeviceSelect: vi.fn<(o: { kind: string }) => { devices: MediaDeviceInfo[] }>(), + previewOptions: undefined as unknown, + attach: vi.fn<(el: HTMLElement) => void>(), + detach: vi.fn<(el: HTMLElement) => void>(), +})); + +vi.mock('@livekit/components-react', () => ({ + // Mirror the real hook: it only returns tracks for the sources requested. + usePreviewTracks: (options: { audio: unknown; video: unknown }) => { + mocks.previewOptions = options; + const tracks = mocks.usePreviewTracks() ?? []; + return tracks.filter((track) => { + const { kind } = track as { kind: string }; + return kind === 'video' ? options.video !== false : options.audio !== false; + }); + }, + useMediaDeviceSelect: (options: { kind: string }) => mocks.useMediaDeviceSelect(options), + useTrackVolume: () => 0.25, +})); + +vi.mock('livekit-client', () => ({ + Track: { Kind: { Video: 'video', Audio: 'audio' } }, +})); + +const device = (deviceId: string, label: string, kind: string) => + ({ deviceId, label, kind }) as MediaDeviceInfo; + +const videoTrack = { kind: 'video', attach: mocks.attach, detach: mocks.detach }; +const audioTrack = { kind: 'audio' }; + +const props = { + microphone: true, + video: true, + onAudioDeviceChange: vi.fn<(id: string) => void>(), + onVideoDeviceChange: vi.fn<(id: string) => void>(), +}; + +beforeEach(() => { + mocks.usePreviewTracks.mockReset().mockReturnValue([videoTrack, audioTrack]); + mocks.useMediaDeviceSelect.mockReset().mockImplementation(({ kind }) => ({ + devices: + kind === 'audioinput' + ? [device('mic-1', 'Built-in Mic', kind), device('mic-2', 'USB Mic', kind)] + : [device('cam-1', 'FaceTime HD', kind)], + })); + mocks.attach.mockReset(); + mocks.detach.mockReset(); + props.onAudioDeviceChange.mockReset(); + props.onVideoDeviceChange.mockReset(); +}); + +describe('CallDevicePreview', () => { + it('attaches the preview camera track to a video element', () => { + render(); + + expect(mocks.attach).toHaveBeenCalledOnce(); + expect(screen.queryByText('Camera is off')).not.toBeInTheDocument(); + }); + + it('detaches the camera track on unmount so the call can claim it', () => { + const { unmount } = render(); + unmount(); + + expect(mocks.detach).toHaveBeenCalledOnce(); + }); + + it('requests only the devices the user actually enabled', () => { + render(); + + expect(mocks.previewOptions).toEqual({ audio: { deviceId: 'mic-2' }, video: false }); + expect(screen.getByText('Camera is off')).toBeInTheDocument(); + }); + + it('reports a chosen microphone so the call can honour it', async () => { + render(); + + await userEvent.selectOptions(screen.getByLabelText('Microphone'), 'mic-2'); + + expect(props.onAudioDeviceChange).toHaveBeenCalledWith('mic-2'); + }); + + it('shows a microphone level only while the microphone is on', () => { + const { rerender } = render(); + expect(screen.getByRole('meter', { name: 'Microphone level' })).toBeInTheDocument(); + + rerender(); + expect(screen.queryByRole('meter', { name: 'Microphone level' })).not.toBeInTheDocument(); + }); +}); diff --git a/src/app/features/call/CallDevicePreview.tsx b/src/app/features/call/CallDevicePreview.tsx new file mode 100644 index 0000000000..c5573c0a13 --- /dev/null +++ b/src/app/features/call/CallDevicePreview.tsx @@ -0,0 +1,141 @@ +import { useEffect, useMemo, useRef } from 'react'; +import { Box, Text, config, toRem } from 'folds'; +import { useMediaDeviceSelect, usePreviewTracks, useTrackVolume } from '@livekit/components-react'; +import { Track, type LocalAudioTrack, type LocalVideoTrack } from 'livekit-client'; +import { VideoCameraSlash, sizedIcon } from '$components/icons/phosphor'; +import * as css from './CallDevicePreview.css'; + +type DeviceSelectProps = { + label: string; + kind: MediaDeviceKind; + deviceId?: string; + onChange: (deviceId: string) => void; + permissionsGranted: boolean; +}; + +function DeviceSelect({ label, kind, deviceId, onChange, permissionsGranted }: DeviceSelectProps) { + const { devices } = useMediaDeviceSelect({ kind, requestPermissions: permissionsGranted }); + + return ( + + {label} + + + ); +} + +function MicrophoneLevel({ track }: { track?: LocalAudioTrack }) { + const volume = useTrackVolume(track); + const level = Math.min(1, volume * 3); + + return ( + + Microphone level +
+
+
+ + ); +} + +function VideoPreview({ track }: { track?: LocalVideoTrack }) { + const videoRef = useRef(null); + + useEffect(() => { + const element = videoRef.current; + if (!track || !element) return undefined; + track.attach(element); + return () => { + track.detach(element); + }; + }, [track]); + + if (!track) { + return ( + + {sizedIcon(VideoCameraSlash, '400')} + Camera is off + + ); + } + + return ( + + + ); +} + +export type CallDevicePreviewProps = { + microphone: boolean; + video: boolean; + audioDeviceId?: string; + videoDeviceId?: string; + onAudioDeviceChange: (deviceId: string) => void; + onVideoDeviceChange: (deviceId: string) => void; +}; + +export function CallDevicePreview({ + microphone, + video, + audioDeviceId, + videoDeviceId, + onAudioDeviceChange, + onVideoDeviceChange, +}: CallDevicePreviewProps) { + const tracks = usePreviewTracks({ + audio: microphone ? { deviceId: audioDeviceId } : false, + video: video ? { deviceId: videoDeviceId } : false, + }); + + const videoTrack = useMemo( + () => tracks?.find((track) => track.kind === Track.Kind.Video) as LocalVideoTrack | undefined, + [tracks] + ); + const audioTrack = useMemo( + () => tracks?.find((track) => track.kind === Track.Kind.Audio) as LocalAudioTrack | undefined, + [tracks] + ); + + return ( + + + {microphone && } + + + + + + ); +} diff --git a/src/app/features/call/CallView.test.tsx b/src/app/features/call/CallView.test.tsx new file mode 100644 index 0000000000..40d4acfd90 --- /dev/null +++ b/src/app/features/call/CallView.test.tsx @@ -0,0 +1,144 @@ +import { render, screen } from '@testing-library/react'; +import { describe, expect, it, vi } from 'vitest'; +import { LivekitJsCallStatus } from './CallView'; +import { NativeCallSurface } from './NativeCallSurface'; +import type { NativeCallSession } from '$state/nativeCall'; + +vi.mock('./livekitMobileBridge', () => ({ + setNativeCallRemoteVideoOverlay: vi.fn<() => Promise>(() => Promise.resolve({})), + clearNativeCallRemoteVideoOverlay: vi.fn<() => Promise>(() => Promise.resolve({})), + setNativeCallLocalVideoOverlay: vi.fn<() => Promise>(() => Promise.resolve({})), + clearNativeCallLocalVideoOverlay: vi.fn<() => Promise>(() => Promise.resolve({})), +})); + +vi.mock('$hooks/useRoom', () => ({ useRoom: () => ({ roomId: '!room:example.org' }) })); +vi.mock('$hooks/router/useSelectedRoom', () => ({ + useSelectedRoom: () => '!room:example.org', +})); +vi.mock('$hooks/useCall', () => ({ useCallSession: () => ({}), useCallMembers: () => [] })); +vi.mock('./LivekitCallParticipant', () => ({ + useCallParticipantProfile: () => ({ name: 'Bob' }), + CallParticipantAvatar: () =>
, +})); + +const nativeSession = (lifecycle: NativeCallSession['lifecycle']): NativeCallSession => ({ + backend: 'livekit-mobile', + roomId: '!room:example.org', + callId: 'call-id', + lifecycle, + participants: [], + microphoneEnabled: true, + cameraEnabled: false, + setMicrophoneEnabled: async () => {}, + setCameraEnabled: async () => {}, + switchCamera: async () => {}, + listAudioRoutes: async () => [], + selectAudioRoute: async () => {}, + hangup: vi.fn<() => Promise>().mockResolvedValue(undefined), +}); + +describe('LiveKit JS call status', () => { + it('reports progress without exposing backend or transport details', () => { + render( + {}} + /> + ); + + expect(screen.getByText('Preparing call')).toBeInTheDocument(); + expect(screen.queryByText(/livekit|token|url|secret|e2ee/i)).not.toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'End' })).toBeInTheDocument(); + }); + + it('explains a setup failure in plain language', () => { + render( + {}} + /> + ); + + expect(screen.getByText('Call failed')).toBeInTheDocument(); + expect(screen.getByText('Could not connect to the call.')).toBeInTheDocument(); + expect(screen.queryByText(/token|url|secret|error:/i)).not.toBeInTheDocument(); + }); + + it('gives an unsupported-encryption failure a dismiss route', () => { + const onHangup = vi.fn<() => void>(); + render( + + ); + + expect( + screen.getByText('Encrypted calls are not supported on this device.') + ).toBeInTheDocument(); + screen.getByRole('button', { name: 'Dismiss' }).click(); + expect(onHangup).toHaveBeenCalledOnce(); + }); +}); + +describe('native call surface', () => { + it('shows the local tile and call controls when connected', () => { + render( {}} />); + + expect(screen.getByText('You')).toBeInTheDocument(); + expect(screen.getAllByRole('button')).toHaveLength(3); + expect(screen.getByRole('button', { name: 'Mute microphone' })).toBeEnabled(); + expect(screen.getByRole('button', { name: 'Start camera' })).toBeEnabled(); + expect(screen.getByRole('button', { name: 'End call' })).toBeInTheDocument(); + }); + + it('keeps media toggles disabled while connecting', () => { + render( {}} />); + + expect(screen.getByText('Connecting')).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Mute microphone' })).toBeDisabled(); + expect(screen.getByRole('button', { name: 'Start camera' })).toBeDisabled(); + expect(screen.getByRole('button', { name: 'End call' })).toBeEnabled(); + }); + + it('renders a remote tile from the participants the session carries', () => { + render( + {}} + /> + ); + + expect(screen.getByText('Bob')).toBeInTheDocument(); + expect(screen.getByRole('img', { name: 'Poor connection' })).toBeInTheDocument(); + expect(screen.getByLabelText('Camera off')).toBeInTheDocument(); + }); + + it('gives failed calls an explicit dismiss route', () => { + const onHangup = vi.fn<() => void>(); + render( + + ); + + expect(screen.getByText('Call failed')).toBeInTheDocument(); + expect(screen.getByText('Native call connection failed.')).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Dismiss' })).toBeInTheDocument(); + screen.getByRole('button', { name: 'Dismiss' }).click(); + expect(onHangup).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/app/features/call/CallView.tsx b/src/app/features/call/CallView.tsx index 62d79cd6d7..f250d0bb5f 100644 --- a/src/app/features/call/CallView.tsx +++ b/src/app/features/call/CallView.tsx @@ -13,6 +13,13 @@ import { CallMemberRenderer } from './CallMemberCard'; import { PrescreenControls } from './PrescreenControls'; import { callEmbedAtom, callEmbedStartErrorAtom } from '$state/callEmbed'; import { canJoinCall } from './callStartCapabilities'; +import type { LivekitJsCallSession } from '$state/livekitJsCall'; +import { livekitJsCallAtom } from '$state/livekitJsCall'; +import { nativeCallAtom } from '$state/nativeCall'; +import { LivekitJsCallSurface } from './LivekitJsCallSurface'; +import { NativeCallSurface } from './NativeCallSurface'; +import { CallStatusBar } from './callChrome'; +import { livekitJsCallStatus } from './callClient'; function LivekitServerMissingMessage() { return ( @@ -80,6 +87,16 @@ function WidgetPreparationErrorMessage({ message }: { message: string }) { ); } +export function LivekitJsCallStatus({ + session, + onHangup, +}: { + session: Pick; + onHangup: () => void; +}) { + return ; +} + function CallPrescreen() { const room = useRoom(); const callEmbed = useAtomValue(callEmbedAtom); @@ -168,11 +185,22 @@ export function CallView({ resizable }: CallViewProps) { const callEmbed = useCallEmbed(); const callJoined = useCallJoined(callEmbed); - - const currentJoined = callEmbed?.roomId === room.roomId && callJoined; - + const livekitJsCall = useAtomValue(livekitJsCallAtom); + const nativeCall = useAtomValue(nativeCallAtom); + + const livekitJsCallForRoom = livekitJsCall?.roomId === room.roomId ? livekitJsCall : undefined; + const nativeCallForRoom = nativeCall?.roomId === room.roomId ? nativeCall : undefined; + const livekitJsRoom = + livekitJsCallForRoom?.lifecycle === 'active' ? livekitJsCallForRoom.room : undefined; + const currentJoined = + !livekitJsCallForRoom && !nativeCallForRoom && callEmbed?.roomId === room.roomId && callJoined; + + // A native call renders video tiles and a control bar, which need most of the + // viewport; the 0.3 default is sized for the Element Call participant list. const [heightRatio, setHeightRatio] = useState(isMobile ? 0.3 : 0.72); const [availableHeight, setAvailableHeight] = useState(0); + const effectiveHeightRatio = + isMobile && nativeCallForRoom ? Math.max(heightRatio, 0.75) : heightRatio; useEffect(() => { if (!resizable || !callViewRef.current) return undefined; @@ -263,12 +291,18 @@ export function CallView({ resizable }: CallViewProps) { minWidth: toRem(280), height: resizable ? availableHeight > 0 - ? `${availableHeight * heightRatio}px` - : `${heightRatio * 100}dvh` + ? `${availableHeight * effectiveHeightRatio}px` + : `${effectiveHeightRatio * 100}dvh` : undefined, borderBottom: `1px solid var(--sable-surface-container-line)`, zIndex: 20, - backgroundColor: currentJoined ? 'transparent' : undefined, + backgroundColor: + livekitJsRoom || nativeCallForRoom + ? color.Background.Container + : currentJoined + ? 'transparent' + : undefined, + overflow: livekitJsRoom || nativeCallForRoom ? 'hidden' : undefined, pointerEvents: currentJoined ? 'none' : 'all', }} > @@ -284,8 +318,27 @@ export function CallView({ resizable }: CallViewProps) { /> )} - {!currentJoined && } - + {!currentJoined && !livekitJsCallForRoom && !nativeCallForRoom && } + {livekitJsCallForRoom && livekitJsRoom ? ( + void livekitJsCallForRoom.hangup()} + /> + ) : livekitJsCallForRoom ? ( + void livekitJsCallForRoom.hangup()} + /> + ) : nativeCallForRoom ? ( + void nativeCallForRoom.hangup()} + /> + ) : ( + + )} {resizable && ( + ), + color: { + Critical: { Container: 'red', OnContainer: 'white' }, + Surface: { OnContainer: 'white' }, + Warning: { Container: 'yellow', OnContainer: 'black' }, + }, + config: { + radii: { R500: '5px' }, + space: { S100: '4px', S200: '8px', S300: '12px' }, + }, + Text: ({ children }: { children: ReactNode }) => {children}, + toRem: (value: number) => `${value}px`, +})); + +const room = {} as Room; +const initialMedia = { microphone: true, camera: false, sound: true }; + +const videoTrack = (source: string, isLocal: boolean, identity = 'hash-alice') => ({ + source, + participant: { isLocal, identity }, + publication: {}, +}); + +const placeholderTrack = (source: string, isLocal: boolean, identity = 'hash-alice') => ({ + source, + participant: { isLocal, identity }, + publication: undefined, +}); + +const surfaceElement = () => document.body.querySelector('[data-livekit-call-surface]'); +const controlsElement = () => document.body.querySelector('[data-livekit-controls]'); + +beforeEach(() => { + getDefaultStore().set(livekitJsCallInitialMediaAppliedAtom, false); + mocks.useConnectionState.mockReset().mockReturnValue('connected'); + mocks.useTracks.mockReset().mockReturnValue([]); + mocks.useParticipants.mockReset().mockReturnValue([]); + mocks.setMicrophoneEnabled.mockReset().mockResolvedValue(undefined); + mocks.setCameraEnabled.mockReset().mockResolvedValue(undefined); + mocks.onDeviceError = undefined; +}); + +describe('LiveKit JS call surface', () => { + it('publishes with the devices chosen in the prescreen preview', () => { + render( + {}} + /> + ); + + expect(mocks.setMicrophoneEnabled).toHaveBeenCalledWith(true, { deviceId: 'mic-2' }); + expect(mocks.setCameraEnabled).toHaveBeenCalledWith(true, { deviceId: 'cam-1' }); + }); + + it('leaves device selection to the browser when the user never picked one', () => { + render( + {}} + /> + ); + + expect(mocks.setMicrophoneEnabled).toHaveBeenCalledWith(true, undefined); + expect(mocks.setCameraEnabled).toHaveBeenCalledWith(true, undefined); + }); + + it('applies the prescreen media choice once media is ready', () => { + render( + {}} + /> + ); + + expect(mocks.setMicrophoneEnabled).toHaveBeenCalledWith(true, undefined); + expect(mocks.setCameraEnabled).toHaveBeenCalledWith(true, undefined); + }); + + it('joins muted when the prescreen microphone was off', () => { + render( + {}} + /> + ); + + expect(mocks.setMicrophoneEnabled).toHaveBeenCalledWith(false, undefined); + expect(mocks.setCameraEnabled).not.toHaveBeenCalled(); + }); + + it('keeps the call bar state when the surface remounts, instead of re-muting', () => { + const media = { microphone: false, camera: false, sound: true }; + const view = render( + {}} /> + ); + + expect(mocks.setMicrophoneEnabled).toHaveBeenCalledTimes(1); + view.unmount(); + + render( + {}} /> + ); + + expect(mocks.setMicrophoneEnabled).toHaveBeenCalledTimes(1); + }); + + it('publishes nothing until media is ready', () => { + render( + {}} + /> + ); + + expect(mocks.setMicrophoneEnabled).not.toHaveBeenCalled(); + expect(mocks.setCameraEnabled).not.toHaveBeenCalled(); + }); + + it('mutes incoming audio while the shared sound toggle is off', () => { + const store = createStore(); + store.set(livekitJsCallSoundAtom, false); + + render( + + {}} + /> + + ); + + expect(screen.getByTestId('room-audio')).toHaveAttribute('data-muted', 'true'); + }); + + it('lets LiveKit derive publish controls from the token grants', () => { + render( + {}} + /> + ); + + expect(screen.getByTestId('control-bar')).toHaveAttribute('data-controls', '{"leave":false}'); + expect(screen.getByTestId('device-menu')).toHaveAttribute('data-kind', 'audiooutput'); + }); + + it('surfaces a denied microphone instead of failing silently', async () => { + render( + {}} + /> + ); + + act(() => mocks.onDeviceError?.({ source: 'microphone' })); + + expect( + await screen.findByText('Microphone unavailable. Check your browser permissions.') + ).toBeInTheDocument(); + }); + + it('gives an unencrypted call its controls and devices with no key wait', () => { + // `mediaReady` is true from the start for an unencrypted room, where + // MSC4143 forbids MatrixRTC encryption and no Matrix key ever arrives. + render( + {}} + /> + ); + + expect(screen.getByTestId('control-bar')).toBeInTheDocument(); + expect(screen.queryByText('Securing call…')).not.toBeInTheDocument(); + expect(mocks.setMicrophoneEnabled).toHaveBeenCalledWith(true, undefined); + expect(mocks.setCameraEnabled).toHaveBeenCalledWith(true, undefined); + }); + + it('withholds media controls until call encryption is ready', () => { + render( + {}} + /> + ); + + expect(screen.queryByTestId('control-bar')).not.toBeInTheDocument(); + expect(screen.getByText('Securing call…')).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'End call' })).toBeInTheDocument(); + }); + + it('keeps audio-only calls understandable with persistent controls', () => { + render( + {}} + /> + ); + + expect(screen.getByTestId('room-audio')).toBeInTheDocument(); + expect(screen.getByText('Audio call')).toBeInTheDocument(); + expect(screen.getByTestId('control-bar')).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'End call' })).toBeInTheDocument(); + }); + + it('shows every participant when no camera is published, not an empty grid', () => { + mocks.useTracks.mockReturnValue([ + placeholderTrack('camera', true), + placeholderTrack('camera', false), + ]); + mocks.useParticipants.mockReturnValue([{ identity: 'hash-alice' }, { identity: 'hash-bob' }]); + + render( + {}} + /> + ); + + expect(screen.queryByTestId('grid-layout')).not.toBeInTheDocument(); + expect(screen.queryByTestId('focus-layout')).not.toBeInTheDocument(); + expect(screen.getAllByTestId('participant-avatar')).toHaveLength(2); + expect(screen.getByText('@alice:example.org')).toBeInTheDocument(); + expect(screen.getByText('@bob:example.org')).toBeInTheDocument(); + }); + + it('labels video tiles with the Matrix display name, never the LiveKit identity hash', () => { + mocks.useTracks.mockReturnValue([videoTrack('camera', false)]); + + render( + {}} + /> + ); + + expect(screen.getByTestId('grid-layout')).toBeInTheDocument(); + expect(screen.getByText('@alice:example.org')).toBeInTheDocument(); + expect(screen.queryByText(/hash-/)).not.toBeInTheDocument(); + }); + + it('keeps the iOS webview from popping its own picture-in-picture overlay', () => { + mocks.useTracks.mockReturnValue([videoTrack('camera', false)]); + + render( + {}} + /> + ); + + expect(screen.getByTestId('video-track')).toHaveAttribute('data-disable-pip', 'true'); + }); + + it('fills the call view container rather than the viewport', () => { + const { container } = render( + {}} + /> + ); + + const surface = container.querySelector('[data-livekit-call-surface]'); + expect(surface).not.toBeNull(); + expect(surface).toHaveStyle({ position: 'relative', width: '100%', height: '100%' }); + expect(screen.getByRole('region', { name: 'Call' })).toBe(surface); + }); + + it('puts the screen share in focus and keeps other tracks in a carousel', () => { + mocks.useTracks.mockReturnValue([ + videoTrack('screen_share', false), + videoTrack('camera', false), + ]); + + render( + {}} + /> + ); + + expect(screen.getByTestId('focus-layout-container')).toBeInTheDocument(); + expect(screen.getByTestId('focus-layout')).toHaveAttribute('data-focus-source', 'screen_share'); + expect(screen.getByTestId('carousel-layout')).toHaveAttribute('data-track-count', '1'); + expect(screen.queryByTestId('grid-layout')).not.toBeInTheDocument(); + expect(screen.queryByText('Audio call')).not.toBeInTheDocument(); + }); + + it('labels the focused screen share instead of leaving the default tile on top', () => { + mocks.useTracks.mockReturnValue([ + videoTrack('screen_share', false), + videoTrack('camera', false), + ]); + + render( + {}} + /> + ); + + const focus = within(screen.getByTestId('focus-layout')); + expect(focus.getByText("@alice:example.org's screen")).toBeInTheDocument(); + expect(focus.queryByText(/hash-/)).not.toBeInTheDocument(); + }); + + it('keeps an active local screen share on stage instead of suppressing it', () => { + mocks.useTracks.mockReturnValue([ + videoTrack('screen_share', true), + videoTrack('camera', false), + placeholderTrack('camera', true), + ]); + + render( + {}} + /> + ); + + expect(screen.getByTestId('focus-layout')).toHaveAttribute('data-focus-source', 'screen_share'); + expect(screen.getByRole('status')).toHaveTextContent('Sharing your screen'); + expect(screen.getByTestId('carousel-layout')).toHaveAttribute('data-track-count', '2'); + }); + + it('shows connection loss feedback without replacing the media canvas', () => { + mocks.useConnectionState.mockReturnValue('reconnecting'); + + render( + {}} + /> + ); + + expect(screen.getByRole('status')).toHaveTextContent('Reconnecting…'); + expect(screen.getByTestId('room-audio')).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'End call' })).toBeInTheDocument(); + }); + + it('hides idle controls from view and hit testing, then reveals them on a canvas tap', () => { + vi.useFakeTimers(); + try { + render( + {}} + /> + ); + + const surface = surfaceElement()!; + const controls = controlsElement()!; + const pill = screen.getByTestId('control-bar').parentElement!; + + act(() => vi.advanceTimersByTime(3500)); + expect(controls).toHaveStyle({ opacity: '0', visibility: 'hidden' }); + expect(pill).toHaveStyle({ pointerEvents: 'none' }); + + act(() => fireEvent.pointerDown(surface)); + expect(controls).toHaveStyle({ opacity: '1', visibility: 'visible' }); + expect(pill).toHaveStyle({ pointerEvents: 'auto' }); + } finally { + vi.useRealTimers(); + } + }); + + it('reveals hidden controls on a key press, the only route a keyboard user has', () => { + vi.useFakeTimers(); + try { + render( + {}} + /> + ); + + const surface = surfaceElement()!; + const controls = controlsElement()!; + + act(() => vi.advanceTimersByTime(3500)); + expect(controls).toHaveStyle({ visibility: 'hidden' }); + + // focusIn cannot fire from a Tab press: visibility:hidden removes the + // controls from the tab order and nothing else inside the surface is + // focusable, so keydown is the reveal a keyboard user can actually reach. + act(() => fireEvent.keyDown(surface, { key: 'Tab' })); + expect(controls).toHaveStyle({ visibility: 'visible' }); + } finally { + vi.useRealTimers(); + } + }); + + it('labels the shared control surface for assistive technology', () => { + render( + {}} + /> + ); + + expect(screen.getByRole('group', { name: 'Call controls' })).toBe(controlsElement()); + }); + + it('ends the call from the control bar', () => { + const onHangup = vi.fn<() => void>(); + render( + + ); + + screen.getByRole('button', { name: 'End call' }).click(); + expect(onHangup).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/app/features/call/LivekitJsCallSurface.tsx b/src/app/features/call/LivekitJsCallSurface.tsx new file mode 100644 index 0000000000..f14c15b1b7 --- /dev/null +++ b/src/app/features/call/LivekitJsCallSurface.tsx @@ -0,0 +1,477 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { useAtom, useAtomValue } from 'jotai'; +import { Box, Button, color, config, Text, toRem } from 'folds'; +import { + CarouselLayout, + ConnectionQualityIndicator, + ControlBar, + MediaDeviceMenu, + FocusLayout, + FocusLayoutContainer, + GridLayout, + ParticipantTile, + RoomAudioRenderer, + RoomContext, + TrackMutedIndicator, + useConnectionState, + useEnsureTrackRef, + useIsSpeaking, + useLocalParticipant, + useParticipants, + useTracks, + VideoTrack, +} from '@livekit/components-react'; +import { ConnectionState, Track, type Participant, type Room } from 'livekit-client'; +import { SpeakerHigh, sizedIcon } from '$components/icons/phosphor'; +import { useCallMembers, useCallSession } from '$hooks/useCall'; +import { useRoom } from '$hooks/useRoom'; +import { + livekitJsCallInitialMediaAppliedAtom, + livekitJsCallSoundAtom, + type LivekitJsCallMedia, +} from '$state/livekitJsCall'; +import { buildRtcIdentityMap, type UserIdByRtcIdentity } from './livekitCallIdentity'; +import { + CallParticipantAvatar, + CallParticipantName, + useCallParticipantProfile, +} from './LivekitCallParticipant'; +import { CallControlBar, CallLayout } from './callChrome'; +import * as css from './LivekitJsCallSurface.css'; + +const controlIdleDelay = 3500; + +// Camera carries a placeholder so participants without video still get a tile; +// screen share must not, or every participant would fake a shared screen. +const trackSources = [ + { source: Track.Source.Camera, withPlaceholder: true }, + { source: Track.Source.ScreenShare, withPlaceholder: false }, +]; + +const trackOptions = { onlySubscribed: false }; + +function CallTileContent({ userIdByIdentity }: { userIdByIdentity: UserIdByRtcIdentity }) { + const trackRef = useEnsureTrackRef(); + const { participant, publication, source } = trackRef; + const profile = useCallParticipantProfile( + participant.identity, + participant.isLocal, + userIdByIdentity + ); + const isScreenShare = source === Track.Source.ScreenShare; + + return ( + <> + {publication ? ( + // Without this, iOS WKWebView pops its own PiP overlay over the call. + + ) : ( +
+ +
+ )} +
+
+ {!isScreenShare && ( + + )} + + {isScreenShare ? `${profile.name}'s screen` : profile.name} + +
+ +
+ + ); +} + +function GridTile({ userIdByIdentity }: { userIdByIdentity: UserIdByRtcIdentity }) { + return ( + + + + ); +} + +function CarouselTile({ userIdByIdentity }: { userIdByIdentity: UserIdByRtcIdentity }) { + return ( + + + + ); +} + +function AudioCallParticipant({ + participant, + userIdByIdentity, +}: { + participant: Participant; + userIdByIdentity: UserIdByRtcIdentity; +}) { + const profile = useCallParticipantProfile( + participant.identity, + participant.isLocal, + userIdByIdentity, + 192 + ); + const speaking = useIsSpeaking(participant); + + return ( + + + + + ); +} + +function AudioCallLayout({ userIdByIdentity }: { userIdByIdentity: UserIdByRtcIdentity }) { + const participants = useParticipants(); + + return ( + + + Audio call + + + {participants.map((participant) => ( + + ))} + + + ); +} + +function MediaLayout({ + tracks, + userIdByIdentity, +}: { + tracks: ReturnType; + userIdByIdentity: UserIdByRtcIdentity; +}) { + const screenShare = tracks.find((track) => track.source === Track.Source.ScreenShare); + + if (screenShare) { + const remainingTracks = tracks.filter((track) => track !== screenShare); + return ( + + + + + {remainingTracks.length > 0 && ( + + + + )} + + ); + } + + return ( + + + + ); +} + +function ConnectionFeedback() { + const connectionState = useConnectionState(); + if (connectionState === ConnectionState.Connected) return null; + + // Only Disconnected is an actual loss. SignalReconnecting keeps media flowing + // while the signal link re-establishes, and Connecting is the initial + // handshake, so neither deserves the critical treatment. + const lost = connectionState === ConnectionState.Disconnected; + const label = + connectionState === ConnectionState.Connecting + ? 'Connecting…' + : lost + ? 'Connection lost' + : 'Reconnecting…'; + return ( + + {label} + + ); +} + +const deviceErrorMessages: Partial> = { + [Track.Source.Microphone]: 'Microphone unavailable. Check your browser permissions.', + [Track.Source.Camera]: 'Camera unavailable. Check your browser permissions.', + [Track.Source.ScreenShare]: 'Screen sharing was not started.', +}; + +function LivekitJsCallContent({ + mediaReady, + initialMedia, + onHangup, +}: { + mediaReady: boolean; + initialMedia: LivekitJsCallMedia; + onHangup: () => void; +}) { + const tracks = useTracks(trackSources, trackOptions); + const localScreenShare = tracks.some( + (track) => track.source === Track.Source.ScreenShare && track.participant.isLocal + ); + const hasVideo = tracks.some((track) => track.publication !== undefined); + const matrixRoom = useRoom(); + const callSession = useCallSession(matrixRoom); + const callMembers = useCallMembers(matrixRoom, callSession); + const userIdByIdentity = useMemo(() => buildRtcIdentityMap(callMembers), [callMembers]); + const [controlsVisible, setControlsVisible] = useState(true); + const [deviceError, setDeviceError] = useState(undefined); + const hideTimer = useRef | undefined>(undefined); + const { localParticipant } = useLocalParticipant(); + const soundEnabled = useAtomValue(livekitJsCallSoundAtom); + const [appliedInitialMedia, setAppliedInitialMedia] = useAtom( + livekitJsCallInitialMediaAppliedAtom + ); + + const handleDeviceError = useCallback(({ source }: { source: Track.Source }) => { + setDeviceError(deviceErrorMessages[source] ?? 'A media device is unavailable.'); + }, []); + + // In an encrypted room, publishing before the local key is imported would + // send frames in the clear, so the prescreen choice waits for `mediaReady`. + // An unencrypted room is ready straight away and never waits on Matrix keys. + useEffect(() => { + if (!mediaReady || appliedInitialMedia) return; + setAppliedInitialMedia(true); + localParticipant + .setMicrophoneEnabled( + initialMedia.microphone, + initialMedia.audioDeviceId ? { deviceId: initialMedia.audioDeviceId } : undefined + ) + .catch(() => handleDeviceError({ source: Track.Source.Microphone })); + if (initialMedia.camera) { + localParticipant + .setCameraEnabled( + true, + initialMedia.videoDeviceId ? { deviceId: initialMedia.videoDeviceId } : undefined + ) + .catch(() => handleDeviceError({ source: Track.Source.Camera })); + } + }, [ + mediaReady, + initialMedia, + localParticipant, + handleDeviceError, + appliedInitialMedia, + setAppliedInitialMedia, + ]); + + const revealControls = useCallback(() => { + setControlsVisible(true); + if (hideTimer.current) clearTimeout(hideTimer.current); + hideTimer.current = setTimeout(() => { + if (!document.activeElement?.closest('[data-livekit-controls]')) { + setControlsVisible(false); + } + }, controlIdleDelay); + }, []); + + useEffect(() => { + revealControls(); + return () => { + if (hideTimer.current) clearTimeout(hideTimer.current); + }; + }, [revealControls]); + + return ( + setControlsVisible(true)} + > + + + {hasVideo ? ( + + ) : ( + + )} + + + {deviceError && ( + + {deviceError} + + + )} + {localScreenShare && ( + + Sharing your screen + + )} + setControlsVisible(true)} + > + {mediaReady ? ( + <> + + + {sizedIcon(SpeakerHigh, '300')} + + + ) : ( + + Securing call… + + )} + + + + ); +} + +export function LivekitJsCallSurface({ + room, + mediaReady, + initialMedia, + onHangup, +}: { + room: Room; + mediaReady: boolean; + initialMedia: LivekitJsCallMedia; + onHangup: () => void; +}) { + return ( + + + + ); +} diff --git a/src/app/features/call/NativeCallSurface.css.ts b/src/app/features/call/NativeCallSurface.css.ts new file mode 100644 index 0000000000..e1212f1dea --- /dev/null +++ b/src/app/features/call/NativeCallSurface.css.ts @@ -0,0 +1,171 @@ +import { style } from '@vanilla-extract/css'; +import { color, config, toRem } from 'folds'; + +export const StatusRow = style({ + flexShrink: 0, + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + padding: `calc(${config.space.S200} + env(safe-area-inset-top, 0px)) ${config.space.S300} ${config.space.S100}`, + textAlign: 'center', +}); + +export const DominantStage = style({ + flex: 1, + minHeight: 0, + minWidth: 0, + position: 'relative', + display: 'flex', + flexDirection: 'column', +}); + +export const DominantTile = style({ + flex: 1, + position: 'relative', + display: 'flex', + flexDirection: 'column', + minWidth: 0, + minHeight: 0, + overflow: 'hidden', + background: color.Background.Container, +}); + +export const FloatingLocal = style({ + position: 'absolute', + bottom: `calc(${config.space.S200} + env(safe-area-inset-bottom, 0px))`, + right: `calc(${config.space.S200} + env(safe-area-inset-right, 0px))`, + width: '120px', + aspectRatio: '3 / 4', + // TileSlot sizes itself as a flex child (`flex: 1`); without a flex parent it + // collapses to the placeholder badge and the native video lands in a strip at + // the top of the card instead of filling it. + display: 'flex', + zIndex: 3, + borderRadius: config.radii.R400, + overflow: 'hidden', + border: `1px solid ${color.Surface.ContainerLine}`, + background: color.Surface.Container, +}); + +export const TileGrid = style({ + flex: 1, + minHeight: 0, + minWidth: 0, + display: 'grid', + gap: config.space.S200, + padding: config.space.S200, + overflowY: 'auto', + selectors: { + '&[data-cols="2"]': { + gridTemplateColumns: 'repeat(2, 1fr)', + gridAutoRows: '1fr', + }, + '&[data-cols="3"]': { + gridTemplateColumns: 'repeat(3, 1fr)', + gridAutoRows: 'min-content', + }, + }, +}); + +export const Tile = style({ + position: 'relative', + display: 'flex', + flexDirection: 'column', + minWidth: 0, + minHeight: 0, + overflow: 'hidden', + borderRadius: config.radii.R400, + background: color.Surface.Container, + outline: `1px solid ${color.Surface.ContainerLine}`, + outlineOffset: '-1px', +}); + +export const TileFixed = style({ + aspectRatio: '3 / 4', + flexShrink: 0, +}); + +// The rect this element occupies is what JS reports to the native side; keep +// it free of labels so the native video never covers them. +export const TileSlot = style({ + position: 'relative', + flex: 1, + minHeight: 0, + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + justifyContent: 'center', + gap: config.space.S100, +}); + +export const InitialsBadge = style({ + display: 'grid', + placeItems: 'center', + width: 'min(72px, 45%)', + aspectRatio: '1 / 1', + borderRadius: '50%', + background: color.SurfaceVariant.Container, + color: color.SurfaceVariant.OnContainer, + fontSize: toRem(22), + fontWeight: 600, + lineHeight: 1, + userSelect: 'none', +}); + +export const TileLabel = style({ + flexShrink: 0, + display: 'flex', + alignItems: 'center', + gap: config.space.S100, + padding: `${config.space.S100} ${config.space.S200}`, + color: color.Surface.OnContainer, + fontSize: toRem(13), + lineHeight: toRem(18), + borderTop: `1px solid ${color.Surface.ContainerLine}`, +}); + +export const TileLabelName = style({ + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap', +}); + +export const QualityDot = style({ + width: '8px', + height: '8px', + borderRadius: '50%', + flexShrink: 0, + selectors: { + '&[data-quality="good"]': { background: color.Success.Main }, + '&[data-quality="poor"]': { background: color.Warning.Main }, + '&[data-quality="lost"]': { background: color.Critical.Main }, + '&[data-quality="excellent"]': { background: color.Success.Main }, + // Neutral rather than a token: it reads as "no signal yet" against the dark + // video stage, where a themed surface colour would look like a real state. + '&[data-quality="unknown"]': { background: color.Surface.ContainerLine }, + }, +}); + +export const HangupButton = style({ + display: 'inline-flex', + alignItems: 'center', + justifyContent: 'center', + width: toRem(44), + height: toRem(44), + padding: 0, + border: 'none', + borderRadius: '50%', + background: '#f54336', + color: '#ffffff', + cursor: 'pointer', + transition: 'filter 120ms ease', + selectors: { + '&:hover': { + filter: 'brightness(1.12)', + }, + '&:focus-visible': { + outline: `2px solid ${color.Primary.Main}`, + outlineOffset: '2px', + }, + }, +}); diff --git a/src/app/features/call/NativeCallSurface.tsx b/src/app/features/call/NativeCallSurface.tsx new file mode 100644 index 0000000000..4098c718bf --- /dev/null +++ b/src/app/features/call/NativeCallSurface.tsx @@ -0,0 +1,633 @@ +import { type ReactNode, useEffect, useMemo, useState } from 'react'; +import { Box, config, Menu, MenuItem, Text, toRem } from 'folds'; +import type { NativeCallSession } from '$state/nativeCall'; +import { + ArrowsClockwise, + Check, + MicrophoneSlash, + PhoneDisconnect, + SpeakerHigh, + User, + VideoCameraSlash, + sizedIcon, +} from '$components/icons/phosphor'; +import { ResponsiveMenu } from '$components/ResponsiveMenu'; +import { useMenuAnchor } from '$hooks/useMenuAnchor'; +import { + clearNativeCallLocalVideoOverlay, + clearNativeCallRemoteVideoOverlay, + setNativeCallLocalVideoOverlay, + setNativeCallRemoteVideoOverlay, + type NativeCallAudioRoute, +} from './livekitMobileBridge'; +import type { CallParticipant, CallTrack } from './callTransport'; +import { useCallMembers, useCallSession } from '$hooks/useCall'; +import { useRoom } from '$hooks/useRoom'; +import { useSelectedRoom } from '$hooks/router/useSelectedRoom'; +import { buildRtcIdentityMap, type UserIdByRtcIdentity } from './livekitCallIdentity'; +import { CallParticipantAvatar, useCallParticipantProfile } from './LivekitCallParticipant'; +import { CallControlBar, CallLayout, CallMediaControls, CallStatusBar } from './callChrome'; +import { controlButton } from './callChrome.css'; +import { nativeCallLifecycleLabels, nativeCallStatus } from './callClient'; +import * as css from './NativeCallSurface.css'; + +export type NativeCallSurfaceProps = { + session: NativeCallSession; + onHangup: () => void; +}; + +type OverlayTarget = { + participantIdentity: string; + trackId: string; +}; + +/** Detect overlays that do not change the slot's geometry. */ +function nativeSlotOccluded(slotNode: HTMLDivElement, rect: DOMRect): boolean { + const cx = rect.x + rect.width / 2; + const cy = rect.y + rect.height / 2; + if (cx < 0 || cy < 0 || cx > window.innerWidth || cy > window.innerHeight) return true; + const topElement = document.elementFromPoint(cx, cy); + return topElement === null || (topElement !== slotNode && !slotNode.contains(topElement)); +} + +/** Positions the native remote-video overlay over its DOM slot. */ +function useNativeVideoOverlay( + callId: string, + active: boolean, + target: OverlayTarget | undefined, + slotNode: HTMLDivElement | null +): void { + // Destructured to primitives: `target` is rebuilt on every transport update, + // so depending on the object would re-arm the observers and re-report an + // unchanged rect each time. + const participantIdentity = active ? target?.participantIdentity : undefined; + const trackId = active ? target?.trackId : undefined; + + useEffect(() => { + if (!participantIdentity || !trackId || !slotNode) return undefined; + + let lastGeometryKey = ''; + const report = () => { + const rect = slotNode.getBoundingClientRect(); + // Slot hidden, outside the viewport (display:none page, scrolled away), or + // occluded by an overlaying page/drawer: hide the native overlay too, + // otherwise it stays painted at a stale position over unrelated content. + const offscreen = + rect.width <= 0 || + rect.height <= 0 || + rect.right < 0 || + rect.bottom < 0 || + rect.left > window.innerWidth || + rect.top > window.innerHeight || + nativeSlotOccluded(slotNode, rect); + if (offscreen) { + if (lastGeometryKey !== '') { + lastGeometryKey = ''; + void clearNativeCallRemoteVideoOverlay({ callId }).catch(() => undefined); + } + return; + } + const geometryKey = `${Math.round(rect.x)},${Math.round(rect.y)},${Math.round(rect.width)},${Math.round(rect.height)}`; + if (geometryKey === lastGeometryKey) return; + lastGeometryKey = geometryKey; + void setNativeCallRemoteVideoOverlay({ + callId, + participantIdentity, + trackId, + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + devicePixelRatio: window.devicePixelRatio || 1, + }).catch(() => undefined); + }; + + report(); + const observer = new ResizeObserver(report); + observer.observe(slotNode); + window.addEventListener('resize', report); + // Scroll is the dominant rect-changing event: the message list (or any + // nested scroll container) moves the slot without resizing it. Capture + // phase reaches scrolls from nested containers, which don't bubble. + document.addEventListener('scroll', report, { capture: true, passive: true }); + return () => { + observer.disconnect(); + window.removeEventListener('resize', report); + document.removeEventListener('scroll', report, { capture: true }); + }; + }, [participantIdentity, trackId, slotNode, callId]); + + // No eligible target (camera muted/unpublished, reconnecting, error): the + // overlay must not linger over the tile grid. + useEffect(() => { + if (participantIdentity && trackId) return; + void clearNativeCallRemoteVideoOverlay({ callId }).catch(() => undefined); + }, [participantIdentity, trackId, callId]); + + useEffect( + () => () => { + // Unmount safety: a stale callId is a native no-op. + void clearNativeCallRemoteVideoOverlay({ callId }).catch(() => undefined); + }, + [callId] + ); +} + +/** Positions the native local-video overlay over its DOM slot. */ +function useNativeLocalVideoOverlay( + callId: string, + enabled: boolean, + cameraEnabled: boolean, + slotNode: HTMLDivElement | null +): void { + const active = enabled && cameraEnabled && slotNode; + + useEffect(() => { + if (!active || !slotNode) return undefined; + + let lastGeometryKey = ''; + const report = () => { + // Never tear down the preview while the page is hidden (e.g. PiP owns + // the layer during backgrounding). + if (document.visibilityState !== 'visible') return; + const rect = slotNode.getBoundingClientRect(); + // Slot hidden, outside the viewport (scrolled away), or occluded by an + // overlaying page/drawer: hide the native overlay too, otherwise it + // stays painted at a stale position over unrelated content. + const hidden = + rect.width <= 0 || + rect.height <= 0 || + rect.right < 0 || + rect.bottom < 0 || + rect.left > window.innerWidth || + rect.top > window.innerHeight || + nativeSlotOccluded(slotNode, rect); + if (hidden) { + if (lastGeometryKey !== '') { + lastGeometryKey = ''; + void clearNativeCallLocalVideoOverlay({ callId }).catch(() => undefined); + } + return; + } + const geometryKey = `${Math.round(rect.x)},${Math.round(rect.y)},${Math.round(rect.width)},${Math.round(rect.height)}`; + if (geometryKey === lastGeometryKey) return; + lastGeometryKey = geometryKey; + void setNativeCallLocalVideoOverlay({ + callId, + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + devicePixelRatio: window.devicePixelRatio || 1, + }).catch(() => undefined); + }; + + report(); + const observer = new ResizeObserver(report); + observer.observe(slotNode); + // Fires during slide transitions (transforms move the slot without any + // scroll/resize event), which is how recovery after a room change works. + const intersectionObserver = new IntersectionObserver(report, { + threshold: [0, 0.25, 0.5, 0.75, 1], + }); + intersectionObserver.observe(slotNode); + window.addEventListener('resize', report); + // Scroll is the dominant rect-changing event: any nested scroll container + // moves the slot without resizing it. Capture phase reaches scrolls from + // nested containers, which don't bubble. + document.addEventListener('scroll', report, { capture: true, passive: true }); + return () => { + observer.disconnect(); + intersectionObserver.disconnect(); + window.removeEventListener('resize', report); + document.removeEventListener('scroll', report, { capture: true }); + }; + }, [callId, slotNode, active]); + + useEffect(() => { + if (active) return; + void clearNativeCallLocalVideoOverlay({ callId }).catch(() => undefined); + }, [callId, active]); + + useEffect( + () => () => { + void clearNativeCallLocalVideoOverlay({ callId }).catch(() => undefined); + }, + [callId] + ); +} + +function LocalTile({ + session, + userIdByIdentity, + slotRef, + fixed, +}: { + session: NativeCallSession; + userIdByIdentity: UserIdByRtcIdentity; + slotRef?: (node: HTMLDivElement | null) => void; + fixed?: boolean; +}) { + const profile = useCallParticipantProfile('', true, userIdByIdentity); + + return ( +
+ {/* When the camera is on, the native local preview renders over this + slot; the avatar stays mounted underneath as the placeholder. */} +
+
+ +
+
+
+ {!session.microphoneEnabled && ( + + {sizedIcon(MicrophoneSlash, '200')} + + )} + You +
+
+ ); +} + +type RemoteTileProps = { + participant: CallParticipant; + userIdByIdentity: UserIdByRtcIdentity; + videoBound: boolean; + slotRef?: (node: HTMLDivElement | null) => void; + fixed?: boolean; +}; + +function RemoteTile({ + participant, + userIdByIdentity, + videoBound, + slotRef, + fixed, +}: RemoteTileProps) { + const profile = useCallParticipantProfile(participant.identity, false, userIdByIdentity); + return ( +
+ {/* When video is bound, the native view renders exactly over this slot; + the avatar stays mounted underneath as the pre-video placeholder. */} +
+
+ +
+
+
+ + {participant.camera?.muted && ( + + {sizedIcon(VideoCameraSlash, '200')} + + )} + {profile.name} +
+
+ ); +} + +/** Output picker, hidden where the platform has no routes. */ +function AudioRouteControl({ + session, + onMenuOpenChange, +}: { + session: NativeCallSession; + onMenuOpenChange: (open: boolean) => void; +}) { + const menu = useMenuAnchor(); + const [routes, setRoutes] = useState([]); + const open = menu.anchor !== undefined; + + useEffect(() => { + let disposed = false; + const load = async () => { + const next = await session.listAudioRoutes(); + if (!disposed) setRoutes(next); + }; + void load(); + return () => { + disposed = true; + }; + }, [session]); + + // The native video view sits above the webview, so it would paint over this + // menu. Report the open state so the surface can drop the overlay while it is + // up, the same way an occluding drawer does. + useEffect(() => { + onMenuOpenChange(open); + return () => onMenuOpenChange(false); + }, [open, onMenuOpenChange]); + + if (routes.length === 0) return null; + + return ( + + + {routes.map((route) => ( + + } + onClick={() => { + void session.selectAudioRoute(route.id); + menu.close(); + }} + > + {route.name} + + ))} + + + } + > + + + ); +} + +const qualityLabels: Record = { + excellent: 'Excellent connection', + good: 'Good connection', + poor: 'Poor connection', + lost: 'Connection lost', + unknown: 'Connection quality unknown', +}; + +/** The dot carries state through colour alone, so it needs its own label. */ +function QualityDot({ quality }: { quality: string | undefined }) { + const value = quality ?? 'unknown'; + return ( + + ); +} + +function RemoteDominantLabel({ + participant, + userIdByIdentity, +}: { + participant: CallParticipant; + userIdByIdentity: UserIdByRtcIdentity; +}) { + const profile = useCallParticipantProfile(participant.identity, false, userIdByIdentity); + return ( + <> + + {participant.camera?.muted && ( + + {sizedIcon(VideoCameraSlash, '200')} + + )} + {profile.name} + + ); +} + +function RemoteDominantPlaceholder({ + participant, + userIdByIdentity, +}: { + participant: CallParticipant; + userIdByIdentity: UserIdByRtcIdentity; +}) { + const profile = useCallParticipantProfile(participant.identity, false, userIdByIdentity, 192); + return ; +} + +/** Full-stage tile for a lone participant. */ +function DominantTile({ + slotRef, + videoBound, + placeholder, + label, +}: { + slotRef?: (node: HTMLDivElement | null) => void; + videoBound?: boolean; + placeholder: ReactNode; + label: ReactNode; +}) { + return ( +
+ {/* The slot's rect is what JS reports to the native video overlay; the + placeholder stays mounted underneath as the pre-video fallback. */} +
+
+ {placeholder} +
+
+
{label}
+
+ ); +} + +export function NativeCallSurface({ session, onHangup }: NativeCallSurfaceProps) { + const isError = session.lifecycle === 'error'; + const connected = session.lifecycle === 'connected'; + const remoteParticipants = session.participants; + const matrixRoom = useRoom(); + const callSession = useCallSession(matrixRoom); + const callMembers = useCallMembers(matrixRoom, callSession); + const userIdByIdentity = useMemo(() => buildRtcIdentityMap(callMembers), [callMembers]); + + const featured = useMemo(() => { + const live = (track: CallTrack | undefined): boolean => + track !== undefined && track.subscribed && !track.muted; + // A shared screen is the thing people are actually looking at, so it + // outranks any camera. + const sharing = remoteParticipants.find((p) => live(p.screenShare)); + if (sharing?.screenShare) { + return { participantIdentity: sharing.identity, trackId: sharing.screenShare.id }; + } + const onCamera = remoteParticipants.find((p) => live(p.camera)); + if (!onCamera?.camera) return undefined; + return { participantIdentity: onCamera.identity, trackId: onCamera.camera.id }; + }, [remoteParticipants]); + + // The room page stays mounted for the whole mobile slide-out transition, so + // geometry alone leaves the native video painted over the outgoing page until + // the slot finally clears the viewport. Route selection flips when the + // transition starts, which is the earliest honest "no longer on screen". + const selectedRoom = useSelectedRoom(); + const [routeMenuOpen, setRouteMenuOpen] = useState(false); + const overlayActive = connected && selectedRoom === session.roomId && !routeMenuOpen; + + const [slotNode, setSlotNode] = useState(null); + useNativeVideoOverlay(session.callId, overlayActive, featured, slotNode); + + const [localSlotNode, setLocalSlotNode] = useState(null); + useNativeLocalVideoOverlay(session.callId, overlayActive, session.cameraEnabled, localSlotNode); + + const remoteCount = remoteParticipants.length; + // Total tiles once the local self-tile joins the grid. 7+ switches the grid + // to its compact, scrollable three-column variant. + const tileCount = remoteCount + 1; + const compactGrid = tileCount > 6; + + // Two-person layout: the remote owns the stage and is video-bound here + // whenever its camera is the featured track. + const duoRemote = remoteCount === 1 ? remoteParticipants[0] : undefined; + const duoLive = + duoRemote !== undefined && connected && featured?.participantIdentity === duoRemote.identity; + + if (isError) { + return ( + + + + ); + } + + return ( + + {!connected && ( +
+ + {nativeCallLifecycleLabels[session.lifecycle]} + +
+ )} + {duoRemote ? ( +
+ + } + label={ + + } + /> + {session.cameraEnabled && ( +
+
+
+ {sizedIcon(User, '200')} +
+
+
+ )} +
+ ) : remoteCount === 0 ? ( +
+ + {!session.microphoneEnabled && ( + + {sizedIcon(MicrophoneSlash, '200')} + + )} + You + + } + /> +
+ ) : ( +
+ {remoteParticipants.map((participant) => { + const live = connected && featured?.participantIdentity === participant.identity; + return ( + + ); + })} + +
+ )} + + + {connected && } + {session.cameraEnabled && connected && ( + + )} + + + + ); +} diff --git a/src/app/features/call/PrescreenControls.tsx b/src/app/features/call/PrescreenControls.tsx index dd721c3cda..464fdf89cb 100644 --- a/src/app/features/call/PrescreenControls.tsx +++ b/src/app/features/call/PrescreenControls.tsx @@ -8,6 +8,9 @@ import { useIsDirectRoom, useRoom } from '../../hooks/useRoom'; import { useCallEmbed, useCallJoined, useCallStart } from '../../hooks/useCallEmbed'; import { useCallPreferences } from '../../state/hooks/callPreferences'; import { ScreenSize, useScreenSizeContext } from '../../hooks/useScreenSize'; +import { settingsAtom } from '$state/settings'; +import { useSetting } from '$state/hooks/settings'; +import { CallDevicePreview } from './CallDevicePreview'; type PrescreenControlsProps = { canJoin?: boolean; @@ -28,70 +31,98 @@ export function PrescreenControls({ canJoin }: PrescreenControlsProps) { const disabled = inOtherCall || !canJoin; - const { microphone, video, sound, toggleMicrophone, toggleVideo, toggleSound } = - useCallPreferences(); + const { + microphone, + video, + sound, + audioDeviceId, + videoDeviceId, + toggleMicrophone, + toggleVideo, + toggleSound, + setAudioDeviceId, + setVideoDeviceId, + } = useCallPreferences(); + // Only the new-call path applies the chosen devices; Element Call picks its + // own, so previewing there would promise something we cannot honour. + const [newCallsEnabled] = useSetting(settingsAtom, 'newCallsEnabled'); + const showPreview = newCallsEnabled && !disabled && !joining && !compact; return ( - - + {showPreview && ( + + )} + - - - - + + + {!compact && } + - {!compact && } + {!compact && } - - - {room?.isCallRoom() && } - - - - - - + + {room?.isCallRoom() && } + + + + + + + ); } diff --git a/src/app/features/call/callChrome.css.ts b/src/app/features/call/callChrome.css.ts new file mode 100644 index 0000000000..430d1ffffa --- /dev/null +++ b/src/app/features/call/callChrome.css.ts @@ -0,0 +1,82 @@ +import { style } from '@vanilla-extract/css'; +import { color, config, toRem } from 'folds'; + +export const callLayout = style({ + minHeight: 0, + overflow: 'hidden', + background: color.Background.Container, +}); + +export const controlBarOverlay = style({ + position: 'absolute', + left: 0, + right: 0, + bottom: 0, + zIndex: 4, + display: 'flex', + justifyContent: 'center', + padding: `${config.space.S200} ${config.space.S300} calc(${config.space.S300} + env(safe-area-inset-bottom, 0px))`, + transition: 'opacity 160ms ease, visibility 160ms ease', + pointerEvents: 'none', +}); + +export const controlBarFlow = style({ + flexShrink: 0, + display: 'flex', + justifyContent: 'center', + padding: `${config.space.S100} ${config.space.S300} calc(${config.space.S300} + env(safe-area-inset-bottom, 0px))`, +}); + +export const controlPill = style({ + display: 'flex', + alignItems: 'center', + // Wraps rather than overflowing: the parent clips, so an unwrapped row loses + // the End call button off the edge on a narrow viewport. + flexWrap: 'wrap', + justifyContent: 'center', + gap: config.space.S200, + maxWidth: '100%', + padding: config.space.S100, + border: `1px solid ${color.Surface.ContainerLine}`, + borderRadius: config.radii.R500, + background: color.Surface.Container, + backdropFilter: 'blur(12px)', +}); + +export const controlButton = style({ + display: 'inline-flex', + alignItems: 'center', + justifyContent: 'center', + minWidth: toRem(44), + minHeight: toRem(44), + padding: 0, + border: 'none', + borderRadius: '50%', + font: 'inherit', + cursor: 'pointer', + transition: 'background-color 120ms ease', + selectors: { + '&[data-on="true"]': { + background: color.SurfaceVariant.Container, + color: color.SurfaceVariant.OnContainer, + }, + '&[data-on="true"]:hover:not(:disabled)': { + background: color.SurfaceVariant.ContainerHover, + }, + '&[data-on="false"]': { + background: color.Critical.Container, + color: color.Critical.OnContainer, + }, + '&[data-on="false"]:hover:not(:disabled)': { + background: color.Critical.ContainerHover, + }, + '&:focus-visible': { + outline: `2px solid var(--sable-primary-main, #7aa2ff)`, + outlineOffset: '2px', + }, + '&:disabled': { + opacity: 0.45, + cursor: 'default', + }, + }, +}); diff --git a/src/app/features/call/callChrome.tsx b/src/app/features/call/callChrome.tsx new file mode 100644 index 0000000000..a08e6ccd8f --- /dev/null +++ b/src/app/features/call/callChrome.tsx @@ -0,0 +1,207 @@ +import type { + CSSProperties, + FocusEventHandler, + KeyboardEventHandler, + PointerEventHandler, + ReactNode, +} from 'react'; +import { Box, Button, color, Text } from 'folds'; +import { + Microphone, + MicrophoneSlash, + VideoCamera, + VideoCameraSlash, + sizedIcon, +} from '$components/icons/phosphor'; +import type { CallStatusView } from './callClient'; +import * as css from './callChrome.css'; + +/** Shared layout for web and native call surfaces. */ +export type CallLayoutProps = { + children: ReactNode; + stack?: boolean; + callSurfaceMarker?: boolean; + className?: string; + style?: CSSProperties; + onPointerMove?: PointerEventHandler; + onPointerDown?: PointerEventHandler; + onFocusCapture?: FocusEventHandler; + onKeyDown?: KeyboardEventHandler; +}; + +export function CallLayout({ + children, + stack, + callSurfaceMarker, + className, + style, + onPointerMove, + onPointerDown, + onFocusCapture, + onKeyDown, +}: CallLayoutProps) { + const mergedClassName = [css.callLayout, className].filter(Boolean).join(' ') || undefined; + return ( + + {children} + + ); +} + +/** Shared shell for the web overlay and native in-flow controls. */ +export type CallControlBarProps = { + children: ReactNode; + layout?: 'overlay' | 'flow'; + visible?: boolean; + onFocusCapture?: FocusEventHandler; +}; + +export function CallControlBar({ + children, + layout = 'flow', + visible, + onFocusCapture, +}: CallControlBarProps) { + if (layout === 'overlay') { + return ( + +
+ {children} +
+ + ); + } + + return ( + +
{children}
+ + ); +} + +/** Status display for call setup and failures. */ +export function CallStatusBar({ + status, + onHangup, +}: { + status: CallStatusView; + onHangup: () => void; +}) { + const failed = status.phase === 'failed'; + return ( + + + {status.statusLabel} + {status.error && ( + + {status.error} + + )} + + + + ); +} + +/** Shared mic and camera toggle. */ +export type CallMediaToggleButtonProps = { + on: boolean; + disabled?: boolean; + label: string; + children: ReactNode; + onToggle: () => void; +}; + +export function CallMediaToggleButton({ + on, + disabled, + label, + children, + onToggle, +}: CallMediaToggleButtonProps) { + return ( + + ); +} + +/** Native mic and camera controls. */ +export type CallMediaControlsProps = { + microphoneEnabled: boolean; + cameraEnabled: boolean; + setMicrophoneEnabled: (enabled: boolean) => Promise; + setCameraEnabled: (enabled: boolean) => Promise; + disabled?: boolean; +}; + +export function CallMediaControls({ + microphoneEnabled, + cameraEnabled, + setMicrophoneEnabled, + setCameraEnabled, + disabled, +}: CallMediaControlsProps) { + return ( + <> + void setMicrophoneEnabled(!microphoneEnabled)} + > + {sizedIcon(microphoneEnabled ? Microphone : MicrophoneSlash, '300', { + filled: !microphoneEnabled, + })} + + void setCameraEnabled(!cameraEnabled)} + > + {sizedIcon(cameraEnabled ? VideoCamera : VideoCameraSlash, '300', { + filled: cameraEnabled, + })} + + + ); +} diff --git a/src/app/features/call/callClient.ts b/src/app/features/call/callClient.ts new file mode 100644 index 0000000000..293839e320 --- /dev/null +++ b/src/app/features/call/callClient.ts @@ -0,0 +1,89 @@ +import type { + LivekitJsControllerFailure, + LivekitJsControllerLifecycle, +} from './livekitJsController'; +import type { NativeCallLifecycle } from '$state/nativeCall'; + +type CallPhase = 'idle' | 'connecting' | 'connected' | 'reconnecting' | 'stopping' | 'failed'; + +export type CallStatusView = { + phase: CallPhase; + statusLabel: string; + error?: string; +}; + +const livekitJsLifecycleLabels: Record = { + idle: 'Idle', + 'joining-matrix': 'Joining call', + provisioning: 'Preparing call', + 'connecting-livekit': 'Connecting', + active: 'Connected', + stopping: 'Ending call', + failed: 'Call failed', +}; + +const livekitJsFailureMessages: Record = { + 'e2ee-unsupported': 'Encrypted calls are not supported on this device.', + 'e2ee-import-failed': 'Could not set up call encryption.', + 'setup-failed': 'Could not connect to the call.', +}; + +export const nativeCallLifecycleLabels: Record = { + starting: 'Starting call', + connecting: 'Connecting', + connected: 'Connected', + reconnecting: 'Reconnecting', + error: 'Call failed', +}; + +function livekitJsPhase(lifecycle: LivekitJsControllerLifecycle): CallPhase { + switch (lifecycle) { + case 'idle': + return 'idle'; + case 'active': + return 'connected'; + case 'stopping': + return 'stopping'; + case 'failed': + return 'failed'; + default: + // joining-matrix, provisioning, connecting-livekit are all "connecting". + return 'connecting'; + } +} + +function nativePhase(lifecycle: NativeCallLifecycle): CallPhase { + switch (lifecycle) { + case 'connected': + return 'connected'; + case 'reconnecting': + return 'reconnecting'; + case 'error': + return 'failed'; + default: + // starting, connecting are both "connecting". + return 'connecting'; + } +} + +export function livekitJsCallStatus(session: { + lifecycle: LivekitJsControllerLifecycle; + failure: LivekitJsControllerFailure | null; +}): CallStatusView { + return { + phase: livekitJsPhase(session.lifecycle), + statusLabel: livekitJsLifecycleLabels[session.lifecycle], + error: session.failure ? livekitJsFailureMessages[session.failure] : undefined, + }; +} + +export function nativeCallStatus(session: { + lifecycle: NativeCallLifecycle; + error?: string; +}): CallStatusView { + return { + phase: nativePhase(session.lifecycle), + statusLabel: nativeCallLifecycleLabels[session.lifecycle], + error: session.error, + }; +} diff --git a/src/app/features/call/callStartCapabilities.ts b/src/app/features/call/callStartCapabilities.ts index 9de7772303..b3d30dce9d 100644 --- a/src/app/features/call/callStartCapabilities.ts +++ b/src/app/features/call/callStartCapabilities.ts @@ -1,6 +1,7 @@ -import type { Room } from '$types/matrix-sdk'; +import { EventType, type Room } from '$types/matrix-sdk'; -const CALL_MEMBER_EVENT_TYPE = 'org.matrix.msc3401.call.member'; +const CALL_MEMBER_EVENT_TYPE = EventType.RTCMembership; +const LEGACY_CALL_MEMBER_EVENT_TYPE = EventType.GroupCallMemberPrefix; export type CallStartBlocker = | 'missing_webrtc' @@ -45,7 +46,9 @@ export const evaluateCallStartCapabilities = ({ }: EvaluateCallStartCapabilitiesInput): CallStartCapabilities => { const blockers: CallStartBlocker[] = []; const hasCallMemberPermission = - room.currentState?.maySendStateEvent(CALL_MEMBER_EVENT_TYPE, myUserId) ?? false; + room.currentState?.maySendStateEvent(CALL_MEMBER_EVENT_TYPE, myUserId) || + room.currentState?.maySendStateEvent(LEGACY_CALL_MEMBER_EVENT_TYPE, myUserId) || + false; const inAnotherCall = !!activeCallRoomId && activeCallRoomId !== room.roomId; if (!rtcSupported) blockers.push('missing_webrtc'); diff --git a/src/app/features/call/callStartSelection.test.ts b/src/app/features/call/callStartSelection.test.ts new file mode 100644 index 0000000000..ec61222054 --- /dev/null +++ b/src/app/features/call/callStartSelection.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest'; +import { selectCallStartOwner } from './callStartSelection'; + +describe('selectCallStartOwner', () => { + it('selects the native transport when available, even over LiveKit JS', () => { + expect(selectCallStartOwner({ newCallsEnabled: true, nativeCallAvailable: true })).toBe( + 'livekit-mobile' + ); + }); + + it('retains LiveKit JS when the native transport is unavailable', () => { + expect(selectCallStartOwner({ newCallsEnabled: true, nativeCallAvailable: false })).toBe( + 'livekit-js' + ); + }); + + it('selects LiveKit JS when new calls are enabled', () => { + expect(selectCallStartOwner({ newCallsEnabled: true })).toBe('livekit-js'); + }); + + it('falls back to Element Call when new calls are disabled', () => { + expect(selectCallStartOwner({ newCallsEnabled: false })).toBe('element'); + }); + + it('keeps the Element fallback when new calls are disabled, even if native is available', () => { + expect(selectCallStartOwner({ newCallsEnabled: false, nativeCallAvailable: true })).toBe( + 'element' + ); + }); +}); diff --git a/src/app/features/call/callStartSelection.ts b/src/app/features/call/callStartSelection.ts new file mode 100644 index 0000000000..0f6951347f --- /dev/null +++ b/src/app/features/call/callStartSelection.ts @@ -0,0 +1,13 @@ +export type CallStartOwner = 'livekit-mobile' | 'livekit-js' | 'element'; + +export const selectCallStartOwner = ({ + newCallsEnabled, + nativeCallAvailable = false, +}: { + newCallsEnabled: boolean; + nativeCallAvailable?: boolean; +}): CallStartOwner => { + if (!newCallsEnabled) return 'element'; + if (nativeCallAvailable) return 'livekit-mobile'; + return 'livekit-js'; +}; diff --git a/src/app/features/call/livekitCallIdentity.test.ts b/src/app/features/call/livekitCallIdentity.test.ts new file mode 100644 index 0000000000..b5f66b00ba --- /dev/null +++ b/src/app/features/call/livekitCallIdentity.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; +import type { CallMembership } from '$types/matrix-sdk'; +import { buildRtcIdentityMap } from './livekitCallIdentity'; + +const membership = (rtcBackendIdentity: string, userId?: string, deviceId?: string) => + ({ + rtcBackendIdentity, + userId, + deviceId, + memberId: userId && deviceId ? `${userId}:${deviceId}` : rtcBackendIdentity, + }) as CallMembership; + +describe('buildRtcIdentityMap', () => { + it('maps each anonymised LiveKit identity back to its Matrix user', () => { + const identities = buildRtcIdentityMap([ + membership('sha256-alice', '@alice:example.org'), + membership('sha256-bob', '@bob:example.org'), + ]); + + expect(identities.get('sha256-alice')).toBe('@alice:example.org'); + expect(identities.get('sha256-bob')).toBe('@bob:example.org'); + }); + + it('skips memberships with no sender rather than mapping them to undefined', () => { + const identities = buildRtcIdentityMap([ + membership('sha256-alice', '@alice:example.org'), + membership('sha256-orphan'), + ]); + + expect(identities.has('sha256-orphan')).toBe(false); + expect(identities.size).toBe(1); + }); + + it('returns an empty map before any membership is known', () => { + expect(buildRtcIdentityMap([]).size).toBe(0); + }); + + it('also indexes the legacy user:device identity the SFU may report instead', () => { + const identities = buildRtcIdentityMap([ + membership('sha256-alice', '@alice:example.org', 'DEVICE'), + ]); + + expect(identities.get('sha256-alice')).toBe('@alice:example.org'); + expect(identities.get('@alice:example.org:DEVICE')).toBe('@alice:example.org'); + }); +}); diff --git a/src/app/features/call/livekitJsCallManager.test.tsx b/src/app/features/call/livekitJsCallManager.test.tsx new file mode 100644 index 0000000000..853eef834e --- /dev/null +++ b/src/app/features/call/livekitJsCallManager.test.tsx @@ -0,0 +1,313 @@ +import type { ReactNode } from 'react'; +import { useEffect } from 'react'; +import { act, render, waitFor } from '@testing-library/react'; +import { createStore, Provider as JotaiProvider } from 'jotai'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MatrixClient, Room } from '$types/matrix-sdk'; +import { MatrixClientProvider } from '$hooks/useMatrixClient'; +import { AutoDiscoveryInfoProvider } from '$hooks/useAutoDiscoveryInfo'; +import { CallEmbedRefContextProvider } from '$hooks/useCallEmbed'; +import { getSettings, settingsAtom, type Settings } from '$state/settings'; +import { livekitJsCallAtom } from '$state/livekitJsCall'; +import { nativeCallAtom, type NativeCallSession } from '$state/nativeCall'; +import { + type LivekitJsCallManager, + LivekitJsCallManagerProvider, + useLivekitJsCallManager, +} from './livekitJsCallManager'; +import type { LivekitJsControllerState } from './livekitJsController'; + +vi.mock('$hooks/useTheme', () => ({ + ThemeKind: { Light: 'light', Dark: 'dark' }, + useTheme: () => ({ kind: 'light' }), +})); + +vi.mock('$hooks/useClientConfig', () => ({ + useClientConfig: () => ({ elementCallUrl: undefined }), +})); + +const { createControllerMock } = vi.hoisted(() => ({ + createControllerMock: vi.fn<(...args: unknown[]) => unknown>(), +})); + +vi.mock('./livekitJsController', () => ({ + createLivekitJsController: createControllerMock, +})); + +type ControllerListener = (state: Readonly) => void; + +type FakeController = { + connect: ReturnType; + disconnect: ReturnType; + subscribe: (listener: ControllerListener) => () => void; + getState: () => LivekitJsControllerState; + emit: (state: Partial) => void; + listenerCount: () => number; +}; + +const makeFakeController = (): FakeController => { + const listeners = new Set(); + let state: LivekitJsControllerState = { + lifecycle: 'idle', + failure: null, + mediaReady: false, + e2ee: { + ready: false, + localOutboundIdentity: null, + keyIndex: null, + lastImportFailure: null, + }, + }; + const snapshot = (): LivekitJsControllerState => ({ + ...state, + e2ee: { ...state.e2ee }, + }); + return { + connect: vi.fn<(...args: unknown[]) => Promise>(async () => undefined), + disconnect: vi.fn<() => Promise>(async () => undefined), + getState: () => snapshot(), + subscribe: (listener) => { + listeners.add(listener); + listener(snapshot()); + return () => { + listeners.delete(listener); + }; + }, + emit: (next) => { + state = { ...state, ...next }; + listeners.forEach((listener) => listener(snapshot())); + }, + listenerCount: () => listeners.size, + }; +}; + +const room = { roomId: '!room:example.org' } as Room; + +const makeNativeSession = (lifecycle: NativeCallSession['lifecycle']): NativeCallSession => ({ + backend: 'livekit-mobile', + roomId: room.roomId, + callId: 'native-call-id', + lifecycle, + participants: [], + microphoneEnabled: true, + cameraEnabled: false, + setMicrophoneEnabled: async () => {}, + setCameraEnabled: async () => {}, + switchCamera: async () => {}, + listAudioRoutes: async () => [], + selectAudioRoute: async () => {}, + hangup: async () => undefined, +}); + +const memberships: string[] = []; +const mx = { + matrixRTC: { + getRoomSession: vi.fn<() => unknown>(() => ({ memberships })), + }, +} as unknown as MatrixClient; + +type Harness = { + store: ReturnType; + controllers: FakeController[]; + wrapper: ({ children }: { children?: ReactNode }) => ReactNode; + managers: Array; + Consumer: () => null; +}; + +const createHarness = (settingsOverrides: Partial = {}): Harness => { + const store = createStore(); + store.set(settingsAtom, { ...getSettings(), ...settingsOverrides }); + const controllers: FakeController[] = []; + createControllerMock.mockImplementation(() => { + const controller = makeFakeController(); + controllers.push(controller); + return controller; + }); + const managers: Array = []; + const Consumer = (): null => { + managers.push(useLivekitJsCallManager()); + return null; + }; + const embedRef = { current: null }; + const wrapper = ({ children }: { children?: ReactNode }) => ( + + + + + {children} + + + + + ); + return { store, controllers, wrapper, managers, Consumer }; +}; + +const currentManager = (harness: Harness): LivekitJsCallManager => { + const manager = harness.managers[harness.managers.length - 1]; + expect(manager).toBeDefined(); + return manager!; +}; + +describe('LivekitJsCallManagerProvider', () => { + beforeEach(() => { + createControllerMock.mockReset(); + }); + + it('start then consumer unmount/replacement does not disconnect the controller', () => { + const harness = createHarness(); + const view = render( + + + + ); + const controller = harness.controllers[0]!; + act(() => { + currentManager(harness).start({ room, video: true }); + }); + expect(controller.connect).toHaveBeenCalledTimes(1); + expect(controller.connect.mock.calls[0]![0]).toMatchObject({ + room, + callIntent: 'video', + }); + + // Consumer unmounts (prescreen replaced by call view) while provider persists. + view.rerender({undefined}); + + expect(controller.disconnect).not.toHaveBeenCalled(); + expect(controller.listenerCount()).toBe(1); + + // A replacement consumer mounts and still sees the same controller/manager. + view.rerender( + + + + ); + expect(controller.disconnect).not.toHaveBeenCalled(); + }); + + it('explicit end via session hangup disconnects exactly once', async () => { + const harness = createHarness(); + render( + + + + ); + const controller = harness.controllers[0]!; + act(() => { + currentManager(harness).start({ room, video: false }); + }); + act(() => { + controller.emit({ lifecycle: 'active', room: {} as never }); + }); + const session = harness.store.get(livekitJsCallAtom); + expect(session?.roomId).toBe(room.roomId); + expect(session?.lifecycle).toBe('active'); + + await act(async () => { + await session!.hangup(); + }); + + expect(controller.disconnect).toHaveBeenCalledTimes(1); + }); + + it('child passive effect (auto-join) can start on initial mount', () => { + const harness = createHarness({ newCallsEnabled: true }); + const AutoJoinChild = (): null => { + const manager = useLivekitJsCallManager(); + useEffect(() => { + manager?.start({ room, video: false }); + }, [manager]); + return null; + }; + + render( + + + + ); + + const controller = harness.controllers[0]!; + expect(controller.connect).toHaveBeenCalledTimes(1); + expect(controller.connect.mock.calls[0]![0]).toMatchObject({ + room, + callIntent: 'audio', + }); + }); + + it('provider unmount disconnects exactly once', async () => { + const harness = createHarness(); + const view = render({undefined}); + const controller = harness.controllers[0]!; + + view.unmount(); + + await waitFor(() => { + expect(controller.disconnect).toHaveBeenCalledTimes(1); + }); + expect(controller.listenerCount()).toBe(0); + expect(harness.store.get(livekitJsCallAtom)).toBeUndefined(); + }); + + it('multiple consumers share one controller for one call', () => { + const harness = createHarness({ newCallsEnabled: true }); + render( + + + + + + ); + const controller = harness.controllers[0]!; + + // Prescreen starts the call. + act(() => { + harness.managers[0]!.start({ room, video: false }); + }); + act(() => { + controller.emit({ lifecycle: 'joining-matrix' }); + }); + // Room header and auto-join attempt the same call while one is active. + act(() => { + harness.managers[1]!.start({ room, video: false }); + harness.managers[2]!.start({ room, video: false }); + }); + + expect(controller.connect).toHaveBeenCalledTimes(1); + }); + + it('refuses to start while a native call is active and publishes no failed session', () => { + const harness = createHarness(); + harness.store.set(nativeCallAtom, makeNativeSession('connected')); + render( + + + + ); + const controller = harness.controllers[0]!; + + act(() => { + currentManager(harness).start({ room, video: false }); + }); + + expect(controller.connect).not.toHaveBeenCalled(); + expect(harness.store.get(livekitJsCallAtom)).toBeUndefined(); + }); + + it('starts when the native session is in a terminal error state', () => { + const harness = createHarness(); + harness.store.set(nativeCallAtom, makeNativeSession('error')); + render( + + + + ); + const controller = harness.controllers[0]!; + + act(() => { + currentManager(harness).start({ room, video: false }); + }); + + expect(controller.connect).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/app/features/call/livekitJsCallManager.tsx b/src/app/features/call/livekitJsCallManager.tsx new file mode 100644 index 0000000000..e1a04e3346 --- /dev/null +++ b/src/app/features/call/livekitJsCallManager.tsx @@ -0,0 +1,148 @@ +import type { ReactNode } from 'react'; +import { createContext, useCallback, useContext, useEffect, useMemo, useRef } from 'react'; +import { useStore } from 'jotai'; +import type { Room } from '$types/matrix-sdk'; +import { createLivekitJsController } from './livekitJsController'; +import { + livekitJsCallAtom, + livekitJsCallInitialMediaAppliedAtom, + livekitJsCallSoundAtom, + type LivekitJsCallMedia, +} from '$state/livekitJsCall'; +import { callInProgressAtom } from '$state/nativeCall'; +import { isCallOngoing } from './callSession'; +import { useMatrixClient } from '$hooks/useMatrixClient'; +import { useAutoDiscoveryInfo } from '$hooks/useAutoDiscoveryInfo'; + +export type LivekitJsCallStartOptions = { + room: Room; + dm?: boolean; + video?: boolean; + microphone?: boolean; + sound?: boolean; + audioDeviceId?: string; + videoDeviceId?: string; +}; + +export type LivekitJsCallManager = { + start: (options: LivekitJsCallStartOptions) => void; +}; + +const LivekitJsCallManagerContext = createContext(undefined); + +export const useLivekitJsCallManager = (): LivekitJsCallManager | undefined => + useContext(LivekitJsCallManagerContext); + +type LivekitJsController = ReturnType; + +type LivekitJsCallManagerProviderProps = { + children?: ReactNode; +}; + +/** + * Provider-level owner of the single LiveKit JS call controller. Mounted with + * CallEmbedProvider so transient consumers (prescreen, room header, auto-join) + * never own or disconnect the controller themselves. Owns: creation, + * subscription -> livekitJsCallAtom, and provider-unmount disconnect. + */ +export function LivekitJsCallManagerProvider({ children }: LivekitJsCallManagerProviderProps) { + const mx = useMatrixClient(); + const discovery = useAutoDiscoveryInfo(); + const store = useStore(); + const roomIdRef = useRef(undefined); + const initialMediaRef = useRef({ + microphone: true, + camera: false, + sound: true, + }); + const controllerRef = useRef(undefined); + const generationRef = useRef(0); + + // Create and expose the controller synchronously during render so child + // passive effects (e.g. AutoJoinManager calling start()) always see the + // current controller before the provider's own subscription effect runs. + const controller = useMemo(() => { + generationRef.current += 1; + return createLivekitJsController(); + }, []); + const generation = generationRef.current; + if (controllerRef.current !== controller) controllerRef.current = controller; + + useEffect(() => { + controllerRef.current = controller; + const unsubscribe = controller.subscribe((controllerState) => { + const roomId = roomIdRef.current; + if (!roomId) return; + if (controllerState.lifecycle === 'idle') { + store.set(livekitJsCallAtom, undefined); + return; + } + store.set(livekitJsCallAtom, { + roomId, + initialMedia: initialMediaRef.current, + lifecycle: controllerState.lifecycle, + failure: controllerState.failure, + room: controllerState.lifecycle === 'active' ? controllerState.room : undefined, + mediaReady: controllerState.mediaReady, + hangup: () => controller.disconnect(), + }); + }); + return () => { + unsubscribe(); + roomIdRef.current = undefined; + if (controllerRef.current === controller) controllerRef.current = undefined; + void controller.disconnect().finally(() => { + // A stale disconnect completing after a deliberate replacement must + // not clear the newer controller's session from the atom. + if (generationRef.current === generation) { + store.set(livekitJsCallAtom, undefined); + } + }); + }; + }, [controller, generation, store]); + + const start = useCallback( + ({ + room, + dm, + video, + microphone, + sound, + audioDeviceId, + videoDeviceId, + }: LivekitJsCallStartOptions) => { + if (store.get(callInProgressAtom)) return; + const activeController = controllerRef.current; + if (!activeController) return; + roomIdRef.current = room.roomId; + initialMediaRef.current = { + microphone: microphone ?? true, + camera: video ?? false, + sound: sound ?? true, + audioDeviceId, + videoDeviceId, + }; + store.set(livekitJsCallSoundAtom, sound ?? true); + store.set(livekitJsCallInitialMediaAppliedAtom, false); + void activeController + .connect({ + mx, + room, + discovery, + callIntent: video ? 'video' : 'audio', + dm, + ongoing: isCallOngoing(mx, room), + }) + .catch(() => undefined); + }, + [mx, discovery, store] + ); + + const manager = useMemo(() => ({ start }), [start]); + + return ( + + {children} + + ); +} diff --git a/src/app/features/call/livekitJsController.test.ts b/src/app/features/call/livekitJsController.test.ts index b60044866b..6ba1cfbb94 100644 --- a/src/app/features/call/livekitJsController.test.ts +++ b/src/app/features/call/livekitJsController.test.ts @@ -111,8 +111,7 @@ type FakeProvider = { const makeProvider = (state: Partial = {}): FakeProvider => { const listeners = new Set<(state: Readonly) => void>(); const provider = { - emit: (next: LivekitMatrixKeyProviderState) => - listeners.forEach((listener) => listener(next)), + emit: (next: LivekitMatrixKeyProviderState) => listeners.forEach((listener) => listener(next)), state: { ready: false, localOutboundIdentity: null, diff --git a/src/app/features/call/matrixRtcCallLifecycle.ts b/src/app/features/call/matrixRtcCallLifecycle.ts index f9ade34257..e80c4915ac 100644 --- a/src/app/features/call/matrixRtcCallLifecycle.ts +++ b/src/app/features/call/matrixRtcCallLifecycle.ts @@ -85,7 +85,10 @@ const hydrateCallRoster = async (room: Room): Promise => { // the room is encrypted, so an unencrypted room can be answered from a stale // cache. Clearing it forces the next load to fetch. if (!rosterComplete(room) && !fromServer) { - debugLog.warn('call', `roster stale for ${room.roomId}, refetching: ${rosterState(room, false)}`); + debugLog.warn( + 'call', + `roster stale for ${room.roomId}, refetching: ${rosterState(room, false)}` + ); try { await room.clearLoadedMembersIfNeeded(); } catch (error) { @@ -96,7 +99,10 @@ const hydrateCallRoster = async (room: Room): Promise => { } if (!rosterComplete(room)) { - debugLog.error('call', `roster incomplete for ${room.roomId}: ${rosterState(room, fromServer)}`); + debugLog.error( + 'call', + `roster incomplete for ${room.roomId}: ${rosterState(room, fromServer)}` + ); throw new Error(ROSTER_HYDRATION_ERROR); } debugLog.info('call', `roster hydrated for ${room.roomId}: ${rosterState(room, fromServer)}`); diff --git a/src/app/features/call/nativeCallManager.test.ts b/src/app/features/call/nativeCallManager.test.ts new file mode 100644 index 0000000000..77c6c5fb2f --- /dev/null +++ b/src/app/features/call/nativeCallManager.test.ts @@ -0,0 +1,145 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { createStore } from 'jotai'; +import type { MatrixClient, Room } from '$types/matrix-sdk'; +import { createNativeCallManager, getNativeCallManager } from './nativeCallManager'; +import type { + NativeCallController, + NativeCallControllerDependencies, + NativeCallStartOptions, +} from './nativeCallController'; +import { nativeCallAtom, type NativeCallSession } from '$state/nativeCall'; +import { callEmbedAtom } from '$state/callEmbed'; +import type { CallEmbed } from '$plugins/call'; +import { resetCallOwnerForTests } from '$state/callOwner'; + +const room = { roomId: '!room:example.org' } as Room; + +const makeClient = (memberships: unknown[] = []) => + ({ + matrixRTC: { + getRoomSession: () => ({ memberships }), + }, + }) as unknown as MatrixClient; + +const makeNativeSession = (lifecycle: NativeCallSession['lifecycle']): NativeCallSession => ({ + backend: 'livekit-mobile', + roomId: room.roomId, + callId: 'call-id', + lifecycle, + participants: [], + microphoneEnabled: true, + cameraEnabled: false, + setMicrophoneEnabled: async () => {}, + setCameraEnabled: async () => {}, + switchCamera: async () => {}, + listAudioRoutes: async () => [], + selectAudioRoute: async () => {}, + hangup: async () => undefined, +}); + +type FakeController = { + controller: NativeCallController; + start: ReturnType Promise>>; + setSession?: (session: NativeCallSession | undefined) => void; +}; + +const makeFakeControllerFactory = () => { + const fake: FakeController = { + controller: undefined as unknown as NativeCallController, + start: vi.fn<(options: NativeCallStartOptions) => Promise>(async () => undefined), + }; + fake.controller = { start: fake.start } as unknown as NativeCallController; + const createController = ( + dependencies: Pick + ): NativeCallController => { + fake.setSession = dependencies.setSession; + return fake.controller; + }; + return { fake, createController }; +}; + +beforeEach(() => { + resetCallOwnerForTests(); +}); + +describe('createNativeCallManager', () => { + it('starts the controller with the computed call options', () => { + const store = createStore(); + const { fake, createController } = makeFakeControllerFactory(); + const manager = createNativeCallManager(store, createController); + + manager.start({ + mx: makeClient([{ userId: '@other:example.org' }]), + room, + dm: true, + video: true, + microphone: false, + }); + + expect(fake.start).toHaveBeenCalledWith({ + mx: expect.anything(), + room, + dm: true, + video: true, + microphone: false, + ongoing: true, + }); + }); + + it('defaults audio/direct-message flags', () => { + const store = createStore(); + const { fake, createController } = makeFakeControllerFactory(); + const manager = createNativeCallManager(store, createController); + + manager.start({ mx: makeClient(), room }); + + expect(fake.start).toHaveBeenCalledWith( + expect.objectContaining({ dm: false, video: false, microphone: true, ongoing: false }) + ); + }); + + it('publishes controller sessions into the native call atom', () => { + const store = createStore(); + const { fake, createController } = makeFakeControllerFactory(); + createNativeCallManager(store, createController); + + const session = makeNativeSession('connected'); + fake.setSession?.(session); + expect(store.get(nativeCallAtom)).toBe(session); + + fake.setSession?.(undefined); + expect(store.get(nativeCallAtom)).toBeUndefined(); + }); + + it('does not start while a native call session is active', () => { + const store = createStore(); + const { fake, createController } = makeFakeControllerFactory(); + const manager = createNativeCallManager(store, createController); + + store.set(nativeCallAtom, makeNativeSession('connected')); + manager.start({ mx: makeClient(), room }); + + expect(fake.start).not.toHaveBeenCalled(); + }); + + it('does not start while a failed native session is cleared but Element Call is active', () => { + const store = createStore(); + const { fake, createController } = makeFakeControllerFactory(); + const manager = createNativeCallManager(store, createController); + + store.set(callEmbedAtom, { + roomId: '!other:example.org', + dispose: vi.fn<() => void>(), + } as unknown as CallEmbed); + manager.start({ mx: makeClient(), room }); + + expect(fake.start).not.toHaveBeenCalled(); + }); +}); + +describe('getNativeCallManager', () => { + it('reuses one manager per store', () => { + const store = createStore(); + expect(getNativeCallManager(store)).toBe(getNativeCallManager(store)); + }); +}); diff --git a/src/app/features/call/nativeCallManager.ts b/src/app/features/call/nativeCallManager.ts new file mode 100644 index 0000000000..9260cc67dd --- /dev/null +++ b/src/app/features/call/nativeCallManager.ts @@ -0,0 +1,63 @@ +import type { useStore } from 'jotai'; + +type Store = ReturnType; +import type { MatrixClient, Room } from '$types/matrix-sdk'; +import type { AutoDiscoveryInfo } from '../../cs-api'; +import { callInProgressAtom, nativeCallAtom } from '$state/nativeCall'; +import { isCallOngoing } from './callSession'; +import { + createNativeCallController, + type NativeCallController, + type NativeCallControllerDependencies, +} from './nativeCallController'; + +export type NativeCallManagerStartOptions = { + mx: MatrixClient; + room: Room; + discovery?: Pick; + dm?: boolean; + video?: boolean; + microphone?: boolean; +}; + +export type NativeCallManager = { + start: (options: NativeCallManagerStartOptions) => void; +}; + +export const createNativeCallManager = ( + store: Pick, + createController: ( + dependencies: Pick + ) => NativeCallController = createNativeCallController +): NativeCallManager => { + const controller = createController({ + setSession: (session) => store.set(nativeCallAtom, session), + }); + + return { + start: ({ mx, room, discovery, dm, video, microphone }) => { + if (store.get(callInProgressAtom)) return; + void controller + .start({ + mx, + room, + discovery, + dm: dm ?? false, + video: video ?? false, + microphone: microphone ?? true, + ongoing: isCallOngoing(mx, room), + }) + .catch(() => undefined); + }, + }; +}; + +const managers = new WeakMap, NativeCallManager>(); + +export const getNativeCallManager = (store: Pick): NativeCallManager => { + const cached = managers.get(store); + if (cached) return cached; + const manager = createNativeCallManager(store); + managers.set(store, manager); + return manager; +}; diff --git a/src/app/features/call/useCallSignaling.ts b/src/app/features/call/useCallSignaling.ts index a652d21ab5..36dffec86a 100644 --- a/src/app/features/call/useCallSignaling.ts +++ b/src/app/features/call/useCallSignaling.ts @@ -1,7 +1,12 @@ import { useCallback, useEffect, useRef } from 'react'; import * as Sentry from '@sentry/react'; import { useAtomValue, useSetAtom, useStore } from 'jotai'; -import type { RoomEventHandlerMap, MatrixEvent, Room } from '$types/matrix-sdk'; +import { + EventType, + type RoomEventHandlerMap, + type MatrixEvent, + type Room, +} from '$types/matrix-sdk'; import { MatrixRTCSessionManagerEvents, RoomEvent } from '$types/matrix-sdk'; import { mDirectAtom } from '$state/mDirectList'; import { @@ -11,6 +16,8 @@ import { mutedCallRoomIdAtom, type IncomingCall, } from '$state/callEmbed'; +import { livekitJsCallAtom } from '$state/livekitJsCall'; +import { nativeCallAtom, selectActiveCallSessionIncludingNative } from '$state/nativeCall'; import { settingsAtom } from '$state/settings'; import { parseIncomingRtcNotification, @@ -46,12 +53,16 @@ import { createDebugLogger } from '$utils/debugLogger'; const debugLog = createDebugLogger('CallSignaling'); const canSenderStartCalls = (room: Room, senderId: string): boolean => - room.currentState?.maySendStateEvent('org.matrix.msc3401.call.member', senderId) ?? false; + room.currentState?.maySendStateEvent(EventType.RTCMembership, senderId) || + room.currentState?.maySendStateEvent(EventType.GroupCallMemberPrefix, senderId) || + false; export function useIncomingCallSignaling() { const mx = useMatrixClient(); const store = useStore(); const callEmbed = useAtomValue(callEmbedAtom); + const livekitJsCall = useAtomValue(livekitJsCallAtom); + const nativeCall = useAtomValue(nativeCallAtom); const mDirects = useAtomValue(mDirectAtom); const settings = useAtomValue(settingsAtom); const incomingCall = useAtomValue(incomingCallAtom); @@ -89,6 +100,8 @@ export function useIncomingCallSignaling() { type SignalingHandlerRefs = { callEmbed: typeof callEmbed; + livekitJsCall: typeof livekitJsCall; + nativeCall: typeof nativeCall; mDirects: typeof mDirects; outgoingRingbackAllowed: boolean; handleIncomingCall: (incoming: IncomingCall) => void; @@ -117,7 +130,7 @@ export function useIncomingCallSignaling() { hasCallBeenActiveRef.current = false; outgoingRingRoomIdRef.current = null; outgoingStartRef.current = null; - }, [callEmbed]); + }, [callEmbed?.roomId, livekitJsCall?.roomId, nativeCall?.roomId]); useEffect(() => { ringtoneManager.syncSources( @@ -147,7 +160,12 @@ export function useIncomingCallSignaling() { const handleOutgoingDecline = useCallback( (decline: OutgoingDeclineEvent) => { - if (!callEmbed || callEmbed.roomId !== decline.roomId) { + const activeCall = selectActiveCallSessionIncludingNative( + callEmbed, + livekitJsCall, + nativeCall + ); + if (!activeCall || activeCall.roomId !== decline.roomId) { return; } @@ -207,8 +225,10 @@ export function useIncomingCallSignaling() { Sentry.metrics.count('sable.call.outgoing.declined', 1); stopOutgoingRing(); - void callEmbed - .hangup() + const hangup = + selectActiveCallSessionIncludingNative(callEmbed, livekitJsCall, nativeCall)?.hangup() ?? + Promise.resolve(); + void hangup .catch((error) => { debugLog.warn('call', 'Failed to hang up after outgoing decline', { roomId: decline.roomId, @@ -219,12 +239,11 @@ export function useIncomingCallSignaling() { .finally(() => { window.setTimeout(() => { const activeEmbed = store.get(callEmbedAtom); - if (activeEmbed !== callEmbed) return; - setCallEmbed(undefined); + if (activeEmbed === callEmbed) setCallEmbed(undefined); }, OUTGOING_DECLINE_EMBED_CLEAR_MS); }); }, - [callEmbed, mDirects, mx, setCallEmbed, stopOutgoingRing, store] + [callEmbed, livekitJsCall, nativeCall, mDirects, mx, setCallEmbed, stopOutgoingRing, store] ); const callAudioAllowed = canPlayCallAudio({ @@ -283,6 +302,8 @@ export function useIncomingCallSignaling() { signalingHandlerRefs.current = { callEmbed, + livekitJsCall, + nativeCall, mDirects, outgoingRingbackAllowed, handleIncomingCall, @@ -421,7 +442,12 @@ export function useIncomingCallSignaling() { if (!senderId || !eventId) return; if (senderId === myUserId) { - if (type === RTC_NOTIFICATION_EVENT_TYPE && handlers().callEmbed?.roomId === room.roomId) { + const activeCall = selectActiveCallSessionIncludingNative( + handlers().callEmbed, + handlers().livekitJsCall, + handlers().nativeCall + ); + if (type === RTC_NOTIFICATION_EVENT_TYPE && activeCall?.roomId === room.roomId) { activeOutgoingNotificationIdRef.current = eventId; } return; @@ -436,8 +462,12 @@ export function useIncomingCallSignaling() { // Only inspect declines for the active outgoing call room. Cleartext declines are // cheap; encrypted events are decrypted only when they might be RTC declines. - const activeEmbed = handlers().callEmbed; - if (!activeEmbed || activeEmbed.roomId !== room.roomId) { + const activeCall = selectActiveCallSessionIncludingNative( + handlers().callEmbed, + handlers().livekitJsCall, + handlers().nativeCall + ); + if (!activeCall || activeCall.roomId !== room.roomId) { return; } if (event.isDecryptionFailure()) { @@ -496,7 +526,11 @@ export function useIncomingCallSignaling() { let outgoingRingTimeoutId: number | undefined; const evaluateOutgoingFallback = () => { - const activeCallRoomId = handlers().callEmbed?.roomId; + const activeCallRoomId = selectActiveCallSessionIncludingNative( + handlers().callEmbed, + handlers().livekitJsCall, + handlers().nativeCall + )?.roomId; const stop = () => { handlers().stopOutgoingRing(); diff --git a/src/app/features/call/useCallStartCapabilities.test.tsx b/src/app/features/call/useCallStartCapabilities.test.tsx new file mode 100644 index 0000000000..111b476646 --- /dev/null +++ b/src/app/features/call/useCallStartCapabilities.test.tsx @@ -0,0 +1,146 @@ +import type { ReactNode } from 'react'; +import { act, render } from '@testing-library/react'; +import { createStore, Provider as JotaiProvider } from 'jotai'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MatrixClient, Room } from '$types/matrix-sdk'; +import type { CallEmbed } from '$plugins/call'; +import { MatrixClientProvider } from '$hooks/useMatrixClient'; +import { AutoDiscoveryInfoProvider } from '$hooks/useAutoDiscoveryInfo'; +import { nativeCallAtom, type NativeCallSession } from '$state/nativeCall'; +import type { CallStartCapabilities } from './callStartCapabilities'; +import { useCallStartCapabilities } from './useCallStartCapabilities'; + +const { webRtcSupportedMock, useCallEmbedMock } = vi.hoisted(() => ({ + webRtcSupportedMock: vi.fn<() => boolean>(() => true), + useCallEmbedMock: vi.fn<() => CallEmbed | undefined>(() => undefined), +})); + +vi.mock('$utils/rtc', () => ({ + webRTCSupported: () => webRtcSupportedMock(), +})); + +vi.mock('$hooks/useCallEmbed', () => ({ + useCallEmbed: () => useCallEmbedMock(), +})); + +const room = { + roomId: '!room:example.org', + currentState: { + maySendStateEvent: () => true, + }, +} as unknown as Room; + +const mx = { + getSafeUserId: () => '@me:example.org', + on: vi.fn<(...args: unknown[]) => void>(), + removeListener: vi.fn<(...args: unknown[]) => void>(), + // This homeserver advertises no MSC4143 transports, so these cases exercise + // the `.well-known` fallback that the fixtures below configure. + _unstable_getRTCTransports: () => Promise.reject(new Error('M_NOT_FOUND')), +} as unknown as MatrixClient; + +const makeNativeSession = ( + roomId: string, + lifecycle: NativeCallSession['lifecycle'] +): NativeCallSession => ({ + backend: 'livekit-mobile', + roomId, + callId: 'call-id', + lifecycle, + participants: [], + microphoneEnabled: true, + cameraEnabled: false, + setMicrophoneEnabled: async () => {}, + setCameraEnabled: async () => {}, + switchCamera: async () => {}, + listAudioRoutes: async () => [], + selectAudioRoute: async () => {}, + hangup: async () => undefined, +}); + +type Harness = { + store: ReturnType; + wrapper: ({ children }: { children?: ReactNode }) => ReactNode; + latest: () => CallStartCapabilities; +}; + +const createHarness = (): Harness => { + const store = createStore(); + let current: CallStartCapabilities | undefined; + const Probe = (): null => { + current = useCallStartCapabilities(room); + return null; + }; + const wrapper = ({ children }: { children?: ReactNode }) => ( + + + + {children ?? } + + + + ); + const latest = (): CallStartCapabilities => { + expect(current).toBeDefined(); + return current!; + }; + return { store, wrapper, latest }; +}; + +describe('useCallStartCapabilities', () => { + beforeEach(() => { + webRtcSupportedMock.mockReset().mockReturnValue(true); + useCallEmbedMock.mockReset().mockReturnValue(undefined); + }); + + it('blocks starting in another room while a native call is active there', () => { + const harness = createHarness(); + harness.store.set(nativeCallAtom, makeNativeSession('!other:example.org', 'connected')); + + render({undefined}); + + const capabilities = harness.latest(); + expect(capabilities.inAnotherCall).toBe(true); + expect(capabilities.blockers).toContain('already_in_another_call'); + expect(capabilities.canStart).toBe(false); + }); + + it('stops treating a native session as active once it reaches the error state', () => { + const harness = createHarness(); + render({undefined}); + + act(() => { + harness.store.set(nativeCallAtom, makeNativeSession('!other:example.org', 'starting')); + }); + expect(harness.latest().inAnotherCall).toBe(true); + + act(() => { + harness.store.set(nativeCallAtom, makeNativeSession('!other:example.org', 'error')); + }); + const capabilities = harness.latest(); + expect(capabilities.inAnotherCall).toBe(false); + expect(capabilities.blockers).not.toContain('already_in_another_call'); + expect(capabilities.canStart).toBe(true); + }); + + it('still counts an active Element call embed as an active call', () => { + const harness = createHarness(); + useCallEmbedMock.mockReturnValue({ + roomId: '!other:example.org', + } as CallEmbed); + + render({undefined}); + + const capabilities = harness.latest(); + expect(capabilities.inAnotherCall).toBe(true); + expect(capabilities.blockers).toContain('already_in_another_call'); + }); +}); diff --git a/src/app/features/call/useCallStartCapabilities.ts b/src/app/features/call/useCallStartCapabilities.ts index bad149ca5f..beff9f078d 100644 --- a/src/app/features/call/useCallStartCapabilities.ts +++ b/src/app/features/call/useCallStartCapabilities.ts @@ -2,6 +2,9 @@ import { useCallback, useMemo } from 'react'; import type { Room } from '$types/matrix-sdk'; import { EventType } from '$types/matrix-sdk'; import { useCallEmbed } from '$hooks/useCallEmbed'; +import { useAtomValue } from 'jotai'; +import { isLivekitJsCallActive, livekitJsCallAtom } from '$state/livekitJsCall'; +import { isNativeCallActive, nativeCallAtom } from '$state/nativeCall'; import { useLivekitSupport } from '$hooks/useLivekitSupport'; import { useMatrixClient } from '$hooks/useMatrixClient'; import { useStateEventCallback } from '$hooks/useStateEventCallback'; @@ -15,10 +18,18 @@ import { export const useCallStartCapabilities = (room: Room): CallStartCapabilities => { const mx = useMatrixClient(); const callEmbed = useCallEmbed(); + const livekitJsCall = useAtomValue(livekitJsCallAtom); + const nativeCall = useAtomValue(nativeCallAtom); const livekitSupported = useLivekitSupport(); const rtcSupported = webRTCSupported(); const myUserId = mx.getSafeUserId(); const [updateCount, forceUpdate] = useForceUpdate(); + // Terminal sessions stay in their atom for display, but they are not active + // calls and must not block starting a later one. + const activeLivekitJsRoomId = isLivekitJsCallActive(livekitJsCall) + ? livekitJsCall?.roomId + : undefined; + const activeNativeCallRoomId = isNativeCallActive(nativeCall) ? nativeCall?.roomId : undefined; useStateEventCallback( mx, @@ -42,9 +53,18 @@ export const useCallStartCapabilities = (room: Room): CallStartCapabilities => { return evaluateCallStartCapabilities({ room, myUserId, - activeCallRoomId: callEmbed?.roomId, + activeCallRoomId: callEmbed?.roomId ?? activeLivekitJsRoomId ?? activeNativeCallRoomId, livekitSupported, rtcSupported, }); - }, [room, myUserId, callEmbed?.roomId, livekitSupported, rtcSupported, updateCount]); + }, [ + room, + myUserId, + callEmbed?.roomId, + activeLivekitJsRoomId, + activeNativeCallRoomId, + livekitSupported, + rtcSupported, + updateCount, + ]); }; diff --git a/src/app/features/settings/experimental/Experimental.test.tsx b/src/app/features/settings/experimental/Experimental.test.tsx new file mode 100644 index 0000000000..9a23131849 --- /dev/null +++ b/src/app/features/settings/experimental/Experimental.test.tsx @@ -0,0 +1,94 @@ +import type { ReactNode } from 'react'; +import type * as SettingsModule from '$state/settings'; +import { fireEvent, render, screen } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { Experimental } from './Experimental'; + +const { mockSetNewCallsEnabled, mockUseSetting } = vi.hoisted(() => ({ + mockSetNewCallsEnabled: vi.fn<(value: boolean) => void>(), + mockUseSetting: + vi.fn<(_atom: unknown, key: string) => readonly [boolean, (value: boolean) => void]>(), +})); + +vi.mock('$state/settings', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, settingsAtom: {} }; +}); + +vi.mock('$state/hooks/settings', () => ({ + useSetting: mockUseSetting, +})); + +vi.mock('$components/page', () => ({ + PageContent: ({ children }: { children: ReactNode }) =>
{children}
, + SettingsSectionPage: ({ children }: { children: ReactNode }) =>
{children}
, +})); + +vi.mock('$components/info-card', () => ({ + InfoCard: () =>
Experimental warning
, +})); + +vi.mock('$components/setting-tile', () => ({ + SettingToggle: ({ + title, + description, + focusId, + value, + onChange, + }: { + title: string; + description: ReactNode; + focusId: string; + value: boolean; + onChange: (value: boolean) => void; + }) => ( +
+ {title} + {description} + +
+ ), +})); + +vi.mock('./BandwithSavingEmojis', () => ({ BandwidthSavingEmojis: () => null })); +vi.mock('./MSC4268HistoryShare', () => ({ MSC4268HistoryShare: () => null })); +vi.mock('./MSC4274MediaGalleries', () => ({ MSC4274MediaGalleries: () => null })); + +vi.mock('folds', () => ({ + Box: ({ children }: { children: ReactNode }) =>
{children}
, + Scroll: ({ children }: { children: ReactNode }) =>
{children}
, + Text: ({ children }: { children: ReactNode }) => {children}, +})); + +beforeEach(() => { + mockSetNewCallsEnabled.mockReset(); + mockUseSetting.mockImplementation((_atom: unknown, key: string) => { + if (key === 'newCallsEnabled') return [false, mockSetNewCallsEnabled]; + return [false, vi.fn<() => void>()]; + }); +}); + +describe('Experimental new calls setting', () => { + it('shows one new calls toggle', () => { + render( {}} />); + + expect(screen.getByText('New calls')).toBeInTheDocument(); + expect(screen.getByText('Enable new calls')).toBeInTheDocument(); + expect( + screen.getByText( + 'Uses LiveKit JS on web and desktop, and native LiveKit on supported mobile devices. Element Call remains the fallback.' + ) + ).toBeInTheDocument(); + expect(screen.queryByText(/manual media|connection probe/i)).not.toBeInTheDocument(); + }); + + it('persists the opt-in through the settings hook', () => { + render( {}} />); + + fireEvent.click(screen.getByRole('button', { name: 'new-calls' })); + + expect(mockSetNewCallsEnabled).toHaveBeenCalledWith(true); + }); +}); diff --git a/src/app/features/settings/experimental/Experimental.tsx b/src/app/features/settings/experimental/Experimental.tsx index 5d38fb06b7..b89ec3c6e1 100644 --- a/src/app/features/settings/experimental/Experimental.tsx +++ b/src/app/features/settings/experimental/Experimental.tsx @@ -29,6 +29,23 @@ function PersonaToggle() { ); } +function NewCallsToggle() { + const [newCallsEnabled, setNewCallsEnabled] = useSetting(settingsAtom, 'newCallsEnabled'); + + return ( + + New calls + + + ); +} + type ExperimentalProps = { requestBack?: () => void; requestClose: () => void; @@ -56,6 +73,7 @@ export function Experimental({ requestBack, requestClose }: Readonly + diff --git a/src/app/features/settings/notifications/AllMessages.tsx b/src/app/features/settings/notifications/AllMessages.tsx index 66d4d4895f..87d5a699ec 100644 --- a/src/app/features/settings/notifications/AllMessages.tsx +++ b/src/app/features/settings/notifications/AllMessages.tsx @@ -25,7 +25,7 @@ const getAllMessageDefaultRule = ( conditions.push({ kind: ConditionKind.EventMatch, key: 'type', - pattern: encrypted ? EventType.RoomMessageEncrypted : EventType.RoomMessage, + pattern: encrypted ? 'm.room.encrypted' : 'm.room.message', }); return { diff --git a/src/app/features/settings/notifications/UnifiedPushNotifications.ts b/src/app/features/settings/notifications/UnifiedPushNotifications.ts index 31ed4dda7c..3375271e63 100644 --- a/src/app/features/settings/notifications/UnifiedPushNotifications.ts +++ b/src/app/features/settings/notifications/UnifiedPushNotifications.ts @@ -364,7 +364,7 @@ async function decryptPreviewFromPayload( const crypto = mx.getCrypto(); if (!crypto || !pushData.content) return undefined; const mEvent = new MatrixEvent({ - type: EventType.RoomMessageEncrypted, + type: 'm.room.encrypted', content: pushData.content, room_id: roomId, event_id: eventId, diff --git a/src/app/features/settings/settingsLink.ts b/src/app/features/settings/settingsLink.ts index 4d85fd0d89..25267cf039 100644 --- a/src/app/features/settings/settingsLink.ts +++ b/src/app/features/settings/settingsLink.ts @@ -241,6 +241,7 @@ export const settingsLinkFocusIdsBySection: Record = exe: async (payload) => { const target = payload.trim(); await mx.sendMessage(room.roomId, { - msgtype: MsgType.Emote, + msgtype: 'm.emote', 'm.mentions': { user_ids: target ? [target] : [], }, diff --git a/src/app/hooks/commands/misc.ts b/src/app/hooks/commands/misc.ts index 23da23bbe4..83952551ae 100644 --- a/src/app/hooks/commands/misc.ts +++ b/src/app/hooks/commands/misc.ts @@ -1,5 +1,5 @@ import type { RoomMessageEventContent } from '$types/matrix-sdk'; -import { MatrixError, MsgType } from '$types/matrix-sdk'; +import { MatrixError } from '$types/matrix-sdk'; import { sendFeedback } from '$utils/sendFeedbackToUser'; import { CustomStateEvent } from '$types/matrix/room'; import { ErrorCode } from '../../cs-errorcode'; @@ -140,7 +140,7 @@ export const createMiscCommands = (ctx: CommandContext): Partial if (mlat && mlon) await mx.sendMessage(room.roomId, { - msgtype: MsgType.Location, + msgtype: 'm.location', geo_uri: `geo:${mlat},${mlon};u=0`, body: `https://www.openstreetmap.org/?mlat=${mlat}&mlon=${mlon}#map=16/${mlat}/${mlon}"`, } as RoomMessageEventContent); @@ -176,7 +176,7 @@ export const createMiscCommands = (ctx: CommandContext): Partial return; } mx.sendMessage(room.roomId, { - msgtype: MsgType.Location, + msgtype: 'm.location', geo_uri: `geo:${mlat},${mlon}${malt ? `,${malt}` : ''};u=0`, body: `https://www.openstreetmap.org/?mlat=${mlat}&mlon=${mlon}#map=16/${mlat}/${mlon}"`, } as unknown as RoomMessageEventContent); diff --git a/src/app/hooks/timeline/useProcessedTimeline.test.tsx b/src/app/hooks/timeline/useProcessedTimeline.test.tsx index aa636988a6..18e6545c93 100644 --- a/src/app/hooks/timeline/useProcessedTimeline.test.tsx +++ b/src/app/hooks/timeline/useProcessedTimeline.test.tsx @@ -164,22 +164,6 @@ const dividerIds = (processed: ProcessedEvent[]) => processed.filter((e) => e.willRenderNewDivider).map((e) => e.id); describe('useProcessedTimeline new-messages divider', () => { - it('renders an event that is still encrypted', () => { - const processed = processTimeline( - [ - createEvent({ id: '$a' }), - createEvent({ - id: '$encrypted', - type: EventType.RoomMessageEncrypted as string, - content: { algorithm: 'm.megolm.v1.aes-sha2', ciphertext: 'AwgAEnB...' }, - }), - ], - undefined - ); - - expect(renderedIds(processed)).toEqual(['$a', '$encrypted']); - }); - it('keeps poll start events with default hidden-event settings', () => { const processed = processTimeline( [ diff --git a/src/app/hooks/timeline/useProcessedTimeline.ts b/src/app/hooks/timeline/useProcessedTimeline.ts index 8b8689e47f..fcc91e1bb2 100644 --- a/src/app/hooks/timeline/useProcessedTimeline.ts +++ b/src/app/hooks/timeline/useProcessedTimeline.ts @@ -69,27 +69,27 @@ export function getProcessedRowIndexForRawTimelineIndex( return bestRowIndex >= 0 ? { rowIndex: bestRowIndex, focusRawIndex: bestRawIndex } : undefined; } -const MESSAGE_EVENT_TYPES = new Set([ - EventType.RoomMessage, - EventType.Sticker, - EventType.RoomMessageEncrypted, +const MESSAGE_EVENT_TYPES = new Set([ + 'm.room.message', + 'm.room.message.encrypted', + 'm.sticker', + 'm.room.encrypted', ]); -export const STANDARD_RENDERED_EVENT_TYPES = new Set([ - EventType.RoomMessage, - // getType() reports this until decryption resolves the real type. - EventType.RoomMessageEncrypted, - EventType.Sticker, +export const STANDARD_RENDERED_EVENT_TYPES = new Set([ + 'm.room.message', + 'm.room.message.encrypted', + 'm.sticker', M_POLL_START.name, - EventType.RoomMember, - EventType.RoomName, - EventType.RoomTopic, - EventType.RoomAvatar, - EventType.GroupCallMemberPrefix, + 'm.room.member', + 'm.room.name', + 'm.room.topic', + 'm.room.avatar', + 'org.matrix.msc3401.call.member', ]); const normalizeMessageType = (t: string): string => - t === (EventType.RoomMessageEncrypted as string) ? EventType.RoomMessage : t; + t === 'm.room.encrypted' || t === 'm.room.message.encrypted' ? 'm.room.message' : t; const isMessageRow = (mEvent: MatrixEvent): boolean => MESSAGE_EVENT_TYPES.has(mEvent.getType()) && !isEditEvent(mEvent); diff --git a/src/app/hooks/timeline/useTimelineActions.ts b/src/app/hooks/timeline/useTimelineActions.ts index b68a71f71f..aee5028b49 100644 --- a/src/app/hooks/timeline/useTimelineActions.ts +++ b/src/app/hooks/timeline/useTimelineActions.ts @@ -2,7 +2,7 @@ import type { MouseEventHandler } from 'react'; import { useCallback } from 'react'; import type { MatrixClient, Room, MatrixEvent } from '$types/matrix-sdk'; import type { UserProfile } from '$hooks/useUserProfile'; -import { EventStatus, RelationType } from '$types/matrix-sdk'; +import { EventStatus } from '$types/matrix-sdk'; import type { Editor } from 'slate'; import { ReactEditor } from 'slate-react'; @@ -154,7 +154,7 @@ export function useTimelineActions({ userId: mx.getUserId() ?? '', eventId: threadRootId, body: '', - relation: { rel_type: RelationType.Thread, event_id: threadRootId }, + relation: { rel_type: 'm.thread', event_id: threadRootId }, } : undefined ); @@ -165,7 +165,7 @@ export function useTimelineActions({ const { body, formattedBody } = extractReplyDraftBody(replyEvt, timelineSet); const { 'm.relates_to': relation } = startThread - ? { 'm.relates_to': { rel_type: RelationType.Thread, event_id: draftEventId } } + ? { 'm.relates_to': { rel_type: 'm.thread', event_id: draftEventId } } : replyEvt.getWireContent(); const senderId = replyEvt.getSender(); diff --git a/src/app/hooks/timeline/useTimelineRendererContext.ts b/src/app/hooks/timeline/useTimelineRendererContext.ts index 9b87ec8926..ce47d892d9 100644 --- a/src/app/hooks/timeline/useTimelineRendererContext.ts +++ b/src/app/hooks/timeline/useTimelineRendererContext.ts @@ -1,6 +1,5 @@ import { useMemo } from 'react'; import type { Room } from '$types/matrix-sdk'; -import { EventType } from '$types/matrix-sdk'; import type { HTMLReactParserOptions } from 'html-react-parser'; import type { Opts as LinkifyOpts } from 'linkifyjs'; import { @@ -110,9 +109,9 @@ export function useTimelineRendererContext(room: Room): TimelineRendererContextV const creators = useRoomCreators(room); const permissions = useRoomPermissions(creators, powerLevels); const canRedact = permissions.action('redact', mx.getSafeUserId()); - const canDeleteOwn = permissions.event(EventType.RoomRedaction, mx.getSafeUserId()); - const canSendReaction = permissions.event(EventType.Reaction, mx.getSafeUserId()); - const canPinEvent = permissions.stateEvent(EventType.RoomPinnedEvents, mx.getSafeUserId()); + const canDeleteOwn = permissions.event('m.room.redaction', mx.getSafeUserId()); + const canSendReaction = permissions.event('m.reaction', mx.getSafeUserId()); + const canPinEvent = permissions.stateEvent('m.room.pinned_events', mx.getSafeUserId()); const isReadOnly = !permissions.message(room.hasEncryptionStateEvent(), mx.getSafeUserId()); const getMemberPowerTag = useGetMemberPowerTag(room, creators, powerLevels); const parseMemberEvent = useMemberEventParser(); diff --git a/src/app/hooks/timeline/useTimelineSync.test.tsx b/src/app/hooks/timeline/useTimelineSync.test.tsx index 664aabbad7..aa0b29033a 100644 --- a/src/app/hooks/timeline/useTimelineSync.test.tsx +++ b/src/app/hooks/timeline/useTimelineSync.test.tsx @@ -238,7 +238,7 @@ describe('useTimelineSync', () => { renderHook(() => useTimelineSync({ room: room as Room, - mx: makeMx(), + mx: { getUserId: () => '@alice:test' } as never, isAtBottom: false, isAtBottomRef: { current: false }, scrollToBottom, @@ -270,7 +270,7 @@ describe('useTimelineSync', () => { renderHook(() => useTimelineSync({ room: room as Room, - mx: makeMx(), + mx: { getUserId: () => '@alice:test' } as never, isAtBottom: true, isAtBottomRef: { current: true }, scrollToBottom, @@ -298,7 +298,7 @@ describe('useTimelineSync', () => { ({ room, eventId }) => useTimelineSync({ room, - mx: makeMx(), + mx: { getUserId: () => '@alice:test' } as never, eventId, isAtBottom: false, isAtBottomRef: { current: false }, @@ -335,7 +335,7 @@ describe('useTimelineSync', () => { ({ room, eventId }) => useTimelineSync({ room, - mx: makeMx(), + mx: { getUserId: () => '@alice:test' } as never, eventId, isAtBottom: false, isAtBottomRef: { current: false }, @@ -370,7 +370,7 @@ describe('useTimelineSync', () => { ({ room }) => useTimelineSync({ room, - mx: makeMx(), + mx: { getUserId: () => '@alice:test' } as never, eventId: undefined, isAtBottom: false, isAtBottomRef: { current: false }, @@ -404,7 +404,7 @@ describe('useTimelineSync', () => { renderHook(() => useTimelineSync({ room: room as Room, - mx: makeMx(), + mx: { getUserId: () => '@alice:test' } as never, isAtBottom: true, isAtBottomRef: { current: true }, scrollToBottom, @@ -431,7 +431,7 @@ describe('useTimelineSync', () => { renderHook(() => useTimelineSync({ room: room as Room, - mx: makeMx(), + mx: { getUserId: () => '@alice:test' } as never, isAtBottom: true, isAtBottomRef: { current: true }, scrollToBottom, @@ -461,7 +461,7 @@ describe('useTimelineSync', () => { renderHook(() => useTimelineSync({ room: room as Room, - mx: makeMx(), + mx: { getUserId: () => '@alice:test' } as never, isAtBottom: true, isAtBottomRef: { current: true }, scrollToBottom, @@ -489,7 +489,7 @@ describe('useTimelineSync', () => { renderHook(() => useTimelineSync({ room: room as Room, - mx: makeMx(), + mx: { getUserId: () => '@alice:test' } as never, isAtBottom: false, isAtBottomRef: { current: false }, scrollToBottom, @@ -515,7 +515,7 @@ describe('useTimelineSync', () => { renderHook(() => useTimelineSync({ room: room as Room, - mx: makeMx(), + mx: { getUserId: () => '@alice:test' } as never, isAtBottom: true, isAtBottomRef: { current: true }, scrollToBottom, @@ -545,7 +545,7 @@ describe('useTimelineSync', () => { renderHook(() => useTimelineSync({ room: room as Room, - mx: makeMx(), + mx: { getUserId: () => '@alice:test' } as never, isAtBottom: true, isAtBottomRef: { current: true }, scrollToBottom, @@ -581,7 +581,7 @@ const syncOpts = ( isEventVisible?: () => boolean ) => ({ room: room as Room, - mx: makeMx({ paginateEventTimeline }), + mx: { getUserId: () => '@alice:test', paginateEventTimeline } as never, isAtBottom: true, isAtBottomRef: { current: true }, scrollToBottom: vi.fn<() => void>(), @@ -956,7 +956,7 @@ describe('live-arrive edge cases', () => { renderHook(() => useTimelineSync({ room: room as Room, - mx: makeMx(), + mx: { getUserId: () => '@alice:test' } as never, isAtBottom: false, isAtBottomRef: { current: false }, scrollToBottom, @@ -980,32 +980,6 @@ describe('live-arrive edge cases', () => { expect(scrollToBottom).not.toHaveBeenCalled(); }); - it('re-renders when an event finishes decrypting', async () => { - const { room } = createRoom(); - const { result } = renderSyncHook(room); - const before = result.current.timeline; - - await act(async () => { - mxEmitter.emit(MatrixEventEvent.Decrypted, { getRoomId: () => room.roomId }); - await Promise.resolve(); - }); - - expect(result.current.timeline).not.toBe(before); - }); - - it('ignores decryption of an event in another room', async () => { - const { room } = createRoom(); - const { result } = renderSyncHook(room); - const before = result.current.timeline; - - await act(async () => { - mxEmitter.emit(MatrixEventEvent.Decrypted, { getRoomId: () => '!other:test' }); - await Promise.resolve(); - }); - - expect(result.current.timeline).toBe(before); - }); - it('re-renders when a late local echo updates (slow send acknowledgement)', async () => { const { room } = createRoom(); const { result } = renderSyncHook(room); @@ -1051,11 +1025,12 @@ describe('event jump recovery', () => { targetTimeline, roomInitialSync, getLatestTimeline, - mx: makeMx({ + mx: { + getUserId: () => '@alice:test', roomInitialSync, getLatestTimeline, getEventTimeline: vi.fn<() => Promise>(() => Promise.resolve(targetTimeline)), - }), + }, }; }; @@ -1101,12 +1076,13 @@ describe('event jump recovery', () => { vi.useFakeTimers(); try { const { room } = createRoom(); - const mx = makeMx({ + const mx = { + getUserId: () => '@alice:test', roomInitialSync: vi.fn<() => Promise>(() => Promise.resolve(undefined)), getLatestTimeline: vi.fn<() => Promise>(() => Promise.resolve(undefined)), // The homeserver never answers /context: hit the 12 s timeout. getEventTimeline: () => new Promise(() => {}), - }); + }; const { result, scrollToBottom } = renderSyncHook(room, { isAtBottom: false, mx }); await act(async () => { @@ -1132,7 +1108,7 @@ describe('sliding sync chain relink', () => { undefined; const paginateEventTimeline = vi.fn<() => Promise>(() => Promise.resolve(false)); const { result } = renderSyncHook(room, { - mx: makeMx({ paginateEventTimeline }), + mx: { getUserId: () => '@alice:test', paginateEventTimeline }, }); expect(result.current.eventsLength).toBe(1); @@ -1183,7 +1159,7 @@ describe('sync transport fuzz', () => { const { result, unmount } = renderHook(() => useTimelineSync({ room: room as Room, - mx: makeMx({ paginateEventTimeline }), + mx: { getUserId: () => '@alice:test', paginateEventTimeline } as never, isAtBottom: false, isAtBottomRef: { current: false }, scrollToBottom, diff --git a/src/app/hooks/timeline/useTimelineSync.ts b/src/app/hooks/timeline/useTimelineSync.ts index b7f1475a93..190a8b8478 100644 --- a/src/app/hooks/timeline/useTimelineSync.ts +++ b/src/app/hooks/timeline/useTimelineSync.ts @@ -580,16 +580,6 @@ export function useTimelineSync({ useMatrixEvent(room, RoomEvent.LocalEchoUpdated, handleLocalEchoUpdated); - const handleDecrypted = useCallback( - (mEvent: MatrixEvent) => { - if (mEvent.getRoomId() !== room.roomId) return; - setTimeline((ct) => ({ ...ct })); - }, - [room, setTimeline] - ); - - useMatrixEvent(mx, MatrixEventEvent.Decrypted, handleDecrypted); - useLiveTimelineRefresh( room, useCallback(() => { diff --git a/src/app/hooks/useAutoJoinCall.ts b/src/app/hooks/useAutoJoinCall.ts index 0f95d94855..c4804cb966 100644 --- a/src/app/hooks/useAutoJoinCall.ts +++ b/src/app/hooks/useAutoJoinCall.ts @@ -26,6 +26,8 @@ export function useAutoJoinCall() { microphone: callPreferences.microphone, video: autoJoinIntent.video, sound: callPreferences.sound, + audioDeviceId: callPreferences.audioDeviceId, + videoDeviceId: callPreferences.videoDeviceId, }); setAutoJoinIntent(null); } @@ -38,6 +40,8 @@ export function useAutoJoinCall() { mDirects, callPreferences.microphone, callPreferences.sound, + callPreferences.audioDeviceId, + callPreferences.videoDeviceId, startDirectCall, startRoomCall, ]); diff --git a/src/app/hooks/useCallEmbed.ts b/src/app/hooks/useCallEmbed.ts index c55cfe8319..a3a4721f56 100644 --- a/src/app/hooks/useCallEmbed.ts +++ b/src/app/hooks/useCallEmbed.ts @@ -1,7 +1,7 @@ import type { RefObject } from 'react'; import { createContext, useCallback, useContext, useEffect, useState } from 'react'; import type { MatrixClient, Room } from '$types/matrix-sdk'; -import { useSetAtom } from 'jotai'; +import { useSetAtom, useStore } from 'jotai'; import * as Sentry from '@sentry/react'; import type { ElementCallThemeKind } from '../plugins/call'; import { CallEmbed, ElementWidgetActions, useClientWidgetApiEvent } from '../plugins/call'; @@ -15,6 +15,14 @@ import type { CallPreferences } from '../state/callPreferences'; import { createDebugLogger } from '../utils/debugLogger'; import { useClientConfig } from './useClientConfig'; import { callEmbedStartErrorAtom } from '$state/callEmbed'; +import { settingsAtom } from '$state/settings'; +import { useSetting } from '$state/hooks/settings'; +import { acquireCallOwner } from '$state/callOwner'; +import { selectCallStartOwner } from '$features/call/callStartSelection'; +import { useLivekitJsCallManager } from '$features/call/livekitJsCallManager'; +import { getNativeCallAvailability } from '$features/call/nativeCallProbe'; +import { getNativeCallManager } from '$features/call/nativeCallManager'; +import { useAutoDiscoveryInfo } from './useAutoDiscoveryInfo'; const debugLog = createDebugLogger('useCallEmbed'); @@ -66,12 +74,54 @@ export const useCallStart = (dm = false) => { const setCallEmbed = useSetAtom(callEmbedAtom); const setCallEmbedStartError = useSetAtom(callEmbedStartErrorAtom); const callEmbedRef = useCallEmbedRef(); + const store = useStore(); + const discovery = useAutoDiscoveryInfo(); + const [newCallsEnabled] = useSetting(settingsAtom, 'newCallsEnabled'); + const livekitJsCallManager = useLivekitJsCallManager(); const startCall = useCallback( (room: Room, pref?: CallPreferences) => { + if (newCallsEnabled) { + if (!livekitJsCallManager) { + throw new Error('LiveKit JS call manager is not provided!'); + } + // Resolved rather than cached in state so the first tap cannot race the + // native capability probe and fall through to the JS backend. + void getNativeCallAvailability(true).then((nativeCallAvailable) => { + if (selectCallStartOwner({ newCallsEnabled, nativeCallAvailable }) === 'livekit-mobile') { + getNativeCallManager(store).start({ + mx, + room, + discovery, + dm, + video: pref?.video, + microphone: pref?.microphone, + }); + return; + } + livekitJsCallManager.start({ + room, + dm, + video: pref?.video, + microphone: pref?.microphone, + sound: pref?.sound, + audioDeviceId: pref?.audioDeviceId, + videoDeviceId: pref?.videoDeviceId, + }); + }); + return; + } + const ownerLease = acquireCallOwner('element', room.roomId); + if (!ownerLease) { + debugLog.warn('call', 'Failed to start call: another call is already active', { + roomId: room.roomId, + }); + return; + } const container = callEmbedRef.current; if (!container) { - debugLog.error('call', 'Failed to start call — no embed container', { + ownerLease.release(); + debugLog.error('call', 'Failed to start call: no embed container', { roomId: room.roomId, }); Sentry.metrics.count('sable.call.start.error', 1, { @@ -96,6 +146,7 @@ export const useCallStart = (dm = false) => { ); setCallEmbed(callEmbed); } catch (err) { + ownerLease.release(); debugLog.error('call', 'Call embed creation failed', { roomId: room.roomId, error: err instanceof Error ? err.message : String(err), @@ -106,7 +157,19 @@ export const useCallStart = (dm = false) => { throw err; } }, - [mx, dm, theme, setCallEmbed, callEmbedRef, clientConfig.elementCallUrl, setCallEmbedStartError] + [ + mx, + dm, + theme, + setCallEmbed, + callEmbedRef, + store, + discovery, + clientConfig.elementCallUrl, + setCallEmbedStartError, + newCallsEnabled, + livekitJsCallManager, + ] ); return startCall; diff --git a/src/app/hooks/useLivekitSupport.test.tsx b/src/app/hooks/useLivekitSupport.test.tsx new file mode 100644 index 0000000000..7328144efd --- /dev/null +++ b/src/app/hooks/useLivekitSupport.test.tsx @@ -0,0 +1,77 @@ +import type { ReactNode } from 'react'; +import { render, waitFor } from '@testing-library/react'; +import { describe, expect, it, vi } from 'vitest'; +import type { MatrixClient } from '$types/matrix-sdk'; +import type { AutoDiscoveryInfo } from '../cs-api'; +import { MatrixClientProvider } from './useMatrixClient'; +import { AutoDiscoveryInfoProvider } from './useAutoDiscoveryInfo'; +import { useLivekitSupport } from './useLivekitSupport'; + +const wellKnownWithFocus = { + 'org.matrix.msc4143.rtc_foci': [{ type: 'livekit', livekit_service_url: 'https://sfu.example' }], +} as unknown as AutoDiscoveryInfo; + +const wellKnownWithoutFocus = {} as unknown as AutoDiscoveryInfo; + +const makeClient = (getTransports: () => Promise): MatrixClient => + ({ _unstable_getRTCTransports: getTransports }) as unknown as MatrixClient; + +const renderSupport = ( + mx: MatrixClient, + autoDiscoveryInfo: AutoDiscoveryInfo +): (() => boolean | undefined) => { + let latest: boolean | undefined; + function Probe() { + latest = useLivekitSupport(); + return null; + } + const wrapper = ({ children }: { children?: ReactNode }) => ( + + {children} + + ); + render(, { wrapper }); + return () => latest; +}; + +describe('useLivekitSupport', () => { + it('accepts a transport advertised by the homeserver', async () => { + const mx = makeClient(() => + Promise.resolve([{ type: 'livekit', livekit_service_url: 'https://sfu.example' }]) + ); + const latest = renderSupport(mx, wellKnownWithoutFocus); + + await waitFor(() => expect(latest()).toBe(true)); + }); + + it('falls back to the legacy well-known focus', async () => { + const mx = makeClient(() => Promise.reject(new Error('M_NOT_FOUND'))); + const latest = renderSupport(mx, wellKnownWithFocus); + + await waitFor(() => expect(latest()).toBe(true)); + }); + + it('reports no support when neither source advertises a livekit transport', async () => { + const mx = makeClient(() => Promise.resolve([{ type: 'not-livekit' }])); + const latest = renderSupport(mx, wellKnownWithoutFocus); + + await waitFor(() => expect(latest()).toBe(false)); + }); + + it('does not report unavailable while the homeserver probe is in flight', () => { + const mx = makeClient(() => new Promise(() => {})); + const latest = renderSupport(mx, wellKnownWithFocus); + + expect(latest()).toBe(true); + }); + + it('probes the homeserver once per client', async () => { + const getTransports = vi.fn<() => Promise>().mockResolvedValue([]); + const mx = { _unstable_getRTCTransports: getTransports } as unknown as MatrixClient; + + renderSupport(mx, wellKnownWithoutFocus); + renderSupport(mx, wellKnownWithoutFocus); + + await waitFor(() => expect(getTransports).toHaveBeenCalledOnce()); + }); +}); diff --git a/src/app/hooks/useLivekitSupport.ts b/src/app/hooks/useLivekitSupport.ts index 6c87d43749..ee90d01b65 100644 --- a/src/app/hooks/useLivekitSupport.ts +++ b/src/app/hooks/useLivekitSupport.ts @@ -1,6 +1,16 @@ +import { useEffect, useState } from 'react'; import type { AutoDiscoveryInfo } from '../cs-api'; import { useAutoDiscoveryInfo } from './useAutoDiscoveryInfo'; +import { useMatrixClient } from './useMatrixClient'; +import { isLivekitTransportConfig } from '$features/call/livekitProvisioning'; +import type { MatrixClient } from '$types/matrix-sdk'; +import { createDebugLogger } from '$utils/debugLogger'; +const debugLog = createDebugLogger('useLivekitSupport'); + +type TransportClient = Pick; + +/** Legacy discovery: the pre-MSC4143 `.well-known` advertisement. */ export const livekitSupport = (autoDiscoveryInfo: AutoDiscoveryInfo): boolean => { const rtcFoci = autoDiscoveryInfo['org.matrix.msc4143.rtc_foci']; @@ -9,8 +19,44 @@ export const livekitSupport = (autoDiscoveryInfo: AutoDiscoveryInfo): boolean => ); }; +// `/rtc/transports` is a network round trip and every room view asks. One probe +// per client is enough; the answer only changes when the homeserver does. +const probes = new WeakMap>(); + +export const probeServerLivekitTransport = (mx: TransportClient): Promise => { + const existing = probes.get(mx); + if (existing) return existing; + + const probe = mx['_unstable_getRTCTransports']() + .then((transports) => transports.some(isLivekitTransportConfig)) + .catch((error: unknown) => { + // A homeserver without MSC4143 answers M_NOT_FOUND, which is a legitimate + // "no transport here" rather than an outage, so it stays at info level. + debugLog.info('call', 'homeserver advertised no RTC transports', error); + return false; + }); + probes.set(mx, probe); + return probe; +}; + export const useLivekitSupport = (): boolean => { + const mx = useMatrixClient(); const autoDiscoveryInfo = useAutoDiscoveryInfo(); + const wellKnownSupported = livekitSupport(autoDiscoveryInfo); + const [serverSupported, setServerSupported] = useState(undefined); + + useEffect(() => { + let active = true; + void probeServerLivekitTransport(mx).then((supported) => { + if (active) setServerSupported(supported); + }); + return () => { + active = false; + }; + }, [mx]); - return livekitSupport(autoDiscoveryInfo); + // The server-advertised transport is the MSC4143 answer and `.well-known` is + // the legacy fallback, so either one is enough. While the probe is in flight + // we report the legacy answer rather than a premature "unavailable". + return serverSupported === true || wellKnownSupported; }; diff --git a/src/app/pages/CallStatusRenderer.test.tsx b/src/app/pages/CallStatusRenderer.test.tsx new file mode 100644 index 0000000000..3093d642cb --- /dev/null +++ b/src/app/pages/CallStatusRenderer.test.tsx @@ -0,0 +1,151 @@ +import { render, screen } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { createStore, Provider } from 'jotai'; +import type { ReactNode } from 'react'; +import { livekitJsCallAtom, type LivekitJsCallSession } from '../state/livekitJsCall'; +import { nativeCallAtom, type NativeCallSession } from '../state/nativeCall'; +import { CallStatusRenderer } from './CallStatusRenderer'; + +const mocks = vi.hoisted(() => ({ + callEmbed: undefined as { roomId: string } | undefined, + selectedRoom: undefined as string | undefined, + screenSize: 'Desktop', +})); + +vi.mock('../hooks/useCallEmbed', () => ({ useCallEmbed: () => mocks.callEmbed })); +vi.mock('../hooks/router/useSelectedRoom', () => ({ + useSelectedRoom: () => mocks.selectedRoom, +})); +vi.mock('../hooks/useScreenSize', () => ({ + ScreenSize: { Mobile: 'Mobile', Desktop: 'Desktop' }, + useScreenSizeContext: () => mocks.screenSize, +})); +vi.mock('../features/call-status', () => ({ + CallStatus: () =>
, +})); +vi.mock('../features/call-status/LivekitCallStatus', () => ({ + LivekitCallStatus: () =>
, +})); +vi.mock('../features/call-status/NativeCallStatus', () => ({ + NativeCallStatus: () =>
, +})); + +const session = (lifecycle: LivekitJsCallSession['lifecycle']): LivekitJsCallSession => ({ + roomId: '!room:example.org', + lifecycle, + failure: null, + mediaReady: true, + initialMedia: { microphone: true, camera: false, sound: true }, + hangup: async () => undefined, +}); + +const renderWith = (children: ReactNode, store = createStore()) => + render({children}); + +beforeEach(() => { + mocks.callEmbed = undefined; + mocks.selectedRoom = undefined; + mocks.screenSize = 'Desktop'; +}); + +describe('CallStatusRenderer', () => { + it('shows the status bar during a LiveKit call', () => { + const store = createStore(); + store.set(livekitJsCallAtom, session('active')); + + renderWith(, store); + + expect(screen.getByTestId('livekit-call-status')).toBeInTheDocument(); + }); + + it('keeps showing it while the call is still connecting', () => { + const store = createStore(); + store.set(livekitJsCallAtom, session('joining-matrix')); + + renderWith(, store); + + expect(screen.getByTestId('livekit-call-status')).toBeInTheDocument(); + }); + + it('hides it once the call has failed', () => { + const store = createStore(); + store.set(livekitJsCallAtom, session('failed')); + + renderWith(, store); + + expect(screen.queryByTestId('livekit-call-status')).not.toBeInTheDocument(); + }); + + it('hides it on mobile while the call room is open, as the embed path does', () => { + mocks.screenSize = 'Mobile'; + mocks.selectedRoom = '!room:example.org'; + const store = createStore(); + store.set(livekitJsCallAtom, session('active')); + + renderWith(, store); + + expect(screen.queryByTestId('livekit-call-status')).not.toBeInTheDocument(); + }); + + it('prefers the Element Call bar when an embed is active', () => { + mocks.callEmbed = { roomId: '!other:example.org' }; + const store = createStore(); + store.set(livekitJsCallAtom, session('active')); + + renderWith(, store); + + expect(screen.getByTestId('element-call-status')).toBeInTheDocument(); + expect(screen.queryByTestId('livekit-call-status')).not.toBeInTheDocument(); + }); + + it('shows the status bar during a native call', () => { + const store = createStore(); + store.set(nativeCallAtom, { + backend: 'livekit-mobile', + roomId: '!room:example.org', + callId: 'call-id', + lifecycle: 'connected', + participants: [], + microphoneEnabled: true, + cameraEnabled: false, + setMicrophoneEnabled: async () => {}, + setCameraEnabled: async () => {}, + switchCamera: async () => {}, + listAudioRoutes: async () => [], + selectAudioRoute: async () => {}, + hangup: async () => undefined, + } satisfies NativeCallSession); + + renderWith(, store); + + expect(screen.getByTestId('native-call-status')).toBeInTheDocument(); + }); + + it('hides the native bar once the call has failed', () => { + const store = createStore(); + store.set(nativeCallAtom, { + backend: 'livekit-mobile', + roomId: '!room:example.org', + callId: 'call-id', + lifecycle: 'error', + participants: [], + microphoneEnabled: false, + cameraEnabled: false, + setMicrophoneEnabled: async () => {}, + setCameraEnabled: async () => {}, + switchCamera: async () => {}, + listAudioRoutes: async () => [], + selectAudioRoute: async () => {}, + hangup: async () => undefined, + } satisfies NativeCallSession); + + renderWith(, store); + + expect(screen.queryByTestId('native-call-status')).not.toBeInTheDocument(); + }); + + it('renders nothing with no call at all', () => { + const { container } = renderWith(); + expect(container).toBeEmptyDOMElement(); + }); +}); diff --git a/src/app/pages/CallStatusRenderer.tsx b/src/app/pages/CallStatusRenderer.tsx index d0fb4732ec..dbd76bde75 100644 --- a/src/app/pages/CallStatusRenderer.tsx +++ b/src/app/pages/CallStatusRenderer.tsx @@ -1,17 +1,36 @@ +import { useAtomValue } from 'jotai'; import { useCallEmbed } from '../hooks/useCallEmbed'; import { CallStatus } from '../features/call-status'; +import { LivekitCallStatus } from '../features/call-status/LivekitCallStatus'; +import { NativeCallStatus } from '../features/call-status/NativeCallStatus'; +import { isLivekitJsCallActive, livekitJsCallAtom } from '../state/livekitJsCall'; +import { isNativeCallActive, nativeCallAtom } from '../state/nativeCall'; import { useSelectedRoom } from '../hooks/router/useSelectedRoom'; import { ScreenSize, useScreenSizeContext } from '../hooks/useScreenSize'; export function CallStatusRenderer() { const callEmbed = useCallEmbed(); + const livekitJsCall = useAtomValue(livekitJsCallAtom); + const nativeCall = useAtomValue(nativeCallAtom); const selectedRoom = useSelectedRoom(); const screenSize = useScreenSizeContext(); + const mobileInCallRoom = screenSize === ScreenSize.Mobile; - if (!callEmbed) return null; + if (callEmbed) { + if (mobileInCallRoom && callEmbed.roomId === selectedRoom) return null; + return ; + } - if (screenSize === ScreenSize.Mobile && callEmbed.roomId === selectedRoom) return null; + if (isLivekitJsCallActive(livekitJsCall) && livekitJsCall) { + if (mobileInCallRoom && livekitJsCall.roomId === selectedRoom) return null; + return ; + } - return ; + if (isNativeCallActive(nativeCall) && nativeCall) { + if (mobileInCallRoom && nativeCall.roomId === selectedRoom) return null; + return ; + } + + return null; } diff --git a/src/app/pages/client/BackgroundNotifications.tsx b/src/app/pages/client/BackgroundNotifications.tsx index 3a9ba270ba..8f40a14435 100644 --- a/src/app/pages/client/BackgroundNotifications.tsx +++ b/src/app/pages/client/BackgroundNotifications.tsx @@ -346,7 +346,7 @@ export function BackgroundNotifications() { if (!eventId) return; const eventType = mEvent.getType(); - const isEncryptedType = eventType === (EventType.RoomMessageEncrypted as string); + const isEncryptedType = eventType === 'm.room.encrypted'; // For encrypted events that haven't been decrypted yet, wait for decryption // before processing the notification. The SDK's Timeline re-emission after diff --git a/src/app/pages/client/client-non-ui/notifications.tsx b/src/app/pages/client/client-non-ui/notifications.tsx index 80b43bc51a..88419c64d3 100644 --- a/src/app/pages/client/client-non-ui/notifications.tsx +++ b/src/app/pages/client/client-non-ui/notifications.tsx @@ -248,7 +248,7 @@ export function MessageNotifications() { // For encrypted events that haven't been decrypted yet, wait for decryption // before processing the notification. The SDK's Timeline re-emission after // decryption comes with data.liveEvent=false which would wrongly block it. - if (mEvent.getType() === (EventType.RoomMessageEncrypted as string) && mEvent.isEncrypted()) { + if (mEvent.getType() === 'm.room.encrypted' && mEvent.isEncrypted()) { if (eventId) { // Mark this event to skip focus check when decrypted, so we use the focus // state from when the encrypted event originally arrived, not when it decrypts. diff --git a/src/app/plugins/call/CallEmbed.ts b/src/app/plugins/call/CallEmbed.ts index 0de990d719..edde57b3e9 100644 --- a/src/app/plugins/call/CallEmbed.ts +++ b/src/app/plugins/call/CallEmbed.ts @@ -1,11 +1,5 @@ import type { MatrixClient, MatrixEvent, Room } from '$types/matrix-sdk'; -import { - ClientEvent, - EventType, - KnownMembership, - MatrixEventEvent, - RoomStateEvent, -} from '$types/matrix-sdk'; +import { ClientEvent, KnownMembership, MatrixEventEvent, RoomStateEvent } from '$types/matrix-sdk'; import { invoke } from '@tauri-apps/api/core'; import type { IRoomEvent, IWidget, WidgetDriver } from 'matrix-widget-api'; import { @@ -339,11 +333,11 @@ export class CallEmbed { }); // Sliding sync may not have delivered m.room.member yet. - if (!this.room.currentState.getStateEvents(EventType.RoomMember, myUserId)) { + if (!this.room.currentState.getStateEvents('m.room.member', myUserId)) { const membership = this.room.getMyMembership(); if (membership) { const memberRaw = { - type: EventType.RoomMember, + type: 'm.room.member', state_key: myUserId, room_id: this.roomId, sender: myUserId, diff --git a/src/app/state/callEmbed.ts b/src/app/state/callEmbed.ts index 3dcc6c7459..6a81124540 100644 --- a/src/app/state/callEmbed.ts +++ b/src/app/state/callEmbed.ts @@ -2,6 +2,7 @@ import { atom } from 'jotai'; import * as Sentry from '@sentry/react'; import type { CallEmbed } from '../plugins/call'; import type { CallEmbedStartError } from '$plugins/call/callEmbedError'; +import { releaseCallOwner } from './callOwner'; const baseCallEmbedAtom = atom(undefined); const baseCallEmbedStartErrorAtom = atom(null); @@ -25,6 +26,7 @@ export const callEmbedAtom = atom resetCallOwnerForTests()); + +describe('call owner guard', () => { + it('allows one owner and rejects competing owners until release', () => { + const element = acquireCallOwner('element', '!room:example.org'); + + expect(element).toBeDefined(); + expect(acquireCallOwner('livekit-js', '!room:example.org')).toBeUndefined(); + expect(acquireCallOwner('livekit-mobile', '!room:example.org')).toBeUndefined(); + expect(getActiveCallOwner()).toMatchObject({ kind: 'element' }); + + element?.release(); + expect(acquireCallOwner('livekit-js', '!room:example.org')).toBeDefined(); + }); + + it('blocks competing owners while the native transport owns the call', () => { + const native = acquireCallOwner('livekit-mobile', '!room:example.org'); + + expect(native).toBeDefined(); + expect(acquireCallOwner('element', '!room:example.org')).toBeUndefined(); + expect(acquireCallOwner('livekit-js', '!room:example.org')).toBeUndefined(); + expect(getActiveCallOwner()).toMatchObject({ kind: 'livekit-mobile' }); + + native?.release(); + expect(acquireCallOwner('element', '!room:example.org')).toBeDefined(); + }); + + it('makes release idempotent and cannot release a replacement lease', () => { + const first = acquireCallOwner('element', '!room:example.org'); + first?.release(); + const second = acquireCallOwner('livekit-js', '!room:example.org'); + + first?.release(); + expect(getActiveCallOwner()).toMatchObject({ kind: 'livekit-js' }); + second?.release(); + expect(getActiveCallOwner()).toBeUndefined(); + }); +}); diff --git a/src/app/state/callOwner.ts b/src/app/state/callOwner.ts new file mode 100644 index 0000000000..a6c95fac72 --- /dev/null +++ b/src/app/state/callOwner.ts @@ -0,0 +1,40 @@ +export type CallOwnerKind = 'element' | 'livekit-js' | 'livekit-mobile'; + +export type CallOwnerLease = { + kind: CallOwnerKind; + roomId: string; + release: () => void; +}; + +let activeOwner: CallOwnerLease | undefined; + +export const acquireCallOwner = ( + kind: CallOwnerKind, + roomId: string +): CallOwnerLease | undefined => { + if (activeOwner) return undefined; + + let released = false; + const lease: CallOwnerLease = { + kind, + roomId, + release: () => { + if (released || activeOwner !== lease) return; + released = true; + activeOwner = undefined; + }, + }; + activeOwner = lease; + return lease; +}; + +export const getActiveCallOwner = (): Pick | undefined => + activeOwner; + +export const releaseCallOwner = (kind: CallOwnerKind, roomId: string): void => { + if (activeOwner?.kind === kind && activeOwner.roomId === roomId) activeOwner.release(); +}; + +export const resetCallOwnerForTests = (): void => { + activeOwner = undefined; +}; diff --git a/src/app/state/callPreferences.ts b/src/app/state/callPreferences.ts index db452f0b54..3c082b2f02 100644 --- a/src/app/state/callPreferences.ts +++ b/src/app/state/callPreferences.ts @@ -9,6 +9,8 @@ export type CallPreferences = { microphone: boolean; video: boolean; sound: boolean; + audioDeviceId?: string; + videoDeviceId?: string; }; const CALL_PREFERENCES = 'callPreferences'; diff --git a/src/app/state/hooks/callPreferences.ts b/src/app/state/hooks/callPreferences.ts index 3bc5f20ce5..5350fa06af 100644 --- a/src/app/state/hooks/callPreferences.ts +++ b/src/app/state/hooks/callPreferences.ts @@ -18,6 +18,8 @@ export const useCallPreferences = (): CallPreferences & { toggleMicrophone: () => void; toggleVideo: () => void; toggleSound: () => void; + setAudioDeviceId: (deviceId: string) => void; + setVideoDeviceId: (deviceId: string) => void; setPreferences: (prefs: CallPreferences) => void; } => { const callPrefAtom = useCallPreferencesAtom(); @@ -27,8 +29,8 @@ export const useCallPreferences = (): CallPreferences & { const microphone = !pref.microphone; setPref({ + ...pref, microphone, - video: pref.video, sound: !pref.sound && microphone ? true : pref.sound, }); }, [setPref, pref]); @@ -36,25 +38,33 @@ export const useCallPreferences = (): CallPreferences & { const toggleVideo = useCallback(() => { const video = !pref.video; - setPref({ - microphone: pref.microphone, - video, - sound: pref.sound, - }); + setPref({ ...pref, video }); }, [setPref, pref]); const toggleSound = useCallback(() => { const sound = !pref.sound; setPref({ + ...pref, microphone: !sound ? false : pref.microphone, - video: pref.video, sound, }); }, [setPref, pref]); + const setAudioDeviceId = useCallback( + (audioDeviceId: string) => setPref({ ...pref, audioDeviceId }), + [setPref, pref] + ); + + const setVideoDeviceId = useCallback( + (videoDeviceId: string) => setPref({ ...pref, videoDeviceId }), + [setPref, pref] + ); + return { ...pref, + setAudioDeviceId, + setVideoDeviceId, toggleMicrophone, toggleVideo, toggleSound, diff --git a/src/app/state/livekitJsCall.test.ts b/src/app/state/livekitJsCall.test.ts new file mode 100644 index 0000000000..15bccead97 --- /dev/null +++ b/src/app/state/livekitJsCall.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest'; +import { selectActiveCallSession } from './livekitJsCall'; +import { acquireCallOwner, getActiveCallOwner, resetCallOwnerForTests } from './callOwner'; + +describe('selectActiveCallSession', () => { + it('selects the JS owner when Element Call is absent', () => { + const livekitSession = { + roomId: '!room:example.org', + lifecycle: 'active' as const, + failure: null, + mediaReady: true, + initialMedia: { microphone: true, camera: false, sound: true }, + hangup: async () => undefined, + }; + + expect(selectActiveCallSession(undefined, livekitSession)).toBe(livekitSession); + }); + + it('preserves Element Call precedence over the JS owner', () => { + const element = { roomId: '!element:example.org' }; + const livekit = { + roomId: '!livekit:example.org', + lifecycle: 'active' as const, + failure: null, + mediaReady: true, + initialMedia: { microphone: true, camera: false, sound: true }, + hangup: async () => undefined, + }; + + expect(selectActiveCallSession(element, livekit)).toBe(element); + }); + + it('routes past a failed JS session to Element Call and releases its lease', async () => { + resetCallOwnerForTests(); + const lease = acquireCallOwner('element', '!room:example.org'); + const element = { + roomId: '!room:example.org', + hangup: async () => lease?.release(), + }; + const livekit = { + roomId: '!room:example.org', + lifecycle: 'failed' as const, + failure: 'setup-failed' as const, + mediaReady: false, + initialMedia: { microphone: true, camera: false, sound: true }, + hangup: async () => undefined, + }; + + await selectActiveCallSession(element, livekit)?.hangup(); + + expect(getActiveCallOwner()).toBeUndefined(); + }); +}); diff --git a/src/app/state/livekitJsCall.ts b/src/app/state/livekitJsCall.ts new file mode 100644 index 0000000000..e9aa4b70f3 --- /dev/null +++ b/src/app/state/livekitJsCall.ts @@ -0,0 +1,52 @@ +import { atom } from 'jotai'; +import type { + LivekitJsControllerFailure, + LivekitJsControllerLifecycle, +} from '$features/call/livekitJsController'; +import type { Room as LivekitRoom } from 'livekit-client'; + +export type LivekitJsCallMedia = { + microphone: boolean; + camera: boolean; + sound: boolean; + audioDeviceId?: string; + videoDeviceId?: string; +}; + +export type LivekitJsCallSession = { + roomId: string; + /** What the user chose on the prescreen; applied once media is ready. */ + initialMedia: LivekitJsCallMedia; + lifecycle: LivekitJsControllerLifecycle; + failure: LivekitJsControllerFailure | null; + room?: LivekitRoom; + /** Immediate for an unencrypted room, after the Matrix key for an encrypted one. */ + mediaReady: boolean; + hangup: () => Promise; +}; + +export const livekitJsCallAtom = atom(undefined); + +export const livekitJsCallSoundAtom = atom(true); + +/** + * Whether `initialMedia` has been published for the current call. The surface + * unmounts when the user navigates out of the room while the call keeps + * running, so this has to outlive it or coming back re-applies the prescreen + * choice and undoes anything toggled from the call bar. + */ +export const livekitJsCallInitialMediaAppliedAtom = atom(false); + +export const isLivekitJsCallActive = (session: LivekitJsCallSession | undefined): boolean => + session?.lifecycle !== undefined && + session.lifecycle !== 'idle' && + session.lifecycle !== 'failed'; + +export const selectActiveCallSession = ( + elementCall: Element | undefined, + livekitJsCall: LivekitJsCallSession | undefined +): Element | LivekitJsCallSession | undefined => { + if (elementCall) return elementCall; + if (isLivekitJsCallActive(livekitJsCall)) return livekitJsCall; + return undefined; +}; diff --git a/src/app/state/nativeCall.test.ts b/src/app/state/nativeCall.test.ts new file mode 100644 index 0000000000..01df3b8815 --- /dev/null +++ b/src/app/state/nativeCall.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from 'vitest'; +import { + isNativeCallActive, + selectActiveCallSessionIncludingNative, + type NativeCallSession, +} from './nativeCall'; +import type { LivekitJsCallSession } from './livekitJsCall'; + +const makeNativeSession = (lifecycle: NativeCallSession['lifecycle']): NativeCallSession => ({ + backend: 'livekit-mobile', + roomId: '!room:example.org', + callId: 'call-id', + lifecycle, + participants: [], + microphoneEnabled: true, + cameraEnabled: false, + setMicrophoneEnabled: async () => {}, + setCameraEnabled: async () => {}, + switchCamera: async () => {}, + listAudioRoutes: async () => [], + selectAudioRoute: async () => {}, + hangup: async () => undefined, +}); + +const livekitSession: LivekitJsCallSession = { + roomId: '!room:example.org', + lifecycle: 'active', + failure: null, + mediaReady: true, + initialMedia: { microphone: true, camera: false, sound: true }, + hangup: async () => undefined, +}; + +describe('isNativeCallActive', () => { + it('treats missing or errored sessions as inactive', () => { + expect(isNativeCallActive(undefined)).toBe(false); + expect(isNativeCallActive(makeNativeSession('error'))).toBe(false); + expect(isNativeCallActive(makeNativeSession('connected'))).toBe(true); + }); +}); + +describe('selectActiveCallSessionIncludingNative', () => { + it('prefers Element Call and LiveKit JS over the native session', () => { + const element = { roomId: '!room:example.org' }; + const native = makeNativeSession('connected'); + + expect(selectActiveCallSessionIncludingNative(element, livekitSession, native)).toBe(element); + expect(selectActiveCallSessionIncludingNative(undefined, livekitSession, native)).toBe( + livekitSession + ); + }); + + it('returns the native session only when nothing else is active', () => { + const native = makeNativeSession('connected'); + expect(selectActiveCallSessionIncludingNative(undefined, undefined, native)).toBe(native); + expect( + selectActiveCallSessionIncludingNative(undefined, undefined, makeNativeSession('error')) + ).toBeUndefined(); + expect(selectActiveCallSessionIncludingNative(undefined, undefined, undefined)).toBeUndefined(); + }); +}); diff --git a/src/app/state/nativeCall.ts b/src/app/state/nativeCall.ts new file mode 100644 index 0000000000..195537d335 --- /dev/null +++ b/src/app/state/nativeCall.ts @@ -0,0 +1,64 @@ +import { atom } from 'jotai'; +import type { NativeCallAudioRoute } from '$features/call/livekitMobileBridge'; +import type { CallParticipant } from '$features/call/callTransport'; +import { + isLivekitJsCallActive, + livekitJsCallAtom, + selectActiveCallSession, + type LivekitJsCallSession, +} from './livekitJsCall'; +import { callEmbedAtom } from './callEmbed'; + +export type NativeCallBackend = 'livekit-mobile'; + +export type NativeCallLifecycle = + | 'starting' + | 'connecting' + | 'connected' + | 'reconnecting' + | 'error'; + +export type NativeCallSession = { + backend: NativeCallBackend; + roomId: string; + callId: string; + lifecycle: NativeCallLifecycle; + error?: string; + /** Remote peers only, as the transport reports them. */ + participants: CallParticipant[]; + microphoneEnabled: boolean; + cameraEnabled: boolean; + setMicrophoneEnabled: (enabled: boolean) => Promise; + setCameraEnabled: (enabled: boolean) => Promise; + switchCamera: () => Promise; + listAudioRoutes: () => Promise; + selectAudioRoute: (routeId: string) => Promise; + hangup: () => Promise; +}; + +export const nativeCallAtom = atom(undefined); + +export const isNativeCallActive = (session: NativeCallSession | undefined): boolean => + session?.lifecycle !== undefined && session.lifecycle !== 'error'; + +/** + * Whether any engine already owns a call. Only one may run at a time: a second + * start would lose the call-owner lease and publish a failure of its own while + * the first call carries on. + */ +export const callInProgressAtom = atom( + (get) => + get(callEmbedAtom) !== undefined || + isLivekitJsCallActive(get(livekitJsCallAtom)) || + isNativeCallActive(get(nativeCallAtom)) +); + +export const selectActiveCallSessionIncludingNative = ( + elementCall: Element | undefined, + livekitJsCall: LivekitJsCallSession | undefined, + nativeCall: NativeCallSession | undefined +): Element | LivekitJsCallSession | NativeCallSession | undefined => { + const selected = selectActiveCallSession(elementCall, livekitJsCall); + if (selected) return selected; + return isNativeCallActive(nativeCall) ? nativeCall : undefined; +}; diff --git a/src/app/state/settings.defaults.test.ts b/src/app/state/settings.defaults.test.ts index 2b798084d6..c2d84bf664 100644 --- a/src/app/state/settings.defaults.test.ts +++ b/src/app/state/settings.defaults.test.ts @@ -12,6 +12,46 @@ beforeEach(() => { }); describe('mergePersistedSettings', () => { + it('defaults new calls off and persists the opt-in', () => { + expect(defaultSettings.newCallsEnabled).toBe(false); + + localStorage.setItem('settings', JSON.stringify({ newCallsEnabled: true })); + expect(mergePersistedSettings(localStorage.getItem('settings'), {}).newCallsEnabled).toBe(true); + }); + + it('enables new calls when either legacy experimental setting was on', () => { + localStorage.setItem('settings', JSON.stringify({ livekitJsCallsEnabled: true })); + const merged = mergePersistedSettings(localStorage.getItem('settings'), {}); + expect(merged.newCallsEnabled).toBe(true); + expect(merged).not.toHaveProperty('livekitJsCallsEnabled'); + + localStorage.setItem('settings', JSON.stringify({ livekitJsMediaTestEnabled: true })); + const mergedMedia = mergePersistedSettings(localStorage.getItem('settings'), {}); + expect(mergedMedia.newCallsEnabled).toBe(true); + expect(mergedMedia).not.toHaveProperty('livekitJsMediaTestEnabled'); + }); + + it('keeps new calls off when both legacy settings were off or absent', () => { + localStorage.setItem( + 'settings', + JSON.stringify({ livekitJsCallsEnabled: false, livekitJsMediaTestEnabled: false }) + ); + const merged = mergePersistedSettings(localStorage.getItem('settings'), {}); + expect(merged.newCallsEnabled).toBe(false); + expect(merged).not.toHaveProperty('livekitJsCallsEnabled'); + expect(merged).not.toHaveProperty('livekitJsMediaTestEnabled'); + }); + + it('does not override an explicit new calls preference with legacy keys', () => { + localStorage.setItem( + 'settings', + JSON.stringify({ newCallsEnabled: false, livekitJsCallsEnabled: true }) + ); + const merged = mergePersistedSettings(localStorage.getItem('settings'), {}); + expect(merged.newCallsEnabled).toBe(false); + expect(merged).not.toHaveProperty('livekitJsCallsEnabled'); + }); + it('layers deployer defaults over code defaults when localStorage is empty', () => { const merged = mergePersistedSettings(null, { twitterEmoji: false }); expect(merged.twitterEmoji).toBe(false); @@ -88,6 +128,19 @@ describe('sanitizeSettingsDefaults', () => { }); }); + it('accepts the new calls setting', () => { + expect(sanitizeSettingsDefaults({ newCallsEnabled: true })).toEqual({ + newCallsEnabled: true, + }); + expect(sanitizeSettingsDefaults({ newCallsEnabled: 'yes' })).toEqual({}); + }); + + it('drops the legacy LiveKit JS experimental settings', () => { + expect( + sanitizeSettingsDefaults({ livekitJsCallsEnabled: true, livekitJsMediaTestEnabled: true }) + ).toEqual({}); + }); + it('drops unknown keys', () => { expect(sanitizeSettingsDefaults({ notARealSetting: true, hour24Clock: true })).toEqual({ hour24Clock: true, diff --git a/src/app/state/settings.ts b/src/app/state/settings.ts index a296dbfe4b..0255d84f2a 100644 --- a/src/app/state/settings.ts +++ b/src/app/state/settings.ts @@ -180,6 +180,7 @@ export interface Settings { developerTools: boolean; enableMSC4268CMD: boolean; enableMediaGalleries: boolean; + newCallsEnabled: boolean; settingsSyncEnabled: boolean; // Cosmetics! @@ -339,6 +340,7 @@ export const defaultSettings: Settings = { enableMSC4268CMD: false, enableMediaGalleries: false, + newCallsEnabled: false, // Push notifications (SW/Sygnal): default on for mobile, opt-in on desktop. // In-app pill banner: default on for mobile (primary foreground alert), opt-in on desktop. @@ -524,6 +526,16 @@ function migrateParsedLocalStorage(parsed: Record): void { delete parsed.themeChatPreviewAnyUrl; delete parsed.themeChatPreviewApprovedCatalogOnly; + // Consolidate the legacy LiveKit JS experiments into the single new-call gate. + if ( + typeof parsed.newCallsEnabled !== 'boolean' && + (parsed.livekitJsCallsEnabled === true || parsed.livekitJsMediaTestEnabled === true) + ) { + parsed.newCallsEnabled = true; + } + delete parsed.livekitJsCallsEnabled; + delete parsed.livekitJsMediaTestEnabled; + if (typeof parsed.callRingtoneVolume === 'number' && Number.isFinite(parsed.callRingtoneVolume)) { parsed.callRingtoneVolume = clampPercent(parsed.callRingtoneVolume); } diff --git a/src/app/utils/messageReaction.ts b/src/app/utils/messageReaction.ts index ea71caa51e..ae74030ad3 100644 --- a/src/app/utils/messageReaction.ts +++ b/src/app/utils/messageReaction.ts @@ -1,6 +1,5 @@ import type { MatrixReactionEvent } from '$types/matrix/common'; -import type { MatrixClient, Room } from '$types/matrix-sdk'; -import { RelationType } from '$types/matrix-sdk'; +import type { MatrixClient, Room } from 'matrix-js-sdk'; import { ImageUsage } from '$plugins/custom-emoji'; import { getImagePackReferencesForMxcWrappedInMap } from './msc4459helper'; import { MATRIX_UNSTABLE_IMAGE_SOURCE_PACK_PROPERTY_NAME } from '$unstable/prefixes'; @@ -15,7 +14,7 @@ export const getReactionContent = ( 'm.relates_to': { event_id: eventId, key, - rel_type: RelationType.Annotation, + rel_type: 'm.annotation', }, shortcode, 'com.beeper.reaction.shortcode': shortcode, diff --git a/src/app/utils/notificationStyle.ts b/src/app/utils/notificationStyle.ts index dfc16091c3..c06345716b 100644 --- a/src/app/utils/notificationStyle.ts +++ b/src/app/utils/notificationStyle.ts @@ -1,5 +1,3 @@ -import { EventType } from 'matrix-js-sdk/lib/@types/event'; - export const DEFAULT_NOTIFICATION_ICON = '/public/res/logo-maskable/logo-maskable-180x180.png'; export const DEFAULT_NOTIFICATION_BADGE = '/public/res/logo-maskable/logo-maskable-72x72.png'; const DEFAULT_MESSAGE_PREVIEW = 'new message'; @@ -65,7 +63,7 @@ export const resolveNotificationPreviewText = ({ return 'Added a reaction'; } - const encryptedContext = isEncryptedRoom || eventType === EventType.RoomMessageEncrypted; + const encryptedContext = isEncryptedRoom || eventType === 'm.room.encrypted'; if (!showMessageContent) { return encryptedContext ? ENCRYPTED_MESSAGE_PREVIEW : DEFAULT_MESSAGE_PREVIEW; diff --git a/src/app/utils/room/relations.ts b/src/app/utils/room/relations.ts index 56474ea7aa..53de309035 100644 --- a/src/app/utils/room/relations.ts +++ b/src/app/utils/room/relations.ts @@ -211,6 +211,7 @@ export const collectRelationEditEvents = ( export const isRedactableMessageType = (type: string): boolean => type === (EventType.RoomMessage as string) || type === (EventType.RoomMessageEncrypted as string) || + type === 'm.room.encrypted' || type === (EventType.Sticker as string); export const getRedactionTargetId = (redactionEvent: MatrixEvent): string | undefined => { diff --git a/src/app/utils/room/unread.ts b/src/app/utils/room/unread.ts index 85ac4f1d48..6d04eddaa9 100644 --- a/src/app/utils/room/unread.ts +++ b/src/app/utils/room/unread.ts @@ -1,10 +1,5 @@ import type { IPushRule, IPushRules, MatrixClient, MatrixEvent, Room } from '$types/matrix-sdk'; -import { - EventType, - NotificationCountType, - PushRuleActionName, - RelationType, -} from '$types/matrix-sdk'; +import { EventType, NotificationCountType, PushRuleActionName } from '$types/matrix-sdk'; import type { UnreadInfo } from '$types/matrix/room'; import { NotificationType } from '$types/matrix/room'; @@ -91,13 +86,13 @@ export const getNotificationType = (mx: MatrixClient, roomId: string): Notificat return NotificationType.MentionsAndKeywords; }; -const NOTIFICATION_EVENT_TYPES = new Set([ - EventType.RoomCreate, - EventType.RoomMessage, - EventType.RoomMessageEncrypted, - EventType.RoomMember, - EventType.Sticker, - EventType.Reaction, +const NOTIFICATION_EVENT_TYPES = new Set([ + 'm.room.create', + 'm.room.message', + 'm.room.encrypted', + 'm.room.member', + 'm.sticker', + 'm.reaction', ]); export const isNotificationEvent = (mEvent: MatrixEvent, room?: Room, userId?: string) => { const eType = mEvent.getType(); @@ -114,7 +109,7 @@ export const isNotificationEvent = (mEvent: MatrixEvent, room?: Room, userId?: s if (relationType === 'm.replace') return false; // For reactions: only count them if they're reactions to the current user's messages - if (relationType === RelationType.Annotation) { + if (relationType === 'm.annotation') { if (!room || !userId || !relation) { // If we don't have room/userId/relation context, filter out all reactions (safe default) return false; diff --git a/src/app/utils/sendFeedbackToUser.ts b/src/app/utils/sendFeedbackToUser.ts index f8b935e48c..03fd3a6fa9 100644 --- a/src/app/utils/sendFeedbackToUser.ts +++ b/src/app/utils/sendFeedbackToUser.ts @@ -1,10 +1,10 @@ import type { Room } from '$types/matrix-sdk'; -import { DuplicateStrategy, EventType, MatrixEvent, MsgType } from '$types/matrix-sdk'; +import { DuplicateStrategy, MatrixEvent } from '$types/matrix-sdk'; export function sendFeedback(msg: string, room: Room, userId: string) { const localNotice = new MatrixEvent({ - type: EventType.RoomMessage, - content: { msgtype: MsgType.Notice, body: msg }, + type: 'm.room.message', + content: { msgtype: 'm.notice', body: msg }, event_id: `~sable-feedback-${Date.now()}`, room_id: room.roomId, sender: userId, From 0e4d1d31832cd642909b801be2410f12a8c76bf5 Mon Sep 17 00:00:00 2001 From: Erwan Leboucher Date: Sun, 2 Aug 2026 00:16:20 +0200 Subject: [PATCH 6/6] fix(timeline): refresh rows after decryption --- .../hooks/timeline/useTimelineSync.test.tsx | 26 +++++++++++++++++++ src/app/hooks/timeline/useTimelineSync.ts | 14 ++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/app/hooks/timeline/useTimelineSync.test.tsx b/src/app/hooks/timeline/useTimelineSync.test.tsx index aa0b29033a..53bc91659c 100644 --- a/src/app/hooks/timeline/useTimelineSync.test.tsx +++ b/src/app/hooks/timeline/useTimelineSync.test.tsx @@ -992,6 +992,32 @@ describe('live-arrive edge cases', () => { expect(result.current.timeline).not.toBe(before); }); + + it('re-renders when an event finishes decrypting', async () => { + const { room } = createRoom(); + const { result } = renderSyncHook(room); + const before = result.current.timeline; + + await act(async () => { + mxEmitter.emit(MatrixEventEvent.Decrypted, { getRoomId: () => room.roomId }); + await Promise.resolve(); + }); + + expect(result.current.timeline).not.toBe(before); + }); + + it('ignores decryption of an event in another room', async () => { + const { room } = createRoom(); + const { result } = renderSyncHook(room); + const before = result.current.timeline; + + await act(async () => { + mxEmitter.emit(MatrixEventEvent.Decrypted, { getRoomId: () => '!other:test' }); + await Promise.resolve(); + }); + + expect(result.current.timeline).toBe(before); + }); }); describe('event jump recovery', () => { diff --git a/src/app/hooks/timeline/useTimelineSync.ts b/src/app/hooks/timeline/useTimelineSync.ts index 190a8b8478..66f1a8c26a 100644 --- a/src/app/hooks/timeline/useTimelineSync.ts +++ b/src/app/hooks/timeline/useTimelineSync.ts @@ -580,6 +580,20 @@ export function useTimelineSync({ useMatrixEvent(room, RoomEvent.LocalEchoUpdated, handleLocalEchoUpdated); + const handleDecrypted = useCallback( + (mEvent: MatrixEvent) => { + if (mEvent.getRoomId() !== room.roomId) return; + setTimeline((ct) => ({ ...ct })); + }, + [room, setTimeline] + ); + + useMatrixEvent( + typeof (mx as Partial>).on === 'function' ? mx : undefined, + MatrixEventEvent.Decrypted, + handleDecrypted + ); + useLiveTimelineRefresh( room, useCallback(() => {