From 286dd2996ca1fe1aa0efa1dbd0696fa3e63acd0b Mon Sep 17 00:00:00 2001 From: TomOdellSheetMusic Date: Sun, 9 Aug 2026 23:32:44 +0200 Subject: [PATCH 1/8] add stream tiles in grid view --- src/grid/CallLayout.ts | 15 ++ src/grid/GridLayout.module.css | 12 +- src/grid/GridLayout.tsx | 28 ++- src/state/CallViewModel/CallViewModel.test.ts | 152 ++++++++-------- src/state/CallViewModel/CallViewModel.ts | 35 ++-- src/state/CallViewModel/LayoutSwitch.test.ts | 61 +------ src/state/CallViewModel/LayoutSwitch.ts | 27 +-- src/state/GridLikeLayout.ts | 2 +- src/state/TileStore.ts | 15 +- src/state/TileViewModel.ts | 8 +- src/state/layout-types.ts | 3 +- src/tile/GridTile.tsx | 167 ++++++++++++++++++ 12 files changed, 329 insertions(+), 196 deletions(-) diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts index 3128087bc4..e53c48b0d0 100644 --- a/src/grid/CallLayout.ts +++ b/src/grid/CallLayout.ts @@ -107,3 +107,18 @@ export function arrangeTiles( return { tileWidth, tileHeight, gap, columns }; } + +/** + * @param cameraCount - Number of regular participant tiles + * @param streamCount - Number of screen shares + */ +export function arrangeTilesWithStreams( + width: number, + minHeight: number, + cameraCount: number, + streamCount: number, +): GridArrangement { + // Each stream occupies a 2x2 block, so it consumes four regular cells worth of space. + const effectiveTileCount = cameraCount + streamCount * 4; + return arrangeTiles(width, minHeight, effectiveTileCount); +} diff --git a/src/grid/GridLayout.module.css b/src/grid/GridLayout.module.css index 984755d4cf..b22bcd05e6 100644 --- a/src/grid/GridLayout.module.css +++ b/src/grid/GridLayout.module.css @@ -11,8 +11,10 @@ Please see LICENSE in the repository root for full details. } .scrolling { - display: flex; - flex-wrap: wrap; + display: grid; + grid-template-columns: repeat(var(--columns), var(--width)); + grid-auto-rows: var(--height); + grid-auto-flow: dense; justify-content: center; align-content: center; gap: var(--gap); @@ -23,6 +25,12 @@ Please see LICENSE in the repository root for full details. height: var(--height); } +/* Larger tiles for Screen shares in Grids */ +.scrolling > .slot[data-stream="true"] { + grid-column: span 2; + grid-row: span 2; +} + .fixed { position: relative; } diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index 79c2b3a4a1..3d4cd1c224 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -17,13 +17,14 @@ import { useObservableEagerState } from "observable-hooks"; import { type GridLayout as GridLayoutModel } from "../state/layout-types.ts"; import styles from "./GridLayout.module.css"; import { useInitial } from "../useInitial"; -import { type CallLayout, arrangeTiles } from "./CallLayout"; +import { type CallLayout, arrangeTilesWithStreams } from "./CallLayout"; import { type DragCallback, useUpdateLayout, useVisibleTiles } from "./Grid"; interface GridCSSProperties extends CSSProperties { "--gap": string; "--width": string; "--height": string; + "--columns": string; } /** @@ -79,10 +80,18 @@ export const makeGridLayout: CallLayout = ({ useUpdateLayout(); useVisibleTiles(model.setVisibleTiles); const { width, height: minHeight } = useObservableEagerState(minBounds$); - const { gap, tileWidth, tileHeight } = useMemo( - () => arrangeTiles(width, minHeight, model.grid.length), - [width, minHeight, model.grid.length], - ); + // Screen shares are shown as larger 2x2 tiles + const { gap, tileWidth, tileHeight, columns } = useMemo(() => { + const streamCount = model.grid.filter( + (m) => m.media$.value.type === "screen share", + ).length; + return arrangeTilesWithStreams( + width, + minHeight, + model.grid.length - streamCount, + streamCount, + ); + }, [width, minHeight, model.grid]); return (
= ({ "--gap": `${gap}px`, "--width": `${Math.floor(tileWidth)}px`, "--height": `${Math.floor(tileHeight)}px`, + "--columns": `${columns}`, } as GridCSSProperties } > {model.grid.map((m) => ( - + ))}
); diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 440ae35e9f..53e86f8ff8 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -302,82 +302,81 @@ describe.each([ }); }); - test("remote screen sharing activates spotlight layout", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - // Start with no screen shares, then have Alice and Bob share their screens, - // then return to no screen shares, then have just Alice share for a bit - const aliceSharingInputMarbles = " ny-n--yn"; - const bobSharingInputMarbles = " n-y-n---"; - // While there are no screen shares, switch to spotlight manually, and then - // switch back to grid at the end - const modeInputMarbles = " -----s--g"; - // We should automatically enter spotlight for the first round of screen - // sharing, then return to grid, then manually go into spotlight, and - // remain in spotlight until we manually go back to grid - const expectedLayoutMarbles = " abcdaefeg"; - const expectedShowSpeakingMarbles = "y----nyny"; + test("remote screen sharing shows streams in grid", () => { + withTestScheduler(({ expectObservable }) => { + // Both Alice and Bob share their screens at the same time. withCallViewModel( { remoteParticipants$: constant([aliceParticipant, bobParticipant]), - rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), + rtcMembers$: constant([ + localRtcMember, + aliceRtcMember, + bobRtcMember, + ]), sharingScreen: new Map([ - [aliceParticipant, behavior(aliceSharingInputMarbles, yesNo)], - [bobParticipant, behavior(bobSharingInputMarbles, yesNo)], + [aliceParticipant, constant(true)], + [bobParticipant, constant(true)], ]), }, (vm) => { - schedule(modeInputMarbles, { + expectObservable(summarizeLayout$(vm.layout$)).toBe("a", { + a: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + `${aliceId}:0`, + `${bobId}:0`, + `${aliceId}:0:screen-share`, + `${bobId}:0:screen-share`, + ], + }, + }); + expectObservable(vm.showSpeakingIndicators$).toBe("y", yesNo); + }, + ); + }); + }); + + test("manually switching to spotlight still spotlights screen shares", () => { + withTestScheduler(({ schedule, expectObservable }) => { + // Alice shares her screen; the user manually switches to spotlight and + // back to grid. + withCallViewModel( + { + remoteParticipants$: constant([aliceParticipant, bobParticipant]), + rtcMembers$: constant([ + localRtcMember, + aliceRtcMember, + bobRtcMember, + ]), + sharingScreen: new Map([ + [aliceParticipant, constant(true)], + ]), + }, + (vm) => { + schedule(" s g", { s: () => vm.setGridMode("spotlight"), g: () => vm.setGridMode("grid"), }); - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - b: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0:screen-share`], - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - c: { - type: "spotlight-landscape", - spotlight: [ - `${aliceId}:0:screen-share`, - `${bobId}:0:screen-share`, - ], - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - d: { - type: "spotlight-landscape", - spotlight: [`${bobId}:0:screen-share`], - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - e: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0`], - grid: [`${localId}:0`, `${bobId}:0`], - }, - f: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0:screen-share`], - grid: [`${localId}:0`, `${bobId}:0`, `${aliceId}:0`], - }, - g: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${bobId}:0`, `${aliceId}:0`], - }, + expectObservable(summarizeLayout$(vm.layout$)).toBe("ba", { + a: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + `${aliceId}:0`, + `${bobId}:0`, + `${aliceId}:0:screen-share`, + ], }, - ); - expectObservable(vm.showSpeakingIndicators$).toBe( - expectedShowSpeakingMarbles, - yesNo, - ); + b: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0:screen-share`], + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + }); }, ); }); @@ -387,12 +386,16 @@ describe.each([ withTestScheduler(({ behavior, expectObservable }) => { // Local participant shares their screen, then stops sharing const sharingInputMarbles = " nyn"; - // Layout should show the screen share but stay in type: "grid" + // Layout should show the screen share as a grid tile but stay in grid const expectedLayoutMarbles = "aba"; withCallViewModel( { remoteParticipants$: constant([aliceParticipant, bobParticipant]), - rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), + rtcMembers$: constant([ + localRtcMember, + aliceRtcMember, + bobRtcMember, + ]), sharingScreen: new Map([ [localParticipant, behavior(sharingInputMarbles, yesNo)], ]), @@ -408,8 +411,13 @@ describe.each([ }, b: { type: "grid", - spotlight: [`${localId}:0:screen-share`], - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + spotlight: undefined, + grid: [ + `${localId}:0`, + `${aliceId}:0`, + `${bobId}:0`, + `${localId}:0:screen-share`, + ], }, }, ); @@ -443,8 +451,12 @@ describe.each([ }, b: { type: "grid", - spotlight: [`${localId}:0:screen-share`], - grid: [`${localId}:0`, `${aliceId}:0`], + spotlight: undefined, + grid: [ + `${localId}:0`, + `${aliceId}:0`, + `${localId}:0:screen-share`, + ], }, }, ); diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index d34e9160f8..8ce076025b 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -1016,14 +1016,6 @@ export function createCallViewModel$( ), ); - const hasRemoteScreenShares$ = scope.behavior( - spotlight$.pipe( - map((spotlight) => - spotlight.some((vm) => vm.type === "screen share" && !vm.local), - ), - ), - ); - const pipEnabled$ = scope.behavior(setPipEnabled$, false); const windowSize$ = @@ -1066,22 +1058,23 @@ export function createCallViewModel$( spotlightExpandedToggle$, ); - const { setGridMode, gridMode$ } = createLayoutModeSwitch( - scope, - windowMode$, - hasRemoteScreenShares$, - ); + const { setGridMode, gridMode$ } = createLayoutModeSwitch(scope, windowMode$); const gridLayoutMedia$: Observable = combineLatest( [grid$, spotlight$], - (grid, spotlight) => ({ - type: "grid", - edgeToEdge: false, - spotlight: spotlight.some((vm) => vm.type === "screen share") - ? spotlight - : undefined, - grid, - }), + (grid, spotlight) => { + // Screen shares are rendered as larger tiles inside the + // grid layout, so multiple screen shares can be seen at once. + // May be not elegant to get them from spotlight. + const screenShares = spotlight.filter( + (vm): vm is ScreenShareViewModel => vm.type === "screen share", + ); + return { + type: "grid", + edgeToEdge: false, + grid: [...grid, ...screenShares], + }; + }, ); const spotlightLandscapeLayoutMedia$ = ( diff --git a/src/state/CallViewModel/LayoutSwitch.test.ts b/src/state/CallViewModel/LayoutSwitch.test.ts index 0d184017b1..b2b976640e 100644 --- a/src/state/CallViewModel/LayoutSwitch.test.ts +++ b/src/state/CallViewModel/LayoutSwitch.test.ts @@ -12,12 +12,10 @@ import { testScope, withTestScheduler } from "../../utils/test"; function testLayoutSwitch({ windowMode = "n", - hasScreenShares = "n", userSelection = "", expectedGridMode, }: { windowMode?: string; - hasScreenShares?: string; userSelection?: string; expectedGridMode: string; }): void { @@ -25,7 +23,6 @@ function testLayoutSwitch({ const { gridMode$, setGridMode } = createLayoutModeSwitch( testScope(), behavior(windowMode, { n: "normal", N: "narrow", f: "flat" }), - behavior(hasScreenShares, { y: true, n: false }), ); schedule(userSelection, { g: () => setGridMode("grid"), @@ -57,49 +54,6 @@ test("allows switching modes manually", () => expectedGridMode: "g-sgs", })); -test("switches to spotlight mode when there is a remote screen share", () => - testLayoutSwitch({ - hasScreenShares: " n--y", - expectedGridMode: "g--s", - })); - -test("can manually switch to grid when there is a screenshare", () => - testLayoutSwitch({ - hasScreenShares: " n-y", - userSelection: " ---g", - expectedGridMode: "g-sg", - })); - -test("auto-switches after manually selecting grid", () => - testLayoutSwitch({ - // Two screenshares will happen in sequence. There is a screen share that - // forces spotlight, then the user manually switches back to grid. - hasScreenShares: " n-y-ny", - userSelection: " ---g", - expectedGridMode: "g-sg-s", - // If we did want to respect manual selection, the expectation would be: g-sg - })); - -test("switches back to grid mode when the remote screen share ends", () => - testLayoutSwitch({ - hasScreenShares: " n--y--n", - expectedGridMode: "g--s--g", - })); - -test("auto-switches to spotlight again after first screen share ends", () => - testLayoutSwitch({ - hasScreenShares: " nyny", - expectedGridMode: "gsgs", - })); - -test("switches manually to grid after screen share while manually in spotlight", () => - testLayoutSwitch({ - // Initially, no one is sharing. Then the user manually switches to spotlight. - // After a screen share starts, the user manually switches to grid. - hasScreenShares: " n-y", - userSelection: " -s-g", - expectedGridMode: "gs-g", - })); test("auto-switches to spotlight when in flat window mode", () => testLayoutSwitch({ @@ -117,16 +71,9 @@ test("allows switching modes manually when in flat window mode", () => expectedGridMode: "gsgsg", })); -test("stays in spotlight while there are screen shares even when window mode changes", () => - testLayoutSwitch({ - windowMode: " nfn", - hasScreenShares: " y", - expectedGridMode: "s", - })); - -test("ignores end of screen share until window mode returns to normal", () => +test("returns to grid mode when the window returns to a normal shape", () => testLayoutSwitch({ - windowMode: " nf-n", - hasScreenShares: " y-n", - expectedGridMode: "s--g", + // Window starts flat (spotlight), then returns to a normal shape. + windowMode: "f n", + expectedGridMode: "sg", })); diff --git a/src/state/CallViewModel/LayoutSwitch.ts b/src/state/CallViewModel/LayoutSwitch.ts index 97a4ee6fe4..5962a810a5 100644 --- a/src/state/CallViewModel/LayoutSwitch.ts +++ b/src/state/CallViewModel/LayoutSwitch.ts @@ -5,14 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - combineLatest, - map, - Subject, - startWith, - skipWhile, - switchMap, -} from "rxjs"; +import { map, Subject, startWith, skipWhile, switchMap } from "rxjs"; import { type GridMode, type WindowMode } from "./CallViewModel.ts"; import { constant, type Behavior } from "../Behavior.ts"; @@ -25,12 +18,10 @@ import { type ObservableScope } from "../ObservableScope.ts"; * * @param scope - The observable scope to manage subscriptions. * @param windowMode$ - The current window mode. - * @param hasRemoteScreenShares$ - A behavior indicating if there are remote screen shares active. */ export function createLayoutModeSwitch( scope: ObservableScope, windowMode$: Behavior, - hasRemoteScreenShares$: Behavior, ): { gridMode$: Behavior; setGridMode: (value: GridMode) => void; @@ -38,7 +29,7 @@ export function createLayoutModeSwitch( const userSelection$ = new Subject(); // Callback to set the grid mode desired by the user. // Notice that this is only a preference, the actual grid mode can be overridden - // if there is a remote screen share active. + // if the window mode is flat. const setGridMode = (value: GridMode): void => userSelection$.next(value); /** @@ -46,15 +37,11 @@ export function createLayoutModeSwitch( * not accounting for the user's manual selections. */ const naturalGridMode$ = scope.behavior( - combineLatest( - [hasRemoteScreenShares$, windowMode$], - (hasRemoteScreenShares, windowMode) => - // When there are screen shares or the window is flat (as with a phone - // in landscape orientation), spotlight is a better experience. - // We want screen shares to be big and readable, and we want flipping - // your phone into landscape to be a quick way of maximising the - // spotlight tile. - hasRemoteScreenShares || windowMode === "flat" ? "spotlight" : "grid", + // When the window is flat (as with a phone in landscape orientation), + // spotlight is a better experience: flipping your phone into landscape is + // a quick way of maximising the spotlight tile. + windowMode$.pipe( + map((windowMode) => (windowMode === "flat" ? "spotlight" : "grid")), ), ); diff --git a/src/state/GridLikeLayout.ts b/src/state/GridLikeLayout.ts index f91f8e310b..38c19317a4 100644 --- a/src/state/GridLikeLayout.ts +++ b/src/state/GridLikeLayout.ts @@ -31,7 +31,7 @@ export function gridLikeLayout( prevTiles: TileStore, ): [Layout & { type: GridLikeLayoutType }, TileStore] { const update = prevTiles.from(visibleTiles); - if (media.spotlight !== undefined) + if (media.type !== "grid") update.registerSpotlight( media.spotlight, media.type === "spotlight-portrait", diff --git a/src/state/TileStore.ts b/src/state/TileStore.ts index 132d1b9461..e70a75be07 100644 --- a/src/state/TileStore.ts +++ b/src/state/TileStore.ts @@ -13,7 +13,6 @@ import { fillGaps } from "../utils/iter"; import { debugTileLayout } from "../settings/settings"; import { type MediaViewModel } from "./media/MediaViewModel"; import { type UserMediaViewModel } from "./media/UserMediaViewModel"; -import { type RingingMediaViewModel } from "./media/RingingMediaViewModel"; type SpotlightBackground = "solid" | "transparent"; @@ -68,19 +67,17 @@ class SpotlightTileData { } class GridTileData { - private readonly media$: BehaviorSubject< - UserMediaViewModel | RingingMediaViewModel - >; - public get media(): UserMediaViewModel | RingingMediaViewModel { + private readonly media$: BehaviorSubject; + public get media(): MediaViewModel { return this.media$.value; } - public set media(value: UserMediaViewModel) { + public set media(value: MediaViewModel) { this.media$.next(value); } public readonly vm: GridTileViewModel; - public constructor(media: UserMediaViewModel | RingingMediaViewModel) { + public constructor(media: MediaViewModel) { this.media$ = new BehaviorSubject(media); this.vm = new GridTileViewModel(this.media$); } @@ -205,9 +202,7 @@ export class TileStoreBuilder { * Sets up a grid tile for the given media. If this is never called for some * media, then that media will have no grid tile. */ - public registerGridTile( - media: UserMediaViewModel | RingingMediaViewModel, - ): void { + public registerGridTile(media: MediaViewModel): void { if (DEBUG_ENABLED) logger.debug( `[TileStore, ${this.generation}] register grid tile: ${media.displayName$.value}`, diff --git a/src/state/TileViewModel.ts b/src/state/TileViewModel.ts index 6a5d9175da..bdc06b7e2d 100644 --- a/src/state/TileViewModel.ts +++ b/src/state/TileViewModel.ts @@ -9,8 +9,6 @@ import { BehaviorSubject } from "rxjs"; import { type Behavior } from "./Behavior"; import { type MediaViewModel } from "./media/MediaViewModel"; -import { type RingingMediaViewModel } from "./media/RingingMediaViewModel"; -import { type UserMediaViewModel } from "./media/UserMediaViewModel"; let nextId = 0; function createId(): string { @@ -22,11 +20,7 @@ export class GridTileViewModel { private readonly _showOutline$ = new BehaviorSubject(false); public readonly showOutline$: Behavior = this._showOutline$; - public constructor( - public readonly media$: Behavior< - UserMediaViewModel | RingingMediaViewModel - >, - ) {} + public constructor(public readonly media$: Behavior) {} public setShowOutline(value: boolean): void { this._showOutline$.next(value); diff --git a/src/state/layout-types.ts b/src/state/layout-types.ts index 2b0d459daa..7f5a0dbed2 100644 --- a/src/state/layout-types.ts +++ b/src/state/layout-types.ts @@ -20,8 +20,7 @@ import { type Behavior } from "./Behavior.ts"; export interface GridLayoutMedia { type: "grid"; edgeToEdge: false; - spotlight?: MediaViewModel[]; - grid: UserMediaViewModel[]; + grid: MediaViewModel[]; } export interface SpotlightLandscapeLayoutMedia { diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 1544e41da1..3b24d83715 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -27,6 +27,7 @@ import { MicrophoneSlash, DotsThreeOutline, Eye, + Monitor, } from "@phosphor-icons/react"; import { ContextMenu, @@ -48,6 +49,8 @@ import { useBehavior } from "../useBehavior"; import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel"; import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel"; import { type UserMediaViewModel } from "../state/media/UserMediaViewModel"; +import { type ScreenShareViewModel } from "../state/media/ScreenShareViewModel"; +import { type RemoteScreenShareViewModel } from "../state/media/RemoteScreenShareViewModel"; import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel"; import { RingingStatus } from "./RingingStatus"; @@ -402,6 +405,159 @@ const RemoteUserMediaTile: FC = ({ RemoteUserMediaTile.displayName = "RemoteUserMediaTile"; +interface ScreenShareTileProps extends TileProps { + vm: ScreenShareViewModel; +} + +/** + * New Tile for screen sharing participants. + */ +const ScreenShareTile: FC = (props) => { + const { vm, ...rest } = props; + return vm.local ? ( + + ) : ( + + ); +}; + +const RemoteScreenShareTileContent: FC< + Omit & { vm: RemoteScreenShareViewModel } +> = ({ vm, ...props }) => { + const { t } = useTranslation(); + const videoEnabled = useBehavior(vm.videoEnabled$); + const playbackMuted = useBehavior(vm.playbackMuted$); + const playbackVolume = useBehavior(vm.playbackVolume$); + + const onSelectMute = useCallback( + (e: Event) => { + e.preventDefault(); + vm.togglePlaybackMuted(); + }, + [vm], + ); + + const VolumeIcon = playbackMuted ? SpeakerSlash : SpeakerHigh; + + return ( + + + {/* TODO: Figure out how to make this slider keyboard accessible */} + + + + + } + /> + ); +}; + +RemoteScreenShareTileContent.displayName = "RemoteScreenShareTileContent"; + +interface ScreenShareTileContentProps extends ScreenShareTileProps { + videoEnabled: boolean; + menu?: ReactNode; +} + +const ScreenShareTileContent: FC = ({ + ref, + vm, + videoEnabled, + menu, + className, + focusable, + targetWidth, + targetHeight, + displayName, + mxcAvatarUrl, + ...props +}) => { + const { t } = useTranslation(); + const video = useBehavior(vm.video$); + const unencryptedWarning = useBehavior(vm.unencryptedWarning$); + const focusUrl = useBehavior(vm.focusUrl$); + const [menuOpen, setMenuOpen] = useState(false); + + const tile = ( + } + displayName={displayName} + mxcAvatarUrl={mxcAvatarUrl} + focusable={focusable} + primaryButton={ + menu === undefined ? undefined : ( + + + + } + side="left" + align="start" + > + {menu} + + ) + } + focusUrl={focusUrl} + targetWidth={targetWidth} + targetHeight={targetHeight} + {...props} + /> + ); + + return menu === undefined ? ( + tile + ) : ( + + {menu} + + ); +}; + +ScreenShareTileContent.displayName = "ScreenShareTileContent"; + interface GridTileProps { ref?: Ref; vm: GridTileViewModel; @@ -445,6 +601,17 @@ export const GridTile: FC = ({ {...props} /> ); + } else if (media.type === "screen share") { + return ( + + ); } else if (media.local) { return ( Date: Mon, 10 Aug 2026 00:29:42 +0200 Subject: [PATCH 2/8] add fullscreen button for screen shares --- src/grid/GridLayout.module.css | 14 ++- src/grid/GridLayout.tsx | 9 +- src/room/InCallView.tsx | 2 + src/state/CallViewModel/CallViewModel.test.ts | 64 +++++++++++++ src/state/CallViewModel/CallViewModel.ts | 37 +++++++- src/state/GridLikeLayout.ts | 1 + src/state/layout-types.ts | 2 + src/tile/GridTile.module.css | 10 ++ src/tile/GridTile.tsx | 92 ++++++++++++++----- 9 files changed, 203 insertions(+), 28 deletions(-) diff --git a/src/grid/GridLayout.module.css b/src/grid/GridLayout.module.css index b22bcd05e6..d776529591 100644 --- a/src/grid/GridLayout.module.css +++ b/src/grid/GridLayout.module.css @@ -21,8 +21,8 @@ Please see LICENSE in the repository root for full details. } .scrolling > .slot { - width: var(--width); - height: var(--height); + min-width: 0; + min-height: 0; } /* Larger tiles for Screen shares in Grids */ @@ -31,6 +31,16 @@ Please see LICENSE in the repository root for full details. grid-row: span 2; } +/* Focused tile takes up the entire grid area */ +.scrolling.focused { + display: block; +} + +.scrolling.focused > .slot { + width: 100%; + height: 100%; +} + .fixed { position: relative; } diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index 3d4cd1c224..531a612ff5 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -13,6 +13,7 @@ import { } from "react"; import { distinctUntilChanged } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; +import classNames from "classnames"; import { type GridLayout as GridLayoutModel } from "../state/layout-types.ts"; import styles from "./GridLayout.module.css"; @@ -96,7 +97,9 @@ export const makeGridLayout: CallLayout = ({ return (
= ({ {model.grid.map((m) => ( = ({ showRingingStatus={showRingingStatus} showOutline={showOutline} focusable={!contentObscured} + focusedStream$={vm.focusedStream$} + onToggleFocusedStream={vm.setFocusedStream} /> ) : ( ): Observable { type: l.type, spotlight: spotlight?.map((vm) => vm.id), grid: grid.map((vm) => vm.id), + ...(l.focused ? { focused: true as const } : {}), }), ); case "spotlight-landscape": @@ -382,6 +384,68 @@ describe.each([ }); }); + test("focused stream fills the grid and hides other tiles", () => { + withTestScheduler(({ schedule, expectObservable }) => { + withCallViewModel( + { + remoteParticipants$: constant([aliceParticipant, bobParticipant]), + rtcMembers$: constant([ + localRtcMember, + aliceRtcMember, + bobRtcMember, + ]), + sharingScreen: new Map([ + [aliceParticipant, constant(true)], + [bobParticipant, constant(true)], + ]), + }, + (vm) => { + // Focus Alice's screen share using the live view model from the + // current layout, then unfocus it again. + const focusAlice = (): void => { + const layout = vm.layout$.value; + if (layout.type !== "grid") return; + const share = layout.grid + .map((tile) => tile.media$.value) + .find( + (m) => + m.type === "screen share" && + m.id === `${aliceId}:0:screen-share`, + ); + if (share !== undefined && share.type === "screen share") + vm.setFocusedStream(share); + }; + schedule(" f u", { + f: focusAlice, + u: (): void => vm.setFocusedStream(null), + }); + + expectObservable(summarizeLayout$(vm.layout$)).toBe("ba", { + a: { + type: "grid", + spotlight: undefined, + grid: [ + // After unfocusing, the TileStore keeps the previously focused + // stream tile in its spot (index 0) and appends the rest. + `${aliceId}:0:screen-share`, + `${localId}:0`, + `${aliceId}:0`, + `${bobId}:0`, + `${bobId}:0:screen-share`, + ], + }, + b: { + type: "grid", + focused: true, + spotlight: undefined, + grid: [`${aliceId}:0:screen-share`], + }, + }); + }, + ); + }); + }); + test("local screen sharing stays in grid layout", () => { withTestScheduler(({ behavior, expectObservable }) => { // Local participant shares their screen, then stops sharing diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 8ce076025b..a1f8126fce 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -357,6 +357,8 @@ export interface CallViewModel { toggleSpotlightExpanded$: Behavior<(() => void) | null>; gridMode$: Behavior; setGridMode: (value: GridMode) => void; + focusedStream$: Behavior; + setFocusedStream: (vm: ScreenShareViewModel | null) => void; // header/footer visibility showHeader$: Behavior; @@ -1060,9 +1062,37 @@ export function createCallViewModel$( const { setGridMode, gridMode$ } = createLayoutModeSwitch(scope, windowMode$); + // A single screen share can be focused (maximised) to fill the grid + const focusedStreamRequest$ = new Subject(); + const focusedStream$ = scope.behavior( + focusedStreamRequest$.pipe( + startWith(null), + switchMap((requested) => + requested === null + ? of(null) + : screenShares$.pipe( + map( + (shares) => + shares.find((s) => s.id === requested.id) ?? null, + ), + distinctUntilChanged(), + ), + ), + ), + ); + const setFocusedStream = (requested: ScreenShareViewModel | null): void => + focusedStreamRequest$.next(requested); + const gridLayoutMedia$: Observable = combineLatest( - [grid$, spotlight$], - (grid, spotlight) => { + [grid$, spotlight$, focusedStream$], + (grid, spotlight, focusedStream) => { + if (focusedStream !== null) + return { + type: "grid", + edgeToEdge: false, + focused: true, + grid: [focusedStream], + }; // Screen shares are rendered as larger tiles inside the // grid layout, so multiple screen shares can be seen at once. // May be not elegant to get them from spotlight. @@ -1072,6 +1102,7 @@ export function createCallViewModel$( return { type: "grid", edgeToEdge: false, + focused: false, grid: [...grid, ...screenShares], }; }, @@ -1774,6 +1805,8 @@ export function createCallViewModel$( toggleSpotlightExpanded$: toggleSpotlightExpanded$, gridMode$: gridMode$, setGridMode: setGridMode, + focusedStream$, + setFocusedStream, layout$: layout$, localMatrixLivekitMember$, remoteMatrixLivekitMembers$: scope.behavior( diff --git a/src/state/GridLikeLayout.ts b/src/state/GridLikeLayout.ts index 38c19317a4..3adda0efdc 100644 --- a/src/state/GridLikeLayout.ts +++ b/src/state/GridLikeLayout.ts @@ -44,6 +44,7 @@ export function gridLikeLayout( type: media.type, spotlight: tiles.spotlightTile, grid: tiles.gridTiles, + focused: media.type === "grid" ? media.focused ?? false : undefined, spotlightAlignment$, setVisibleTiles, } as Layout & { type: GridLikeLayoutType }, diff --git a/src/state/layout-types.ts b/src/state/layout-types.ts index 7f5a0dbed2..d813d99b8f 100644 --- a/src/state/layout-types.ts +++ b/src/state/layout-types.ts @@ -21,6 +21,7 @@ export interface GridLayoutMedia { type: "grid"; edgeToEdge: false; grid: MediaViewModel[]; + focused?: boolean; } export interface SpotlightLandscapeLayoutMedia { @@ -84,6 +85,7 @@ export interface GridLayout { grid: GridTileViewModel[]; spotlightAlignment$: BehaviorSubject; setVisibleTiles: (value: number) => void; + focused?: boolean; } export interface SpotlightLandscapeLayout { diff --git a/src/tile/GridTile.module.css b/src/tile/GridTile.module.css index 3ebb9bf757..1c3d50a226 100644 --- a/src/tile/GridTile.module.css +++ b/src/tile/GridTile.module.css @@ -94,6 +94,16 @@ borders don't support gradients */ width: 100%; } +.maximise { + display: flex; + align-items: center; +} + +.maximise > svg { + display: block; + color: var(--cpd-color-icon-primary); +} + .tile .switchCamera { opacity: 1; background: var(--cpd-color-bg-action-secondary-rest); diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 3b24d83715..a6f7f9f82d 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -36,6 +36,10 @@ import { Menu, Text, } from "@vector-im/compound-web"; +import { + ExpandIcon, + CollapseIcon, +} from "@vector-im/compound-design-tokens/assets/web/icons"; import { useObservableEagerState } from "observable-hooks"; import styles from "./GridTile.module.css"; @@ -52,6 +56,7 @@ import { type UserMediaViewModel } from "../state/media/UserMediaViewModel"; import { type ScreenShareViewModel } from "../state/media/ScreenShareViewModel"; import { type RemoteScreenShareViewModel } from "../state/media/RemoteScreenShareViewModel"; import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel"; +import { constant, type Behavior } from "../state/Behavior"; import { RingingStatus } from "./RingingStatus"; interface TileProps { @@ -407,6 +412,16 @@ RemoteUserMediaTile.displayName = "RemoteUserMediaTile"; interface ScreenShareTileProps extends TileProps { vm: ScreenShareViewModel; + /** + * The currently focused (maximised) stream, used to decide whether this tile + * shows a "maximise" or "restore" button. + */ + focusedStream$?: Behavior; + /** + * Focuses (maximises) the given stream so it fills the grid and hides every + * other tile, or unfocuses when passed null. + */ + onToggleFocusedStream?: (vm: ScreenShareViewModel | null) => void; } /** @@ -483,6 +498,8 @@ const ScreenShareTileContent: FC = ({ vm, videoEnabled, menu, + focusedStream$, + onToggleFocusedStream, className, focusable, targetWidth, @@ -496,6 +513,10 @@ const ScreenShareTileContent: FC = ({ const unencryptedWarning = useBehavior(vm.unencryptedWarning$); const focusUrl = useBehavior(vm.focusUrl$); const [menuOpen, setMenuOpen] = useState(false); + const focusedStream = useBehavior(focusedStream$ ?? constant(null)); + const isFocused = focusedStream?.id === vm.id; + + const FocusIcon = isFocused ? CollapseIcon : ExpandIcon; const tile = ( = ({ mxcAvatarUrl={mxcAvatarUrl} focusable={focusable} primaryButton={ - menu === undefined ? undefined : ( - + {onToggleFocusedStream !== undefined && ( - } - side="left" - align="start" - > - {menu} - + )} + {menu !== undefined && ( + + + + } + side="left" + align="start" + > + {menu} + + )} + ) } focusUrl={focusUrl} @@ -571,6 +613,8 @@ interface GridTileProps { showRingingStatus: boolean; showOutline: boolean; focusable: boolean; + focusedStream$?: Behavior; + onToggleFocusedStream?: (vm: ScreenShareViewModel | null) => void; } export const GridTile: FC = ({ @@ -580,6 +624,8 @@ export const GridTile: FC = ({ showRingingStatus, showOutline, onOpenProfile, + focusedStream$, + onToggleFocusedStream, className, ...props }) => { @@ -606,6 +652,8 @@ export const GridTile: FC = ({ Date: Mon, 10 Aug 2026 01:03:06 +0200 Subject: [PATCH 3/8] fix: screen shares occupying user grid slots --- src/grid/GridLayout.module.css | 2 +- src/grid/GridLayout.tsx | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/grid/GridLayout.module.css b/src/grid/GridLayout.module.css index d776529591..febe694551 100644 --- a/src/grid/GridLayout.module.css +++ b/src/grid/GridLayout.module.css @@ -14,7 +14,7 @@ Please see LICENSE in the repository root for full details. display: grid; grid-template-columns: repeat(var(--columns), var(--width)); grid-auto-rows: var(--height); - grid-auto-flow: dense; + grid-auto-flow: row; justify-content: center; align-content: center; gap: var(--gap); diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index 531a612ff5..5d48c2731b 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -94,6 +94,15 @@ export const makeGridLayout: CallLayout = ({ ); }, [width, minHeight, model.grid]); + // Render camera tiles before screen shares + const orderedTiles = useMemo( + () => [ + ...model.grid.filter((m) => m.media$.value.type !== "screen share"), + ...model.grid.filter((m) => m.media$.value.type === "screen share"), + ], + [model.grid], + ); + return (
= ({ } as GridCSSProperties } > - {model.grid.map((m) => ( + {orderedTiles.map((m) => ( Date: Mon, 10 Aug 2026 01:17:04 +0200 Subject: [PATCH 4/8] add stop watching screen shares button --- locales/de/app.json | 4 +- locales/en/app.json | 4 +- src/state/media/ScreenShareViewModel.ts | 8 +++ src/tile/GridTile.module.css | 33 ++++++++++++ src/tile/GridTile.tsx | 67 ++++++++++++++++++++++++- src/tile/MediaView.module.css | 15 ++++++ src/tile/MediaView.tsx | 5 ++ 7 files changed, 133 insertions(+), 3 deletions(-) diff --git a/locales/de/app.json b/locales/de/app.json index 32ee930cdf..86ba77d474 100644 --- a/locales/de/app.json +++ b/locales/de/app.json @@ -252,6 +252,8 @@ "muted_for_me": "Für mich stumm geschaltet", "screen_share_volume": "Lautstärke der Bildschirmfreigabe", "volume": "Lautstärke", - "waiting_for_media": "Warten auf Medien..." + "waiting_for_media": "Warten auf Medien...", + "stop_watching": "Nicht mehr zuschauen", + "watch_stream": "Stream zuschauen" } } diff --git a/locales/en/app.json b/locales/en/app.json index c5f8b34320..c0788cb553 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -291,6 +291,8 @@ "muted_for_me": "Muted for me", "screen_share_volume": "Screen share volume", "volume": "Volume", - "waiting_for_media": "Waiting for media..." + "waiting_for_media": "Waiting for media...", + "stop_watching": "Stop watching", + "watch_stream": "Watch stream" } } diff --git a/src/state/media/ScreenShareViewModel.ts b/src/state/media/ScreenShareViewModel.ts index 8336f0a6ba..adb2b16b92 100644 --- a/src/state/media/ScreenShareViewModel.ts +++ b/src/state/media/ScreenShareViewModel.ts @@ -7,7 +7,9 @@ Please see LICENSE in the repository root for full details. */ import { Track } from "livekit-client"; +import { Subject, startWith } from "rxjs"; +import { type Behavior } from "../Behavior"; import { type ObservableScope } from "../ObservableScope"; import { type LocalScreenShareViewModel } from "./LocalScreenShareViewModel"; import { @@ -29,6 +31,8 @@ export type ScreenShareViewModel = */ export interface BaseScreenShareViewModel extends BaseMemberMediaViewModel { type: "screen share"; + watching$: Behavior; + setWatching: (watching: boolean) => void; } export type BaseScreenShareInputs = Omit< @@ -40,6 +44,8 @@ export function createBaseScreenShare( scope: ObservableScope, inputs: BaseScreenShareInputs, ): BaseScreenShareViewModel { + const watchingRequest$ = new Subject(); + const watching$ = scope.behavior(watchingRequest$.pipe(startWith(true))); return { ...createMemberMedia(scope, { ...inputs, @@ -47,5 +53,7 @@ export function createBaseScreenShare( videoSource: Track.Source.ScreenShare, }), type: "screen share", + watching$, + setWatching: (watching: boolean): void => watchingRequest$.next(watching), }; } diff --git a/src/tile/GridTile.module.css b/src/tile/GridTile.module.css index 1c3d50a226..977f83c1a1 100644 --- a/src/tile/GridTile.module.css +++ b/src/tile/GridTile.module.css @@ -104,6 +104,39 @@ borders don't support gradients */ color: var(--cpd-color-icon-primary); } +/* The "Watch stream" button shown on a stopped screen share. */ +.watchStream { + appearance: none; + border: none; + border-radius: var(--cpd-radius-pill-effect); + padding: var(--cpd-space-3x) var(--cpd-space-5x); + background: var(--cpd-color-bg-action-primary-rest); + color: var(--cpd-color-text-on-primary); + box-shadow: var(--small-drop-shadow); + cursor: pointer; + display: flex; + align-items: center; + gap: var(--cpd-space-2x); + font: inherit; + font-weight: 600; + font-size: var(--cpd-font-size-body-lg); +} + +.watchStream > svg { + display: block; + color: var(--cpd-color-text-on-primary); +} + +@media (hover) { + .watchStream:hover { + background: var(--cpd-color-bg-action-primary-hovered); + } +} + +.watchStream:active { + background: var(--cpd-color-bg-action-primary-pressed); +} + .tile .switchCamera { opacity: 1; background: var(--cpd-color-bg-action-secondary-rest); diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index a6f7f9f82d..2fe163b825 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -27,7 +27,9 @@ import { MicrophoneSlash, DotsThreeOutline, Eye, + EyeSlash, Monitor, + Play, } from "@phosphor-icons/react"; import { ContextMenu, @@ -443,6 +445,7 @@ const RemoteScreenShareTileContent: FC< const videoEnabled = useBehavior(vm.videoEnabled$); const playbackMuted = useBehavior(vm.playbackMuted$); const playbackVolume = useBehavior(vm.playbackVolume$); + const watching = useBehavior(vm.watching$); const onSelectMute = useCallback( (e: Event) => { @@ -452,6 +455,14 @@ const RemoteScreenShareTileContent: FC< [vm], ); + const onSelectWatching = useCallback( + (e: Event) => { + e.preventDefault(); + vm.setWatching(!watching); + }, + [vm, watching], + ); + const VolumeIcon = playbackMuted ? SpeakerSlash : SpeakerHigh; return ( @@ -461,6 +472,16 @@ const RemoteScreenShareTileContent: FC< {...props} menu={ <> + = ({ const video = useBehavior(vm.video$); const unencryptedWarning = useBehavior(vm.unencryptedWarning$); const focusUrl = useBehavior(vm.focusUrl$); + const watching = useBehavior(vm.watching$); const [menuOpen, setMenuOpen] = useState(false); const focusedStream = useBehavior(focusedStream$ ?? constant(null)); const isFocused = focusedStream?.id === vm.id; + // A ref to the tile root so we can freeze the video element when the user + // stops watching the stream. + const contentRef = useRef(null); + const mergedRef = useMergedRefs(contentRef, ref); + + // Freeze the video (pause it) while not watching, and resume when watching. + // While stopped we also watch for new video elements (e.g. LiveKit + // re-attaching) and pause those too. + useEffect(() => { + const root = contentRef.current; + if (root === null) return; + const apply = (): void => { + root.querySelectorAll("video").forEach((v) => { + if (watching) void v.play().catch(() => {}); + else v.pause(); + }); + }; + apply(); + if (watching) return; + const observer = new MutationObserver(apply); + observer.observe(root, { childList: true, subtree: true }); + return (): void => observer.disconnect(); + }, [watching]); + const FocusIcon = isFocused ? CollapseIcon : ExpandIcon; const tile = ( { + vm.setWatching(true); + // Resume playback within the click gesture. + contentRef.current + ?.querySelectorAll("video") + .forEach((v) => void v.play().catch(() => {})); + }} + tabIndex={focusable ? undefined : -1} + > + + {t("video_tile.watch_stream")} + + ) + } userId={vm.userId} unencryptedWarning={unencryptedWarning} videoEnabled={videoEnabled} diff --git a/src/tile/MediaView.module.css b/src/tile/MediaView.module.css index 13d0fd1b1b..9ea287d5a1 100644 --- a/src/tile/MediaView.module.css +++ b/src/tile/MediaView.module.css @@ -16,6 +16,21 @@ Please see LICENSE in the repository root for full details. place-items: stretch; } +.streamOverlay { + grid-area: content; + place-self: stretch; + z-index: 1; + display: grid; + place-items: center; + background: rgb(0 0 0 / 0.35); + backdrop-filter: blur(10px); + pointer-events: none; +} + +.streamOverlay > * { + pointer-events: auto; +} + .media video { inline-size: 100%; block-size: 100%; diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index 4035eec8ce..c1a3425af6 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -55,6 +55,7 @@ interface Props extends ComponentProps { rtcBackendIdentity?: string; // The focus url, mainly for debugging purposes focusUrl?: string; + streamOverlay?: ReactNode; } export const MediaView: FC = ({ @@ -85,6 +86,7 @@ export const MediaView: FC = ({ videoStreamStats, rtcBackendIdentity, focusUrl, + streamOverlay, ...props }) => { const { t } = useTranslation(); @@ -211,6 +213,9 @@ export const MediaView: FC = ({ )} {primaryButton}
+ {streamOverlay !== undefined && ( +
{streamOverlay}
+ )} ); }; From d3cd81c1673369d824b8ac0bbcf7fed5bbc55bd4 Mon Sep 17 00:00:00 2001 From: TomOdellSheetMusic Date: Mon, 10 Aug 2026 01:26:45 +0200 Subject: [PATCH 5/8] fix: frozen frame overlay uses renamed streamOverlay class --- src/tile/MediaView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index c1a3425af6..5b51ec35c6 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -214,7 +214,7 @@ export const MediaView: FC = ({ {primaryButton}
{streamOverlay !== undefined && ( -
{streamOverlay}
+
{streamOverlay}
)} ); From 51571570104a9ad92bb4ff2ea2475a8ca366bc12 Mon Sep 17 00:00:00 2001 From: TomOdellSheetMusic Date: Mon, 10 Aug 2026 02:23:20 +0200 Subject: [PATCH 6/8] Stop streaming data when not watching screen shares --- src/state/media/MediaViewModel.test.ts | 82 +++++++++++++++++++ src/state/media/RemoteScreenShareViewModel.ts | 56 +++++++++++-- src/tile/GridTile.module.css | 32 +++++++- src/tile/GridTile.tsx | 61 ++++++++++---- src/tile/MediaView.module.css | 6 -- 5 files changed, 210 insertions(+), 27 deletions(-) diff --git a/src/state/media/MediaViewModel.test.ts b/src/state/media/MediaViewModel.test.ts index 9d873ccba2..ba06a23d76 100644 --- a/src/state/media/MediaViewModel.test.ts +++ b/src/state/media/MediaViewModel.test.ts @@ -9,6 +9,8 @@ import { expect, onTestFinished, test, vi } from "vitest"; import { type LocalTrackPublication, LocalVideoTrack, + ParticipantEvent, + RemoteTrackPublication, Track, TrackEvent, } from "livekit-client"; @@ -160,6 +162,86 @@ test("control a participant's screen share volume", () => { }); }); +test("stop watching a remote screen share actually unsubscribes from the LiveKit track", () => { + const videoPublication = new RemoteTrackPublication( + Track.Kind.Video, + { + sid: "TR_screen", + name: "screen", + muted: false, + } as unknown as ConstructorParameters[1], + true, + {}, + ); + const audioPublication = new RemoteTrackPublication( + Track.Kind.Audio, + { + sid: "TR_screen_audio", + name: "screen_audio", + muted: false, + } as unknown as ConstructorParameters[1], + true, + {}, + ); + const setVideoSubscribedSpy = vi.spyOn(videoPublication, "setSubscribed"); + const setAudioSubscribedSpy = vi.spyOn(audioPublication, "setSubscribed"); + const vm = mockRemoteScreenShare( + rtcMembership, + {}, + mockRemoteParticipant({ + getTrackPublication: (source) => { + if (source === Track.Source.ScreenShare) return videoPublication; + if (source === Track.Source.ScreenShareAudio) return audioPublication; + return undefined; + }, + }), + ); + + // Watching starts out enabled, so we should be subscribed to both the video + // and the screen share audio track. + expect(setVideoSubscribedSpy).toHaveBeenCalledWith(true); + expect(setAudioSubscribedSpy).toHaveBeenCalledWith(true); + + // Stopping watching should unsubscribe both so that the data stops flowing. + vm.setWatching(false); + expect(setVideoSubscribedSpy).toHaveBeenLastCalledWith(false); + expect(setAudioSubscribedSpy).toHaveBeenLastCalledWith(false); + + // Watching again should resubscribe both. + vm.setWatching(true); + expect(setVideoSubscribedSpy).toHaveBeenLastCalledWith(true); + expect(setAudioSubscribedSpy).toHaveBeenLastCalledWith(true); +}); + +test("screen share mute is re-applied when the audio track is re-subscribed", () => { + const setVolumeSpy = vi.fn(); + const participant = mockRemoteParticipant({ setVolume: setVolumeSpy }); + const vm = mockRemoteScreenShare(rtcMembership, {}, participant); + + // Muting should set the screen share audio volume to 0. + vm.togglePlaybackMuted(); + expect(setVolumeSpy).toHaveBeenLastCalledWith( + 0, + Track.Source.ScreenShareAudio, + ); + + // Simulate the audio track being re-attached (e.g. after the user resumes + // watching): the current volume must be re-applied, otherwise the mute + // would be lost and the sound would come back. + const callsBefore = setVolumeSpy.mock.calls.length; + ( + participant.emit as unknown as ( + event: string, + ...args: unknown[] + ) => boolean + )(ParticipantEvent.TrackSubscribed, {}); + expect(setVolumeSpy.mock.calls.length).toBeGreaterThan(callsBefore); + expect(setVolumeSpy).toHaveBeenLastCalledWith( + 0, + Track.Source.ScreenShareAudio, + ); +}); + test("local media remembers whether it should always be shown", () => { const vm1 = mockLocalMedia( rtcMembership, diff --git a/src/state/media/RemoteScreenShareViewModel.ts b/src/state/media/RemoteScreenShareViewModel.ts index cc3221cfa3..7477281955 100644 --- a/src/state/media/RemoteScreenShareViewModel.ts +++ b/src/state/media/RemoteScreenShareViewModel.ts @@ -6,8 +6,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { Track, type RemoteParticipant } from "livekit-client"; -import { map, of, switchMap } from "rxjs"; +import { + ParticipantEvent, + RemoteTrackPublication, + Track, + type RemoteParticipant, +} from "livekit-client"; +import { observeParticipantEvents } from "@livekit/components-core"; +import { combineLatest, distinctUntilChanged, map, of, switchMap } from "rxjs"; import { type Behavior } from "../Behavior"; import { @@ -43,17 +49,57 @@ export function createRemoteScreenShare( scope: ObservableScope, { pretendToBeDisconnected$, ...inputs }: RemoteScreenShareInputs, ): RemoteScreenShareViewModel { + const base = createBaseScreenShare(scope, inputs); + + // The screen share's video and audio publications. + const videoPublication$ = base.video$.pipe(map((ref) => ref?.publication)); + const audioPublication$ = inputs.participant$.pipe( + switchMap((p) => + p ? observeTrackReference$(p, Track.Source.ScreenShareAudio) : of(undefined), + ), + map((ref) => ref?.publication), + ); + combineLatest([base.watching$, videoPublication$, audioPublication$]) + .pipe( + scope.bind(), + distinctUntilChanged( + ([watching, video, audio], [nextWatching, nextVideo, nextAudio]) => + watching === nextWatching && + video === nextVideo && + audio === nextAudio, + ), + ) + .subscribe(([watching, video, audio]) => { + for (const publication of [video, audio]) { + if (publication instanceof RemoteTrackPublication) + publication.setSubscribed(watching); + } + }); + + // Emits whenever any remote track subscribes or unsubscribes. + const audioTrackEvents$ = inputs.participant$.pipe( + switchMap((p) => + p === null + ? of(undefined) + : observeParticipantEvents( + p, + ParticipantEvent.TrackSubscribed, + ParticipantEvent.TrackUnsubscribed, + ), + ), + ); + // Screen share audio gets its own saved volume, separate from the // participant's voice volume. const savedVolumeKey = `${inputs.rtcBackendIdentity}:screen-share`; return { - ...createBaseScreenShare(scope, inputs), + ...base, ...createVolumeControls(scope, { pretendToBeDisconnected$, sink$: scope.behavior( - inputs.participant$.pipe( + combineLatest([inputs.participant$, audioTrackEvents$]).pipe( map( - (p) => (volume) => + ([p]) => (volume) => p?.setVolume(volume, Track.Source.ScreenShareAudio), ), ), diff --git a/src/tile/GridTile.module.css b/src/tile/GridTile.module.css index 977f83c1a1..2d67e33117 100644 --- a/src/tile/GridTile.module.css +++ b/src/tile/GridTile.module.css @@ -104,7 +104,36 @@ borders don't support gradients */ color: var(--cpd-color-icon-primary); } -/* The "Watch stream" button shown on a stopped screen share. */ +.streamOverlayInner { + position: relative; + width: 100%; + height: 100%; + display: grid; + place-items: center; +} + +.streamOverlayInner > * { + pointer-events: auto; +} + +.streamOverlayScrim { + position: absolute; + inset: 0; + background: rgb(0 0 0 / 0.35); + backdrop-filter: blur(10px); +} + +.frozenFrame { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + object-fit: contain; + background: var(--cpd-color-bg-canvas-default); + filter: blur(8px); + transform: scale(1.05); +} + .watchStream { appearance: none; border: none; @@ -120,6 +149,7 @@ borders don't support gradients */ font: inherit; font-weight: 600; font-size: var(--cpd-font-size-body-lg); + z-index: 1; } .watchStream > svg { diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 2fe163b825..2188dd3779 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -542,6 +542,25 @@ const ScreenShareTileContent: FC = ({ // stops watching the stream. const contentRef = useRef(null); const mergedRef = useMergedRefs(contentRef, ref); + + const [frozenFrame, setFrozenFrame] = useState(null); + + useEffect(() => { + if (watching) { + setFrozenFrame(null); + return; + } + const video = contentRef.current?.querySelector("video"); + if (video && video.videoWidth > 0 && video.videoHeight > 0) { + const canvas = document.createElement("canvas"); + canvas.width = video.videoWidth; + canvas.height = video.videoHeight; + canvas.getContext("2d")?.drawImage(video, 0, 0); + setFrozenFrame(canvas.toDataURL()); + } else { + setFrozenFrame(null); + } + }, [watching]); // Freeze the video (pause it) while not watching, and resume when watching. // While stopped we also watch for new video elements (e.g. LiveKit @@ -570,21 +589,33 @@ const ScreenShareTileContent: FC = ({ video={video} streamOverlay={ watching ? undefined : ( - +
+ {frozenFrame !== null ? ( + + ) : ( +
+ )} + +
) } userId={vm.userId} diff --git a/src/tile/MediaView.module.css b/src/tile/MediaView.module.css index 9ea287d5a1..ff93c6e7bb 100644 --- a/src/tile/MediaView.module.css +++ b/src/tile/MediaView.module.css @@ -22,15 +22,9 @@ Please see LICENSE in the repository root for full details. z-index: 1; display: grid; place-items: center; - background: rgb(0 0 0 / 0.35); - backdrop-filter: blur(10px); pointer-events: none; } -.streamOverlay > * { - pointer-events: auto; -} - .media video { inline-size: 100%; block-size: 100%; From 512d77fca43030c7bd09a72a2591c7cffbbe83d5 Mon Sep 17 00:00:00 2001 From: TomOdellSheetMusic Date: Sun, 16 Aug 2026 01:14:01 +0200 Subject: [PATCH 7/8] add a setting to hide avatar tiles in calls --- locales/en/app.json | 5 ++-- src/settings/PreferencesSettingsTab.tsx | 19 +++++++++++++++ src/settings/settings.ts | 5 ++++ src/state/CallViewModel/CallViewModel.ts | 30 ++++++++++++++++++++---- 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/locales/en/app.json b/locales/en/app.json index c0788cb553..0261129443 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -255,8 +255,9 @@ "preferences_tab": { "developer_mode_label": "Developer mode", "developer_mode_label_description": "Enable developer mode and show developer settings tab.", - "introduction": "Here you can configure extra options for an improved experience.", - "reactions_play_sound_description": "Play a sound effect when anyone sends a reaction into a call.", + "hide_avatars_when_camera_off_description": "Hide avatar tiles of participants whose camera is off.", + "hide_avatars_when_camera_off_label": "Hide avatar tiles when camera is off", + "call.", "reactions_play_sound_label": "Play reaction sounds", "reactions_show_description": "Show an animation when anyone sends a reaction.", "reactions_show_label": "Show reactions", diff --git a/src/settings/PreferencesSettingsTab.tsx b/src/settings/PreferencesSettingsTab.tsx index 82306e7b7c..ed89ea78e6 100644 --- a/src/settings/PreferencesSettingsTab.tsx +++ b/src/settings/PreferencesSettingsTab.tsx @@ -15,6 +15,7 @@ import { showReactions as showReactionsSetting, playReactionsSound as playReactionsSoundSetting, developerMode as developerModeSetting, + hideAvatarTilesWhenCameraOff as hideAvatarTilesWhenCameraOffSetting, useSetting, } from "./settings"; @@ -30,6 +31,10 @@ export const PreferencesSettingsTab: FC = () => { playReactionsSoundSetting, ); + const [hideAvatarTilesWhenCameraOff, setHideAvatarTilesWhenCameraOff] = useSetting( + hideAvatarTilesWhenCameraOffSetting, + ); + const onChangeSetting = ( e: ChangeEvent, fn: (value: boolean) => void, @@ -76,6 +81,20 @@ export const PreferencesSettingsTab: FC = () => { onChange={(e) => onChangeSetting(e, setPlayReactionSound)} /> + + onChangeSetting(e, setHideAvatarTilesWhenCameraOff)} + /> + ("mute-all-audio", false); export const alwaysShowSelf = new Setting("always-show-self", true); +export const hideAvatarTilesWhenCameraOff = new Setting( + "hide-avatars-when-camera-off", + false, +); + export const alwaysShowIphoneEarpiece = new Setting( "always-show-iphone-earpiece", false, diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index a1f8126fce..167acbcb9b 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -61,6 +61,7 @@ import { import { duplicateTiles, echoCancellationSetting, + hideAvatarTilesWhenCameraOff, noiseSuppressionSetting, playReactionsSound, rnnoiseNoiseSuppression, @@ -943,7 +944,29 @@ export function createCallViewModel$( ); const grid$ = scope.behavior( - userMedia$.pipe( + combineLatest([userMedia$, hideAvatarTilesWhenCameraOff.value$]).pipe( + switchMap(([mediaItems, hideAvatars]) => + hideAvatars + ? // When enabled, only generate tiles for participants whose camera + // is on. Participants with their camera off remain audible but + // have no tile, keeping voice calls tidy. + mediaItems.length === 0 + ? of([]) + : combineLatest( + mediaItems.map((m) => + m.videoEnabled$.pipe( + map((videoEnabled) => [m, videoEnabled] as const), + ), + ), + ).pipe( + map((pairs) => + pairs + .filter(([, videoEnabled]) => videoEnabled) + .map(([m]) => m), + ), + ) + : of(mediaItems), + ), switchMap((mediaItems) => { const bins = mediaItems.map((m) => m.bin$.pipe(map((bin) => [m, bin] as const)), @@ -1062,7 +1085,7 @@ export function createCallViewModel$( const { setGridMode, gridMode$ } = createLayoutModeSwitch(scope, windowMode$); - // A single screen share can be focused (maximised) to fill the grid + // A single screen share can be focused (maximised) to fill the grid const focusedStreamRequest$ = new Subject(); const focusedStream$ = scope.behavior( focusedStreamRequest$.pipe( @@ -1072,8 +1095,7 @@ export function createCallViewModel$( ? of(null) : screenShares$.pipe( map( - (shares) => - shares.find((s) => s.id === requested.id) ?? null, + (shares) => shares.find((s) => s.id === requested.id) ?? null, ), distinctUntilChanged(), ), From ac62a3334fdea8a77f1c045937510918d9651140 Mon Sep 17 00:00:00 2001 From: TomOdellSheetMusic Date: Sun, 16 Aug 2026 02:46:45 +0200 Subject: [PATCH 8/8] fix key --- locales/en/app.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/locales/en/app.json b/locales/en/app.json index 0261129443..df02958c00 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -257,7 +257,8 @@ "developer_mode_label_description": "Enable developer mode and show developer settings tab.", "hide_avatars_when_camera_off_description": "Hide avatar tiles of participants whose camera is off.", "hide_avatars_when_camera_off_label": "Hide avatar tiles when camera is off", - "call.", + "introduction": "Here you can configure extra options for an improved experience.", + "reactions_play_sound_description": "Play a sound effect when anyone sends a reaction into a call.", "reactions_play_sound_label": "Play reaction sounds", "reactions_show_description": "Show an animation when anyone sends a reaction.", "reactions_show_label": "Show reactions",