From 33c39b2d4746cf132f8c22bc1814357266edc922 Mon Sep 17 00:00:00 2001 From: Joseph Myers Date: Mon, 2 Nov 2026 14:05:43 -0600 Subject: [PATCH 1/8] Reimplemented Audio Player and overhauled Record step to use it. - Audio Player now has an explicit, defined interface, usable through a handle. Record tracks the audio player imperatively. - Audio edits autosave after 3 seconds. - Audio Player and future audio components will use the new AudioManager, which ensures in a central location that only one audio plays at a time. - Recording should work for iOS Safari (it may not have been beforehand) - New display component for AI Processing, Convert Voice, Microphone Selection. - Version Count treats only counts 1 autosave (to avoid Versions button flickering in some cases) - Removed forced .5s delay and step unset on page nav, since this can be handled in a single place (PassageDetailContext) without a delay - useMediaUpload has its own compression ability. saving now reads mimeType property and compresses if omitted (or not if WAV is specified) - BOLD no longers moves to the next step for the user, but it does still make the step complete once audio is given --- web/src/business/voice/SelectVoice.tsx | 4 +- web/src/components/AiProcessing.tsx | 51 + web/src/components/AudioPlayer.test.tsx | 343 ++++++ web/src/components/AudioPlayer.tsx | 1097 +++++++++++++++++ .../usePassageVersionAudioRows.test.ts | 5 + .../AudioTab/usePassageVersionAudioRows.ts | 18 +- web/src/components/ConvertVoiceDialog.tsx | 36 + web/src/components/MicrophoneDialog.tsx | 122 ++ .../PassageDetail/PassageDetailChooser.tsx | 12 +- .../PassageDetailRecord.test.tsx | 277 +++-- .../PassageDetail/PassageDetailRecord.tsx | 667 +++++----- .../PassageDetailStepComplete.tsx | 11 +- .../mobile/MobileWorkflowSteps.tsx | 6 +- .../mobile/PassageDetailMobileFooter.tsx | 6 +- .../PassageDetail/usePassageAudioSave.test.ts | 320 +++++ .../PassageDetail/usePassageAudioSave.ts | 119 ++ .../PassageDetail/usePassageNavigate.ts | 5 - web/src/context/PassageDetailContext.tsx | 8 +- web/src/crud/autosaveMedia.test.ts | 173 +++ web/src/crud/autosaveMedia.ts | 90 ++ web/src/crud/convertToWebM.ts | 6 +- web/src/crud/useMediaUpload.test.ts | 10 + web/src/crud/useMediaUpload.ts | 29 +- web/src/crud/useWavRecorder.tsx | 12 + web/src/crud/waveformRenderer.ts | 159 +++ web/src/store/upload/pendingMediaUploads.ts | 36 + web/src/theme.ts | 14 +- web/src/utils/audioManager.test.ts | 100 ++ web/src/utils/audioManager.ts | 60 + web/src/utils/audioUtils.ts | 480 ++++++++ web/src/utils/formatTime.ts | 114 ++ web/src/utils/useAudioConversion.ts | 129 ++ web/src/utils/useRecorder.ts | 100 ++ 33 files changed, 4096 insertions(+), 523 deletions(-) create mode 100644 web/src/components/AiProcessing.tsx create mode 100644 web/src/components/AudioPlayer.test.tsx create mode 100644 web/src/components/AudioPlayer.tsx create mode 100644 web/src/components/ConvertVoiceDialog.tsx create mode 100644 web/src/components/MicrophoneDialog.tsx create mode 100644 web/src/components/PassageDetail/usePassageAudioSave.test.ts create mode 100644 web/src/components/PassageDetail/usePassageAudioSave.ts create mode 100644 web/src/crud/autosaveMedia.test.ts create mode 100644 web/src/crud/autosaveMedia.ts create mode 100644 web/src/crud/waveformRenderer.ts create mode 100644 web/src/utils/audioManager.test.ts create mode 100644 web/src/utils/audioManager.ts create mode 100644 web/src/utils/audioUtils.ts create mode 100644 web/src/utils/formatTime.ts create mode 100644 web/src/utils/useAudioConversion.ts create mode 100644 web/src/utils/useRecorder.ts diff --git a/web/src/business/voice/SelectVoice.tsx b/web/src/business/voice/SelectVoice.tsx index 7b62015f2..709e41000 100644 --- a/web/src/business/voice/SelectVoice.tsx +++ b/web/src/business/voice/SelectVoice.tsx @@ -25,7 +25,7 @@ interface ISelectVoice { noNewVoice?: boolean; onlySettings?: boolean; onOpen: () => void; - begin?: () => void; + begin?: (voice: string) => void; refresh?: () => void; } @@ -104,7 +104,7 @@ export default function SelectVoice({ diff --git a/web/src/components/AiProcessing.tsx b/web/src/components/AiProcessing.tsx new file mode 100644 index 000000000..175b12205 --- /dev/null +++ b/web/src/components/AiProcessing.tsx @@ -0,0 +1,51 @@ +import { Box, Typography } from '@mui/material'; +import { shallowEqual, useSelector } from 'react-redux'; +import { Button } from '../control'; +import { ISharedStrings, IWsAudioPlayerStrings } from '../model'; +import { sharedSelector, wsAudioPlayerSelector } from '../selector'; +import AiTimeProgress from './AiTimeProgress'; + +interface AiProcessingProps { + startedAt: number; + estimateSec: number; + onCancel: () => void; +} + +/** Shown in place of a player while an AI request runs. */ +export function AiProcessing({ + startedAt, + estimateSec, + onCancel, +}: AiProcessingProps) { + const t: IWsAudioPlayerStrings = useSelector( + wsAudioPlayerSelector, + shallowEqual + ); + const ts: ISharedStrings = useSelector(sharedSelector, shallowEqual); + + return ( + + + {t.aiInProgress} + + + + + ); +} diff --git a/web/src/components/AudioPlayer.test.tsx b/web/src/components/AudioPlayer.test.tsx new file mode 100644 index 000000000..c53cfc530 --- /dev/null +++ b/web/src/components/AudioPlayer.test.tsx @@ -0,0 +1,343 @@ +import { createRef } from 'react'; +import { act, render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +type Handler = (...args: any[]) => void; + +// Wavesurfer needs a real audio stack. +let mockListeners: Record = {}; +const mockWs = { + on: jest.fn((evt: string, cb: Handler) => { + (mockListeners[evt] ||= []).push(cb); + return () => {}; + }), + once: jest.fn(), + play: jest.fn(), + pause: jest.fn(), + stop: jest.fn(), + setTime: jest.fn(), + setOptions: jest.fn(), + load: jest.fn(), + empty: jest.fn(), + zoom: jest.fn(), + destroy: jest.fn(), + registerPlugin: jest.fn(), + isPlaying: jest.fn(() => false), + getDuration: jest.fn(() => 10), + getCurrentTime: jest.fn(() => 0), + getMediaElement: jest.fn(() => ({ load: jest.fn() })), + getWrapper: jest.fn(() => ({ + getBoundingClientRect: () => ({ width: 100 }), + })), + options: { minPxPerSec: 0 }, +}; +const mockRegions = { + on: jest.fn(), + getRegions: jest.fn(() => []), + addRegion: jest.fn(), + enableDragSelection: jest.fn(() => () => {}), +}; +const mockSpliceAudio = jest.fn(); +const mockInsertSilenceAudio = jest.fn(); +const mockStartRecording = jest.fn(); +// Stable reference: the player's mic effect depends on this function. +const mockGetMic = jest.fn(() => Promise.resolve({ stream: {} })); + +jest.mock('wavesurfer.js', () => ({ + __esModule: true, + default: { create: () => mockWs }, +})); +jest.mock('wavesurfer.js/dist/plugins/regions', () => ({ + __esModule: true, + default: { create: () => mockRegions }, +})); +jest.mock('wavesurfer.js/dist/plugins/zoom', () => ({ + __esModule: true, + default: { create: jest.fn() }, +})); +jest.mock('wavesurfer.js/dist/plugins/minimap', () => ({ + __esModule: true, + default: { create: jest.fn() }, +})); +jest.mock('../crud/waveformRenderer', () => ({ + createWaveformRenderer: jest.fn(), + disableProgressSplit: jest.fn(), +})); +jest.mock('../utils/audioUtils', () => ({ + spliceAudio: (...args: unknown[]) => mockSpliceAudio(...args), + insertSilenceAudio: (...args: unknown[]) => mockInsertSilenceAudio(...args), + clampSelectionToHighlights: jest.fn(), +})); +jest.mock('../utils/useRecorder', () => ({ + useRecorder: () => ({ + startRecording: mockStartRecording, + stopRecording: jest.fn(), + }), +})); +jest.mock('../crud/useUserMedia', () => ({ + useUserMedia: () => mockGetMic, +})); +jest.mock('../context/usePassageDetailContext', () => ({ + __esModule: true, + default: () => ({ + recording: false, + setRecording: jest.fn(), + isBoldWorkflow: false, + }), +})); +jest.mock('../utils/useAudioConversion', () => ({ + useAudioConversion: () => ({ + progress: undefined, + suppressNoise: jest.fn(), + convertVoice: jest.fn(), + cancel: jest.fn(), + }), +})); +jest.mock('react-redux', () => ({ + useSelector: () => ({ reduceNoiseAi: 'Reduce', convertVoiceAi: 'Convert' }), + shallowEqual: jest.fn(), +})); +jest.mock('../selector', () => ({ wsAudioPlayerSelector: jest.fn() })); +jest.mock('./MicrophoneDialog', () => ({ MicrophoneDialog: () => null })); +jest.mock('./ConvertVoiceDialog', () => ({ ConvertVoiceDialog: () => null })); +jest.mock('./AiProcessing', () => ({ AiProcessing: () => null })); +jest.mock('../control/NoChickenIcon', () => ({ + __esModule: true, + default: () => null, +})); +jest.mock('../control/VoiceConversionLogo', () => ({ + __esModule: true, + default: () => null, +})); + +// imported after the mocks so the player picks them up +import { + AudioPlayer, + type AudioPlayerHandle, + type AudioPlayerProps, +} from './AudioPlayer'; + +const AUDIO = new Blob(['audio']); +const DURATION = 10; + +const emit = (evt: string, ...args: unknown[]) => + act(() => (mockListeners[evt] ?? []).forEach((cb) => cb(...args))); + +const renderPlayer = (props: AudioPlayerProps = {}) => { + const ref = createRef(); + render(); + return ref; +}; + +/** Renders the player and decodes its audio, as when a clip finishes loading. */ +const renderLoaded = (props: AudioPlayerProps = {}) => { + const ref = renderPlayer({ audioSource: AUDIO, ...props }); + emit('decode', DURATION); + return ref; +}; + +const button = (name: string) => screen.getByRole('button', { name }); + +describe('AudioPlayer', () => { + beforeAll(() => { + // jsdom does not implement object URLs. + URL.createObjectURL = jest.fn(() => 'blob:audio'); + URL.revokeObjectURL = jest.fn(); + }); + + beforeEach(() => { + mockListeners = {}; + }); + + describe('loading', () => { + it('keeps play disabled until the audio decodes', () => { + const onReady = jest.fn(); + renderPlayer({ audioSource: AUDIO, onReady }); + expect(button('passage play')).toBeDisabled(); + + emit('decode', DURATION); + + expect(button('passage play')).toBeEnabled(); + expect(screen.getByText('0:00 / 0:10')).toBeInTheDocument(); + expect(onReady).toHaveBeenCalledWith(DURATION); + }); + + it('prefixes aria-labels with the label prop', () => { + renderPlayer({ label: 'source' }); + expect(button('source play')).toBeInTheDocument(); + }); + }); + + describe('playback', () => { + it('toggles between play and pause', async () => { + renderLoaded(); + + await userEvent.click(button('passage play')); + expect(mockWs.play).toHaveBeenCalledTimes(1); + + emit('play'); + await userEvent.click(button('passage pause')); + expect(mockWs.pause).toHaveBeenCalledTimes(1); + + emit('pause'); + expect(button('passage play')).toBeInTheDocument(); + }); + + it('shows and reports the current time', () => { + const onTimeUpdate = jest.fn(); + renderLoaded({ onTimeUpdate }); + + emit('timeupdate', 3); + + expect(screen.getByText('0:03 / 0:10')).toBeInTheDocument(); + expect(onTimeUpdate).toHaveBeenCalledWith(3); + }); + + it('stops at the end of a sticky selection and seeks back', () => { + mockWs.isPlaying.mockReturnValueOnce(true); + renderLoaded({ stickySelection: { start: 2, end: 4 } }); + emit('play'); + + emit('timeupdate', 4); + + expect(mockWs.pause).toHaveBeenCalledTimes(1); + expect(mockWs.setTime).toHaveBeenLastCalledWith(2); + }); + + it('calls onFinish at the end of the clip', () => { + const onFinish = jest.fn(); + renderLoaded({ onFinish }); + + emit('finish'); + + expect(onFinish).toHaveBeenCalledTimes(1); + }); + }); + + describe('recording', () => { + it('shows record instead of play when no audio is loaded', () => { + renderPlayer({ showRecordButton: true }); + expect(button('passage start recording')).toBeInTheDocument(); + expect( + screen.queryByRole('button', { name: 'passage play' }) + ).not.toBeInTheDocument(); + }); + + it('shows play instead of record once audio is loaded', () => { + renderLoaded({ showRecordButton: true }); + expect(button('passage play')).toBeInTheDocument(); + }); + + it('starts the recorder when record is clicked', async () => { + renderPlayer({ showRecordButton: true }); + await userEvent.click(button('passage start recording')); + expect(mockStartRecording).toHaveBeenCalledTimes(1); + }); + }); + + describe('selection', () => { + it('shows the selected range and clears it on waveform click', () => { + const onSelectionChange = jest.fn(); + const ref = renderLoaded({ onSelectionChange }); + + act(() => ref.current!.updateSelection({ start: 2, end: 4 })); + expect(screen.getByText('0:02 - 0:04')).toBeInTheDocument(); + expect(onSelectionChange).toHaveBeenLastCalledWith( + { start: 2, end: 4 }, + 'user' + ); + + emit('click', 0.5); + expect(screen.getByText('0:00 / 0:10')).toBeInTheDocument(); + expect(onSelectionChange).toHaveBeenLastCalledWith(null, 'user'); + }); + + it('keeps a sticky selection on waveform click', () => { + renderLoaded({ stickySelection: { start: 2, end: 4 } }); + + emit('click', 0.5); + + expect(screen.getByText('0:02 - 0:04')).toBeInTheDocument(); + }); + }); + + describe('onWaveformClick', () => { + it('passes the nearest marker within the grace distance', () => { + const onWaveformClick = jest.fn(); + // 100px wide over 10s, so 2px away from the marker at 5s. + renderLoaded({ markers: [5], onWaveformClick }); + + emit('click', 0.52); + + expect(onWaveformClick).toHaveBeenCalledWith(5.2, 5); + }); + + it('passes no marker when the click is outside the grace distance', () => { + const onWaveformClick = jest.fn(); + renderLoaded({ markers: [5], onWaveformClick }); + + emit('click', 0.8); + + expect(onWaveformClick).toHaveBeenCalledWith(8, undefined); + }); + }); + + describe('editing', () => { + it('hides undo until there is something to undo', () => { + renderLoaded(); + expect( + screen.queryByRole('button', { name: 'passage undo' }) + ).not.toBeInTheDocument(); + }); + + it('disables cut until there is a selection', () => { + renderLoaded({ showCut: true }); + expect(button('passage cut')).toBeDisabled(); + }); + + it('cuts the selection, and undo restores the audio', async () => { + const spliced = new Blob(['spliced']); + mockSpliceAudio.mockResolvedValueOnce(spliced); + const onAudioChange = jest.fn(); + const ref = renderLoaded({ showCut: true, onAudioChange }); + act(() => ref.current!.updateSelection({ start: 2, end: 4 })); + + await userEvent.click(button('passage cut')); + + expect(mockSpliceAudio).toHaveBeenCalledWith(AUDIO, 2, 4); + expect(onAudioChange).toHaveBeenLastCalledWith(spliced); + + await userEvent.click(button('passage undo')); + + expect(onAudioChange).toHaveBeenLastCalledWith(AUDIO, undefined); + expect(screen.getByText('0:02 - 0:04')).toBeInTheDocument(); + }); + + it('inserts silence at the current time', async () => { + const withSilence = new Blob(['silence']); + mockInsertSilenceAudio.mockResolvedValueOnce(withSilence); + const onAudioChange = jest.fn(); + renderLoaded({ showSilence: true, onAudioChange }); + emit('timeupdate', 3); + + await userEvent.click(button('passage insert silence')); + + expect(mockInsertSilenceAudio).toHaveBeenCalledWith(AUDIO, 3, 0.5); + expect(onAudioChange).toHaveBeenLastCalledWith(withSilence); + }); + + it('clears the audio from the menu', async () => { + const onAudioChange = jest.fn(); + renderLoaded({ showTrash: true, onAudioChange }); + + await userEvent.click(button('passage audio menu')); + await userEvent.click( + screen.getByRole('menuitem', { name: 'passage reset audio' }) + ); + + expect(onAudioChange).toHaveBeenLastCalledWith(null); + expect(screen.getByText('0:00 / 0:00')).toBeInTheDocument(); + expect(button('passage play')).toBeDisabled(); + }); + }); +}); diff --git a/web/src/components/AudioPlayer.tsx b/web/src/components/AudioPlayer.tsx new file mode 100644 index 000000000..531499702 --- /dev/null +++ b/web/src/components/AudioPlayer.tsx @@ -0,0 +1,1097 @@ +import React, { + forwardRef, + useEffect, + useImperativeHandle, + useRef, + useState, +} from 'react'; +import WaveSurfer from 'wavesurfer.js'; +import RegionsPlugin from 'wavesurfer.js/dist/plugins/regions'; +import ZoomPlugin from 'wavesurfer.js/dist/plugins/zoom'; +import MinimapPlugin from 'wavesurfer.js/dist/plugins/minimap'; +import { + Box, + IconButton, + ListItemIcon, + ListItemText, + Menu, + MenuItem, + Stack, + Typography, + useMediaQuery, + useTheme, +} from '@mui/material'; +import PlayArrowIcon from '@mui/icons-material/PlayArrow'; +import PauseIcon from '@mui/icons-material/Pause'; +import MoreVertIcon from '@mui/icons-material/MoreVert'; +import FiberManualRecordIcon from '@mui/icons-material/FiberManualRecord'; +import StopIcon from '@mui/icons-material/Stop'; +import ContentCutIcon from '@mui/icons-material/ContentCut'; +import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline'; +import VoiceOverOffOutlinedIcon from '@mui/icons-material/VoiceOverOffOutlined'; +import UndoIcon from '@mui/icons-material/Undo'; +import MicIcon from '@mui/icons-material/Mic'; +import { formatTime } from '../utils/formatTime'; +import { localUserKey, LocalKey } from '../utils/localUserKey'; +import { + spliceAudio, + clampSelectionToHighlights, + insertSilenceAudio, +} from '../utils/audioUtils'; +import { useRecorder } from '../utils/useRecorder'; +import { useUserMedia } from '../crud/useUserMedia'; +import { buildCaptureConstraints } from '../crud/captureConstraints'; +import { + createWaveformRenderer, + disableProgressSplit, +} from '../crud/waveformRenderer'; +import { audioManager, type AudioHandle } from '../utils/audioManager'; +import usePassageDetailContext from '../context/usePassageDetailContext'; +import { MicrophoneDialog } from './MicrophoneDialog'; +import { ConvertVoiceDialog } from './ConvertVoiceDialog'; +import { AiProcessing } from './AiProcessing'; +import NoChickenIcon from '../control/NoChickenIcon'; +import VoiceConversionLogo from '../control/VoiceConversionLogo'; +import { useAudioConversion } from '../utils/useAudioConversion'; +import { shallowEqual, useSelector } from 'react-redux'; +import { IWsAudioPlayerStrings } from '../model'; +import { wsAudioPlayerSelector } from '../selector'; + +/* ------------------------------------------------------------------ */ +/* Types */ +/* ------------------------------------------------------------------ */ + +export interface AudioPlayerHandle { + /** Start playback */ + play: () => void; + /** Pause playback */ + pause: () => void; + /** Seek to a specific time */ + setTime: (time: number) => void; + /** Start recording from the microphone */ + startRecording: () => Promise; + /** Stop recording */ + stopRecording: () => void; + /** Push current audio + selection onto the undo stack, with an optional payload returned to onAudioChange when this entry is restored */ + pushUndo: (payload?: unknown) => void; + /** Programmatically set the waveform selection */ + updateSelection: (sel: { start: number; end: number } | null) => void; + /** Reset zoom to the default */ + resetZoom: () => void; +} + +export interface AudioPlayerProps { + /** Audio source — either a Blob or a URL string */ + audioSource?: Blob; + + /** Allow user to drag-create a selection on the waveform */ + enableDragSelection?: boolean; + + /** Double-tap the waveform to select the entire clip (default: false) */ + selectAllOnDoubleClick?: boolean; + + /** WaveSurfer waveColor */ + waveColor?: string; + /** WaveSurfer progressColor */ + progressColor?: string; + /** Waveform height in px (default: 80) */ + height?: number; + + /** Custom time display. If omitted, shows "currentTime / duration" (or selection range when active) */ + formatTimeDisplay?: (currentTime: number, duration: number) => string; + /** Called on every timeupdate */ + onTimeUpdate?: (time: number) => void; + /** Called when WaveSurfer decodes audio (provides duration) */ + onReady?: (duration: number) => void; + /** Called when a recording completes (provides the recorded Blob) */ + onRecordingComplete?: (blob: Blob) => void; + /** Called once when playback reaches the end of the clip. */ + onFinish?: () => void; + + /** Menu items to show in the overflow menu (e.g. elements) */ + menuItems?: React.ReactNode; + + /** Called when the drag-selection changes (null when cleared) */ + onSelectionChange?: ( + selection: { start: number; end: number } | null, + source: 'user' | 'undo' + ) => void; + + /** Show a Record button instead of Play when no audio is loaded (default: false) */ + showRecordButton?: boolean; + /** Is recording intended for this player (default: showRecordButton) */ + allowRecord?: boolean; + /** Mic echo cancellation while recording (default: false) */ + captureEchoCancellation?: boolean; + /** Mic noise suppression while recording (default: false) */ + captureNoiseSuppression?: boolean; + /** Show a cut icon in the controls row (default: false) */ + showCut?: boolean; + /** Show a trash option to clear the audio (default: false) */ + showTrash?: boolean; + /** Show a silence icon in the controls row (default: false) */ + showSilence?: boolean; + /** Show an undo icon in the controls row (default: true) */ + showUndo?: boolean; + + /** Create an unclearable selection region on the waveform */ + stickySelection?: { start: number; end: number }; + + /** Stop playback at stickySelection end and seek back to start (default: true) */ + shouldStopAfterStickySelection?: boolean; + + /** Called when the internal audio changes. When the change is an undo restore, undoPayload carries whatever was passed to pushUndo for that entry. */ + onAudioChange?: (audio: Blob | null, undoPayload?: unknown) => void; + + /** Enable mouse-wheel and touch-pinch zoom on the waveform (default: true) */ + enableZoom?: boolean; + + /** Colored regions to highlight on the waveform */ + highlights?: { start: number; end: number; color: string }[]; + + /** Children rendered below the waveform (e.g. helper text) */ + children?: React.ReactNode; + + /** Optional label rendered on the top row (after time display) */ + topRowLabel?: React.ReactNode; + + /** Static markers — timestamps in seconds, optionally with a color (default grey). */ + markers?: (number | AudioMarker)[]; + + /** Fires on every waveform click. `marker` is the nearest marker timestamp within ~10px, if any. */ + onWaveformClick?: (timestamp: number, marker?: number) => void; + + /** + * Prefixes every aria-label inside the player (e.g. "passage play"). + * Override when a screen shows more than one player, so the labels stay unique. + */ + label?: string; +} + +export interface AudioMarker { + /** Timestamp in seconds */ + time: number; + /** Marker color */ + color?: string; +} + +/* ------------------------------------------------------------------ */ +/* Marker helpers */ +/* ------------------------------------------------------------------ */ + +const GRACE_PX = 10; +const DEFAULT_MARKER_COLOR = '#868686c5'; + +/** + * Nearest marker drawn within GRACE_PX on-screen pixels of the click, or undefined. + */ +function nearestMarkerWithin( + markers: AudioMarker[], + clickXPercent: number, + durationInSec: number, + waveformWidthPx: number +): number | undefined { + const clickPx = clickXPercent * waveformWidthPx; + let best: number | undefined; + let bestD = Infinity; + for (const m of markers) { + const markerPx = (m.time / durationInSec) * waveformWidthPx; + const d = Math.abs(markerPx - clickPx); + if (d <= GRACE_PX && d < bestD) { + best = m.time; + bestD = d; + } + } + return best; +} + +/* ------------------------------------------------------------------ */ +/* Component */ +/* ------------------------------------------------------------------ */ + +export const AudioPlayer = forwardRef( + ( + { + audioSource, + enableDragSelection = false, + selectAllOnDoubleClick = false, + waveColor: waveColorProp, + progressColor: progressColorProp, + height = 80, + formatTimeDisplay, + onTimeUpdate, + onReady, + menuItems: menuItemsProp, + onSelectionChange, + showRecordButton = false, + allowRecord = showRecordButton, + captureEchoCancellation = false, + captureNoiseSuppression = false, + showCut = false, + showTrash = false, + showSilence = false, + showUndo = true, + stickySelection, + shouldStopAfterStickySelection = true, + onRecordingComplete, + onFinish, + onAudioChange, + highlights = [], + enableZoom = true, + children, + topRowLabel, + markers, + onWaveformClick, + label = 'passage', + }, + ref + ) => { + const containerRef = useRef(null); + const minimapContainerRef = useRef(null); + const wsRef = useRef(null); + const regionsRef = useRef(null); + + const [currentTime, setCurrentTime] = useState(0); + const [duration, setDuration] = useState(0); + const [playing, setPlaying] = useState(false); + const [internalAudio, setInternalAudio] = useState( + audioSource ?? null + ); + const [menuAnchorEl, setMenuAnchorEl] = useState(null); + const [isZoomed, setIsZoomed] = useState(false); + const theme = useTheme(); + const waveColor = waveColorProp ?? theme.palette.primary.extraLight; + const progressColor = progressColorProp ?? theme.palette.primary.extraLight; + const isSmallScreen = useMediaQuery(theme.breakpoints.down('md')); + const isSelectionSticky = !!stickySelection; + + // Undo stack + interface UndoEntry { + audio: Blob | null; + selection: { start: number; end: number } | null; + payload?: unknown; + } + const [undoStack, setUndoStack] = useState([]); + const pushUndo = (payload?: unknown) => { + setUndoStack((prev) => [ + ...prev, + { audio: internalAudio, selection, payload }, + ]); + }; + + const selfHandleRef = useRef>({ + play: () => wsRef.current?.play(), + pause: () => wsRef.current?.pause(), + stop: () => wsRef.current?.stop(), + }); + + const [selection, setSelection] = useState<{ + start: number; + end: number; + } | null>(stickySelection ?? null); + const selectionRef = useRef(selection); + useEffect(() => { + selectionRef.current = selection; + }, [selection]); + const stickySelectionRef = useRef(stickySelection); + useEffect(() => { + stickySelectionRef.current = stickySelection; + }, [stickySelection]); + const shouldStopAfterStickySelectionRef = useRef( + shouldStopAfterStickySelection + ); + useEffect(() => { + shouldStopAfterStickySelectionRef.current = + shouldStopAfterStickySelection; + }, [shouldStopAfterStickySelection]); + const highlightsRef = useRef(highlights); + useEffect(() => { + highlightsRef.current = highlights; + }, [highlights]); + const markerObjs = (markers ?? []).map((m) => + typeof m === 'number' ? { time: m } : m + ); + const markersRef = useRef(markerObjs); + markersRef.current = markerObjs; + + useEffect(() => { + if (audioSource && audioSource !== internalAudio) { + setUndoStack([]); + applySelection(stickySelection ?? null); + setInternalAudio(audioSource); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [audioSource]); + + // Helper: only fire onSelectionChange when the value actually changed + const fireSelectionChange = ( + sel: { start: number; end: number } | null, + source: 'user' | 'undo' + ) => { + const prev = selectionRef.current; + if ( + prev === sel || + (prev && sel && prev.start === sel.start && prev.end === sel.end) + ) + return; + onSelectionChange?.(sel, source); + }; + const onAudioChangeRef = useRef(onAudioChange); + useEffect(() => { + onAudioChangeRef.current = onAudioChange; + }, [onAudioChange]); + + // Ref to track whether a region add is programmatic + const programmaticRef = useRef(false); + + const { recording, setRecording, isBoldWorkflow } = + usePassageDetailContext(); + const { + progress: conversionProcessing, + suppressNoise, + convertVoice, + cancel, + } = useAudioConversion(); + const t: IWsAudioPlayerStrings = useSelector( + wsAudioPlayerSelector, + shallowEqual + ); + const [microphoneId, setMicrophoneId] = useState( + () => localStorage.getItem(localUserKey(LocalKey.microphoneId)) ?? '' + ); + const [micDialogOpen, setMicDialogOpen] = useState(false); + const [voiceDialogOpen, setVoiceDialogOpen] = useState(false); + const getMic = useUserMedia( + buildCaptureConstraints( + microphoneId, + captureEchoCancellation, + captureNoiseSuppression + ) + ); + const [micStream, setMicStream] = useState(); + useEffect(() => { + if (allowRecord) + getMic(true).then( + ({ stream }) => setMicStream(stream), + () => {} + ); + }, [allowRecord, getMic, microphoneId]); + + const recorder = useRecorder(micStream); + const startRecording = async () => { + if (conversionProcessing) return; + await recorder.startRecording( + internalAudio, + currentTime, + wsRef.current, + (audio) => { + if (internalAudio) { + pushUndo(); + wsRef.current?.once('decode', () => + wsRef.current?.setTime(currentTime) + ); + } + setInternalAudio(audio); + setRecording(false); + onAudioChangeRef.current?.(audio); + onRecordingComplete?.(audio); + } + ); + setRecording(true); + }; + const stopRecording = () => recorder.stopRecording(); + + // Apply or clear the single selection region. + const applySelection = (sel: { start: number; end: number } | null) => { + const rp = regionsRef.current; + if (rp) { + rp.getRegions().forEach((r) => { + if (r.start !== r.end) r.remove(); + }); + if (sel) { + programmaticRef.current = true; + rp.addRegion({ start: sel.start, end: sel.end }); + programmaticRef.current = false; + wsRef.current?.setTime(sel.start); + } + } + setSelection(sel); + fireSelectionChange(sel, 'user'); + }; + + // Imperative handle + useImperativeHandle(ref, () => ({ + play: selfHandleRef.current.play, + pause: selfHandleRef.current.pause, + setTime: (t: number) => wsRef.current?.setTime(t), + get container() { + return containerRef.current; + }, + startRecording, + stopRecording, + pushUndo, + updateSelection: applySelection, + resetZoom: () => { + const ws = wsRef.current; + if (ws) ws.zoom(1); + }, + })); + + /* ----- Init WaveSurfer ----- */ + useEffect(() => { + if (!containerRef.current) return; + const handle = selfHandleRef.current; + + const wsRegions = RegionsPlugin.create(); + regionsRef.current = wsRegions; + + const plugins: Array = [wsRegions]; + if (enableZoom) { + plugins.push( + ZoomPlugin.create({ + scale: 0.5, + maxZoom: 500, + deltaThreshold: 5, + exponentialZooming: true, + iterations: 20, + }) + ); + } + + const wsOptions = { + container: containerRef.current, + waveColor, + progressColor, + cursorColor: '#333', + cursorWidth: 4, + barWidth: 2, + height, + normalize: false, + plugins, + hideScrollbar: isSmallScreen, + }; + + const renderFunction = createWaveformRenderer( + { + ws: wsRef, + highlights: highlightsRef, + }, + wsOptions + ); + + const ws = WaveSurfer.create({ ...wsOptions, renderFunction }); + wsRef.current = ws; + + disableProgressSplit(ws); + + if (enableZoom && minimapContainerRef.current) { + ws.registerPlugin( + MinimapPlugin.create({ + container: minimapContainerRef.current, + height: 12, + waveColor: '#dddddd', + progressColor: '#dddddd', + cursorWidth: 0, + overlayColor: '#c4c4c448', + }) + ); + } + + ws.on('zoom', () => { + requestAnimationFrame(() => { + const parent = wsRef.current?.getWrapper().parentElement; + setIsZoomed(!!parent && parent.scrollWidth > parent.clientWidth + 1); + }); + }); + + const container = containerRef.current!; + let dragCleanup: (() => void) | undefined; + if (enableDragSelection) { + dragCleanup = wsRegions.enableDragSelection({}); + + // Work around a bug in wavesurfer's createDragStream: multi-touch leaves + // stale entries in its activePointers map, permanently breaking drag selection. + // Re-initialize drag selection after each pinch gesture to reset the state. + let wasPinching = false; + const onTouchStart = (e: TouchEvent) => { + if (e.touches.length >= 2) wasPinching = true; + }; + const onTouchEnd = (e: TouchEvent) => { + if (wasPinching && e.touches.length === 0) { + wasPinching = false; + dragCleanup?.(); + dragCleanup = wsRegions.enableDragSelection({}); + } + }; + container.addEventListener('touchstart', onTouchStart, { + passive: true, + }); + container.addEventListener('touchend', onTouchEnd, { passive: true }); + } + + // --- Real-time highlight clamping --- + wsRegions.on('region-initialized', (region) => { + region.on('update', (side) => { + if (region.start === region.end) return; // marker + const hl = highlightsRef.current; + if (!hl.length) return; + + const mode = + side === 'start' ? 'start' : side === 'end' ? 'end' : 'pan'; + const userSelection = selectionRef.current; + //filter out when the selection is highlighted (e.g. replacement dialog) + const filtered = userSelection + ? hl.filter( + (h) => + h.start >= userSelection.end || h.end <= userSelection.start + ) + : hl; + const clamped = clampSelectionToHighlights( + { start: region.start, end: region.end }, + filtered, + mode, + userSelection + ); + + if (!clamped) { + const prev = selectionRef.current; + if (prev) region.setOptions({ start: prev.start, end: prev.end }); + else region.remove(); + return; + } + if (clamped.start !== region.start || clamped.end !== region.end) { + region.setOptions({ + start: clamped.start, + end: clamped.end, + }); + } + }); + }); + + // --- Region events (drag-selection) --- + wsRegions.on('region-created', (region) => { + if (region.start === region.end) return; // marker, ignore + if (programmaticRef.current) return; + + // Remove other non-marker regions (single-selection mode) + wsRegions.getRegions().forEach((r) => { + if (r.id !== region.id && r.start !== r.end) r.remove(); + }); + const sel = { start: region.start, end: region.end }; + setSelection(sel); + fireSelectionChange(sel, 'user'); + ws.setTime(region.start); + }); + + wsRegions.on('region-updated', (region) => { + const sel = { start: region.start, end: region.end }; + setSelection(sel); + fireSelectionChange(sel, 'user'); + ws.setTime(region.start); + }); + + wsRegions.on('region-clicked', (region, e) => { + if (region.start !== region.end) { + e.stopPropagation(); + } + }); + + // --- Waveform click --- + ws.on('click', () => { + if (!isSelectionSticky) { + // Clear selection regions (but not when initialSelection is set) + wsRegions.getRegions().forEach((r) => { + if (r.start !== r.end) r.remove(); + }); + setSelection(null); + fireSelectionChange(null, 'user'); + } + }); + + // --- Waveform double-click: select the entire clip --- + if (selectAllOnDoubleClick) { + ws.on('dblclick', () => { + const dur = ws.getDuration(); + if (!dur) return; + applySelection({ start: 0, end: dur }); + }); + } + + // --- Playback events --- + let playStartTime = 0; + ws.on('play', () => { + playStartTime = ws.getCurrentTime(); + setPlaying(true); + }); + ws.on('pause', () => { + setPlaying(false); + audioManager.release(handle); + }); + ws.on('timeupdate', (time) => { + setCurrentTime(time); + onTimeUpdate?.(time); + // Stop playback at sticky/regular selection end and seek back to start. + const sel = + shouldStopAfterStickySelectionRef.current && + stickySelectionRef.current + ? stickySelectionRef.current + : enableDragSelection + ? selectionRef.current + : null; + if ( + ws.isPlaying() && + sel && + time >= sel.end && + playStartTime < sel.end + ) { + ws.pause(); + ws.setTime(sel.start); + setCurrentTime(sel.start); + } + }); + ws.on('decode', (d) => { + if (d < 0.01) { + return; + } + setDuration(d); + if (recording) return; + onReady?.(d); + if (stickySelectionRef.current && ws.options.minPxPerSec < 20) + ws.zoom(20); + + if (!regionsRef.current) return; + + // Clear any pre-existing regions + regionsRef.current.getRegions().forEach((r) => { + r.remove(); + }); + + // Preserve previous selection + const prevSelection = selectionRef.current; + if (prevSelection) { + programmaticRef.current = true; + regionsRef.current.addRegion({ + start: prevSelection.start, + end: prevSelection.end, + color: + prevSelection.start === prevSelection.end + ? '#303030aa' + : undefined, + }); + programmaticRef.current = false; + ws.setTime(prevSelection.start); + } + + // Re-add static markers + for (const m of markersRef.current) { + regionsRef.current.addRegion({ + start: m.time, + drag: false, + resize: false, + color: m.color ?? DEFAULT_MARKER_COLOR, + }); + } + }); + ws.on('finish', () => { + onFinish?.(); + }); + + return () => { + dragCleanup?.(); + audioManager.release(handle); + ws.destroy(); + wsRef.current = null; + regionsRef.current = null; + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + /* ----- Sync audio source ----- */ + useEffect(() => { + const ws = wsRef.current; + if (!ws || !internalAudio) return; + // Restore cursor visibility (may have been hidden by trash) + ws.setOptions({ cursorWidth: 4 }); + const url = URL.createObjectURL(internalAudio); + ws.load(url); + return () => URL.revokeObjectURL(url); + }, [internalAudio]); + + /* ----- Sync sticky selection → selection ----- */ + useEffect(() => { + if (!stickySelection) return; + setSelection({ + start: stickySelection.start, + end: stickySelection.end, + }); + wsRef.current?.setTime(stickySelection.start); + }, [stickySelection]); + + /* ----- Re-render waveform when highlights change ----- */ + useEffect(() => { + const ws = wsRef.current; + if (!ws || !duration) return; + ws.setOptions({}); + }, [selection, duration, waveColor, highlights]); + + /* ----- Sync static markers when the prop changes during runtime ----- */ + const markerKey = markerObjs + .map((m) => `${m.time}|${m.color ?? ''}`) + .join(','); + useEffect(() => { + const rp = regionsRef.current; + const ws = wsRef.current; + if (!rp || !ws || !ws.getDuration()) return; + rp.getRegions().forEach((r) => { + if (r.end === r.start) r.remove(); + }); + for (const m of markersRef.current) { + rp.addRegion({ + start: m.time, + drag: false, + resize: false, + color: m.color ?? DEFAULT_MARKER_COLOR, + }); + } + }, [markerKey]); + + /* ----- Fire onWaveformClick (alongside the click handler) ----- */ + useEffect(() => { + if (!onWaveformClick) return; + const ws = wsRef.current; + if (!ws) return; + return ws.on('click', (relX: number) => { + const duration = ws.getDuration(); + const waveformWidth = ws.getWrapper().getBoundingClientRect().width; + + const marker = nearestMarkerWithin( + markersRef.current ?? [], + relX, + duration, + waveformWidth + ); + onWaveformClick(relX * duration, marker); + }); + }, [onWaveformClick]); + + const hasLoadedAudio = duration > 0; + + const handlePlayToggle = () => { + if (playing) audioManager.pause(); + else audioManager.play(selfHandleRef.current); + }; + + const handleUndo = () => { + const stack = [...undoStack]; + const entry = stack.pop(); + if (!entry) return; + setUndoStack(stack); + setInternalAudio(entry.audio); + onAudioChangeRef.current?.(entry.audio, entry.payload); + wsRef.current?.once('decode', () => wsRef.current?.setTime(currentTime)); + setSelection(entry.selection); + fireSelectionChange(entry.selection, 'undo'); + }; + const handleCutClick = async () => { + if (!internalAudio || !selection) return; + pushUndo(); + try { + const spliced = await spliceAudio( + internalAudio, + selection.start, + selection.end + ); + // Clear selection and regions before loading new audio + setSelection(null); + fireSelectionChange(null, 'user'); + regionsRef.current?.getRegions().forEach((r) => { + if (r.start !== r.end) r.remove(); + }); + setInternalAudio(spliced); + onAudioChangeRef.current?.(spliced); + wsRef.current?.once('decode', () => + wsRef.current?.setTime(currentTime) + ); + } catch { + // Splice failed — leave audio unchanged + } + }; + const handleTrashClick = () => { + pushUndo(); + setInternalAudio(null); + onAudioChangeRef.current?.(null); + const ws = wsRef.current; + if (ws) { + ws.setTime(0); + ws.empty(); + ws.getMediaElement().load(); + ws.setOptions({ cursorWidth: 0 }); + } + setDuration(0); + setSelection(null); + }; + const handleInsertSilenceClick = async () => { + if (!internalAudio) return; + pushUndo(); + const withSilence = await insertSilenceAudio( + internalAudio, + currentTime, + 0.5 + ); + setInternalAudio(withSilence); + onAudioChangeRef.current?.(withSilence); + wsRef.current?.once('decode', () => wsRef.current?.setTime(currentTime)); + }; + const applyAi = async ( + func: ( + audio: Blob, + range: { start: number; end: number } + ) => Promise + ) => { + wsRef.current?.pause(); + onAudioChangeRef.current?.(null); + const result = await func( + internalAudio!, + selection ?? { start: 0, end: duration } + ); + if (!result) { + onAudioChangeRef.current?.(internalAudio); + return; + } + pushUndo(); + setInternalAudio(result); + onAudioChangeRef.current?.(result); + wsRef.current?.once('decode', () => wsRef.current?.setTime(currentTime)); + }; + + const handleMenuOpen = (event: React.MouseEvent) => { + setMenuAnchorEl(event.currentTarget); + }; + const handleMenuClose = () => { + setMenuAnchorEl(null); + }; + const isMenuOpen = Boolean(menuAnchorEl); + const hasMenu = + React.Children.count(menuItemsProp) > 0 || showTrash || allowRecord; + + const timeDisplay = formatTimeDisplay + ? formatTimeDisplay(currentTime, duration) + : selection + ? `${formatTime(selection.start)} - ${formatTime(selection.end)}` + : `${formatTime(currentTime)} / ${formatTime(duration || 0)}`; + + return ( + + + + {/* Left button: Record/Stop when showRecordButton and no audio (or actively recording), otherwise Play/Pause */} + {showRecordButton && (!hasLoadedAudio || recording) ? ( + recording ? ( + + + + ) : ( + + + + ) + ) : ( + + {playing ? ( + + ) : ( + + )} + + )} + + {timeDisplay} + + {topRowLabel ?? } + {showUndo && undoStack.length > 0 && ( + + + + )} + {showCut && ( + + + + )} + {showSilence && ( + + + + )} + {hasMenu && ( + + + + )} + + + {hasMenu && ( + + {allowRecord && ( + { + handleMenuClose(); + setMicDialogOpen(true); + }} + > + + + + Select Microphone... + + )} + {allowRecord && !isBoldWorkflow && ( + { + handleMenuClose(); + applyAi(suppressNoise); + }} + > + + + + {t.reduceNoiseAi} + + )} + {allowRecord && !isBoldWorkflow && ( + { + handleMenuClose(); + setVoiceDialogOpen(true); + }} + > + + + + {t.convertVoiceAi}... + + )} + {showTrash && ( + { + handleMenuClose(); + handleTrashClick(); + }} + > + + + + Reset Audio + + )} + {menuItemsProp} + + )} + {micDialogOpen && ( + setMicDialogOpen(false)} + onConfirm={(id) => { + localStorage.setItem(localUserKey(LocalKey.microphoneId), id); + setMicrophoneId(id); + setMicDialogOpen(false); + }} + /> + )} + {voiceDialogOpen && ( + setVoiceDialogOpen(false)} + begin={(voice) => { + setVoiceDialogOpen(false); + applyAi((audio, range) => convertVoice(audio, range, voice)); + }} + /> + )} + + + {enableZoom && ( + + )} + + {conversionProcessing && ( + + )} + {children} + + ); + } +); + +AudioPlayer.displayName = 'AudioPlayer'; diff --git a/web/src/components/AudioTab/usePassageVersionAudioRows.test.ts b/web/src/components/AudioTab/usePassageVersionAudioRows.test.ts index da077d74f..2a186db23 100644 --- a/web/src/components/AudioTab/usePassageVersionAudioRows.test.ts +++ b/web/src/components/AudioTab/usePassageVersionAudioRows.test.ts @@ -30,6 +30,11 @@ jest.mock('./getMedia', () => ({ getMedia: jest.fn(), })); +jest.mock('../../utils/logErrorService', () => ({ + logError: jest.fn(), + Severity: { error: 'error' }, +})); + import type { IState } from '../../model'; import type { IBookNameData } from '../../model/bookName'; import type { MediaFileD, Passage, Section } from '../../model'; diff --git a/web/src/components/AudioTab/usePassageVersionAudioRows.ts b/web/src/components/AudioTab/usePassageVersionAudioRows.ts index 928bbaa93..b6a15ec58 100644 --- a/web/src/components/AudioTab/usePassageVersionAudioRows.ts +++ b/web/src/components/AudioTab/usePassageVersionAudioRows.ts @@ -20,6 +20,7 @@ import { } from '../../crud'; import { IRow, IGetMedia } from '.'; import { getMedia } from './getMedia'; +import { isAutosaveMedia } from '../../crud/autosaveMedia'; import { useOrbitData } from '../../hoc/useOrbitData'; import { useSelector } from 'react-redux'; import { @@ -28,18 +29,19 @@ import { } from '../../crud/useProjectDefaults'; /** - * Count of vernacular media files for a passage (same filter as VersionDlg rows). + * Count of vernacular media files for a passage (same filter as VersionDlg + * rows), with all autosaves together counted as one. */ export function usePassageVernacularVersionCount(passId: string): number { const mediaFiles = useOrbitData('mediafile'); const { IsVernacularMedia } = useArtifactType(); - return useMemo( - () => - mediaFiles.filter( - (m) => related(m, 'passage') === passId && IsVernacularMedia(m) - ).length, - [mediaFiles, passId, IsVernacularMedia] - ); + return useMemo(() => { + const vern = mediaFiles.filter( + (m) => related(m, 'passage') === passId && IsVernacularMedia(m) + ); + const autosaves = vern.filter(isAutosaveMedia).length; + return vern.length - autosaves + Math.min(autosaves, 1); + }, [mediaFiles, passId, IsVernacularMedia]); } /** diff --git a/web/src/components/ConvertVoiceDialog.tsx b/web/src/components/ConvertVoiceDialog.tsx new file mode 100644 index 000000000..cc80b7a68 --- /dev/null +++ b/web/src/components/ConvertVoiceDialog.tsx @@ -0,0 +1,36 @@ +import { Typography, useMediaQuery, useTheme } from '@mui/material'; +import { shallowEqual, useSelector } from 'react-redux'; +import BigDialog from '../hoc/BigDialog'; +import BigDialogBp from '../hoc/BigDialogBp'; +import SelectVoice from '../business/voice/SelectVoice'; +import { IWsAudioPlayerStrings } from '../model'; +import { wsAudioPlayerSelector } from '../selector'; + +export interface ConvertVoiceDialogProps { + onCancel: () => void; + begin: (voice: string) => void; +} + +/** Lets the user pick the voice to convert the recording to. */ +export function ConvertVoiceDialog({ + onCancel, + begin, +}: ConvertVoiceDialogProps) { + const t: IWsAudioPlayerStrings = useSelector( + wsAudioPlayerSelector, + shallowEqual + ); + const theme = useTheme(); + const isSmallScreen = useMediaQuery(theme.breakpoints.down('md')); + return ( + {t.selectVoicePrompt}} + isOpen + onOpen={onCancel} + bp={isSmallScreen ? BigDialogBp.mobile : BigDialogBp.sm} + > + + + ); +} diff --git a/web/src/components/MicrophoneDialog.tsx b/web/src/components/MicrophoneDialog.tsx new file mode 100644 index 000000000..ec52943e1 --- /dev/null +++ b/web/src/components/MicrophoneDialog.tsx @@ -0,0 +1,122 @@ +import { useCallback, useEffect, useState } from 'react'; +import { + Box, + Button, + Dialog, + DialogActions, + DialogContent, + DialogTitle, + MenuItem, + TextField, +} from '@mui/material'; + +export interface MicrophoneDialogProps { + microphoneId: string; + onCancel: () => void; + onConfirm: (deviceId: string) => void; +} + +/** Lets the user pick a microphone, with a live level meter. */ +export function MicrophoneDialog({ + microphoneId, + onCancel, + onConfirm, +}: MicrophoneDialogProps) { + const [inputs, setInputs] = useState([]); + const [pickedId, setPickedId] = useState(microphoneId); + // The pick if present, else the OS default, else the first mic. + const chosenId = ( + inputs.find((d) => d.deviceId === pickedId) ?? + inputs.find((d) => d.deviceId === 'default') ?? + inputs[0] + )?.deviceId; + + useEffect(() => { + navigator.mediaDevices.enumerateDevices().then( + (devices) => setInputs(devices.filter((d) => d.kind === 'audioinput')), + () => {} + ); + }, []); + + const meterRef = useCallback( + (bar: HTMLDivElement | null) => { + if (!bar || !chosenId) return; + const context = new AudioContext(); + const analyser = context.createAnalyser(); + const samples = new Float32Array(analyser.fftSize); + let frame = 0; + const draw = () => { + analyser.getFloatTimeDomainData(samples); + const rms = Math.sqrt( + samples.reduce((sum, v) => sum + v * v, 0) / samples.length + ); + // -60 dB (silence) to 0 dB (full scale) + bar.style.transform = `scaleX(${Math.max(0, 1 + Math.log10(rms) / 3)})`; + frame = requestAnimationFrame(draw); + }; + draw(); + const stream = navigator.mediaDevices.getUserMedia({ + audio: { deviceId: { exact: chosenId }, autoGainControl: false }, + }); + stream + .then((s) => context.createMediaStreamSource(s).connect(analyser)) + .catch(() => {}); // mic unavailable, or dialog closed before it opened + return () => { + cancelAnimationFrame(frame); + void context.close(); + stream.then( + (s) => s.getTracks().forEach((t) => t.stop()), + () => {} + ); + }; + }, + [chosenId] + ); + + if (!chosenId) return null; + return ( + + Select Microphone + + setPickedId(e.target.value)} + > + {inputs.map((d) => ( + + {d.label} + + ))} + + + + + + + + + + + ); +} diff --git a/web/src/components/PassageDetail/PassageDetailChooser.tsx b/web/src/components/PassageDetail/PassageDetailChooser.tsx index 48704905c..c3fa7a73e 100644 --- a/web/src/components/PassageDetail/PassageDetailChooser.tsx +++ b/web/src/components/PassageDetail/PassageDetailChooser.tsx @@ -39,7 +39,6 @@ export const PassageDetailChooser = ({ sx }: IProps) => { section, prjId, allBookData, - setCurrentStep, isNavigationBlocked, } = usePassageDetailContext(); const [passageCount, setPassageCount] = useState(0); @@ -47,13 +46,9 @@ export const PassageDetailChooser = ({ sx }: IProps) => { const marks = useRef>([]); const [view, setView] = useState(''); const { getSharedResource } = useSharedResRead(); - const passageNavigate = usePassageNavigate( - () => { - setView(''); - }, - setCurrentStep, - isNavigationBlocked - ); + const passageNavigate = usePassageNavigate(() => { + setView(''); + }, isNavigationBlocked); const t = useSelector( passageChooserSelector, @@ -138,6 +133,7 @@ export const PassageDetailChooser = ({ sx }: IProps) => { false); const mockMemoryUpdate = jest.fn().mockResolvedValue(undefined); +const mockStartRecording = jest.fn().mockResolvedValue(undefined); +const mockIsAutosavePending = jest.fn(); +const mockClearPendingUploads = jest.fn(); const PENDING_WARN_TEXT = 'A recording for this passage is already pending upload. Recording it again will replace the existing pending recording. Do you want to continue?'; -let capturedMediaRecordProps: { - afterUploadCb: (mediaId: string | undefined) => Promise; - onRecordingCleared?: () => void | Promise; - allowRecord?: boolean; - onBeforeStartRecord?: () => boolean | Promise; +const CLEAR_ARGS = [expect.anything(), 'plan1', 'p1']; +// Pending upload rows store server ids, not the local ids the step holds. +const REMOTE_IDS: Record = { plan1: '101', p1: '201' }; + +let capturedPlayerProps: { + onAudioChange?: (audio: Blob | null) => void; +} | null = null; + +let capturedUploaderProps: { + finish?: (planId: string, mediaRemoteIds?: string[]) => Promise; } | null = null; const linkedSharedResource = { @@ -38,11 +46,10 @@ const passageDetailCtx = { sharedResource: undefined as unknown, mediafileId: 'mf1', recording: false, - setRecording: jest.fn(), currentstep: 'step-record', isBoldWorkflow: true, setStepComplete: mockSetStepComplete, - gotoNextStep: mockGotoNextStep, + stepComplete: mockStepComplete, }; jest.mock( @@ -50,23 +57,41 @@ jest.mock( () => () => passageDetailCtx ); -jest.mock('../MediaRecord', () => ({ - __esModule: true, - default: (props: { - afterUploadCb: (mediaId: string | undefined) => Promise; - onRecordingCleared?: () => void | Promise; - allowRecord?: boolean; - onBeforeStartRecord?: () => boolean | Promise; - }) => { - capturedMediaRecordProps = props; - return
; - }, +// The real player needs wavesurfer.js, whose ESM-only dist files jest can't load. +jest.mock('../AudioPlayer', () => { + const ReactActual = jest.requireActual('react'); + const MockAudioPlayer = ReactActual.forwardRef( + (props: { onAudioChange?: (audio: Blob | null) => void }, ref) => { + capturedPlayerProps = props; + ReactActual.useImperativeHandle(ref, () => ({ + startRecording: mockStartRecording, + setTime: jest.fn(), + })); + return
; + } + ); + MockAudioPlayer.displayName = 'MockAudioPlayer'; + return { + __esModule: true, + AudioPlayer: MockAudioPlayer, + }; +}); + +jest.mock('./usePassageAudioSave', () => ({ + usePassageAudioSave: () => ({ + save: jest.fn().mockResolvedValue(undefined), + saving: false, + }), })); jest.mock( '../Uploader', () => - function MockUploader(props: { isOpen?: boolean }) { + function MockUploader(props: { + isOpen?: boolean; + finish?: (planId: string, mediaRemoteIds?: string[]) => Promise; + }) { + capturedUploaderProps = props; return props.isOpen ?
: null; } ); @@ -88,16 +113,14 @@ jest.mock('../AlertDialog', () => ({
), })); -jest.mock('../Sheet/AudacityManager', () => () => null); jest.mock('../../hoc/BigDialog', () => () => null); jest.mock('../AudioTab/VersionDlg', () => () => null); +// Recording needs a speaker, so name one as the real control would on load. jest.mock('../SpeakerName', () => { const ReactActual = jest.requireActual('react'); - const MockSpeakerName = (props: { - onRights?: (hasRights: boolean) => void; - }) => { + const MockSpeakerName = (props: { onChange?: (name: string) => void }) => { ReactActual.useEffect(() => { - props.onRights?.(true); + props.onChange?.('Speaker'); }, []); return null; }; @@ -123,17 +146,15 @@ jest.mock('../../control', () => ({ ) : null, })); -jest.mock('../../hoc/useOrbitData', () => ({ - useOrbitData: () => [], -})); - jest.mock('../../context/useGlobal', () => ({ useGlobal: (key: string) => { if (key === 'memory') { - return [{ keyMap: {}, update: mockMemoryUpdate }, jest.fn()]; + const keyMap = { + idToKey: (_table: string, _key: string, id: string) => REMOTE_IDS[id], + }; + return [{ keyMap, update: mockMemoryUpdate }, jest.fn()]; } if (key === 'plan') return ['plan1', jest.fn()]; - if (key === 'offline') return [false, jest.fn()]; if (key === 'errorReporter') return [jest.fn(), jest.fn()]; if (key === 'importexportBusy') return [false, jest.fn()]; return [undefined, jest.fn()]; @@ -146,6 +167,7 @@ jest.mock('../../context/UnsavedContext', () => { UnsavedContext: ReactActual.createContext({ state: { startSave: jest.fn(), + saveCompleted: jest.fn(), toolChanged: jest.fn(), toolsChanged: 0, saveRequested: jest.fn(() => false), @@ -158,8 +180,7 @@ jest.mock('../../context/UnsavedContext', () => { }); jest.mock('../../crud', () => ({ - VernacularTag: null, - findRecord: jest.fn(), + getVernacularMediaRec: jest.fn(() => undefined), related: jest.fn( ( rec: { @@ -175,19 +196,37 @@ jest.mock('../../crud', () => ({ } ), remoteIdGuid: jest.fn((_t: string, id: string) => id), - remoteIdNum: jest.fn(() => NaN), - MediaSt: { FETCHED: 'FETCHED' }, + MediaSt: { FETCHED: 'FETCHED', ERROR: 'ERROR' }, useFetchMediaUrl: () => ({ fetchMediaUrl: jest.fn(), mediaState: { id: 'mf1', status: 'FETCHED' }, }), - useSharedResRead: () => ({ getSharedResource: () => undefined }), +})); + +jest.mock('../../crud/autosaveMedia', () => ({ + isAutosaveMedia: () => false, + isAutosavePending: mockIsAutosavePending, +})); + +jest.mock('../../store/upload/pendingMediaUploads', () => ({ + ...jest.requireActual< + typeof import('../../store/upload/pendingMediaUploads') + >('../../store/upload/pendingMediaUploads'), + clearPendingUploads: mockClearPendingUploads, })); jest.mock('../../crud/useStepTool', () => ({ useStepTool: () => ({ settings: {} }), })); +jest.mock('../../crud/useWavRecorder', () => ({ + parseRecordCaptureAudioProcessing: () => ({ + echoCancellation: false, + noiseSuppression: false, + }), + parseRecordSaveAsWav: () => false, +})); + jest.mock('../../utils/useStepPermission', () => ({ useStepPermissions: () => ({ canDoVernacular: () => true, @@ -195,11 +234,7 @@ jest.mock('../../utils/useStepPermission', () => ({ })); jest.mock('../../utils', () => ({ - useMobile: () => ({ isMobile: false }), -})); - -jest.mock('../../utils/passageDefaultFilename', () => ({ - passageDefaultFilename: () => 'file.ogg', + loadBlobAsync: jest.fn().mockResolvedValue(undefined), })); jest.mock('../../selector', () => ({ @@ -234,18 +269,21 @@ import { PassageDetailRecord } from './PassageDetailRecord'; const renderRecord = () => render(); +const recordButton = () => + document.querySelector('[aria-label="record"]'); + const seedPendingForCurrentPassage = () => { appendPendingMediaUpload({ localAbsolutePath: '/pending/luke.mp3', fileSize: 100, uploadType: UploadType.Media, record: { - planId: 'plan1', + planId: REMOTE_IDS.plan1, versionNumber: 1, originalFile: 'luke.mp3', contentType: 'audio/mpeg', artifactTypeId: '', - passageId: 'p1', + passageId: REMOTE_IDS.p1, userId: 'user-1', recordedbyUserId: 'user-1', sourceMediaId: '', @@ -254,51 +292,88 @@ const seedPendingForCurrentPassage = () => { }); }; +const resetState = () => { + jest.clearAllMocks(); + localStorage.clear(); + capturedPlayerProps = null; + capturedUploaderProps = null; + mockStepComplete.mockReturnValue(false); + passageDetailCtx.isBoldWorkflow = true; + passageDetailCtx.mediafileId = 'mf1'; + passageDetailCtx.currentstep = 'step-record'; + passageDetailCtx.sharedResource = undefined; +}; + describe('PassageDetailRecord BOLD step completion', () => { - beforeEach(() => { - jest.clearAllMocks(); - localStorage.clear(); - capturedMediaRecordProps = null; - passageDetailCtx.isBoldWorkflow = true; - passageDetailCtx.mediafileId = 'mf1'; - passageDetailCtx.currentstep = 'step-record'; - passageDetailCtx.sharedResource = undefined; + beforeEach(resetState); + + it('completes the step when audio changes when BOLD', () => { + renderRecord(); + expect(capturedPlayerProps).not.toBeNull(); + + act(() => { + capturedPlayerProps!.onAudioChange!(new Blob(['audio'])); + }); + + expect(mockSetStepComplete).toHaveBeenCalledWith('step-record', true); + }); + + it('does not complete the step when audio changes when not BOLD', () => { + passageDetailCtx.isBoldWorkflow = false; + renderRecord(); + + act(() => { + capturedPlayerProps!.onAudioChange!(new Blob(['audio'])); + }); + + expect(mockSetStepComplete).not.toHaveBeenCalled(); }); - it('auto-completes and advances after successful save when BOLD', async () => { + it('does not complete a step that is already complete', () => { + mockStepComplete.mockReturnValue(true); + renderRecord(); + + act(() => { + capturedPlayerProps!.onAudioChange!(new Blob(['audio'])); + }); + + expect(mockSetStepComplete).not.toHaveBeenCalled(); + }); + + it('completes the step and clears pending uploads after Load File uploads media', async () => { renderRecord(); - expect(capturedMediaRecordProps).not.toBeNull(); await act(async () => { - await capturedMediaRecordProps!.afterUploadCb('new-media-id'); + await capturedUploaderProps!.finish!('plan1', ['new-media-id']); }); expect(mockSetStepComplete).toHaveBeenCalledWith('step-record', true); - expect(mockGotoNextStep).toHaveBeenCalled(); + expect(mockClearPendingUploads).toHaveBeenCalledWith(...CLEAR_ARGS); }); - it('does not auto-complete after save when not BOLD', async () => { - passageDetailCtx.isBoldWorkflow = false; + it('does not complete the step when Load File uploads nothing', async () => { renderRecord(); await act(async () => { - await capturedMediaRecordProps!.afterUploadCb('new-media-id'); + await capturedUploaderProps!.finish!('plan1', []); }); expect(mockSetStepComplete).not.toHaveBeenCalled(); - expect(mockGotoNextStep).not.toHaveBeenCalled(); + expect(mockClearPendingUploads).not.toHaveBeenCalled(); }); it('marks step incomplete and deletes mediafile when recording cleared', async () => { renderRecord(); - expect(capturedMediaRecordProps?.onRecordingCleared).toBeDefined(); - await act(async () => { - await capturedMediaRecordProps!.onRecordingCleared!(); + act(() => { + capturedPlayerProps!.onAudioChange!(null); }); + expect(mockClearPendingUploads).toHaveBeenCalledWith( + ...CLEAR_ARGS, + mockIsAutosavePending + ); expect(mockSetStepComplete).toHaveBeenCalledWith('step-record', false); - expect(mockMemoryUpdate).toHaveBeenCalled(); const updateFn = mockMemoryUpdate.mock.calls[0][0] as ( tr: RecordTransformBuilder ) => unknown; @@ -313,24 +388,10 @@ describe('PassageDetailRecord BOLD step completion', () => { id: 'mf1', }); }); - - it('does not pass onRecordingCleared when not BOLD', () => { - passageDetailCtx.isBoldWorkflow = false; - renderRecord(); - expect(capturedMediaRecordProps?.onRecordingCleared).toBeUndefined(); - }); }); describe('PassageDetailRecord linked note play-only (TT-5873)', () => { - beforeEach(() => { - jest.clearAllMocks(); - localStorage.clear(); - capturedMediaRecordProps = null; - passageDetailCtx.isBoldWorkflow = true; - passageDetailCtx.mediafileId = 'mf1'; - passageDetailCtx.currentstep = 'step-record'; - passageDetailCtx.sharedResource = undefined; - }); + beforeEach(resetState); it('keeps Load File and recording enabled on the source note', async () => { passageDetailCtx.sharedResource = sourceOwnedSharedResource; @@ -338,10 +399,10 @@ describe('PassageDetailRecord linked note play-only (TT-5873)', () => { await waitFor(() => { expect(document.getElementById('pdRecordLoadFile')).not.toBeNull(); - expect(capturedMediaRecordProps?.allowRecord).toBe(true); + expect(recordButton()).not.toBeNull(); }); expect( - document.querySelector('[data-testid="media-record"]') + document.querySelector('[data-testid="audio-player"]') ).not.toBeNull(); }); @@ -350,26 +411,18 @@ describe('PassageDetailRecord linked note play-only (TT-5873)', () => { renderRecord(); await waitFor(() => { - expect(capturedMediaRecordProps).not.toBeNull(); + expect(capturedPlayerProps).not.toBeNull(); }); expect(document.getElementById('pdRecordLoadFile')).toBeNull(); - expect(capturedMediaRecordProps?.allowRecord).toBe(false); + expect(recordButton()).toBeNull(); expect( - document.querySelector('[data-testid="media-record"]') + document.querySelector('[data-testid="audio-player"]') ).not.toBeNull(); }); }); describe('PassageDetailRecord pending upload warn (TT-7366)', () => { - beforeEach(() => { - jest.clearAllMocks(); - localStorage.clear(); - capturedMediaRecordProps = null; - passageDetailCtx.isBoldWorkflow = true; - passageDetailCtx.mediafileId = 'mf1'; - passageDetailCtx.currentstep = 'step-record'; - passageDetailCtx.sharedResource = undefined; - }); + beforeEach(resetState); it('opens Load File immediately when there is no pending upload', async () => { renderRecord(); @@ -382,9 +435,10 @@ describe('PassageDetailRecord pending upload warn (TT-7366)', () => { expect( document.querySelector('[data-testid="uploader-open"]') ).not.toBeNull(); + expect(mockClearPendingUploads).not.toHaveBeenCalled(); }); - it('warns on Load File when a pending upload exists; No cancels; Yes opens Uploader', async () => { + it('warns on Load File when a pending upload exists; No cancels; Yes opens Uploader and keeps pending', async () => { seedPendingForCurrentPassage(); renderRecord(); @@ -404,6 +458,7 @@ describe('PassageDetailRecord pending upload warn (TT-7366)', () => { document.getElementById('alertNo')?.click(); }); expect(document.querySelector('[data-testid="uploader-open"]')).toBeNull(); + expect(mockClearPendingUploads).not.toHaveBeenCalled(); await act(async () => { document.getElementById('pdRecordLoadFile')?.click(); @@ -419,45 +474,44 @@ describe('PassageDetailRecord pending upload warn (TT-7366)', () => { document.querySelector('[data-testid="uploader-open"]') ).not.toBeNull(); }); + expect(mockClearPendingUploads).not.toHaveBeenCalled(); + expect(mockStartRecording).not.toHaveBeenCalled(); }); - it('allows onBeforeStartRecord immediately when there is no pending upload', async () => { + it('starts recording immediately when there is no pending upload', async () => { renderRecord(); - expect(capturedMediaRecordProps?.onBeforeStartRecord).toBeDefined(); - let allowed: boolean | undefined; + await act(async () => { - allowed = await capturedMediaRecordProps!.onBeforeStartRecord!(); + recordButton()?.click(); }); - expect(allowed).toBe(true); + + expect(mockStartRecording).toHaveBeenCalledTimes(1); expect( document.querySelector('[data-testid="pending-warn-confirm"]') ).toBeNull(); }); - it('warns before record start when pending exists; No blocks; Yes allows', async () => { + it('warns before record start when pending exists; No blocks; Yes records and clears pending', async () => { seedPendingForCurrentPassage(); renderRecord(); - expect(capturedMediaRecordProps?.onBeforeStartRecord).toBeDefined(); - let blockedPromise: Promise; await act(async () => { - blockedPromise = Promise.resolve( - capturedMediaRecordProps!.onBeforeStartRecord!() - ); + recordButton()?.click(); }); await waitFor(() => { expect(document.getElementById('alertNo')).not.toBeNull(); }); + expect(document.body.textContent).toContain(PENDING_WARN_TEXT); + expect(mockStartRecording).not.toHaveBeenCalled(); + await act(async () => { document.getElementById('alertNo')?.click(); }); - expect(await blockedPromise!).toBe(false); + expect(mockStartRecording).not.toHaveBeenCalled(); + expect(mockClearPendingUploads).not.toHaveBeenCalled(); - let allowedPromise: Promise; await act(async () => { - allowedPromise = Promise.resolve( - capturedMediaRecordProps!.onBeforeStartRecord!() - ); + recordButton()?.click(); }); await waitFor(() => { expect(document.getElementById('alertYes')).not.toBeNull(); @@ -465,6 +519,9 @@ describe('PassageDetailRecord pending upload warn (TT-7366)', () => { await act(async () => { document.getElementById('alertYes')?.click(); }); - expect(await allowedPromise!).toBe(true); + + expect(mockClearPendingUploads).toHaveBeenCalledWith(...CLEAR_ARGS); + expect(mockStartRecording).toHaveBeenCalledTimes(1); + expect(document.querySelector('[data-testid="uploader-open"]')).toBeNull(); }); }); diff --git a/web/src/components/PassageDetail/PassageDetailRecord.tsx b/web/src/components/PassageDetail/PassageDetailRecord.tsx index a2a5cbd06..584dab657 100644 --- a/web/src/components/PassageDetail/PassageDetailRecord.tsx +++ b/web/src/components/PassageDetail/PassageDetailRecord.tsx @@ -1,71 +1,56 @@ import { shallowEqual, useSelector } from 'react-redux'; -import { Typography, Box, Stack } from '@mui/material'; +import { Box, Stack } from '@mui/material'; import FolderOpenOutlinedIcon from '@mui/icons-material/FolderOpenOutlined'; +import PauseIcon from '@mui/icons-material/Pause'; +import StopIcon from '@mui/icons-material/Stop'; +import { InsertDriveFileOutlined } from '@mui/icons-material'; +import { useContext, useEffect, useMemo, useRef, useState } from 'react'; import { - useContext, - useEffect, - useMemo, - useRef, - useState, - useCallback, -} from 'react'; -import { - findRecord, - IMediaState, + getVernacularMediaRec, MediaSt, related, remoteIdGuid, - remoteIdNum, useFetchMediaUrl, - useSharedResRead, - VernacularTag, } from '../../crud'; +import { isAutosaveMedia, isAutosavePending } from '../../crud/autosaveMedia'; +import { usePassageAudioSave } from './usePassageAudioSave'; import { useGlobal } from '../../context/useGlobal'; import usePassageDetailContext from '../../context/usePassageDetailContext'; -import { passageDefaultFilename } from '../../utils/passageDefaultFilename'; import { useStepTool } from '../../crud/useStepTool'; -import MediaRecord from '../MediaRecord'; +import { + parseRecordCaptureAudioProcessing, + parseRecordSaveAsWav, +} from '../../crud/useWavRecorder'; +import { AudioPlayer, AudioPlayerHandle } from '../AudioPlayer'; import { UnsavedContext } from '../../context/UnsavedContext'; import Uploader from '../Uploader'; -import AudacityManager from '../Sheet/AudacityManager'; import { Button } from '../../control'; import BigDialog from '../../hoc/BigDialog'; -import BigDialogBp from '../../hoc/BigDialogBp'; import VersionDlg from '../AudioTab/VersionDlg'; import { usePassageVernacularVersionCount } from '../AudioTab/usePassageVersionAudioRows'; import SpeakerName from '../SpeakerName'; import { mediaTabSelector, sharedSelector } from '../../selector'; -import { useMobile } from '../../utils'; -import { useOrbitData } from '../../hoc/useOrbitData'; -import { - RecordIdentity, - RecordKeyMap, - RecordTransformBuilder, -} from '@orbit/records'; +import { loadBlobAsync } from '../../utils'; +import { RecordKeyMap, RecordTransformBuilder } from '@orbit/records'; import { useStepPermissions } from '../../utils/useStepPermission'; import { isLinkedNote } from '../../crud/isLinkedNote'; import Confirm from '../AlertDialog'; -import { hasPendingUploadForPassage } from '../../store/upload/pendingMediaUploads'; import { - IMediaTabStrings, - ISharedStrings, - IState, - MediaFileD, -} from '../../model'; - -interface IProps { - ready?: () => boolean; -} + clearPendingUploads, + hasPendingUploadForPassage, + pendingUploadIds, +} from '../../store/upload/pendingMediaUploads'; +import { IMediaTabStrings, ISharedStrings } from '../../model'; +import { useSnackBar } from '../../hoc/useSnackBar'; const SaveWait = 500; -export function PassageDetailRecord(props: IProps) { - const { ready } = props; - const mediafiles = useOrbitData('mediafile'); +export function PassageDetailRecord() { const ts: ISharedStrings = useSelector(sharedSelector, shallowEqual); const mt: IMediaTabStrings = useSelector(mediaTabSelector, shallowEqual); const { startSave, + saveCompleted, toolChanged, toolsChanged, saveRequested, @@ -74,301 +59,233 @@ export function PassageDetailRecord(props: IProps) { waitForSave, } = useContext(UnsavedContext).state; const [reporter] = useGlobal('errorReporter'); - const [, setBigBusy] = useGlobal('importexportBusy'); const [plan] = useGlobal('plan'); //will be constant here const { fetchMediaUrl, mediaState } = useFetchMediaUrl(reporter); - const [statusText, setStatusText] = useState(''); - const [canSave, setCanSave] = useState(false); - const [offline] = useGlobal('offline'); //verified this is not used in a function 2/18/25 const [memory] = useGlobal('memory'); + const keyMap = memory?.keyMap as RecordKeyMap; + const [, setBigBusy] = useGlobal('importexportBusy'); const { passage, sharedResource, mediafileId, recording, - setRecording, currentstep, isBoldWorkflow, setStepComplete, - gotoNextStep, + stepComplete, } = usePassageDetailContext(); const toolId = 'RecordTool'; + const { settings: toolSettings } = useStepTool(currentstep); - const onSaving = () => { - setBigBusy(true); - }; - const onReady = () => { - setBigBusy(false); - }; - const [importList, setImportList] = useState(); - const cancelled = useRef(false); - const [uploadVisible, setUploadVisiblex] = useState(false); - const pendingConfirmRef = useRef<{ - promise: Promise; - resolve: (ok: boolean) => void; - } | null>(null); - const [pendingWarnOpen, setPendingWarnOpen] = useState(false); - const [audacityVisible, setAudacityVisible] = useState(false); + const { showMessage } = useSnackBar(); + const { echoCancellation, noiseSuppression } = + parseRecordCaptureAudioProcessing(toolSettings); + const saveAsWav = parseRecordSaveAsWav(toolSettings); + + const [uploadVisible, setUploadVisible] = useState(false); const [versionVisible, setVersionVisible] = useState(false); - const [preload, setPreload] = useState(0); - const [recorderState, setRecorderState] = useState(); - const [hasExistingVersion, setHasExistingVersion] = useState(false); - /** Prevents handleReload ↔ mediaState PENDING/FETCHED loops; reset when mediafileId changes. */ - const recordPreloadInitiatedRef = useRef(null); - const [resetMedia, setResetMedia] = useState(false); + const [pendingUploadWarning, setPendingUploadWarning] = useState< + 'upload' | 'record' + >(); + const playerRef = useRef(null); + const [selection, setSelection] = useState<{ + start: number; + end: number; + } | null>(null); + // Audio of the passage's latest stored version (autosave or permanent). + const [savedBlob, setSavedBlob] = useState(); + // Audio in the player. + const [currentBlob, setCurrentBlob] = useState(); const [speaker, setSpeaker] = useState(''); - const [hasRights, setHasRight] = useState(false); + + const passageId = useMemo( + () => related(sharedResource, 'passage') ?? passage.id, + [sharedResource, passage] + ); + const vernacularVersionCount = usePassageVernacularVersionCount(passageId); + const { canDoVernacular } = useStepPermissions(); - const { isMobile: isMobileView } = useMobile(); - const allBookData = useSelector((state: IState) => state.books.bookData); - const { getSharedResource } = useSharedResRead(); - - const defaultFilename = useMemo(() => { - const sr = getSharedResource(passage); - return passageDefaultFilename( - passage, - plan, - memory, - VernacularTag, - offline, - '', - toolSettings, - allBookData, - sr?.attributes.title - ); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [getSharedResource, passage, plan, offline, toolSettings, allBookData]); + const edited = !!currentBlob && currentBlob !== savedBlob; + const isLatestAutosave = isAutosaveMedia( + getVernacularMediaRec(passageId, memory) + ); + const canSave = edited || (!!currentBlob && isLatestAutosave); + const busy = saveRequested(toolId); + const canRecord = + canDoVernacular(related(passage, 'section')) && + !isLinkedNote(passage, sharedResource); - const setUploadVisible = (value: boolean) => { - if (value) { - cancelled.current = false; - } - setUploadVisiblex(value); - }; + const { save } = usePassageAudioSave({ + passageId, + currentBlob, + edited, + enabled: canRecord && !recording && !busy, + performedBy: speaker, + mimeType: saveAsWav ? 'audio/wav' : undefined, + onStored: (blob) => + setSavedBlob((prev) => (prev === savedBlob ? blob : prev)), + }); useEffect(() => { - toolChanged(toolId, canSave); + if (!busy) toolChanged(toolId, edited); // eslint-disable-next-line react-hooks/exhaustive-deps - }, [canSave]); + }, [edited, busy]); useEffect(() => { - if (saveRequested(toolId)) handleSave(); - else if (clearRequested(toolId)) { - clearCompleted(toolId); + if (!saveRequested(toolId)) { + if (clearRequested(toolId)) clearCompleted(toolId); + return; } + void save().then( + () => saveCompleted(toolId), + () => saveCompleted(toolId, ts.NoSaveWoMedia) + ); // eslint-disable-next-line react-hooks/exhaustive-deps }, [toolsChanged]); - useEffect(() => { - if (!mediafileId) { - fetchMediaUrl({ id: mediafileId }); - setResetMedia(true); - } else if (mediafileId !== mediaState.id) { - fetchMediaUrl({ id: mediafileId }); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [mediafileId, passage]); - const performedByFromMedia = useMemo(() => { - const mediaRec = findRecord(memory, 'mediafile', mediafileId) as - MediaFileD | undefined; - return mediaRec?.attributes?.performedBy; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [memory, mediafileId, mediafiles]); + // `stale` drops audio that arrives after the user moved to another passage. + const showBlob = (url: string, stale = () => false) => + loadBlobAsync(url) + .then((blob) => { + if (stale()) return; + if (!blob) throw new Error(ts.mediaError); + setSavedBlob(blob); + setCurrentBlob(blob); + }) + .catch(() => { + if (stale()) return; + setBigBusy(false); + showMessage(ts.mediaError); + }); - useEffect(() => { - if (performedByFromMedia) { - setSpeaker(performedByFromMedia); - setHasRight(true); - } - }, [mediafileId, performedByFromMedia]); + const loadLatest = () => { + const mediaFile = getVernacularMediaRec(passageId, memory); + if (mediaFile?.attributes?.performedBy) + setSpeaker(mediaFile.attributes.performedBy); + if (mediaFile) setBigBusy(true); + // fetchMediaUrl skips an id it already has. + if (mediaFile?.id === mediaState.id) void showBlob(mediaState.url); + else fetchMediaUrl({ id: mediaFile?.id ?? '' }); + }; useEffect(() => { - recordPreloadInitiatedRef.current = null; - }, [mediafileId]); + loadLatest(); + return () => setBigBusy(false); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); useEffect(() => { - const hasExisting = - Boolean(mediafileId) && - recorderState?.status === MediaSt.FETCHED && - recorderState?.id === mediafileId; - const shouldAutoPreload = - hasExisting && recordPreloadInitiatedRef.current !== mediafileId; - if (shouldAutoPreload) { - recordPreloadInitiatedRef.current = mediafileId; - setStatusText(ts.loading); - handleReload(); + if (mediaState.status === MediaSt.ERROR) { + setBigBusy(false); + showMessage(mediaState.error || ts.mediaError); + return; } - setHasExistingVersion(hasExisting); - }, [mediafileId, recorderState, ts.loading]); + if (mediaState.status !== MediaSt.FETCHED || !mediaState.url) return; - const passageId = useMemo( - () => related(sharedResource, 'passage') ?? passage.id, - [sharedResource, passage] - ); - const vernacularVersionCount = usePassageVernacularVersionCount(passageId); - - const uploadShapedIds = useCallback(() => { - const keyMap = memory?.keyMap as RecordKeyMap; - const planRemote = remoteIdNum('plan', plan, keyMap) || plan; - const passageRemote = - remoteIdNum('passage', passageId, keyMap) || passageId; - return { - planId: String(planRemote), - passageId: String(passageRemote), + let stale = false; + void showBlob(mediaState.url, () => stale); + return () => { + stale = true; }; - }, [memory, plan, passageId]); - - const confirmIfPending = useCallback((): Promise => { - const ids = uploadShapedIds(); - if ( - !hasPendingUploadForPassage({ - ...ids, - artifactTypeId: null, - }) - ) { - return Promise.resolve(true); - } - if (pendingConfirmRef.current) { - return pendingConfirmRef.current.promise; - } - let resolve!: (ok: boolean) => void; - const promise = new Promise((res) => { - resolve = res; - }); - pendingConfirmRef.current = { promise, resolve }; - setPendingWarnOpen(true); - return promise; - }, [uploadShapedIds]); - - const handlePendingWarnYes = () => { - pendingConfirmRef.current?.resolve(true); - pendingConfirmRef.current = null; - setPendingWarnOpen(false); - }; - const handlePendingWarnNo = () => { - pendingConfirmRef.current?.resolve(false); - pendingConfirmRef.current = null; - setPendingWarnOpen(false); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [mediaState]); + + const markCompleteForBold = () => { + if (isBoldWorkflow && !stepComplete(currentstep)) + void setStepComplete(currentstep, true); }; - const onBeforeStartRecord = useCallback( - () => confirmIfPending(), - [confirmIfPending] - ); + const hasPendingUpload = () => + hasPendingUploadForPassage({ + ...pendingUploadIds(keyMap, plan, passageId), + artifactTypeId: null, + }); - const handleSave = () => { - startSave(toolId); - }; - const afterUploadCb = async (mediaId: string | undefined) => { - if (mediaId) { - setStatusText(''); - if (isBoldWorkflow) { - await setStepComplete(currentstep, true); - gotoNextStep(); - } - } else setStatusText(ts.NoSaveWoMedia); - }; - const handleRecordingCleared = useCallback(async () => { - if (!isBoldWorkflow) return; - await setStepComplete(currentstep, false); - if (mediafileId) { - await memory.update((tr: RecordTransformBuilder) => - tr.removeRecord({ type: 'mediafile', id: mediafileId }).toOperation() + const startRecording = async () => { + clearPendingUploads(keyMap, plan, passageId); + try { + await playerRef.current?.startRecording(); + } catch { + showMessage( + 'Could not access microphone. Please allow microphone access and try again.' ); } - recordPreloadInitiatedRef.current = null; - setResetMedia(true); - }, [isBoldWorkflow, setStepComplete, currentstep, mediafileId, memory]); + }; + + const pendingUploadWarningClosed = (confirm: boolean) => () => { + if (confirm) { + if (pendingUploadWarning === 'upload') setUploadVisible(true); + else void startRecording(); + } + setPendingUploadWarning(undefined); + }; + const afterUpload = async (mediaRemoteIds?: string[]) => { const id = mediaRemoteIds && mediaRemoteIds.length > 0 - ? remoteIdGuid( - 'mediafile', - mediaRemoteIds[0], - memory?.keyMap as RecordKeyMap - ) || mediaRemoteIds[0] + ? remoteIdGuid('mediafile', mediaRemoteIds[0], keyMap) || + mediaRemoteIds[0] : undefined; - if (cancelled.current && id) { - await memory.update((tr: RecordTransformBuilder) => - tr.removeRecord({ type: 'mediafile', id }).toOperation() - ); - } else { - await afterUploadCb(id); - } - if (id) handleReload(); - cancelled.current = false; - if (importList) { - setImportList(undefined); - setUploadVisible(false); - setAudacityVisible(false); + if (id) { + clearPendingUploads(keyMap, plan, passageId); + loadLatest(); + markCompleteForBold(); } + playerRef.current?.setTime(0); + }; + + const handleVersionApplied = () => { + clearPendingUploads(keyMap, plan, passageId, isAutosavePending); + loadLatest(); }; const saveIfChanged = (cb: () => void) => { if (canSave) { startSave(toolId); - waitForSave(() => cb(), SaveWait); + waitForSave(cb, SaveWait).catch((err: Error) => showMessage(err.message)); } else cb(); }; - // TODO cleanup: the importList upload flow doesn't involve showing the Upload dialog, - // so it would make sense for us to remove the importList from the Uploader component and - // use useMediaUpload directly for such flows instead. Or possibly get rid of importList - // altogether and do `await run(stagedFiles)` instead. And then here we need to look into - // whether to a) switch PassageDetailRecord entirely off of Uploader b) have it use both - // Uploader and useMediaUpload separately c) leave Uploader as is or give it useImperativeHandle - // or something like that. - // Compare with what ResourceUploadStep.tsx and see if we should make the consistent. - // Claude says: "the current effect catches rejections (logs them and shows the error). If you move to await run(...), keep that catch at the call site so a total-failure rejection doesn't escape." - const handleAudacityImport = (i: number, list: File[]) => { - saveIfChanged(() => { - void confirmIfPending().then((ok) => { - if (!ok) return; - setImportList(list); - setUploadVisible(true); - }); - }); - }; + async function handleRecordToggle() { + if (busy || !speaker) return; + + if (!recording) { + if (selection) { + showMessage( + 'Recording is not supported while there is a selected time range.' + ); + return; + } + if (hasPendingUpload()) setPendingUploadWarning('record'); + else await startRecording(); + } else { + playerRef.current?.stopRecording(); + } + } - const handleAudacityClose = () => { - setAudacityVisible(false); - }; const handleUpload = () => { saveIfChanged(() => { - void confirmIfPending().then((ok) => { - if (ok) setUploadVisible(true); - }); + if (hasPendingUpload()) setPendingUploadWarning('upload'); + else setUploadVisible(true); }); }; - // const handleAudacity = () => { - // saveIfChanged(() => { - // setAudacityVisible(true); - // }); - // }; - const handleVersions = () => { - setVersionVisible(true); - }; - const handleVerHistClose = () => { - setVersionVisible(false); - }; - const handleNameChange = (name: string) => { - setSpeaker(name); - }; - const handleRights = (hasRights: boolean) => setHasRight(hasRights); - const handleReload = () => { - setPreload((p) => p + 1); - }; - const handleTrackRecorder = (state: IMediaState) => setRecorderState(state); - const handleRecording = (recording: boolean) => { - setRecording(recording); - }; - const canVern = - canDoVernacular(related(passage, 'section')) && - !isLinkedNote(passage, sharedResource); + const handleAudioChange = (audio: Blob | null) => { + setCurrentBlob(audio); + if (audio) markCompleteForBold(); + else { + clearPendingUploads(keyMap, plan, passageId, isAutosavePending); + if (isBoldWorkflow) { + setStepComplete(currentstep, false); + memory.update((tr: RecordTransformBuilder) => + tr.removeRecord({ type: 'mediafile', id: mediafileId }).toOperation() + ); + } + } + }; return ( - + - {canVern && ( + {canRecord && ( + )} + + {currentBlob && ( + + )} + + + {/* Record */} + {canRecord && ( + + {!mediafileId || !recording ? ( + - {statusText} - - {!isMobileView && canSave && ( - - )} - - } - onVersions={ - !isBoldWorkflow && hasExistingVersion && vernacularVersionCount > 1 - ? handleVersions - : undefined - } - onRecordingCleared={isBoldWorkflow ? handleRecordingCleared : undefined} - /> + {recording && } + + ) : ( + + )} + + )} setUploadVisible(false)} multiple={false} finish={afterUpload} - cancelled={cancelled} + cancelled={useRef(false)} passageId={passageId} speaker={speaker} - onSpeaker={handleNameChange} + onSpeaker={setSpeaker} skipImportExportWait /> - + setVersionVisible(false)} + onVersionApplied={handleVersionApplied} /> - {pendingWarnOpen && ( + + {pendingUploadWarning && ( )} diff --git a/web/src/components/PassageDetail/PassageDetailStepComplete.tsx b/web/src/components/PassageDetail/PassageDetailStepComplete.tsx index 2a54cf4bc..ff70ced53 100644 --- a/web/src/components/PassageDetail/PassageDetailStepComplete.tsx +++ b/web/src/components/PassageDetail/PassageDetailStepComplete.tsx @@ -27,7 +27,6 @@ import { RecordKeyMap } from '@orbit/records'; export const PassageDetailStepComplete = () => { const { currentstep, - setCurrentStep, setStepCompleteTo, section, recording, @@ -50,13 +49,9 @@ export const PassageDetailStepComplete = () => { passageDetailStepCompleteSelector, shallowEqual ); - const passageNavigate = usePassageNavigate( - () => { - setView(''); - }, - setCurrentStep, - isNavigationBlocked - ); + const passageNavigate = usePassageNavigate(() => { + setView(''); + }, isNavigationBlocked); const { isChanged } = useContext(UnsavedContext).state; const { complete, toggleComplete } = useToggleStepComplete(); diff --git a/web/src/components/PassageDetail/mobile/MobileWorkflowSteps.tsx b/web/src/components/PassageDetail/mobile/MobileWorkflowSteps.tsx index 0e2cd9dce..db2208a4b 100644 --- a/web/src/components/PassageDetail/mobile/MobileWorkflowSteps.tsx +++ b/web/src/components/PassageDetail/mobile/MobileWorkflowSteps.tsx @@ -58,11 +58,7 @@ const MobileWorkflowSteps = () => { const ts = useSelector(sharedSelector, shallowEqual); const { showMessage } = useSnackBar(); const getWfLabel = useWfLabel(); - const passageNavigate = usePassageNavigate( - () => {}, - setCurrentStep, - isNavigationBlocked - ); + const passageNavigate = usePassageNavigate(() => {}, isNavigationBlocked); const { getOrgDefault } = useOrgDefaults(); const isStepProgression = getOrgDefault(orgDefaultWorkflowProgression) === WorkflowProgression.Step; diff --git a/web/src/components/PassageDetail/mobile/PassageDetailMobileFooter.tsx b/web/src/components/PassageDetail/mobile/PassageDetailMobileFooter.tsx index 46ef26d53..1381b50af 100644 --- a/web/src/components/PassageDetail/mobile/PassageDetailMobileFooter.tsx +++ b/web/src/components/PassageDetail/mobile/PassageDetailMobileFooter.tsx @@ -72,11 +72,7 @@ export default function PassageDetailMobileFooter() { const t: IMobileStrings = useSelector(mobileSelector, shallowEqual); const [memory] = useGlobal('memory'); const { prjId } = useParams(); - const passageNavigate = usePassageNavigate( - () => {}, - setCurrentStep, - isNavigationBlocked - ); + const passageNavigate = usePassageNavigate(() => {}, isNavigationBlocked); const { getOrgDefault } = useOrgDefaults(); const { localizedWorkStepFromId } = useOrgWorkflowSteps(); diff --git a/web/src/components/PassageDetail/usePassageAudioSave.test.ts b/web/src/components/PassageDetail/usePassageAudioSave.test.ts new file mode 100644 index 000000000..788a0f87d --- /dev/null +++ b/web/src/components/PassageDetail/usePassageAudioSave.test.ts @@ -0,0 +1,320 @@ +import { act, renderHook } from '@testing-library/react'; + +interface IUploadProps { + afterUploadCb: (mediaId: string) => Promise; +} + +const mockUpload = jest.fn(); +let mockGlobals: Record = {}; +// The passage's stored versions, by file name. +let mockVersions: string[] = []; + +jest.mock('react-redux', () => ({ useSelector: jest.fn() })); + +jest.mock('../../context/useGlobal', () => ({ + useGlobal: (key: string) => [mockGlobals[key], jest.fn()], +})); + +jest.mock('../../context/usePassageDetailContext', () => () => ({ + passage: { id: 'p1' }, + currentstep: 'step1', +})); + +jest.mock('../../crud', () => ({ + VernacularTag: null, + useSharedResRead: () => ({ getSharedResource: jest.fn() }), + // An upload reports to the callback of the render that started it, so a save + // running across an edit must still report the audio it stored. + useMediaUpload: (props: IUploadProps) => (files: File[]) => + mockUpload(files, props), +})); + +jest.mock('../../crud/autosaveMedia', () => ({ + AutosaveSuffix: '_autosave', + removeAllAutosaves: jest.fn(() => { + mockVersions = mockVersions.filter((name) => !name.includes('_autosave')); + }), + // Keeps the newest autosave, and the test reading mockVersions stores only one. + removeOldAutosaves: jest.fn(), +})); + +jest.mock('../../crud/useStepTool', () => ({ + useStepTool: () => ({ settings: '' }), +})); + +jest.mock('../../utils/passageDefaultFilename', () => ({ + passageDefaultFilename: () => 'base', +})); + +import { + removeAllAutosaves, + removeOldAutosaves, +} from '../../crud/autosaveMedia'; +import { AutosaveIntervalMs, usePassageAudioSave } from './usePassageAudioSave'; + +type Props = Parameters[0]; + +const memory = {}; +const reporter = {}; +const autosaveProps = { + memory, + passageId: 'p1', + planId: 'plan1', + offline: false, + reporter, +}; + +// The file extension shows which of these an upload carried. +const wavBlob = new Blob(['first'], { type: 'audio/wav' }); +const webmBlob = new Blob(['second'], { type: 'audio/webm' }); + +// The real upload calls afterUploadCb and then settles; a failed one has no +// media id. +const succeed = async (files: File[], props: IUploadProps) => { + mockVersions.push(...files.map((file) => file.name)); + await props.afterUploadCb('media1'); +}; + +const fail = async (_files: File[], props: IUploadProps) => { + await props.afterUploadCb(''); + throw new Error('upload failed'); +}; + +/** Keeps the next upload running until the returned function is called. */ +const holdNextUpload = (outcome = succeed) => { + let finish = () => {}; + const finished = new Promise((resolve) => { + finish = () => resolve(); + }); + mockUpload.mockImplementationOnce( + async (files: File[], props: IUploadProps) => { + await finished; + await outcome(files, props); + } + ); + return () => finish(); +}; + +/** Lets time pass, along with the promises that settle during it. */ +const elapse = (ms = 0) => + act(async () => { + await jest.advanceTimersByTimeAsync(ms); + }); + +const uploadedNames = () => + mockUpload.mock.calls.map(([files]) => (files as File[])[0]?.name); + +const setup = (overrides: Partial = {}) => { + const onStored = jest.fn(); + const initialProps: Props = { + passageId: 'p1', + currentBlob: wavBlob, + edited: true, + enabled: true, + performedBy: 'Speaker', + onStored, + ...overrides, + }; + const view = renderHook((props: Props) => usePassageAudioSave(props), { + initialProps, + }); + const update = (changes: Partial = {}) => + view.rerender({ ...initialProps, ...changes }); + return { ...view, onStored, update }; +}; + +describe('usePassageAudioSave', () => { + beforeEach(() => { + jest.useFakeTimers(); + mockUpload.mockReset(); + mockUpload.mockImplementation(succeed); + mockVersions = []; + mockGlobals = { + offline: false, + connected: true, + memory, + plan: 'plan1', + errorReporter: reporter, + }; + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + describe('autosave', () => { + it('stores an edit once the interval has passed', async () => { + setup(); + + await elapse(AutosaveIntervalMs - 1); + expect(uploadedNames()).toEqual([]); + + await elapse(1); + expect(uploadedNames()).toEqual(['base_autosave.wav']); + }); + + it('restarts the wait when the audio changes again', async () => { + const { update } = setup(); + + await elapse(AutosaveIntervalMs - 1); + update({ currentBlob: webmBlob }); + + await elapse(AutosaveIntervalMs - 1); + expect(uploadedNames()).toEqual([]); + + await elapse(1); + expect(uploadedNames()).toEqual(['base_autosave.webm']); + }); + + it.each<[string, Partial]>([ + ['there is no audio', { currentBlob: null }], + ['nothing is edited', { edited: false }], + ['it is not enabled', { enabled: false }], + ])('does not run when %s', async (_name, overrides) => { + setup(overrides); + + await elapse(AutosaveIntervalMs); + + expect(uploadedNames()).toEqual([]); + }); + + it('drops a waiting autosave when it stops being enabled', async () => { + const { update } = setup(); + + await elapse(AutosaveIntervalMs - 1); + update({ enabled: false }); + await elapse(AutosaveIntervalMs); + + expect(uploadedNames()).toEqual([]); + }); + + it('waits out a network loss and runs once the connection returns', async () => { + mockGlobals.connected = false; + const { update } = setup(); + + await elapse(AutosaveIntervalMs); + expect(uploadedNames()).toEqual([]); + + mockGlobals.connected = true; + update(); + await elapse(AutosaveIntervalMs); + expect(uploadedNames()).toEqual(['base_autosave.wav']); + }); + + it('runs without a connection under Work Offline', async () => { + mockGlobals.offline = true; + mockGlobals.connected = false; + setup(); + + await elapse(AutosaveIntervalMs); + + expect(uploadedNames()).toEqual(['base_autosave.wav']); + }); + + it('reports the stored audio and removes only the older autosaves', async () => { + const { onStored } = setup(); + + await elapse(AutosaveIntervalMs); + + expect(onStored).toHaveBeenCalledWith(wavBlob); + expect(removeOldAutosaves).toHaveBeenCalledWith(autosaveProps); + expect(removeAllAutosaves).not.toHaveBeenCalled(); + }); + + it('holds a newer edit until the running autosave finishes', async () => { + const finish = holdNextUpload(); + const { onStored, update } = setup(); + + await elapse(AutosaveIntervalMs); + update({ currentBlob: webmBlob }); + await elapse(AutosaveIntervalMs); + expect(uploadedNames()).toEqual(['base_autosave.wav']); + + finish(); + await elapse(); + // The newer edit isn't stored yet, so it must stay marked as unsaved. + expect(onStored).toHaveBeenCalledTimes(1); + expect(onStored).toHaveBeenCalledWith(wavBlob); + + await elapse(AutosaveIntervalMs); + expect(uploadedNames()).toEqual([ + 'base_autosave.wav', + 'base_autosave.webm', + ]); + }); + + it('tries again an interval after a failure', async () => { + // Held because a real upload settles after the render that shows it running. + const finish = holdNextUpload(fail); + const { result, onStored } = setup(); + + await elapse(AutosaveIntervalMs); + finish(); + await elapse(); + expect(onStored).not.toHaveBeenCalled(); + expect(removeOldAutosaves).not.toHaveBeenCalled(); + expect(result.current.saving).toBe(false); + + await elapse(AutosaveIntervalMs); + expect(uploadedNames()).toHaveLength(2); + expect(onStored).toHaveBeenCalledWith(wavBlob); + }); + }); + + describe('save', () => { + it('stores a permanent version and removes all autosaves', async () => { + const { result, onStored } = setup(); + + await act(async () => { + await expect(result.current.save()).resolves.toBe(wavBlob); + }); + + expect(uploadedNames()).toEqual(['base.wav']); + expect(onStored).toHaveBeenCalledWith(wavBlob); + expect(removeAllAutosaves).toHaveBeenCalledWith(autosaveProps); + }); + + it('uploads nothing when there is no audio', async () => { + const { result } = setup({ currentBlob: null }); + + await act(async () => { + await expect(result.current.save()).resolves.toBeNull(); + }); + + expect(mockUpload).not.toHaveBeenCalled(); + expect(removeAllAutosaves).not.toHaveBeenCalled(); + }); + + it('rejects and keeps the autosaves when the upload fails', async () => { + mockUpload.mockImplementationOnce(fail); + const { result, onStored } = setup(); + + await act(async () => { + await expect(result.current.save()).rejects.toThrow('upload failed'); + }); + + expect(onStored).not.toHaveBeenCalled(); + expect(removeAllAutosaves).not.toHaveBeenCalled(); + expect(result.current.saving).toBe(false); + }); + + it('keeps the saved audio and no autosave when an autosave is running', async () => { + const finishAutosave = holdNextUpload(); + const finishSave = holdNextUpload(); + const { result, update } = setup(); + + await elapse(AutosaveIntervalMs); + update({ currentBlob: webmBlob }); + act(() => { + void result.current.save(); + }); + + finishAutosave(); + await elapse(); + finishSave(); + await elapse(); + + expect(mockVersions).toEqual(['base.webm']); + }); + }); +}); diff --git a/web/src/components/PassageDetail/usePassageAudioSave.ts b/web/src/components/PassageDetail/usePassageAudioSave.ts new file mode 100644 index 000000000..373c5b808 --- /dev/null +++ b/web/src/components/PassageDetail/usePassageAudioSave.ts @@ -0,0 +1,119 @@ +import { useEffect, useMemo, useState } from 'react'; +import { useSelector } from 'react-redux'; +import { useGlobal } from '../../context/useGlobal'; +import usePassageDetailContext from '../../context/usePassageDetailContext'; +import { useMediaUpload, useSharedResRead, VernacularTag } from '../../crud'; +import { + AutosaveSuffix, + removeAllAutosaves, + removeOldAutosaves, +} from '../../crud/autosaveMedia'; +import { useStepTool } from '../../crud/useStepTool'; +import { IState } from '../../model'; +import { extensionFromAudioContentType } from '../../utils/mimeTypes'; +import { passageDefaultFilename } from '../../utils/passageDefaultFilename'; + +export const AutosaveIntervalMs = 3000; + +interface IProps { + passageId: string; + currentBlob: Blob | null | undefined; + edited: boolean; + enabled: boolean; + performedBy: string; + mimeType?: string; + onStored: (blob: Blob) => void; +} + +/** + * Stores the passage's audio. `save()` makes it a permanent version; while + * `enabled`, edits are also autosaved AutosaveIntervalMs after they stop. + */ +export const usePassageAudioSave = ({ + passageId, + currentBlob, + edited, + enabled, + performedBy, + mimeType, + onStored, +}: IProps) => { + const [offline] = useGlobal('offline'); + const [connected] = useGlobal('connected'); + const [memory] = useGlobal('memory'); + const [plan] = useGlobal('plan'); + const [reporter] = useGlobal('errorReporter'); + const { passage, currentstep } = usePassageDetailContext(); + const { settings: toolSettings } = useStepTool(currentstep); + const allBookData = useSelector((state: IState) => state.books.bookData); + const { getSharedResource } = useSharedResRead(); + // The running autosave or save, resolving to the audio it stored. + const [saving, setSaving] = useState>(); + + const fileBase = useMemo(() => { + const sr = getSharedResource(passage); + return passageDefaultFilename( + passage, + plan, + memory, + VernacularTag, + offline, + '', + toolSettings, + allBookData, + sr?.attributes.title + ); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [getSharedResource, passage, plan, offline, toolSettings, allBookData]); + + const autosaveProps = { memory, passageId, planId: plan, offline, reporter }; + + const upload = useMediaUpload({ + artifactId: VernacularTag, + passageId, + performedBy, + mimeType, + afterUploadCb: async (mediaId) => { + if (!mediaId || !currentBlob) return; + onStored(currentBlob); + await removeOldAutosaves(autosaveProps); + }, + }); + + const saveAs = async (blob: Blob, suffix = '') => { + const ext = extensionFromAudioContentType(blob.type); + const name = `${fileBase}${suffix}.${ext}`; + await upload([new File([blob], name, { type: blob.type })]); + return blob; + }; + + const track = (running: Promise) => { + setSaving(running); + // A save that waited on this one has replaced it. + const done = () => setSaving((s) => (s === running ? undefined : s)); + running.then(done, done); + return running; + }; + + // A manual save + const save = async () => { + if (currentBlob) { + await saveAs(currentBlob); + await removeAllAutosaves(autosaveProps); + } + return currentBlob; + }; + + useEffect(() => { + if (!currentBlob || !edited || !enabled || saving) return; + if (!offline && !connected) return; + const timer = setTimeout( + () => void track(saveAs(currentBlob, AutosaveSuffix)).catch(() => {}), + AutosaveIntervalMs + ); + return () => clearTimeout(timer); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [currentBlob, edited, enabled, saving, offline, connected]); + + return { save: () => track(save()), saving: !!saving }; +}; diff --git a/web/src/components/PassageDetail/usePassageNavigate.ts b/web/src/components/PassageDetail/usePassageNavigate.ts index 8cfd13013..0f3dfb465 100644 --- a/web/src/components/PassageDetail/usePassageNavigate.ts +++ b/web/src/components/PassageDetail/usePassageNavigate.ts @@ -5,7 +5,6 @@ import { LocalKey, localUserKey, useMyNavigate } from '../../utils'; export const usePassageNavigate = ( cb: () => void, - setCurrentStep: (step: string) => void, isNavigationBlocked: () => boolean ) => { const { pathname } = useLocation(); @@ -20,10 +19,6 @@ export const usePassageNavigate = ( localStorage.setItem(localUserKey(LocalKey.url), view); setTimeout(() => { navigate(view); - setTimeout(() => { - // Jump to first uncompleted step - setCurrentStep(''); - }, 500); // go to first step }, 500); // go to next passage cb(); }); diff --git a/web/src/context/PassageDetailContext.tsx b/web/src/context/PassageDetailContext.tsx index 11ccd3c54..7892ce961 100644 --- a/web/src/context/PassageDetailContext.tsx +++ b/web/src/context/PassageDetailContext.tsx @@ -400,11 +400,7 @@ const PassageDetailProvider = (props: IProps) => { } }; - const passageNavigate = usePassageNavigate( - () => {}, - setCurrentStep, - isNavigationBlocked - ); + const passageNavigate = usePassageNavigate(() => {}, isNavigationBlocked); const forceRefresh = (rowData?: IRow[]) => { refreshRef.current = refreshRef.current + 1; @@ -967,6 +963,7 @@ const PassageDetailProvider = (props: IProps) => { section: s as SectionD, sharedResource, psgCompleted: [...complete], + currentstep: '', }; }); } else { @@ -1138,6 +1135,7 @@ const PassageDetailProvider = (props: IProps) => { return () => { forceClearPending(); }; + // eslint-disable-next-line react-hooks/exhaustive-deps }, []); useEffect(() => { diff --git a/web/src/crud/autosaveMedia.test.ts b/web/src/crud/autosaveMedia.test.ts new file mode 100644 index 000000000..dfc56c1a6 --- /dev/null +++ b/web/src/crud/autosaveMedia.test.ts @@ -0,0 +1,173 @@ +jest.mock('../utils/evictMediaCache', () => ({ + evictMediaCache: jest.fn(() => Promise.resolve()), +})); + +jest.mock('../utils/logErrorService', () => ({ + logError: jest.fn(), + Severity: { error: 'error' }, +})); + +import MemorySource from '@orbit/memory'; +import { RecordKeyMap } from '@orbit/records'; +import { UploadType } from '../components/UploadType'; +import type { MediaFileD } from '../model'; +import { schema } from '../schema'; +import { + appendPendingMediaUpload, + loadPendingMediaUploads, + type PendingUploadMediaRecord, +} from '../store/upload/pendingMediaUploads'; +import { evictMediaCache } from '../utils/evictMediaCache'; +import { logError, Severity } from '../utils/logErrorService'; +import { + isAutosaveFileName, + removeAllAutosaves, + removeOldAutosaves, +} from './autosaveMedia'; + +interface Version { + id: string; + version: number; + file?: string; + audioUrl?: string; + remoteId?: string; + passage?: string; + artifactType?: string; +} + +/** A store holding these versions, which are autosaves unless given a file. */ +const seed = async (...versions: Version[]) => { + const memory = new MemorySource({ schema, keyMap: new RecordKeyMap() }); + await memory.update((t) => + versions.map((v) => + t.addRecord({ + type: 'mediafile', + id: v.id, + ...(v.remoteId && { keys: { remoteId: v.remoteId } }), + attributes: { + versionNumber: v.version, + originalFile: v.file ?? 'base_autosave.wav', + ...(v.audioUrl && { audioUrl: v.audioUrl }), + }, + relationships: { + passage: { data: { type: 'passage', id: v.passage ?? 'p1' } }, + ...(v.artifactType && { + artifactType: { + data: { type: 'artifacttype', id: v.artifactType }, + }, + }), + }, + }) + ) + ); + return memory; +}; + +const props = (memory: MemorySource, offline = false) => ({ + memory, + passageId: 'p1', + planId: 'plan1', + offline, + reporter: 'error.log', +}); + +const storedIds = (memory: MemorySource) => + (memory.cache.query((q) => q.findRecords('mediafile')) as MediaFileD[]) + .map((m) => m.id) + .sort(); + +const stagePending = (originalFile: string, passageId = 'p1') => + appendPendingMediaUpload({ + localAbsolutePath: `/media/${originalFile}`, + fileSize: 1, + uploadType: UploadType.Media, + record: { + planId: 'plan1', + passageId, + originalFile, + } as PendingUploadMediaRecord, + }); + +describe('autosaveMedia', () => { + beforeEach(() => { + jest.clearAllMocks(); + localStorage.clear(); + }); + + it.each([ + [undefined, false], + [null, false], + ['', false], + ['base.wav', false], + ['base_autosave.wav', true], + ])('isAutosaveFileName(%p) is %p', (name, expected) => { + expect(isAutosaveFileName(name)).toBe(expected); + }); + + it('removeOldAutosaves keeps the newest autosave and permanent versions', async () => { + const memory = await seed( + { id: 'auto2', version: 2 }, + { id: 'auto3', version: 3 }, + { id: 'perm4', version: 4, file: 'base.wav' }, + { id: 'auto1', version: 1 } + ); + + await removeOldAutosaves(props(memory)); + + expect(storedIds(memory)).toEqual(['auto3', 'perm4']); + }); + + it('removeAllAutosaves keeps only the permanent versions', async () => { + const memory = await seed( + { id: 'auto1', version: 1 }, + { id: 'perm2', version: 2, file: 'base.wav' }, + { id: 'auto3', version: 3 } + ); + + await removeAllAutosaves(props(memory)); + + expect(storedIds(memory)).toEqual(['perm2']); + }); + + it('leaves other passages and non-vernacular media', async () => { + const memory = await seed( + { id: 'otherPassage', version: 1, passage: 'p2' }, + { id: 'resource', version: 1, artifactType: 'backtranslation' } + ); + + await removeAllAutosaves(props(memory)); + + expect(storedIds(memory)).toEqual(['otherPassage', 'resource']); + }); + + it('under Work Offline leaves the autosaves already on the server', async () => { + const memory = await seed( + { id: 'server', version: 2, remoteId: '101' }, + { id: 'local', version: 1 } + ); + + await removeAllAutosaves(props(memory, true)); + + expect(storedIds(memory)).toEqual(['server']); + }); + + it('outside Work Offline removes the autosaves already on the server', async () => { + const memory = await seed({ id: 'server', version: 1, remoteId: '101' }); + + await removeAllAutosaves(props(memory)); + + expect(storedIds(memory)).toEqual([]); + }); + + it("clears the passage's pending autosave uploads", async () => { + stagePending('base_autosave.wav'); + stagePending('base.wav'); + stagePending('other_autosave.wav', 'p2'); + + await removeOldAutosaves(props(await seed())); + + expect(loadPendingMediaUploads().map((p) => p.record.originalFile)).toEqual( + ['base.wav', 'other_autosave.wav'] + ); + }); +}); diff --git a/web/src/crud/autosaveMedia.ts b/web/src/crud/autosaveMedia.ts new file mode 100644 index 000000000..83030c69d --- /dev/null +++ b/web/src/crud/autosaveMedia.ts @@ -0,0 +1,90 @@ +import Memory from '@orbit/memory'; +import { RecordKeyMap, RecordTransformBuilder } from '@orbit/records'; +import { MediaFile } from '../model'; +import { getAllMediaRecs } from './media'; +import { VernacularTag } from './useArtifactType'; +import { + clearPendingUploads, + PendingUploadMediaRecord, +} from '../store/upload/pendingMediaUploads'; +import { evictMediaCache } from '../utils/evictMediaCache'; +import { logError, Severity } from '../utils/logErrorService'; +import bugsnagClient from '../auth/bugsnagClient'; + +export const AutosaveSuffix = '_autosave'; + +export const isAutosaveFileName = (name?: string | null) => + (name ?? '').includes(AutosaveSuffix); + +export const isAutosaveMedia = (m?: MediaFile | null) => + isAutosaveFileName(m?.attributes?.originalFile); + +export const isAutosavePending = (record: PendingUploadMediaRecord) => + isAutosaveFileName(record.originalFile); + +const fileOf = (url?: string) => url?.split('?')[0]?.split(/[\\/]/).pop(); + +interface RemoveAutosavesProps { + memory: Memory; + passageId: string; + planId: string; + offline: boolean; + reporter: typeof bugsnagClient | string; +} + +/** Evict the removed records' files, except any a kept record still plays. */ +const evictFiles = async (removed: MediaFile[], kept: MediaFile[]) => { + const inUse = new Set(kept.map((m) => fileOf(m.attributes.audioUrl))); + const files = removed + .flatMap((m) => [m.attributes.audioUrl, m.attributes.originalFile]) + .filter((f) => f && !inUse.has(fileOf(f))); + await Promise.all(files.map((f) => evictMediaCache(f))); +}; + +const removeAutosaves = async ({ + memory, + passageId, + planId, + offline, + reporter, + keepNewest, +}: RemoveAutosavesProps & { keepNewest: boolean }) => { + try { + clearPendingUploads( + memory?.keyMap as RecordKeyMap, + planId, + passageId, + isAutosavePending + ); + const versions = getAllMediaRecs(passageId, memory, VernacularTag); + const newestId = keepNewest + ? versions.find(isAutosaveMedia)?.id + : undefined; + const oldAutosaves = versions.filter( + (m) => + m.id !== newestId && + isAutosaveMedia(m) && + // Sync can't delete server records, so Work Offline leaves them + // for the next online autosave or Save of this passage. + !(offline && m.keys?.remoteId) + ); + if (!oldAutosaves.length) return; + await memory.update((t: RecordTransformBuilder) => + oldAutosaves.map((m) => t.removeRecord(m).toOperation()) + ); + await evictFiles( + oldAutosaves, + versions.filter((m) => !oldAutosaves.includes(m)) + ); + } catch (err) { + logError(Severity.error, reporter, err as Error); + } +}; + +/** Remove the passage's vernacular autosaves, except its newest one. Best-effort. */ +export const removeOldAutosaves = (props: RemoveAutosavesProps) => + removeAutosaves({ ...props, keepNewest: true }); + +/** Remove all the passage's vernacular autosaves. Best-effort. */ +export const removeAllAutosaves = (props: RemoveAutosavesProps) => + removeAutosaves({ ...props, keepNewest: false }); diff --git a/web/src/crud/convertToWebM.ts b/web/src/crud/convertToWebM.ts index 194666b74..fd0d91c6c 100644 --- a/web/src/crud/convertToWebM.ts +++ b/web/src/crud/convertToWebM.ts @@ -218,11 +218,13 @@ async function convertToOggWithWorker( }); } +export const DEFAULT_COMPRESSED_MIME = 'audio/ogg;codecs=opus'; + /** * Converts any audio Blob to a target format using MediaRecorder API or worker-based conversion. * Currently only OGG and WebM are supported output formats. * @param audioBlob - The input audio Blob (WAV, MP3, MP4, etc.) - * @param targetMimeType - The target MIME type. Supported values: + * @param targetMimeType - The target MIME type. Defaults to DEFAULT_COMPRESSED_MIME. Supported values: * - 'audio/webm' * - 'audio/webm;codecs=opus' * - 'audio/ogg' @@ -232,7 +234,7 @@ async function convertToOggWithWorker( */ export async function convertToFormat( audioBlob: Blob, - targetMimeType: string + targetMimeType: string = DEFAULT_COMPRESSED_MIME ): Promise { // Normalize the target MIME type to avoid subtle mismatches const normalizedMimeType = targetMimeType.trim().toLowerCase(); diff --git a/web/src/crud/useMediaUpload.test.ts b/web/src/crud/useMediaUpload.test.ts index 49faaa025..e8b0dfde9 100644 --- a/web/src/crud/useMediaUpload.test.ts +++ b/web/src/crud/useMediaUpload.test.ts @@ -46,6 +46,16 @@ jest.mock('../context/TokenProvider', () => ({ TokenContext: React.createContext({ state: { accessToken: 'tok' } }), })); +jest.mock('../utils/logErrorService', () => ({ + logError: jest.fn(), + Severity: { info: 0, error: 1, retry: 2 }, +})); + +jest.mock('./convertToWebM', () => ({ + DEFAULT_COMPRESSED_MIME: 'audio/ogg;codecs=opus', + convertToFormat: (blob: Blob) => Promise.resolve(blob), +})); + let mockMediaFiles: MediaFileD[] = []; const mockMemory = { cache: { diff --git a/web/src/crud/useMediaUpload.ts b/web/src/crud/useMediaUpload.ts index f935a1d1e..cb601d87e 100644 --- a/web/src/crud/useMediaUpload.ts +++ b/web/src/crud/useMediaUpload.ts @@ -27,6 +27,7 @@ import { toError } from '../utils/toError'; import path from 'path-browserify'; import type { PendingRestoreInput } from '../store/upload/pendingMediaUploads'; import { findRecords } from './tryFindRecord'; +import { convertToFormat, DEFAULT_COMPRESSED_MIME } from './convertToWebM'; /** * The values that become the uploaded mediafile (its attributes and @@ -87,6 +88,11 @@ export interface IMediaUploadProps { /** Opt in to passage filename versioning (upload tab); recordings omit it. */ versionFilenames?: boolean | undefined; importList?: File[] | undefined; + /** + * Target save file type. Omit for normal compression, supply 'audio/wav' + * to skip compression, or supply another type to convert to that before saving. + */ + mimeType?: string; } /** @@ -109,6 +115,7 @@ export const useMediaUpload = ({ skipImportExportWait, versionFilenames, importList, + mimeType, }: IMediaUploadProps) => { const dispatch = useDispatch(); const uploadFiles = (files: File[]) => @@ -280,6 +287,20 @@ export const useMediaUpload = ({ return ''; }; + const compressFile = async (file: File): Promise => { + const type = mimeType ?? DEFAULT_COMPRESSED_MIME; + if (file.type === type) return file; + try { + const blob = await convertToFormat(file, mimeType); + const ext = type.split(/[/;]/)[1]; + const name = file.name.replace(/\.[^.]*$/, '') + `.${ext}`; + return new File([blob], name, { type }); + } catch (err) { + logError(Severity.info, reporter, err as Error); + return file; + } + }; + // TODO cleanup: the snackbar holds one message at a time (showMessage // replaces snackMessage), so back-to-back messages overwrite each other. // Centralize spacing/queueing in useSnackBar? We should consider if MUI or a @@ -386,7 +407,13 @@ export const useMediaUpload = ({ const run = async (files: File[]): Promise => { if (!files || files.length === 0) return false; if (beforeUpload) await beforeUpload(); - const staged = applyVersionSuffix(files); + + const filesToSave = + mimeType === 'audio/wav' + ? files + : await Promise.all(files.map(compressFile)); + + const staged = applyVersionSuffix(filesToSave); if (!NON_FILE_UPLOAD_TYPES.includes(effectiveType)) uploadFiles(staged); const startedBusy = Boolean(getGlobal('importexportBusy')); if (!noBusy) setBusy(true); diff --git a/web/src/crud/useWavRecorder.tsx b/web/src/crud/useWavRecorder.tsx index eda2a23f6..3e6ddf3bb 100644 --- a/web/src/crud/useWavRecorder.tsx +++ b/web/src/crud/useWavRecorder.tsx @@ -45,6 +45,18 @@ export function parseRecordCaptureAudioProcessing( } } +/** Record step `saveAsWav` toolSetting (written by RecordStepSettings). */ +export function parseRecordSaveAsWav(toolSettings: string | undefined | null) { + if (!toolSettings?.trim()) return false; + try { + return Boolean( + (JSON.parse(toolSettings) as { saveAsWav?: boolean }).saveAsWav + ); + } catch { + return false; + } +} + const noop = () => {}; export interface MimeInfo { diff --git a/web/src/crud/waveformRenderer.ts b/web/src/crud/waveformRenderer.ts new file mode 100644 index 000000000..a76e8a53b --- /dev/null +++ b/web/src/crud/waveformRenderer.ts @@ -0,0 +1,159 @@ +import type WaveSurfer from 'wavesurfer.js'; +import type { WaveSurferOptions } from 'wavesurfer.js'; +import { + calculateBarRenderConfig, + calculateBarSegments, + calculateVerticalScale, +} from 'wavesurfer.js/dist/renderer-utils.js'; + +export interface WaveformRenderRefs { + ws: React.RefObject; + highlights: React.RefObject<{ start: number; end: number; color: string }[]>; +} + +/** + * Creates a WaveSurfer `renderFunction` that draws bars with a gradient + * to highlight colored regions. + */ +export function createWaveformRenderer( + refs: WaveformRenderRefs, + wsOptions: WaveSurferOptions +) { + return ( + peaks: Array, + ctx: CanvasRenderingContext2D + ) => { + const dur = refs.ws.current?.getDuration() || 0; + const baseColor = wsOptions.waveColor as string; + const highlights = refs.highlights.current; + + if (highlights && highlights.length > 0 && dur > 0) { + applyRegionsGradient(ctx, highlights, dur, baseColor, refs.ws); + } + + drawBars(peaks, ctx, wsOptions); + }; +} + +/** + * Applies a horizontal gradient to `ctx.fillStyle` so that bars inside any + * of the given regions use that region's color and bars outside use `baseColor`. + * Accounts for zoom by mapping regions to the canvas chunk's local coordinate space. + */ +function applyRegionsGradient( + ctx: CanvasRenderingContext2D, + regions: { start: number; end: number; color: string }[], + duration: number, + baseColor: string, + wsRef: React.RefObject +) { + // When zoomed, WaveSurfer splits into multiple canvas chunks. + // Map region time ranges to this chunk's local [0,1] fraction. + const canvasOffsetCSS = parseFloat(ctx.canvas.style.left) || 0; + const canvasWidthCSS = parseFloat(ctx.canvas.style.width) || ctx.canvas.width; + const wrapper = wsRef.current?.getWrapper(); + const totalWidthCSS = wrapper?.scrollWidth || canvasWidthCSS; + + const chunkStart = canvasOffsetCSS / totalWidthCSS; + const chunkSpan = canvasWidthCSS / totalWidthCSS; + + // Convert to local coordinates, filter to those overlapping this chunk, sort + const localRegions = regions + .map((r) => ({ + localStart: (r.start / duration - chunkStart) / chunkSpan, + localEnd: (r.end / duration - chunkStart) / chunkSpan, + color: r.color, + })) + .filter((r) => r.localEnd > 0 && r.localStart < 1) + .sort((a, b) => a.localStart - b.localStart); + + if (localRegions.length === 0) return; + + const eps = 0.0001; + const gradient = ctx.createLinearGradient(0, 0, ctx.canvas.width, 0); + let cursor = 0; + + for (const region of localRegions) { + const s = Math.max(0, region.localStart); + const e = Math.min(1, region.localEnd); + + // Base color from cursor to region start (only if there's a gap) + if (s > cursor + eps) { + if (cursor === 0) { + gradient.addColorStop(0, baseColor); + } else { + gradient.addColorStop(cursor, baseColor); + } + gradient.addColorStop(s - eps, baseColor); + } + + // Region color + gradient.addColorStop(s, region.color); + gradient.addColorStop(e, region.color); + + cursor = e + eps; + } + + // Base color to the end + if (cursor < 1) { + gradient.addColorStop(Math.min(cursor, 1), baseColor); + gradient.addColorStop(1, baseColor); + } + + ctx.fillStyle = gradient; +} + +/** + * Draws waveform bars using WaveSurfer's exported utilities. + */ +function drawBars( + peaks: Array, + ctx: CanvasRenderingContext2D, + options: WaveSurferOptions +) { + const pixelRatio = Math.max(1, window.devicePixelRatio || 1); + const { width, height } = ctx.canvas; + + const vScale = calculateVerticalScale({ + channelData: peaks, + normalize: options.normalize, + }); + const config = calculateBarRenderConfig({ + width, + height, + length: (peaks[0] || []).length, + options, + pixelRatio, + }); + const segments = calculateBarSegments({ + channelData: peaks, + barIndexScale: config.barIndexScale, + barSpacing: config.barSpacing, + barWidth: config.barWidth, + halfHeight: config.halfHeight, + vScale, + canvasHeight: height, + barAlign: undefined, + barMinHeight: config.barMinHeight, + }); + + ctx.beginPath(); + for (const seg of segments) { + ctx.rect(seg.x, seg.y, seg.width, seg.height); + } + ctx.fill(); + ctx.closePath(); +} + +/** + * Injects a `