From 7560dbf9f9396d6e925ea08a08a8823d45e1b086 Mon Sep 17 00:00:00 2001 From: Nick Beaugeard Date: Tue, 1 Sep 2026 12:24:05 +1000 Subject: [PATCH] feat: sync subtitle timing and add end fade --- docs/ARCHITECTURE.md | 15 +++-- docs/PRODUCT.md | 8 ++- server/index.ts | 49 +++++++-------- server/subtitleExport.ts | 63 ++++++++++++++++--- src/components/Studio.tsx | 114 +++++++++++++++++++++++++++++------ src/lib/studioPreferences.ts | 3 + src/lib/videoExport.ts | 26 +++++--- src/styles.css | 12 ++++ tests/studioControls.test.ts | 4 ++ tests/videoExport.test.ts | 41 ++++++++++++- 10 files changed, 266 insertions(+), 69 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 24b6307..60eeeee 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -86,12 +86,15 @@ available, it records a WebM/Opus take and posts it only to the loopback API. The API converts it to H.264/AAC MP4 with the bundled FFmpeg executable, returns the file for review/save, and removes its temporary working directory. -Speech-confirmed cursor advances are timestamped during recording. Review keeps -the clean recording as the source of truth and can preview those words as a -single lower-third line. A subtitle export sends the clean take, selected font, -and validated word timings only to the loopback API. The API builds a temporary -ASS track, re-encodes the video with libass so the active word is enlarged and -accented, returns an H.264/AAC MP4, and removes all temporary inputs. +Speech-confirmed cursor advances are timestamped during recording with a +1.5-second correction for the local model's observed recognition delay. Review +keeps the clean recording as the source of truth and can preview those words as +a single lower-third line. A subtitle export sends the clean take, selected +font, and validated word timings only to the loopback API. The API builds a +temporary ASS track, re-encodes the video with libass so the active word is +enlarged and accented, optionally fades the final second to black, returns an +H.264/AAC MP4, and removes all temporary inputs. The final caption remains +visible for 1.5 seconds after its last timed word. ## Alignment approach diff --git a/docs/PRODUCT.md b/docs/PRODUCT.md index 29b8961..e0074d5 100644 --- a/docs/PRODUCT.md +++ b/docs/PRODUCT.md @@ -21,7 +21,8 @@ fixed scroll speed. 8. Resume from nearby script text and prompt movement catches up. 9. Stop and review the finished take. 10. Save a clean video, or choose a font and export a subtitled MP4 whose - lower-third line highlights the word currently being spoken. + lower-third line highlights the word currently being spoken. Either export + can optionally fade to black over its final second. ## Information architecture @@ -88,8 +89,9 @@ recording and lets Studio devote most of the screen to eye-line and readability. - A user can complete the full journey without a developer tool. - A saved MP4 contains synchronized H.264 video and AAC microphone audio. -- Subtitle export shows one lower-third line and highlights the active spoken - word without adding the teleprompter overlay to the clean source. +- Subtitle export compensates for local-model recognition delay, keeps the + final lower-third on screen briefly, and highlights the active spoken word + without adding the teleprompter overlay to the clean source. - Prompt movement responds to matching speech and stops for unmatched speech. - Recording duration continues to increase while prompt status is off-script. - Script CRUD persists after a refresh. diff --git a/server/index.ts b/server/index.ts index daca197..37c5ead 100644 --- a/server/index.ts +++ b/server/index.ts @@ -16,6 +16,7 @@ import { } from "./localSpeech.js"; import { buildAssSubtitles, + buildVideoFilter, parseCaptionExportBody, } from "./subtitleExport.js"; @@ -45,13 +46,6 @@ function resolveFfmpegPath(): string | null { ); } -function assVideoFilter(): string { - // The bundled Windows FFmpeg build uses fontconfig's system-font provider. - // Keeping the ASS path relative also avoids drive-letter escaping being - // misread by libass as another filter option. - return "ass=captions.ass"; -} - const ffmpegPath = resolveFfmpegPath(); app.disable("x-powered-by"); @@ -142,7 +136,7 @@ app.post( ); app.post( - "/api/recordings/subtitles", + "/api/recordings/render", express.raw({ type: "application/x-prompter-export", limit: "1gb", @@ -150,7 +144,7 @@ app.post( async (request, response) => { if (!ffmpegPath) { response.status(503).json({ - error: "Subtitle export is not available on this computer.", + error: "Video rendering is not available on this computer.", }); return; } @@ -177,20 +171,24 @@ app.post( ); const inputPath = path.join(workingDirectory, "take.recording"); const subtitlePath = path.join(workingDirectory, "captions.ass"); - const outputPath = path.join(workingDirectory, "take-subtitled.mp4"); + const outputPath = path.join(workingDirectory, "take-rendered.mp4"); try { - await Promise.all([ - writeFile(inputPath, parsedExport.recording), - writeFile( - subtitlePath, - buildAssSubtitles( - parsedExport.request.words, - parsedExport.request.fontFamily, + const writes = [writeFile(inputPath, parsedExport.recording)]; + if (parsedExport.request.mode === "subtitles") { + writes.push( + writeFile( + subtitlePath, + buildAssSubtitles( + parsedExport.request.words, + parsedExport.request.fontFamily, + ), + "utf8", ), - "utf8", - ), - ]); + ); + } + await Promise.all(writes); + const videoFilter = buildVideoFilter(parsedExport.request); await execFileAsync( ffmpegPath, [ @@ -204,8 +202,7 @@ app.post( "0:v:0", "-map", "0:a:0", - "-vf", - assVideoFilter(), + ...(videoFilter ? ["-vf", videoFilter] : []), "-c:v", "libx264", "-preset", @@ -235,17 +232,17 @@ app.post( void rm(workingDirectory, { recursive: true, force: true }); if (sendError && !response.headersSent) { response.status(500).json({ - error: "The subtitled MP4 could not be returned.", + error: "The rendered MP4 could not be returned.", }); } }); } catch (error) { await rm(workingDirectory, { recursive: true, force: true }); const message = - error instanceof Error ? error.message : "Unknown subtitle export error"; - console.error("Subtitle export failed:", message); + error instanceof Error ? error.message : "Unknown video render error"; + console.error("Video rendering failed:", message); response.status(500).json({ - error: "Subtitle export failed. Your clean recording is still safe.", + error: "Video rendering failed. Your clean recording is still safe.", }); } }, diff --git a/server/subtitleExport.ts b/server/subtitleExport.ts index f1f07aa..e0f0543 100644 --- a/server/subtitleExport.ts +++ b/server/subtitleExport.ts @@ -1,9 +1,10 @@ import { VIDEO_EXPORT_FONTS, + FINAL_CAPTION_HOLD_MS, captionPages, - type CaptionExportRequest, type TimedWord, type VideoExportFont, + type VideoRenderRequest, } from "../src/lib/videoExport.js"; const MAX_METADATA_BYTES = 512 * 1024; @@ -12,9 +13,11 @@ const MAX_VIDEO_DURATION_MS = 4 * 60 * 60 * 1_000; export interface ParsedCaptionExport { recording: Buffer; - request: CaptionExportRequest; + request: VideoRenderRequest; } +const FADE_TO_BLACK_DURATION_MS = 1_000; + function isExportFont(value: unknown): value is VideoExportFont { return VIDEO_EXPORT_FONTS.some((font) => font.family === value); } @@ -67,17 +70,33 @@ export function parseCaptionExportBody(body: Buffer): ParsedCaptionExport { if (typeof metadata !== "object" || metadata === null) { throw new Error("The caption export options are invalid."); } - const candidate = metadata as Partial; + const candidate = metadata as Partial; + if (candidate.mode !== "clean" && candidate.mode !== "subtitles") { + throw new Error("Choose a supported video export style."); + } if (!isExportFont(candidate.fontFamily)) { throw new Error("Choose a supported subtitle font."); } if ( !Array.isArray(candidate.words) || - candidate.words.length === 0 || candidate.words.length > MAX_CAPTION_WORDS ) { + throw new Error("The spoken-word timings are invalid."); + } + if (candidate.mode === "subtitles" && candidate.words.length === 0) { throw new Error("No spoken-word timings were supplied."); } + if (typeof candidate.fadeToBlack !== "boolean") { + throw new Error("The fade option is invalid."); + } + const videoDurationMs = Number(candidate.videoDurationMs); + if ( + !Number.isFinite(videoDurationMs) || + videoDurationMs <= 0 || + videoDurationMs > MAX_VIDEO_DURATION_MS + ) { + throw new Error("The video duration is invalid."); + } const words = candidate.words.map(parseWord); if (words.some((word) => word === null)) { @@ -92,10 +111,32 @@ export function parseCaptionExportBody(body: Buffer): ParsedCaptionExport { return { recording: body.subarray(metadataLength + 4), - request: { fontFamily: candidate.fontFamily, words: parsedWords }, + request: { + mode: candidate.mode, + fontFamily: candidate.fontFamily, + words: parsedWords, + fadeToBlack: candidate.fadeToBlack, + videoDurationMs: Math.round(videoDurationMs), + }, }; } +export function buildVideoFilter(request: VideoRenderRequest): string | null { + const filters = request.mode === "subtitles" ? ["ass=captions.ass"] : []; + if (request.fadeToBlack) { + const fadeStartMs = Math.max( + 0, + request.videoDurationMs - FADE_TO_BLACK_DURATION_MS, + ); + filters.push( + `fade=t=out:st=${(fadeStartMs / 1_000).toFixed(3)}:d=${( + FADE_TO_BLACK_DURATION_MS / 1_000 + ).toFixed(3)}:color=black`, + ); + } + return filters.length > 0 ? filters.join(",") : null; +} + function escapeAssText(text: string): string { return text .replaceAll("\\", "\\\\") @@ -131,11 +172,19 @@ export function buildAssSubtitles( fontFamily: VideoExportFont, ): string { const events: string[] = []; - for (const page of captionPages(words)) { + const pages = captionPages(words); + for (let pageIndex = 0; pageIndex < pages.length; pageIndex += 1) { + const page = pages[pageIndex]; for (let index = 0; index < page.length; index += 1) { const word = page[index]; const nextWord = page[index + 1]; - const endMs = Math.max(word.startMs + 80, nextWord?.startMs ?? word.endMs); + const nextPageStartMs = pages[pageIndex + 1]?.[0]?.startMs; + const endMs = Math.max( + word.startMs + 80, + nextWord?.startMs ?? + nextPageStartMs ?? + word.endMs + FINAL_CAPTION_HOLD_MS, + ); events.push( [ "Dialogue: 0", diff --git a/src/components/Studio.tsx b/src/components/Studio.tsx index 562442d..042b66b 100644 --- a/src/components/Studio.tsx +++ b/src/components/Studio.tsx @@ -46,6 +46,7 @@ import { import { VIDEO_EXPORT_FONTS, VIDEO_EXPORT_MIME_TYPE, + FINAL_CAPTION_HOLD_MS, activeCaptionPage, makeCaptionExportBody, type TimedWord, @@ -108,27 +109,36 @@ async function makeMp4(recording: Blob): Promise { return new Blob([converted], { type: "video/mp4" }); } -async function makeSubtitledMp4( +async function renderMp4( recording: Blob, words: TimedWord[], fontFamily: VideoExportFont, + mode: VideoExportMode, + fadeToBlack: boolean, + videoDurationMs: number, ): Promise { - const response = await fetch("/api/recordings/subtitles", { + const response = await fetch("/api/recordings/render", { method: "POST", headers: { "Content-Type": VIDEO_EXPORT_MIME_TYPE }, - body: makeCaptionExportBody(recording, { fontFamily, words }), + body: makeCaptionExportBody(recording, { + mode, + fontFamily, + words, + fadeToBlack, + videoDurationMs, + }), }); if (!response.ok) { const body = (await response.json().catch(() => null)) as { error?: string; } | null; - throw new Error(body?.error || "The subtitled MP4 could not be exported."); + throw new Error(body?.error || "The rendered MP4 could not be exported."); } const exported = await response.blob(); if (exported.size === 0) { - throw new Error("The subtitle exporter returned an empty file."); + throw new Error("The video renderer returned an empty file."); } return new Blob([exported], { type: "video/mp4" }); } @@ -166,8 +176,12 @@ export function Studio({ const [exportFont, setExportFont] = useState( initialStudioPreferences.exportFont, ); + const [exportFadeToBlack, setExportFadeToBlack] = useState( + initialStudioPreferences.exportFadeToBlack, + ); const [exporting, setExporting] = useState(false); const [reviewTimeMs, setReviewTimeMs] = useState(0); + const [recordingDurationMs, setRecordingDurationMs] = useState(0); const [recordingUrl, setRecordingUrl] = useState(""); const [recordingType, setRecordingType] = useState(""); const [availableCameras, setAvailableCameras] = useState([]); @@ -380,8 +394,16 @@ export function Studio({ captionMode, exportMode, exportFont, + exportFadeToBlack, }); - }, [captionMode, exportFont, exportMode, fontSize, promptPosition]); + }, [ + captionMode, + exportFadeToBlack, + exportFont, + exportMode, + fontSize, + promptPosition, + ]); const changeDevice = useCallback( async (kind: "camera" | "microphone", deviceId: string) => { @@ -480,6 +502,7 @@ export function Studio({ recordingBlobRef.current = null; setRecordingUrl(""); setReviewTimeMs(0); + setRecordingDurationMs(0); recorder.ondataavailable = (event) => { if (event.data.size > 0) chunksRef.current.push(event.data); @@ -558,39 +581,70 @@ export function Studio({ setError(""); setElapsed(0); setReviewTimeMs(0); + setRecordingDurationMs(0); setExporting(false); resetFollower(); setStudioState(streamRef.current ? "ready" : "setup"); window.setTimeout(attachPreview, 0); }, [attachPreview, resetFollower]); - const exportSubtitledTake = useCallback(async () => { + const exportRenderedTake = useCallback(async () => { const recording = recordingBlobRef.current; - const words = getTimedWords(); - if (!recording || words.length === 0 || exporting) return; + const timedWords = getTimedWords(); + const words = exportMode === "subtitles" ? timedWords : []; + if ( + !recording || + (exportMode === "subtitles" && words.length === 0) || + exporting + ) return; + const videoDurationMs = + recordingDurationMs > 0 + ? recordingDurationMs + : Math.max( + elapsed * 1_000, + (timedWords.at(-1)?.endMs ?? 0) + FINAL_CAPTION_HOLD_MS, + ); setError(""); setExporting(true); try { - const subtitled = await makeSubtitledMp4(recording, words, exportFont); + const rendered = await renderMp4( + recording, + words, + exportFont, + exportMode, + exportFadeToBlack, + videoDurationMs, + ); const timestamp = new Date() .toISOString() .slice(0, 19) .replaceAll(":", "-"); downloadBlob( - subtitled, - `${safeFileName(script.title)}-${timestamp}-subtitled.mp4`, + rendered, + `${safeFileName(script.title)}-${timestamp}-${ + exportMode === "subtitles" ? "subtitled" : "faded" + }.mp4`, ); } catch (caught) { setError( caught instanceof Error ? `${caught.message} Your clean recording is still available.` - : "Subtitle export failed. Your clean recording is still available.", + : "Video rendering failed. Your clean recording is still available.", ); } finally { setExporting(false); } - }, [exportFont, exporting, getTimedWords, script.title]); + }, [ + elapsed, + exportFadeToBlack, + exportFont, + exportMode, + exporting, + getTimedWords, + recordingDurationMs, + script.title, + ]); const handleBack = useCallback(() => { onBack(); @@ -924,6 +978,12 @@ export function Studio({ onSeeked={(event) => setReviewTimeMs(event.currentTarget.currentTime * 1_000) } + onLoadedMetadata={(event) => { + const durationMs = event.currentTarget.duration * 1_000; + if (Number.isFinite(durationMs) && durationMs > 0) { + setRecordingDurationMs(Math.round(durationMs)); + } + }} /> ) : (