From 445e5f5ac704d4bf15c8b9de4a641227f970de65 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 06:27:44 +0000 Subject: [PATCH 1/2] Feat: Add "Render Selected" support for video generation When a user highlights text in the editor, it's now automatically detected and passed to the video renderer. If selection exists, only that portion is rendered; otherwise, the full note is rendered. Changes: - Frontend: Detect selection using getEditorSelectionMarkdown(), filter diagram rasterization to selected blocks only, pass selectedContent through the render pipeline - Backend: Accept optional selected_content in VideoRenderRequest, store in VideoRenderJob, use in estimate and render endpoints - Database: Add nullable selected_content column to video_render_jobs via migration UX: Selection detection is transparent - single "Video (MP4)" button automatically uses selection if present, falls back to full note if not. Estimate endpoint returns accurate segment/duration for selection. --- backend/app/database.py | 6 ++++++ backend/app/models.py | 1 + backend/app/routers/video.py | 3 ++- backend/app/schemas.py | 1 + backend/app/video/worker.py | 2 +- frontend/src/api/videoGen.ts | 7 ++++--- frontend/src/components/VideoGenModal.tsx | 8 +++++--- frontend/src/stores/activity.ts | 4 ++-- frontend/src/views/EditorView.tsx | 17 ++++++++++++++--- 9 files changed, 36 insertions(+), 13 deletions(-) diff --git a/backend/app/database.py b/backend/app/database.py index e90d8af..e609701 100644 --- a/backend/app/database.py +++ b/backend/app/database.py @@ -679,6 +679,12 @@ def _run_migrations(): conn.commit() except Exception: pass + # Video render: selected content (render only selection instead of full note) + try: + conn.execute(text("ALTER TABLE video_render_job ADD COLUMN selected_content TEXT")) + conn.commit() + except Exception: + pass def _seed_after_migrations(): diff --git a/backend/app/models.py b/backend/app/models.py index 5452714..e1ca8ba 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -292,6 +292,7 @@ class VideoRenderJob(SQLModel, table=True): detail: str = Field(default="") # e.g. "shot 7 of 19" options: str = Field(default="{}") # RenderOptions, JSON as text quality: str = Field(default="full") # "preview" | "full" + selected_content: Optional[str] = None # If set, render only this BlockNote content; otherwise render full note note_title: str = Field(default="") # snapshot, for download filenames result_filename: Optional[str] = None # rendered .mp4 subtitle_filename: Optional[str] = None # .srt sidecar diff --git a/backend/app/routers/video.py b/backend/app/routers/video.py index 7f4fdab..485d8c7 100644 --- a/backend/app/routers/video.py +++ b/backend/app/routers/video.py @@ -126,6 +126,7 @@ def create_job(payload: VideoRenderRequest, request: Request, session: Session = stage="Queued", options=json.dumps(options.model_dump()), quality="preview" if payload.quality == "preview" else "full", + selected_content=payload.selected_content, created_at=now, updated_at=now, ) @@ -201,7 +202,7 @@ def estimate_render(payload: VideoRenderRequest, request: Request, session: Sess shots, chars, seconds, warnings = estimate( user_id=user_id, media_dir=MEDIA_DIR, - note_content=note.content or "[]", + note_content=payload.selected_content or note.content or "[]", note_title=note.title or "Untitled", author=(user.username if user else "") or "", options=options, diff --git a/backend/app/schemas.py b/backend/app/schemas.py index 056a062..51990e2 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -712,6 +712,7 @@ class VideoRenderRequest(BaseModel): # option schema lives in one place next to the renderer that consumes it. options: Dict[str, Any] = {} quality: str = "full" # "preview" renders small and fast, reusing cached narration + selected_content: Optional[str] = None # If provided, render only this content instead of full note class VideoEstimateRead(BaseModel): diff --git a/backend/app/video/worker.py b/backend/app/video/worker.py index 9625654..b44672e 100644 --- a/backend/app/video/worker.py +++ b/backend/app/video/worker.py @@ -210,7 +210,7 @@ def raise_cancelled() -> None: job_id=job_id, user_id=job.user_id, media_dir=_media_dir(), - note_content=note.content or "[]", + note_content=job.selected_content or note.content or "[]", note_title=note.title or "Untitled", author=author, options=options, diff --git a/frontend/src/api/videoGen.ts b/frontend/src/api/videoGen.ts index 8d72ae1..d7d1a5e 100644 --- a/frontend/src/api/videoGen.ts +++ b/frontend/src/api/videoGen.ts @@ -210,9 +210,10 @@ export const DEFAULT_RENDER_OPTIONS: RenderOptions = { export const videoGenApi = { /** Queue a render. `preview` renders small and fast; because narration is - * cached by voice+text, a full render afterwards reuses it at no extra cost. */ - createJob(noteId: string, options: RenderOptions, quality: 'preview' | 'full' = 'full'): Promise<{ data: VideoRenderJob }> { - return client.post('/video/jobs', { note_id: noteId, options, quality }).then((r) => r.data) + * cached by voice+text, a full render afterwards reuses it at no extra cost. + * `selectedContent` (if provided) renders only that portion instead of the full note. */ + createJob(noteId: string, options: RenderOptions, quality: 'preview' | 'full' = 'full', selectedContent?: string): Promise<{ data: VideoRenderJob }> { + return client.post('/video/jobs', { note_id: noteId, options, quality, selected_content: selectedContent }).then((r) => r.data) }, // Reading and cancelling a render go through `api/activity` now, which covers diff --git a/frontend/src/components/VideoGenModal.tsx b/frontend/src/components/VideoGenModal.tsx index 17116ac..2435603 100644 --- a/frontend/src/components/VideoGenModal.tsx +++ b/frontend/src/components/VideoGenModal.tsx @@ -31,7 +31,9 @@ interface Props { noteTitle: string /** blockId -> /media URL for rasterised diagram blocks, gathered by the caller. */ diagramImages: Record - onGenerate: (options: RenderOptions, quality: 'preview' | 'full') => Promise + /** Selected text to render (if any); if undefined, full note is rendered. */ + selectedContent?: string + onGenerate: (options: RenderOptions, quality: 'preview' | 'full', selected?: string) => Promise onClose: () => void } @@ -186,7 +188,7 @@ function formatDuration(seconds: number): string { return m > 0 ? `${m}m ${s}s` : `${s}s` } -export default function VideoGenModal({ noteId, noteTitle, diagramImages, onGenerate, onClose }: Props) { +export default function VideoGenModal({ noteId, noteTitle, diagramImages, selectedContent, onGenerate, onClose }: Props) { const [options, setOptions] = useState(loadStoredOptions) const [tab, setTab] = useState('format') const [estimate, setEstimate] = useState(null) @@ -302,7 +304,7 @@ export default function VideoGenModal({ noteId, noteTitle, diagramImages, onGene setBusy(quality) setError(null) try { - await onGenerate(payload, quality) + await onGenerate(payload, quality, selectedContent) onClose() } catch (e) { setError(apiErrorMessage(e, 'Could not start the render')) diff --git a/frontend/src/stores/activity.ts b/frontend/src/stores/activity.ts index a966a7e..e46631a 100644 --- a/frontend/src/stores/activity.ts +++ b/frontend/src/stores/activity.ts @@ -154,8 +154,8 @@ export const useActivityStore = create((set, get) => { } }, - async startVideo(noteId, options, quality = 'full') { - await videoGenApi.createJob(noteId, options, quality) + async startVideo(noteId, options, quality = 'full', selectedContent?: string) { + await videoGenApi.createJob(noteId, options, quality, selectedContent) // Read it straight back through the activity view rather than mapping the // richer render shape by hand, so there is one place that knows the wire form. await get().resume() diff --git a/frontend/src/views/EditorView.tsx b/frontend/src/views/EditorView.tsx index c29cd39..47d6a93 100644 --- a/frontend/src/views/EditorView.tsx +++ b/frontend/src/views/EditorView.tsx @@ -52,7 +52,7 @@ import { annotationsApi, type Annotation } from '@/api/annotations' import { useDictation } from '@/hooks/useDictation' import { insertDictationAtSelection, normalizeDictatedText } from '@/utils/dictationInsert' import { useTextToSpeech } from '@/hooks/useTextToSpeech' -import { extractPlainText } from '@/utils/blocks' +import { extractPlainText, getEditorSelectionMarkdown } from '@/utils/blocks' import { noteToMarkdownBody, svgToPngData } from '@/utils/export' import { ARCHIVE_SYSTEM_KEY } from '@/utils/folderTree' @@ -166,6 +166,7 @@ export default function EditorView() { const [showImageGen, setShowImageGen] = useState(false) const [showVideoGen, setShowVideoGen] = useState(false) const [diagramImages, setDiagramImages] = useState>({}) + const [selectedContent, setSelectedContent] = useState(undefined) const [showHistory, setShowHistory] = useState(false) const [showStats, setShowStats] = useState(false) const [findOpen, setFindOpen] = useState(false) @@ -957,9 +958,18 @@ export default function EditorView() { if (!id) return if (hasPendingChanges.current) await doSave(true) + // Detect selected content and get selected block IDs + const selected = getEditorSelectionMarkdown(editor) + const selectedBlockIds = new Set(editor?.getSelection?.()?.blocks?.map((b) => b.id) ?? []) + setSelectedContent(selected || undefined) + + // Rasterize only diagrams within selected blocks (if selection exists) or all diagrams (if no selection) const rasterised: Record = {} const blocks = (editor?.document ?? []) as { id?: string; type?: string; props?: { source?: string } }[] for (const block of blocks) { + // Skip if selection exists and this block is not in the selection + if (selected && !selectedBlockIds.has(block.id ?? '')) continue + const source = block.type === 'diagram' ? block.props?.source : undefined if (!block.id || !source) continue try { @@ -977,10 +987,10 @@ export default function EditorView() { setShowVideoGen(true) } - async function runVideoGen(options: RenderOptions, quality: 'preview' | 'full') { + async function runVideoGen(options: RenderOptions, quality: 'preview' | 'full', selected?: string) { const id = createdNoteId.current || latestNoteId.current if (!id) throw new Error('Save the note first') - await startVideoJob(id, options, quality) + await startVideoJob(id, options, quality, selected) showToast(quality === 'preview' ? 'Rendering a preview…' : 'Rendering your video…') } @@ -2086,6 +2096,7 @@ export default function EditorView() { noteId={createdNoteId.current || latestNoteId.current || ''} noteTitle={title || 'Untitled'} diagramImages={diagramImages} + selectedContent={selectedContent} onGenerate={runVideoGen} onClose={() => setShowVideoGen(false)} /> From 2b97dde12245c3ea078e26ea8047aa999f13bab9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 06:29:02 +0000 Subject: [PATCH 2/2] Fix: Update startVideo type signature to include selectedContent parameter The function implementation accepted selectedContent but the interface didn't, causing TypeScript compilation error TS2554 in EditorView.tsx. --- frontend/src/stores/activity.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/stores/activity.ts b/frontend/src/stores/activity.ts index e46631a..393b6ed 100644 --- a/frontend/src/stores/activity.ts +++ b/frontend/src/stores/activity.ts @@ -32,6 +32,7 @@ interface ActivityState { noteId: string, options: RenderOptions, quality?: 'preview' | 'full', + selectedContent?: string, ) => Promise reset: () => void }