Add high-quality scene replacement and flexible video exports

package.json

@@ -8,7 +8,7 @@
   "type": "module",
   "main": "desktop/main.mjs",
   "scripts": {
-    "dev": "concurrently -k -n API,WEB -c cyan,magenta \"tsx watch server/index.ts\" \"vite\"",
+    "dev": "concurrently -k -n API,WEB -c cyan,magenta \"node scripts/dev-api.mjs\" \"vite\"",
     "site:render": "tsx site/render-static.tsx",
     "site:dev": "npm run site:render && vite site",
     "site:build": "npm run site:render && tsc -p site/tsconfig.json --pretty false && vite build site",
@@ -19,13 +19,14 @@
     "package:windows": "npm run build:desktop && node scripts/package-windows.mjs",
     "speech:model": "node scripts/install-speech-model.mjs",
     "speech:smoke:arm64": "node scripts/smoke-arm64-sidecar.mjs",
-    "speech:smoke": "tsx scripts/smoke-speech.ts",
+    "speech:smoke": "node scripts/smoke-speech.mjs",
     "start": "tsx server/index.ts --production",
     "test": "vitest run",
     "test:watch": "vitest",
     "typecheck": "tsc -b --pretty false"
   },
   "dependencies": {
+    "@mediapipe/tasks-vision": "0.10.21",
     "dotenv": "^17.2.1",
     "express": "^5.1.0",
     "ffmpeg-static": "^5.2.0",

public/mediapipe/NOTICE.txt

@@ -0,0 +1,8 @@
+MediaPipe Tasks Vision runtime and Selfie Segmentation model
+Copyright 2023 Google LLC
+
+Licensed under the Apache License, Version 2.0.
+https://www.apache.org/licenses/LICENSE-2.0
+
+The bundled model performs human/background segmentation locally on the
+user's device. Source project: https://github.com/google-ai-edge/mediapipe

scripts/dev-api.mjs

@@ -0,0 +1,39 @@
+import { spawn } from "node:child_process";
+import { once } from "node:events";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { ensureWindowsX64SpeechRuntime } from "./ensure-windows-x64-speech-runtime.mjs";
+
+const scriptDirectory = path.dirname(fileURLToPath(import.meta.url));
+const rootDirectory = path.resolve(scriptDirectory, "..");
+const nodeExecutable = await ensureWindowsX64SpeechRuntime();
+const usesX64Sidecar = nodeExecutable !== process.execPath;
+if (usesX64Sidecar) {
+  await import(`./build-server.mjs?dev=${Date.now()}`);
+}
+const childArguments = usesX64Sidecar
+  ? [path.join(rootDirectory, "dist-server", "index.mjs")]
+  : [
+      path.join(rootDirectory, "node_modules", "tsx", "dist", "cli.mjs"),
+      "watch",
+      "server/index.ts",
+    ];
+const child = spawn(
+  nodeExecutable,
+  childArguments,
+  {
+    cwd: rootDirectory,
+    env: process.env,
+    stdio: "inherit",
+    windowsHide: true,
+  },
+);
+
+for (const signal of ["SIGINT", "SIGTERM"]) {
+  process.on(signal, () => {
+    if (!child.killed) child.kill();
+  });
+}
+
+const [exitCode] = await once(child, "exit");
+process.exitCode = exitCode ?? 1;

scripts/ensure-windows-x64-speech-runtime.mjs

@@ -0,0 +1,150 @@
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { once } from "node:events";
+import { access, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const scriptDirectory = path.dirname(fileURLToPath(import.meta.url));
+const rootDirectory = path.resolve(scriptDirectory, "..");
+const runtimeDirectory = path.join(
+  rootDirectory,
+  ".tmp",
+  "windows-x64-speech",
+);
+
+async function fileExists(filePath) {
+  try {
+    await access(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function assertX64Executable(filePath) {
+  const executable = await readFile(filePath);
+  if (executable.toString("ascii", 0, 2) !== "MZ") {
+    throw new Error(`${filePath} is not a Windows executable.`);
+  }
+  const peOffset = executable.readUInt32LE(0x3c);
+  if (executable.readUInt16LE(peOffset + 4) !== 0x8664) {
+    throw new Error(`${filePath} is not an x64 executable.`);
+  }
+}
+
+async function verifiedDownload(url, integrity, destination) {
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Runtime download failed with HTTP ${response.status}.`);
+  }
+  const bytes = Buffer.from(await response.arrayBuffer());
+  const [algorithm, expected] = integrity.split("-", 2);
+  const actual = createHash(algorithm).update(bytes).digest("base64");
+  if (actual !== expected) {
+    throw new Error(`Runtime download from ${url} failed its integrity check.`);
+  }
+  await writeFile(destination, bytes);
+}
+
+async function ensureX64Node() {
+  const nodeExecutable = path.join(runtimeDirectory, "node.exe");
+  if (await fileExists(nodeExecutable)) {
+    await assertX64Executable(nodeExecutable);
+    return nodeExecutable;
+  }
+
+  const version = `v${process.versions.node}`;
+  const baseUrl = `https://nodejs.org/dist/${version}`;
+  const checksumResponse = await fetch(`${baseUrl}/SHASUMS256.txt`);
+  if (!checksumResponse.ok) {
+    throw new Error(
+      `Could not read the Node.js ${version} release checksums.`,
+    );
+  }
+  const checksumLine = (await checksumResponse.text())
+    .split(/\r?\n/u)
+    .find((line) => line.endsWith("  win-x64/node.exe"));
+  const checksum = checksumLine?.split(/\s+/u, 1)[0];
+  if (!checksum) {
+    throw new Error(`Node.js ${version} has no Windows x64 runtime.`);
+  }
+
+  console.log("Preparing the Windows x64 speech sidecar (one-time setup)…");
+  const response = await fetch(`${baseUrl}/win-x64/node.exe`);
+  if (!response.ok) {
+    throw new Error(`Node.js runtime download failed with HTTP ${response.status}.`);
+  }
+  const bytes = Buffer.from(await response.arrayBuffer());
+  const actual = createHash("sha256").update(bytes).digest("hex");
+  if (actual !== checksum) {
+    throw new Error("The Node.js x64 runtime failed its checksum validation.");
+  }
+  await writeFile(nodeExecutable, bytes);
+  await assertX64Executable(nodeExecutable);
+  return nodeExecutable;
+}
+
+async function ensureSherpaX64() {
+  const sherpaNodePackage = JSON.parse(
+    await readFile(
+      path.join(rootDirectory, "node_modules", "sherpa-onnx-node", "package.json"),
+      "utf8",
+    ),
+  );
+  const requestedVersion =
+    sherpaNodePackage.optionalDependencies?.["sherpa-onnx-win-x64"];
+  const version = requestedVersion?.replace(/^[^0-9]*/u, "");
+  if (!version) {
+    throw new Error("sherpa-onnx-node does not declare a Windows x64 runtime.");
+  }
+
+  const destination = path.join(
+    rootDirectory,
+    "node_modules",
+    "sherpa-onnx-win-x64",
+  );
+  const nativeAddon = path.join(destination, "sherpa-onnx.node");
+  if (await fileExists(nativeAddon)) return;
+
+  console.log("Installing the Windows x64 Sherpa speech runtime…");
+  const metadataResponse = await fetch(
+    `https://registry.npmjs.org/sherpa-onnx-win-x64/${version}`,
+  );
+  if (!metadataResponse.ok) {
+    throw new Error(
+      `Sherpa runtime metadata returned HTTP ${metadataResponse.status}.`,
+    );
+  }
+  const metadata = await metadataResponse.json();
+  const tarballUrl = metadata.dist?.tarball;
+  const integrity = metadata.dist?.integrity;
+  if (typeof tarballUrl !== "string" || typeof integrity !== "string") {
+    throw new Error("Sherpa runtime metadata is incomplete.");
+  }
+
+  const archivePath = path.join(runtimeDirectory, "sherpa-onnx-win-x64.tgz");
+  await verifiedDownload(tarballUrl, integrity, archivePath);
+  await rm(destination, { recursive: true, force: true });
+  await mkdir(destination, { recursive: true });
+  const extraction = spawn(
+    "tar",
+    ["-xzf", archivePath, "--strip-components", "1", "-C", destination],
+    { cwd: rootDirectory, stdio: "inherit", windowsHide: true },
+  );
+  const [exitCode] = await once(extraction, "exit");
+  await rm(archivePath, { force: true });
+  if (exitCode !== 0 || !(await fileExists(nativeAddon))) {
+    throw new Error("The Windows x64 Sherpa runtime could not be extracted.");
+  }
+}
+
+export async function ensureWindowsX64SpeechRuntime() {
+  if (process.platform !== "win32" || process.arch !== "arm64") {
+    return process.execPath;
+  }
+  await mkdir(runtimeDirectory, { recursive: true });
+  const nodeExecutable = await ensureX64Node();
+  await ensureSherpaX64();
+  return nodeExecutable;
+}

scripts/smoke-speech.mjs

@@ -0,0 +1,57 @@
+import { spawn } from "node:child_process";
+import { once } from "node:events";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { ensureWindowsX64SpeechRuntime } from "./ensure-windows-x64-speech-runtime.mjs";
+
+const scriptDirectory = path.dirname(fileURLToPath(import.meta.url));
+const rootDirectory = path.resolve(scriptDirectory, "..");
+const nodeExecutable = await ensureWindowsX64SpeechRuntime();
+const usesX64Sidecar = nodeExecutable !== process.execPath;
+let childArguments;
+if (usesX64Sidecar) {
+  const { build } = await import("esbuild");
+  const bundledSmokeTest = path.join(
+    rootDirectory,
+    ".tmp",
+    "windows-x64-speech",
+    "smoke-speech.mjs",
+  );
+  await build({
+    absWorkingDir: rootDirectory,
+    entryPoints: ["scripts/smoke-speech.ts"],
+    outfile: bundledSmokeTest,
+    bundle: true,
+    external: ["sherpa-onnx-node"],
+    format: "esm",
+    platform: "node",
+    target: "node22",
+    banner: {
+      js:
+        'import { createRequire as __nodeCreateRequire } from "node:module";' +
+        "const require = __nodeCreateRequire(import.meta.url);",
+    },
+  });
+  childArguments = [bundledSmokeTest];
+} else {
+  childArguments = [
+    path.join(rootDirectory, "node_modules", "tsx", "dist", "cli.mjs"),
+    path.join(scriptDirectory, "smoke-speech.ts"),
+  ];
+}
+const smokeTest = spawn(
+  nodeExecutable,
+  childArguments,
+  {
+    cwd: rootDirectory,
+    env: {
+      ...process.env,
+      PROMPTER_ROOT_DIR: rootDirectory,
+    },
+    stdio: "inherit",
+    windowsHide: true,
+  },
+);
+
+const [exitCode] = await once(smokeTest, "exit");
+process.exitCode = exitCode ?? 1;

scripts/smoke-speech.ts

@@ -25,7 +25,9 @@ interface SpeechMessage {
 const require = createRequire(import.meta.url);
 const sherpa = require("sherpa-onnx-node") as SherpaModule;
 const scriptDirectory = path.dirname(fileURLToPath(import.meta.url));
-const rootDirectory = path.resolve(scriptDirectory, "..");
+const rootDirectory = process.env.PROMPTER_ROOT_DIR
+  ? path.resolve(process.env.PROMPTER_ROOT_DIR)
+  : path.resolve(scriptDirectory, "..");
 const modelDirectory = path.join(
   rootDirectory,
   ".models",

server/index.ts

@@ -102,6 +102,7 @@ app.post(
           "aac",
           "-b:a",
           "192k",
+          "-shortest",
           "-movflags",
           "+faststart",
           outputPath,
@@ -170,18 +171,25 @@ app.post(
       path.join(tmpdir(), "prompter-subtitle-export-"),
     );
     const inputPath = path.join(workingDirectory, "take.recording");
+    const audioPath = path.join(workingDirectory, "take-audio.recording");
     const subtitlePath = path.join(workingDirectory, "captions.ass");
     const outputPath = path.join(workingDirectory, "take-rendered.mp4");
 
     try {
       const writes = [writeFile(inputPath, parsedExport.recording)];
+      if (parsedExport.audioRecording) {
+        writes.push(writeFile(audioPath, parsedExport.audioRecording));
+      }
       if (parsedExport.request.mode === "subtitles") {
         writes.push(
           writeFile(
             subtitlePath,
             buildAssSubtitles(
               parsedExport.request.words,
               parsedExport.request.fontFamily,
+              parsedExport.request.aspectRatio,
+              parsedExport.request.highlightColor,
+              parsedExport.request.subtitleTreatment,
             ),
             "utf8",
           ),
@@ -198,23 +206,25 @@ app.post(
           "-y",
           "-i",
           inputPath,
+          ...(parsedExport.audioRecording ? ["-i", audioPath] : []),
           "-map",
           "0:v:0",
           "-map",
-          "0:a:0",
+          parsedExport.audioRecording ? "1:a:0" : "0:a:0",
           ...(videoFilter ? ["-vf", videoFilter] : []),
           "-c:v",
           "libx264",
           "-preset",
-          "veryfast",
+          parsedExport.request.preserveQuality ? "medium" : "veryfast",
           "-crf",
-          "20",
+          parsedExport.request.preserveQuality ? "17" : "20",
           "-pix_fmt",
           "yuv420p",
           "-c:a",
           "aac",
           "-b:a",
           "192k",
+          "-shortest",
           "-movflags",
           "+faststart",
           outputPath,