fix(setup): make the model step honest and interruptible

app/core/model_cache.py

@@ -28,6 +28,7 @@
 
 import logging
 import shutil
+import traceback
 from collections.abc import Callable, Iterable
 from pathlib import Path
 from typing import TypeVar
@@ -91,6 +92,11 @@ def load_or_heal(load: Callable[[], T], stale: Callable[[], Iterable[Path]]) ->
         fresh = [p for p in paths if p not in _healed]
         if not fresh:
             raise
+        # audio-separator reads its indexes with json.load(open(...)) and
+        # never closes the file, so the one that failed is still open in the
+        # traceback's frames, and Windows will not delete an open file.
+        # Clearing the frames' locals closes it; the traceback itself stays.
+        traceback.clear_frames(first.__traceback__)
         removed = [p for p in fresh if _remove(p)]
         _healed.update(fresh)
         if not removed:
@@ -125,10 +131,16 @@ def vocal_split_artifacts(models_dir: Path, model_file: str) -> list[Path]:
     they are the names audio-separator actually writes: a single
     "model_data.json" is not one of them, so listing that instead heals the
     checkpoint and leaves the index that produced the error in place.
+
+    download_checks.json, the list of models audio-separator offers, is read
+    before any of them, and is written in place like the rest: a setup ended
+    mid-fetch (the wizard's Skip, #405) leaves it half written, and json.load
+    fails on it every time after.
     """
     root = models_dir / "audio-separator"
     return [
         root / model_file,
         root / "vr_model_data.json",
         root / "mdx_model_data.json",
+        root / "download_checks.json",
     ]

app/pipeline/warmup.py

@@ -8,10 +8,23 @@
 steps; Docker has no equivalent step and keeps the pre-existing
 lazy-download-on-first-use behavior (see docs/models.md).
 
-One line per model is printed to stdout so the caller can report per-model
-status: "WARMUP_OK <name>" on success, "WARMUP_FAILED <name> <error>" on
-failure. A model failing to download here is never fatal to setup -- exactly
-like the lazy path it replaces, a missing model degrades only that one
+Lines on stdout tell the caller where it is (#405): "WARMUP_START <name> <n>
+<count>" before each model, then "WARMUP_OK <name>" on success or
+"WARMUP_FAILED <name> <error>" on failure, followed by "WARMUP_OFFLINE <name>"
+when the download host could not be reached or stopped answering. When the
+disk cannot hold the models, the only line is "WARMUP_NO_SPACE <needed bytes>
+<free bytes>" and nothing is downloaded.
+
+The steps run inside bounded_downloads (app/core/model_download.py): a
+torch.hub download (the separation and beat models) gets the separation
+model's read timeout and tries, so a dead connection ends it in about a
+minute and a half, and every other socket that sets no timeout of its own
+(Whisper's) gets the read timeout. The Hugging Face hub and audio-separator
+set their own, audio-separator 300 s per read, which the shell's stall limit
+outlasts.
+
+A model failing to download here is never fatal to setup -- exactly like the
+lazy path it replaces, a missing model degrades only that one
 feature (e.g. no beat grid, or the karaoke split unavailable until it can
 download later) rather than blocking the app from starting.
 
@@ -31,18 +44,38 @@
 
 from __future__ import annotations
 
+import http.client
 import os
+import shutil
+import socket
 import sys
 
 from app.core.config import (
     BEAT_MODEL_CHECKPOINT,
     DEMUCS_MODEL,
+    MODEL_DOWNLOAD_ATTEMPTS,
     MODELS_DIR,
     SECTION_MODEL,
+    TIMEOUT_MODEL_DOWNLOAD_READ,
     TRANSCRIBE_MODEL_CPU,
     TRANSCRIBE_MODEL_GPU,
     VOCAL_SPLIT_MODEL,
 )
+from app.core.model_download import bounded_downloads
+
+# What each model adds to the data folder, in bytes: measured on a finished
+# install (models/ and cache/huggingface/, 2026-11-03), except Whisper's
+# "small", which is its published size. Only the space check before anything
+# downloads uses these, so a model renamed through the environment is just not
+# counted.
+_DOWNLOAD_BYTES = {
+    "demucs": 54_996_327,
+    "beat_this": 81_058_141,
+    "sections": 11_204_803,
+    "vocal_split": 52_834_731,
+    "turbo": 1_617_941_637,
+    "small": 483_617_219,
+}
 
 
 def _warm_demucs() -> None:
@@ -90,24 +123,30 @@ def _warm_sections() -> None:
     load_pretrained_model(model_name=SECTION_MODEL, device="cpu")
 
 
-def _warm_whisper() -> None:
-    """The Whisper model the lyrics stage would use here, and only when that
-    stage would run: under "auto" only a CUDA install transcribes, and the
-    GPU model is 1.6 GB. Last in the list for the same reason, so a slow
-    connection costs the other models nothing. A download cut short by the
-    setup timeout heals itself: Whisper checks the file's SHA-256 on every
-    load and fetches it again when it does not match."""
+def _whisper_model() -> str | None:
+    """The Whisper model the lyrics stage would use here, or None when that
+    stage would not run: under "auto" only a CUDA install transcribes."""
     from app.core.settings import get_demucs_device, transcribe_lyrics_enabled
 
     device = get_demucs_device()
     if not transcribe_lyrics_enabled(device):
+        return None
+    return TRANSCRIBE_MODEL_GPU if device == "cuda" else TRANSCRIBE_MODEL_CPU
+
+
+def _warm_whisper() -> None:
+    """The GPU model is 1.6 GB, so this is last in the list: a slow
+    connection costs the other models nothing. A download cut short by the
+    setup timeout heals itself: Whisper checks the file's SHA-256 on every
+    load and fetches it again when it does not match."""
+    name = _whisper_model()
+    if name is None:
         return
 
     import whisper
 
     from app.pipeline.transcribe import whisper_models_dir
 
-    name = TRANSCRIBE_MODEL_GPU if device == "cuda" else TRANSCRIBE_MODEL_CPU
     whisper.load_model(name, device="cpu", download_root=str(whisper_models_dir()))
 
 
@@ -137,17 +176,85 @@ def _remember_vocal_split(error: Exception | None) -> None:
         print(f"WARMUP_NOTE vocal_split status not recorded: {e}", flush=True)
 
 
+def _free_bytes() -> int:
+    """Free space on the disk MODELS_DIR is on, which may not exist yet."""
+    folder = MODELS_DIR
+    while not folder.exists() and folder != folder.parent:
+        folder = folder.parent
+    return shutil.disk_usage(folder).free
+
+
+def _bytes_needed() -> int:
+    """About what the downloads still need: every model's size, less what
+    MODELS_DIR already holds, so an install that has them all needs nothing.
+
+    It errs towards trying. Whatever else sits in MODELS_DIR counts as
+    present, and the sections model's few MB in the Hugging Face cache are
+    always counted as needed rather than walking a cache the user may share
+    with other tools. A check that passes wrongly costs a download that fails
+    on the full disk with its own error, which the wizard reports; one that
+    fails wrongly would skip every model for nothing. A torch.hub ".partial"
+    is a download cut short, not a model."""
+    total = 0
+    for name, _ in _STEPS:
+        key = _whisper_model() if name == "whisper" else name
+        total += _DOWNLOAD_BYTES.get(key or "", 0)
+    present = sum(
+        path.stat().st_size
+        for path in (MODELS_DIR.rglob("*") if MODELS_DIR.is_dir() else ())
+        if path.is_file() and not path.is_symlink() and path.suffix != ".partial"
+    )
+    return max(0, total - present)
+
+
+def _enough_space() -> bool:
+    try:
+        needed, free = _bytes_needed(), _free_bytes()
+    except Exception as e:
+        # Only a courtesy: a full disk still fails each download with its own
+        # error. Not a reason to download nothing.
+        print(f"WARMUP_NOTE space not checked: {e}", flush=True)
+        return True
+    if needed > free:
+        print(f"WARMUP_NO_SPACE {needed} {free}", flush=True)
+        return False
+    return True
+
+
+def _unreachable(error: BaseException) -> bool:
+    """Whether a download failed because its host could not be reached or
+    stopped answering: no network, no DNS, a refused or dropped connection, a
+    read that timed out. The whole chain is read, since every library wraps
+    these in errors of its own (requests, httpx, the hub, ModelDownloadError)."""
+    seen: set[int] = set()
+    current: BaseException | None = error
+    while current is not None and id(current) not in seen:
+        if isinstance(
+            current, (socket.gaierror, TimeoutError, ConnectionError, http.client.IncompleteRead)
+        ):
+            return True
+        seen.add(id(current))
+        current = current.__cause__ or current.__context__
+    return False
+
+
 def main() -> int:
-    for name, fn in _STEPS:
-        try:
-            fn()
-            print(f"WARMUP_OK {name}", flush=True)
-            if name == "vocal_split":
-                _remember_vocal_split(None)
-        except Exception as e:
-            print(f"WARMUP_FAILED {name} {e}", flush=True)
-            if name == "vocal_split":
-                _remember_vocal_split(e)
+    if not _enough_space():
+        return 0
+    with bounded_downloads(TIMEOUT_MODEL_DOWNLOAD_READ, MODEL_DOWNLOAD_ATTEMPTS):
+        for number, (name, fn) in enumerate(_STEPS, start=1):
+            print(f"WARMUP_START {name} {number} {len(_STEPS)}", flush=True)
+            try:
+                fn()
+                print(f"WARMUP_OK {name}", flush=True)
+                if name == "vocal_split":
+                    _remember_vocal_split(None)
+            except Exception as e:
+                print(f"WARMUP_FAILED {name} {e}", flush=True)
+                if _unreachable(e):
+                    print(f"WARMUP_OFFLINE {name}", flush=True)
+                if name == "vocal_split":
+                    _remember_vocal_split(e)
     return 0  # best-effort throughout -- see module docstring
 
 

desktop/src-tauri/src/backendjob.rs

@@ -54,7 +54,7 @@ mod windows_impl {
     };
     use windows_sys::Win32::System::JobObjects::{
         AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation,
-        SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
+        SetInformationJobObject, TerminateJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
         JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
     };
     use windows_sys::Win32::System::Threading::{
@@ -67,8 +67,21 @@ mod windows_impl {
     // SAFETY: a job handle is a kernel handle, usable from any thread; it is
     // only closed once, in Drop.
     unsafe impl Send for KillOnCloseJob {}
+    // SAFETY: shared use only terminates the job, which Windows allows from
+    // any thread on the same handle at once.
+    unsafe impl Sync for KillOnCloseJob {}
 
     impl KillOnCloseJob {
+        /// End every process in the job now, while StemDeck keeps running:
+        /// the setup wizard's Skip on the model step (#405).
+        pub fn terminate(&self) -> io::Result<()> {
+            // SAFETY: the handle is owned by `self` and open until Drop.
+            if unsafe { TerminateJobObject(self.0, 1) } == 0 {
+                return Err(io::Error::last_os_error());
+            }
+            Ok(())
+        }
+
         fn new() -> io::Result<Self> {
             // SAFETY: plain Win32 calls; the handle is owned by the returned
             // value, and `info` is a zeroed plain-data struct of the size given.