From a48d3573a3ef7dc180763ddc6531134c7c67a627 Mon Sep 17 00:00:00 2001 From: pgoode41 Date: Tue, 22 Sep 2026 03:44:08 -0400 Subject: [PATCH 01/13] Add managed llama.cpp as a third engine across the services Register llama.cpp in the shared engine table and give it a facade on the unified proxy at 8080, with the PAIR-managed engine on 8081. Engine Manager installs the official llama app into an owned directory from a checksum-pinned installer or archive recipe (Windows ARM64 tries the current upstream build first and falls back to pinned CUDA archives; confirmed non-NVIDIA ARM and Intel Macs use the pinned CPU recipes), starts and stops it, keeps a private model cache, and exposes download, import, load, unload, delete and download cancellation. Uninstall removes only the runtime slots and keeps models. Routing eligibility for llama.cpp is loaded models only, because the engine runs with --no-models-autoload. The proxy keeps per-facade cancellation isolated and delivers exactly one terminal workload event even when a committed upstream stream is truncated; the classification of that case follows the foundation's existing mid-stream test. The broker advertises the engine, probes manual peers for it, and applies the same engine-settings port path it uses for Ollama and LM Studio, made profile-generic so a third engine does not fall through two-engine branches. llama.cpp declares reviewed launch controls (--port, --host) with serve --no-models-autoload fixed; its owned cache environment is injected on every launch rather than edited, and the settings preview refuses attempts to set it. This is the focused integration replayed onto the public develop foundation from the internal review branch. Runtime updating is not offered for llama.cpp (the desktop bridge refuses update rather than substituting uninstall and reinstall), and the GPU/OS telemetry work is kept as a separate patch candidate. Co-authored-by: Terve Co-Authored-By: Claude Fable 5.1 Signed-off-by: pgoode41 --- services/nvpair-engine-manager/LAUNCH_TEXT.md | 3 +- services/nvpair-engine-manager/README.md | 225 +++++- services/nvpair-engine-manager/actions.go | 27 +- services/nvpair-engine-manager/childenv.go | 119 ++++ .../nvpair-engine-manager/childenv_test.go | 46 ++ .../nvpair-engine-manager/controlmodels.go | 14 +- .../nvpair-engine-manager/controlserver.go | 1 + services/nvpair-engine-manager/executor.go | 38 +- services/nvpair-engine-manager/install.go | 70 +- .../launch_controls_test.go | 13 +- services/nvpair-engine-manager/launch_test.go | 12 +- services/nvpair-engine-manager/lifecycle.go | 99 ++- .../nvpair-engine-manager/llamaactions.go | 186 +++++ .../nvpair-engine-manager/llamaarchives.go | 142 ++++ .../llamaarchives_test.go | 118 ++++ services/nvpair-engine-manager/llamaarmcpu.go | 119 ++++ .../nvpair-engine-manager/llamaarmcpu_test.go | 131 ++++ services/nvpair-engine-manager/llamacache.go | 78 +++ .../nvpair-engine-manager/llamacache_test.go | 82 +++ services/nvpair-engine-manager/llamacompat.go | 221 ++++++ .../nvpair-engine-manager/llamacompat_test.go | 327 +++++++++ .../nvpair-engine-manager/llamainstall.go | 316 +++++++++ .../llamainstall_test.go | 641 ++++++++++++++++++ services/nvpair-engine-manager/llamamodels.go | 461 +++++++++++++ .../nvpair-engine-manager/llamamodels_test.go | 182 +++++ .../llamapath_windows_test.go | 65 ++ services/nvpair-engine-manager/llamatar.go | 168 +++++ .../nvpair-engine-manager/llamatar_test.go | 143 ++++ .../nvpair-engine-manager/llamaupstream.go | 217 ++++++ .../llamaupstream_test.go | 558 +++++++++++++++ services/nvpair-engine-manager/loadedwatch.go | 9 + services/nvpair-engine-manager/main.go | 5 + services/nvpair-engine-manager/manager.go | 6 + .../manifests/llamacpp.json | 188 +++++ services/nvpair-engine-manager/modelpath.go | 32 + services/nvpair-engine-manager/models.go | 4 +- .../nvpair-engine-manager/proc_windows.go | 30 + services/nvpair-engine-manager/pull.go | 8 + services/nvpair-engine-manager/registry.go | 64 +- .../nvpair-engine-manager/remediation_test.go | 2 +- services/nvpair-engine-manager/remote.go | 4 +- .../nvpair-engine-manager/remoteclient.go | 2 +- .../remoteclient_test.go | 1 + services/nvpair-engine-manager/settings.go | 11 +- .../settings_llama_test.go | 67 ++ services/nvpair-engine-manager/status.go | 70 +- .../testdata/fakeengine/main.go | 31 + services/nvpair-engine-manager/usererrors.go | 32 + .../nvpair-job-scheduler/schedule_test.go | 12 + services/nvpair-manual-nodes/README.md | 10 +- services/nvpair-manual-nodes/manager.go | 91 ++- services/nvpair-manual-nodes/manager_test.go | 76 +++ services/nvpair-proxy/cancel_test.go | 198 ++++++ services/nvpair-proxy/eligibility_test.go | 128 ++++ services/nvpair-proxy/enginecase_test.go | 50 +- services/nvpair-proxy/engines.go | 41 +- services/nvpair-proxy/facade.go | 79 +++ services/nvpair-proxy/main.go | 11 + services/nvpair-proxy/portstore_test.go | 22 +- services/nvpair-proxy/proxy.go | 142 +++- services/nvpair-proxy/subscribed_test.go | 9 +- services/nvpair-proxy/terminal_test.go | 85 +++ services/nvpair-proxy/zombie_test.go | 48 +- services/nvpair-tui/README.md | 13 +- services/nvpair-tui/ui/engines.go | 264 +++++++- services/nvpair-tui/ui/engines_test.go | 147 +++- services/nvpair-tui/ui/proxies_test.go | 41 ++ services/nvpair-tui/ui/rpccmd.go | 16 +- services/nvpair-tui/ui/workloads.go | 115 +++- services/nvpair-tui/ui/workloads_test.go | 43 ++ services/nvpair-ui-broker/ENGINE_SETTINGS.md | 69 +- services/nvpair-ui-broker/README.md | 42 +- services/nvpair-ui-broker/advertiser.go | 72 +- services/nvpair-ui-broker/advertiser_test.go | 3 + services/nvpair-ui-broker/broker.go | 139 +++- .../nvpair-ui-broker/broker_lifecycle_test.go | 47 +- services/nvpair-ui-broker/engineproxy.go | 11 + services/nvpair-ui-broker/enginesettings.go | 79 ++- .../enginesettings_recovery.go | 85 ++- .../enginesettings_review_test.go | 95 ++- .../nvpair-ui-broker/enginesettings_test.go | 194 +++++- .../llamacpp_advertise_test.go | 215 ++++++ services/nvpair-ui-broker/llamacppport.go | 458 +++++++++++++ .../nvpair-ui-broker/llamacppport_test.go | 284 ++++++++ services/nvpair-ui-broker/llamacppproxy.go | 152 +++++ services/nvpair-ui-broker/lmstudioport.go | 29 +- services/nvpair-ui-broker/lmstudioproxy.go | 18 +- services/nvpair-ui-broker/manualnodes.go | 32 +- services/nvpair-ui-broker/manualnodes_test.go | 150 ++++ services/nvpair-ui-broker/proxyport.go | 16 +- services/nvpair-ui-broker/settingsport.go | 19 +- services/readme.md | 9 +- services/shared/appdir/appdir.go | 10 + services/shared/appdir/appdir_test.go | 44 ++ services/shared/engines/engines.go | 26 +- services/shared/engines/engines_test.go | 2 +- services/shared/noderec/noderec.go | 14 +- services/shared/noderec/noderec_test.go | 42 ++ .../shared/splitlisten/splitlisten_test.go | 10 +- services/tests/llama_headless_test.go | 69 ++ services/tests/main_test.go | 2 - services/tests/remote_engine_test.go | 62 +- services/tests/scheduler_interop_test.go | 27 +- 103 files changed, 9129 insertions(+), 426 deletions(-) create mode 100644 services/nvpair-engine-manager/childenv.go create mode 100644 services/nvpair-engine-manager/childenv_test.go create mode 100644 services/nvpair-engine-manager/llamaactions.go create mode 100644 services/nvpair-engine-manager/llamaarchives.go create mode 100644 services/nvpair-engine-manager/llamaarchives_test.go create mode 100644 services/nvpair-engine-manager/llamaarmcpu.go create mode 100644 services/nvpair-engine-manager/llamaarmcpu_test.go create mode 100644 services/nvpair-engine-manager/llamacache.go create mode 100644 services/nvpair-engine-manager/llamacache_test.go create mode 100644 services/nvpair-engine-manager/llamacompat.go create mode 100644 services/nvpair-engine-manager/llamacompat_test.go create mode 100644 services/nvpair-engine-manager/llamainstall.go create mode 100644 services/nvpair-engine-manager/llamainstall_test.go create mode 100644 services/nvpair-engine-manager/llamamodels.go create mode 100644 services/nvpair-engine-manager/llamamodels_test.go create mode 100644 services/nvpair-engine-manager/llamapath_windows_test.go create mode 100644 services/nvpair-engine-manager/llamatar.go create mode 100644 services/nvpair-engine-manager/llamatar_test.go create mode 100644 services/nvpair-engine-manager/llamaupstream.go create mode 100644 services/nvpair-engine-manager/llamaupstream_test.go create mode 100644 services/nvpair-engine-manager/manifests/llamacpp.json create mode 100644 services/nvpair-engine-manager/modelpath.go create mode 100644 services/nvpair-engine-manager/settings_llama_test.go create mode 100644 services/nvpair-proxy/cancel_test.go create mode 100644 services/nvpair-proxy/eligibility_test.go create mode 100644 services/nvpair-proxy/terminal_test.go create mode 100644 services/nvpair-tui/ui/proxies_test.go create mode 100644 services/nvpair-tui/ui/workloads_test.go create mode 100644 services/nvpair-ui-broker/llamacpp_advertise_test.go create mode 100644 services/nvpair-ui-broker/llamacppport.go create mode 100644 services/nvpair-ui-broker/llamacppport_test.go create mode 100644 services/nvpair-ui-broker/llamacppproxy.go create mode 100644 services/nvpair-ui-broker/manualnodes_test.go create mode 100644 services/shared/appdir/appdir_test.go create mode 100644 services/tests/llama_headless_test.go diff --git a/services/nvpair-engine-manager/LAUNCH_TEXT.md b/services/nvpair-engine-manager/LAUNCH_TEXT.md index 12a59b91..5f3e9ea7 100644 --- a/services/nvpair-engine-manager/LAUNCH_TEXT.md +++ b/services/nvpair-engine-manager/LAUNCH_TEXT.md @@ -149,7 +149,7 @@ The existing engine-manager suite also exercises the JSON-RPC stdio path. ## Networking adapter coverage -The bundled engines are Ollama and LM Studio. The shared parser contains no engine +The bundled engines are Ollama, LM Studio and llama.cpp. The shared parser contains no engine name checks. Their mandatory networking declarations and all platform variants are pinned by TestBundledNetworkingControls; adding an engine requires extending that inventory and reviewing its networking alternatives. Other settings have no catalog. @@ -158,6 +158,7 @@ that inventory and reviewing its networking alternatives. Other settings have no | --- | --- | --- | --- | | Ollama | OLLAMA_HOST | OLLAMA_ORIGINS, including vendor quote stripping | [Environment source](https://github.com/ollama/ollama/blob/main/envconfig/config.go) | | LM Studio | --port / -p; --bind / LMS_SERVER_HOST | --cors (no short alias) | [Server command source](https://github.com/lmstudio-ai/lms/blob/main/src/subcommands/server.ts) | +| llama.cpp | --port; --host | none declared: the llama app exposes no CORS switch, so browser access follows its own responses | [llama.cpp repository](https://github.com/ggml-org/llama.cpp) | Remote CORS comparisons use normalized policy. The broker derives the internal preserveCORS preview guard from the authenticated caller and repeats validation diff --git a/services/nvpair-engine-manager/README.md b/services/nvpair-engine-manager/README.md index b401c883..b467018a 100644 --- a/services/nvpair-engine-manager/README.md +++ b/services/nvpair-engine-manager/README.md @@ -5,11 +5,12 @@ SPDX-License-Identifier: Apache-2.0 # nvpair-engine-manager -A config-driven control plane for local inference engines (Ollama today; -Intel/others via a dropped-in manifest). It manages everything about an -engine **except serving inference**: detect, user-mode install, -start/stop/restart, health, and config-declared actions. Adding an engine -is a JSON manifest, not code. +A config-driven control plane for local inference engines, including Ollama, +LM Studio and llama.cpp. It manages everything about an engine **except serving +inference**: detect, user-mode install, start/stop/restart, health, and +config-declared actions. Manifests describe common operations; engine-specific +backend drivers handle ownership or lifecycle behavior a command recipe cannot +safely express. The desktop and TUI remain clients of this control plane. The bundled manifests under `manifests/` are the working reference for manifest authoring. @@ -134,6 +135,34 @@ SIGKILL escalation), and `taskkill /T /F` on Windows — where the windowless engines we spawn can't receive a graceful (non-`/F`) close, so a forced terminate is the only signal that actually stops them. +### Managed install/uninstall contract + +This is the required recipe standard shared by Ollama, LM Studio and llama.cpp. +It defines acceptance requirements, not a blanket certification of legacy +recipes. Each engine must provide evidence for its actual platform and layout. + +| Operation or boundary | Required behavior | +| --- | --- | +| Install | Put the managed runtime in a PAIR-owned location using the supported vendor path. Do not overwrite a detected external installation or take ownership of its data. | +| Detection/adoption | Finding an executable or serving endpoint does not grant uninstall authority. Keep existing engine-specific detection, start/stop and port behavior; prove ownership separately before removal. | +| Uninstall | Stop the correct managed instance, then remove only its owned runtime. Refuse removal of external, shared or legacy installations when ownership cannot be established, including command-mode engines. | +| Retention | Keep normal separate model libraries, settings and user data. Runtime removal is not profile reset or model cleanup. Persisting the user's Off intent is allowed; erasing their configuration is not. | +| Reinstall | Reuse retained data without requiring models to be downloaded again. A vendor cache is not disposable merely because the runtime was removed. | +| Failure | Return an actionable error and the observed state. A successful command exit alone does not prove runtime removal or data preservation. | + +Parity here is **install/uninstall ownership and retention**. It does not add a +new detach interface, job-drain mechanism, profile-reset/model-cleanup feature, +API-first rewrite or generic updater redesign. Update behavior remains +engine-specific: Ollama and LM Studio keep their existing update paths, and +managed llama has no update action. Uninstall-then-install is not a substitute +for one and must not be wired up as such for superficial similarity. + +Minimum recipe evidence: managed install/detection, bounded start/stop, runtime +removal with model/settings retention, reinstall using retained data, and +external/shared-install refusal. A manifest or mock alone does not establish +native vendor-package behavior. For llama, only `runtime` and `previous` are +removable installation slots; model/cache and settings paths remain separate. + ### Adoption — start may attach to an engine it didn't launch Before spawning, `engine:start` **probes the chosen port's readiness @@ -294,3 +323,189 @@ termination, console hiding) are the only build-tagged Go Shuts down on stdin EOF (parent closed the pipe), `SIGINT`/`SIGTERM`, or a `shutdown` JSON-RPC request — stopping any running engines first so none are orphaned. + +## Managed llama app + +On Windows x64, the qualified b10826 / 73a43d1f6 Vulkan runtime automatically +receives a B580 compatibility profile before a managed start when +`llama cli --list-devices` actually enumerates Intel Arc B580. Automatic device +selection and explicit device lists containing that B580 receive the profile; +explicit CPU (including zero GPU layers), CUDA, or other Vulkan device selections retain their options. +PAIR checks the existing managed install receipt, pinned installer identity, +and exact qualified executable SHA-256 before enumeration. Unknown or adopted +runtimes do not receive defaults. Windows on ARM with CUDA, Apple Silicon, +other engines, and other Vulkan devices do not acquire this profile. + +The process-local defaults are `GGML_VK_DISABLE_COOPMAT=1`, +`GGML_VK_DISABLE_COOPMAT2=1`, `GGML_VK_DISABLE_INTEGER_DOT_PRODUCT=1`, +`GGML_VK_DISABLE_F16=1`, `GGML_VK_DISABLE_BFLOAT16=1`, and +`LLAMA_ARG_FLASH_ATTN=off`. No ASYNC override is added. These defaults also +reach model-serving children of `llama serve`. They apply to fresh and older +managed installations on their next start after a PAIR upgrade; no reinstall, +model change, or receipt rewrite is needed. Install on an already-installed +runtime stays a no-op, and saved Off stays Off. No global environment, registry, driver, +upstream source, or safety/bounds check is changed. + +Existing runtime environment options override inherited environment options; +explicit CLI options take precedence where the vendor supports them. Compatible +explicit values are preserved. A conflicting flash-attention value, ambiguous +Windows spelling of a value-bearing option, or a per-model preset that could override flash attention stops Start +with an actionable retry error. Remove or correct the named override in the +per-user engine manifest or inherited environment before retrying. Disable +variables use presence semantics: any existing value, including empty or `0`, +already disables that feature and is preserved. Missing disables receive `1`. +The vendor's equivalent flash-attention-off values (`off`, `disabled`, `false`, +and `0`) are accepted without rewriting the user's option. +The profile ID `b10826-73a43d1f6-windows-vulkan-b580` and its effective options +appear in the existing manager and engine logs without inference content. + +This is a bounded compatibility workaround based on repeated correct requests, +not a uniquely isolated root cause, globally minimal option set, or broad +numerical guarantee. Options affect the whole serving process, including other +Vulkan adapters used together with B580. Maintainers must requalify or remove +the profile when changing vendor identity; `llamacompat.go` pins the qualified +executable so an unrelated future build cannot silently inherit the workaround. +Automatic mixed-device inference still requires native runtime validation. + +`llamacpp` installs the official llama app. Windows x64, Linux and Apple Silicon +use the checksum-pinned installer from ggml-org/llama-install.sh commit +`27a82f3a6e0f259f88c2c31cd6b20d858a975f27`. Pins refer to raw repository +bytes, before Windows checkout line-ending conversion. Their supported runtime +is `b10826`. Install reports an already-installed managed runtime as +`already-installed` and changes nothing; there is no update action that moves an +older managed runtime to a newer build, and uninstall-then-install is not run as +a substitute for one. + +NVIDIA Windows ARM64 Install first tries the current official +`ggml-org/llama-install.sh` PowerShell installer. `install.upstream_first` is +restricted to that platform and driver, with two pinned fallback archives. +The fixed official version endpoint resolves one numeric build, which is then +passed to the installer; the whole attempt, including validation, is limited to +three minutes. The version response is limited to 64 bytes and the script to +1 MiB. Builds older than the qualified fallback are refused. The installer gets +CUDA enabled and Vulkan skipped; the app still owns artifact/device selection. + +Before promotion, PAIR checks the exact selected build, nonempty vendor licenses, +and an actual `CUDA0:` (or other numbered CUDA device) row from +`llama cli --list-devices`. This check starts no server and loads no model. +If acquisition, the installer, or validation fails or times out, PAIR uses the +tested b10826 app ZIP plus CUDA 13.4 runtime ZIP declared in `install.archives`, +in a separate clean stage. Parent cancellation stops the operation without +starting a fallback. CUDA on this platform is an upstream preview and requires +a compatible NVIDIA driver; PAIR installs no driver or toolkit. + +Each fallback archive is verified before bounded extraction into its owned stage. +Unsafe paths, nonregular entries and file collisions are refused. Both bundles +move together through the existing validation, promotion, rollback and runtime-only +removal flow. Whichever source succeeds is what Install promotes; an installed +managed runtime is never re-fetched or replaced by a later Install request. + +`runtime/pair-install.json` records the actual source, selected build, executable +hash, CUDA validation, and any fallback reason/attempt metadata. The latest +installer is pinned to an upstream commit and **verified against a prequalified +SHA-256 before it is executed**, because it is a script PAIR runs rather than an +artifact it only unpacks; an upstream change fails the download and the pinned +CUDA archives take over. The build installed is still whatever the version +endpoint resolves to, so pinning the installer does not pin the engine. The +fallback retains its verified archive pins and recipe identity. Maintainers update those pins together +and repeat platform/failure/retention checks when changing the fallback. Acquired +runtime provenance and vendor license output remain with the managed installation; +normal release signing/notarization belongs to CI/CD, not a local bypass. +Security reports follow the repository's `SECURITY.md` process. + +Windows ARM policy is selected inside the install transaction from successful +native CPU and PNP inventory. NVIDIA CPU/hardware identity or a retained verified +CUDA receipt keeps the CUDA-required path above, including with an unbound or +broken driver. Failed/incomplete inventory never selects CPU. Confirmed +non-NVIDIA ARM uses `install.cpu_fetch`: the pinned b10826 official PowerShell +installer with CUDA/Vulkan probes explicitly skipped. Its receipt records +`source: official-pinned-cpu`, `acceleration_policy: cpu`, installer/binary hashes +and `cuda_device_verified: false`. + +Intel macOS uses the checksum-pinned official b10826 x64 CPU unified-app tar +archive. `install.archive_root` selects its fixed `llama-b10826` prefix. Bounded +extraction rejects escaping/duplicate paths, hardlinks and special files; +contained versioned dylib links become regular files, never filesystem symlinks. +The existing version/license, candidate promotion/rollback and persistent model +cache lifecycle apply. This artifact requires macOS 13.3 or newer and does not +provide Radeon acceleration. + +Ordinary Windows x64 and Linux recipes retain vendor accelerator-to-CPU selection; +Apple Silicon retains its Metal installer. Selection is not automatic recovery +from a GPU hang or incorrect model answer. `install_supported` and +`install_reason` describe recipe availability and selection requirements, not +proof of a GPU or a particular model. Native validation verifies the actual host. + +The per-user `engine-bin/llamacpp` directory contains `runtime` and `previous`. +Models live outside removable application data, in the sibling +`Nvidia Corporation/Personal AI Router Models/llamacpp` directory under the +platform configuration base: LocalAppData on Windows, XDG_CONFIG_HOME (or +`~/.config`) on Linux, and `~/Library/Application Support` on macOS. +An old `engine-bin/llamacpp/models` cache is atomically migrated before use. +Migration refuses existing-destination collisions, redirected/absolute/external +links, and a live configured listener; it never merges or overwrites caches. +Reset/uninstall preserve an unmigrated cache rather than deleting it. +Script installer subprocesses receive a fresh private home, +`SKIP_INSTALL=1`, and the selected vendor build, so user-global llama binaries and +PATH are untouched. Version and bundled license output are checked before +promotion; `runtime/pair-install.json` records installer and executable identity. +On Windows script attempts, the acquired official installer runs +with a process-scoped PowerShell execution-policy override; no saved execution +policy is changed. The Windows ARM64 fallback stage extracts archives without +executing another installer script. +Install stages and verifies the candidate before promoting it: promotion moves +any existing `runtime` slot to `previous` and the candidate into `runtime` with +two renames. If the manager exits between those renames, the next manager +restores the retained runtime when the current slot is absent. Saved Off +remains Off. Uninstall removes +the two runtime slots while retaining models and failed diagnostic stages. +Redirected managed directories are refused rather than mutating external data. +The first headless mutation detects the installed runtime and reconciles listener +ownership itself; it does not require a preceding status request or UI poll. + +The foreground `llama serve` process binds loopback, uses the same `LLAMA_CACHE` +and `HF_HUB_CACHE` as downloads, and runs with `--no-models-autoload`. Cached, +unloaded, loading, and loaded are separate states. Readiness checks the llama.cpp +server identity and router model-list shape. An externally started instance can +be inspected but cannot be mutated; `managed` is false for an adopted listener. +On Windows, both subprocess paths use the standard extended-length cache path +form to support long Hugging Face filenames without changing OS settings. + +Launch settings follow the shared editable-launch contract: the fixed startup arguments are `serve --no-models-autoload`, the reviewed networking controls are `--port` ({server.port}) and `--host` ({server.host}, loopback only), no CORS switch is declared, and the owned model cache environment (`LLAMA_CACHE`, `HF_HUB_CACHE`) is injected on every launch rather than edited; the settings preview rejects assignments to those two names. + +These actions use the existing `engine:action` request with `engine: "llamacpp"`: + +| Action | Parameters and result | +| --- | --- | +| `list_models` | Current router `/models` response; `data[].id`, nested `status.value` | +| `loaded_models` | Same response; residency extraction keeps only `status.value == loaded` | +| `list_downloaded` | Managed cache IDs as `data[].id`; works while stopped or after uninstall | +| `pull_model` | `{model: "owner/repository:TAG", file?: "file.gguf"}`; official CLI download | +| `import_model` | `{path: "/absolute/model-Q4_K_M.gguf"}`; copies a single GGUF into managed cache | +| `load_model`, `unload_model`, `delete_model` | `{model: "exact ID from inventory"}` | +| `cancel_pull` | `{model: "same requested model"}`; acknowledges the cancellation request | +| `get_version` | Vendor version string | + +There is no `update` action for `llamacpp`; a request for one is refused rather +than translated into uninstall followed by install. + +Import preserves the source file and requires a single primary GGUF with a +quantization suffix; split-file and auxiliary-only imports are refused. Pulls +require returned owned GGUF files with a supported header and model tensors; +preset/configuration-only results are explicitly unsupported even if the vendor +downloader exits successfully. This format check is not full tensor validation. +Downloads run as cancellable owned CLI processes. Their progress is indeterminate until +completion because vendor CLI output does not provide a reliable percentage. +Terminal completion/cancellation is distinct from a cancellation request. +Load/unload responses wait for observed vendor state instead of treating the +vendor's asynchronous acceptance response as completed work; failed loads surface +their exit status and waiting honors cancellation. +Downloads/imports/deletes refresh the router catalogue; deletion unloads an +observed loaded model first and removes only matching cache artifacts. Missing +residency observations publish unknown instead of retaining a current-looking +loaded set. + +Paired control adds `engine:remote-cancel-pull {node, engine, model}` through +the existing pinned-mTLS boundary at `POST /v1/models/cancel-pull`. Mixed-version +peers that lack that route return an explicit error. This layer does not claim +vendor acknowledgement of an inference cancellation or aggregate GPU memory. diff --git a/services/nvpair-engine-manager/actions.go b/services/nvpair-engine-manager/actions.go index 372ee104..280a912b 100644 --- a/services/nvpair-engine-manager/actions.go +++ b/services/nvpair-engine-manager/actions.go @@ -30,6 +30,9 @@ func (e *Executor) Action(ctx context.Context, engine, action string, params jso if !ok { return nil, fmt.Errorf("engine %q has no action %q", engine, action) } + if engine == "llamacpp" { + return e.actionLlama(ctx, st, action, act, params) + } res, err := e.dispatchAction(ctx, st, engine, action, act, params) if err != nil { return nil, err @@ -126,7 +129,7 @@ func (e *Executor) dispatchAction(ctx context.Context, st *engineState, engine, } req.Header.Set(engineIdentityProbeHeader, "1") client := e.client - if engine == "ollama" && action == "run_model" && e.ollamaLoadClient != nil { + if ((engine == "ollama" && action == "run_model") || (engine == "llamacpp" && action == "load_model")) && e.ollamaLoadClient != nil { client = e.ollamaLoadClient } resp, err := client.Do(req) @@ -234,7 +237,15 @@ func (e *Executor) runCmdAction(ctx context.Context, st *engineState, act Action vars["port"] = strconv.Itoa(port) vars["install_dir"] = st.installDir if cli := st.plat.Runtime.CLI; cli != "" { - vars["cli"] = expandPath(cli) + resolved, err := resolvePlaceholders(cli, vars) + if err != nil { + return nil, err + } + vars["cli"] = expandPath(resolved) + } + env, err := childEnv(st) + if err != nil { + return nil, err } // Most cmd actions run once with the params as given. An action that @@ -263,9 +274,9 @@ func (e *Executor) runCmdAction(ctx context.Context, st *engineState, act Action // failure is *not* retried in place — it falls through to the next // source below (and runWithResume returns it immediately). if lmsGet { - out, lastErr = e.runWithResume(ctx, argv) + out, lastErr = e.runWithResume(ctx, argv, env) } else { - out, lastErr = e.runCommandOutput(ctx, argv) + out, lastErr = e.runCommandOutput(ctx, argv, env) } if lastErr == nil { break @@ -290,12 +301,14 @@ func (e *Executor) runCmdAction(ctx context.Context, st *engineState, act Action // runCommandOutput runs argv and returns its stdout; on failure it // returns the error with stderr attached for diagnostics. -func (e *Executor) runCommandOutput(ctx context.Context, argv []string) (string, error) { +func (e *Executor) runCommandOutput(ctx context.Context, argv []string, env ...map[string]string) (string, error) { if len(argv) == 0 { return "", nil } cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + cmd.Env = commandEnv(env...) configureSysProcAttr(cmd) + configureCommandCancel(cmd) out, err := cmd.Output() if err != nil { if ee, ok := err.(*exec.ExitError); ok { @@ -323,11 +336,11 @@ var ( // model-pull path, where re-running resumes the partial download. A // resolution failure or any hard error returns immediately, so the caller's // source-fallback (or final error) is reached. -func (e *Executor) runWithResume(ctx context.Context, argv []string) (string, error) { +func (e *Executor) runWithResume(ctx context.Context, argv []string, env ...map[string]string) (string, error) { var out string var err error for attempt := 1; ; attempt++ { - out, err = e.runCommandOutput(ctx, argv) + out, err = e.runCommandOutput(ctx, argv, env...) if err == nil { return out, nil } diff --git a/services/nvpair-engine-manager/childenv.go b/services/nvpair-engine-manager/childenv.go new file mode 100644 index 00000000..b14d9db6 --- /dev/null +++ b/services/nvpair-engine-manager/childenv.go @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" +) + +// childEnv gives installer, control CLI and serving processes the same cache +// and runtime settings. Caller action parameters cannot change this environment. +func childEnv(st *engineState, overrides ...map[string]string) (map[string]string, error) { + st.mu.Lock() + vars := map[string]string{"install_dir": st.installDir, "port": strconv.Itoa(st.port), "bin": st.binPath, "host": st.plat.Runtime.Bind} + vars["model_dir"] = llamaModelDir(st) + st.mu.Unlock() + for _, values := range overrides { + for key, value := range values { + vars[key] = value + } + } + if vars["host"] == "" { + vars["host"] = "127.0.0.1" + } + env, err := resolveChildEnv(st.plat.Runtime.Env, vars) + if err != nil { + return nil, err + } + if st.manifest != nil && st.manifest.Engine == "llamacpp" { + if err := validateLlamaPath(vars["model_dir"]); err != nil { + return nil, err + } + cache, err := llamaCachePath(vars["model_dir"]) + if err != nil { + return nil, err + } + env["LLAMA_CACHE"], env["HF_HUB_CACHE"] = cache, cache + } + return env, nil +} + +// llamaOwnedEnvironmentKey reports whether key names environment the engine +// manager sets on every llama launch from the owned model directory. Launch +// settings may not replace these: a saved value would never reach the engine. +func llamaOwnedEnvironmentKey(key string) bool { + return environmentKey(key) == environmentKey("LLAMA_CACHE") || environmentKey(key) == environmentKey("HF_HUB_CACHE") +} + +func resolveChildEnv(spec, vars map[string]string) (map[string]string, error) { + env := make(map[string]string, len(spec)) + for k, v := range spec { + resolved, err := resolvePlaceholders(v, vars) + if err != nil { + return nil, err + } + if (k == "LLAMA_CACHE" || k == "HF_HUB_CACHE") && resolved != "" { + resolved, err = llamaCachePath(resolved) + if err != nil { + return nil, err + } + } + env[k] = resolved + } + return env, nil +} + +func llamaCachePath(path string) (string, error) { + if runtime.GOOS != "windows" { + return path, nil + } + abs, err := filepath.Abs(path) + if err != nil { + return "", err + } + abs = filepath.Clean(abs) + if strings.HasPrefix(abs, `\\?\`) { + return abs, nil + } + if strings.HasPrefix(abs, `\\`) { + return `\\?\UNC\` + strings.TrimPrefix(abs, `\\`), nil + } + return `\\?\` + abs, nil +} + +func plainWindowsPath(path string) string { + if runtime.GOOS != "windows" { + return path + } + if strings.HasPrefix(path, `\\?\UNC\`) { + return `\\` + strings.TrimPrefix(path, `\\?\UNC\`) + } + return strings.TrimPrefix(path, `\\?\`) +} + +func commandEnv(extra ...map[string]string) []string { + env := os.Environ() + for _, values := range extra { + for k, v := range values { + env = append(env, k+"="+v) + } + } + return env +} + +// Cancel every process spawned by a bounded CLI/install operation. The default +// CommandContext kill targets only the parent and can leave a downloader alive. +func configureCommandCancel(cmd *exec.Cmd) { + cmd.Cancel = func() error { + if cmd.Process == nil { + return os.ErrProcessDone + } + return signalPID(cmd.Process.Pid, true) + } +} diff --git a/services/nvpair-engine-manager/childenv_test.go b/services/nvpair-engine-manager/childenv_test.go new file mode 100644 index 00000000..d5ab0880 --- /dev/null +++ b/services/nvpair-engine-manager/childenv_test.go @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "fmt" + "os" + "strings" + "testing" +) + +func TestEngineChildEnvironment(t *testing.T) { + if os.Getenv("PAIR_ENV_CHILD") == "1" { + fmt.Print(os.Getenv("PAIR_CACHE_TEST")) + os.Exit(0) + } + st := &engineState{installDir: t.TempDir(), port: 8081, plat: &Platform{Runtime: Runtime{Env: map[string]string{"PAIR_ENV_CHILD": "1", "PAIR_CACHE_TEST": "{install_dir}/cache"}}}} + env, err := childEnv(st) + if err != nil { + t.Fatal(err) + } + argv := []string{os.Args[0], "-test.run=^TestEngineChildEnvironment$"} + e := &Executor{} + out, err := e.runCommandOutput(context.Background(), argv, env) + if err != nil || out != st.installDir+"/cache" { + t.Fatalf("action environment: %q, %v", out, err) + } + var lines []string + p, err := startManagedProc(argv[0], argv[1:], env, func(_, line string) { lines = append(lines, line) }) + if err != nil { + t.Fatal(err) + } + <-p.done + if strings.Join(lines, "") != out { + t.Fatalf("serving environment differs: %q", lines) + } + if err := e.runCommand(context.Background(), argv, env); err != nil { + t.Fatal(err) + } + st.plat.Runtime.Env["PAIR_CACHE_TEST"] = "{unresolved}" + if _, err := childEnv(st); err == nil { + t.Fatal("accepted unresolved environment") + } +} diff --git a/services/nvpair-engine-manager/controlmodels.go b/services/nvpair-engine-manager/controlmodels.go index 8bdce1ab..a1cf9eb7 100644 --- a/services/nvpair-engine-manager/controlmodels.go +++ b/services/nvpair-engine-manager/controlmodels.go @@ -14,9 +14,10 @@ import ( ) const ( - controlLoadPath = "/v1/models/load" - controlUnloadPath = "/v1/models/unload" - controlDeletePath = "/v1/models/delete" + controlLoadPath = "/v1/models/load" + controlUnloadPath = "/v1/models/unload" + controlDeletePath = "/v1/models/delete" + controlCancelPullPath = "/v1/models/cancel-pull" ) func (s *controlServer) handleLoad(w http.ResponseWriter, r *http.Request) { @@ -31,6 +32,10 @@ func (s *controlServer) handleDelete(w http.ResponseWriter, r *http.Request) { s.handleModelAction(w, r, "delete") } +func (s *controlServer) handleCancelPull(w http.ResponseWriter, r *http.Request) { + s.handleModelAction(w, r, "cancel-pull") +} + func (s *controlServer) handleModelAction(w http.ResponseWriter, r *http.Request, op string) { if r.Method != http.MethodPost { w.Header().Set("Allow", "POST") @@ -61,6 +66,9 @@ func (s *controlServer) handleModelAction(w http.ResponseWriter, r *http.Request res, err = s.exec.ModelUnload(r.Context(), req.Engine, req.Model) case "delete": res, err = s.exec.ModelDelete(r.Context(), req.Engine, req.Model) + case "cancel-pull": + params, _ := json.Marshal(map[string]string{"model": req.Model}) + res, err = s.exec.Action(r.Context(), req.Engine, "cancel_pull", params) default: http.Error(w, "unknown model op", http.StatusInternalServerError) return diff --git a/services/nvpair-engine-manager/controlserver.go b/services/nvpair-engine-manager/controlserver.go index 8a23a5f5..c2cd7cb3 100644 --- a/services/nvpair-engine-manager/controlserver.go +++ b/services/nvpair-engine-manager/controlserver.go @@ -66,6 +66,7 @@ func (s *controlServer) mux() *http.ServeMux { mux.HandleFunc(controlLoadPath, s.requirePin(s.handleLoad)) mux.HandleFunc(controlUnloadPath, s.requirePin(s.handleUnload)) mux.HandleFunc(controlDeletePath, s.requirePin(s.handleDelete)) + mux.HandleFunc(controlCancelPullPath, s.requirePin(s.handleCancelPull)) mux.HandleFunc(controlStartPath, s.requirePin(s.handleStart)) mux.HandleFunc(controlStopPath, s.requirePin(s.handleStop)) return mux diff --git a/services/nvpair-engine-manager/executor.go b/services/nvpair-engine-manager/executor.go index de6bcfdd..a97ce74c 100644 --- a/services/nvpair-engine-manager/executor.go +++ b/services/nvpair-engine-manager/executor.go @@ -29,12 +29,15 @@ const ( // EngineStatus is the snapshot returned by engine:status and // engine:get-installed. type EngineStatus struct { - Engine string `json:"engine"` - DisplayName string `json:"display_name"` - Installed bool `json:"installed"` - Running bool `json:"running"` - Healthy bool `json:"healthy"` - Port int `json:"port,omitempty"` + InstallSupported bool `json:"install_supported"` + InstallReason string `json:"install_reason,omitempty"` + Managed bool `json:"managed"` + Engine string `json:"engine"` + DisplayName string `json:"display_name"` + Installed bool `json:"installed"` + Running bool `json:"running"` + Healthy bool `json:"healthy"` + Port int `json:"port,omitempty"` } // engineState is the per-engine runtime state. @@ -43,6 +46,7 @@ type engineState struct { plat *Platform logs *logBuffer installDir string + modelDir string // opMu serializes lifecycle operations (install / start / stop / // restart / uninstall) for this engine, so concurrent calls can't @@ -63,7 +67,11 @@ type engineState struct { proc *managedProc healthStop context.CancelFunc // startCancel lets StopAll unblock doStart before waiting on opMu. - startCancel context.CancelFunc + startCancel context.CancelFunc + pullCancel context.CancelFunc + mutationCancel context.CancelFunc + stopPending int + pullModel string } // Executor owns engine lifecycle for every engine known on this host. @@ -77,12 +85,16 @@ type Executor struct { reporter *Reporter emit func(method string, params any) client *http.Client + armHardwareQuery func(context.Context, map[string]string) (string, error) // nil uses the native inventory command ollamaLoadClient *http.Client // progress fans install/pull progress to transient subscribers (the ec // streaming handlers) in addition to the local engine:install-progress // notification path. See progress.go. progress *progressHub baseDir string // user-scoped install base + // main injects appdir.ModelsDir; executor-only fixtures keep their cache + // beneath the explicitly supplied isolated install base. + modelBaseDir string // desired persists explicit per-engine ON/OFF intent. Runtime state remains // in-memory; shutdown cleanup must not rewrite this store. desired *desiredStateStore @@ -211,6 +223,18 @@ func (e *Executor) state(engine string) (*engineState, error) { port: plat.Runtime.Port, installDir: filepath.Join(e.baseDir, engine), } + st.modelDir = filepath.Join(st.installDir, "models") + if engine == "llamacpp" { + if e.modelBaseDir != "" { + st.modelDir = filepath.Join(e.modelBaseDir, "llamacpp") + if err := migrateLlamaCache(st); err != nil { + return nil, fmt.Errorf("preserve llama model cache: %w", err) + } + } + if err := recoverLlamaRuntime(st.installDir); err != nil { + return nil, fmt.Errorf("recover interrupted llama replacement: %w", err) + } + } e.engines[engine] = st return st, nil } diff --git a/services/nvpair-engine-manager/install.go b/services/nvpair-engine-manager/install.go index 268bd0b9..54e1a2d2 100644 --- a/services/nvpair-engine-manager/install.go +++ b/services/nvpair-engine-manager/install.go @@ -28,6 +28,19 @@ func (e *Executor) Install(ctx context.Context, engine string) error { } st.opMu.Lock() defer st.opMu.Unlock() + if engine == "llamacpp" { + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + st.mu.Lock() + if e.shuttingDown.Load() || st.stopPending > 0 { + st.mu.Unlock() + cancel() + return context.Canceled + } + st.mutationCancel = cancel + st.mu.Unlock() + defer func() { cancel(); st.mu.Lock(); st.mutationCancel = nil; st.mu.Unlock() }() + } if ok, _ := e.Detect(engine); ok { e.reporter.clear(installFailedID(engine)) e.emitInstallProgress(engine, "already-installed", 100) @@ -60,11 +73,21 @@ func (e *Executor) Install(ctx context.Context, engine string) error { e.reportInstallFailed(engine, err) return err } + if inst.Driver == "llama-app" { + if engine != "llamacpp" { + return fmt.Errorf("llama-app driver requires llamacpp engine") + } + return e.installLlamaApp(ctx, st) + } if err := os.MkdirAll(st.installDir, 0o755); err != nil { return fmt.Errorf("create install dir: %w", err) } vars := map[string]string{"install_dir": st.installDir} + env, err := childEnv(st) + if err != nil { + return err + } if len(inst.Script) > 0 { // Escape hatch: vendor-script install with no checksum. Logged @@ -79,7 +102,7 @@ func (e *Executor) Install(ctx context.Context, engine string) error { for i := range argv { argv[i] = expandPath(argv[i]) } - if err := e.runCommand(ctx, argv); err != nil { + if err := e.runCommand(ctx, argv, env); err != nil { werr := fmt.Errorf("script install failed: %w", err) e.reportInstallFailed(engine, werr) return werr @@ -106,7 +129,7 @@ func (e *Executor) Install(ctx context.Context, engine string) error { for i := range args { args[i] = expandPath(args[i]) } - if err := e.runCommand(ctx, args); err != nil { + if err := e.runCommand(ctx, args, env); err != nil { werr := fmt.Errorf("install command failed: %w", err) e.reportInstallFailed(engine, werr) return werr @@ -138,7 +161,7 @@ func (e *Executor) Uninstall(ctx context.Context, engine string) error { return e.setDesiredEnabled(engine, false) // already gone } un := st.plat.Uninstall - if un == nil || len(un.Run) == 0 { + if un == nil || (len(un.Run) == 0 && un.Driver == "") { return fmt.Errorf("engine %q has no uninstall defined for this platform", engine) } st.mu.Lock() @@ -169,6 +192,18 @@ func (e *Executor) Uninstall(ctx context.Context, engine string) error { e.reporter.report(serviceError{ID: uninstallFailedID(engine), Message: werr.Error(), Severity: "error", Action: "none", EngineType: engine, Operation: "uninstall"}) return werr } + if un.Driver == "llama-app" { + if engine != "llamacpp" { + return fmt.Errorf("llama-app driver requires llamacpp engine") + } + if err := removeLlamaRuntime(st); err != nil { + return err + } + e.Detect(engine) + e.reporter.clear(uninstallFailedID(engine)) + e.emitState(engine) + return e.setDesiredEnabled(engine, false) + } args, err := resolveArgs(un.Run, map[string]string{"install_dir": st.installDir}) if err != nil { @@ -178,8 +213,12 @@ func (e *Executor) Uninstall(ctx context.Context, engine string) error { args[i] = expandPath(args[i]) } var runErr error + env, err := childEnv(st) + if err != nil { + return err + } for attempt := 1; attempt <= uninstallRetries; attempt++ { - if runErr = e.runCommand(ctx, args); runErr == nil { + if runErr = e.runCommand(ctx, args, env); runErr == nil { break } if attempt < uninstallRetries { @@ -248,6 +287,10 @@ func validateDownloadURL(raw string) error { } func (e *Executor) download(ctx context.Context, engine string, f *Fetch) (string, error) { + return e.downloadLimited(ctx, engine, f, maxDownloadBytes) +} + +func (e *Executor) downloadLimited(ctx context.Context, engine string, f *Fetch, limit int64) (string, error) { if err := validateDownloadURL(f.URL); err != nil { return "", err } @@ -278,15 +321,22 @@ func (e *Executor) download(ctx context.Context, engine string, f *Fetch) (strin }} h := sha256.New() // Read one byte past the cap so we can detect (and reject) overflow. - n, err := io.Copy(io.MultiWriter(tmp, h), io.TeeReader(io.LimitReader(resp.Body, maxDownloadBytes+1), pw)) - tmp.Close() + n, err := io.Copy(io.MultiWriter(tmp, h), io.TeeReader(io.LimitReader(resp.Body, limit+1), pw)) + // A close error means the last buffered bytes never reached disk, so the + // file on disk is not what the digest above was computed over. Report it + // here rather than letting a truncated artifact fail later in extraction, + // where the cause is no longer visible. + closeErr := tmp.Close() + if err == nil { + err = closeErr + } if err != nil { os.Remove(tmp.Name()) return "", fmt.Errorf("download %s: %w", f.URL, err) } - if n > maxDownloadBytes { + if n > limit { os.Remove(tmp.Name()) - return "", fmt.Errorf("download %s exceeds the %d-byte limit", f.URL, int64(maxDownloadBytes)) + return "", fmt.Errorf("download %s exceeds the %d-byte limit", f.URL, limit) } sum := hex.EncodeToString(h.Sum(nil)) @@ -309,12 +359,14 @@ func (e *Executor) download(ctx context.Context, engine string, f *Fetch) (strin // runCommand executes a manifest-declared argv (an install or uninstall // step), hiding the console window on Windows; on failure it returns the // combined output for diagnostics. -func (e *Executor) runCommand(ctx context.Context, argv []string) error { +func (e *Executor) runCommand(ctx context.Context, argv []string, env ...map[string]string) error { if len(argv) == 0 { return nil } cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + cmd.Env = commandEnv(env...) configureSysProcAttr(cmd) // hide the console window on Windows + configureCommandCancel(cmd) out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out))) diff --git a/services/nvpair-engine-manager/launch_controls_test.go b/services/nvpair-engine-manager/launch_controls_test.go index f8718996..6c06bb55 100644 --- a/services/nvpair-engine-manager/launch_controls_test.go +++ b/services/nvpair-engine-manager/launch_controls_test.go @@ -109,7 +109,7 @@ func TestSavedControlsCannotBypassLaunchValidation(t *testing.T) { func TestBundledNetworkingControls(t *testing.T) { reg := loadWithOverrides(t, t.TempDir()) // Adding a bundled engine requires an explicit networking review and cases. - wantEngines := []string{"lmstudio", "ollama"} + wantEngines := []string{"llamacpp", "lmstudio", "ollama"} names := reg.Names() slices.Sort(names) if !slices.Equal(names, wantEngines) { @@ -126,13 +126,20 @@ func TestBundledNetworkingControls(t *testing.T) { t.Fatal("missing reviewed networking controls") } var valid, invalid []string - if name == "lmstudio" { + switch name { + case "llamacpp": + if !reflect.DeepEqual(policy.Controls, []LaunchControl{{Value: "{server.port}", Flags: []string{"--port"}}, {Value: "{server.host}", Flags: []string{"--host"}}}) { + t.Fatal("incomplete llama.cpp controls") + } + valid = []string{"--port 23456", "--port=23456", `"--port" "23456"`, "--host 127.0.0.1 --port 23456", "--port 23456 --host=127.0.0.1", "--port 23456 --ctx-size 4096"} + invalid = []string{"--port 0", "--port 65536", "--port", "--port no", "--port 23456 --port 23457", "--host 0.0.0.0", "--host=::", "--host 0.0.0.0 --port 23456"} + case "lmstudio": if !reflect.DeepEqual(policy.Controls, []LaunchControl{{Value: "{server.port}", Flags: []string{"--port", "-p"}}, {Value: "{server.host}", Flags: []string{"--bind"}, Env: []string{"LMS_SERVER_HOST"}}, {Value: "{cors.enabled}", Implicit: implicitLaunchValue("true"), Flags: []string{"--cors"}}}) { t.Fatal("incomplete LM Studio controls") } valid = []string{"--port 23456", "--port=23456", "-p 23456", "-p23456", "-p=23456", `"-p" "23456"`, "--port 23456 -p23456", "-- -p23456"} invalid = []string{"-p0", "-p65536", "-p", "-pno", "--port 23456 -p23457", "-vp23456", "-vp=23456", "--bind 0.0.0.0", "--bind=::", "LMS_SERVER_HOST=0.0.0.0", "--cors=false", "--cors=true", "--cors=", "-- --bind 0.0.0.0"} - } else { + default: if !reflect.DeepEqual(policy.Controls, []LaunchControl{{Value: "{server.host}:{server.port}", Env: []string{"OLLAMA_HOST"}}, {Value: "{cors.origins}", Env: []string{"OLLAMA_ORIGINS"}}}) { t.Fatal("incomplete Ollama controls") } diff --git a/services/nvpair-engine-manager/launch_test.go b/services/nvpair-engine-manager/launch_test.go index d2b8eb32..e62e126d 100644 --- a/services/nvpair-engine-manager/launch_test.go +++ b/services/nvpair-engine-manager/launch_test.go @@ -21,8 +21,8 @@ func (command launchCommand) text() (string, error) { func TestResolvedLaunchMatchesBundledEngines(t *testing.T) { reg := loadWithOverrides(t, t.TempDir()) - vars := map[string]string{"host": "127.0.0.1", "port": "12345", "cli": "/test path/lms", "install_dir": "/test path"} - for _, engine := range []string{"ollama", "lmstudio"} { + vars := map[string]string{"host": "127.0.0.1", "port": "12345", "cli": "/test path/lms", "install_dir": "/test path", "model_dir": "/test path/models"} + for _, engine := range []string{"ollama", "lmstudio", "llamacpp"} { manifest, ok := reg.Get(engine) if !ok { t.Fatalf("missing bundled engine %q", engine) @@ -34,13 +34,17 @@ func TestResolvedLaunchMatchesBundledEngines(t *testing.T) { var launch launchCommand var err error var want []string - if engine == "ollama" { + switch engine { + case "llamacpp": + launch, err = resolveProcessLaunch(platform.Runtime, "/test path/llama", vars) + want = []string{"/test path/llama", "serve", "--no-models-autoload", "--host", "127.0.0.1", "--port", "12345"} + case "ollama": launch, err = resolveProcessLaunch(platform.Runtime, "/test path/ollama", vars) want = []string{"OLLAMA_HOST=127.0.0.1:12345", "/test path/ollama", "serve"} if strings.HasPrefix(platformKey, "linux/") { want = append([]string{"LD_LIBRARY_PATH=/test path/lib/ollama"}, want...) } - } else { + default: launch, err = resolveCommandLaunch(platform.Runtime.Start[0], vars) want = []string{"/test path/lms", "server", "start", "--port", "12345", "--bind", "127.0.0.1"} } diff --git a/services/nvpair-engine-manager/lifecycle.go b/services/nvpair-engine-manager/lifecycle.go index cea7aa16..ec23a1ca 100644 --- a/services/nvpair-engine-manager/lifecycle.go +++ b/services/nvpair-engine-manager/lifecycle.go @@ -5,8 +5,10 @@ package main import ( "context" + "encoding/json" "errors" "fmt" + "io" "log/slog" "net" "net/http" @@ -174,6 +176,7 @@ func (e *Executor) doStart(ctx context.Context, st *engineState, engine string, "host": effectiveBind(rt.Bind, opts.Bind), "port": strconv.Itoa(port), "install_dir": st.installDir, + "model_dir": llamaModelDir(st), } if rt.CLI != "" { vars["cli"] = expandPath(rt.CLI) @@ -237,10 +240,19 @@ func (e *Executor) bringUpProcess(ctx context.Context, st *engineState, engine s if err := validateEffectiveLaunch(rt, launch, vars["host"], vars["port"]); err != nil { return err } + shared, err := childEnv(st, vars) + if err != nil { + return err + } + env := layerLaunchEnvironment(shared, launch.Env) + if err := e.prepareLlamaCompatibility(ctx, st, launch.Bin, launch.Args, env); err != nil { + e.reportStartFailedUnlessShuttingDown(ctx, engine, err) + return err + } - diagnostics := newStartupOutput(launchDiagnosticArgs(rt), launch.Env) + diagnostics := newStartupOutput(launchDiagnosticArgs(rt), env) defer diagnostics.close() - proc, err := startManagedProc(launch.Bin, launch.Args, launch.Env, func(stream, line string) { + proc, err := startManagedProc(launch.Bin, launch.Args, env, func(stream, line string) { _, _ = diagnostics.Write([]byte(line + "\n")) // Vendor diagnostics may echo arbitrary custom arguments. Keep those // out of exported engine logs; report exit/readiness failures separately. @@ -293,6 +305,10 @@ func (e *Executor) bringUpProcess(ctx context.Context, st *engineState, engine s // daemon-style engine such as LM Studio), then waits for readiness. // There is no owned process — liveness comes from the probe. func (e *Executor) bringUpCommand(ctx context.Context, st *engineState, engine string, rt Runtime, port int, vars map[string]string) error { + shared, err := childEnv(st, vars) + if err != nil { + return err + } st.mu.Lock() st.proc = nil st.mu.Unlock() @@ -326,10 +342,13 @@ func (e *Executor) bringUpCommand(ctx context.Context, st *engineState, engine s } } argv := append([]string{launch.Bin}, launch.Args...) - run := e.runCommand + env := layerLaunchEnvironment(shared, launch.Env) + run := func(ctx context.Context, argv []string) error { + return e.runCommand(ctx, argv, env) + } if rt.LaunchArgs != nil || rt.LaunchEnv != nil || len(launch.Env) > 0 { run = func(ctx context.Context, argv []string) error { - return runPrivateLaunchCommand(ctx, argv, launch.Env, launchDiagnosticArgs(rt)) + return runPrivateLaunchCommand(ctx, argv, env, launchDiagnosticArgs(rt)) } // A vendor command may spawn its daemon and then exit unsuccessfully. // Attempt its official cleanup even if the first command failed. @@ -354,6 +373,30 @@ func (e *Executor) bringUpCommand(ctx context.Context, st *engineState, engine s return nil } +// layerLaunchEnvironment builds the environment an engine start receives: the +// shared child environment (manifest defaults plus the owned llama model cache), +// then the resolved launch environment so saved launch settings and managed +// controls override manifest defaults. The llama cache location is not a launch +// setting; the validated, normalized path childEnv resolved stays authoritative. +func layerLaunchEnvironment(shared, launch map[string]string) map[string]string { + env := make(map[string]string, len(shared)+len(launch)) + for key, value := range shared { + env[key] = value + } + for key, value := range launch { + if _, owned := shared[key]; owned && llamaOwnedEnvironmentKey(key) { + continue + } + for existing := range env { + if environmentKey(existing) == environmentKey(key) { + delete(env, existing) + } + } + env[key] = value + } + return env +} + // Capture vendor startup diagnostics without interpreting engine options. func runPrivateLaunchCommand(ctx context.Context, argv []string, environment map[string]string, privateArgs []string) error { if len(argv) == 0 { @@ -411,6 +454,19 @@ func (e *Executor) Stop(engine string) error { if err != nil { return err } + if engine == "llamacpp" { + st.mu.Lock() + st.stopPending++ + cancelMutation, cancelStart := st.mutationCancel, st.startCancel + st.mu.Unlock() + defer func() { st.mu.Lock(); st.stopPending--; st.mu.Unlock() }() + if cancelMutation != nil { + cancelMutation() + } + if cancelStart != nil { + cancelStart() + } + } st.opMu.Lock() defer st.opMu.Unlock() stopErr := e.doStop(st, engine) @@ -526,7 +582,12 @@ func (e *Executor) runCommandStop(st *engineState, engine string, rt Runtime, po // force-kills engine-manager on a timeout, so an unbounded stop command // would wedge StopAll and, in turn, the whole app shutdown. stopCtx, cancelStop := context.WithTimeout(context.Background(), commandStopTimeout(rt)) - runErr := e.runCommand(stopCtx, argv) + env, envErr := childEnv(st) + if envErr != nil { + cancelStop() + return envErr + } + runErr := e.runCommand(stopCtx, argv, env) cancelStop() if runErr != nil { return fmt.Errorf("stop %s: %w", engine, runErr) @@ -773,10 +834,18 @@ func (e *Executor) StopAll() { defer wg.Done() st.mu.Lock() cancel := st.startCancel + cancelMutation := st.mutationCancel + cancelPull := st.pullCancel st.mu.Unlock() if cancel != nil { cancel() } + if cancelMutation != nil { + cancelMutation() + } + if cancelPull != nil { + cancelPull() + } st.opMu.Lock() defer st.opMu.Unlock() if err := e.doStop(st, n); err != nil { @@ -871,12 +940,28 @@ func (e *Executor) probe(ctx context.Context, p *Probe, port int) bool { if err != nil { return false } - resp.Body.Close() + defer resp.Body.Close() want := p.Status if want == 0 { want = 200 } - return resp.StatusCode == want + if resp.StatusCode != want { + return false + } + if p.Identity == "llamacpp" { + if resp.Header.Get("Server") != "llama.cpp" { + return false + } + var payload struct { + Data json.RawMessage `json:"data"` + } + if json.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(&payload) != nil { + return false + } + var rows []json.RawMessage + return len(payload.Data) > 0 && string(payload.Data) != "null" && json.Unmarshal(payload.Data, &rows) == nil + } + return true } if p.TCP != "" { addr, err := resolvePlaceholders(p.TCP, vars) diff --git a/services/nvpair-engine-manager/llamaactions.go b/services/nvpair-engine-manager/llamaactions.go new file mode 100644 index 00000000..f7fc976f --- /dev/null +++ b/services/nvpair-engine-manager/llamaactions.go @@ -0,0 +1,186 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" +) + +func (e *Executor) actionLlama(ctx context.Context, st *engineState, action string, act Action, params json.RawMessage) (json.RawMessage, error) { + if action == "cancel_pull" { + model := modelFromParams(params) + st.mu.Lock() + defer st.mu.Unlock() + if st.pullCancel == nil || (model != "" && model != st.pullModel) { + return nil, fmt.Errorf("no matching llama model download is active") + } + st.pullCancel() + return json.RawMessage(`{"cancel_requested":true}`), nil + } + readOnly := action == "list_models" || action == "loaded_models" || action == "list_downloaded" || action == "get_version" + if !readOnly { + st.opMu.Lock() + defer st.opMu.Unlock() + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + st.mu.Lock() + if e.shuttingDown.Load() || st.stopPending > 0 { + st.mu.Unlock() + cancel() + return nil, context.Canceled + } + st.mutationCancel = cancel + st.mu.Unlock() + defer func() { cancel(); st.mu.Lock(); st.mutationCancel = nil; st.mu.Unlock() }() + if err := validateLlamaOwnedPaths(st.installDir); err != nil { + return nil, err + } + } + st.mu.Lock() + running, bin, port, adopted := st.running, st.binPath, st.port, st.adopted + st.mu.Unlock() + if !readOnly && adopted { + return nil, errors.New("this llama listener was started externally; stop it in its own application before managing it with PAIR") + } + if !readOnly && bin == "" { + if _, err := e.Detect("llamacpp"); err != nil { + return nil, err + } + st.mu.Lock() + bin = st.binPath + st.mu.Unlock() + } + if !readOnly && (!isManagedInstallPath(bin, st.installDir) || !fileExists(bin)) { + return nil, errors.New("this llama instance is externally owned; install a managed instance to change it") + } + if !readOnly { + presence := e.reconcilePresence(ctx, "llamacpp", st, true, port, false) + st.mu.Lock() + running, adopted = st.running, st.adopted + st.mu.Unlock() + if adopted || (presence.Occupied && (!presence.Identified || !running)) { + return nil, errors.New("the llama port has an external or unidentified listener; stop it in its own application before managing it with PAIR") + } + } + if !readOnly && running && !e.probe(ctx, st.plat.Runtime.Ready, port) { + return nil, errors.New("llama identity/readiness could not be confirmed") + } + if act.Builtin != "llama-models" { + result, err := e.dispatchAction(ctx, st, "llamacpp", action, act, params) + if err == nil && (action == "load_model" || action == "unload_model") { + want := "loaded" + if action == "unload_model" { + want = "unloaded" + } + err = e.waitLlamaModelState(ctx, st, modelFromParams(params), want) + if err == nil { + e.pokeLoaded() + } + } + return result, err + } + if action == "pull_model" { + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + st.mu.Lock() + st.pullCancel, st.pullModel = cancel, modelFromParams(params) + st.mu.Unlock() + defer func() { cancel(); st.mu.Lock(); st.pullCancel, st.pullModel = nil, ""; st.mu.Unlock() }() + } + if action == "delete_model" && running { + // Unload only an observed resident model; deleting a cold model needs no + // runtime mutation and cannot turn another cached model off. + raw, err := e.dispatchAction(ctx, st, "llamacpp", "loaded_models", st.manifest.Actions["loaded_models"], nil) + if err != nil { + return nil, err + } + loaded, ok := extractStringsResult(raw, st.manifest.Actions["loaded_models"].Result) + if !ok { + return nil, errors.New("llama returned an invalid loaded-model inventory") + } + for _, id := range loaded { + if id == modelFromParams(params) { + if _, err := e.dispatchAction(ctx, st, "llamacpp", "unload_model", st.manifest.Actions["unload_model"], params); err != nil { + return nil, err + } + if err := e.waitLlamaModelState(ctx, st, id, "unloaded"); err != nil { + return nil, err + } + } + } + } + result, err := e.llamaModelAction(ctx, st, action, params) + if err != nil { + return nil, err + } + if !readOnly && running { + refresh := Action{HTTP: &ActionHTTP{Method: "GET", Path: "/models?reload=1"}} + if _, err := e.dispatchAction(ctx, st, "llamacpp", "refresh_models", refresh, nil); err != nil { + return nil, fmt.Errorf("model files changed but llama catalogue refresh failed: %w", err) + } + } + if !readOnly { + e.pokeLoaded() + } + return result, nil +} + +func (e *Executor) waitLlamaModelState(ctx context.Context, st *engineState, model, want string) error { + ctx, cancel := context.WithTimeout(ctx, e.actionTimeout) + defer cancel() + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + if err := ctx.Err(); err != nil { + return err + } + raw, err := e.dispatchAction(ctx, st, "llamacpp", "list_models", st.manifest.Actions["list_models"], nil) + if err != nil { + return err + } + var response struct { + Data []struct { + ID string `json:"id"` + Status struct { + Value string `json:"value"` + Failed bool `json:"failed"` + ExitCode int `json:"exit_code"` + } `json:"status"` + } `json:"data"` + } + if json.Unmarshal(raw, &response) != nil || response.Data == nil { + return errors.New("llama returned an invalid model-state response") + } + found := false + for _, row := range response.Data { + if row.ID == model { + found = true + if row.Status.Value == want { + return nil + } + if row.Status.Failed { + return fmt.Errorf("llama model failed to load (exit code %d)", row.Status.ExitCode) + } + if row.Status.Value == "" { + return errors.New("llama model status is missing") + } + } + } + if !found { + if want == "unloaded" { + return nil + } + return errors.New("requested model is absent from the llama catalogue") + } + select { + case <-ctx.Done(): + return fmt.Errorf("waiting for llama model %s: %w", want, ctx.Err()) + case <-ticker.C: + } + } +} diff --git a/services/nvpair-engine-manager/llamaarchives.go b/services/nvpair-engine-manager/llamaarchives.go new file mode 100644 index 00000000..18ceef00 --- /dev/null +++ b/services/nvpair-engine-manager/llamaarchives.go @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "archive/zip" + "context" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" +) + +const maxLlamaArchiveEntries = 10000 + +// Merge the pinned official app and dependency bundles into a fresh owned stage. +// The actual ARM64 bundles are flat; relative directories are preserved, never flattened. +func (e *Executor) stageLlamaArchives(ctx context.Context, st *engineState, candidate string) error { + if err := validateLlamaOwnedPaths(st.installDir); err != nil { + return err + } + if !isManagedInstallPath(candidate, st.installDir) { + return fmt.Errorf("llama archive stage is outside the managed install") + } + if err := validateLlamaPath(candidate); err != nil { + return err + } + if err := os.MkdirAll(candidate, 0700); err != nil { + return err + } + remaining, entries := maxDownloadBytes, 0 + for _, fetch := range st.plat.Install.Archives { + if err := ctx.Err(); err != nil { + return err + } + if strings.TrimSpace(fetch.SHA256) == "" { + return fmt.Errorf("llama archive requires a pinned checksum") + } + archive, err := e.download(ctx, "llamacpp", &fetch) + if err != nil { + return err + } + if st.plat.Install.ArchiveRoot != "" { + err = extractLlamaTar(ctx, archive, candidate, st.plat.Install.ArchiveRoot, &remaining, &entries) + } else { + err = extractLlamaArchive(ctx, archive, candidate, &remaining, &entries) + } + _ = os.Remove(archive) // Failed extracted stages remain available for diagnosis. + if err != nil { + return err + } + } + return ctx.Err() +} + +func extractLlamaArchive(ctx context.Context, archive, candidate string, remaining *int64, entries *int) error { + z, err := zip.OpenReader(archive) + if err != nil { + return err + } + defer z.Close() + if len(z.File) > maxLlamaArchiveEntries-*entries { + return fmt.Errorf("llama archives exceed the entry limit") + } + *entries += len(z.File) + for _, entry := range z.File { + if err := ctx.Err(); err != nil { + return err + } + name := strings.TrimSuffix(entry.Name, "/") + if name == "" || path.IsAbs(name) || path.Clean(name) != name || strings.ContainsAny(name, "\\:\x00") { + return fmt.Errorf("llama archive has an unsafe entry name") + } + for _, part := range strings.Split(name, "/") { + if part == "." || part == ".." || strings.TrimRight(part, ". ") != part || !filepath.IsLocal(part) { + return fmt.Errorf("llama archive has an unsafe path component") + } + } + mode := entry.Mode() + if !mode.IsRegular() && !mode.IsDir() { + return fmt.Errorf("llama archive entry is not a regular file or directory") + } + dest := filepath.Join(candidate, filepath.FromSlash(name)) + rel, err := filepath.Rel(candidate, dest) + if err != nil || !filepath.IsLocal(rel) { + return fmt.Errorf("llama archive entry escapes its stage") + } + if err := validateLlamaPath(dest); err != nil { + return err + } + if mode.IsDir() { + if err := os.MkdirAll(dest, 0700); err != nil { + return err + } + continue + } + if *remaining < 0 || entry.UncompressedSize64 > uint64(*remaining) { + return fmt.Errorf("llama archives exceed the uncompressed byte limit") + } + if err := os.MkdirAll(filepath.Dir(dest), 0700); err != nil { + return err + } + src, err := entry.Open() + if err != nil { + return err + } + dst, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0700) + if err != nil { + src.Close() + return err + } + n, copyErr := io.Copy(llamaArchiveWriter{ctx, dst}, io.LimitReader(src, *remaining+1)) + closeErr := dst.Close() + src.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + if n > *remaining { + return fmt.Errorf("llama archives exceed the uncompressed byte limit") + } + *remaining -= n + } + return ctx.Err() +} + +type llamaArchiveWriter struct { + ctx context.Context + io.Writer +} + +func (w llamaArchiveWriter) Write(p []byte) (int, error) { + if err := w.ctx.Err(); err != nil { + return 0, err + } + return w.Writer.Write(p) +} diff --git a/services/nvpair-engine-manager/llamaarchives_test.go b/services/nvpair-engine-manager/llamaarchives_test.go new file mode 100644 index 00000000..bf1b5eb6 --- /dev/null +++ b/services/nvpair-engine-manager/llamaarchives_test.go @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "net/http" + "os" + "path/filepath" + "testing" +) + +func llamaArchiveFixture(t *testing.T, name, body string, mode os.FileMode) []byte { + t.Helper() + var buf bytes.Buffer + z := zip.NewWriter(&buf) + h := &zip.FileHeader{Name: name, Method: zip.Deflate} + h.SetMode(mode) + w, err := z.CreateHeader(h) + if err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(w, body); err != nil { + t.Fatal(err) + } + if err := z.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestLlamaArchivesMergeAndRefuseCollision(t *testing.T) { + for _, collision := range []bool{false, true} { + t.Run(map[bool]string{false: "merge", true: "collision"}[collision], func(t *testing.T) { + root := t.TempDir() + candidate := filepath.Join(root, "stage") + second := "deps/runtime.dll" + if collision { + second = "llama.exe" + } + bundles := [][]byte{llamaArchiveFixture(t, "llama.exe", "app", 0600), llamaArchiveFixture(t, second, "dependency", 0600)} + st := &engineState{installDir: root, plat: &Platform{Install: &Install{}}} + for _, bundle := range bundles { + digest := sha256.Sum256(bundle) + st.plat.Install.Archives = append(st.plat.Install.Archives, Fetch{URL: "https://example.invalid/pinned.zip", SHA256: hex.EncodeToString(digest[:])}) + } + e := NewExecutor(buildRegistry(""), NewReporter(nil), nil, root) + index := 0 + e.client = &http.Client{Transport: llamaFixtureTransport(func(*http.Request) (*http.Response, error) { + data := bundles[index] + index++ + return &http.Response{StatusCode: 200, ContentLength: int64(len(data)), Body: io.NopCloser(bytes.NewReader(data))}, nil + })} + err := e.stageLlamaArchives(context.Background(), st, candidate) + if (err != nil) != collision { + t.Fatalf("collision=%v: %v", collision, err) + } + body, err := os.ReadFile(filepath.Join(candidate, "llama.exe")) + if err != nil || string(body) != "app" { + t.Fatalf("first bundle overwritten: %q %v", body, err) + } + if !collision { + body, err = os.ReadFile(filepath.Join(candidate, "deps", "runtime.dll")) + if err != nil || string(body) != "dependency" { + t.Fatalf("relative directory lost: %q %v", body, err) + } + } + }) + } +} + +func TestLlamaArchivesRejectUnsafeEntriesAndBounds(t *testing.T) { + for _, tc := range []struct { + name string + mode os.FileMode + budget int64 + entries int + }{ + {"../outside", 0600, 100, 0}, {"/absolute", 0600, 100, 0}, {`dir\file`, 0600, 100, 0}, + {"file:stream", 0600, 100, 0}, {"dir/../file", 0600, 100, 0}, {"file.", 0600, 100, 0}, + {"link", os.ModeSymlink | 0600, 100, 0}, {"pipe", os.ModeNamedPipe | 0600, 100, 0}, + {"large", 0600, 2, 0}, {"entry", 0600, 100, maxLlamaArchiveEntries}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + archive := filepath.Join(root, "fixture.zip") + if err := os.WriteFile(archive, llamaArchiveFixture(t, tc.name, "data", tc.mode), 0600); err != nil { + t.Fatal(err) + } + if err := extractLlamaArchive(context.Background(), archive, filepath.Join(root, "stage"), &tc.budget, &tc.entries); err == nil { + t.Fatal("entry/bound accepted") + } + }) + } +} + +func TestLlamaArchiveCopyHonorsCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + var output bytes.Buffer + w := llamaArchiveWriter{ctx, &output} + if _, err := w.Write([]byte("first")); err != nil { + t.Fatal(err) + } + cancel() + if _, err := w.Write([]byte("later")); !errors.Is(err, context.Canceled) { + t.Fatalf("copy after cancellation: %v", err) + } + if output.String() != "first" { + t.Fatal("wrote bytes after cancellation") + } +} diff --git a/services/nvpair-engine-manager/llamaarmcpu.go b/services/nvpair-engine-manager/llamaarmcpu.go new file mode 100644 index 00000000..e8a02f08 --- /dev/null +++ b/services/nvpair-engine-manager/llamaarmcpu.go @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// Static PNP identities survive an unbound/broken GPU driver. Do not classify +// hardware by a failed CUDA runtime probe or by the absence of working devices. +const llamaARMInventoryCommand = `$ErrorActionPreference='Stop'; $cpus=@(Get-CimInstance Win32_Processor -ErrorAction Stop | Select-Object Architecture,Manufacturer); $devices=@(Get-CimInstance Win32_PnPEntity -ErrorAction Stop); $nv=@($devices | Where-Object { $_.PNPDeviceID -match 'VEN_10DE' -or ($_.HardwareID -join ' ') -match 'VEN_10DE' -or $_.Manufacturer -match 'NVIDIA' }); @{cpus=$cpus;device_count=$devices.Count;nvidia_hardware=($nv.Count -gt 0)} | ConvertTo-Json -Depth 4 -Compress` + +type llamaARMInventory struct { + CPUs []struct { + Architecture *int `json:"Architecture"` + Manufacturer string `json:"Manufacturer"` + } `json:"cpus"` + DeviceCount int `json:"device_count"` + NVIDIAHardware *bool `json:"nvidia_hardware"` +} + +func llamaARMRequiresCUDA(data []byte) (bool, error) { + var inventory llamaARMInventory + if err := json.Unmarshal(data, &inventory); err != nil { + return false, fmt.Errorf("read Windows ARM hardware policy: %w", err) + } + if len(inventory.CPUs) == 0 || inventory.DeviceCount <= 0 || inventory.NVIDIAHardware == nil { + return false, errors.New("Windows ARM hardware inventory incomplete; refusing to infer CPU-only support") + } + cuda := *inventory.NVIDIAHardware + for _, cpu := range inventory.CPUs { + if cpu.Architecture == nil || *cpu.Architecture != 12 || strings.TrimSpace(cpu.Manufacturer) == "" { + return false, errors.New("Windows ARM CPU identity unavailable; refusing to infer CPU-only support") + } + cuda = cuda || strings.Contains(strings.ToLower(cpu.Manufacturer), "nvidia") + } + return cuda, nil +} + +func (e *Executor) prepareLlamaWindowsARM(ctx context.Context, st *engineState, stage string) (string, map[string]any, error) { + if err := ctx.Err(); err != nil { + return "", nil, err + } + // Legacy test/custom recipes without CPUFetch stay CUDA-required. + if st.plat.Install.CPUFetch == nil { + return e.prepareLlamaUpstream(ctx, st, stage) + } + queryCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + env, err := childEnv(st) + if err != nil { + return "", nil, err + } + query := e.armHardwareQuery + if query == nil { + query = func(ctx context.Context, env map[string]string) (string, error) { + return e.runCommandOutput(ctx, []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", llamaARMInventoryCommand}, env) + } + } + raw, err := query(queryCtx, env) + if err != nil { + if ctx.Err() != nil { + return "", nil, ctx.Err() + } + return "", nil, fmt.Errorf("Windows ARM hardware query failed; CPU fallback was not selected: %w", err) + } + cuda, err := llamaARMRequiresCUDA([]byte(raw)) + if err != nil { + return "", nil, err + } + if cuda { + return e.prepareLlamaUpstream(ctx, st, stage) + } + return e.prepareLlamaARMCPU(ctx, st, stage) +} + +func (e *Executor) prepareLlamaARMCPU(ctx context.Context, st *engineState, stage string) (string, map[string]any, error) { + fetch := st.plat.Install.CPUFetch + if fetch == nil || fetch.SHA256 == "" { + return "", nil, errors.New("Windows ARM CPU installation requires its pinned official installer") + } + home := filepath.Join(stage, "cpu") + if err := os.MkdirAll(home, 0700); err != nil { + return "", nil, err + } + env, err := llamaInstallerEnv(st, home) + if err != nil { + return "", nil, err + } + env["SKIP_CUDA"], env["SKIP_VULKAN"] = "1", "1" + script, err := e.downloadLimited(ctx, "llamacpp", fetch, maxLlamaInstallerBytes) + if err != nil { + return "", nil, err + } + defer os.Remove(script) + e.emitInstallProgress("llamacpp", "installing", -1) + if err := e.runCommand(ctx, llamaInstallerArgs("windows", script), env); err != nil { + return "", nil, fmt.Errorf("official Windows ARM CPU installer: %w", err) + } + candidate := filepath.Join(home, "llama-app") + identity, licenses, err := e.validateLlamaARMApp(ctx, st, candidate, 10826, false) + if err != nil { + return "", nil, err + } + identity["source"], identity["acceleration_policy"] = "official-pinned-cpu", "cpu" + identity["installer_url"], identity["installer_sha256"] = fetch.URL, fetch.SHA256 + if err := os.WriteFile(filepath.Join(candidate, "THIRD-PARTY-LICENSES.txt"), []byte(licenses), 0600); err != nil { + return "", nil, err + } + return candidate, identity, ctx.Err() +} diff --git a/services/nvpair-engine-manager/llamaarmcpu_test.go b/services/nvpair-engine-manager/llamaarmcpu_test.go new file mode 100644 index 00000000..d8fe9f6d --- /dev/null +++ b/services/nvpair-engine-manager/llamaarmcpu_test.go @@ -0,0 +1,131 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "net/http" + "os" + "path/filepath" + "testing" +) + +const nonNvidiaARM = `{"cpus":[{"Architecture":12,"Manufacturer":"Qualcomm"}],"device_count":20,"nvidia_hardware":false}` + +func TestLlamaARMHardwarePolicy(t *testing.T) { + for _, tc := range []struct { + name, data string + cuda, fail bool + }{ + {"generic", nonNvidiaARM, false, false}, + {"unbound-nvidia-hardware", `{"cpus":[{"Architecture":12,"Manufacturer":"Qualcomm"}],"device_count":20,"nvidia_hardware":true}`, true, false}, + {"nvidia-cpu-driver-absent", `{"cpus":[{"Architecture":12,"Manufacturer":"NVIDIA"}],"device_count":20,"nvidia_hardware":false}`, true, false}, + {"missing", "{}", false, true}, {"invalid", "not-json", false, true}, + {"wrong-arch", `{"cpus":[{"Architecture":9,"Manufacturer":"Qualcomm"}],"device_count":20,"nvidia_hardware":false}`, false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + cuda, err := llamaARMRequiresCUDA([]byte(tc.data)) + if (err != nil) != tc.fail || cuda != tc.cuda { + t.Fatalf("cuda=%v err=%v", cuda, err) + } + }) + } +} + +func cpuARMFixture(t *testing.T) *llamaUpstreamFixture { + f := newLlamaUpstreamFixture(t, llamaRuntimeFixture{Version: "version: 0.4.0-dev (build 10826, commit fixture)", Licenses: "CPU fixture license"}) + script := []byte(`$ErrorActionPreference='Stop' +if ($env:LLAMA_VERSION -ne 'b10826' -or $env:SKIP_CUDA -ne '1' -or $env:SKIP_VULKAN -ne '1' -or $env:SKIP_INSTALL -ne '1') {throw 'CPU policy mismatch'} +$stagePath=Join-Path $env:USERPROFILE 'llama-app' +New-Item -ItemType Directory -Path $stagePath -Force | Out-Null +Copy-Item -LiteralPath $env:FAKE_LLAMA_BIN_SOURCE -Destination (Join-Path $stagePath 'llama.exe') +Copy-Item -LiteralPath $env:FAKE_LLAMA_FIXTURE_SOURCE -Destination (Join-Path $stagePath '.llama-fixture.json') +`) + digest := sha256.Sum256(script) + fetch := &Fetch{URL: "https://example.invalid/pinned-cpu.ps1", SHA256: hex.EncodeToString(digest[:])} + f.st.plat.Install.CPUFetch = fetch + f.e.armHardwareQuery = func(context.Context, map[string]string) (string, error) { return nonNvidiaARM, nil } + f.e.client = &http.Client{Transport: llamaFixtureTransport(func(r *http.Request) (*http.Response, error) { + f.requests[r.URL.String()]++ + if r.URL.String() != fetch.URL { + return nil, errors.New("CPU path attempted an upstream/CUDA request") + } + return &http.Response{StatusCode: 200, Body: io.NopCloser(bytes.NewReader(script)), ContentLength: int64(len(script))}, nil + })} + return f +} + +// Native inventory that positively identifies a non-NVIDIA ARM host selects the +// pinned CPU installer and nothing else; the resulting runtime is detected, and +// its later removal retains models. +func TestLlamaARMCPUInstallAndRetention(t *testing.T) { + f := cpuARMFixture(t) + if err := f.e.installLlamaApp(context.Background(), f.st); err != nil { + t.Fatal(err) + } + receipt := f.receipt(t) + if receipt["acceleration_policy"] != "cpu" || receipt["cuda_device_verified"] != false || receipt["installer_sha256"] != f.st.plat.Install.CPUFetch.SHA256 { + t.Fatalf("wrong CPU provenance: %#v", receipt) + } + if len(f.requests) != 1 || f.requests[f.st.plat.Install.CPUFetch.URL] != 1 { + t.Fatalf("CPU install did not fetch exactly its pinned installer: %v", f.requests) + } + if installed, err := f.e.Detect("llamacpp"); err != nil || !installed { + t.Fatalf("CPU install not detected: %v %v", installed, err) + } + models := filepath.Join(f.st.installDir, "models") + if err := os.MkdirAll(models, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(models, "keep.gguf"), []byte("model"), 0600); err != nil { + t.Fatal(err) + } + if err := removeLlamaRuntime(f.st); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(f.st.installDir, "runtime")); !os.IsNotExist(err) { + t.Fatal("uninstall left the runtime slot") + } + data, _ := os.ReadFile(filepath.Join(models, "keep.gguf")) + if string(data) != "model" { + t.Fatal("model lost") + } +} + +// Without a complete native inventory the ARM policy cannot tell CPU-only from +// NVIDIA-with-a-broken-driver, so install refuses before any download. +func TestLlamaARMCPURefusesUncertainPolicy(t *testing.T) { + for _, name := range []string{"query-error", "incomplete", "cancel"} { + t.Run(name, func(t *testing.T) { + f := cpuARMFixture(t) + ctx := context.Background() + switch name { + case "query-error": + f.e.armHardwareQuery = func(context.Context, map[string]string) (string, error) { + return "", errors.New("inventory unavailable") + } + case "incomplete": + f.e.armHardwareQuery = func(context.Context, map[string]string) (string, error) { return "{}", nil } + case "cancel": + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + } + if err := f.e.installLlamaApp(ctx, f.st); err == nil { + t.Fatal("uncertainty accepted") + } + if len(f.requests) != 0 { + t.Fatal("uncertainty started download/fallback") + } + if _, err := os.Stat(filepath.Join(f.st.installDir, "runtime")); !os.IsNotExist(err) { + t.Fatal("refused install promoted a runtime") + } + }) + } +} diff --git a/services/nvpair-engine-manager/llamacache.go b/services/nvpair-engine-manager/llamacache.go new file mode 100644 index 00000000..28c62241 --- /dev/null +++ b/services/nvpair-engine-manager/llamacache.go @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" +) + +func llamaModelDir(st *engineState) string { + if st.modelDir != "" { + return st.modelDir + } + return filepath.Join(st.installDir, "models") +} + +// Move the owned cache once, atomically on the same platform data volume. +// Never merge or replace user content, and never move a live engine's cache. +func migrateLlamaCache(st *engineState) error { + legacy, target := filepath.Join(st.installDir, "models"), llamaModelDir(st) + if legacy == target { + return nil + } + info, err := os.Lstat(legacy) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if !info.IsDir() { + return fmt.Errorf("legacy model cache is not an owned directory") + } + for _, path := range []string{legacy, target} { + if err := validateLlamaPath(path); err != nil { + return err + } + } + if _, err := os.Lstat(target); err == nil { + return fmt.Errorf("both legacy and persistent model caches exist; neither was changed") + } else if !os.IsNotExist(err) { + return err + } + if err := filepath.WalkDir(legacy, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.Type()&os.ModeSymlink == 0 { + return nil + } + link, err := os.Readlink(path) + if err != nil { + return err + } + if filepath.IsAbs(link) { + return fmt.Errorf("legacy cache has an absolute link; migration left it unchanged") + } + _, err = llamaOwnedFile(legacy, path) + return err + }); err != nil { + return err + } + if st.port > 0 && st.plat.Runtime.Ready != nil { + ctx, cancel := context.WithTimeout(context.Background(), presenceRefusalWindow) + defer cancel() + if probeListener(ctx, st.plat.Runtime.Ready, st.port) != listenerProbeRefused { + return fmt.Errorf("stop the listener on llama's configured port before migrating its model cache") + } + } + if err := os.MkdirAll(filepath.Dir(target), 0700); err != nil { + return err + } + return os.Rename(legacy, target) +} diff --git a/services/nvpair-engine-manager/llamacache_test.go b/services/nvpair-engine-manager/llamacache_test.go new file mode 100644 index 00000000..c919ddf0 --- /dev/null +++ b/services/nvpair-engine-manager/llamacache_test.go @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLlamaCacheMigrationSurvivesAppReset(t *testing.T) { + root := t.TempDir() + app := filepath.Join(root, "app") + st := &engineState{installDir: filepath.Join(app, "engine-bin", "llamacpp"), modelDir: filepath.Join(root, "persistent-models", "llamacpp")} + legacy := filepath.Join(st.installDir, "models") + if err := os.MkdirAll(legacy, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(legacy, "retained.gguf"), []byte("GGUFretained"), 0600); err != nil { + t.Fatal(err) + } + if err := migrateLlamaCache(st); err != nil { + t.Fatal(err) + } + if err := os.RemoveAll(app); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(st.modelDir, "retained.gguf")) + if err != nil || string(data) != "GGUFretained" { + t.Fatalf("application reset lost migrated model: %q %v", data, err) + } + if err := migrateLlamaCache(st); err != nil { + t.Fatalf("migration is not restart-safe: %v", err) + } +} + +func TestLlamaCacheMigrationPreservesBothOnCollision(t *testing.T) { + root := t.TempDir() + st := &engineState{installDir: filepath.Join(root, "runtime"), modelDir: filepath.Join(root, "persistent")} + legacy := filepath.Join(st.installDir, "models") + for _, dir := range []string{legacy, st.modelDir} { + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "keep"), []byte(dir), 0600); err != nil { + t.Fatal(err) + } + } + if err := migrateLlamaCache(st); err == nil { + t.Fatal("merged distinct model caches") + } + for _, dir := range []string{legacy, st.modelDir} { + if data, err := os.ReadFile(filepath.Join(dir, "keep")); err != nil || string(data) != dir { + t.Fatalf("collision changed existing content: %v", err) + } + } +} + +func TestLlamaServingAndControlSharePersistentCache(t *testing.T) { + root := t.TempDir() + st := &engineState{ + installDir: filepath.Join(root, "app", "engine-bin", "llamacpp"), + modelDir: filepath.Join(root, "persistent", "llamacpp"), + manifest: &Manifest{Engine: "llamacpp"}, + plat: &Platform{Runtime: Runtime{Env: map[string]string{ + "LLAMA_CACHE": "{install_dir}/models", "HF_HUB_CACHE": "{model_dir}", "TEST_PORT": "{port}", + }}}, + } + env, err := childEnv(st, map[string]string{"port": "18082"}) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"LLAMA_CACHE", "HF_HUB_CACHE"} { + if plainWindowsPath(env[key]) != st.modelDir { + t.Fatalf("%s escaped persistent cache: %q", key, env[key]) + } + } + if env["TEST_PORT"] != "18082" { + t.Fatal("serving-specific environment was lost") + } +} diff --git a/services/nvpair-engine-manager/llamacompat.go b/services/nvpair-engine-manager/llamacompat.go new file mode 100644 index 00000000..3f8c1161 --- /dev/null +++ b/services/nvpair-engine-manager/llamacompat.go @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "time" +) + +// This is the qualified Windows Vulkan executable, not a rolling build rule. +// Requalify or remove this profile when the vendor identity changes. +const llamaB580BinarySHA256 = "58a3601bb2760652050595bba9b1f8cc9b4380d63427b045f0ba8099716b9ee6" +const llamaB580Profile = "b10826-73a43d1f6-windows-vulkan-b580" + +var llamaB580Device = regexp.MustCompile(`(?m)^[\t ]*(Vulkan[0-9]+):[\t ]*Intel(?:\(R\))? Arc(?:\(TM\))? B580 Graphics(?:[\t \r\n]|$)`) + +func llamaB580Receipt(data []byte) bool { + var receipt struct { + BinarySHA256 string `json:"binary_sha256"` + InstallerSHA256 string `json:"installer_sha256"` + Platform string `json:"platform"` + } + return json.Unmarshal(data, &receipt) == nil && receipt.Platform == "windows/amd64" && + receipt.BinarySHA256 == llamaB580BinarySHA256 && + receipt.InstallerSHA256 == "455084203db0c864f4eb218bc82792b4304458a96211c385275d8337a5049851" +} + +// prepareLlamaCompatibility runs only at the owned process-start boundary, so +// older installs receive defaults on their next start without a reinstall. +// The caller holds opMu; no durable state or runtime/model bytes are changed. +func (e *Executor) prepareLlamaCompatibility(ctx context.Context, st *engineState, bin string, args []string, env map[string]string) error { + if runtime.GOOS != "windows" || runtime.GOARCH != "amd64" || st.manifest == nil || st.manifest.Engine != "llamacpp" { + return nil + } + st.mu.Lock() + adopted := st.adopted + st.mu.Unlock() + if adopted || !isOurEngineImage(bin, filepath.Join(st.installDir, "runtime", "llama.exe")) { + return nil + } + if err := validateLlamaOwnedPaths(st.installDir); err != nil { + return err + } + data, err := os.ReadFile(filepath.Join(st.installDir, "runtime", "pair-install.json")) + if err != nil { + return nil // Unknown/external provenance is not a managed compatibility target. + } + if !llamaB580Receipt(data) { + return nil + } + f, err := os.Open(bin) + if err != nil { + return err + } + h := sha256.New() + _, err = io.Copy(llamaArchiveWriter{ctx, h}, f) + f.Close() + if err != nil { + return err + } + if hex.EncodeToString(h.Sum(nil)) != llamaB580BinarySHA256 { + return fmt.Errorf("llama compatibility identity changed: reinstall the managed runtime before starting") + } + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + devices, err := e.runCommandOutput(ctx, []string{bin, "cli", "--list-devices"}, env) + if ctx.Err() != nil { + return ctx.Err() + } + if err != nil { + return fmt.Errorf("llama compatibility device enumeration failed; retry Start after checking the managed runtime") + } + selected, err := llamaB580Selected(devices, args, env) + if err != nil || !selected { + return err + } + if err := applyLlamaB580Defaults(args, env); err != nil { + return err + } + const detail = "COOPMAT, COOPMAT2, INTEGER_DOT_PRODUCT, F16 and BFLOAT16 disabled; flash attention off" + slog.Info("llama compatibility profile", "profile", llamaB580Profile, "options", detail) + st.logs.append("stderr", "compatibility profile "+llamaB580Profile+": "+detail) + return nil +} + +func llamaB580Selected(devices string, args []string, env map[string]string) (bool, error) { + rows := llamaB580Device.FindAllStringSubmatch(devices, -1) + if len(rows) == 0 { + return false, nil + } + layers, explicitLayers, err := llamaCompatibilityOption(args, env, "LLAMA_ARG_N_GPU_LAYERS", "--gpu-layers", "--n-gpu-layers", "-ngl") + if err != nil { + return false, err + } + if n, parseErr := strconv.Atoi(strings.TrimSpace(layers)); explicitLayers && parseErr == nil && n == 0 { + return false, nil + } + selection, explicit, err := llamaCompatibilityOption(args, env, "LLAMA_ARG_DEVICE", "--device", "-dev") + if err != nil { + return false, err + } + if !explicit || selection == "" { + return true, nil // Vendor automatic selection can use the enumerated B580. + } + for _, device := range strings.Split(selection, ",") { + for _, row := range rows { + if device == row[1] { + return true, nil + } + } + } + return false, nil +} + +func applyLlamaB580Defaults(args []string, env map[string]string) error { + // Model presets can override router defaults for individual children. Keep + // their contents private and require explicit reconciliation before serving. + if preset, _, err := llamaCompatibilityOption(args, env, "LLAMA_ARG_MODELS_PRESET", "--models-preset"); err != nil { + return err + } else if preset != "" { + return fmt.Errorf("llama B580 compatibility cannot verify per-model preset options; remove --models-preset / LLAMA_ARG_MODELS_PRESET and use runtime options with flash attention off, then retry Start") + } + defaults := map[string]string{ + "GGML_VK_DISABLE_COOPMAT": "1", "GGML_VK_DISABLE_COOPMAT2": "1", + "GGML_VK_DISABLE_INTEGER_DOT_PRODUCT": "1", "GGML_VK_DISABLE_F16": "1", + "GGML_VK_DISABLE_BFLOAT16": "1", "LLAMA_ARG_FLASH_ATTN": "off", + } + // Validate everything before changing this process's environment. Windows + // names are case-insensitive; manifest options override inherited options. + missing := map[string]string{} + for key, want := range defaults { + var flags []string + if key == "LLAMA_ARG_FLASH_ATTN" { + flags = []string{"--flash-attn", "-fa"} + } + value, present, err := llamaCompatibilityOption(args, env, key, flags...) + if err != nil { + return err + } + if present && value != want { + return fmt.Errorf("llama B580 compatibility requires %s=%s; remove or change the explicit option and retry Start", key, want) + } + if !present { + missing[key] = want + } + } + for key, value := range missing { + env[key] = value + } + return nil +} + +// Match the existing child environment precedence, then the vendor's CLI +// precedence. Conflicting Windows spellings in an unordered map are ambiguous. +func llamaCompatibilityOption(args []string, env map[string]string, key string, flags ...string) (string, bool, error) { + // The five Vulkan disable variables use getenv presence, not Boolean values. + // Preserve existing values (including empty and "0") without changing behavior. + if len(flags) == 0 { + if _, present := os.LookupEnv(key); present { + return "1", true, nil + } + for name := range env { + if strings.EqualFold(name, key) { + return "1", true, nil + } + } + return "", false, nil + } + value, present := os.LookupEnv(key) + canonical := func(value string) string { + if key == "LLAMA_ARG_FLASH_ATTN" && (value == "0" || value == "false" || value == "disabled") { + return "off" + } + return value + } + value = canonical(value) + found := false + for k, v := range env { + if strings.EqualFold(k, key) { + v = canonical(v) + if found && v != value { + return "", false, fmt.Errorf("llama compatibility: conflicting spellings of %s in runtime.env; keep one value and retry Start", key) + } + value, present, found = v, true, true + } + } + for i, arg := range args { + name, v, inline := strings.Cut(arg, "=") + for _, flag := range flags { + if name != flag { + continue + } + if !inline { + negativeLayer := false + if key == "LLAMA_ARG_N_GPU_LAYERS" && i+1 < len(args) { + _, parseErr := strconv.Atoi(args[i+1]) + negativeLayer = parseErr == nil + } + if i+1 == len(args) || (strings.HasPrefix(args[i+1], "-") && !negativeLayer) { + return "", false, fmt.Errorf("llama compatibility: %s needs an explicit value; correct runtime.args and retry Start", flag) + } + v = args[i+1] + } + value, present = canonical(v), true + } + } + return value, present, nil +} diff --git a/services/nvpair-engine-manager/llamacompat_test.go b/services/nvpair-engine-manager/llamacompat_test.go new file mode 100644 index 00000000..dfae23cc --- /dev/null +++ b/services/nvpair-engine-manager/llamacompat_test.go @@ -0,0 +1,327 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +const b580Devices = "Available devices:\r\n Vulkan0: Intel(R) Arc(TM) B580 Graphics (12116 MiB, 11347 MiB free)\r\n Vulkan1: AMD Radeon(TM) Graphics (16188 MiB, 15379 MiB free)\r\n" + +// No profile marker is required: this is the receipt format from before the +// compatibility default. Runtime identity, not a PAIR version, admits it. +func b580OldReceipt(t *testing.T) []byte { + t.Helper() + data, err := json.Marshal(map[string]string{ + "platform": "windows/amd64", "version": "version: 0.4.0-dev (build 10826, commit 73a43d1f6)", + "binary_sha256": llamaB580BinarySHA256, + "installer_sha256": "455084203db0c864f4eb218bc82792b4304458a96211c385275d8337a5049851", + }) + if err != nil { + t.Fatal(err) + } + return data +} + +func TestLlamaB580Receipt(t *testing.T) { + old := b580OldReceipt(t) + for _, tc := range []struct { + name string + data []byte + want bool + }{ + {"old-install", old, true}, + {"new-build", bytes.ReplaceAll(old, []byte(llamaB580BinarySHA256), []byte(strings.Repeat("a", 64))), false}, + {"new-installer", bytes.ReplaceAll(old, []byte("45508420"), []byte("55508420")), false}, + {"windows-arm-cuda", bytes.ReplaceAll(old, []byte("windows/amd64"), []byte("windows/arm64")), false}, + {"apple-silicon", bytes.ReplaceAll(old, []byte("windows/amd64"), []byte("darwin/arm64")), false}, + {"linux", bytes.ReplaceAll(old, []byte("windows/amd64"), []byte("linux/amd64")), false}, + {"unknown", []byte(`{"version":"b10826-73a43d1f6"}`), false}, + {"invalid", []byte("{"), false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := llamaB580Receipt(tc.data); got != tc.want { + t.Fatalf("receipt match = %v, want %v", got, tc.want) + } + }) + } +} + +func TestLlamaB580Selection(t *testing.T) { + t.Setenv("LLAMA_ARG_DEVICE", "") + t.Setenv("LLAMA_ARG_N_GPU_LAYERS", "") + for _, tc := range []struct { + name, devices string + args []string + env map[string]string + want bool + }{ + {"automatic-mixed", b580Devices, nil, nil, true}, + {"explicit-b580", b580Devices, []string{"serve", "-dev", "Vulkan0"}, nil, true}, + {"explicit-mixed", b580Devices, []string{"--device=Vulkan1,Vulkan0"}, nil, true}, + {"explicit-other-vulkan", b580Devices, []string{"--device", "Vulkan1"}, nil, false}, + {"explicit-cuda", b580Devices, []string{"--device", "CUDA0"}, nil, false}, + {"explicit-cpu", b580Devices, nil, map[string]string{"LLAMA_ARG_DEVICE": "none"}, false}, + {"zero-gpu-layers", b580Devices, []string{"-ngl", "0", "-fa", "on"}, nil, false}, + {"zero-layers-env", b580Devices, nil, map[string]string{"LLAMA_ARG_N_GPU_LAYERS": "0"}, false}, + {"negative-auto-layers", b580Devices, []string{"--gpu-layers", "-1"}, nil, true}, + {"negative-all-layers", b580Devices, []string{"--gpu-layers=-2"}, nil, true}, + {"cli-over-env", b580Devices, []string{"-dev", "Vulkan0"}, map[string]string{"llama_arg_device": "CUDA0"}, true}, + {"last-cli-wins", b580Devices, []string{"-dev", "Vulkan0", "--device=Vulkan1"}, nil, false}, + {"renumbered", strings.ReplaceAll(b580Devices, "Vulkan0", "Vulkan12"), []string{"-dev", "Vulkan12"}, nil, true}, + {"other-intel", strings.ReplaceAll(b580Devices, "B580", "B570"), nil, nil, false}, + {"model-name-is-not-device", "model: Intel(R) Arc(TM) B580 Graphics", nil, nil, false}, + {"diagnostic-is-not-row", "error: Vulkan0: Intel(R) Arc(TM) B580 Graphics", nil, nil, false}, + {"cuda-only", " CUDA0: NVIDIA GPU", nil, nil, false}, + {"empty", "", nil, nil, false}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := llamaB580Selected(tc.devices, tc.args, tc.env) + if err != nil || got != tc.want { + t.Fatalf("selection = %v, %v; want %v", got, err, tc.want) + } + }) + } +} + +func clearB580Environment(t *testing.T) { + t.Helper() + for _, key := range []string{"GGML_VK_DISABLE_COOPMAT", "GGML_VK_DISABLE_COOPMAT2", "GGML_VK_DISABLE_INTEGER_DOT_PRODUCT", "GGML_VK_DISABLE_F16", "GGML_VK_DISABLE_BFLOAT16", "LLAMA_ARG_FLASH_ATTN", "LLAMA_ARG_MODELS_PRESET"} { + t.Setenv(key, "") // Register restoration before making the key absent. + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } +} + +func TestLlamaB580Environment(t *testing.T) { + clearB580Environment(t) + for _, tc := range []struct { + name string + args []string + env, parent map[string]string + conflict bool + }{ + {"defaults", nil, map[string]string{"UNCHANGED": "keep"}, nil, false}, + {"compatible", []string{"-fa", "off"}, map[string]string{"ggml_vk_disable_f16": "1"}, nil, false}, + {"inherited-compatible", nil, nil, map[string]string{"GGML_VK_DISABLE_F16": "1"}, false}, + {"inherited-zero", nil, nil, map[string]string{"GGML_VK_DISABLE_F16": "0"}, false}, + {"manifest-over-parent", nil, map[string]string{"ggml_vk_disable_f16": "1"}, map[string]string{"GGML_VK_DISABLE_F16": "0"}, false}, + {"explicit-zero", nil, map[string]string{"GGML_VK_DISABLE_COOPMAT": "0"}, nil, false}, + {"explicit-empty", nil, map[string]string{"GGML_VK_DISABLE_COOPMAT2": ""}, nil, false}, + {"equivalent-presence-case", nil, map[string]string{"GGML_VK_DISABLE_F16": "1", "ggml_vk_disable_f16": "0"}, nil, false}, + {"ambiguous-flash-case", nil, map[string]string{"LLAMA_ARG_FLASH_ATTN": "off", "llama_arg_flash_attn": "on"}, nil, true}, + {"flash-env-on", nil, nil, map[string]string{"LLAMA_ARG_FLASH_ATTN": "on"}, true}, + {"flash-cli-on", []string{"--flash-attn=on"}, nil, nil, true}, + {"flash-cli-auto", []string{"-fa", "auto"}, nil, nil, true}, + {"flash-cli-missing", []string{"--flash-attn"}, nil, nil, true}, + {"flash-cli-over-env", []string{"--flash-attn", "off"}, map[string]string{"LLAMA_ARG_FLASH_ATTN": "on"}, nil, false}, + {"flash-inherited-zero", nil, nil, map[string]string{"LLAMA_ARG_FLASH_ATTN": "0"}, false}, + {"flash-manifest-disabled", nil, map[string]string{"LLAMA_ARG_FLASH_ATTN": "disabled"}, nil, false}, + {"flash-cli-false", []string{"-fa", "false"}, nil, nil, false}, + {"flash-inline-zero", []string{"--flash-attn=0"}, nil, nil, false}, + {"equivalent-flash-case", nil, map[string]string{"LLAMA_ARG_FLASH_ATTN": "off", "llama_arg_flash_attn": "false"}, nil, false}, + {"flash-last-cli-wins", []string{"-fa", "on", "--flash-attn=off"}, nil, nil, false}, + {"model-preset", []string{"--models-preset", "private.ini"}, nil, nil, true}, + {"inherited-preset", nil, nil, map[string]string{"LLAMA_ARG_MODELS_PRESET": "private.ini"}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.parent { + t.Setenv(k, v) + } + env := map[string]string{} + for k, v := range tc.env { + env[k] = v + } + before, _ := json.Marshal(env) + parent := os.Environ() + err := applyLlamaB580Defaults(tc.args, env) + if (err != nil) != tc.conflict { + t.Fatalf("conflict = %v, want %v", err, tc.conflict) + } + if !reflect.DeepEqual(os.Environ(), parent) { + t.Fatal("changed parent environment") + } + if err != nil { + after, _ := json.Marshal(env) + if !bytes.Equal(before, after) || !strings.Contains(err.Error(), "retry Start") || strings.Contains(err.Error(), "private.ini") { + t.Fatalf("conflict changed options or lacked private actionable error: %v", err) + } + return + } + for k, v := range tc.env { + if env[k] != v { + t.Fatalf("changed explicit option %s", k) + } + } + for _, key := range []string{"GGML_VK_DISABLE_COOPMAT", "GGML_VK_DISABLE_COOPMAT2", "GGML_VK_DISABLE_INTEGER_DOT_PRODUCT", "GGML_VK_DISABLE_F16", "GGML_VK_DISABLE_BFLOAT16", "LLAMA_ARG_FLASH_ATTN"} { + flags := []string{} + want := "1" + if key == "LLAMA_ARG_FLASH_ATTN" { + want, flags = "off", []string{"--flash-attn", "-fa"} + } + got, present, err := llamaCompatibilityOption(tc.args, env, key, flags...) + if err != nil || !present || got != want { + t.Fatalf("effective %s = %q, %v", key, got, err) + } + } + first, _ := json.Marshal(env) + if err := applyLlamaB580Defaults(tc.args, env); err != nil { + t.Fatal(err) + } + second, _ := json.Marshal(env) + if !bytes.Equal(first, second) || env["GGML_VK_DISABLE_ASYNC"] != "" { + t.Fatal("defaults are not idempotent or added ASYNC") + } + }) + } +} + +func TestLlamaB580StartAdmission(t *testing.T) { + clearB580Environment(t) + for _, tc := range []struct { + name, engine string + adopted, external, known bool + }{ + {"other-engine", "ollama", false, false, true}, + {"adopted", "llamacpp", true, false, true}, + {"external-path", "llamacpp", false, true, true}, + {"unknown-receipt", "llamacpp", false, false, false}, + {"changed-binary", "llamacpp", false, false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + bin := filepath.Join(root, "runtime", "llama.exe") + if err := os.MkdirAll(filepath.Dir(bin), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(bin, []byte("never execute"), 0600); err != nil { + t.Fatal(err) + } + if tc.known { + if err := os.WriteFile(filepath.Join(filepath.Dir(bin), "pair-install.json"), b580OldReceipt(t), 0600); err != nil { + t.Fatal(err) + } + } + if tc.external { + bin = filepath.Join(t.TempDir(), "llama.exe") + } + st := &engineState{manifest: &Manifest{Engine: tc.engine}, installDir: root, adopted: tc.adopted} + env := map[string]string{"UNCHANGED": "keep"} + err := (&Executor{}).prepareLlamaCompatibility(context.Background(), st, bin, nil, env) + wantErr := tc.name == "changed-binary" && runtime.GOOS == "windows" && runtime.GOARCH == "amd64" + if (err != nil) != wantErr || len(env) != 1 { + t.Fatalf("admission = %v, options = %v", err, env) + } + if wantErr { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := (&Executor{}).prepareLlamaCompatibility(ctx, st, bin, nil, env); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation = %v", err) + } + } + }) + } +} + +// An old B580 install the user turned Off must stay Off across restores, and a +// repeat Install over it must be a pure no-op: no fetch, no receipt rewrite, no +// runtime byte change and no staging, so the compatibility identity keeps +// qualifying on the next Start. +func TestLlamaB580OldInstallStaysOffAndInstallIsIdempotent(t *testing.T) { + e := NewExecutor(buildRegistry(""), NewReporter(nil), nil, t.TempDir()) + st, err := e.state("llamacpp") + if err != nil { + t.Fatal(err) + } + bin := filepath.Join(st.installDir, "runtime", llamaExecutable()) + if err := os.MkdirAll(filepath.Dir(bin), 0700); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(fakeEngineBin) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(bin, data, 0700); err != nil { + t.Fatal(err) + } + receiptPath := filepath.Join(filepath.Dir(bin), "pair-install.json") + receipt := b580OldReceipt(t) + if err := os.WriteFile(receiptPath, receipt, 0600); err != nil { + t.Fatal(err) + } + if err := e.setDesiredEnabled("llamacpp", false); err != nil { + t.Fatal(err) + } + e.client = &http.Client{Transport: llamaFixtureTransport(func(*http.Request) (*http.Response, error) { + t.Error("repeat install over an existing runtime fetched an artifact") + return nil, errors.New("unexpected acquisition") + })} + for range 2 { + if err := e.Install(context.Background(), "llamacpp"); err != nil { + t.Fatal(err) + } + if err := e.RestoreEnabled(context.Background()); err != nil { + t.Fatal(err) + } + } + after, err := os.ReadFile(receiptPath) + if err != nil || !bytes.Equal(receipt, after) || !llamaB580Receipt(after) { + t.Fatal("old receipt changed or stopped qualifying") + } + after, err = os.ReadFile(bin) + if err != nil || !bytes.Equal(data, after) { + t.Fatal("runtime bytes changed") + } + if enabled, known, err := e.desired.get("llamacpp"); err != nil || enabled || !known || st.running || st.proc != nil { + t.Fatal("Off intent changed") + } + entries, err := os.ReadDir(st.installDir) + if err != nil || len(entries) != 1 || entries[0].Name() != "runtime" { + t.Fatal("repeat install created model or runtime stages") + } +} + +func TestLlamaB580ChildEnvironment(t *testing.T) { + if os.Getenv("PAIR_B580_ENV_CHILD") == "1" { + for _, key := range []string{"GGML_VK_DISABLE_COOPMAT", "GGML_VK_DISABLE_COOPMAT2", "GGML_VK_DISABLE_INTEGER_DOT_PRODUCT", "GGML_VK_DISABLE_F16", "GGML_VK_DISABLE_BFLOAT16", "LLAMA_ARG_FLASH_ATTN"} { + fmt.Print(os.Getenv(key), "/") + } + os.Exit(0) + } + clearB580Environment(t) + t.Setenv("GGML_VK_DISABLE_F16", "0") + key := "GGML_VK_DISABLE_F16" + if runtime.GOOS == "windows" { + key = strings.ToLower(key) + } + env := map[string]string{key: "1", "PAIR_B580_ENV_CHILD": "1"} + if err := applyLlamaB580Defaults(nil, env); err != nil { + t.Fatal(err) + } + args := []string{"-test.run=^TestLlamaB580ChildEnvironment$"} + var output strings.Builder + proc, err := startManagedProc(os.Args[0], args, env, func(_, line string) { output.WriteString(line) }) + if err != nil { + t.Fatal(err) + } + <-proc.done + if got := output.String(); got != "1/1/1/1/1/off/" { + t.Fatalf("actual child environment = %q", got) + } + if os.Getenv("GGML_VK_DISABLE_F16") != "0" { + t.Fatal("child options changed the parent") + } +} diff --git a/services/nvpair-engine-manager/llamainstall.go b/services/nvpair-engine-manager/llamainstall.go new file mode 100644 index 00000000..a68d7758 --- /dev/null +++ b/services/nvpair-engine-manager/llamainstall.go @@ -0,0 +1,316 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" +) + +func llamaExecutable() string { + if runtime.GOOS == "windows" { + return "llama.exe" + } + return "llama" +} + +var llamaPinnedVersion = regexp.MustCompile(`(?m)^(?:b10826-[a-zA-Z0-9]+|version: .*\(build 10826, commit [a-zA-Z0-9]+\))`) + +func llamaInstallSupport(goos, arch string) (bool, string) { + if arch != "amd64" && arch != "arm64" { + return false, "The official llama app has no build for this CPU architecture." + } + if goos != "windows" && goos != "linux" && goos != "darwin" { + return false, "The official llama app installer is unavailable on this operating system." + } + if goos == "darwin" && arch == "amd64" { + return true, "The official Intel Mac llama app uses CPU inference; Radeon acceleration is not provided by this recipe." + } + if goos == "windows" && arch == "arm64" { + return true, "Native hardware inventory selects CPU for non-NVIDIA ARM; NVIDIA ARM remains CUDA-required, including when its driver needs repair." + } + return true, "The vendor installer selects an available accelerator or CPU build; acceleration is verified after installation." +} + +// The vendor installer owns a fixed home-relative staging directory. Give it +// a fresh, private home and request download-only; never let it replace a user +// binary, mutate PATH, or update the live runtime in place. +func llamaInstallerEnv(st *engineState, stage string) (map[string]string, error) { + env, err := childEnv(st) + if err != nil { + return nil, err + } + for _, key := range []string{"HOME", "USERPROFILE", "LOCALAPPDATA", "APPDATA"} { + env[key] = stage + } + env["SKIP_INSTALL"] = "1" + env["LLAMA_VERSION"] = "b10826" + env["LLAMA_BUCKET"] = "ggml-org/install.sh" + return env, nil +} + +// installLlamaApp stages, verifies and promotes a fresh managed runtime. Install +// returns before reaching here when a managed runtime is already detected or an +// external identified listener is adopted, so no runtime is running and the +// runtime slot is empty (or holds an incomplete image) when promotion happens. +func (e *Executor) installLlamaApp(ctx context.Context, st *engineState) (err error) { + const engine = "llamacpp" + defer func() { + if err != nil { + e.reportInstallFailed(engine, err) + } + }() + if ok, reason := llamaInstallSupport(runtime.GOOS, runtime.GOARCH); !ok { + return errors.New(reason) + } + if st.plat.Install == nil || (len(st.plat.Install.Archives) == 0 && (st.plat.Install.Fetch == nil || st.plat.Install.Fetch.SHA256 == "")) { + return errors.New("llama app requires a pinned official installer or archive recipe") + } + if err = os.MkdirAll(st.installDir, 0o700); err != nil { + return err + } + if err = validateLlamaOwnedPaths(st.installDir); err != nil { + return err + } + stage, err := os.MkdirTemp(st.installDir, ".llama-install-") + if err != nil { + return err + } + // Preserve failed stages for diagnosis. A successful stage contains no models. + defer func() { + if err == nil { + _ = safeRemoveUnderRoot(st.installDir, stage) + } + }() + ctx, cancel := context.WithTimeout(ctx, 30*time.Minute) + defer cancel() + env, err := llamaInstallerEnv(st, stage) + if err != nil { + return err + } + candidate := filepath.Join(stage, ".llama-app") + if runtime.GOOS == "windows" { + candidate = filepath.Join(stage, "llama-app") + } + var provenance map[string]any + if st.plat.Install.UpstreamFirst { + candidate, provenance, err = e.prepareLlamaWindowsARM(ctx, st, stage) + if err != nil { + return err + } + } else if len(st.plat.Install.Archives) > 0 { + if err = e.stageLlamaArchives(ctx, st, candidate); err != nil { + return fmt.Errorf("official llama archives: %w", err) + } + } else { + script, downloadErr := e.download(ctx, engine, st.plat.Install.Fetch) + if downloadErr != nil { + return downloadErr + } + defer os.Remove(script) + e.emitInstallProgress(engine, "installing", -1) + if err = e.runCommand(ctx, llamaInstallerArgs(runtime.GOOS, script), env); err != nil { + return fmt.Errorf("official llama installer: %w", err) + } + } + e.emitInstallProgress(engine, "installing", -1) + bin := filepath.Join(candidate, llamaExecutable()) + if provenance == nil { + version, err := e.runCommandOutput(ctx, []string{bin, "version"}, env) + if err != nil { + return fmt.Errorf("validate downloaded llama: %w", err) + } + if !llamaPinnedVersion.MatchString(version) { + return fmt.Errorf("official llama version did not match pinned build b10826") + } + licenses, err := e.runCommandOutput(ctx, []string{bin, "licenses"}, env) + if err != nil { + return fmt.Errorf("read llama third-party licenses: %w", err) + } + if err = os.WriteFile(filepath.Join(candidate, "THIRD-PARTY-LICENSES.txt"), []byte(licenses), 0o600); err != nil { + return err + } + f, err := os.Open(bin) + if err != nil { + return err + } + h := sha256.New() + _, err = io.Copy(h, f) + f.Close() + if err != nil { + return err + } + provenance = map[string]any{"version": strings.TrimSpace(version), "binary_sha256": hex.EncodeToString(h.Sum(nil)), "platform": runtime.GOOS + "/" + runtime.GOARCH} + if len(st.plat.Install.Archives) > 0 { + provenance["archives"] = st.plat.Install.Archives + provenance["recipe_sha256"] = llamaArchiveRecipeHash(st.plat.Install.Archives) + } else { + provenance["installer_sha256"] = st.plat.Install.Fetch.SHA256 + provenance["installer_url"] = st.plat.Install.Fetch.URL + } + } + receipt, err := json.MarshalIndent(provenance, "", " ") + if err != nil { + return err + } + if err = os.WriteFile(filepath.Join(candidate, "pair-install.json"), receipt, 0o600); err != nil { + return err + } + // A cancellation (Stop, shutdown, or the caller's context) that arrives + // after the candidate is complete must still leave the runtime slot alone. + if err = ctx.Err(); err != nil { + return err + } + if err = promoteLlamaRuntime(st.installDir, candidate); err != nil { + return err + } + e.Detect(engine) + e.reporter.clear(installFailedID(engine)) + e.emitInstallProgress(engine, "done", 100) + e.emitState(engine) + return nil +} + +// llamaArchiveRecipeHash identifies the exact pinned archive set that produced a +// runtime, so the receipt distinguishes recipes that share a build number but +// differ in a companion bundle (for example the CUDA runtime). +func llamaArchiveRecipeHash(archives []Fetch) string { + h := sha256.New() + for _, archive := range archives { + fmt.Fprintf(h, "%q %q\n", archive.URL, archive.SHA256) + } + return hex.EncodeToString(h.Sum(nil)) +} + +// Called after acquiring the official installer, whose checksum every caller +// has verified by this point: pinned recipes against their manifest entry, and +// Windows ARM64 latest against the commit pin in llamaupstream.go. +// The Windows policy override lasts for this installer process only. +func llamaInstallerArgs(goos, script string) []string { + if goos == "windows" { + return []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", script} + } + return []string{"sh", script} +} + +// Recover the gap between the two promotion renames after process termination. +// Called once when creating the engine state, before any operation can start. +func recoverLlamaRuntime(root string) error { + current, previous := filepath.Join(root, "runtime"), filepath.Join(root, "previous") + if _, err := os.Lstat(current); err == nil { + return nil + } else if !os.IsNotExist(err) { + return err + } + if _, err := os.Lstat(previous); os.IsNotExist(err) { + return nil + } else if err != nil { + return err + } + if err := validateLlamaOwnedPaths(root); err != nil { + return err + } + return os.Rename(previous, current) +} + +func promoteLlamaRuntime(root, candidate string) error { + if err := validateLlamaOwnedPaths(root); err != nil { + return err + } + if !isManagedInstallPath(candidate, root) { + return errors.New("candidate is outside managed install") + } + if info, err := os.Stat(candidate); err != nil { + return err + } else if !info.IsDir() { + return errors.New("candidate runtime is not a directory") + } + current, previous := filepath.Join(root, "runtime"), filepath.Join(root, "previous") + if _, err := os.Lstat(previous); err == nil { + if err := safeRemoveUnderRoot(root, previous); err != nil { + return err + } + } else if !os.IsNotExist(err) { + return err + } + hadCurrent := false + if _, err := os.Lstat(current); err == nil { + if err := os.Rename(current, previous); err != nil { + return err + } + hadCurrent = true + } else if !os.IsNotExist(err) { + return err + } + if err := os.Rename(candidate, current); err != nil { + if hadCurrent { + return errors.Join(err, os.Rename(previous, current)) + } + return err + } + return nil +} + +func removeLlamaRuntime(st *engineState) error { + if err := validateLlamaOwnedPaths(st.installDir); err != nil { + return err + } + // Models and diagnostic stages are deliberately retained. Only the two + // runtime slots are PAIR's executable installation. + for _, name := range []string{"runtime", "previous"} { + target := filepath.Join(st.installDir, name) + if _, err := os.Lstat(target); os.IsNotExist(err) { + continue + } else if err != nil { + return err + } + if err := safeRemoveUnderRoot(st.installDir, target); err != nil { + return err + } + } + st.mu.Lock() + st.binPath = "" + st.mu.Unlock() + return nil +} + +func validateLlamaOwnedPaths(root string) error { + for _, name := range []string{"", "runtime", "previous", "models"} { + p, err := filepath.Abs(filepath.Join(root, name)) + if err != nil { + return err + } + if err := validateLlamaPath(p); err != nil { + return err + } + } + return nil +} + +func llamaPrerequisite() string { + if runtime.GOOS == "windows" { + if _, err := exec.LookPath("powershell.exe"); err != nil { + return "PowerShell is required by the official llama installer." + } + } else { + for _, tool := range []string{"sh", "curl"} { + if _, err := exec.LookPath(tool); err != nil { + return tool + " is required by the official llama installer." + } + } + } + return "" +} diff --git a/services/nvpair-engine-manager/llamainstall_test.go b/services/nvpair-engine-manager/llamainstall_test.go new file mode 100644 index 00000000..a4375304 --- /dev/null +++ b/services/nvpair-engine-manager/llamainstall_test.go @@ -0,0 +1,641 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + "time" +) + +func TestLlamaManagedReplacementRetainsModels(t *testing.T) { + root := t.TempDir() + write := func(name, value string) { + t.Helper() + p := filepath.Join(root, name) + if err := os.MkdirAll(filepath.Dir(p), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(value), 0600); err != nil { + t.Fatal(err) + } + } + read := func(name string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + t.Fatal(err) + } + return string(b) + } + write("runtime/llama", "old") + write("models/model.gguf", "model") + write("settings/config.json", `{"port":18082,"preference":"retained"}`) + write("stage/llama", "new") + if err := promoteLlamaRuntime(root, filepath.Join(root, "missing")); err == nil { + t.Fatal("promoted missing candidate") + } + if read("runtime/llama") != "old" { + t.Fatal("failed promotion lost prior runtime") + } + if err := promoteLlamaRuntime(root, filepath.Join(root, "stage")); err != nil { + t.Fatal(err) + } + if read("runtime/llama") != "new" || read("previous/llama") != "old" { + t.Fatal("replacement slots disagree") + } + if err := removeLlamaRuntime(&engineState{installDir: root}); err != nil { + t.Fatal(err) + } + if read("models/model.gguf") != "model" { + t.Fatal("uninstall touched models") + } + if read("settings/config.json") != `{"port":18082,"preference":"retained"}` { + t.Fatal("runtime-only uninstall touched separate settings") + } + for _, slot := range []string{"runtime", "previous"} { + if _, err := os.Stat(filepath.Join(root, slot)); !os.IsNotExist(err) { + t.Fatalf("%s slot survived uninstall", slot) + } + } +} + +func TestLlamaInstallerUsesOnlyProcessScopedWindowsPolicy(t *testing.T) { + script := `C:\isolated\verified installer.ps1` + want := []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", script} + if got := llamaInstallerArgs("windows", script); !slices.Equal(got, want) { + t.Fatalf("Windows installer argv = %q, want only process-scoped policy", got) + } + for _, goos := range []string{"linux", "darwin"} { + if got := llamaInstallerArgs(goos, "/isolated/pinned.sh"); !slices.Equal(got, []string{"sh", "/isolated/pinned.sh"}) { + t.Fatalf("%s installer argv changed: %q", goos, got) + } + } +} + +func TestLlamaStartupRecoversInterruptedReplacement(t *testing.T) { + base := t.TempDir() + root := filepath.Join(base, "llamacpp") + previous := filepath.Join(root, "previous") + if err := os.MkdirAll(previous, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(previous, llamaExecutable()), []byte("retained runtime"), 0600); err != nil { + t.Fatal(err) + } + model := filepath.Join(root, "models", "retained.gguf") + if err := os.MkdirAll(filepath.Dir(model), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(model, []byte("GGUFretained model"), 0600); err != nil { + t.Fatal(err) + } + e := NewExecutor(buildRegistry(""), NewReporter(nil), nil, base) + if installed, err := e.Detect("llamacpp"); err != nil || !installed { + t.Fatalf("restarted manager did not recover installation: %v %v", installed, err) + } + if got, err := os.ReadFile(model); err != nil || string(got) != "GGUFretained model" { + t.Fatalf("recovery changed model: %v", err) + } + // A retained older slot must never replace an already present runtime. + if err := os.MkdirAll(previous, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(previous, llamaExecutable()), []byte("older runtime"), 0600); err != nil { + t.Fatal(err) + } + e = NewExecutor(buildRegistry(""), NewReporter(nil), nil, base) + if _, err := e.Detect("llamacpp"); err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(filepath.Join(root, "runtime", llamaExecutable())); err != nil || string(got) != "retained runtime" { + t.Fatalf("recovery replaced current runtime: %v", err) + } +} + +func TestLlamaHeadlessMutationDetectsInstalledRuntime(t *testing.T) { + base := t.TempDir() + runtimeDir := filepath.Join(base, "llamacpp", "runtime") + if err := os.MkdirAll(runtimeDir, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(runtimeDir, llamaExecutable()), []byte("owned runtime"), 0600); err != nil { + t.Fatal(err) + } + source := filepath.Join(t.TempDir(), "local-Q4_K_M.gguf") + if err := os.WriteFile(source, []byte("GGUFfixture"), 0600); err != nil { + t.Fatal(err) + } + e := NewExecutor(buildRegistry(""), NewReporter(nil), nil, base) + params, _ := json.Marshal(map[string]string{"path": source}) + if _, err := e.Action(context.Background(), "llamacpp", "import_model", params); err != nil { + t.Fatalf("first headless action requires an unrelated status call: %v", err) + } +} + +func TestLlamaRecoveryErrorIsNotUnsupportedPlatform(t *testing.T) { + base := t.TempDir() + root := filepath.Join(base, "llamacpp") + if err := os.MkdirAll(root, 0700); err != nil { + t.Fatal(err) + } + previous, outside := filepath.Join(root, "previous"), t.TempDir() + if runtime.GOOS == "windows" { + cmd := exec.Command("cmd.exe", "/c", "mklink", "/J", previous, outside) + configureSysProcAttr(cmd) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("create harmless junction fixture: %v: %s", err, output) + } + } else if err := os.Symlink(outside, previous); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Remove(previous) }) + e := NewExecutor(buildRegistry(""), NewReporter(nil), nil, base) + if _, err := e.Status("llamacpp"); err == nil || !strings.Contains(err.Error(), "recover interrupted") { + t.Fatalf("status hid recovery failure as platform support: %v", err) + } + for _, status := range e.GetInstalled() { + if status.Engine == "llamacpp" && !strings.Contains(status.InstallReason, "recover interrupted") { + t.Fatalf("inventory hid recovery failure: %+v", status) + } + } +} + +func TestLlamaManifestAndInstallerIsolation(t *testing.T) { + reg := buildRegistry("") + mf, ok := reg.Get("llamacpp") + if !ok { + t.Fatal("manifest missing") + } + // These are SHA256 of the raw Git blobs at the URL's commit, not checkout + // files that Git may have converted to CRLF on Windows. + for platform, p := range mf.Platforms { + if platform == "darwin/amd64" { + if p.Install.Fetch != nil || p.Install.ArchiveRoot != "llama-b10826" || len(p.Install.Archives) != 1 || p.Install.Archives[0].SHA256 != "adcd2066b2a1a3d8e774e36c8f97d166defccd01d947d9e39667b0435e8361b0" { + t.Fatal("Intel Mac must use the pinned official CPU archive") + } + continue + } + if platform == "windows/arm64" { + if p.Install.Fetch != nil || len(p.Install.Archives) != 2 { + t.Fatal("Windows ARM64 must use the official app and CUDA runtime archives") + } + for _, archive := range p.Install.Archives { + if len(archive.SHA256) != 64 || !strings.HasPrefix(archive.URL, "https://github.com/ggml-org/llama.cpp/releases/download/b10826/") { + t.Fatal("Windows ARM64 archive lacks pinned official provenance") + } + } + continue + } + want := "cccdfcbd1b55bf6003ac3037588c9f5b3b79aa0a75fe991e97bb218ccdb55e4d" + if strings.HasPrefix(platform, "windows/") { + want = "455084203db0c864f4eb218bc82792b4304458a96211c385275d8337a5049851" + } + if p.Install.Fetch.SHA256 != want || !strings.Contains(p.Install.Fetch.URL, "27a82f3a6e0f259f88c2c31cd6b20d858a975f27") { + t.Fatalf("%s: installer provenance mismatch", platform) + } + } + if supported, reason := llamaInstallSupport("darwin", "amd64"); !supported || !strings.Contains(reason, "CPU") { + t.Fatal("Intel Mac CPU support must be explicit") + } + st := &engineState{installDir: t.TempDir(), plat: &Platform{Runtime: Runtime{Env: map[string]string{"LLAMA_CACHE": "{install_dir}/models"}}}} + stage := filepath.Join(st.installDir, "stage") + env, err := llamaInstallerEnv(st, stage) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"HOME", "USERPROFILE", "LOCALAPPDATA", "APPDATA"} { + if env[key] != stage { + t.Fatalf("%s escapes stage", key) + } + } + if env["SKIP_INSTALL"] != "1" || filepath.Clean(plainWindowsPath(env["LLAMA_CACHE"])) != filepath.Join(st.installDir, "models") { + t.Fatal("installer/cache isolation missing") + } + for _, version := range []string{"version: 0.0.0 (build 10826, commit abc123)\nbuilt with clang", "b10826-abc123"} { + if !llamaPinnedVersion.MatchString(version) { + t.Fatalf("reject actual vendor format %q", version) + } + } + if llamaPinnedVersion.MatchString("version: 0 (build 108260, commit abc)") { + t.Fatal("accepted different build") + } +} + +func TestLlamaExtendedCachePathAndSafeDiagnostic(t *testing.T) { + root := filepath.Join(t.TempDir(), strings.Repeat("long-directory-", 12), strings.Repeat("nested-", 12)) + if err := os.MkdirAll(root, 0700); err != nil { + t.Fatal(err) + } + file := filepath.Join(root, "model-Q4_K_M.gguf") + if err := os.WriteFile(file, []byte("GGUFfixture"), 0600); err != nil { + t.Fatal(err) + } + cache, err := llamaCachePath(root) + if err != nil { + t.Fatal(err) + } + returned, err := llamaCachePath(file) + if err != nil { + t.Fatal(err) + } + owned, err := llamaOwnedFile(cache, returned) + ownedInfo, ownedErr := os.Stat(owned) + fileInfo, fileErr := os.Stat(file) + if err != nil || ownedErr != nil || fileErr != nil || !os.SameFile(ownedInfo, fileInfo) { + t.Fatalf("extended vendor path rejected: %q %v", owned, err) + } + server, err := resolveChildEnv(map[string]string{"LLAMA_CACHE": "{install_dir}"}, map[string]string{"install_dir": root}) + if err != nil { + t.Fatal(err) + } + if server["LLAMA_CACHE"] != cache { + t.Fatal("server and downloader caches differ") + } + secret := "hf_test_secret_value" + diagnostic := llamaDownloadError(&exec.ExitError{Stderr: []byte("error opening C:/private/path.downloadInProgress Authorization: Bearer " + secret)}) + if !strings.Contains(diagnostic.Error(), "path length") || strings.Contains(diagnostic.Error(), secret) || strings.Contains(diagnostic.Error(), "private") { + t.Fatalf("unsafe or unactionable diagnostic: %s", diagnostic) + } +} + +type llamaFixtureTransport func(*http.Request) (*http.Response, error) + +func (f llamaFixtureTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestLlamaIdentityAndNestedState(t *testing.T) { + body := `{"data":[{"id":"cold","status":{"value":"unloaded"}},{"id":"hot","status":{"value":"loaded"}},{"id":"uncertain","status":null}]}` + spec := &ActionResult{Array: "data", Field: "id", Match: &ResultMatch{Field: "status.value", In: []string{"loaded"}}} + got, ok := extractStringsResult(json.RawMessage(body), spec) + if !ok || len(got) != 1 || got[0] != "hot" { + t.Fatalf("loaded state %v %v", got, ok) + } + var row map[string]json.RawMessage + _ = json.Unmarshal([]byte(`{"value":"wrong","status":{}}`), &row) + if _, ok := lookupField(row, "status.value"); ok { + t.Fatal("nested lookup reused top-level value") + } + for _, server := range []string{"llama.cpp", "unrelated"} { + e := &Executor{client: &http.Client{Transport: llamaFixtureTransport(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 200, Header: http.Header{"Server": []string{server}}, Body: io.NopCloser(strings.NewReader(body))}, nil + })}} + if e.probe(context.Background(), &Probe{HTTP: "http://127.0.0.1:{port}/models", Identity: "llamacpp"}, 8082) != (server == "llama.cpp") { + t.Fatal("identity decision wrong") + } + } +} + +func TestLlamaShutdownCancelsMutationBeforeWaiting(t *testing.T) { + e := NewExecutor(buildRegistry(""), NewReporter(nil), nil, t.TempDir()) + st, err := e.state("llamacpp") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + st.opMu.Lock() + st.mu.Lock() + st.pullCancel = cancel + st.mu.Unlock() + go func() { <-ctx.Done(); st.opMu.Unlock() }() + done := make(chan struct{}) + go func() { e.StopAll(); close(done) }() + select { + case <-done: + case <-time.After(time.Second): + cancel() + t.Fatal("shutdown waited on a pull it did not cancel") + } + _, err = e.actionLlama(context.Background(), st, "import_model", Action{Builtin: "llama-models"}, json.RawMessage(`{"path":"unused"}`)) + if !errors.Is(err, context.Canceled) { + t.Fatalf("post-shutdown mutation admitted: %v", err) + } +} + +func TestLlamaAdoptedListenerRejectsMutationBeforeHTTP(t *testing.T) { + root := t.TempDir() + bin := filepath.Join(root, "llama") + if err := os.WriteFile(bin, []byte("owned executable placeholder"), 0600); err != nil { + t.Fatal(err) + } + st := &engineState{installDir: root, binPath: bin, running: true, adopted: true, plat: &Platform{Runtime: Runtime{Ready: &Probe{HTTP: "http://127.0.0.1:{port}/models", Identity: "llamacpp"}}}} + called := false + e := &Executor{client: &http.Client{Transport: llamaFixtureTransport(func(*http.Request) (*http.Response, error) { called = true; return nil, errors.New("unexpected HTTP") })}} + for _, action := range []string{"load_model", "unload_model", "pull_model", "delete_model"} { + _, err := e.actionLlama(context.Background(), st, action, Action{}, json.RawMessage(`{"model":"owner/model:Q4_K_M"}`)) + if err == nil || !strings.Contains(err.Error(), "externally") { + t.Fatalf("%s: %v", action, err) + } + } + if called { + t.Fatal("sent mutation or identity probe to external listener") + } +} + +func TestLlamaLostResidencyPublishesUnknown(t *testing.T) { + e := NewExecutor(NewRegistry(), NewReporter(nil), nil, t.TempDir()) + changed, next, result := e.sweepLoaded(context.Background(), map[string][]string{"llamacpp": {"owner/model:Q4_K_M"}, "other": {"unchanged"}}) + if len(changed) != 1 || changed[0] != "llamacpp" { + t.Fatalf("missing observation was not published: %v", changed) + } + if _, ok := next["llamacpp"]; ok { + t.Fatal("retained stale llama residency") + } + if _, ok := result.LoadedByEngine["llamacpp"]; ok { + t.Fatal("unknown observation represented as an empty successful set") + } + if len(next["other"]) != 1 { + t.Fatal("changed unrelated engine policy") + } +} + +func TestLlamaLoadWaitsForObservedState(t *testing.T) { + polls := 0 + e := &Executor{actionTimeout: time.Second, client: &http.Client{Transport: llamaFixtureTransport(func(*http.Request) (*http.Response, error) { + polls++ + state := "loading" + if polls >= 2 { + state = "loaded" + } + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"data":[{"id":"tiny","status":{"value":"` + state + `"}}]}`))}, nil + })}} + st := &engineState{running: true, port: 8082, manifest: &Manifest{Actions: map[string]Action{"list_models": {HTTP: &ActionHTTP{Method: "GET", Path: "/models"}}}}} + if err := e.waitLlamaModelState(context.Background(), st, "tiny", "loaded"); err != nil { + t.Fatal(err) + } + if polls != 2 { + t.Fatalf("returned before observed loaded state: %d polls", polls) + } + e.client.Transport = llamaFixtureTransport(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"data":[{"id":"tiny","status":{"value":"unloaded","failed":true,"exit_code":7}}]}`))}, nil + }) + if err := e.waitLlamaModelState(context.Background(), st, "tiny", "loaded"); err == nil || !strings.Contains(err.Error(), "exit code 7") { + t.Fatalf("failed load was not surfaced: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := e.waitLlamaModelState(ctx, st, "tiny", "loaded"); !errors.Is(err, context.Canceled) { + t.Fatalf("cancel: %v", err) + } +} + +func TestLlamaBusyStatusAndStopRemainResponsive(t *testing.T) { + reg := NewRegistry() + mf, _ := buildRegistry("").Get("llamacpp") + reg.engines["llamacpp"] = mf + e := NewExecutor(reg, NewReporter(nil), nil, t.TempDir()) + st, err := e.state("llamacpp") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + st.opMu.Lock() + st.mu.Lock() + st.mutationCancel = cancel + st.mu.Unlock() + statusDone := make(chan error, 1) + go func() { + _, err := e.Status("llamacpp") + if err == nil && len(e.GetInstalled()) != 1 { + err = errors.New("missing busy engine") + } + statusDone <- err + }() + select { + case err := <-statusDone: + if err != nil { + st.opMu.Unlock() + t.Fatal(err) + } + case <-time.After(time.Second): + st.opMu.Unlock() + t.Fatal("status waited behind model operation") + } + go func() { <-ctx.Done(); st.opMu.Unlock() }() + stopDone := make(chan error, 1) + go func() { stopDone <- e.Stop("llamacpp") }() + select { + case err := <-stopDone: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + cancel() + t.Fatal("Stop did not cancel the active model operation") + } + st.mu.Lock() + pending := st.stopPending + st.mu.Unlock() + if pending != 0 { + t.Fatal("Stop left mutation admission blocked") + } +} + +func TestLlamaInstallRegistersCancellationAndAdmission(t *testing.T) { + for _, shutdown := range []bool{false, true} { + t.Run(fmt.Sprintf("shutdown-%t", shutdown), func(t *testing.T) { + reg := NewRegistry() + mf, _ := buildRegistry("").Get("llamacpp") + reg.engines["llamacpp"] = mf + e := NewExecutor(reg, NewReporter(nil), nil, t.TempDir()) + st, err := e.state("llamacpp") + if err != nil { + t.Fatal(err) + } + st.plat.Runtime.Ready = nil // No listener/probe in this cancellation fixture. + started := make(chan struct{}) + e.client = &http.Client{Transport: llamaFixtureTransport(func(r *http.Request) (*http.Response, error) { + close(started) + <-r.Context().Done() + return nil, r.Context().Err() + })} + installed := make(chan error, 1) + go func() { installed <- e.Install(context.Background(), "llamacpp") }() + select { + case <-started: + case err := <-installed: + t.Fatalf("Install did not reach acquisition: %v", err) + case <-time.After(time.Second): + t.Fatal("Install acquisition did not start") + } + stopped := make(chan error, 1) + go func() { + if shutdown { + e.StopAll() + stopped <- nil + } else { + stopped <- e.Stop("llamacpp") + } + }() + select { + case err := <-installed: + if !errors.Is(err, context.Canceled) { + t.Fatalf("install result: %v", err) + } + case <-time.After(time.Second): + t.Fatal("actual Install was not cancelled") + } + select { + case err := <-stopped: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("Stop blocked behind Install") + } + st.mu.Lock() + registered := st.mutationCancel != nil + st.mu.Unlock() + if registered { + t.Fatal("install cancellation remained registered") + } + }) + } + t.Run("reject-before-effects", func(t *testing.T) { + e := NewExecutor(buildRegistry(""), NewReporter(nil), nil, t.TempDir()) + e.StopAll() + if err := e.Install(context.Background(), "llamacpp"); !errors.Is(err, context.Canceled) { + t.Fatalf("post-shutdown install: %v", err) + } + st, _ := e.state("llamacpp") + if _, err := os.Stat(st.installDir); !os.IsNotExist(err) { + t.Fatal("post-shutdown install created staging state") + } + }) +} + +func TestLlamaUnsupportedStatusExplainsWhy(t *testing.T) { + status := unavailableEngineStatus("llamacpp", "llama.cpp") + if status.InstallSupported || status.InstallReason == "" { + t.Fatalf("unsupported engine lacks a reason: %+v", status) + } +} + +// Install is idempotent: with a managed runtime already detected it reports +// already-installed and touches nothing, so a repeat cannot become a hidden +// replacement of the runtime, its receipt, or its models. +func TestLlamaInstallOnExistingRuntimeIsIdempotent(t *testing.T) { + reg := NewRegistry() + mf, _ := buildRegistry("").Get("llamacpp") + reg.engines["llamacpp"] = mf + stage := "" + e := NewExecutor(reg, NewReporter(nil), func(method string, value any) { + if method == "engine:install-progress" { + stage = value.(map[string]any)["stage"].(string) + } + }, t.TempDir()) + st, err := e.state("llamacpp") + if err != nil { + t.Fatal(err) + } + path := filepath.Join(st.installDir, "runtime", llamaExecutable()) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("owned runtime"), 0700); err != nil { + t.Fatal(err) + } + receipt := filepath.Join(filepath.Dir(path), "pair-install.json") + if err := os.WriteFile(receipt, []byte(`{"version":"fixture"}`), 0600); err != nil { + t.Fatal(err) + } + model := filepath.Join(st.installDir, "models", "retained.gguf") + if err := os.MkdirAll(filepath.Dir(model), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(model, []byte("model"), 0600); err != nil { + t.Fatal(err) + } + e.client = &http.Client{Transport: llamaFixtureTransport(func(*http.Request) (*http.Response, error) { + t.Error("repeat install fetched an artifact") + return nil, errors.New("unexpected acquisition") + })} + for range 2 { + if err := e.Install(context.Background(), "llamacpp"); err != nil { + t.Fatal(err) + } + if stage != "already-installed" { + t.Fatalf("repeat install stage = %q", stage) + } + } + for name, want := range map[string]string{path: "owned runtime", receipt: `{"version":"fixture"}`, model: "model"} { + if got, err := os.ReadFile(name); err != nil || string(got) != want { + t.Fatalf("repeat install changed %s: %q %v", filepath.Base(name), got, err) + } + } + if _, err := os.Stat(filepath.Join(st.installDir, "previous")); !os.IsNotExist(err) { + t.Fatal("repeat install replaced the runtime") + } + if matches, _ := filepath.Glob(filepath.Join(st.installDir, ".llama-install-*")); len(matches) != 0 { + t.Fatalf("repeat install staged an acquisition: %v", matches) + } +} + +// llama has no update action. The manifest does not declare one, the registry +// rejects the builtin, and the executor refuses the request outright rather +// than substituting an uninstall-and-reinstall; callers that offer a generic +// update must refuse this engine before reaching the manager. +func TestLlamaUpdateIsRefusedNotSubstituted(t *testing.T) { + mf, ok := buildRegistry("").Get("llamacpp") + if !ok { + t.Fatal("manifest missing") + } + if _, declared := mf.Actions["update"]; declared { + t.Fatal("llama manifest still declares an update action") + } + for name, act := range mf.Actions { + if act.Builtin == "llama-update" { + t.Fatalf("action %q still uses the removed llama-update builtin", name) + } + } + if err := (&Action{Builtin: "llama-update"}).validate("update"); err == nil || !strings.Contains(err.Error(), "unknown builtin") { + t.Fatalf("llama-update builtin accepted: %v", err) + } + e := NewExecutor(buildRegistry(""), NewReporter(nil), nil, t.TempDir()) + e.client = &http.Client{Transport: llamaFixtureTransport(func(*http.Request) (*http.Response, error) { + t.Error("refused update touched the network") + return nil, errors.New("unexpected request") + })} + if _, err := e.Action(context.Background(), "llamacpp", "update", nil); err == nil || !strings.Contains(err.Error(), `no action "update"`) { + t.Fatalf("update was not refused explicitly: %v", err) + } + st, err := e.state("llamacpp") + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(st.installDir); !os.IsNotExist(err) { + t.Fatal("refused update created install state") + } +} + +// The receipt's recipe_sha256 must identify the exact pinned archive set, so a +// runtime built from a changed companion bundle is never mistaken for one built +// from the recipe currently shipped. +func TestLlamaArchiveRecipeHashTracksEveryPin(t *testing.T) { + archives := []Fetch{{URL: "https://example.invalid/app.zip", SHA256: "app-pin"}, {URL: "https://example.invalid/runtime.zip", SHA256: "runtime-pin"}} + base := llamaArchiveRecipeHash(archives) + if base != llamaArchiveRecipeHash(slices.Clone(archives)) { + t.Fatal("identical archive recipe produced different provenance") + } + companion := slices.Clone(archives) + companion[1].SHA256 = "new-runtime-pin" + if llamaArchiveRecipeHash(companion) == base { + t.Fatal("changed companion runtime pin was not recorded in provenance") + } + if llamaArchiveRecipeHash(archives[:1]) == base { + t.Fatal("dropped companion archive was not recorded in provenance") + } +} diff --git a/services/nvpair-engine-manager/llamamodels.go b/services/nvpair-engine-manager/llamamodels.go new file mode 100644 index 00000000..3a3e742d --- /dev/null +++ b/services/nvpair-engine-manager/llamamodels.go @@ -0,0 +1,461 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "sort" + "strconv" + "strings" + "time" +) + +var llamaRepoRE = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]*/[A-Za-z0-9_][A-Za-z0-9_.-]*(?::[A-Za-z0-9_]+)?$`) +var llamaTagRE = regexp.MustCompile(`(?i)[-.]([A-Z0-9_]+)$`) +var llamaSplitRE = regexp.MustCompile(`(?i)^(.+)-([0-9]{5})-of-([0-9]{5})$`) + +var llamaCommitRE = regexp.MustCompile(`^[0-9a-fA-F]{40}$`) + +type llamaCacheModel struct { + ID string + Files []string +} + +// llamaCacheModels follows llama.cpp b10826 hf-cache.cpp and download.cpp. +// Only the current ref and complete primary GGUF groups are visible. +func llamaCacheModels(root string) ([]llamaCacheModel, error) { + entries, err := os.ReadDir(root) + if os.IsNotExist(err) { + return []llamaCacheModel{}, nil + } + if err != nil { + return nil, err + } + groups := map[string][]string{} + primary := map[string]bool{} + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "models--") { + continue + } + repo := strings.ReplaceAll(strings.TrimPrefix(entry.Name(), "models--"), "--", "/") + if !llamaRepoRE.MatchString(repo) || strings.Contains(repo, ":") { + continue + } + base := filepath.Join(root, entry.Name()) + refs, err := os.ReadDir(filepath.Join(base, "refs")) + if os.IsNotExist(err) { + continue + } + if err != nil { + return nil, err + } + ref := "" + for _, r := range refs { + if !r.IsDir() { + ref = r.Name() + break + } + } + for _, r := range refs { + if r.Name() == "main" { + ref = "main" + } + } + if ref == "" { + continue + } + body, err := os.ReadFile(filepath.Join(base, "refs", ref)) + if err != nil { + return nil, err + } + commit := strings.TrimSpace(string(body)) + if !llamaCommitRE.MatchString(commit) { + continue + } + snapshot := filepath.Join(base, "snapshots", commit) + err = filepath.WalkDir(snapshot, func(path string, d fs.DirEntry, walkErr error) error { + if os.IsNotExist(walkErr) { + return nil + } + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + name := d.Name() + if !strings.HasSuffix(name, ".gguf") { + return nil + } + prefix := strings.TrimSuffix(name, ".gguf") + first := true + if split := llamaSplitRE.FindStringSubmatch(prefix); split != nil { + prefix = split[1] + first = split[2] == "00001" + count, _ := strconv.Atoi(split[3]) + if count < 1 { + return nil + } + for i := 1; i <= count; i++ { + part := filepath.Join(filepath.Dir(path), fmt.Sprintf("%s-%05d-of-%05d.gguf", prefix, i, count)) + info, err := os.Stat(part) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + return nil + } + if _, err := llamaOwnedFile(root, part); err != nil { + return nil + } + } + } + tag := llamaTagRE.FindStringSubmatch(prefix) + if tag == nil { + return nil + } + for _, aux := range []string{"mmproj", "mtp-", "eagle3-", "dflash-", "dspark-"} { + if strings.Contains(prefix, aux) { + return nil + } + } + if _, err := llamaOwnedFile(root, path); err != nil { + return nil + } + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + return nil + } + id := repo + ":" + strings.ToUpper(tag[1]) + groups[id] = append(groups[id], path) + primary[id] = primary[id] || first + return nil + }) + if err != nil { + return nil, err + } + } + out := []llamaCacheModel{} + for id, files := range groups { + if primary[id] { + out = append(out, llamaCacheModel{id, files}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func llamaOwnedFile(root, path string) (string, error) { + root, path = plainWindowsPath(root), plainWindowsPath(path) + realRoot, err := filepath.EvalSymlinks(root) + if err != nil { + return "", err + } + real, err := filepath.EvalSymlinks(path) + if err != nil { + return "", err + } + rel, err := filepath.Rel(realRoot, real) + if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) { + return "", fmt.Errorf("model is outside managed cache") + } + return real, nil +} + +// Caller serializes mutations with st.opMu; this function never starts a server. +func (e *Executor) llamaModelAction(ctx context.Context, st *engineState, action string, params json.RawMessage) (json.RawMessage, error) { + root := llamaModelDir(st) + if err := validateLlamaPath(root); err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, err + } + var p struct { + Model string `json:"model"` + Name string `json:"name"` + Path string `json:"path"` + File string `json:"file"` + } + if len(params) > 0 { + if err := json.Unmarshal(params, &p); err != nil { + return nil, fmt.Errorf("invalid model parameters: %w", err) + } + } + if p.Model == "" { + p.Model = p.Name + } + switch action { + case "list_downloaded": + models, err := llamaCacheModels(root) + if err != nil { + return nil, err + } + data := []map[string]any{} + for _, m := range models { + data = append(data, map[string]any{"id": m.ID, "status": map[string]string{"value": "unloaded"}}) + } + return json.Marshal(map[string]any{"data": data}) + case "pull_model": + if !llamaRepoRE.MatchString(p.Model) || strings.Contains(p.Model, "..") { + return nil, fmt.Errorf("model must be owner/repository[:TAG]") + } + if err := os.MkdirAll(root, 0700); err != nil { + return nil, err + } + bin := "llama" + if runtime.GOOS == "windows" { + bin += ".exe" + } + args := []string{"download", "--hf-repo", p.Model} + if p.File != "" { + if strings.HasPrefix(p.File, "-") || strings.Contains(p.File, "..") || strings.ContainsAny(p.File, "\\\r\n") { + return nil, fmt.Errorf("invalid model file") + } + args = append(args, "--hf-file", p.File) + } + env, err := childEnv(st) + if err != nil { + return nil, err + } + cachePath, err := llamaCachePath(root) + if err != nil { + return nil, err + } + env["LLAMA_CACHE"] = cachePath + env["HF_HUB_CACHE"] = cachePath + cmd := exec.CommandContext(ctx, filepath.Join(st.installDir, "runtime", bin), args...) + cmd.Env = commandEnv(env) + configureSysProcAttr(cmd) + configureCommandCancel(cmd) + cmd.WaitDelay = 10 * time.Second + // Vendor output contains paths and unstructured diagnostics, not reliable percentages. + e.emitPullProgress(ProgressEvent{Engine: st.manifest.Engine, Op: "pull", Stage: "pulling", Percent: -1, Message: p.Model}) + out, err := cmd.Output() + if ctx.Err() != nil { + return nil, ctx.Err() + } + if err != nil { + return nil, llamaDownloadError(err) + } + if err := validateLlamaDownload(root, string(out)); err != nil { + return nil, err + } + return json.Marshal(map[string]string{"status": "success", "model": p.Model}) + case "import_model": + id, err := llamaImport(ctx, root, p.Path) + if err != nil { + return nil, err + } + return json.Marshal(map[string]string{"status": "success", "model": id}) + case "delete_model": + models, err := llamaCacheModels(root) + if err != nil { + return nil, err + } + for _, m := range models { + if m.ID != p.Model { + continue + } + for _, path := range m.Files { + real, err := llamaOwnedFile(root, path) + if err != nil { + return nil, err + } + if err := os.Remove(path); err != nil { + return nil, err + } + if real != path { + referenced := false + err = filepath.WalkDir(root, func(candidate string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.Type()&os.ModeSymlink != 0 { + resolved, resolveErr := filepath.EvalSymlinks(candidate) + if resolveErr == nil && resolved == real { + referenced = true + } + } + return nil + }) + if err != nil { + return nil, err + } + if !referenced { + if err := os.Remove(real); err != nil && !os.IsNotExist(err) { + return nil, err + } + } + } + } + if strings.HasPrefix(m.ID, "local/") { + // Imports own one flat snapshot; remove only empty scaffolding. + base := filepath.Join(root, "models--"+strings.ReplaceAll(strings.SplitN(m.ID, ":", 2)[0], "/", "--")) + if err := os.Remove(filepath.Dir(m.Files[0])); err == nil { + _ = os.Remove(filepath.Join(base, "snapshots")) + _ = os.Remove(filepath.Join(base, "refs", "main")) + _ = os.Remove(filepath.Join(base, "refs")) + _ = os.Remove(base) + } + } + return json.Marshal(map[string]string{"status": "success", "model": m.ID}) + } + return nil, fmt.Errorf("model not found in managed cache") + default: + return nil, fmt.Errorf("unsupported llama model action %q", action) + } +} + +// The vendor can exit successfully after downloading a preset instead of model +// weights. Validate its returned files, not merely the requested source name. +func validateLlamaDownload(root, output string) error { + if strings.TrimSpace(output) == "" { + return fmt.Errorf("download returned no model path") + } + for _, line := range strings.Split(strings.TrimSpace(output), "\n") { + path := strings.TrimSpace(line) + if !strings.HasSuffix(strings.ToLower(path), ".gguf") { + return fmt.Errorf("download returned an unsupported preset or configuration; a GGUF model is required") + } + real, err := llamaOwnedFile(root, path) + if err != nil { + return fmt.Errorf("download did not return a managed model") + } + info, err := os.Stat(real) + if err != nil || !info.Mode().IsRegular() || info.Size() < 24 { + return fmt.Errorf("download returned an empty or invalid GGUF model") + } + file, err := os.Open(real) + if err != nil { + return fmt.Errorf("cannot read downloaded model") + } + info, statErr := file.Stat() + var header [24]byte + _, readErr := io.ReadFull(file, header[:]) + file.Close() + if statErr != nil || !info.Mode().IsRegular() || info.Size() < 24 || readErr != nil || string(header[:4]) != "GGUF" { + return fmt.Errorf("download returned an empty or invalid GGUF model") + } + version := binary.LittleEndian.Uint32(header[4:8]) + if (version != 2 && version != 3) || binary.LittleEndian.Uint64(header[8:16]) == 0 { + return fmt.Errorf("download returned an unsupported GGUF header or a file without model tensors") + } + } + return nil +} + +func llamaImport(ctx context.Context, root, source string) (string, error) { + if !filepath.IsAbs(source) { + return "", fmt.Errorf("absolute GGUF path is required") + } + name := filepath.Base(source) + prefix := strings.TrimSuffix(name, ".gguf") + tag := llamaTagRE.FindStringSubmatch(prefix) + if prefix == name || tag == nil || llamaSplitRE.MatchString(prefix) { + return "", fmt.Errorf("import requires a single GGUF named with its quantization tag") + } + for _, aux := range []string{"mmproj", "mtp-", "eagle3-", "dflash-", "dspark-"} { + if strings.Contains(prefix, aux) { + return "", fmt.Errorf("import requires a primary model GGUF") + } + } + repo := "local/" + prefix + if !llamaRepoRE.MatchString(repo) || strings.Contains(repo, "..") || strings.Contains(repo, "--") { + return "", fmt.Errorf("unsupported GGUF filename") + } + src, err := os.Open(source) + if err != nil { + return "", err + } + defer src.Close() + info, err := src.Stat() + if err != nil { + return "", err + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("GGUF must be a regular file") + } + magic := make([]byte, 4) + if _, err := io.ReadFull(src, magic); err != nil || string(magic) != "GGUF" { + return "", fmt.Errorf("invalid GGUF header") + } + if _, err := src.Seek(0, 0); err != nil { + return "", err + } + base := filepath.Join(root, "models--local--"+prefix) + if _, err := os.Lstat(base); err == nil { + return "", fmt.Errorf("imported model already exists") + } else if !os.IsNotExist(err) { + return "", err + } + if err := os.MkdirAll(root, 0700); err != nil { + return "", err + } + temp, err := os.MkdirTemp(root, ".import-") + if err != nil { + return "", err + } + defer os.RemoveAll(temp) + if err := os.MkdirAll(filepath.Join(temp, "snapshots", "local"), 0700); err != nil { + return "", err + } + dst, err := os.Create(filepath.Join(temp, "snapshots", "local", name)) + if err != nil { + return "", err + } + buf := make([]byte, 1024*1024) + digest := sha256.New() + writer := io.MultiWriter(dst, digest) + for { + if err = ctx.Err(); err != nil { + break + } + var n int + n, err = src.Read(buf) + if n > 0 { + if _, writeErr := writer.Write(buf[:n]); writeErr != nil { + err = writeErr + break + } + } + if err != nil { + break + } + } + closeErr := dst.Close() + if err != io.EOF { + return "", err + } + if closeErr != nil { + return "", closeErr + } + // The vendor accepts only40hex snapshot refs, even for a local cache entry. + commit := hex.EncodeToString(digest.Sum(nil))[:40] + if err := os.Rename(filepath.Join(temp, "snapshots", "local"), filepath.Join(temp, "snapshots", commit)); err != nil { + return "", err + } + if err := os.Mkdir(filepath.Join(temp, "refs"), 0700); err != nil { + return "", err + } + if err := os.WriteFile(filepath.Join(temp, "refs", "main"), []byte(commit), 0600); err != nil { + return "", err + } + if err := ctx.Err(); err != nil { + return "", err + } + if err := os.Rename(temp, base); err != nil { + return "", err + } + return repo + ":" + strings.ToUpper(tag[1]), nil +} diff --git a/services/nvpair-engine-manager/llamamodels_test.go b/services/nvpair-engine-manager/llamamodels_test.go new file mode 100644 index 00000000..a7f05a2e --- /dev/null +++ b/services/nvpair-engine-manager/llamamodels_test.go @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/binary" + "encoding/json" + "os" + "path/filepath" + "testing" +) + +const llamaTestCommit = "0123456789abcdef0123456789abcdef01234567" + +func TestLlamaDownloadValidatesReturnedModelData(t *testing.T) { + root := t.TempDir() + header := make([]byte, 24) + copy(header, "GGUF") + binary.LittleEndian.PutUint32(header[4:8], 3) + binary.LittleEndian.PutUint64(header[8:16], 1) + for _, tc := range []struct { + name string + body []byte + valid bool + }{ + {"model-Q4_K_M.gguf", header, true}, + {"preset.ini", []byte("[model]\nname=fixture\n"), false}, + {"configuration.gguf", []byte("[model]\nname=fixture\n"), false}, + {"empty.gguf", nil, false}, + {"truncated.gguf", []byte("GGUF"), false}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(root, tc.name) + if err := os.WriteFile(path, tc.body, 0600); err != nil { + t.Fatal(err) + } + if err := validateLlamaDownload(root, path+"\n"); (err == nil) != tc.valid { + t.Fatalf("valid=%v error=%v", tc.valid, err) + } + }) + } + outside := filepath.Join(t.TempDir(), "other.gguf") + if err := os.WriteFile(outside, header, 0600); err != nil { + t.Fatal(err) + } + if err := validateLlamaDownload(root, outside); err == nil { + t.Fatal("accepted external model") + } +} + +func TestLlamaCacheImportListDelete(t *testing.T) { + root := filepath.Join(t.TempDir(), "models") + source := filepath.Join(t.TempDir(), "tiny-Q4_K_M.gguf") + if err := os.WriteFile(source, []byte("GGUFfixture"), 0600); err != nil { + t.Fatal(err) + } + id, err := llamaImport(context.Background(), root, source) + if err != nil { + t.Fatal(err) + } + if id != "local/tiny-Q4_K_M:Q4_K_M" { + t.Fatal(id) + } + models, err := llamaCacheModels(root) + if err != nil || len(models) != 1 || models[0].ID != id { + t.Fatalf("models=%v err=%v", models, err) + } + if _, err := llamaImport(context.Background(), root, source); err == nil { + t.Fatal("duplicate overwrote model") + } + e := &Executor{} + st := &engineState{installDir: filepath.Dir(root)} + params, _ := json.Marshal(map[string]string{"model": id}) + if _, err := e.llamaModelAction(context.Background(), st, "delete_model", params); err != nil { + t.Fatal(err) + } + models, err = llamaCacheModels(root) + if err != nil || len(models) != 0 { + t.Fatalf("models=%v err=%v", models, err) + } + if _, err := os.Stat(source); err != nil { + t.Fatal("source file modified", err) + } + if _, err := llamaImport(context.Background(), root, source); err != nil { + t.Fatal("reimport after delete", err) + } +} + +func TestLlamaCacheIncompleteAndCancellation(t *testing.T) { + root := t.TempDir() + base := filepath.Join(root, "models--owner--repo") + if err := os.MkdirAll(filepath.Join(base, "refs"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(base, "refs", "main"), []byte(llamaTestCommit), 0600); err != nil { + t.Fatal(err) + } + snapshot := filepath.Join(base, "snapshots", llamaTestCommit) + if err := os.MkdirAll(snapshot, 0700); err != nil { + t.Fatal(err) + } + write := func(name string) { + t.Helper() + if err := os.WriteFile(filepath.Join(snapshot, name), []byte("GGUFfixture"), 0600); err != nil { + t.Fatal(err) + } + } + write("tiny-Q4_K_M-00001-of-00002.gguf") + write("mmproj-F16.gguf") + write("tiny-Q8_0.gguf.downloadInProgress") + models, err := llamaCacheModels(root) + if err != nil || len(models) != 0 { + t.Fatalf("incomplete listed: %v %v", models, err) + } + write("tiny-Q4_K_M-00002-of-00002.gguf") + models, err = llamaCacheModels(root) + if err != nil || len(models) != 1 || len(models[0].Files) != 2 { + t.Fatalf("complete missing: %v %v", models, err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + source := filepath.Join(t.TempDir(), "cancel-Q8_0.gguf") + if err := os.WriteFile(source, []byte("GGUFfixture"), 0600); err != nil { + t.Fatal(err) + } + if _, err := llamaImport(ctx, root, source); err != context.Canceled { + t.Fatalf("cancellation: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "models--local--cancel-Q8_0")); !os.IsNotExist(err) { + t.Fatal("canceled import published") + } + outside := filepath.Join(t.TempDir(), "other.gguf") + if err := os.WriteFile(outside, []byte("GGUF"), 0600); err != nil { + t.Fatal(err) + } + if _, err := llamaOwnedFile(root, outside); err == nil { + t.Fatal("outside path admitted") + } +} + +func TestLlamaCacheSharedBlobDeletion(t *testing.T) { + install := t.TempDir() + root := filepath.Join(install, "models") + base := filepath.Join(root, "models--owner--repo") + snapshot := filepath.Join(base, "snapshots", llamaTestCommit) + for _, dir := range []string{snapshot, filepath.Join(base, "refs"), filepath.Join(base, "blobs")} { + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(base, "refs", "main"), []byte(llamaTestCommit), 0600); err != nil { + t.Fatal(err) + } + blob := filepath.Join(base, "blobs", "content") + if err := os.WriteFile(blob, []byte("GGUFfixture"), 0600); err != nil { + t.Fatal(err) + } + for _, name := range []string{"tiny-Q4_K_M.gguf", "tiny-Q8_0.gguf"} { + if err := os.Symlink(blob, filepath.Join(snapshot, name)); err != nil { + t.Skipf("symlink fixture unavailable: %v", err) + } + } + e := &Executor{} + st := &engineState{installDir: install} + deleteModel := func(id string) { + t.Helper() + params, _ := json.Marshal(map[string]string{"model": id}) + if _, err := e.llamaModelAction(context.Background(), st, "delete_model", params); err != nil { + t.Fatal(err) + } + } + deleteModel("owner/repo:Q4_K_M") + if _, err := os.Stat(blob); err != nil { + t.Fatal("shared blob removed", err) + } + deleteModel("owner/repo:Q8_0") + if _, err := os.Stat(blob); !os.IsNotExist(err) { + t.Fatal("orphaned blob retained", err) + } +} diff --git a/services/nvpair-engine-manager/llamapath_windows_test.go b/services/nvpair-engine-manager/llamapath_windows_test.go new file mode 100644 index 00000000..0018c26c --- /dev/null +++ b/services/nvpair-engine-manager/llamapath_windows_test.go @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build windows + +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "golang.org/x/sys/windows" +) + +func TestLlamaWindowsShortAliasAndJunctionRefusal(t *testing.T) { + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + ptr, err := windows.UTF16PtrFromString(root) + if err != nil { + t.Fatal(err) + } + buffer := make([]uint16, 32768) + n, err := windows.GetShortPathName(ptr, &buffer[0], uint32(len(buffer))) + if err != nil { + t.Fatal(err) + } + if n == 0 || n >= uint32(len(buffer)) { + t.Fatal("short-path query failed") + } + alias := windows.UTF16ToString(buffer[:n]) + if strings.EqualFold(alias, root) { + t.Skip("filesystem supplied no distinct8.3 alias") + } + if err := validateLlamaOwnedPaths(alias); err != nil { + t.Fatalf("ordinary8.3 alias rejected: %v", err) + } + t.Log("exercised a distinct native8.3 alias") + + outside := t.TempDir() + marker := filepath.Join(outside, "keep.txt") + if err := os.WriteFile(marker, []byte("outside"), 0600); err != nil { + t.Fatal(err) + } + junction := filepath.Join(root, "models") + cmd := exec.Command("cmd.exe", "/c", "mklink", "/J", junction, outside) + configureSysProcAttr(cmd) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("create harmless junction fixture: %v: %s", err, output) + } + t.Cleanup(func() { _ = os.Remove(junction) }) + if err := validateLlamaOwnedPaths(alias); err == nil { + t.Fatal("junction escape accepted through8.3 alias") + } + if err := validateLlamaOwnedPaths(filepath.Join(junction, "missing-child")); err == nil { + t.Fatal("missing child hid ancestor junction") + } + if bytes, err := os.ReadFile(marker); err != nil || string(bytes) != "outside" { + t.Fatal("external marker changed") + } +} diff --git a/services/nvpair-engine-manager/llamatar.go b/services/nvpair-engine-manager/llamatar.go new file mode 100644 index 00000000..61ad2ff2 --- /dev/null +++ b/services/nvpair-engine-manager/llamatar.go @@ -0,0 +1,168 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "archive/tar" + "compress/gzip" + "context" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" +) + +func safeLlamaTarName(name string) bool { + if name == "" || path.IsAbs(name) || path.Clean(name) != name || strings.ContainsAny(name, "\\:\x00") { + return false + } + for _, part := range strings.Split(name, "/") { + if part == "." || part == ".." || strings.TrimRight(part, ". ") != part || !filepath.IsLocal(part) { + return false + } + } + return true +} + +// Official Intel Mac bundles contain a fixed root and versioned dylib links. +// Materialize links as regular files: no symlink survives into the managed tree. +func extractLlamaTar(ctx context.Context, archive, candidate, prefix string, remaining *int64, entries *int) error { + if !safeLlamaTarName(prefix) || strings.Contains(prefix, "/") { + return fmt.Errorf("unsafe llama tar prefix") + } + f, err := os.Open(archive) + if err != nil { + return err + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + return err + } + defer gz.Close() + tr := tar.NewReader(gz) + seen := map[string]bool{} + files := map[string]bool{} + links := map[string]string{} + write := func(name string, src io.Reader, size int64) error { + if size < 0 || size > *remaining { + return fmt.Errorf("llama archives exceed the uncompressed byte limit") + } + dest := filepath.Join(candidate, filepath.FromSlash(name)) + if err := validateLlamaPath(dest); err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(dest), 0700); err != nil { + return err + } + dst, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0700) + if err != nil { + return err + } + n, copyErr := io.Copy(llamaArchiveWriter{ctx, dst}, io.LimitReader(src, size+1)) + closeErr := dst.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + if n != size { + return fmt.Errorf("llama tar size mismatch") + } + *remaining -= n + files[name] = true + return nil + } + for { + if err := ctx.Err(); err != nil { + return err + } + h, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + *entries++ + if *entries > maxLlamaArchiveEntries { + return fmt.Errorf("llama archives exceed the entry limit") + } + full := strings.TrimSuffix(h.Name, "/") + if !safeLlamaTarName(full) || seen[full] { + return fmt.Errorf("unsafe or duplicate llama tar entry") + } + seen[full] = true + if full == prefix && h.Typeflag == tar.TypeDir { + continue + } + if !strings.HasPrefix(full, prefix+"/") { + return fmt.Errorf("llama tar entry outside declared prefix") + } + name := strings.TrimPrefix(full, prefix+"/") + switch h.Typeflag { + case tar.TypeReg, tar.TypeRegA: + if err := write(name, tr, h.Size); err != nil { + return err + } + case tar.TypeDir: + dest := filepath.Join(candidate, filepath.FromSlash(name)) + if err := validateLlamaPath(dest); err != nil { + return err + } + if err := os.MkdirAll(dest, 0700); err != nil { + return err + } + case tar.TypeSymlink: + if !safeLlamaTarName(h.Linkname) { + return fmt.Errorf("unsafe llama library link") + } + links[name] = path.Join(path.Dir(name), h.Linkname) + default: + return fmt.Errorf("unsupported llama tar entry type") + } + } + // Consume bounded padding/trailer so gzip integrity errors are not hidden by + // tar's earlier end marker. Extra compressed members share the same budget. + n, err := io.Copy(llamaArchiveWriter{ctx, io.Discard}, io.LimitReader(gz, *remaining+1)) + if err != nil { + return err + } + if n > *remaining { + return fmt.Errorf("llama archives exceed the uncompressed byte limit") + } + *remaining -= n + for len(links) > 0 { + progress := false + for name, target := range links { + if !files[target] { + continue + } + src, err := os.Open(filepath.Join(candidate, filepath.FromSlash(target))) + if err != nil { + return err + } + info, err := src.Stat() + if err == nil && !info.Mode().IsRegular() { + err = fmt.Errorf("llama link target is not a regular file") + } + if err == nil { + err = write(name, src, info.Size()) + } + src.Close() + if err != nil { + return err + } + delete(links, name) + progress = true + } + if !progress { + return fmt.Errorf("missing or cyclic llama library link") + } + } + return ctx.Err() +} diff --git a/services/nvpair-engine-manager/llamatar_test.go b/services/nvpair-engine-manager/llamatar_test.go new file mode 100644 index 00000000..6e20125a --- /dev/null +++ b/services/nvpair-engine-manager/llamatar_test.go @@ -0,0 +1,143 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "os" + "path/filepath" + "testing" +) + +type llamaTarEntry struct { + name, body, link string + kind byte +} + +func llamaTarFixture(t *testing.T, entries []llamaTarEntry) string { + t.Helper() + var data bytes.Buffer + gz := gzip.NewWriter(&data) + tw := tar.NewWriter(gz) + for _, e := range entries { + h := &tar.Header{Name: e.name, Typeflag: e.kind, Mode: 0700, Linkname: e.link} + if e.kind == tar.TypeReg { + h.Size = int64(len(e.body)) + } + if err := tw.WriteHeader(h); err != nil { + t.Fatal(err) + } + if e.body != "" { + if _, err := tw.Write([]byte(e.body)); err != nil { + t.Fatal(err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + file := filepath.Join(t.TempDir(), "official.tar.gz") + if err := os.WriteFile(file, data.Bytes(), 0600); err != nil { + t.Fatal(err) + } + return file +} +func TestLlamaTarOfficialLayoutAndLinkChain(t *testing.T) { + archive := llamaTarFixture(t, []llamaTarEntry{{name: "llama-b10826", kind: tar.TypeDir}, {name: "llama-b10826/llama", body: "app", kind: tar.TypeReg}, {name: "llama-b10826/libggml.dylib", link: "libggml.0.dylib", kind: tar.TypeSymlink}, {name: "llama-b10826/libggml.0.dylib", link: "libggml.0.23.0.dylib", kind: tar.TypeSymlink}, {name: "llama-b10826/libggml.0.23.0.dylib", body: "library", kind: tar.TypeReg}}) + stage := t.TempDir() + remaining := int64(100000) + count := 0 + if err := extractLlamaTar(context.Background(), archive, stage, "llama-b10826", &remaining, &count); err != nil { + t.Fatal(err) + } + for _, name := range []string{"libggml.dylib", "libggml.0.dylib", "libggml.0.23.0.dylib"} { + p := filepath.Join(stage, name) + info, err := os.Lstat(p) + if err != nil || !info.Mode().IsRegular() { + t.Fatalf("not a materialized regular library: %s", name) + } + data, _ := os.ReadFile(p) + if string(data) != "library" { + t.Fatal("library changed") + } + } + if count != 5 { + t.Fatalf("entries=%d", count) + } +} +func TestLlamaTarRejectsUnsafeOrIncompleteEntries(t *testing.T) { + for _, tc := range []struct { + name string + entries []llamaTarEntry + }{ + {"outside", []llamaTarEntry{{name: "other/llama", body: "x", kind: tar.TypeReg}}}, + {"traversal", []llamaTarEntry{{name: "llama-b10826/../escape", body: "x", kind: tar.TypeReg}}}, + {"link-escape", []llamaTarEntry{{name: "llama-b10826/lib", link: "../escape", kind: tar.TypeSymlink}}}, + {"link-absolute", []llamaTarEntry{{name: "llama-b10826/lib", link: "/tmp/escape", kind: tar.TypeSymlink}}}, + {"missing", []llamaTarEntry{{name: "llama-b10826/lib", link: "absent", kind: tar.TypeSymlink}}}, + {"cycle", []llamaTarEntry{{name: "llama-b10826/a", link: "b", kind: tar.TypeSymlink}, {name: "llama-b10826/b", link: "a", kind: tar.TypeSymlink}}}, + {"duplicate", []llamaTarEntry{{name: "llama-b10826/a", body: "x", kind: tar.TypeReg}, {name: "llama-b10826/a", body: "y", kind: tar.TypeReg}}}, + {"hardlink", []llamaTarEntry{{name: "llama-b10826/a", link: "b", kind: tar.TypeLink}}}, + } { + t.Run(tc.name, func(t *testing.T) { + remaining := int64(100000) + count := 0 + if err := extractLlamaTar(context.Background(), llamaTarFixture(t, tc.entries), t.TempDir(), "llama-b10826", &remaining, &count); err == nil { + t.Fatal("accepted unsafe tar") + } + }) + } +} +func TestLlamaTarLimitsCancellationAndExistingFiles(t *testing.T) { + archive := llamaTarFixture(t, []llamaTarEntry{{name: "llama-b10826/llama", body: "new", kind: tar.TypeReg}}) + for _, name := range []string{"bytes", "entries", "cancel", "collision", "gzip-trailer"} { + t.Run(name, func(t *testing.T) { + stage := t.TempDir() + remaining := int64(100000) + count := 0 + ctx := context.Background() + input := archive + switch name { + case "bytes": + remaining = 2 + case "entries": + count = maxLlamaArchiveEntries + case "cancel": + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + case "collision": + if err := os.WriteFile(filepath.Join(stage, "llama"), []byte("old"), 0600); err != nil { + t.Fatal(err) + } + case "gzip-trailer": + data, _ := os.ReadFile(archive) + input = filepath.Join(t.TempDir(), "bad.gz") + if err := os.WriteFile(input, data[:len(data)-5], 0600); err != nil { + t.Fatal(err) + } + } + err := extractLlamaTar(ctx, input, stage, "llama-b10826", &remaining, &count) + if err == nil { + t.Fatal("expected refusal") + } + if name == "cancel" && !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if name == "collision" { + data, _ := os.ReadFile(filepath.Join(stage, "llama")) + if string(data) != "old" { + t.Fatal("overwrote existing file") + } + } + }) + } +} diff --git a/services/nvpair-engine-manager/llamaupstream.go b/services/nvpair-engine-manager/llamaupstream.go new file mode 100644 index 00000000..4e3e3301 --- /dev/null +++ b/services/nvpair-engine-manager/llamaupstream.go @@ -0,0 +1,217 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" +) + +// The version endpoint is the only unpinned llama source. The manifest cannot +// supply a latest URL or relax the ordinary fetch pin rules. +// +// The installer is pinned by commit and verified against llamaInstallerSHA256 +// before it runs. A branch ref would be mutable, and this script is executed: +// upstream could change the bytes between review and any user's install, and +// recording the hash afterwards gates nothing, because by then it has already +// run as the engine-manager user. Verifying first means the worst an upstream +// change can do is fail the download, which falls back to the pinned CUDA +// archives below. +// +// Pinning the script does not pin the engine: the build installed is whatever +// llamaLatestVersionURL resolves to, passed in as LLAMA_VERSION. Bumping the +// pin is a deliberate edit of both constants together. +const llamaInstallerCommit = "4ee224e8b16ad6c48be85609e74dd8b1e8d740ae" +const llamaLatestInstallerURL = "https://raw.githubusercontent.com/ggml-org/llama-install.sh/" + + llamaInstallerCommit + "/install.ps1" +const llamaLatestVersionURL = "https://huggingface.co/buckets/ggml-org/install.sh/resolve/latest" +const maxLlamaInstallerBytes = 1 << 20 + +// Digest of the pinned commit's install.ps1. A var only so the fixtures can +// substitute their own script, which stands in for an installer they cannot +// run; nothing in the shipped binary assigns to it. +var llamaInstallerSHA256 = "455084203db0c864f4eb218bc82792b4304458a96211c385275d8337a5049851" + +// Overridable by serial socket-free tests; covers resolution through validation. +var llamaUpstreamAttemptTimeout = 3 * time.Minute + +var llamaBuildTag = regexp.MustCompile(`^b([1-9][0-9]*)$`) +var llamaBuildOutput = regexp.MustCompile(`(?m)^(?:b([1-9][0-9]*)-[a-zA-Z0-9]+|version: [^\r\n]*\(build ([1-9][0-9]*), commit [a-zA-Z0-9]+\))`) +var llamaCUDADevice = regexp.MustCompile(`(?m)^\s*CUDA[0-9]+:\s*\S`) + +func llamaBuildNumber(version string) int64 { + m := llamaBuildOutput.FindStringSubmatch(version) + if m == nil { + return 0 + } + n, _ := strconv.ParseInt(m[1]+m[2], 10, 64) + return n +} + +func (e *Executor) latestLlamaBuild(ctx context.Context) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, llamaLatestVersionURL, nil) + if err != nil { + return "", err + } + resp, err := e.client.Do(req) + if err != nil { + return "", fmt.Errorf("resolve latest official llama build: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("resolve latest official llama build: HTTP %d", resp.StatusCode) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, 65)) + if err != nil { + return "", err + } + build := strings.TrimSpace(string(data)) + if len(data) > 64 || !llamaBuildTag.MatchString(build) { + return "", errors.New("latest official llama build response is invalid") + } + if n, err := strconv.ParseInt(build[1:], 10, 64); err != nil || n == 0 { + return "", errors.New("latest official llama build number is invalid") + } + return build, ctx.Err() +} + +// Checks a staged candidate without a server or model. The candidate must +// report the exact numeric build that was selected for it. +func (e *Executor) validateLlamaCUDA(ctx context.Context, st *engineState, candidate string, expected int64) (map[string]any, string, error) { + return e.validateLlamaARMApp(ctx, st, candidate, expected, true) +} + +func (e *Executor) validateLlamaARMApp(ctx context.Context, st *engineState, candidate string, expected int64, requireCUDA bool) (map[string]any, string, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + env, err := childEnv(st) + if err != nil { + return nil, "", err + } + bin := filepath.Join(candidate, llamaExecutable()) + version, err := e.runCommandOutput(ctx, []string{bin, "version"}, env) + if err != nil { + return nil, "", fmt.Errorf("validate llama version: %w", err) + } + build := llamaBuildNumber(version) + if build == 0 || expected != build { + return nil, "", errors.New("downloaded llama version did not match the selected build") + } + identity := map[string]any{"version": strings.TrimSpace(version), "selected_build": fmt.Sprintf("b%d", build), "platform": "windows/arm64"} + licenses, err := e.runCommandOutput(ctx, []string{bin, "licenses"}, env) + if err != nil || strings.TrimSpace(licenses) == "" { + return identity, "", errors.New("llama third-party licenses could not be read") + } + if requireCUDA { + devices, err := e.runCommandOutput(ctx, []string{bin, "cli", "--list-devices"}, env) + if err != nil || !llamaCUDADevice.MatchString(devices) { + return identity, "", errors.New("llama did not report a CUDA device; check the NVIDIA driver and retry") + } + } + f, err := os.Open(bin) + if err != nil { + return identity, "", err + } + h := sha256.New() + _, err = io.Copy(llamaArchiveWriter{ctx, h}, f) + f.Close() + if err != nil { + return identity, "", err + } + identity["binary_sha256"], identity["cuda_device_verified"] = hex.EncodeToString(h.Sum(nil)), requireCUDA + return identity, licenses, ctx.Err() +} + +func (e *Executor) stageLlamaUpstream(ctx context.Context, st *engineState, stage string) (candidate string, provenance map[string]any, err error) { + ctx, cancel := context.WithTimeout(ctx, llamaUpstreamAttemptTimeout) + defer cancel() + // Command helpers include exit diagnostics but do not preserve cancellation. + defer func() { + if ctx.Err() != nil { + err = ctx.Err() + } + }() + provenance = map[string]any{"installer_url": llamaLatestInstallerURL, "version_url": llamaLatestVersionURL, "source": "official-upstream", "installer_provenance": "HTTPS official source, pinned by commit and verified against a prequalified SHA256 before execution"} + build, err := e.latestLlamaBuild(ctx) + if err != nil { + return "", provenance, err + } + provenance["selected_build"] = build + selectedBuild, _ := strconv.ParseInt(build[1:], 10, 64) + if selectedBuild < 10826 { + return "", provenance, errors.New("latest upstream build is older than supported CUDA fallback b10826") + } + env, err := llamaInstallerEnv(st, stage) + if err != nil { + return "", provenance, err + } + env["LLAMA_VERSION"], env["SKIP_CUDA"], env["SKIP_VULKAN"] = build, "", "1" + if err = os.MkdirAll(stage, 0700); err != nil { + return "", provenance, err + } + // SHA256 makes downloadLimited reject a mismatch before the script reaches + // disk, so the verification happens ahead of runCommand below. + script, err := e.downloadLimited(ctx, "llamacpp", &Fetch{URL: llamaLatestInstallerURL, SHA256: llamaInstallerSHA256}, maxLlamaInstallerBytes) + if err != nil { + return "", provenance, err + } + defer os.Remove(script) + provenance["installer_sha256"] = llamaInstallerSHA256 + e.emitInstallProgress("llamacpp", "installing", -1) + if err = e.runCommand(ctx, llamaInstallerArgs("windows", script), env); err != nil { + return "", provenance, fmt.Errorf("official latest llama installer: %w", err) + } + candidate = filepath.Join(stage, "llama-app") + identity, licenses, err := e.validateLlamaCUDA(ctx, st, candidate, selectedBuild) + if err != nil { + return "", provenance, err + } + for k, v := range identity { + provenance[k] = v + } + err = os.WriteFile(filepath.Join(candidate, "THIRD-PARTY-LICENSES.txt"), []byte(licenses), 0600) + return candidate, provenance, err +} + +// prepareLlamaUpstream stages the latest official CUDA build first and, when +// that attempt fails for any reason other than the caller's own cancellation, +// stages the checksum-pinned CUDA archives instead. The candidate it returns is +// always a complete, validated runtime; the caller promotes it. +func (e *Executor) prepareLlamaUpstream(ctx context.Context, st *engineState, stage string) (string, map[string]any, error) { + candidate, provenance, primaryErr := e.stageLlamaUpstream(ctx, st, filepath.Join(stage, "upstream")) + if err := ctx.Err(); err != nil { + return "", nil, err // A parent cancellation never authorizes a fallback. + } + if primaryErr == nil { + return candidate, provenance, nil + } + e.emitInstallProgress("llamacpp", "fallback", -1) + candidate = filepath.Join(stage, "fallback") // Never mix failed script bytes with the ZIPs. + if err := e.stageLlamaArchives(ctx, st, candidate); err != nil { + return "", nil, fmt.Errorf("upstream attempt failed (%v); official CUDA fallback failed: %w", primaryErr, err) + } + fallback, licenses, err := e.validateLlamaCUDA(ctx, st, candidate, 10826) + if err != nil { + return "", nil, fmt.Errorf("upstream attempt failed (%v); official CUDA fallback validation failed: %w", primaryErr, err) + } + fallback["source"], fallback["fallback_reason"], fallback["upstream_attempt"] = "pinned-cuda-archives", primaryErr.Error(), provenance + fallback["archives"], fallback["recipe_sha256"] = st.plat.Install.Archives, llamaArchiveRecipeHash(st.plat.Install.Archives) + if err = os.WriteFile(filepath.Join(candidate, "THIRD-PARTY-LICENSES.txt"), []byte(licenses), 0600); err != nil { + return "", nil, err + } + return candidate, fallback, ctx.Err() +} diff --git a/services/nvpair-engine-manager/llamaupstream_test.go b/services/nvpair-engine-manager/llamaupstream_test.go new file mode 100644 index 00000000..e583039a --- /dev/null +++ b/services/nvpair-engine-manager/llamaupstream_test.go @@ -0,0 +1,558 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "sync" + "testing" + "time" +) + +type llamaRuntimeFixture struct { + Version string `json:"version"` + Licenses string `json:"licenses"` + Devices string `json:"devices"` + DelayCommand string `json:"delay_command,omitempty"` + DelayMS int `json:"delay_ms,omitempty"` +} + +func cudaLlamaFixture(build int) llamaRuntimeFixture { + return llamaRuntimeFixture{Version: fmt.Sprintf("version: 0.4.0-dev (build %d, commit fixture)", build), Licenses: "fixture third-party license", Devices: "Available devices:\n CUDA0: Fixture NVIDIA device"} +} + +type llamaUpstreamFixture struct { + e *Executor + st *engineState + requests map[string]int + latest string + script string + fail string + onScript func() +} + +// useScript serves script as the pinned installer. The fixture stands in for an +// installer these tests cannot run, so the expected digest has to follow the +// substituted bytes; every other caller of the real URL still gets the pin. +func (f *llamaUpstreamFixture) useScript(t *testing.T, script string) { + t.Helper() + before := llamaInstallerSHA256 + t.Cleanup(func() { llamaInstallerSHA256 = before }) + f.script = script + sum := sha256.Sum256([]byte(script)) + llamaInstallerSHA256 = hex.EncodeToString(sum[:]) +} + +func newLlamaUpstreamFixture(t *testing.T, primary llamaRuntimeFixture) *llamaUpstreamFixture { + t.Helper() + if runtime.GOOS != "windows" { + t.Skip("Windows PowerShell fixture; other-platform pin tests are portable") + } + reg := NewRegistry() + mf, _ := buildRegistry("").Get("llamacpp") + mf.Platforms[runtime.GOOS+"/"+runtime.GOARCH] = mf.Platforms["windows/arm64"] + reg.engines["llamacpp"] = mf + e := NewExecutor(reg, NewReporter(nil), nil, t.TempDir()) + st, err := e.state("llamacpp") + if err != nil { + t.Fatal(err) + } + // Policy fixtures use commands and injected HTTP bodies only. Public Install + // also reconciles presence; leaving the vendor readiness probe here would + // cause a real loopback dial outside the injected HTTP transport. + st.plat.Runtime.Ready, st.plat.Runtime.Health = nil, nil + // This existing suite exercises the CUDA-required policy, not host inventory. + st.plat.Install.CPUFetch = nil + fixtureSource := filepath.Join(t.TempDir(), "primary.json") + data, _ := json.Marshal(primary) + if err := os.WriteFile(fixtureSource, data, 0600); err != nil { + t.Fatal(err) + } + // The manifest no longer declares runtime env (childEnv injects the owned cache paths). + if st.plat.Runtime.Env == nil { + st.plat.Runtime.Env = map[string]string{} + } + st.plat.Runtime.Env["FAKE_LLAMA_BIN_SOURCE"] = fakeEngineBin + st.plat.Runtime.Env["FAKE_LLAMA_FIXTURE_SOURCE"] = fixtureSource + f := &llamaUpstreamFixture{e: e, st: st, requests: map[string]int{}, latest: "b10900"} + f.useScript(t, `$ErrorActionPreference = 'Stop' +if ($env:LLAMA_VERSION -ne 'b10900' -or $env:SKIP_INSTALL -ne '1' -or $env:SKIP_VULKAN -ne '1' -or -not [string]::IsNullOrEmpty($env:SKIP_CUDA)) { throw 'installer env mismatch' } +$stagePath = Join-Path $env:USERPROFILE 'llama-app' +New-Item -ItemType Directory -Path $stagePath -Force | Out-Null +Copy-Item -LiteralPath $env:FAKE_LLAMA_BIN_SOURCE -Destination (Join-Path $stagePath 'llama.exe') +Copy-Item -LiteralPath $env:FAKE_LLAMA_FIXTURE_SOURCE -Destination (Join-Path $stagePath '.llama-fixture.json') +Set-Content -LiteralPath (Join-Path $stagePath 'primary-only.txt') -Value 'primary' +`) + bin, err := os.ReadFile(fakeEngineBin) + if err != nil { + t.Fatal(err) + } + fallbackJSON, _ := json.Marshal(cudaLlamaFixture(10826)) + archive := func(files map[string][]byte) []byte { + var buf bytes.Buffer + z := zip.NewWriter(&buf) + for name, body := range files { + h := &zip.FileHeader{Name: name, Method: zip.Deflate} + h.SetMode(0700) + w, err := z.CreateHeader(h) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write(body); err != nil { + t.Fatal(err) + } + } + if err := z.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() + } + bundles := [][]byte{archive(map[string][]byte{"llama.exe": bin, ".llama-fixture.json": fallbackJSON, "fallback-only.txt": []byte("fallback")}), archive(map[string][]byte{"cuda.dll": []byte("fixture CUDA companion")})} + st.plat.Install.Archives = nil + for i, bundle := range bundles { + h := sha256.Sum256(bundle) + st.plat.Install.Archives = append(st.plat.Install.Archives, Fetch{URL: fmt.Sprintf("https://github.com/ggml-org/llama.cpp/releases/download/b10826/fixture-%d.zip", i), SHA256: hex.EncodeToString(h[:])}) + } + e.client = &http.Client{Transport: llamaFixtureTransport(func(r *http.Request) (*http.Response, error) { + url := r.URL.String() + f.requests[url]++ + var body []byte + switch url { + case llamaLatestVersionURL: + if f.fail == "version" { + return nil, errors.New("fixture latest unavailable") + } + body = []byte(f.latest) + case llamaLatestInstallerURL: + if f.onScript != nil { + f.onScript() + } + if f.fail == "download" { + return nil, errors.New("fixture script unavailable") + } + if f.fail == "timeout" { + <-r.Context().Done() + return nil, r.Context().Err() + } + body = []byte(f.script) + default: + if f.fail == "both" { + return nil, errors.New("fixture fallback unavailable") + } + for i, fetch := range st.plat.Install.Archives { + if fetch.URL == url { + body = bundles[i] + } + } + if body == nil { + return nil, errors.New("unexpected fixture request") + } + } + return &http.Response{StatusCode: http.StatusOK, ContentLength: -1, Body: io.NopCloser(bytes.NewReader(body))}, nil + })} + return f +} + +func (f *llamaUpstreamFixture) seed(t *testing.T, spec llamaRuntimeFixture, receipt map[string]any) { + t.Helper() + dir := filepath.Join(f.st.installDir, "runtime") + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + bin, err := os.ReadFile(fakeEngineBin) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, llamaExecutable()), bin, 0700); err != nil { + t.Fatal(err) + } + data, _ := json.Marshal(spec) + if err := os.WriteFile(filepath.Join(dir, ".llama-fixture.json"), data, 0600); err != nil { + t.Fatal(err) + } + if receipt != nil { + h := sha256.Sum256(bin) + receipt["binary_sha256"] = hex.EncodeToString(h[:]) + if err := writeJSONAtomic(filepath.Join(dir, "pair-install.json"), receipt); err != nil { + t.Fatal(err) + } + } + if err := os.MkdirAll(filepath.Join(f.st.installDir, "models"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(f.st.installDir, "models", "retained.gguf"), []byte("model"), 0600); err != nil { + t.Fatal(err) + } +} + +func (f *llamaUpstreamFixture) receipt(t *testing.T) map[string]any { + t.Helper() + data, err := os.ReadFile(filepath.Join(f.st.installDir, "runtime", "pair-install.json")) + if err != nil { + t.Fatal(err) + } + var result map[string]any + if err := json.Unmarshal(data, &result); err != nil { + t.Fatal(err) + } + return result +} + +func TestLlamaUpstreamInstallAndFallback(t *testing.T) { + for _, failure := range []string{"", "download", "version", "wrong-version", "cpu", "licenses", "timeout", "child-timeout", "oversized-script", "oversized-version", "invalid-version", "old-version"} { + t.Run(map[bool]string{true: "primary-success", false: failure}[failure == ""], func(t *testing.T) { + primary := cudaLlamaFixture(10900) + if failure == "wrong-version" { + primary.Version = cudaLlamaFixture(10901).Version + } + if failure == "cpu" { + primary.Devices = "CPU0: Fixture CPU\nCUDA support compiled but no devices" + } + if failure == "licenses" { + primary.Licenses = "" + } + if failure == "child-timeout" { + primary.DelayCommand, primary.DelayMS = "cli", 10000 + } + f := newLlamaUpstreamFixture(t, primary) + f.fail = failure + if failure == "oversized-script" { + f.useScript(t, strings.Repeat("#", maxLlamaInstallerBytes+1)) + } + if failure == "oversized-version" { + f.latest = "b10900" + strings.Repeat(" ", 64) + } + if failure == "invalid-version" { + f.latest = "b10900;untrusted" + } + if failure == "old-version" { + f.latest = "b10825" + } + if failure == "timeout" || failure == "child-timeout" { + before := llamaUpstreamAttemptTimeout + llamaUpstreamAttemptTimeout = 2 * time.Second + t.Cleanup(func() { llamaUpstreamAttemptTimeout = before }) + } + if err := f.e.installLlamaApp(context.Background(), f.st); err != nil { + t.Fatal(err) + } + receipt := f.receipt(t) + wantSource, wantBuild := "official-upstream", "b10900" + if failure != "" { + wantSource, wantBuild = "pinned-cuda-archives", "b10826" + } + if receipt["source"] != wantSource || receipt["selected_build"] != wantBuild || receipt["cuda_device_verified"] != true { + t.Fatalf("incorrect outcome: %+v", receipt) + } + if failure == "" { + if len(f.requests) != 2 || receipt["installer_sha256"] != llamaInstallerSHA256 { + t.Fatal("primary fetched fallback or lacks the verified script hash") + } + } else { + if receipt["fallback_reason"] == "" || receipt["upstream_attempt"] == nil { + t.Fatal("fallback did not record attempted upstream provenance") + } + wantReason := map[string]string{"wrong-version": "selected build", "cpu": "CUDA device", "licenses": "licenses", "timeout": "deadline exceeded", "child-timeout": "deadline exceeded", "oversized-script": "1048576-byte limit", "oversized-version": "response is invalid", "invalid-version": "response is invalid", "old-version": "older than supported"}[failure] + if wantReason != "" && !strings.Contains(receipt["fallback_reason"].(string), wantReason) { + t.Fatalf("fallback was not caused by %s: %v", failure, receipt["fallback_reason"]) + } + if _, err := os.Stat(filepath.Join(f.st.installDir, "runtime", "primary-only.txt")); !os.IsNotExist(err) { + t.Fatal("primary bytes merged into fallback") + } + } + }) + } +} + +// retainModels seeds a model file beneath the install root without a runtime: +// an earlier uninstall retains models, so a later install, failure, or +// cancellation must leave them exactly as found. +func (f *llamaUpstreamFixture) retainModels(t *testing.T) string { + t.Helper() + model := filepath.Join(f.st.installDir, "models", "retained.gguf") + if err := os.MkdirAll(filepath.Dir(model), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(model, []byte("model"), 0600); err != nil { + t.Fatal(err) + } + return model +} + +// assertNothingPromoted checks the outcome every non-success install shares: +// neither runtime slot exists, the engine is not detected, and models survive. +func (f *llamaUpstreamFixture) assertNothingPromoted(t *testing.T, model string) { + t.Helper() + for _, slot := range []string{"runtime", "previous"} { + if _, err := os.Stat(filepath.Join(f.st.installDir, slot)); !os.IsNotExist(err) { + t.Fatalf("install left a %s slot", slot) + } + } + if installed, err := f.e.Detect("llamacpp"); err != nil || installed { + t.Fatalf("install reported as installed: %v %v", installed, err) + } + if data, err := os.ReadFile(model); err != nil || string(data) != "model" { + t.Fatalf("install changed models: %v", err) + } +} + +func TestLlamaUpstreamInstallFailsClosedWhenBothSourcesFail(t *testing.T) { + f := newLlamaUpstreamFixture(t, cudaLlamaFixture(10900)) + f.fail = "both" + f.useScript(t, "throw 'fixture primary failed'") + model := f.retainModels(t) + err := f.e.installLlamaApp(context.Background(), f.st) + if err == nil || !strings.Contains(err.Error(), "upstream attempt failed") || !strings.Contains(err.Error(), "official CUDA fallback failed") { + t.Fatalf("both-fail outcome did not name both sources: %v", err) + } + if f.requests[llamaLatestInstallerURL] != 1 { + t.Fatal("install skipped the upstream attempt") + } + f.assertNothingPromoted(t, model) + if matches, _ := filepath.Glob(filepath.Join(f.st.installDir, ".llama-install-*")); len(matches) != 1 { + t.Fatalf("failed stage was not retained for diagnosis: %v", matches) + } + found := false + for _, se := range f.e.Errors() { + if se.ID == installFailedID("llamacpp") && se.Operation == "install" { + found = true + } + } + if !found { + t.Fatal("both-fail install did not report an install failure") + } +} + +// A parent cancellation at any point of a fresh install, including after the +// candidate is fully validated, must end the install cancelled without a +// fallback attempt and without promoting anything into the runtime slot. +func TestLlamaUpstreamParentCancelDoesNotPromote(t *testing.T) { + for _, point := range []string{"download", "execution", "validation", "before-promotion"} { + t.Run(point, func(t *testing.T) { + primary := cudaLlamaFixture(10900) + if point == "validation" { + primary.DelayCommand, primary.DelayMS = "cli", 10000 + } + f := newLlamaUpstreamFixture(t, primary) + model := f.retainModels(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var blocked chan bool + if point == "download" { + f.onScript = cancel + } else if point == "execution" || point == "validation" { + if point == "execution" { + f.useScript(t, f.script+"Set-Content -LiteralPath (Join-Path $stagePath '.llama-delay-started') -Value 'installer'\nStart-Sleep -Seconds 10\n") + } + blocked = make(chan bool, 1) + go func() { + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + deadline := time.NewTimer(10 * time.Second) + defer deadline.Stop() + for { + select { + case <-ctx.Done(): + blocked <- false + return + case <-deadline.C: + cancel() + blocked <- false + return + case <-ticker.C: + matches, _ := filepath.Glob(filepath.Join(f.st.installDir, ".llama-install-*", "upstream", "llama-app", ".llama-delay-started")) + if len(matches) > 0 { + cancel() + blocked <- true + return + } + } + } + }() + } else { + f.e.emit = func(method string, value any) { + if method == "engine:install-progress" && value.(map[string]any)["stage"] == "installing" { + // The second installing event occurs after the complete primary + // candidate is validated, immediately before receipt/promotion. + if matches, _ := filepath.Glob(filepath.Join(f.st.installDir, ".llama-install-*", "upstream", "llama-app", "THIRD-PARTY-LICENSES.txt")); len(matches) > 0 { + cancel() + } + } + } + } + err := f.e.installLlamaApp(ctx, f.st) + cancel() + if blocked != nil && !<-blocked { + t.Fatal("did not reach blocked subprocess before parent cancellation") + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("parent cancellation lost: %v", err) + } + if len(f.requests) != 2 { + t.Fatal("cancelled parent attempted fallback") + } + f.assertNothingPromoted(t, model) + }) + } +} + +// Stop and shutdown cancel an in-flight public Install through the mutation +// cancel it registers. When that cancellation lands after the candidate is +// complete but before promotion, the install must still end cancelled with an +// empty runtime slot, Stop must not stay blocked behind it, and admission must +// reopen afterwards. +func TestLlamaInstallStoppedBeforePromotionDoesNotPromote(t *testing.T) { + for _, reason := range []string{"operation", "stop", "shutdown"} { + t.Run(reason, func(t *testing.T) { + f := newLlamaUpstreamFixture(t, cudaLlamaFixture(10900)) + model := f.retainModels(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cancelled := make(chan struct{}) + stopped := make(chan error, 1) + var once sync.Once + f.e.emit = func(method string, value any) { + if method != "engine:install-progress" || value.(map[string]any)["stage"] != "installing" { + return + } + // The second installing event occurs after the complete primary + // candidate is validated, immediately before receipt/promotion. + if matches, _ := filepath.Glob(filepath.Join(f.st.installDir, ".llama-install-*", "upstream", "llama-app", "THIRD-PARTY-LICENSES.txt")); len(matches) == 0 { + return + } + once.Do(func() { + if reason == "operation" { + cancel() + close(cancelled) + return + } + // Install registered its cancel under st.mu. Observe the exact + // moment Stop/StopAll invokes it, then let the install proceed + // to its pre-promotion check with the cancellation in effect. + f.st.mu.Lock() + registered := f.st.mutationCancel + f.st.mutationCancel = func() { registered(); close(cancelled) } + f.st.mu.Unlock() + go func() { + if reason == "shutdown" { + f.e.StopAll() + stopped <- nil + return + } + stopped <- f.e.Stop("llamacpp") + }() + <-cancelled + }) + } + err := f.e.Install(ctx, "llamacpp") + if !errors.Is(err, context.Canceled) { + t.Fatalf("%s: cancellation lost: %v", reason, err) + } + select { + case <-cancelled: + default: + t.Fatal("install finished before reaching the pre-promotion point") + } + if reason != "operation" { + select { + case err := <-stopped: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("Stop blocked behind the cancelled install") + } + } + if len(f.requests) != 2 { + t.Fatal("cancelled install attempted fallback") + } + f.assertNothingPromoted(t, model) + f.st.mu.Lock() + pending, registered := f.st.stopPending, f.st.mutationCancel != nil + f.st.mu.Unlock() + if pending != 0 || registered { + t.Fatal("cancelled install left admission blocked or its cancel registered") + } + }) + } +} + +// The upstream installer is the one artifact PAIR executes rather than merely +// unpacks, so a branch ref here would let upstream change what runs between +// review and a user's install. Guarded on every platform: the Windows fixtures +// above skip elsewhere, and they substitute this digest anyway. +func TestUpstreamInstallerIsPinnedAndVerified(t *testing.T) { + if !regexp.MustCompile(`/llama-install\.sh/[0-9a-f]{40}/`).MatchString(llamaLatestInstallerURL) { + t.Fatalf("installer URL is not pinned to a commit: %s", llamaLatestInstallerURL) + } + if !strings.Contains(llamaLatestInstallerURL, llamaInstallerCommit) { + t.Fatalf("installer URL does not carry llamaInstallerCommit: %s", llamaLatestInstallerURL) + } + if !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(llamaInstallerSHA256) { + t.Fatalf("installer digest is not a SHA256: %q", llamaInstallerSHA256) + } +} + +func TestLlamaUpstreamRegistryAndOtherPlatformPins(t *testing.T) { + mf, _ := buildRegistry("").Get("llamacpp") + for key, p := range mf.Platforms { + if key == "windows/arm64" { + if !p.Install.UpstreamFirst || p.Install.Fetch != nil || len(p.Install.Archives) != 2 { + t.Fatal("Windows ARM64 recipe lost its bounded opt-in/fallback") + } + } else if key == "darwin/amd64" { + if p.Install.UpstreamFirst || p.Install.Fetch != nil || p.Install.ArchiveRoot != "llama-b10826" || len(p.Install.Archives) != 1 || + p.Install.Archives[0].URL != "https://github.com/ggml-org/llama.cpp/releases/download/b10826/llama-b10826-bin-macos-x64.tar.gz" || + p.Install.Archives[0].SHA256 != "adcd2066b2a1a3d8e774e36c8f97d166defccd01d947d9e39667b0435e8361b0" { + t.Fatal("Intel Mac recipe lost its exact pinned CPU archive/root") + } + } else if p.Install.UpstreamFirst || p.Install.Fetch == nil || len(p.Install.Fetch.SHA256) != 64 || !strings.Contains(p.Install.Fetch.URL, "27a82f3a6e0f259f88c2c31cd6b20d858a975f27") { + t.Fatalf("%s lost its existing pin", key) + } + } + for _, mutation := range []string{"other-platform", "other-driver", "one-archive", "unpinned-archive", "unpinned-fetch", "custom-script"} { + t.Run(mutation, func(t *testing.T) { + fresh, _ := buildRegistry("").Get("llamacpp") + p := fresh.Platforms["windows/arm64"] + key := "windows/arm64" + switch mutation { + case "other-platform": + key = "linux/arm64" + case "other-driver": + p.Install.Driver = "" + case "one-archive": + p.Install.Archives = p.Install.Archives[:1] + case "unpinned-archive": + p.Install.Archives[0].SHA256 = "" + case "unpinned-fetch": + p.Install.Archives = nil + p.Install.Fetch = &Fetch{URL: llamaLatestInstallerURL} + case "custom-script": + p.Install.Script = []string{"powershell", "untrusted"} + } + if err := p.validate(key); err == nil { + t.Fatal("invalid latest recipe accepted") + } + }) + } + for _, version := range []string{"b108260-fixture", "version: 0.4 (build 108260, commit fixture)"} { + if llamaBuildNumber(version) != 108260 { + t.Fatal("numeric build comparison truncated a prefix") + } + } +} diff --git a/services/nvpair-engine-manager/loadedwatch.go b/services/nvpair-engine-manager/loadedwatch.go index 16e8bfae..f235bb29 100644 --- a/services/nvpair-engine-manager/loadedwatch.go +++ b/services/nvpair-engine-manager/loadedwatch.go @@ -94,6 +94,15 @@ func (e *Executor) sweepLoaded(ctx context.Context, prevLoaded map[string][]stri for name, ld := range res.LoadedByEngine { next[name] = ld } + // llama routing requires observed residency. A missed observation must + // publish unknown instead of leaving the last loaded set looking current. + if _, wasKnown := prevLoaded["llamacpp"]; wasKnown { + if _, known := res.LoadedByEngine["llamacpp"]; !known { + changed = append(changed, "llamacpp") + delete(next, "llamacpp") + sort.Strings(changed) + } + } return changed, next, res } diff --git a/services/nvpair-engine-manager/main.go b/services/nvpair-engine-manager/main.go index 4a9b5765..c0648054 100644 --- a/services/nvpair-engine-manager/main.go +++ b/services/nvpair-engine-manager/main.go @@ -84,6 +84,11 @@ func main() { reporter := NewReporter(codec) emit := func(method string, params any) { _ = codec.Notify(method, params) } exec := NewExecutor(reg, reporter, emit, installBase) + modelBaseDir, err := appdir.ModelsDir() + if err != nil { + log.Fatalf("resolve persistent model directory: %v", err) + } + exec.modelBaseDir = modelBaseDir if err := exec.SetReservedPort(*reservedPort); err != nil { log.Fatalf("invalid --reserved-port: %v", err) } diff --git a/services/nvpair-engine-manager/manager.go b/services/nvpair-engine-manager/manager.go index e12f62dc..7c4f581b 100644 --- a/services/nvpair-engine-manager/manager.go +++ b/services/nvpair-engine-manager/manager.go @@ -274,6 +274,7 @@ func (m *Manager) handleMessage(ctx context.Context, msg *Message) { case "engine:remote-get-installed", "engine:remote-install", "engine:remote-pull-model", "engine:remote-load-model", "engine:remote-unload-model", "engine:remote-delete-model", + "engine:remote-cancel-pull", "engine:remote-start", "engine:remote-stop": go m.runRemote(ctx, msg) @@ -373,6 +374,11 @@ func (m *Manager) runAction(ctx context.Context, msg *Message) { model := modelFromParams(p.Params) res, err = m.exec.PullModelStream(ctx, p.Engine, model, p.Params) if err != nil { + if errors.Is(err, context.Canceled) { + m.exec.emitPullProgress(ProgressEvent{Engine: p.Engine, Op: "pull", Stage: "cancelled", Percent: -1, Message: model}) + m.codec.RespondError(msg.ID, -32000, "model download cancelled") + return + } // A pull can fail after the client's synchronous call has already // timed out (long downloads), so the RPC error alone can't reach a // UI that stopped waiting. Emit one terminal engine:pull-progress diff --git a/services/nvpair-engine-manager/manifests/llamacpp.json b/services/nvpair-engine-manager/manifests/llamacpp.json new file mode 100644 index 00000000..aeb4933c --- /dev/null +++ b/services/nvpair-engine-manager/manifests/llamacpp.json @@ -0,0 +1,188 @@ +{ + "engine": "llamacpp", + "display_name": "llama.cpp", + "manifest_version": 1, + "install": { + "driver": "llama-app", + "mode": "user", + "fetch": { + "url": "https://raw.githubusercontent.com/ggml-org/llama-install.sh/27a82f3a6e0f259f88c2c31cd6b20d858a975f27/install.sh", + "sha256": "cccdfcbd1b55bf6003ac3037588c9f5b3b79aa0a75fe991e97bb218ccdb55e4d" + } + }, + "uninstall": { + "driver": "llama-app" + }, + "detect": [ + "{install_dir}/runtime/llama" + ], + "runtime": { + "mode": "process", + "bin": "{install_dir}/runtime/llama", + "cli": "{install_dir}/runtime/llama", + "args": [ + "serve", + "--no-models-autoload", + "--host", + "{host}", + "--port", + "{port}" + ], + "editable_launch": { + "fixed_args": [ + "serve", + "--no-models-autoload" + ], + "controls": [ + { + "flags": [ + "--port" + ], + "value": "{server.port}" + }, + { + "flags": [ + "--host" + ], + "value": "{server.host}" + } + ] + }, + "bind": "127.0.0.1", + "port": 8081, + "ready": { + "http": "http://127.0.0.1:{port}/models", + "status": 200, + "timeout_s": 120, + "identity": "llamacpp" + }, + "health": { + "http": "http://127.0.0.1:{port}/models", + "status": 200, + "interval_s": 5, + "identity": "llamacpp" + } + }, + "platforms": { + "windows/amd64": { + "detect": [ + "{install_dir}/runtime/llama.exe" + ], + "runtime": { + "bin": "{install_dir}/runtime/llama.exe", + "cli": "{install_dir}/runtime/llama.exe" + }, + "install": { + "fetch": { + "url": "https://raw.githubusercontent.com/ggml-org/llama-install.sh/27a82f3a6e0f259f88c2c31cd6b20d858a975f27/install.ps1", + "sha256": "455084203db0c864f4eb218bc82792b4304458a96211c385275d8337a5049851" + } + } + }, + "windows/arm64": { + "detect": [ + "{install_dir}/runtime/llama.exe" + ], + "runtime": { + "bin": "{install_dir}/runtime/llama.exe", + "cli": "{install_dir}/runtime/llama.exe" + }, + "install": { + "fetch": null, + "upstream_first": true, + "cpu_fetch": { + "url": "https://raw.githubusercontent.com/ggml-org/llama-install.sh/27a82f3a6e0f259f88c2c31cd6b20d858a975f27/install.ps1", + "sha256": "455084203db0c864f4eb218bc82792b4304458a96211c385275d8337a5049851" + }, + "archives": [ + { + "url": "https://github.com/ggml-org/llama.cpp/releases/download/b10826/llama-b10826-bin-win-cuda-13.4-arm64.zip", + "sha256": "b0b2b071d45ad14b85f935a9a4451e83ca82935dfc4b66327f3490e8e26263c9" + }, + { + "url": "https://github.com/ggml-org/llama.cpp/releases/download/b10826/cudart-llama-bin-win-cuda-13.4-arm64.zip", + "sha256": "5a40dc7c5fa3d0a80ceeba4f16f9e8d25d87bcf1399c9233588953c43436c33c" + } + ] + } + }, + "linux/amd64": {}, + "linux/arm64": {}, + "darwin/arm64": {}, + "darwin/amd64": { + "install": { + "fetch": null, + "archive_root": "llama-b10826", + "archives": [{ + "url": "https://github.com/ggml-org/llama.cpp/releases/download/b10826/llama-b10826-bin-macos-x64.tar.gz", + "sha256": "adcd2066b2a1a3d8e774e36c8f97d166defccd01d947d9e39667b0435e8361b0" + }] + } + } + }, + "actions": { + "list_models": { + "http": { + "method": "GET", + "path": "/models" + }, + "result": { + "array": "data", + "field": "id" + } + }, + "loaded_models": { + "http": { + "method": "GET", + "path": "/models" + }, + "result": { + "array": "data", + "field": "id", + "match": { + "field": "status.value", + "in": [ + "loaded" + ] + } + } + }, + "list_downloaded": { + "builtin": "llama-models", + "result": { + "array": "data", + "field": "id" + } + }, + "pull_model": { + "builtin": "llama-models" + }, + "import_model": { + "builtin": "llama-models" + }, + "delete_model": { + "builtin": "llama-models" + }, + "load_model": { + "http": { + "method": "POST", + "path": "/models/load" + } + }, + "unload_model": { + "http": { + "method": "POST", + "path": "/models/unload" + } + }, + "get_version": { + "cmd": [ + "{cli}", + "version" + ] + }, + "cancel_pull": { + "builtin": "llama-cancel" + } + } +} diff --git a/services/nvpair-engine-manager/modelpath.go b/services/nvpair-engine-manager/modelpath.go new file mode 100644 index 00000000..4f09d919 --- /dev/null +++ b/services/nvpair-engine-manager/modelpath.go @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "strings" +) + +// lookupField retains literal-key precedence and supports nested vendor state. +func lookupField(row map[string]json.RawMessage, field string) (json.RawMessage, bool) { + if value, ok := row[field]; ok { + return value, true + } + parts := strings.Split(field, ".") + for i, part := range parts { + value, ok := row[part] + if !ok { + return nil, false + } + if i == len(parts)-1 { + return value, true + } + var nested map[string]json.RawMessage + if json.Unmarshal(value, &nested) != nil { + return nil, false + } + row = nested + } + return nil, false +} diff --git a/services/nvpair-engine-manager/models.go b/services/nvpair-engine-manager/models.go index de89064e..72a27ddd 100644 --- a/services/nvpair-engine-manager/models.go +++ b/services/nvpair-engine-manager/models.go @@ -201,7 +201,7 @@ func extractStringsResult(raw json.RawMessage, spec *ActionResult) ([]string, bo if spec.Match != nil && !matchRow(el, spec.Match) { continue } - fv, ok := el[spec.Field] + fv, ok := lookupField(el, spec.Field) if !ok { continue } @@ -223,7 +223,7 @@ func extractStringsResult(raw json.RawMessage, spec *ActionResult) ([]string, bo // wrong-typed field fails the match, so a row we cannot classify is excluded // rather than counted as loaded. func matchRow(el map[string]json.RawMessage, m *ResultMatch) bool { - fv, ok := el[m.Field] + fv, ok := lookupField(el, m.Field) if !ok { return false } diff --git a/services/nvpair-engine-manager/proc_windows.go b/services/nvpair-engine-manager/proc_windows.go index c383c83d..1a8850a2 100644 --- a/services/nvpair-engine-manager/proc_windows.go +++ b/services/nvpair-engine-manager/proc_windows.go @@ -7,7 +7,10 @@ package main import ( "context" + "errors" + "fmt" "os/exec" + "path/filepath" "strconv" "syscall" "time" @@ -21,6 +24,33 @@ import ( // force-kills engine-manager on a timeout. const taskkillTimeout = 5 * time.Second +// Windows canonicalization expands ordinary8.3 aliases as well as reparse +// points. Inspect actual filesystem attributes instead of treating every name +// change as redirection. Check ancestors too, including above missing children. +func validateLlamaPath(path string) error { + for current := filepath.Clean(path); ; current = filepath.Dir(current) { + extended, err := llamaCachePath(current) + if err != nil { + return err + } + ptr, err := windows.UTF16PtrFromString(extended) + if err != nil { + return err + } + attrs, err := windows.GetFileAttributes(ptr) + if err != nil && !errors.Is(err, windows.ERROR_FILE_NOT_FOUND) && !errors.Is(err, windows.ERROR_PATH_NOT_FOUND) { + return err + } + if err == nil && attrs&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return fmt.Errorf("llama managed directory contains a reparse point; external data is left untouched") + } + if parent := filepath.Dir(current); parent == current { + break + } + } + return nil +} + // configureSysProcAttr hides the child's console window // (HideWindow + CREATE_NO_WINDOW), matching every other NVPAIR subprocess. func configureSysProcAttr(cmd *exec.Cmd) { diff --git a/services/nvpair-engine-manager/pull.go b/services/nvpair-engine-manager/pull.go index 6ffd8cca..d931a5ed 100644 --- a/services/nvpair-engine-manager/pull.go +++ b/services/nvpair-engine-manager/pull.go @@ -72,6 +72,14 @@ func (e *Executor) PullModelStream(ctx context.Context, engine, model string, pa ctx, cancel := context.WithTimeout(ctx, e.actionTimeout) defer cancel() + if act.Builtin == "llama-models" { + res, err := e.Action(ctx, engine, pullModelAction, params) + if err == nil { + e.reporter.clear(pullFailedID(engine, model)) + e.emitPullProgress(ProgressEvent{Engine: engine, Op: "pull", Stage: "done", Percent: 100, Message: model}) + } + return res, err + } // CLI action (e.g. lms get): no structured line progress; emit a start // marker and return the final result via the existing runner. diff --git a/services/nvpair-engine-manager/registry.go b/services/nvpair-engine-manager/registry.go index b6a79ad6..b4a55309 100644 --- a/services/nvpair-engine-manager/registry.go +++ b/services/nvpair-engine-manager/registry.go @@ -36,6 +36,7 @@ var allowedPlaceholders = map[string]bool{ "port": true, "download": true, "install_dir": true, + "model_dir": true, "models_dir": true, } @@ -71,8 +72,17 @@ type Platform struct { // download (fetch.sha256 set) is checksum-verified before its `run` // command executes; an unpinned fetch is HTTPS-only (see download). type Install struct { - Fetch *Fetch `json:"fetch,omitempty"` - Run []string `json:"run,omitempty"` + Driver string `json:"driver,omitempty"` + Fetch *Fetch `json:"fetch,omitempty"` + // UpstreamFirst opts only Windows ARM64 llama into the fixed official latest + // installer, retaining Archives as its checksum-qualified CUDA fallback. + UpstreamFirst bool `json:"upstream_first,omitempty"` + CPUFetch *Fetch `json:"cpu_fetch,omitempty"` // confirmed non-NVIDIA Windows ARM CPU path + ArchiveRoot string `json:"archive_root,omitempty"` // fixed prefix of the official Intel Mac tar + // Archives is the checksum-pinned official llama app and companion runtime + // bundle set for Windows ARM64, where the script distribution lacks CUDA. + Archives []Fetch `json:"archives,omitempty"` + Run []string `json:"run,omitempty"` // Script is an escape hatch for vendors that only ship a script // installer. It runs without checksum verification — strictly opt-in // and logged as unpinned. Prefer fetch+run whenever the vendor publishes @@ -88,7 +98,8 @@ type Install struct { // uninstaller (or removing its install dir). No download/checksum — // it only runs a local command. type Uninstall struct { - Run []string `json:"run"` + Driver string `json:"driver,omitempty"` + Run []string `json:"run"` } // Fetch is an engine download. SHA256, when set, pins it (verified @@ -167,6 +178,7 @@ func (r *Runtime) hasCustomLaunch() bool { // Probe is an HTTP or TCP reachability check. Exactly one of HTTP/TCP // should be set; HTTP wins if both are. type Probe struct { + Identity string `json:"identity,omitempty"` HTTP string `json:"http,omitempty"` // url template, e.g. "http://127.0.0.1:{port}/" TCP string `json:"tcp,omitempty"` // host:port template, e.g. "127.0.0.1:{port}" Status int `json:"status,omitempty"` // expected HTTP status (default 200) @@ -189,6 +201,7 @@ type StopSpec struct { // placeholders (e.g. {model}); an HTTP action sends params as the JSON // request body. type Action struct { + Builtin string `json:"builtin,omitempty"` Description string `json:"description,omitempty"` HTTP *ActionHTTP `json:"http,omitempty"` Cmd []string `json:"cmd,omitempty"` @@ -566,6 +579,9 @@ func (m *Manifest) Validate() error { return errors.New("at least one platforms entry is required") } for key, p := range m.Platforms { + if m.Engine != "llamacpp" && ((p.Install != nil && p.Install.Driver == "llama-app") || (p.Uninstall != nil && p.Uninstall.Driver == "llama-app")) { + return fmt.Errorf("platform %q: llama-app driver requires engine llamacpp", key) + } if !validPlatformKey(key) { return fmt.Errorf("platform key %q must be \"/\"", key) } @@ -574,6 +590,9 @@ func (m *Manifest) Validate() error { } } for name, a := range m.Actions { + if a.Builtin != "" && m.Engine != "llamacpp" { + return fmt.Errorf("action %q: llama builtin requires engine llamacpp", name) + } if err := a.validate(name); err != nil { return err } @@ -636,6 +655,31 @@ func (p *Platform) validate(key string) error { return fmt.Errorf("platform %q: runtime.mode %q invalid (want \"process\" or \"command\")", key, p.Runtime.Mode) } if p.Install != nil { + if p.Install.CPUFetch != nil && (key != "windows/arm64" || p.Install.Driver != "llama-app" || !p.Install.UpstreamFirst || p.Install.CPUFetch.SHA256 == "" || p.Install.CPUFetch.URL == "") { + return fmt.Errorf("platform %q: cpu_fetch requires the Windows ARM64 llama-app policy and a pinned fetch", key) + } + if p.Install.ArchiveRoot != "" && (key != "darwin/amd64" || p.Install.Driver != "llama-app" || p.Install.ArchiveRoot != "llama-b10826" || len(p.Install.Archives) != 1) { + return fmt.Errorf("platform %q: archive_root requires the pinned Intel Mac llama archive", key) + } + if p.Install.UpstreamFirst && (key != "windows/arm64" || p.Install.Driver != "llama-app" || len(p.Install.Archives) != 2) { + return fmt.Errorf("platform %q: upstream_first requires Windows ARM64 llama-app with two pinned fallback archives", key) + } + if len(p.Install.Archives) > 0 { + if p.Install.Driver != "llama-app" || (key != "windows/arm64" && key != "darwin/amd64") || p.Install.Fetch != nil || len(p.Install.Run) > 0 || len(p.Install.Script) > 0 { + return fmt.Errorf("platform %q: archives require the Windows ARM64 or Intel Mac llama-app driver without fetch/run/script", key) + } + for _, archive := range p.Install.Archives { + if strings.TrimSpace(archive.URL) == "" || archive.SHA256 == "" { + return fmt.Errorf("platform %q: every llama archive requires a URL and checksum", key) + } + } + } + if p.Install.Driver == "llama-app" && len(p.Install.Archives) == 0 && (p.Install.Fetch == nil || p.Install.Fetch.SHA256 == "" || len(p.Install.Run) > 0 || len(p.Install.Script) > 0) { + return fmt.Errorf("platform %q: llama-app requires pinned archives or a pinned fetch without custom run/script", key) + } + if p.Install.Driver != "" && p.Install.Driver != "llama-app" { + return fmt.Errorf("platform %q: unknown install driver", key) + } if len(p.Install.Script) > 0 && (p.Install.Fetch != nil || len(p.Install.Run) > 0) { return fmt.Errorf("platform %q: install.script is mutually exclusive with fetch/run (a script install cannot also be checksum-pinned)", key) } @@ -651,7 +695,10 @@ func (p *Platform) validate(key string) error { return fmt.Errorf("platform %q: install.mode %q invalid (want \"user\" or \"admin\")", key, p.Install.Mode) } } - if p.Uninstall != nil && len(p.Uninstall.Run) == 0 { + if p.Uninstall != nil && p.Uninstall.Driver != "" && p.Uninstall.Driver != "llama-app" { + return fmt.Errorf("platform %q: unknown uninstall driver", key) + } + if p.Uninstall != nil && len(p.Uninstall.Run) == 0 && p.Uninstall.Driver == "" { return fmt.Errorf("platform %q: uninstall.run is required when uninstall is present", key) } if err := validateProbe(key, "ready", p.Runtime.Ready); err != nil { @@ -671,6 +718,9 @@ func validateProbe(key, which string, p *Probe) error { if p == nil { return nil } + if p.Identity != "" && p.Identity != "llamacpp" { + return fmt.Errorf("platform %q: unknown probe identity %q", key, p.Identity) + } if strings.TrimSpace(p.HTTP) == "" && strings.TrimSpace(p.TCP) == "" { return fmt.Errorf("platform %q: runtime.%s must set either http or tcp", key, which) } @@ -682,6 +732,12 @@ func (a *Action) validate(name string) error { hasCmd := len(a.Cmd) > 0 hasRemovePath := a.RemovePath != nil kinds := 0 + if a.Builtin != "" { + if a.Builtin != "llama-models" && a.Builtin != "llama-cancel" { + return fmt.Errorf("action %q: unknown builtin", name) + } + kinds++ + } if hasHTTP { kinds++ } diff --git a/services/nvpair-engine-manager/remediation_test.go b/services/nvpair-engine-manager/remediation_test.go index 1999373b..3d320698 100644 --- a/services/nvpair-engine-manager/remediation_test.go +++ b/services/nvpair-engine-manager/remediation_test.go @@ -453,7 +453,7 @@ func TestBundledManifestsGolden(t *testing.T) { if err := reg.LoadFS(bundledManifests, "manifests"); err != nil { t.Fatalf("bundled manifests invalid: %v", err) } - for _, want := range []string{"ollama", "lmstudio"} { + for _, want := range []string{"ollama", "lmstudio", "llamacpp"} { m, ok := reg.Get(want) if !ok { t.Fatalf("missing bundled engine %q (have %v)", want, reg.Names()) diff --git a/services/nvpair-engine-manager/remote.go b/services/nvpair-engine-manager/remote.go index 5d296441..fce230b6 100644 --- a/services/nvpair-engine-manager/remote.go +++ b/services/nvpair-engine-manager/remote.go @@ -107,7 +107,7 @@ func (m *Manager) runRemote(ctx context.Context, msg *Message) { } m.codec.Respond(msg.ID, map[string]any{"opId": opID, "result": terminal.Result}) - case "engine:remote-load-model", "engine:remote-unload-model", "engine:remote-delete-model": + case "engine:remote-load-model", "engine:remote-unload-model", "engine:remote-delete-model", "engine:remote-cancel-pull": if p.Engine == "" { m.codec.RespondError(msg.ID, -32602, "engine is required") return @@ -122,6 +122,8 @@ func (m *Manager) runRemote(ctx context.Context, msg *Message) { path = controlUnloadPath case "engine:remote-delete-model": path = controlDeletePath + case "engine:remote-cancel-pull": + path = controlCancelPullPath } res, err := client.postJSON(ctx, path, p.Engine, modelActionRequest{Engine: p.Engine, Model: p.Model}) m.respondOrErr(msg, res, err) diff --git a/services/nvpair-engine-manager/remoteclient.go b/services/nvpair-engine-manager/remoteclient.go index 3a19bdf0..cb93548f 100644 --- a/services/nvpair-engine-manager/remoteclient.go +++ b/services/nvpair-engine-manager/remoteclient.go @@ -52,7 +52,7 @@ func waitsForEngineReadiness(path, engine string) bool { // controlDeletePath: LM Studio's delete_model declares restart_after, so the // peer replies only after the post-delete restart is ready. if path == controlLoadPath { - return engine == "ollama" + return engine == "ollama" || engine == "llamacpp" } return path == controlStartPath || path == controlDeletePath } diff --git a/services/nvpair-engine-manager/remoteclient_test.go b/services/nvpair-engine-manager/remoteclient_test.go index c4d16e67..b0c1b93d 100644 --- a/services/nvpair-engine-manager/remoteclient_test.go +++ b/services/nvpair-engine-manager/remoteclient_test.go @@ -83,6 +83,7 @@ func TestRemoteReadinessBudgetCoversEngineStartupAllowance(t *testing.T) { {controlStartPath, "ollama", true}, {controlDeletePath, "lmstudio", true}, {controlLoadPath, "ollama", true}, + {controlLoadPath, "llamacpp", true}, {controlLoadPath, "lmstudio", false}, {controlStopPath, "ollama", false}, {controlUnloadPath, "ollama", false}, diff --git a/services/nvpair-engine-manager/settings.go b/services/nvpair-engine-manager/settings.go index 660c0cf9..975d8bf1 100644 --- a/services/nvpair-engine-manager/settings.go +++ b/services/nvpair-engine-manager/settings.go @@ -97,7 +97,7 @@ func resolveRuntimeCommand(rt Runtime, index int, vars map[string]string) (launc // settings read probes ports, starts processes or writes configuration. func launchForState(st *engineState, port int) (launchCommand, error) { rt := st.plat.Runtime - vars := map[string]string{"host": effectiveBind(rt.Bind, ""), "port": strconv.Itoa(port), "install_dir": st.installDir} + vars := map[string]string{"host": effectiveBind(rt.Bind, ""), "port": strconv.Itoa(port), "install_dir": st.installDir, "model_dir": llamaModelDir(st)} if rt.CLI != "" { vars["cli"] = expandPath(rt.CLI) } @@ -122,7 +122,9 @@ func (e *Executor) launchStateLocked(engine string, st *engineState) settings.La result := settings.LaunchState{Engine: engine, ServerPort: st.plat.Runtime.Port, EffectivePort: st.port, Running: st.running, Adopted: st.adopted, Format: launchTextFormat} st.mu.Unlock() command, err := launchForState(st, result.ServerPort) - if err == nil { + // An engine without an editable launch has no argument text to show; asking + // for it would read as a display failure instead of "not supported". + if err == nil && st.plat.Runtime.EditableLaunch != nil { result.LaunchText, err = command.argumentText(st.plat.Runtime.EditableLaunch) } switch { @@ -260,7 +262,7 @@ func (e *Executor) previewLaunchLocked(st *engineState, request settings.Request host := effectiveBind(rt.Bind, "") vars := map[string]string{ "host": host, "port": strconv.Itoa(request.Settings.ServerPort), - "install_dir": st.installDir, "bin": base.Bin, "cli": expandPath(rt.CLI), + "install_dir": st.installDir, "model_dir": llamaModelDir(st), "bin": base.Bin, "cli": expandPath(rt.CLI), } if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() { return fail("PAIR-managed launch settings require a loopback bind.") @@ -282,6 +284,9 @@ func (e *Executor) previewLaunchLocked(st *engineState, request settings.Request if seenEnv[environmentKey(key)] { return fail("An environment assignment is repeated.") } + if st.manifest != nil && st.manifest.Engine == "llamacpp" && llamaOwnedEnvironmentKey(key) { + return fail("PAIR manages the llama.cpp model cache location; LLAMA_CACHE and HF_HUB_CACHE cannot be set in launch settings.") + } seenEnv[environmentKey(key)] = true managed := false if control := policy.environmentControl(key); control != nil { diff --git a/services/nvpair-engine-manager/settings_llama_test.go b/services/nvpair-engine-manager/settings_llama_test.go new file mode 100644 index 00000000..2b6f82ff --- /dev/null +++ b/services/nvpair-engine-manager/settings_llama_test.go @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "strings" + "testing" + + settings "nvpair-shared/enginesettings" +) + +// The llama manifest resolves its cache environment from {model_dir}. The +// settings display path must supply that placeholder like doStart does, and +// the reviewed --host/--port controls make the launch editable, so a llama.cpp +// row shows its argument text instead of a display failure. +func TestLlamaLaunchSettingsResolveModelDirAndAreEditable(t *testing.T) { + e := NewExecutor(buildRegistry(""), NewReporter(nil), nil, t.TempDir()) + state, err := e.LaunchSettings("llamacpp") + if err != nil { + t.Fatal(err) + } + if strings.Contains(state.Reason, "cannot be displayed") { + t.Fatalf("settings display failed to resolve the llama launch: %q", state.Reason) + } + if !state.Editable { + t.Fatalf("llama.cpp declares reviewed networking controls and must be editable: reason %q", state.Reason) + } + if state.ServerPort != 8081 { + t.Fatalf("server port = %d, want the managed default 8081", state.ServerPort) + } + // Fixed startup arguments (serve, --no-models-autoload) stay out of the + // editable text; the managed loopback bind and port are what the user sees. + if state.LaunchText != "--host 127.0.0.1 --port 8081" { + t.Fatalf("launch text = %q", state.LaunchText) + } + if strings.Contains(state.LaunchText, "no-models-autoload") || strings.Contains(state.LaunchText, "LLAMA_CACHE") { + t.Fatalf("fixed arguments or owned cache environment leaked into editable text: %q", state.LaunchText) + } +} + +// The model cache location is owned by the engine manager and injected on every +// launch. Accepting it in the editor would save a value that never reaches the +// engine, so the preview refuses it instead of dropping it silently. +func TestLlamaLaunchSettingsRefuseOwnedCacheEnvironment(t *testing.T) { + e := NewExecutor(buildRegistry(""), NewReporter(nil), nil, t.TempDir()) + for _, text := range []string{`LLAMA_CACHE="C:/elsewhere" --host 127.0.0.1 --port 8081`, `hf_hub_cache=/elsewhere --port 8081`} { + preview, err := e.PreviewLaunch(settings.Request{Engine: "llamacpp", Settings: settings.Config{ServerPort: 8081, ProxyPort: 8080, LaunchText: text}}) + if err != nil { + t.Fatal(err) + } + if len(preview.Errors) == 0 { + t.Fatalf("accepted owned cache environment %q: %+v", text, preview) + } + if len(preview.Env) != 0 { + t.Fatalf("owned cache environment leaked into saved env for %q: %v", text, preview.Env) + } + } + // An unrelated variable is still an ordinary editable assignment. + preview, err := e.PreviewLaunch(settings.Request{Engine: "llamacpp", Settings: settings.Config{ServerPort: 8081, ProxyPort: 8080, LaunchText: `GGML_EXAMPLE=1 --host 127.0.0.1 --port 8081 --ctx-size 4096`}}) + if err != nil { + t.Fatal(err) + } + if len(preview.Errors) != 0 || len(preview.Env) != 1 || preview.Env[0] != "GGML_EXAMPLE=1" || len(preview.Args) != 2 || preview.Args[0] != "--ctx-size" { + t.Fatalf("unexpected preview for an ordinary llama launch edit: %+v", preview) + } +} diff --git a/services/nvpair-engine-manager/status.go b/services/nvpair-engine-manager/status.go index 75419cb7..8206997f 100644 --- a/services/nvpair-engine-manager/status.go +++ b/services/nvpair-engine-manager/status.go @@ -5,6 +5,8 @@ package main import ( "context" + "fmt" + "runtime" "time" ) @@ -63,11 +65,22 @@ func (e *Executor) StatusAtPort(engine string, probePort int) (EngineStatus, err // (matching get-installed) rather than erroring, so status is // consistent for the same engine across methods. if m, ok := e.reg.Get(engine); ok { - return EngineStatus{Engine: engine, DisplayName: m.DisplayName}, nil + if _, supported := m.HostPlatform(); !supported { + return unavailableEngineStatus(engine, m.DisplayName), nil + } } return EngineStatus{}, err } - st.opMu.Lock() + if engine == "llamacpp" { + if !st.opMu.TryLock() { + if probePort > 0 { + return EngineStatus{}, fmt.Errorf("llama lifecycle is busy; cannot probe an alternate port") + } + return e.snapshot(engine, st), nil + } + } else { + st.opMu.Lock() + } defer st.opMu.Unlock() pathInstalled, _ := e.Detect(engine) st.mu.Lock() @@ -89,13 +102,28 @@ func (e *Executor) GetInstalled() []EngineStatus { // Known engine with no block for this host: surface a shell // status so the UI can still list it as unavailable here. dn := name + reason := "" if m, ok := e.reg.Get(name); ok { dn = m.DisplayName + if _, supported := m.HostPlatform(); supported { + reason = err.Error() + } + } + status := unavailableEngineStatus(name, dn) + if reason != "" { + status.InstallReason = reason } - out = append(out, EngineStatus{Engine: name, DisplayName: dn}) + out = append(out, status) continue } - st.opMu.Lock() + if name == "llamacpp" { + if !st.opMu.TryLock() { + out = append(out, e.snapshot(name, st)) + continue + } + } else { + st.opMu.Lock() + } pathInstalled, _ := e.Detect(name) st.mu.Lock() port := st.port @@ -121,16 +149,38 @@ func (e *Executor) Errors() []serviceError { return e.reporter.snapshot() } +func unavailableEngineStatus(engine, displayName string) EngineStatus { + status := EngineStatus{Engine: engine, DisplayName: displayName} + if engine == "llamacpp" { + status.InstallReason = "No llama installer recipe is available for this operating system and architecture." + if supported, reason := llamaInstallSupport(runtime.GOOS, runtime.GOARCH); !supported { + status.InstallReason = reason + } + } + return status +} + func (e *Executor) snapshot(engine string, st *engineState) EngineStatus { st.mu.Lock() defer st.mu.Unlock() + supported := st.plat.Install != nil + reason := "" + if engine == "llamacpp" { + supported, reason = llamaInstallSupport(runtime.GOOS, runtime.GOARCH) + if missing := llamaPrerequisite(); supported && missing != "" { + supported, reason = false, missing + } + } return EngineStatus{ - Engine: engine, - DisplayName: st.manifest.DisplayName, - Installed: st.installed, - Running: st.running, - Healthy: st.healthy, - Port: st.port, + InstallSupported: supported, + InstallReason: reason, + Managed: st.installed && !st.adopted && isManagedInstallPath(st.binPath, st.installDir), + Engine: engine, + DisplayName: st.manifest.DisplayName, + Installed: st.installed, + Running: st.running, + Healthy: st.healthy, + Port: st.port, } } diff --git a/services/nvpair-engine-manager/testdata/fakeengine/main.go b/services/nvpair-engine-manager/testdata/fakeengine/main.go index 9f12ba82..e165d051 100644 --- a/services/nvpair-engine-manager/testdata/fakeengine/main.go +++ b/services/nvpair-engine-manager/testdata/fakeengine/main.go @@ -19,6 +19,7 @@ import ( "net" "net/http" "os" + "path/filepath" "strconv" "strings" "sync" @@ -95,6 +96,36 @@ func main() { _ = json.NewEncoder(file).Encode(os.Args[1:]) _ = file.Close() } + if len(os.Args) > 1 && (os.Args[1] == "version" || os.Args[1] == "licenses" || (len(os.Args) == 3 && os.Args[1] == "cli" && os.Args[2] == "--list-devices")) { + // Socket-free llama install validation. Each staged copy can describe a + // different build/device result without changing the executor's env. + var fixture struct { + Version string `json:"version"` + Licenses string `json:"licenses"` + Devices string `json:"devices"` + DelayCommand string `json:"delay_command"` + DelayMS int `json:"delay_ms"` + } + bin, _ := os.Executable() + data, _ := os.ReadFile(filepath.Join(filepath.Dir(bin), ".llama-fixture.json")) + _ = json.Unmarshal(data, &fixture) + if version := os.Getenv("FAKE_LLAMA_VERSION"); version != "" { + fixture.Version = version + } + if fixture.DelayCommand == os.Args[1] { + _ = os.WriteFile(filepath.Join(filepath.Dir(bin), ".llama-delay-started"), []byte(os.Args[1]), 0600) + time.Sleep(time.Duration(fixture.DelayMS) * time.Millisecond) + } + switch os.Args[1] { + case "version": + fmt.Println(fixture.Version) + case "licenses": + fmt.Println(fixture.Licenses) + case "cli": + fmt.Println(fixture.Devices) + } + return + } // Subcommands used by command-mode + cmd-action tests. They run and // exit (no server), standing in for a daemon's control CLI. if len(os.Args) > 1 { diff --git a/services/nvpair-engine-manager/usererrors.go b/services/nvpair-engine-manager/usererrors.go index 4b30bbf0..1cdbcdbc 100644 --- a/services/nvpair-engine-manager/usererrors.go +++ b/services/nvpair-engine-manager/usererrors.go @@ -4,13 +4,45 @@ package main import ( + "errors" "fmt" + "os/exec" "regexp" "strings" ) var exitStatusPrefixRe = regexp.MustCompile(`^exit status \d+: `) +// Classify bounded vendor stderr without publishing cache paths, URLs, tokens, +// or arbitrary third-party text through the local/paired error log. +func llamaDownloadError(err error) error { + var exit *exec.ExitError + if !errors.As(err, &exit) { + return fmt.Errorf("cannot start llama downloader: %w", err) + } + detail := exit.Stderr + if len(detail) > 16384 { + detail = detail[:16384] + } + s := strings.ToLower(string(detail)) + message := "the vendor downloader exited unsuccessfully" + switch { + case strings.Contains(s, "no space left"), strings.Contains(s, "disk full"): + message = "the model cache disk is full" + case strings.Contains(s, "error opening"), strings.Contains(s, "failed to open"), strings.Contains(s, "cannot open"): + message = "cannot open the model cache file; check free space, path length and write permissions" + case strings.Contains(s, "401"), strings.Contains(s, "403"): + message = "the model source refused access; check repository access and authentication" + case strings.Contains(s, "404"), strings.Contains(s, "not found"): + message = "the requested repository, quantization or model file was not found" + case strings.Contains(s, "certificate"), strings.Contains(s, "ssl"): + message = "TLS verification failed while contacting the model source" + case strings.Contains(s, "timed out"), strings.Contains(s, "timeout"): + message = "the model source or network timed out; the download can be retried" + } + return fmt.Errorf("llama model download failed: %s", message) +} + // formatEnginePullError renders a user-facing message for a model-pull failure // attributable to the engine (CLI stderr, engine HTTP response, etc.). func formatEnginePullError(displayName string, err error) string { diff --git a/services/nvpair-job-scheduler/schedule_test.go b/services/nvpair-job-scheduler/schedule_test.go index 6c1a402e..5dc89034 100644 --- a/services/nvpair-job-scheduler/schedule_test.go +++ b/services/nvpair-job-scheduler/schedule_test.go @@ -691,3 +691,15 @@ func TestNewManager_Floor(t *testing.T) { t.Fatalf("interval = %v, want floor %v", m.interval, intervalFloor) } } + +func TestSchedulerEnginesIncludesLlamaCpp(t *testing.T) { + found := false + for _, e := range schedulerEngines { + if e == "llamacpp" { + found = true + } + } + if !found { + t.Fatal("schedulerEngines missing llamacpp") + } +} diff --git a/services/nvpair-manual-nodes/README.md b/services/nvpair-manual-nodes/README.md index 52c5de7b..04212ec9 100644 --- a/services/nvpair-manual-nodes/README.md +++ b/services/nvpair-manual-nodes/README.md @@ -5,7 +5,7 @@ SPDX-License-Identifier: Apache-2.0 # nvpair-manual-nodes -A Go service for managing manually configured nodes on networks where mDNS discovery is unavailable. Accepts node addresses via JSON-RPC, probes each for Ollama, LM Studio, and node-info, and emits status events. +A Go service for managing manually configured nodes on networks where mDNS discovery is unavailable. Accepts node addresses via JSON-RPC, probes each for Ollama, LM Studio, llama.cpp, and node-info, and emits status events. ## Communication @@ -50,6 +50,9 @@ Emitted when a manually added node has been probed and its initial status determ "lmstudio_up":true, "lmstudio_port":1234, "lmstudio_models":["qwen2.5-7b-instruct"], + "llamacpp_up":true, + "llamacpp_port":8080, + "llamacpp_models":["loaded-one"], "node_info_up":true, "node_info_port":14318, "gpus":[{"name":"NVIDIA GeForce RTX 3080","utilization_percent":37}], @@ -60,7 +63,7 @@ Emitted when a manually added node has been probed and its initial status determ } ``` -Each node is probed for both inference engines: Ollama on its default `:11434` (`GET /` + `/api/tags`) and LM Studio on its default `:1234` (`GET /v1/models`, which doubles as the liveness check and the model list). `lmstudio_up` / `lmstudio_port` / `lmstudio_models` mirror the `ollama_*` fields and let a supervising broker bridge the node into that engine's `nvpair-proxy` instance the same way it bridges Ollama into its own. A node can run either engine, both, or neither. +Each node is probed for the three inference engines: Ollama on its default `:11434` (`GET /` + `/api/tags`), LM Studio on its default `:1234` (`GET /v1/models`, which doubles as the liveness check and the model list), and llama.cpp on its default `:8080` (`GET /v1/models`, same liveness-plus-list shape). `lmstudio_up` / `lmstudio_port` / `lmstudio_models` and `llamacpp_up` / `llamacpp_port` / `llamacpp_models` mirror the `ollama_*` fields and let a supervising broker bridge the node into that engine's facade on the `nvpair-proxy` process the same way it bridges Ollama into its own. `llamacpp_models` is the loaded subset only (`status.value == "loaded"`; a missing status is not loaded); a 200 from `/v1/models` still sets `llamacpp_up` when that set is empty. A node can run any combination of engines, or none. ### `node/updated` @@ -136,11 +139,12 @@ Each manual node is probed every 10 seconds, with a 3-second timeout per leg, fo - **Ollama** on port 11434: health check (`GET /`) and model list (`GET /api/tags`) - **LM Studio** on port 1234: `GET /v1/models`, which doubles as the liveness check and the model list +- **llama.cpp** on port 8080: `GET /v1/models`, which doubles as the liveness check and the loaded-model list (`status.value == "loaded"`) - **Node Info** on port 14318, or `tls_port` over HTTPS: hardware inventory and identity (`GET /v1/node-info`) A node can have any combination of these, or none if the target is unreachable. Status changes trigger `node/updated` events. Because change detection compares CPU, memory, and GPU values, a node running node-info emits a `node/updated` on most probe cycles as utilization moves. -The three engine ports are compiled in: only the node-info leg's port can be moved, via `tls_port`. A remote engine on a non-default port is not discovered. +The engine ports are compiled in: only the node-info leg's port can be moved, via `tls_port`. A remote engine on a non-default port is not discovered. ## Shutdown diff --git a/services/nvpair-manual-nodes/manager.go b/services/nvpair-manual-nodes/manager.go index 55a4040a..a2b0b6e7 100644 --- a/services/nvpair-manual-nodes/manager.go +++ b/services/nvpair-manual-nodes/manager.go @@ -112,9 +112,17 @@ type ManualNodeStatus struct { // LM Studio is probed on its default OpenAI-API port the same way Ollama // is on 11434, so a manually-added node running LM Studio can be bridged // into lmstudio-proxy by a supervising broker. - LMStudioUp bool `json:"lmstudio_up"` - LMStudioPort int `json:"lmstudio_port"` - LMStudioModels []string `json:"lmstudio_models,omitempty"` + LMStudioUp bool `json:"lmstudio_up"` + LMStudioPort int `json:"lmstudio_port"` + LMStudioModels []string `json:"lmstudio_models,omitempty"` + // llama.cpp is probed on its default OpenAI-API port (8080) the same way + // LM Studio is on 1234. llamacpp_models is the loaded subset only + // (status.value == "loaded"); a 200 from GET /v1/models still counts as + // up when that set is empty so a supervising broker can bridge the node + // into llamacpp-proxy. + LlamaCppUp bool `json:"llamacpp_up"` + LlamaCppPort int `json:"llamacpp_port"` + LlamaCppModels []string `json:"llamacpp_models,omitempty"` NodeInfoUp bool `json:"node_info_up"` NodeInfoPort int `json:"node_info_port"` TLSEnabled bool `json:"tls_enabled,omitempty"` @@ -138,12 +146,11 @@ type trackedNode struct { entry ManualEntry status ManualNodeStatus - // consecutiveFails counts back-to-back probes where neither - // service answered (OllamaUp && NodeInfoUp both false). Reset - // to 0 on any probe where at least one service responded. - // Used to gate probe-failed errors:report emits at - // probeFailThreshold so a single transient failure doesn't - // generate UI noise. + // consecutiveFails counts back-to-back probes where no engine + // and no node-info answered (reachable is false). Reset to 0 + // on any probe where at least one service responded. Used to + // gate probe-failed errors:report emits at probeFailThreshold + // so a single transient failure doesn't generate UI noise. consecutiveFails int } @@ -253,6 +260,7 @@ func (m *Manager) probeNode(entry ManualEntry) { ollamaUp, ollamaModels := m.probeOllama(addr, 11434) lmStudioUp, lmStudioModels := m.probeLMStudio(addr, lmStudioPort) + llamaCppUp, llamaCppModels := m.probeLlamaCpp(addr, llamaCppPort) // Pick scheme + port + client based on the entry's TLS hint. // The operator decides which scheme this manual node uses; we @@ -290,6 +298,9 @@ func (m *Manager) probeNode(entry ManualEntry) { LMStudioUp: lmStudioUp, LMStudioPort: lmStudioPort, LMStudioModels: lmStudioModels, + LlamaCppUp: llamaCppUp, + LlamaCppPort: llamaCppPort, + LlamaCppModels: llamaCppModels, NodeInfoUp: nodeInfoUp, NodeInfoPort: nodeInfoPort, TLSEnabled: entry.TLSPort > 0, @@ -302,7 +313,7 @@ func (m *Manager) probeNode(entry ManualEntry) { HostUUID: info.HostUUID, } - reachable := newStatus.OllamaUp || newStatus.LMStudioUp || newStatus.NodeInfoUp + reachable := newStatus.OllamaUp || newStatus.LMStudioUp || newStatus.LlamaCppUp || newStatus.NodeInfoUp m.mu.Lock() tn, exists := m.nodes[id] @@ -331,10 +342,12 @@ func (m *Manager) probeNode(entry ManualEntry) { changed := prev.OllamaUp != newStatus.OllamaUp || prev.LMStudioUp != newStatus.LMStudioUp || + prev.LlamaCppUp != newStatus.LlamaCppUp || prev.NodeInfoUp != newStatus.NodeInfoUp || prev.HostUUID != newStatus.HostUUID || !sliceEqual(prev.OllamaModels, newStatus.OllamaModels) || !sliceEqual(prev.LMStudioModels, newStatus.LMStudioModels) || + !sliceEqual(prev.LlamaCppModels, newStatus.LlamaCppModels) || !gpusEqual(prev.GPUs, newStatus.GPUs) || !cpuEqual(prev.CPU, newStatus.CPU) || !memoryEqual(prev.Memory, newStatus.Memory) || @@ -446,6 +459,64 @@ func (m *Manager) probeLMStudio(addr string, port int) (bool, []string) { return true, models } +// llamaCppPort is llama.cpp's own default server port, probed the same way LM +// Studio is hardcoded to 1234. A manual node is remote, so (like the other +// engines) we assume the engine's default port rather than resolving it via the +// engine manager, which only governs the local engine. +// +// 8080 answers for either kind of peer, which is why it is the right guess: a +// bare llama.cpp listens there by default, and a peer running PAIR has its +// llama.cpp facade there with the engine relocated above it. Probing the +// relocated port instead would reach the engine directly and bypass routing. +const llamaCppPort = 8080 + +// probeLlamaCpp checks llama-server's OpenAI-compatible API on addr:port. A +// single GET /v1/models doubles as the liveness check and the model list. +// Only ids whose status.value is "loaded" are returned — a missing status is +// treated as not loaded — so the broker bridges a routing-eligible set into +// llamacpp-proxy. A 200 still reports the node up when that set is empty. +func (m *Manager) probeLlamaCpp(addr string, port int) (bool, []string) { + url := "http://" + net.JoinHostPort(addr, strconv.Itoa(port)) + "/v1/models" + start := time.Now() + resp, err := m.client.Get(url) + if err != nil { + slog.Debug("manual probe llamacpp failed", + "addr", addr, "port", port, "duration_ms", time.Since(start).Milliseconds(), "err", err) + return false, nil + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + slog.Debug("manual probe llamacpp non-OK", + "addr", addr, "port", port, "status", resp.StatusCode, + "duration_ms", time.Since(start).Milliseconds()) + return false, nil + } + var result struct { + Data []struct { + ID string `json:"id"` + Status struct { + Value string `json:"value"` + } `json:"status"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + // Reachable, but the model list didn't parse — still report it up. + slog.Debug("manual probe llamacpp up (models parse failed)", + "addr", addr, "port", port, "err", err) + return true, nil + } + models := make([]string, 0, len(result.Data)) + for _, d := range result.Data { + if d.ID != "" && d.Status.Value == "loaded" { + models = append(models, d.ID) + } + } + slog.Debug("manual probe llamacpp up", + "addr", addr, "port", port, "models", len(models), + "duration_ms", time.Since(start).Milliseconds()) + return true, models +} + func (m *Manager) probeOllama(addr string, port int) (bool, []string) { url := "http://" + net.JoinHostPort(addr, strconv.Itoa(port)) + "/" start := time.Now() diff --git a/services/nvpair-manual-nodes/manager_test.go b/services/nvpair-manual-nodes/manager_test.go index e52d9d5d..23a1d075 100644 --- a/services/nvpair-manual-nodes/manager_test.go +++ b/services/nvpair-manual-nodes/manager_test.go @@ -11,6 +11,7 @@ import ( "io" "net" "net/http" + "strconv" "strings" "sync" "testing" @@ -154,6 +155,81 @@ func TestProbeLMStudioReportsModels(t *testing.T) { } } +// configureHealthyLlamaCpp registers a 200 GET /v1/models on llama.cpp's probe port with +// one loaded and one unloaded model, so probeLlamaCpp reports the node up +// with only the loaded id. +func configureHealthyLlamaCpp(rt *fakeRoundTripper, addr string) { + host := net.JoinHostPort(addr, strconv.Itoa(llamaCppPort)) + rt.set(http.MethodGet, host, "/v1/models", func(*http.Request) (*http.Response, error) { + return httpJSON(http.StatusOK, `{"object":"list","data":[{"id":"loaded-one","status":{"value":"loaded"}},{"id":"catalog-only","status":{"value":"unloaded"}}]}`) + }) +} + +// TestProbeLlamaCppReportsModels covers the llama.cpp probe: a reachable +// server reports up with only ids whose status.value is "loaded", a 200 with +// an empty loaded set is still up, and an absent one reports down. +func TestProbeLlamaCppReportsModels(t *testing.T) { + m, _, rt := newTestManager() + configureHealthyLlamaCpp(rt, "node.local") + + up, models := m.probeLlamaCpp("node.local", llamaCppPort) + if !up { + t.Fatal("expected llamacpp up") + } + if len(models) != 1 || models[0] != "loaded-one" { + t.Fatalf("models = %#v, want [loaded-one] (loaded only)", models) + } + + emptyHost := net.JoinHostPort("empty.local", strconv.Itoa(llamaCppPort)) + rt.set(http.MethodGet, emptyHost, "/v1/models", func(*http.Request) (*http.Response, error) { + return httpJSON(http.StatusOK, `{"object":"list","data":[{"id":"catalog-only","status":{"value":"unloaded"}},{"id":"no-status"}]}`) + }) + emptyUp, emptyModels := m.probeLlamaCpp("empty.local", llamaCppPort) + if !emptyUp { + t.Fatal("expected llamacpp up with empty loaded set") + } + if len(emptyModels) != 0 { + t.Fatalf("empty loaded set models = %#v, want empty", emptyModels) + } + + downUp, downModels := m.probeLlamaCpp("absent.local", llamaCppPort) + if downUp || downModels != nil { + t.Fatalf("expected absent llamacpp down, got up=%v models=%#v", downUp, downModels) + } +} + +func TestProbeNodeLlamaCppOnlyIsReachable(t *testing.T) { + m, rw, rt := newTestManager() + entry := ManualEntry{Name: "lab", Address: "node.local"} + m.nodes["lab"] = &trackedNode{entry: entry, status: ManualNodeStatus{ID: "lab", Address: "node.local"}} + configureHealthyLlamaCpp(rt, "node.local") + + m.probeNode(entry) + updated := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, rw, methodIs("node/updated"))) + if !updated.LlamaCppUp { + t.Fatalf("expected llamacpp up: %+v", updated) + } + if updated.LlamaCppPort != llamaCppPort { + t.Fatalf("llamacpp_port = %d, want %d", updated.LlamaCppPort, llamaCppPort) + } + if len(updated.LlamaCppModels) != 1 || updated.LlamaCppModels[0] != "loaded-one" { + t.Fatalf("llamacpp_models = %#v", updated.LlamaCppModels) + } + if m.nodes["lab"].consecutiveFails != 0 { + t.Fatalf("llama.cpp-only node counted as unreachable: fails=%d", m.nodes["lab"].consecutiveFails) + } + + host := net.JoinHostPort("node.local", strconv.Itoa(llamaCppPort)) + rt.set(http.MethodGet, host, "/v1/models", func(*http.Request) (*http.Response, error) { + return httpJSON(http.StatusOK, `{"object":"list","data":[{"id":"loaded-two","status":{"value":"loaded"}}]}`) + }) + m.probeNode(entry) + second := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, rw, methodIs("node/updated"))) + if len(second.LlamaCppModels) != 1 || second.LlamaCppModels[0] != "loaded-two" { + t.Fatalf("second llamacpp_models = %#v", second.LlamaCppModels) + } +} + func requestMessage(id int, method string, params any) *Message { idData, _ := json.Marshal(id) idRaw := json.RawMessage(idData) diff --git a/services/nvpair-proxy/cancel_test.go b/services/nvpair-proxy/cancel_test.go new file mode 100644 index 00000000..eac41c40 --- /dev/null +++ b/services/nvpair-proxy/cancel_test.go @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +// Coverage for cancelling one in-flight request by id. +// +// The registry this drives is per facade rather than per process, and the +// reason is only visible with more than one facade enabled: each facade mints +// request ids from its own counter starting at 1, so "cancel request 1" is +// ambiguous across engines and the runId guard cannot disambiguate it — +// runId names the process, which every facade shares. + +import ( + "context" + "encoding/json" + "testing" + + "nvpair-shared/engines" +) + +// cancelResult drives workload/cancel through the control plane and returns +// what the caller was told. +func cancelResult(t *testing.T, p *Proxy, rec *recordingWriter, engine, id, runID string) bool { + t.Helper() + before := len(rec.lines()) + params, err := json.Marshal(map[string]string{"id": id, "runId": runID}) + if err != nil { + t.Fatalf("marshal cancel params: %v", err) + } + msgID := json.RawMessage(`1`) + p.handleMessage(&Message{ + Method: engines.AddressMethod(engine, "workload/cancel"), + Params: params, + ID: &msgID, + }) + + lines := rec.lines() + if len(lines) <= before { + t.Fatalf("workload/cancel for %s/%s produced no response", engine, id) + } + var reply struct { + Result struct { + Accepted bool `json:"accepted"` + } `json:"result"` + Error *struct { + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(lines[len(lines)-1], &reply); err != nil { + t.Fatalf("decode cancel reply: %v", err) + } + if reply.Error != nil { + t.Fatalf("workload/cancel for %s/%s errored: %s", engine, id, reply.Error.Message) + } + return reply.Result.Accepted +} + +// twoFacadeRecordingProxy is twoFacadeProxy with its upward frames captured, so +// a test can read the reply to a request rather than only observe side effects. +func twoFacadeRecordingProxy(t *testing.T) (*Proxy, *recordingWriter) { + t.Helper() + redirectConfigDir(t) + + rec := &recordingWriter{} + p := NewProxy(NewCodec(rec)) + p.serveCtx = t.Context() + for _, e := range engines.All() { + port := freeTCPPort(t) + if _, err := p.enableFacade(enableFacadeParams{ + Engine: e.Name, + Port: port, + IgnorePersistedPort: true, + }); err != nil { + t.Fatalf("enable %s facade on :%d: %v", e.Name, port, err) + } + } + t.Cleanup(func() { p.shutdown(t.Context()) }) + return p, rec +} + +// A cancel names one request of one engine. Accepting it must cancel exactly +// that request's context and nothing else. +func TestCancelAbortsOnlyTheNamedRequest(t *testing.T) { + p, rec := twoFacadeRecordingProxy(t) + f := p.facadeFor(engines.All()[0].Name) + + first, cancelFirst := context.WithCancel(context.Background()) + second, cancelSecond := context.WithCancel(context.Background()) + _, forgetFirst := f.trackInflight("1", cancelFirst) + _, forgetSecond := f.trackInflight("2", cancelSecond) + defer forgetFirst() + defer forgetSecond() + + if cancelResult(t, p, rec, f.profile.Name, "missing", p.runID) { + t.Error("a request id that was never registered was accepted") + } + if first.Err() != nil || second.Err() != nil { + t.Fatal("an unmatched cancel aborted a live request") + } + + if !cancelResult(t, p, rec, f.profile.Name, "1", p.runID) { + t.Fatal("the named request was not accepted") + } + if first.Err() != context.Canceled { + t.Errorf("named request context = %v, want cancelled", first.Err()) + } + if second.Err() != nil { + t.Errorf("unrelated request was cancelled: %v", second.Err()) + } +} + +// runId names the proxy process, and request ids restart with it. A cancel +// carrying a previous run's id refers to a request that no longer exists, and +// the id it names may now belong to an unrelated one. +func TestCancelRefusesAStaleRun(t *testing.T) { + p, rec := twoFacadeRecordingProxy(t) + f := p.facadeFor(engines.All()[0].Name) + + ctx, cancel := context.WithCancel(context.Background()) + _, forget := f.trackInflight("1", cancel) + defer forget() + + if cancelResult(t, p, rec, f.profile.Name, "1", p.runID+"-previous") { + t.Error("a cancel from a previous run was accepted") + } + if ctx.Err() != nil { + t.Errorf("a stale run cancelled a live request: %v", ctx.Err()) + } +} + +// The property a process-wide registry would break. Both facades have a +// request numbered "1", because each counts from 1; a cancel addressed to one +// engine must leave the other engine's request of the same number running. +func TestCancelIsScopedToTheAddressedFacade(t *testing.T) { + p, rec := twoFacadeRecordingProxy(t) + all := engines.All() + target := p.facadeFor(all[0].Name) + bystander := p.facadeFor(all[1].Name) + + targetCtx, cancelTarget := context.WithCancel(context.Background()) + bystanderCtx, cancelBystander := context.WithCancel(context.Background()) + _, forgetTarget := target.trackInflight("1", cancelTarget) + _, forgetBystander := bystander.trackInflight("1", cancelBystander) + defer forgetTarget() + defer forgetBystander() + + if !cancelResult(t, p, rec, target.profile.Name, "1", p.runID) { + t.Fatalf("%s request 1 was not accepted", target.profile.Name) + } + if targetCtx.Err() != context.Canceled { + t.Errorf("%s request 1 = %v, want cancelled", target.profile.Name, targetCtx.Err()) + } + if bystanderCtx.Err() != nil { + t.Fatalf("cancelling %s request 1 also aborted %s request 1", + target.profile.Name, bystander.profile.Name) + } +} + +// An asked-for cancel and a client hanging up both surface as a cancelled +// request context. The workload's error text is user-visible, so the two must +// not be reported the same way. +func TestCancelIsDistinguishedFromAClientDisconnect(t *testing.T) { + p, _ := twoFacadeRecordingProxy(t) + f := p.facadeFor(engines.All()[0].Name) + + cancelled, cancelOne := context.WithCancel(context.Background()) + defer cancelOne() + req, forget := f.trackInflight("1", cancelOne) + defer forget() + if req.cancelled.Load() { + t.Fatal("a freshly tracked request already reports being cancelled") + } + if !f.cancelInflight("1") { + t.Fatal("cancel was not accepted") + } + if !req.cancelled.Load() { + t.Error("an asked-for cancel was not distinguished from a disconnect") + } + if cancelled.Err() != context.Canceled { + t.Errorf("request context = %v, want cancelled", cancelled.Err()) + } +} + +// Forgetting a finished request keeps a later cancel of the same id from +// reaching a request that has since reused it. +func TestCancelFindsNothingAfterTheRequestFinishes(t *testing.T) { + p, rec := twoFacadeRecordingProxy(t) + f := p.facadeFor(engines.All()[0].Name) + + _, cancel := context.WithCancel(context.Background()) + _, forget := f.trackInflight("1", cancel) + forget() + + if cancelResult(t, p, rec, f.profile.Name, "1", p.runID) { + t.Error("a finished request was still cancellable") + } +} diff --git a/services/nvpair-proxy/eligibility_test.go b/services/nvpair-proxy/eligibility_test.go new file mode 100644 index 00000000..99161781 --- /dev/null +++ b/services/nvpair-proxy/eligibility_test.go @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +// Coverage for which of a node's advertised inventories makes it an owner. +// +// Ollama and LM Studio load a requested model on demand, so a catalog entry is +// a promise they can serve it. PAIR runs llama.cpp with --no-models-autoload, +// so a downloaded model it has not been told to load will not be served, and +// eligibility there has to read the loaded set instead. These assert the +// difference directly, because everything else in the suite goes through +// advertiseEngine and so would keep passing if the two collapsed. + +import ( + "testing" + + "nvpair-shared/noderec" +) + +// nodeAdvertising builds a relay record for one engine with no inventory, ready +// for a test to populate whichever field it means. +func nodeAdvertising(p engineProfile) noderec.DirectoryNode { + return noderec.DirectoryNode{ + HostUUID: "uuid-eligibility", + Name: "host-eligibility", + IP: "10.0.0.9", + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + p.DiscoveryService: {Port: p.FacadePort}, + }, + } +} + +// Exactly one engine reads the loaded set. Pinning the count keeps a new +// engine from silently inheriting llama.cpp's answer, which is the failure +// this field exists to make impossible. +func TestLoadedOnlyEligibilityIsDeclaredNotInherited(t *testing.T) { + loaded := map[string]bool{} + for _, p := range profiles { + if p.ModelEligibility == loadedModels { + loaded[p.Name] = true + } + } + if len(loaded) != 1 || !loaded["llamacpp"] { + t.Fatalf("engines reading the loaded set = %v, want just llamacpp", loaded) + } +} + +// A node whose catalog lists a model it has not loaded is an owner for the +// on-demand engines and not for llama.cpp. +func TestCatalogOnlyNodeIsNotALlamaCppOwner(t *testing.T) { + forEachEngine(t, func(t *testing.T, tc engineCase) { + n := nodeAdvertising(tc.profile) + n.ModelsByEngine = map[string][]string{tc.profile.Name: {"on-disk"}} + n.Models = []string{"on-disk"} + + got, ok := subscribedToNode(tc.profile, n) + if !ok { + t.Fatal("node advertising this engine should still project") + } + if tc.profile.ModelEligibility == loadedModels { + if len(got.Models) != 0 { + t.Fatalf("catalog-only node offered %v as loaded owners", got.Models) + } + return + } + if len(got.Models) != 1 || got.Models[0] != "on-disk" { + t.Fatalf("catalog models = %v, want [on-disk]", got.Models) + } + }) +} + +// The converse: a loaded report is what makes llama.cpp route, and it is not +// read as a catalog by the engines that do consult one. +func TestLoadedReportMakesLlamaCppAnOwner(t *testing.T) { + llamacpp := llamacppCase(t).profile + + n := nodeAdvertising(llamacpp) + n.LoadedByEngine = map[string][]string{llamacpp.Name: {"resident"}} + + got, ok := subscribedToNode(llamacpp, n) + if !ok { + t.Fatal("node advertising llama.cpp should project") + } + if len(got.Models) != 1 || got.Models[0] != "resident" { + t.Fatalf("loaded models = %v, want [resident]", got.Models) + } +} + +// A missing report means nothing is loaded, never "fall back to the catalog". +// Without that, a node that never answered a loaded-models query would look +// like an owner of everything it has on disk. +func TestMissingLoadedReportIsNotACatalogFallback(t *testing.T) { + llamacpp := llamacppCase(t).profile + + n := nodeAdvertising(llamacpp) + n.Models = []string{"on-disk"} + n.ModelsByEngine = map[string][]string{llamacpp.Name: {"on-disk"}} + // LoadedByEngine deliberately absent. + + got, ok := subscribedToNode(llamacpp, n) + if !ok { + t.Fatal("node advertising llama.cpp should project") + } + if len(got.Models) != 0 { + t.Fatalf("a node with no loaded report offered %v", got.Models) + } +} + +// Attribution stays per engine on the loaded side too: a model resident under +// another engine does not make this one an owner. +func TestLoadedAttributionDoesNotLeakAcrossEngines(t *testing.T) { + llamacpp := llamacppCase(t).profile + + n := nodeAdvertising(llamacpp) + n.LoadedByEngine = map[string][]string{ + llamacpp.Name: {"mine"}, + "lmstudio": {"theirs"}, + } + + got, ok := subscribedToNode(llamacpp, n) + if !ok { + t.Fatal("node advertising llama.cpp should project") + } + if len(got.Models) != 1 || got.Models[0] != "mine" { + t.Fatalf("loaded models = %v, want [mine] only", got.Models) + } +} diff --git a/services/nvpair-proxy/enginecase_test.go b/services/nvpair-proxy/enginecase_test.go index 1a4b541d..beb9e28d 100644 --- a/services/nvpair-proxy/enginecase_test.go +++ b/services/nvpair-proxy/enginecase_test.go @@ -21,6 +21,8 @@ import ( "net/http/httptest" "strings" "testing" + + "nvpair-shared/noderec" ) type engineCase struct { @@ -91,6 +93,52 @@ func lmstudioCase(t *testing.T) engineCase { } } +func llamacppCase(t *testing.T) engineCase { + t.Helper() + p, ok := profileFor("llamacpp") + if !ok { + t.Fatal("llamacpp profile missing") + } + return engineCase{ + profile: p, + inferencePath: "/v1/chat/completions", + // llama-server's own status surface, unlisted in Routes and so + // forwarded verbatim. + nonInferencePath: "/props", + modelListPath: "/v1/models", + emptyModelList: `{"object":"list","data":[]}`, + advertisedModel: "qwen3-8b-gguf", + requestedModel: "qwen3-8b-gguf", + } +} + +// advertiseEngine records model on a discovery record as belonging to engine +// p, in whichever per-engine inventory p's eligibility actually reads: the +// installed catalog for the on-demand engines, the loaded set for llama.cpp. +// +// Bodies use this instead of assigning Models or ModelsByEngine directly so a +// shared fixture means "this node can serve model" for every engine, rather +// than silently meaning it only for the ones that consult the catalog. +func advertiseEngine(n *noderec.DirectoryNode, p engineProfile, model string) { + if p.ModelEligibility == loadedModels { + if n.LoadedByEngine == nil { + n.LoadedByEngine = map[string][]string{} + } + n.LoadedByEngine[p.Name] = append(n.LoadedByEngine[p.Name], model) + return + } + if n.ModelsByEngine == nil { + n.ModelsByEngine = map[string][]string{} + } + n.ModelsByEngine[p.Name] = append(n.ModelsByEngine[p.Name], model) + n.Models = append(n.Models, model) +} + +// advertise is advertiseEngine for the engine under test. +func (tc engineCase) advertise(n *noderec.DirectoryNode, model string) { + advertiseEngine(n, tc.profile, model) +} + // inferenceBody is the request body a client sends for requestedModel. It is // the one shape both dialects share: a top-level "model" field, which is all // bufferBodyAndModel reads. @@ -108,7 +156,7 @@ func (tc engineCase) inferenceRequest() *http.Request { // under this is asserting behavior both proxies must share. func engineCases(t *testing.T) []engineCase { t.Helper() - return []engineCase{ollamaCase(t), lmstudioCase(t)} + return []engineCase{ollamaCase(t), lmstudioCase(t), llamacppCase(t)} } // forEachEngine runs body as a subtest per engine. diff --git a/services/nvpair-proxy/engines.go b/services/nvpair-proxy/engines.go index 06d1a54f..2ec8cc32 100644 --- a/services/nvpair-proxy/engines.go +++ b/services/nvpair-proxy/engines.go @@ -114,8 +114,32 @@ type engineProfile struct { // claim one names that variable. Gating on this makes the scoping // enforced rather than left to the broker's restraint in passing the flag. SupportsHostAlias bool + + // ModelEligibility is which advertised inventory makes a node an owner. + // Stated by every engine rather than left to the zero value: which + // inventory to trust is the kind of decision a new engine should have to + // make, not inherit from whichever constant happens to be first. + ModelEligibility modelEligibility } +// modelEligibility is which of a node's advertised inventories makes it a +// routable owner of a requested model. +type modelEligibility int + +const ( + // catalogModels accepts any model the node reports as installed. Ollama + // and LM Studio both load a requested model on demand, so a catalog entry + // is a promise the node can serve it. + catalogModels modelEligibility = iota + + // loadedModels accepts only models the node reports resident in memory, + // and never falls back to the catalog. PAIR runs llama.cpp with + // --no-models-autoload, so a downloaded model it has not been told to load + // will not be served; routing to it on the strength of the catalog + // produces a request its owner cannot answer. + loadedModels +) + // openAIInferenceRoutes is the inference surface every OpenAI-compatible // engine exposes. Ollama serves these alongside its native routes. var openAIInferenceRoutes = []route{ @@ -129,6 +153,7 @@ var profiles = buildProfiles() func buildProfiles() []engineProfile { ollama, _ := engines.ByName("ollama") lmstudio, _ := engines.ByName("lmstudio") + llamacpp, _ := engines.ByName("llamacpp") return []engineProfile{ { @@ -143,6 +168,7 @@ func buildProfiles() []engineProfile { {Path: "/v1/models", Role: roleModelListOpenAIGET}, }, openAIInferenceRoutes...), ModelNaming: impliedLatestTag, + ModelEligibility: catalogModels, ReservedPersistedPort: 0, SupportsHostAlias: true, }, @@ -152,12 +178,25 @@ func buildProfiles() []engineProfile { Routes: append([]route{ {Path: "/v1/models", Role: roleModelListOpenAIGET}, }, openAIInferenceRoutes...), - ModelNaming: exactID, + ModelNaming: exactID, + ModelEligibility: catalogModels, // 1235 is where engine-manager runs a managed LM Studio, so a // proxy that restored it would sit on the engine's own port. The // stored value predates the current default of 1234. ReservedPersistedPort: 1235, }, + { + Engine: llamacpp, + StandalonePort: llamacpp.FacadePort, + Routes: append([]route{ + {Path: "/v1/models", Role: roleModelListOpenAIGET}, + }, openAIInferenceRoutes...), + ModelNaming: exactID, + // 8081 is where engine-manager relocates a managed llama.cpp, so a + // proxy that restored it would sit on the engine's own port. + ReservedPersistedPort: llamacpp.EnginePortBase, + ModelEligibility: loadedModels, + }, } } diff --git a/services/nvpair-proxy/facade.go b/services/nvpair-proxy/facade.go index b53cfa3c..0f34b0cb 100644 --- a/services/nvpair-proxy/facade.go +++ b/services/nvpair-proxy/facade.go @@ -105,6 +105,85 @@ type facade struct { // separates them. A shared counter would hand the second facade "2" and // leave that guarantee untested. nextRequestID atomic.Uint64 + + // inflightMu guards inflight, the cancel handle for every cancellable + // request this facade is currently serving. + // + // Per facade for exactly the reason nextRequestID is: ids restart at 1 in + // each facade and therefore collide across them, so a process-wide map + // keyed by id would let a cancel aimed at one engine abort another + // engine's request of the same number. The runId guard cannot catch that, + // because runId names the process and every facade shares it. + inflightMu sync.Mutex + inflight map[string]*inflightRequest +} + +// inflightRequest is one cancellable request in progress. +type inflightRequest struct { + // cancel aborts the origin request's context, which propagates to the + // upstream connection so the engine stops generating rather than finishing + // a result nobody will read. + cancel context.CancelFunc + + // cancelled records that the abort was asked for, rather than the client + // hanging up. Both reach the disconnect watcher as a cancelled context, + // and the workload's error text is user-visible, so reporting an operator + // cancel as a client disconnect would be a lie about what happened. + cancelled atomic.Bool +} + +// trackInflight registers a request's cancel handle for the life of the +// request. The returned cleanup forgets it and releases the context. +// +// Every section holding inflightMu releases it by defer, for the reason the +// httpMu accessors below do: cancelInflight is reached from handleMessage, +// which recovers panics, and a mutex stranded by a recovered panic would trade +// a contained crash for a facade that can never track or cancel a request +// again. +func (f *facade) trackInflight(id string, cancel context.CancelFunc) (*inflightRequest, func()) { + req := &inflightRequest{cancel: cancel} + f.putInflight(id, req) + return req, func() { + f.forgetInflight(id) + cancel() + } +} + +func (f *facade) putInflight(id string, req *inflightRequest) { + f.inflightMu.Lock() + defer f.inflightMu.Unlock() + if f.inflight == nil { + f.inflight = make(map[string]*inflightRequest) + } + f.inflight[id] = req +} + +func (f *facade) forgetInflight(id string) { + f.inflightMu.Lock() + defer f.inflightMu.Unlock() + delete(f.inflight, id) +} + +func (f *facade) lookupInflight(id string) *inflightRequest { + f.inflightMu.Lock() + defer f.inflightMu.Unlock() + return f.inflight[id] +} + +// cancelInflight aborts one request by id and reports whether there was a live +// request to abort. +// +// Acceptance means that request's context was cancelled, not that the engine +// has acknowledged anything: the request's ordinary terminal workload event +// still reports the outcome. +func (f *facade) cancelInflight(id string) bool { + req := f.lookupInflight(id) + if req == nil { + return false + } + req.cancelled.Store(true) + req.cancel() + return true } func newFacade(host *Proxy, profile engineProfile, discovery *Discovery, port int) *facade { diff --git a/services/nvpair-proxy/main.go b/services/nvpair-proxy/main.go index ebfa902d..94436f50 100644 --- a/services/nvpair-proxy/main.go +++ b/services/nvpair-proxy/main.go @@ -13,6 +13,7 @@ import ( "os" "os/signal" "syscall" + "time" "nvpair-shared/applog" "nvpair-shared/clustertrust" @@ -71,6 +72,16 @@ func main() { codec := NewCodec(transport) proxy := NewProxy(codec) + // Unblock a parked control-plane read at shutdown without disturbing the + // write side, so the terminal workload events emitted while facades drain + // still reach the broker. An IPC connection can have its read deadline + // brought forward; stdio cannot, and ends instead when the parent closes + // the pipe. + proxy.interruptRead = func() { + if conn, ok := transport.(interface{ SetReadDeadline(time.Time) error }); ok { + _ = conn.SetReadDeadline(time.Now()) + } + } // Open a live view of this node's cluster mTLS trust fabric. While unclustered // the proxy serves only the loopback plaintext personality; once this node is // a member the same listener also serves the pin-gated LAN mTLS ingress, and diff --git a/services/nvpair-proxy/portstore_test.go b/services/nvpair-proxy/portstore_test.go index 6484c3b9..1442dcb1 100644 --- a/services/nvpair-proxy/portstore_test.go +++ b/services/nvpair-proxy/portstore_test.go @@ -29,15 +29,25 @@ func redirectConfigDir(t *testing.T) { t.Setenv("LOCALAPPDATA", dir) } -// freeTCPPort returns a port that was free a moment ago. +// freeTCPPort asks the OS for a port that was unused a moment ago, on the +// wildcard address, which is where a facade binds. // -// A bind probe cannot be held open and handed over, so the port is genuinely -// free when checked and may not be by the time a facade binds it. Callers that -// can tolerate a retry should use one — see enableOnFreePort in -// twofacade_test.go — because this helper cannot close that window. +// Two separate things can go wrong here and only one of them is fixable in this +// helper. Probing loopback would answer a different question altogether: a +// facade just torn down can still be lingering on 0.0.0.0:P while the OS +// happily hands out 127.0.0.1:P, and the caller's bind then fails as "address +// already in use". Matching the address the caller binds closes that one. +// +// What it cannot close is the window itself. A bind probe cannot be held open +// and handed over, so a port that really was free when checked may be taken by +// the time a facade binds it. Callers that can tolerate a retry should use one +// — see enableOnFreePort in twofacade_test.go. +// +// Both get likelier with each engine added, because every extra facade runs +// another round of setup and teardown through the same ephemeral range. func freeTCPPort(t *testing.T) int { t.Helper() - l, err := net.Listen("tcp", "127.0.0.1:0") + l, err := net.Listen("tcp", ":0") if err != nil { t.Fatalf("reserve free port: %v", err) } diff --git a/services/nvpair-proxy/proxy.go b/services/nvpair-proxy/proxy.go index 929c3d67..2735b759 100644 --- a/services/nvpair-proxy/proxy.go +++ b/services/nvpair-proxy/proxy.go @@ -399,6 +399,16 @@ type Proxy struct { // exists because each facade's request counter restarts at 1, so without it // a reused id after a restart would collide in the broker's store. runID string + + // interruptRead unblocks a pending control-plane read at shutdown, set by + // the entrypoint because the transport is its to own. Optional: a nil + // value just means the read loop waits for the read to return by itself. + // + // It interrupts the read side only, never the whole transport, so the + // terminal workload events emitted while facades drain still reach the + // broker. That distinction is why this is a callback rather than a Close + // on the codec. + interruptRead func() } // NewProxy builds a facade-less process host. Facades arrive via @@ -1281,6 +1291,16 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) { var wlSeq int64 nextWlSeq := func() int64 { wlSeq++; return wlSeq } + // inflight is the handle workload/cancel reaches this request through, and + // is nil for anything that is not a cancellable workload. The request is + // re-pointed at a context derived from the origin's, so cancelling it does + // everything a client hangup does: the retry loop below reads r.Context() + // before every dispatch and every backoff, each attempt's context is a + // child of it, and the exhaustion response is suppressed on it. A cancel + // therefore tears down the attempt in flight, stops further retries, and + // is classified by the same reporters — once, through terminalOnce. + var inflight *inflightRequest + // Emit workload:submitted the moment the request is admitted, before any // dispatch. A burst of concurrent inference requests must surface as job // cards immediately — the upstream engine serializes work on a single GPU @@ -1294,6 +1314,12 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) { // not tried would inflate its load. Each dispatch and each gap between // attempts re-points it. if isInf && model != "" { + requestCtx, cancelRequest := context.WithCancel(r.Context()) + r = r.WithContext(requestCtx) + var forget func() + inflight, forget = f.trackInflight(reqID, cancelRequest) + defer forget() + createdMs := start.UnixMilli() wl = &Workload{ ID: reqID, @@ -1347,15 +1373,29 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) { }) } + // cancelReason names why the request context ended. An asked-for cancel + // (workload/cancel) arrives as the same cancelled context a vanished client + // or our own shutdown does, and the workload's error text is user-visible, + // so reporting an operator's cancel as a disconnect would misstate what + // happened. Both reporters below go through this so the answer does not + // depend on which of them wins the race to emit. + cancelReason := func(otherwise string) string { + if inflight != nil && inflight.cancelled.Load() { + return "cancelled before completion" + } + return otherwise + } + // Watch for the client going away while the request is in flight. The // terminal event is otherwise emitted only after the stream copy returns; // a client that disconnects mid-stream can leave the copy blocked, so we // emit the terminal here the moment r.Context() is cancelled instead of - // waiting for the unwind. Cancelling r.Context() (client close, or our own - // shutdown) also propagates to the ReverseProxy's upstream request, so the - // engine stops generating. terminalOnce keeps this from double-emitting - // with the normal path. The half-open case (no FIN, r.Context() never - // fires) is caught instead by statusCapture's write deadline below. + // waiting for the unwind. Cancelling r.Context() (client close, a + // workload/cancel, or our own shutdown) also propagates to the + // ReverseProxy's upstream request, so the engine stops generating. + // terminalOnce keeps this from double-emitting with the normal path. The + // half-open case (no FIN, r.Context() never fires) is caught instead by + // statusCapture's write deadline below. if wl != nil { reqCtx := r.Context() finished := make(chan struct{}) @@ -1363,12 +1403,21 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) { go func() { select { case <-reqCtx.Done(): - emitTerminal("cancelled", "client disconnected before completion") + emitTerminal("cancelled", cancelReason("client disconnected before completion")) case <-finished: } }() } + // Backstop terminal. finalize below recovers the ErrAbortHandler panic a + // truncated upstream raises and classifies the outcome, so in the ordinary + // course this is a no-op behind terminalOnce: it is registered before + // finalize and therefore runs after it. It exists for the unwind finalize + // itself does not survive — a panic raised inside finalize before its own + // emitTerminal, say from the codec — so a workload never stays "running" + // for the life of the broker whatever path the handler leaves by. + defer emitTerminal("failed", "request handler exited before completion") + // committedSC is the statusCapture of the candidate we committed to // streaming; its wroteErr tells us after the fact whether the client write // failed (dead/half-open client) so we can mark the workload failed. It is @@ -1445,15 +1494,16 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) { if wl != nil { switch { case r.Context().Err() != nil: - // The request was cancelled before it finished — either the - // client disconnected or, on shutdown, we cancelled it to stop - // the in-flight inference. A mid-stream cancel never reaches - // ErrorHandler (the 200 headers are already sent), so without - // this branch it would be misreported as completed. (The - // watcher above usually beats us to it; emitTerminal makes that - // a no-op.) Cancelled rather than failed: nothing went wrong - // here, the requester stopped waiting. - emitTerminal("cancelled", "request cancelled before completion") + // The request was cancelled before it finished — the client + // disconnected, a workload/cancel asked for it, or, on + // shutdown, we cancelled it to stop the in-flight inference. A + // mid-stream cancel never reaches ErrorHandler (the 200 headers + // are already sent), so without this branch it would be + // misreported as completed. (The watcher above usually beats us + // to it; emitTerminal makes that a no-op.) Cancelled rather + // than failed: nothing went wrong here, the requester stopped + // waiting. + emitTerminal("cancelled", cancelReason("request cancelled before completion")) case committedSC != nil && committedSC.wroteErr != nil: // The response committed but a write to (or flush toward) the // client failed — typically the idle deadline tripping on a @@ -2538,16 +2588,41 @@ func subscribedToNode(p engineProfile, n noderec.DirectoryNode) (Node, bool) { TXT: n.AddressTXT(), IP: n.IP, ClusterUUID: n.ClusterUUID, - // Filter on this node's Ollama models only, not the cross-engine union, so - // a model that a dual-engine node serves solely via LM Studio isn't - // accepted as an Ollama owner here (falls back to the union for a peer - // that sends no attribution — see DirectoryNode.EngineModels). - Models: append([]string(nil), n.EngineModels(p.Name)...), + // Filter on this node's models for this engine only, never the + // cross-engine union, so a model a dual-engine node serves solely via + // another engine is not accepted as an owner here. + Models: append([]string(nil), p.eligibleModels(n)...), }, true } +// eligibleModels is the node's inventory that makes it an owner for this +// engine. +// +// The two on-demand engines read the catalog, falling back to the union for a +// peer that sends no per-engine attribution (see DirectoryNode.EngineModels). +// llama.cpp reads the loaded set with no fallback at all: a missing report +// means nothing is loaded, and treating catalog ids as eligible there would +// route to an owner that will not serve them. +func (p engineProfile) eligibleModels(n noderec.DirectoryNode) []string { + if p.ModelEligibility == loadedModels { + return n.EngineLoadedModels(p.Name) + } + return n.EngineModels(p.Name) +} + +// readLoop serves the control plane until the transport ends or ctx is done. +// +// The context is checked per iteration, and interruptRead is armed to unblock a +// read that is already parked, so teardown does not wait on a message that may +// never arrive. On the stdio transport the parent closing the pipe is what ends +// the read in practice, since Close there is deliberately a no-op; the +// interrupt is what ends it on an IPC connection, whose read deadline can be +// brought forward without disturbing the write side. func (p *Proxy) readLoop(ctx context.Context) error { - for { + if p.interruptRead != nil { + defer context.AfterFunc(ctx, p.interruptRead)() + } + for ctx.Err() == nil { msg, err := p.codec.Read() if err != nil { if err == io.EOF || ctx.Err() != nil { @@ -2558,6 +2633,7 @@ func (p *Proxy) readLoop(ctx context.Context) error { } p.handleMessage(msg) } + return nil } // handleMessage dispatches one control-plane message. @@ -2658,6 +2734,30 @@ func (p *Proxy) handleMessage(msg *Message) { log.Printf("failed to respond to facade/enable: %v", err) } + case "workload/cancel": + f, ok := p.requireFacade(msg, engine) + if !ok { + return + } + var params struct { + ID string `json:"id"` + RunID string `json:"runId"` + } + if json.Unmarshal(msg.Params, ¶ms) != nil || params.ID == "" || params.RunID == "" { + p.codec.RespondError(msg.ID, -32602, "invalid params: expected {\"id\",\"runId\"}") + return + } + // runId names this process, and request ids restart with it. A stale + // runId therefore identifies a request from a previous proxy lifetime + // whose id may now belong to an unrelated request, so it is refused + // rather than matched. The facade is addressed, which is what keeps + // one engine's cancel off another engine's identically numbered + // request. + accepted := params.RunID == p.runID && f.cancelInflight(params.ID) + if err := p.codec.Respond(msg.ID, map[string]bool{"accepted": accepted}); err != nil { + log.Printf("failed to respond to workload/cancel: %v", err) + } + case "nodes/list": f, ok := p.requireFacade(msg, engine) if !ok { diff --git a/services/nvpair-proxy/subscribed_test.go b/services/nvpair-proxy/subscribed_test.go index 70661604..4c5e8c6e 100644 --- a/services/nvpair-proxy/subscribed_test.go +++ b/services/nvpair-proxy/subscribed_test.go @@ -102,11 +102,11 @@ func TestSubscribedToNode(t *testing.T) { HostUUID: "uuid-a", Name: "host-a", IP: "10.0.0.5", - Models: []string{"llama"}, Services: map[noderec.ServiceKey]noderec.ServiceStatus{ tc.profile.DiscoveryService: {Port: tc.profile.FacadePort}, }, } + tc.advertise(&withService, "llama") got, ok := subscribedToNode(tc.profile, withService) if !ok { t.Fatal("node advertising this engine + IP should project") @@ -153,15 +153,12 @@ func TestSubscribedToNode(t *testing.T) { HostUUID: "uuid-d", Name: "host-d", IP: "10.0.0.7", - Models: []string{"mine", "theirs"}, - ModelsByEngine: map[string][]string{ - tc.profile.Name: {"mine"}, - other.Name: {"theirs"}, - }, Services: map[noderec.ServiceKey]noderec.ServiceStatus{ tc.profile.DiscoveryService: {Port: tc.profile.FacadePort}, }, } + advertiseEngine(&dual, tc.profile, "mine") + advertiseEngine(&dual, other, "theirs") got, ok = subscribedToNode(tc.profile, dual) if !ok { t.Fatal("dual-engine node advertising this engine should project") diff --git a/services/nvpair-proxy/terminal_test.go b/services/nvpair-proxy/terminal_test.go new file mode 100644 index 00000000..c90b5106 --- /dev/null +++ b/services/nvpair-proxy/terminal_test.go @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +// Coverage for a workload always reaching a terminal state. +// +// A job card that never leaves "running" is worse than one that reports a +// failure: the UI has no way to retire it and the scheduler keeps counting it +// as pending. These drive the paths that unwind past the normal terminal +// emission and assert one is emitted anyway. +// +// Run for every engine because the handler is shared. It came in with +// llama.cpp, whose streaming exposed it, but nothing about it is llama-specific +// and the other two engines had the same gap. + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// A committed response whose upstream body is truncated makes ReverseProxy +// abort the handler with ErrAbortHandler, which unwinds past every terminal +// path in the request. The workload must still be finalized, exactly once, +// for every engine. How that terminal is classified is pinned separately by +// TestHandleHTTP_UpstreamDiesMidStream_EmitsTerminal: a stream that already +// committed a 2xx terminates as completed, so this test asserts delivery and +// cardinality, not the label. +func TestTruncatedUpstreamStillFinalizesTheWorkload(t *testing.T) { + forEachEngine(t, func(t *testing.T, tc engineCase) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + // A declared length the body never satisfies, then a clean close. + w.Header().Set("Content-Length", "1000") + _, _ = io.WriteString(w, "data: partial\n\n") + })) + defer upstream.Close() + + rec := &recRW{} + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "truncating-node", upstream.URL, tc.advertisedModel)) + p := newTestProxy(tc.profile, NewCodec(rec), disc, tc.profile.FacadePort) + + // Served through a real listener rather than a recorder: the abort is + // raised by net/http while streaming a committed response, which a + // ResponseRecorder never reaches. + finished := make(chan struct{}) + front := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer close(finished) + p.soleFacade().handleHTTP(w, r) + })) + defer front.Close() + + client := &http.Client{Timeout: 3 * time.Second} + resp, err := client.Post(front.URL+tc.inferencePath, "application/json", + strings.NewReader(fmt.Sprintf(`{"model":%q,"stream":true}`, tc.requestedModel))) + if err == nil { + _, err = io.ReadAll(resp.Body) + resp.Body.Close() + } + if err == nil { + t.Fatal("a truncated upstream was presented to the client as a complete response") + } + + select { + case <-finished: + case <-time.After(3 * time.Second): + t.Fatal("the handler never unwound after the upstream truncated") + } + + terminals := func() int { return rec.count("workload:completed") + rec.count("workload:errored") } + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) && terminals() == 0 { + time.Sleep(20 * time.Millisecond) + } + if got := terminals(); got != 1 { + t.Fatalf("terminal workload events = %d, want exactly one after a truncated upstream", got) + } + }) +} diff --git a/services/nvpair-proxy/zombie_test.go b/services/nvpair-proxy/zombie_test.go index 306bfff4..aa9c548b 100644 --- a/services/nvpair-proxy/zombie_test.go +++ b/services/nvpair-proxy/zombie_test.go @@ -371,7 +371,7 @@ func TestHandleHTTP_RealSocketWriteDeadline(t *testing.T) { t.Fatalf("listen: %v", err) } srv := &http.Server{Handler: http.HandlerFunc(p.soleFacade().handleHTTP)} - go func() { _ = srv.Serve(ln) }() + go func() { _ = srv.Serve(cappedSendBufferListener{ln}) }() defer srv.Close() conn, err := net.Dial("tcp", ln.Addr().String()) @@ -379,6 +379,13 @@ func TestHandleHTTP_RealSocketWriteDeadline(t *testing.T) { t.Fatalf("dial proxy: %v", err) } defer conn.Close() + // Close the receive window quickly, for the same reason the send buffer is + // capped: the stall has to arrive well inside this test's own budget. + if tcpConn, ok := conn.(*net.TCPConn); ok { + if err := tcpConn.SetReadBuffer(socketBufferCap); err != nil { + t.Fatalf("cap client read buffer: %v", err) + } + } body := tc.inferenceBody() reqText := fmt.Sprintf("POST %s HTTP/1.1\r\n", tc.inferencePath) + @@ -419,6 +426,36 @@ func TestHandleHTTP_RealSocketWriteDeadline(t *testing.T) { } } +// socketBufferCap bounds both ends of the loopback connection in the two tests +// that need a client to stall. +const socketBufferCap = 4096 + +// cappedSendBufferListener caps the send buffer on every accepted connection. +// +// Both socket-level zombie tests work by letting the proxy's send buffer fill +// until a write or flush blocks, and how long that takes is a kernel tuning +// detail rather than anything the proxy controls. Linux autotunes loopback +// buffers into the megabytes, so the flush test's paced 1500-byte chunks — +// about 750 KB/s, deliberately slow so the block lands in Flush and not in +// Write — took longer to fill them than the test's own five-second budget. +// It failed there while passing on macOS, where the buffers are smaller. +// +// Capping both ends makes the backup arrive in kilobytes instead of megabytes, +// so what these tests measure is the deadline behaviour rather than the +// platform's buffer size. +type cappedSendBufferListener struct{ net.Listener } + +func (l cappedSendBufferListener) Accept() (net.Conn, error) { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + if tcpConn, ok := conn.(*net.TCPConn); ok { + _ = tcpConn.SetWriteBuffer(socketBufferCap) + } + return conn, nil +} + // TestHandleHTTP_RealSocketFlushDeadline is the flush-path counterpart to the // write-deadline test. A streaming response is flushed after every chunk // (ReverseProxy uses immediate flushing for chunked/streaming upstreams), so a @@ -475,7 +512,7 @@ func TestHandleHTTP_RealSocketFlushDeadline(t *testing.T) { t.Fatalf("listen: %v", err) } srv := &http.Server{Handler: http.HandlerFunc(p.soleFacade().handleHTTP)} - go func() { _ = srv.Serve(ln) }() + go func() { _ = srv.Serve(cappedSendBufferListener{ln}) }() defer srv.Close() conn, err := net.Dial("tcp", ln.Addr().String()) @@ -483,6 +520,13 @@ func TestHandleHTTP_RealSocketFlushDeadline(t *testing.T) { t.Fatalf("dial proxy: %v", err) } defer conn.Close() + // Close the receive window quickly, for the same reason the send buffer is + // capped: the stall has to arrive well inside this test's own budget. + if tcpConn, ok := conn.(*net.TCPConn); ok { + if err := tcpConn.SetReadBuffer(socketBufferCap); err != nil { + t.Fatalf("cap client read buffer: %v", err) + } + } body := tc.inferenceBody() reqText := fmt.Sprintf("POST %s HTTP/1.1\r\n", tc.inferencePath) + diff --git a/services/nvpair-tui/README.md b/services/nvpair-tui/README.md index 3963d1a8..d91979ce 100644 --- a/services/nvpair-tui/README.md +++ b/services/nvpair-tui/README.md @@ -30,9 +30,9 @@ Tabs: | **Overview** | Broker liveness/version/uptime (`ping`) and a per-worker health table derived from the broker's `supervisor:subprocess-crashed:*` errors. | | **Errors** | The service-error datastore (`errors:get-initial` + live `errors:update`); `c` clears the selected entry. | | **Nodes** | mDNS-discovered Ollama nodes (`discovery:subscribe` / `discovery:nodes-changed`). | -| **Proxies** | Ollama and LM Studio reverse proxies: status, discovered upstreams, select a node (`enter`/`a`), set the listen port (`p`). | -| **Workloads** | Live cluster workloads (`workloads:subscribe` / `workloads:upsert` / `workloads:remove`). | -| **Engines** | Local inference engines: install (`i`), start (`s`), stop (`x`), restart (`r`), uninstall (`u`). | +| **Proxies** | Ollama, LM Studio and llama.cpp reverse proxies: status, upstream selection and listen port. | +| **Workloads** | Baseline plus live workloads keyed by origin/engine/run/id. `c` requests cancellation of one local-origin llama request; other origins/engines are refused. | +| **Engines** | Install (`i`), start (`s`), stop (`x`), restart (`r`), uninstall (`u`); model inventory (`m`), pull (`p`), load (`L`), unload (`e`), delete (`d`), cancel pull (`c`), local GGUF import (`I`). No engine update key: managed llama has no update action, and the TUI never substitutes uninstall plus reinstall for one. | | **Cluster** | Pairing + membership: invite by address (`i`, shows the six-digit PIN — the first invite auto-founds a cluster of one), accept (`a`) / decline (`d`) an inbound invite, remove a member (`r`), leave (`L`). | | **Manual** | User-added nodes: add by address (`a`), remove (`r`). | | **Settings** | The node-settings store (force-ports, cluster auto-sync, cluster id/name). | @@ -40,6 +40,13 @@ Tabs: ## Keys +The model view uses arrow keys to scroll and `Esc` to return. Model actions +prefill the selected exact model ID and require Enter. llama downloads accept +`owner/repository:QUANT`; imports take a local GGUF path. Install support and +external ownership come from Engine Manager. Downloaded models are not loaded +until the runtime reports them resident. Cancellation acceptance is not a vendor +stop acknowledgement; the workload stream supplies the terminal outcome. + - `tab` / `shift+tab` (or `→` / `←`, `l` / `h`) — switch tabs - `?` — toggle full help - `q` / `ctrl+c` — quit (the broker is shut down cleanly on exit) diff --git a/services/nvpair-tui/ui/engines.go b/services/nvpair-tui/ui/engines.go index 809febfb..f963015e 100644 --- a/services/nvpair-tui/ui/engines.go +++ b/services/nvpair-tui/ui/engines.go @@ -21,12 +21,15 @@ import ( // engineStatus mirrors nvpair-engine-manager's EngineStatus snapshot, the // element of engine:get-installed and the engine:state-changed payload. type engineStatus struct { - Engine string `json:"engine"` - DisplayName string `json:"display_name"` - Installed bool `json:"installed"` - Running bool `json:"running"` - Healthy bool `json:"healthy"` - Port int `json:"port"` + Engine string `json:"engine"` + DisplayName string `json:"display_name"` + Installed bool `json:"installed"` + Running bool `json:"running"` + Healthy bool `json:"healthy"` + Port int `json:"port"` + InstallSupported bool `json:"install_supported"` + InstallReason string `json:"install_reason"` + Managed bool `json:"managed"` } // enginesView manages local inference engines via the engine-manager @@ -35,14 +38,20 @@ type engineStatus struct { // It can also pull a model (engine:action{action:"pull_model"}), rendering // the live engine:pull-progress feed the way remote pulls already show. type enginesView struct { - client *rpc.Client - table table.Model - order []string - byName map[string]engineStatus - status string - input textinput.Model - pulling bool - pullEngine string + client *rpc.Client + table table.Model + order []string + byName map[string]engineStatus + status string + input textinput.Model + pulling bool + pullEngine string + modelAction string + models table.Model + showModels bool + modelNames []string + pendingLoadEngine string + pendingLoadModel string width, height int } @@ -52,6 +61,13 @@ type enginesLoadedMsg struct { err error } +type engineModelsMsg struct { + engine string + names []string + loaded map[string]bool + err error +} + type engineOpMsg struct { what string engine string @@ -65,6 +81,12 @@ var ( engInstallKey = key.NewBinding(key.WithKeys("i"), key.WithHelp("i", "install")) engUninstallKey = key.NewBinding(key.WithKeys("u"), key.WithHelp("u", "uninstall")) engPullKey = key.NewBinding(key.WithKeys("p"), key.WithHelp("p", "pull model")) + engLoadKey = key.NewBinding(key.WithKeys("L"), key.WithHelp("L", "load model")) + engUnloadKey = key.NewBinding(key.WithKeys("e"), key.WithHelp("e", "unload model")) + engDeleteKey = key.NewBinding(key.WithKeys("d"), key.WithHelp("d", "delete model")) + engCancelKey = key.NewBinding(key.WithKeys("c"), key.WithHelp("c", "cancel model pull")) + engModelsKey = key.NewBinding(key.WithKeys("m"), key.WithHelp("m", "models / refresh")) + engImportKey = key.NewBinding(key.WithKeys("I"), key.WithHelp("I", "import GGUF path")) ) func newEnginesView(client *rpc.Client) *enginesView { @@ -72,6 +94,7 @@ func newEnginesView(client *rpc.Client) *enginesView { ti.Placeholder = "model name (e.g. llama3.2)" v := &enginesView{client: client, byName: map[string]engineStatus{}, input: ti} v.table = newTable(nil) + v.models = newTable(nil) return v } @@ -110,10 +133,41 @@ func (v *enginesView) SetSize(w, h int) { }) v.table.SetWidth(w) v.table.SetHeight(clampWidth(h-2, 1)) + v.models.SetColumns([]table.Column{{Title: "MODEL ID", Width: clampWidth(w-18, 10)}, {Title: "STATE", Width: 14}}) + v.models.SetWidth(w) + v.models.SetHeight(clampWidth(h-3, 1)) } func (v *enginesView) Update(msg tea.Msg) tea.Cmd { switch msg := msg.(type) { + case engineModelsMsg: + if msg.engine != "" && msg.engine != v.selectedEngine() { + return nil + } + if msg.err != nil { + v.showModels = false + v.status = "Model inventory unavailable: " + msg.err.Error() + return nil + } + v.modelNames = msg.names + rows := make([]table.Row, 0, len(msg.names)) + for _, name := range msg.names { + state := "downloaded" + if v.selectedEngine() == "llamacpp" && !v.byName[v.selectedEngine()].Managed { + state = "catalogue" + } + if msg.loaded[name] { + state = "loaded" + } + rows = append(rows, table.Row{name, state}) + } + v.models.SetRows(rows) + v.showModels = true + if msg.engine == v.pendingLoadEngine && v.pendingLoadModel != "" && msg.loaded[v.pendingLoadModel] { + v.status = "Model loaded: " + v.pendingLoadModel + v.pendingLoadEngine, v.pendingLoadModel = "", "" + } + return nil case enginesLoadedMsg: if msg.err != nil { v.status = "load engines failed: " + msg.err.Error() @@ -125,15 +179,47 @@ func (v *enginesView) Update(msg tea.Msg) tea.Cmd { return nil case engineOpMsg: - if msg.err != nil { + if errors.Is(msg.err, context.DeadlineExceeded) { + // The RPC client dropped its waiter, not the backend operation. + // Preserve pending residency so a later authoritative load can settle. + loadedModel := strings.TrimPrefix(strings.TrimPrefix(msg.what, "load_model "), "run_model ") + if v.status != "Model loaded: "+loadedModel { + v.status = fmt.Sprintf("%s %s: no final response yet; outcome unknown. Refresh to observe completion.", msg.what, msg.engine) + } + } else if msg.err != nil { v.status = fmt.Sprintf("%s %s failed: %s", msg.what, msg.engine, msg.err.Error()) + if msg.engine == v.pendingLoadEngine { + v.pendingLoadEngine, v.pendingLoadModel = "", "" + } + } else if strings.HasPrefix(msg.what, "load_model ") || strings.HasPrefix(msg.what, "run_model ") { + if v.pendingLoadModel != "" { + v.status = "Load requested; waiting for observed model residency." + } } else { v.status = fmt.Sprintf("%s %s ok", msg.what, msg.engine) } + if v.showModels && msg.engine == v.selectedEngine() { + return v.loadModelsCmd() + } return nil case NotificationMsg: switch msg.Msg.Method { + case "engine:models-changed": + var snapshot struct { + Loaded map[string][]string `json:"loadedByEngine"` + } + if decodeParams(msg.Msg.Params, &snapshot) == nil { + for _, name := range snapshot.Loaded[v.pendingLoadEngine] { + if name == v.pendingLoadModel && name != "" { + v.status = "Model loaded: " + name + v.pendingLoadEngine, v.pendingLoadModel = "", "" + } + } + } + if v.showModels { + return v.loadModelsCmd() + } case "engine:state-changed": var e engineStatus _ = decodeParams(msg.Msg.Params, &e) @@ -147,7 +233,10 @@ func (v *enginesView) Update(msg tea.Msg) tea.Cmd { Percent int `json:"percent"` } _ = decodeParams(msg.Msg.Params, &p) - v.status = fmt.Sprintf("install %s: %s (%d%%)", p.Engine, p.Stage, p.Percent) + v.status = fmt.Sprintf("install %s: %s", p.Engine, p.Stage) + if p.Percent >= 0 && p.Percent <= 100 { + v.status += fmt.Sprintf(" (%d%%)", p.Percent) + } case "engine:pull-progress": var p struct { Engine string `json:"engine"` @@ -170,7 +259,10 @@ func (v *enginesView) Update(msg tea.Msg) tea.Cmd { } v.status = fmt.Sprintf("pull %s failed: %s", p.Engine, detail) default: - v.status = fmt.Sprintf("pull %s: %s (%d%%)", p.Engine, p.Stage, p.Percent) + v.status = fmt.Sprintf("pull %s: %s", p.Engine, p.Stage) + if p.Percent >= 0 && p.Percent <= 100 { + v.status += fmt.Sprintf(" (%d%%)", p.Percent) + } } } return nil @@ -197,14 +289,47 @@ func (v *enginesView) handleKey(msg tea.KeyMsg) tea.Cmd { v.input, cmd = v.input.Update(msg) return cmd } - if key.Matches(msg, engPullKey) { + if v.showModels && msg.String() == "esc" { + v.showModels = false + return nil + } + if key.Matches(msg, engModelsKey) { + return v.loadModelsCmd() + } + var action string + switch { + case key.Matches(msg, engPullKey): + action = "pull_model" + case key.Matches(msg, engLoadKey): + action = "load_model" + case key.Matches(msg, engUnloadKey): + action = "unload_model" + case key.Matches(msg, engDeleteKey): + action = "delete_model" + case key.Matches(msg, engCancelKey): + action = "cancel_pull" + case key.Matches(msg, engImportKey): + action = "import_model" + } + if action != "" { engine := v.selectedEngine() if engine == "" { return nil } + if engine == "llamacpp" && !v.byName[engine].Managed { + v.status = "Model actions require a PAIR-managed llama app." + return nil + } v.pullEngine = engine + if action == "load_model" && engine == "ollama" { + action = "run_model" + } + v.modelAction = action v.pulling = true v.input.SetValue("") + if v.showModels && v.models.Cursor() >= 0 && v.models.Cursor() < len(v.modelNames) { + v.input.SetValue(v.modelNames[v.models.Cursor()]) + } v.input.Focus() return textinput.Blink } @@ -212,10 +337,74 @@ func (v *enginesView) handleKey(msg tea.KeyMsg) tea.Cmd { return cmd } var cmd tea.Cmd + if v.showModels { + v.models, cmd = v.models.Update(msg) + return cmd + } v.table, cmd = v.table.Update(msg) return cmd } +func (v *enginesView) loadModelsCmd() tea.Cmd { + engine := v.selectedEngine() + if engine == "" { + return nil + } + action := "list_models" + if engine == "llamacpp" && v.byName[engine].Managed { + action = "list_downloaded" + } + return func() tea.Msg { + ctx, cancel := context.WithTimeout(context.Background(), callTimeout) + defer cancel() + msg, err := v.client.Call(ctx, "engine:action", map[string]string{"engine": engine, "action": action}) + if err != nil { + return engineModelsMsg{err: err} + } + var inventory struct { + Data []struct { + ID string `json:"id"` + } `json:"data"` + Models []struct { + Name string `json:"name"` + Model string `json:"model"` + } `json:"models"` + } + if err := decodeParams(msg.Result, &inventory); err != nil { + return engineModelsMsg{err: err} + } + result := engineModelsMsg{engine: engine, loaded: map[string]bool{}} + for _, model := range inventory.Data { + if model.ID != "" { + result.names = append(result.names, model.ID) + } + } + for _, model := range inventory.Models { + name := model.Name + if name == "" { + name = model.Model + } + if name != "" { + result.names = append(result.names, name) + } + } + msg, err = v.client.Call(ctx, "engine:models", nil) + if err != nil { + return engineModelsMsg{err: err} + } + var snapshot struct { + Loaded map[string][]string `json:"loadedByEngine"` + } + if err := decodeParams(msg.Result, &snapshot); err != nil { + return engineModelsMsg{err: err} + } + for _, name := range snapshot.Loaded[engine] { + result.loaded[name] = true + } + return result + } +} + // pullParams builds the engine:action{action:"pull_model"} params for a pull. // The model name is sent under BOTH "name" and "model" — mirroring // PullModelStream's own empty-params default — because the two engines key it @@ -228,9 +417,8 @@ func pullParams(engine, model string) map[string]any { // submitPull issues engine:action{action:"pull_model"} for the selected engine. // Live download progress and the terminal result arrive as engine:pull-progress -// notifications; the synchronous response can outlast callTimeout for a large -// model, so a deadline error here is expected and ignored (the progress feed is -// the real signal). +// notifications. Mutations share the backend's long-operation budget; a client +// deadline still cannot establish that the backend operation failed. func (v *enginesView) submitPull() tea.Cmd { v.pulling = false v.input.Blur() @@ -242,12 +430,27 @@ func (v *enginesView) submitPull() tea.Cmd { } v.status = fmt.Sprintf("pull %s: %s...", engine, model) params := pullParams(engine, model) + action := v.modelAction + if action == "" { + action = "pull_model" + } + params["action"] = action + if action == "load_model" || action == "run_model" { + v.pendingLoadEngine, v.pendingLoadModel = engine, model + } + if action == "import_model" { + params["params"] = map[string]string{"path": model} + } + v.status = fmt.Sprintf("%s %s: %s...", action, engine, model) return call(v.client, "engine:action", params, func(_ *rpc.Message, err error) tea.Msg { + if action != "pull_model" { + return engineOpMsg{what: action + " " + model, engine: engine, err: err} + } if err != nil && !errors.Is(err, context.DeadlineExceeded) { return engineOpMsg{what: "pull " + model, engine: engine, err: err} } return nil - }) + }, engineOperationTimeout) } func (v *enginesView) handleAction(msg tea.KeyMsg) (tea.Cmd, bool) { @@ -270,10 +473,18 @@ func (v *enginesView) handleAction(msg tea.KeyMsg) (tea.Cmd, bool) { if engine == "" { return nil, true } + if what == "install" && engine == "llamacpp" && !v.byName[engine].InstallSupported { + v.status = "Install unavailable: " + v.byName[engine].InstallReason + return nil, true + } + if engine == "llamacpp" && v.byName[engine].Installed && !v.byName[engine].Managed { + v.status = "External llama.cpp runtime: lifecycle remains with its owner." + return nil, true + } v.status = what + " " + engine + "..." return call(v.client, method, map[string]string{"engine": engine}, func(_ *rpc.Message, err error) tea.Msg { return engineOpMsg{what: what, engine: engine, err: err} - }), true + }, engineOperationTimeout), true } func (v *enginesView) selectedEngine() string { @@ -323,8 +534,11 @@ func (v *enginesView) View() string { return footerStyle.Render("No engines known on this host.") } out := v.table.View() + if v.showModels { + out = v.models.View() + } if v.pulling { - out += "\npull model: " + v.input.View() + out += "\n" + v.modelAction + " (enter model ID; Esc cancels input): " + v.input.View() } if v.status != "" { out += "\n" + footerStyle.Render(v.status) @@ -333,7 +547,7 @@ func (v *enginesView) View() string { } func (v *enginesView) Help() []key.Binding { - return []key.Binding{engStartKey, engStopKey, engRestartKey, engInstallKey, engUninstallKey, engPullKey} + return []key.Binding{engStartKey, engStopKey, engRestartKey, engInstallKey, engUninstallKey, engModelsKey, engPullKey, engLoadKey, engUnloadKey, engDeleteKey, engCancelKey, engImportKey} } func yesNo(b bool) string { diff --git a/services/nvpair-tui/ui/engines_test.go b/services/nvpair-tui/ui/engines_test.go index e04a2c77..60cf7a3b 100644 --- a/services/nvpair-tui/ui/engines_test.go +++ b/services/nvpair-tui/ui/engines_test.go @@ -3,7 +3,152 @@ package ui -import "testing" +import ( + "context" + "encoding/json" + "errors" + tea "github.com/charmbracelet/bubbletea" + "nvpair-tui/rpc" + "strings" + "testing" + "time" +) + +func TestLongEngineCallKeepsObservedLoadTruth(t *testing.T) { + if engineOperationTimeout <= 30*time.Minute || callTimeout != 35*time.Second { + t.Fatal("engine operation budget must cover the backend without extending ordinary calls") + } + v := newEnginesView(nil) + v.pendingLoadEngine, v.pendingLoadModel = "llamacpp", "owner/model:Q4" + timedOut := engineOpMsg{what: "load_model owner/model:Q4", engine: "llamacpp", err: context.DeadlineExceeded} + v.Update(timedOut) + if v.pendingLoadModel == "" || !strings.Contains(v.status, "outcome unknown") || strings.Contains(v.status, "failed") { + t.Fatalf("client timeout invented backend failure: %s", v.status) + } + v.Update(NotificationMsg{Msg: &rpc.Message{Method: "engine:models-changed", Params: json.RawMessage(`{"loadedByEngine":{"llamacpp":["owner/model:Q4"]}}`)}}) + v.Update(timedOut) + if v.pendingLoadModel != "" || v.status != "Model loaded: owner/model:Q4" { + t.Fatalf("late client timeout erased observed success: %s", v.status) + } + v.pendingLoadEngine, v.pendingLoadModel = "llamacpp", "other/model:Q4" + v.Update(engineOpMsg{what: "load_model other/model:Q4", engine: "llamacpp", err: errors.New("vendor rejected load")}) + if v.pendingLoadModel != "" || !strings.Contains(v.status, "failed") { + t.Fatal("actual backend error was hidden") + } +} + +func TestUnknownEngineProgressIsIndeterminate(t *testing.T) { + for _, method := range []string{"engine:install-progress", "engine:pull-progress"} { + v := newEnginesView(nil) + v.Update(NotificationMsg{Msg: &rpc.Message{Method: method, Params: json.RawMessage(`{"engine":"llamacpp","stage":"installing","percent":-1}`)}}) + if strings.Contains(v.status, "%") { + t.Fatalf("unknown progress shown as percent: %s", v.status) + } + v.Update(NotificationMsg{Msg: &rpc.Message{Method: method, Params: json.RawMessage(`{"engine":"llamacpp","stage":"installing","percent":50}`)}}) + if !strings.Contains(v.status, "50%") { + t.Fatal("known progress omitted") + } + } +} + +func TestModelRefreshPreservesActionFailure(t *testing.T) { + v := newEnginesView(nil) + v.SetSize(90, 20) + v.merge(engineStatus{Engine: "llamacpp", Installed: true, Managed: true}) + v.Update(engineOpMsg{what: "pull invalid-reference", engine: "llamacpp", err: errors.New("expected owner/repository")}) + v.Update(engineModelsMsg{engine: "llamacpp", names: []string{"cached"}, loaded: map[string]bool{}}) + if !strings.Contains(v.status, "expected owner/repository") { + t.Fatal("inventory refresh hid the actionable failure") + } +} + +func TestModelSnapshotSettlesObservedLoad(t *testing.T) { + v := newEnginesView(nil) + v.SetSize(90, 20) + v.merge(engineStatus{Engine: "llamacpp", Installed: true, Managed: true}) + v.pendingLoadEngine, v.pendingLoadModel = "llamacpp", "cached" + v.Update(engineModelsMsg{engine: "llamacpp", names: []string{"cached"}, loaded: map[string]bool{"cached": true}}) + if v.pendingLoadModel != "" || v.status != "Model loaded: cached" { + t.Fatal("observed loaded snapshot left a pending load") + } +} + +func TestLlamaLoadAcceptanceWaitsForObservation(t *testing.T) { + v := newEnginesView(nil) + v.pendingLoadEngine, v.pendingLoadModel = "llamacpp", "owner/model:Q4" + v.Update(engineOpMsg{what: "load_model owner/model:Q4", engine: "llamacpp"}) + if v.pendingLoadModel == "" || !strings.Contains(v.status, "waiting") { + t.Fatal("RPC acceptance completed the load") + } + v.Update(NotificationMsg{Msg: &rpc.Message{Method: "engine:models-changed", Params: json.RawMessage(`{"loadedByEngine":{"llamacpp":["owner/model:Q4"]}}`)}}) + if v.pendingLoadModel != "" || !strings.Contains(v.status, "Model loaded") { + t.Fatal("loaded observation did not settle action") + } + v.Update(engineOpMsg{what: "load_model owner/model:Q4", engine: "llamacpp"}) + if strings.Contains(v.status, "waiting") { + t.Fatal("late RPC response reopened a settled load") + } +} + +func TestLlamaManagedActionsAndOwnership(t *testing.T) { + v := newEnginesView(nil) + v.SetSize(90, 20) + v.merge(engineStatus{Engine: "llamacpp", Installed: true, Managed: true}) + for _, tc := range []struct { + key rune + action string + }{{'L', "load_model"}, {'e', "unload_model"}, {'d', "delete_model"}, {'c', "cancel_pull"}, {'I', "import_model"}} { + v.handleKey(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{tc.key}}) + if !v.pulling || v.modelAction != tc.action { + t.Fatalf("key %c: action %s", tc.key, v.modelAction) + } + v.handleKey(tea.KeyMsg{Type: tea.KeyEsc}) + } + v.merge(engineStatus{Engine: "llamacpp", Installed: true, Managed: false}) + if cmd := v.handleKey(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{'L'}}); cmd != nil || v.pulling { + t.Fatal("external engine accepted model mutation") + } + if cmd, handled := v.handleAction(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{'x'}}); cmd != nil || !handled { + t.Fatal("external engine accepted stop") + } +} + +// TestNoManagedLlamaUpdateControl pins the scoped-down engine surface: the TUI +// offers install, lifecycle, uninstall and model actions for managed llama, but +// no runtime update key, help entry, or engine:action{action:"update"} call. +func TestNoManagedLlamaUpdateControl(t *testing.T) { + v := newEnginesView(nil) + v.SetSize(90, 20) + v.merge(engineStatus{Engine: "llamacpp", Installed: true, Managed: true}) + for _, b := range v.Help() { + if strings.Contains(strings.ToLower(b.Help().Desc), "update") { + t.Fatalf("help still advertises an update control: %q", b.Help().Desc) + } + for _, k := range b.Keys() { + if k == "U" { + t.Fatal("U is still bound in the engines view") + } + } + } + if cmd, handled := v.handleAction(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{'U'}}); handled || cmd != nil { + t.Fatal("U still dispatches an engine lifecycle action") + } + _ = v.handleKey(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{'U'}}) + if v.pulling || v.status != "" { + t.Fatalf("U changed engine view state: pulling=%v status=%q", v.pulling, v.status) + } +} + +func TestLlamaModelInventorySeparatesDownloadedAndLoaded(t *testing.T) { + v := newEnginesView(nil) + v.SetSize(90, 20) + v.merge(engineStatus{Engine: "llamacpp", Installed: true, Managed: true}) + v.Update(engineModelsMsg{names: []string{"cold", "resident"}, loaded: map[string]bool{"resident": true}}) + rows := v.models.Rows() + if rows[0][1] != "downloaded" || rows[1][1] != "loaded" { + t.Fatalf("inventory = %v", rows) + } +} // TestPullParamsSendsBothKeys guards the LM Studio pull fix: the pull params // must carry the model under BOTH "name" (Ollama's /api/pull body key) and diff --git a/services/nvpair-tui/ui/proxies_test.go b/services/nvpair-tui/ui/proxies_test.go new file mode 100644 index 00000000..46bac15a --- /dev/null +++ b/services/nvpair-tui/ui/proxies_test.go @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package ui + +import ( + "encoding/json" + "testing" + + "nvpair-tui/rpc" +) + +func TestProxiesViewIncludesLlamaCpp(t *testing.T) { + v := newProxiesView(nil) + if len(v.engines) != 3 { + t.Fatalf("engines = %d, want 3", len(v.engines)) + } + got := v.engines[2] + if got.label != "llama.cpp" || got.prefix != "llamacpp-proxy" { + t.Fatalf("engine[2] = {%q, %q}, want {llama.cpp, llamacpp-proxy}", got.label, got.prefix) + } +} + +func TestHandleNotificationLlamaCppProxy(t *testing.T) { + v := newProxiesView(nil) + params, err := json.Marshal(map[string]int{"port": 8080}) + if err != nil { + t.Fatal(err) + } + v.handleNotification(&rpc.Message{ + Method: "llamacpp-proxy:ready", + Params: params, + }) + e := v.engines[2] + if !e.ready || e.port != 8080 { + t.Fatalf("llamacpp engine ready=%v port=%d, want ready :8080", e.ready, e.port) + } + if v.engines[0].ready { + t.Fatal("ollama proxy must not consume llamacpp-proxy notifications") + } +} diff --git a/services/nvpair-tui/ui/rpccmd.go b/services/nvpair-tui/ui/rpccmd.go index 007f56dd..1b8546fc 100644 --- a/services/nvpair-tui/ui/rpccmd.go +++ b/services/nvpair-tui/ui/rpccmd.go @@ -12,11 +12,13 @@ import ( tea "github.com/charmbracelet/bubbletea" ) -// callTimeout bounds a single broker request. It is generous enough to -// cover the broker's slowest relay (the 30s cluster-manager path) plus -// headroom, so a healthy call never times out under us. +// callTimeout bounds ordinary broker requests, not long engine mutations. const callTimeout = 35 * time.Second +// Engine Manager allows 30 minutes for acquisition and model operations. +// Keep its terminal response observable, with a little relay headroom. +const engineOperationTimeout = 31 * time.Minute + // NotificationMsg carries one broker server-push frame into the Bubble // Tea update loop. Every view receives it. type NotificationMsg struct{ Msg *rpc.Message } @@ -60,9 +62,13 @@ func waitForNotification(client *rpc.Client) tea.Cmd { // call issues a broker request on a background goroutine and feeds the // outcome back into the update loop via decode, which maps the response // (or error) to a view-specific message. -func call(client *rpc.Client, method string, params any, decode func(*rpc.Message, error) tea.Msg) tea.Cmd { +func call(client *rpc.Client, method string, params any, decode func(*rpc.Message, error) tea.Msg, budget ...time.Duration) tea.Cmd { + timeout := callTimeout + if len(budget) > 0 { + timeout = budget[0] + } return func() tea.Msg { - ctx, cancel := context.WithTimeout(context.Background(), callTimeout) + ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() msg, err := client.Call(ctx, method, params) return decode(msg, err) diff --git a/services/nvpair-tui/ui/workloads.go b/services/nvpair-tui/ui/workloads.go index ea7931df..f09c9af8 100644 --- a/services/nvpair-tui/ui/workloads.go +++ b/services/nvpair-tui/ui/workloads.go @@ -16,16 +16,14 @@ type workload struct { ID string `json:"id"` Model string `json:"model"` Engine string `json:"engine"` + RunID string `json:"runId"` State string `json:"state"` OriginatedFrom string `json:"originatedFrom"` + ScheduledOn string `json:"scheduledOn"` CreatedAt int64 `json:"createdAt"` // Unix millis } -// workloadsView shows cluster-wide inference workloads. The table is built -// purely from the live workloads:upsert / workloads:remove stream after -// subscribing, so a workload already in flight when the TUI starts stays -// invisible until its next transition. The broker does expose -// workloads:get-initial for a baseline; this view does not yet call it. +// workloadsView combines the initial snapshot with live workload events. type workloadsView struct { client *rpc.Client table table.Model @@ -37,6 +35,16 @@ type workloadsView struct { } type workloadsSubscribedMsg struct{ err error } +type workloadsInitialMsg struct { + workloads []workload + err error +} +type workloadCancelMsg struct { + accepted bool + err error +} + +var workloadCancelKey = key.NewBinding(key.WithKeys("c"), key.WithHelp("c", "cancel local llama request")) func newWorkloadsView(client *rpc.Client) *workloadsView { v := &workloadsView{client: client, byKey: map[string]workload{}} @@ -65,14 +73,42 @@ func (v *workloadsView) SetSize(w, h int) { {Title: "AGE", Width: age}, }) v.table.SetWidth(w) - v.table.SetHeight(clampWidth(h-1, 1)) + v.table.SetHeight(clampWidth(h-3, 1)) } func (v *workloadsView) Update(msg tea.Msg) tea.Cmd { switch msg := msg.(type) { + case workloadCancelMsg: + if msg.err != nil { + v.status = "Cancel failed: " + msg.err.Error() + } else if msg.accepted { + v.status = "Cancellation requested; awaiting terminal workload event." + } else { + v.status = "Request is no longer active in that proxy run." + } + return nil case workloadsSubscribedMsg: if msg.err != nil { v.status = "workloads subscribe failed: " + msg.err.Error() + return nil + } + return call(v.client, "workloads:get-initial", nil, func(msg *rpc.Message, err error) tea.Msg { + if err != nil { + return workloadsInitialMsg{err: err} + } + var result struct { + Workloads []workload `json:"workloads"` + } + err = decodeParams(msg.Result, &result) + return workloadsInitialMsg{workloads: result.Workloads, err: err} + }) + case workloadsInitialMsg: + if msg.err != nil { + v.status = "Workload baseline unavailable: " + msg.err.Error() + return nil + } + for _, w := range msg.workloads { + v.upsert(w) } return nil @@ -88,13 +124,40 @@ func (v *workloadsView) Update(msg tea.Msg) tea.Cmd { var p struct { WorkloadID string `json:"workloadId"` OriginatedFrom string `json:"originatedFrom"` + Engine string `json:"engine"` + RunID string `json:"runId"` } _ = decodeParams(msg.Msg.Params, &p) - v.remove(workloadKey(p.OriginatedFrom, p.WorkloadID)) + for key, w := range v.byKey { + if w.OriginatedFrom == p.OriginatedFrom && w.ID == p.WorkloadID && (p.Engine == "" || p.Engine == w.Engine) && (p.RunID == "" || p.RunID == w.RunID) { + v.remove(key) + } + } } return nil case tea.KeyMsg: + if key.Matches(msg, workloadCancelKey) { + index := v.table.Cursor() + if index < 0 || index >= len(v.order) { + return nil + } + w := v.byKey[v.order[index]] + if w.Engine != "llamacpp" || w.RunID == "" || w.State != "running" { + v.status = "Only active llama requests with a run identity can be cancelled." + return nil + } + return call(v.client, "workloads:cancel", map[string]string{"id": w.ID, "runId": w.RunID, "engine": w.Engine, "originatedFrom": w.OriginatedFrom}, func(msg *rpc.Message, err error) tea.Msg { + if err != nil { + return workloadCancelMsg{err: err} + } + var result struct { + Accepted bool `json:"accepted"` + } + err = decodeParams(msg.Result, &result) + return workloadCancelMsg{accepted: result.Accepted, err: err} + }) + } var cmd tea.Cmd v.table, cmd = v.table.Update(msg) return cmd @@ -103,7 +166,10 @@ func (v *workloadsView) Update(msg tea.Msg) tea.Cmd { } func (v *workloadsView) upsert(w workload) { - key := workloadKey(w.OriginatedFrom, w.ID) + key := workloadKey(w.OriginatedFrom, w.ID, w.Engine, w.RunID) + if previous, ok := v.byKey[key]; ok && (previous.State == "completed" || previous.State == "failed") && w.State != "completed" && w.State != "failed" { + return + } if _, ok := v.byKey[key]; !ok { v.order = append(v.order, key) } @@ -141,15 +207,34 @@ func (v *workloadsView) refreshRows() { } func (v *workloadsView) View() string { - if v.status != "" { - return statusErrStyle.Render(v.status) - } if len(v.order) == 0 { - return footerStyle.Render("No active workloads. Live cluster workloads will appear here as they run.") + empty := "No active workloads. Live cluster workloads will appear here as they run." + // An empty list and a failed baseline fetch look identical otherwise, + // so a subscribe or get-initial error would be written to v.status and + // never rendered. + if v.status != "" { + return footerStyle.Render(empty) + "\n" + footerStyle.Render(v.status) + } + return footerStyle.Render(empty) + } + detail := "" + if index := v.table.Cursor(); index >= 0 && index < len(v.order) { + w := v.byKey[v.order[index]] + target := w.ScheduledOn + if target == "" { + target = "unknown (not reported)" + } + detail = "Origin: " + w.OriginatedFrom + " | Runs on: " + target } - return v.table.View() + return v.table.View() + "\n" + footerStyle.Width(v.width).Render(detail) + "\n" + footerStyle.Render(v.status) } -func (v *workloadsView) Help() []key.Binding { return nil } +func (v *workloadsView) Help() []key.Binding { return []key.Binding{workloadCancelKey} } -func workloadKey(origin, id string) string { return origin + "/" + id } +func workloadKey(origin, id string, identity ...string) string { + key := origin + "/" + id + for _, part := range identity { + key += "/" + part + } + return key +} diff --git a/services/nvpair-tui/ui/workloads_test.go b/services/nvpair-tui/ui/workloads_test.go new file mode 100644 index 00000000..1aa3fdc8 --- /dev/null +++ b/services/nvpair-tui/ui/workloads_test.go @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package ui + +import ( + "strings" + "testing" +) + +func TestWorkloadsShowReportedTargetWithoutInferringOrigin(t *testing.T) { + v := newWorkloadsView(nil) + v.SetSize(100, 20) + w := workload{ID: "1", Engine: "llamacpp", OriginatedFrom: "origin-node", ScheduledOn: "serving-node", State: "running"} + v.upsert(w) + if !strings.Contains(v.View(), "Runs on: serving-node") { + t.Fatal("reported serving node missing from selected workload") + } + w.ScheduledOn = "" + v.upsert(w) + if !strings.Contains(v.View(), "Runs on: unknown (not reported)") { + t.Fatal("missing target must remain unknown") + } +} + +func TestWorkloadsKeepEngineRunIdentityAndTerminalTruth(t *testing.T) { + v := newWorkloadsView(nil) + v.SetSize(100, 20) + for _, w := range []workload{ + {ID: "1", Engine: "ollama", RunID: "a", OriginatedFrom: "self", State: "running"}, + {ID: "1", Engine: "llamacpp", RunID: "a", OriginatedFrom: "self", State: "completed"}, + {ID: "1", Engine: "llamacpp", RunID: "b", OriginatedFrom: "self", State: "running"}, + } { + v.upsert(w) + } + v.Update(workloadsInitialMsg{workloads: []workload{{ID: "1", Engine: "llamacpp", RunID: "a", OriginatedFrom: "self", State: "running"}}}) + if len(v.byKey) != 3 { + t.Fatalf("identity collision: %v", v.byKey) + } + if v.byKey[workloadKey("self", "1", "llamacpp", "a")].State != "completed" { + t.Fatal("late baseline regressed completed request") + } +} diff --git a/services/nvpair-ui-broker/ENGINE_SETTINGS.md b/services/nvpair-ui-broker/ENGINE_SETTINGS.md index a6e2351f..37c4963f 100644 --- a/services/nvpair-ui-broker/ENGINE_SETTINGS.md +++ b/services/nvpair-ui-broker/ENGINE_SETTINGS.md @@ -5,10 +5,13 @@ SPDX-License-Identifier: Apache-2.0 # Engine settings protocol -The broker owns combined settings operations for Ollama and LM Studio. Each -node owns its own configuration. `nodeId` selects a discovered, currently pinned -peer; omission or the local host ID selects this node. Bulk propagation is not -part of this API. +The broker owns combined settings operations for every engine in its proxy +table — Ollama, LM Studio and llama.cpp — through one profile-generic path: the +same journal, validation, rebind and recovery code serves each of them, and an +engine's differences live in its `engineProxyProfile` entry rather than in the +settings code. Each node owns its own configuration. `nodeId` selects a +discovered, currently pinned peer; omission or the local host ID selects this +node. Bulk propagation is not part of this API. | Method | Request | Result | | --- | --- | --- | @@ -16,14 +19,45 @@ part of this API. | `engine:preview-settings` | `{engine, nodeId?, expectedRevision, settings, resolution?}` | Normalized settings, errors, conflict, restart/rebind summary | | `engine:apply-settings` | Preview request plus `requestId` | `{revision, phase}` acknowledgement | -`engine` is `ollama` or `lmstudio`. `settings` contains all three fields: -`serverPort`, `proxyPort`, `launchText`. The last field contains arguments and -leading environment assignments, without the executable or startup subcommand. -The argument grammar is +`engine` names an entry in `nvpair-shared/engines`: `ollama`, `lmstudio` or +`llamacpp`; any other value is refused as an engine without settings. `settings` +contains all three fields: `serverPort`, `proxyPort`, `launchText`. The last +field contains arguments and leading environment assignments, without the +executable or startup subcommand. The argument grammar is [`pair-arguments-v1`](../nvpair-engine-manager/LAUNCH_TEXT.md). Preview does not change component configuration or runtime. A snapshot read may persist the initial revision baseline or reconcile an external component change. +## Engines + +Whether a snapshot is `editable`, and why not, comes from engine-manager's +`engine:get-launch`: an engine whose manifest declares no `editable_launch` +block reports "This engine does not support launch settings.", and the broker +then refuses previews and applies for it — port-only ones included, because the +proxy port travels in the same operation. What the broker adds per engine is +the port choreography an explicit choice has to override: + +- **Ollama** is an adopted engine: facade `11434`, managed backend from `11435`. + Its settings validation also reserves an inherited `OLLAMA_HOST` alias port, + and an explicit choice clears the pending backend move and re-syncs that alias + reservation with engine-manager. +- **LM Studio** is a managed engine: facade `1234`, managed backend from `1235`. + Its standalone proxy once wrote `1235` as its own default, so that value is + excluded when a saved proxy-port store is migrated (below). +- **llama.cpp** is a managed engine that PAIR assigns both ports for: facade + `8080`, managed engine port `8081`. An inherited `LLAMA_ARG_PORT` moves the + facade to the port it names and the engine to one above it. It has no + editable launch arguments unless its manifest declares `editable_launch`; as + shipped, engine-manager therefore reports it as not editable and its settings + are read-only until the manifest gains that block. + +For every engine an explicit settings choice disables automatic facade takeover +in the same way: `explicitSettings` is set on the engine's proxy runtime, its +managed-facade claim is dropped, the chosen server and proxy ports become the +backend and startup ports, a bind failure on the chosen proxy port is surfaced +instead of triggering a fallback, and readiness reconciliation opens the +engine's startup gate without moving anything. + Each full snapshot includes desired `settings`, `revision`, `appliedRevision`, `phase` (`idle`, `applying`, `succeeded`, `failed`), `requestId`, `error`, `effectiveServerPort`, `effectiveProxyPort`, `running`, `adopted`, `editable`, @@ -47,8 +81,9 @@ the node lock across validation, journal acceptance and application. The worker holds its engine lock across override persistence, stopping with the old launch context, installing the new context, the narrow proxy rebind callback and one start/readiness wait. The callback validates an active operation token and -acquires no node configuration lock. Legacy public port setters enter this same -operation and retain their response shape. Legacy lifecycle port overrides and +acquires no node configuration lock. The port-only setters — `engine:set-port` +and every engine's `-proxy:set-port` — enter this same operation and +retain their response shape. Legacy lifecycle port overrides and each engine's automatic port reconciliation share the node lock and revision reconciliation. Validate both ports together against the unchanged counterpart, registered @@ -76,12 +111,14 @@ reporting the same startup failure twice. `engine-settings-operations.json` in the app data directory holds previous and desired settings, revision, resume intent and operation receipts. Component -engine overrides and proxy-port files remain their owners' configuration. -Acceptance is written and synced before component changes. Interrupted applying -records replay before enabled-engine restoration. Explicit settings override -automatic facade defaults on later startup. Unreadable journal data is preserved -and suppresses automatic component rewrites. On first upgrade, a valid existing -proxy-port choice is preserved as explicit when it differs from the engine port +engine overrides and each proxy's saved-port file (`proxy-port.json`, +`lmstudio-proxy-port.json`, `llamacpp-proxy-port.json`, as named by the shared +engine table) remain their owners' configuration. Acceptance is written and +synced before component changes. Interrupted applying records replay before +enabled-engine restoration. Explicit settings override automatic facade defaults +on later startup. Unreadable journal data is preserved and suppresses automatic +component rewrites. On first upgrade, a valid existing proxy-port choice in any +engine's store is preserved as explicit when it differs from the engine port (except LM Studio's obsolete 1235 default). Old stores lack choice provenance, so a historical automatic move is conservatively preserved too. Unreadable data blocks automatic recovery. The journal retains up to 256 terminal receipts diff --git a/services/nvpair-ui-broker/README.md b/services/nvpair-ui-broker/README.md index 91cd7f91..20c37010 100644 --- a/services/nvpair-ui-broker/README.md +++ b/services/nvpair-ui-broker/README.md @@ -23,7 +23,7 @@ namespace: | --- | --- | --- | | `nvpair-node-scanner` | Discovery daemon: advertises this host's one `_nvpair-node._tcp` record and browses the LAN | `discovery:*` | | `nvpair-node-info` | Local GPU / CPU / memory inventory over HTTP at `/v1/node-info` | — (HTTP only) | -| `nvpair-proxy` | One process hosting an inference proxy and router facade per enabled engine | `ollama-proxy:*`, `lmstudio-proxy:*` | +| `nvpair-proxy` | One process hosting an inference proxy and router facade per enabled engine | `ollama-proxy:*`, `lmstudio-proxy:*`, `llamacpp-proxy:*` | | `nvpair-engine-manager` | Local engine and model control plane; also serves `GET /v1/models` to peers | `engine:*` | | `nvpair-cluster-manager` | Node identity, trusted-node store, PIN pairing | `cluster:*`, `nodes:*` | | `nvpair-workload-manager` | Cluster workload relay between this node and peers | `workloads:*` | @@ -39,6 +39,24 @@ lifecycle, and relay rules. Two responsibilities live in the broker itself rather than in a worker: +The headless `workloads:cancel` method takes `{id, runId, engine, +originatedFrom}`. Only `engine: "llamacpp"` and this broker's exact local origin +UUID are accepted. It returns `{accepted}` after cancelling that active proxy +request context; the ordinary workload event reports its terminal outcome. +Foreign-origin requests and other engines are unsupported. A stale proxy run or +finished request returns `accepted: false`. Desktop and TUI use this same API. + +`llamacpp-proxy:set-port` is intercepted exactly as `ollama-proxy:set-port` and +`lmstudio-proxy:set-port` are: it runs through the authoritative engine-settings +operation (see [`ENGINE_SETTINGS.md`](ENGINE_SETTINGS.md)), which refuses a port +another configured engine or proxy holds, refuses the move outright when no +engine manager can vouch for port ownership, and records an accepted port as an +explicit choice that later startups honor. An engine port assignment cannot take +the active llama proxy listener. The broker advertises `lc` only while Engine +Manager reports the llama process running and its health check succeeds; +missing manager state is not permission to probe and adopt an unrelated +stock-port listener. + - **Engine advertising.** The broker polls local Ollama and LM Studio every 5 s and registers each running engine's port (`ol` / `lm`) with the discovery daemon, so both are carried in this host's single `_nvpair-node` record. The @@ -73,7 +91,7 @@ Bidirectional newline-delimited JSON-RPC 2.0 — same conventions as every other | `--scanner-path ` | `./nvpair-node-scanner[.exe]` in the CWD | Explicit path to the `nvpair-node-scanner` binary the broker should spawn | | `--node-info-path ` | `./nvpair-node-info[.exe]` in the CWD | Explicit path to the `nvpair-node-info` binary the broker should spawn. When omitted and no default sibling exists, the broker runs without the local inventory server (non-fatal); when set to an invalid path, the broker exits with an error | | `--proxy-path ` | `./nvpair-proxy[.exe]` in the CWD | Explicit path to the `nvpair-proxy` binary. One process fronts every engine: the broker spawns it once and then sends a `facade/enable` per entry in `--proxy-engines`. Same optional semantics as `--node-info-path`: an absent default sibling means no local proxies (non-fatal); an invalid explicit path exits with an error | -| `--proxy-engines ` | every engine in `nvpair-shared/engines` (currently `ollama,lmstudio`) | Which engines to front with a proxy. An unrecognized name exits with an error rather than being skipped, so a typo cannot look like it worked. An engine left out is not started **and not prepared** — the broker will not relocate an engine whose facade nothing is going to claim | +| `--proxy-engines ` | every engine in `nvpair-shared/engines` (currently `ollama,lmstudio,llamacpp`) | Which engines to front with a proxy. An unrecognized name exits with an error rather than being skipped, so a typo cannot look like it worked. An engine left out is not started **and not prepared** — the broker will not relocate an engine whose facade nothing is going to claim | | `--workload-manager-path ` | `./nvpair-workload-manager[.exe]` in the CWD | Explicit path to the `nvpair-workload-manager` binary the broker spawns for the cluster workload relay. Same optional semantics as `--node-info-path`: an absent default sibling means no workload relay (non-fatal); an invalid explicit path exits with an error | | `--errors-path ` | `./nvpair-errors[.exe]` in the CWD | Explicit path to the `nvpair-errors` binary the broker spawns (with `--peer-sync`) for the service-error pipeline. Same optional semantics as `--node-info-path`: an absent default sibling means the error pipeline is disabled — producers' errors are dropped (non-fatal); an invalid explicit path exits with an error | | `--engine-manager-path ` | `./nvpair-engine-manager[.exe]` in the CWD | Explicit path to the `nvpair-engine-manager` binary the broker spawns for engine management. Same optional semantics as `--node-info-path` | @@ -91,13 +109,13 @@ Logs go to **stderr** (shared `applog` format, same as every other NVPAIR binary On startup — **before** emitting `app:ready` — the broker spawns the scanner and (when available) node-info, `nvpair-proxy`, the workload-manager, and the cluster-manager as child processes over stdio. The proxy is spawned up front but doesn't gate `app:ready` — each of its facades announces its listen port asynchronously (see below). None of the auxiliary workers gate `app:ready`. -**`nvpair-node-scanner`** (the consolidated discovery daemon) is spawned first. It pushes `discovery:node-discovered`, `discovery:node-updated`, and `discovery:node-removed` notifications into the broker, which maintains them in an in-memory map keyed by `id`. Clients query that map via `discovery:get-nodes` and — once they've opted in via `discovery:subscribe` — receive a `discovery:nodes-changed` notification on every store mutation. The raw `discovery:node-*` notifications are never forwarded as-is. The scanner polls healthy node-info endpoints on a staggered two-second cadence, backs consecutive remote failures off to a 30-second cap, and emits compact `discovery:node-telemetry` observations containing maximum GPU utilization, validity, and age; these remain internal to broker scheduling. The broker registers this node's local service ports (`ni`/`er`/`wl`/`cl`/`em`, plus `ol`/`lm` from the engine poller) with the daemon over the same link, so the daemon can advertise them all in one `_nvpair-node` record. +**`nvpair-node-scanner`** (the consolidated discovery daemon) is spawned first. It pushes `discovery:node-discovered`, `discovery:node-updated`, and `discovery:node-removed` notifications into the broker, which maintains them in an in-memory map keyed by `id`. Clients query that map via `discovery:get-nodes` and — once they've opted in via `discovery:subscribe` — receive a `discovery:nodes-changed` notification on every store mutation. The raw `discovery:node-*` notifications are never forwarded as-is. The scanner polls healthy node-info endpoints on a staggered two-second cadence, backs consecutive remote failures off to a 30-second cap, and emits compact `discovery:node-telemetry` observations containing maximum GPU utilization, validity, and age; these remain internal to broker scheduling. The broker registers this node's local service ports (`ni`/`er`/`wl`/`cl`/`em`, plus `ol`/`lm`/`lc` from the engine poller) with the daemon over the same link, so the daemon can advertise them all in one `_nvpair-node` record. **`nvpair-node-info`** is spawned next. It's a server, not an event source: it stands up the local `/v1/node-info` HTTP endpoint (GPU/CPU/memory inventory). It does not advertise itself — the broker registers its `ni` port with the scanner daemon, which carries it in the node record, and a peer's daemon fetches `/v1/node-info` over plain HTTP to enrich the node. The broker doesn't read anything back from node-info's stdout (drained and discarded). Spawning it is **optional**: if the binary can't be resolved (and no `--node-info-path` override was given) the broker logs a warning and continues serving discovery without it. -**Engine advertising.** The broker runs an internal 5 s poll loop against local Ollama at its configured backend port and LM Studio (`GET /v1/models`) and reconciles this node's engine registration with the scanner daemon: +**Engine advertising.** The broker runs an internal 5 s poll loop against local Ollama at its configured backend port, LM Studio, and llama.cpp (`GET /v1/models`) and reconciles this node's engine registration with the scanner daemon: -- engine **up** → register `ol` / `lm` at the engine's real port, never the proxy's own, to prevent a self-forward loop; +- engine **up** → register `ol` / `lm` / `lc` at the **proxy** port (never the engine) and hand the engine's loopback port to that proxy via `node/set-local-backend`; equal proxy/engine ports are refused so the proxy cannot self-forward; - engine **down** → unregister it. The daemon folds those registrations into this host's single `_nvpair-node` record, so a peer discovers the engine through the shared channel. The model list is not part of that registration — it's served over HTTP by `nvpair-engine-manager` (the `em` service, `GET /v1/models`) and enriched onto each node by the peer's daemon. There is no separate advertiser subprocess and no manual-advertise RPC. @@ -383,9 +401,9 @@ Two relay-specific error cases: - If no proxy is being supervised (or it has exited), the broker replies with error `-32000` `"ollama-proxy not available"`. - `ollama-proxy:shutdown` is **refused** with error `-32601` — the broker owns the proxy's lifecycle, so a client can't terminate it independently. Shut the broker down instead (which tears the proxy down with it). -#### `ollama-proxy:set-port` +#### `ollama-proxy:set-port` / `lmstudio-proxy:set-port` / `llamacpp-proxy:set-port` -**Intercepted, not relayed verbatim.** A port-only caller — `nvpair-tui` is the one in tree — gets to move a single port without rendering the whole launch settings form, but the change still runs through the same authoritative settings operation the desktop editor uses, so a port set from the terminal cannot diverge from one set from the UI. The broker reads the engine's current settings, substitutes the requested proxy port, and applies the result. +**Intercepted, not relayed verbatim.** A port-only caller — `nvpair-tui` is the one in tree — gets to move a single port without rendering the whole launch settings form, but the change still runs through the same authoritative settings operation the desktop editor uses, so a port set from the terminal cannot diverge from one set from the UI. The broker reads the engine's current settings, substitutes the requested proxy port, and applies the result. Every engine in the proxy table is served by this one path; the examples below use Ollama, and the other engines' methods behave identically for their own facade. A **requested port that is already in use is refused** with error `-32000 "port %d is already in use"`. The broker does not pick a different port on the caller's behalf: silently binding somewhere else left clients pointed at a port nothing was listening on. Retry with a free port. A request that collides with an inherited `OLLAMA_HOST` alias is refused with its own message naming that alias. The response echoes the requested port (`{"port": }`) once it is bound. @@ -393,12 +411,16 @@ A **requested port that is already in use is refused** with error `-32000 "port {"jsonrpc":"2.0","id":9,"method":"ollama-proxy:set-port","params":{"port":11500}} ``` -Automatic conflict resolution still exists, but only for a port the user did not just choose: when the proxy announces a (re)bound port on startup and a running engine has since taken it, the broker steers the proxy to a free port and surfaces a sticky `warning` into the errors pipeline (id `ollama-proxy:port-bumped`, `action:"none"`) explaining the move. That path **never changes an engine's port** — only the proxy is moved. Error `-32000 "ollama-proxy not available"` when no proxy is supervised. +Automatic conflict resolution still exists, but only for a port the user did not just choose: when the proxy announces a (re)bound port on startup and a running engine has since taken it, the broker steers the proxy to a free port and surfaces a sticky `warning` into the errors pipeline (id `ollama-proxy:port-bumped`, `action:"none"`) explaining the move. That path **never changes an engine's port** — only the proxy is moved. When no proxy is supervised the engine's settings snapshot is not editable, so the move is refused with that settings error rather than binding nothing. #### `lmstudio-proxy:get-status` / `lmstudio-proxy:subscribe` / `lmstudio-proxy:unsubscribe` / `lmstudio-proxy:` (generic relay) The LM Studio counterpart of the `ollama-proxy:*` surface runs the supervised `lmstudio-proxy` on compatibility port `:1234` and tracks the managed LM Studio backend on `:1235`. With managed port ownership enabled (the default), the broker identifies and moves an existing LM Studio server through engine-manager before allowing the proxy to claim `1234`; unknown owners are left untouched and force a warned proxy fallback. Disabling managed ownership preserves explicit custom backend and proxy ports. `lmstudio-proxy:get-status` reports the actual bound port; `lmstudio-proxy:subscribe` / `lmstudio-proxy:unsubscribe` opt into / out of its `lmstudio-proxy:` stream; and any other `lmstudio-proxy:` is relayed verbatim with the prefix stripped (`nodes/list`, `node/select`, `node/add-manual`, `node/remove-manual`, ...). `lmstudio-proxy:shutdown` is refused because the broker owns lifecycle ordering. Workload and error events feed the shared streams exactly as Ollama's do. +#### `llamacpp-proxy:get-status` / `llamacpp-proxy:subscribe` / `llamacpp-proxy:unsubscribe` / `llamacpp-proxy:` (generic relay) + +The llama.cpp counterpart of the same surface. Its facade claims `:8080` — or the port an inherited `LLAMA_ARG_PORT` names, since that is where a user's clients already point — and the managed llama.cpp engine runs one above it, on `:8081` by default. PAIR assigns both ports: engine-manager installs and owns the engine, so with managed port ownership enabled the broker moves the engine first and judges the facade afterwards, exactly as for LM Studio, and an unknown owner of the facade port is left untouched behind a warned proxy fallback. An explicit settings choice for either port disables that takeover, as it does for every engine. `llamacpp-proxy:get-status`, the subscription methods, the verbatim relay and the refused `llamacpp-proxy:shutdown` behave as for the other two engines, and `llamacpp-proxy:set-port` is the intercepted settings operation described above. + #### `ollama-proxy:subscribe` Opts the peer into the `ollama-proxy:` stream (off by default). On a fresh subscription the broker immediately replays the proxy's last `ready` payload as a baseline `ollama-proxy:ready` (if the proxy has come up), so a subscriber learns the port without a separate `ollama-proxy:get-status`. @@ -483,7 +505,7 @@ Opt into / out of the `engine:` stream (off by default). Acks `{ subscrib Any other `engine:*` request is forwarded to `nvpair-engine-manager` verbatim and its response relayed straight back. This covers the whole engine control plane: `engine:get-installed`, `engine:describe`, `engine:status`, `engine:install`, `engine:uninstall`, `engine:start`, `engine:stop`, `engine:restart`, `engine:action`, `engine:logs`, `engine:errors`, `engine:models`. Lifecycle ops run for minutes (reporting progress via the `engine:install-progress` / `engine:state-changed` push events), so the relay imposes **no broker-side timeout** — fire the request and watch the event stream for the outcome. Error `-32000 "engine-manager not available"` when no engine-manager is supervised. -`engine:set-port` is **not** in that generic set. Like `proxy:set-port` it is intercepted and run through the authoritative settings operation, so moving an engine's server port from a port-only caller validates and restarts exactly as the full editor does, and persists as a manifest override that survives a restart. Its response is the engine's `engine:status` result. +`engine:set-port` is **not** in that generic set. Like every `-proxy:set-port` it is intercepted and run through the authoritative settings operation, so moving an engine's server port from a port-only caller validates and restarts exactly as the full editor does, and persists as a manifest override that survives a restart. Its response is the engine's `engine:status` result. #### `settings/` (generic relay) @@ -493,7 +515,7 @@ Any `settings/*` request is forwarded to `nvpair-node-settings` and its response Relayed to `nvpair-manual-nodes`. `node/add` (`{ address, name?, tls_port?, mtls? }`) registers a user-added node and probes it; `node/remove` (`{ id }`) drops it; `nodes/list` returns the tracked manual nodes. Manually added nodes also surface in the shared `discovery:get-nodes` / `discovery:nodes-changed` snapshot — the broker merges `nvpair-manual-nodes`' `node/discovered|updated|removed` into the same store the scanner feeds. A `nvpair-manual-nodes` restart loses the in-memory entries because neither that worker nor the broker persists an authoritative copy, so clients must re-add manual nodes after a restart. Error `-32000 "manual-nodes not available"` when no manual-nodes worker is supervised. -**Manual → proxy bridge.** When the broker supervises both `nvpair-manual-nodes` and a proxy, it also bridges a manual node whose engine is reachable into that proxy via `node/add-manual` (host/port from the node's per-engine status), so inference can route to it through `ollama-proxy:node/select` / `lmstudio-proxy:node/select` just like a relay-discovered node. This is per-engine: a node whose `ollama_*` status is up is bridged into `ollama-proxy` (host/port from `ollama_port`), and one whose `lmstudio_*` status is up into `lmstudio-proxy` (from `lmstudio_port`) — a node running both is bridged into both. Manual nodes are by definition the ones that never appear via the daemon's `_nvpair-node` discovery, so this explicit add is what makes them routable. The bridge tracks reachability: an engine that goes down (or a node that is removed, or whose prober crashes) is pulled back out with `node/remove-manual`. A proxy that isn't supervised → that leg is a no-op; manual nodes still appear in the discovery snapshot as before. +**Manual → proxy bridge.** When the broker supervises both `nvpair-manual-nodes` and a proxy, it also bridges a manual node whose engine is reachable into that proxy via `node/add-manual` (host/port from the node's per-engine status), so inference can route to it through `ollama-proxy:node/select` / `lmstudio-proxy:node/select` / `llamacpp-proxy:node/select` just like a relay-discovered node. This is per-engine: a node whose `ollama_*` status is up is bridged into `ollama-proxy` (host/port from `ollama_port`), one whose `lmstudio_*` status is up into `lmstudio-proxy` (from `lmstudio_port`), and one whose `llamacpp_*` status is up into `llamacpp-proxy` (from `llamacpp_port`) — a node running more than one is bridged into each. Manual nodes are by definition the ones that never appear via the daemon's `_nvpair-node` discovery, so this explicit add is what makes them routable. The bridge tracks reachability: an engine that goes down (or a node that is removed, or whose prober crashes) is pulled back out with `node/remove-manual`. A proxy that isn't supervised → that leg is a no-op; manual nodes still appear in the discovery snapshot as before. #### `cluster:` / `nodes:` (generic relay) diff --git a/services/nvpair-ui-broker/advertiser.go b/services/nvpair-ui-broker/advertiser.go index 05d85fdf..060f3dc0 100644 --- a/services/nvpair-ui-broker/advertiser.go +++ b/services/nvpair-ui-broker/advertiser.go @@ -19,9 +19,15 @@ import ( // a fixed 11434 would advertise the proxy as Ollama and make the proxy — and // peers — self-forward into a loop. The real port is resolved per poll via // localEnginePort. +// +// llama.cpp is the one engine whose client-facing port PAIR assigns rather than +// inherits, so its engine port is the table's EnginePortBase and its facade +// port is what a client targets. Nothing claims llama.cpp's upstream default. var ( - defaultOllamaPort = ollamaProxyProfile.FacadePort - defaultLMStudioPort = lmstudioProxyProfile.FacadePort + defaultOllamaPort = ollamaProxyProfile.FacadePort + defaultLMStudioPort = lmstudioProxyProfile.FacadePort + defaultLlamaCppPort = llamacppProxyProfile.EnginePortBase + defaultLlamaCppProxyPort = llamacppProxyProfile.FacadePort ) const ( @@ -187,6 +193,64 @@ func (b *Broker) reconcileAdvertiseLMStudio(client *http.Client) { } } +// runAutoAdvertiseLlamaCpp is the llama.cpp sibling of runAutoAdvertiseLMStudio: +// it polls the local llama-server and reconciles this node's lc service +// registration against it. +// +// The three advertise loops are still one per engine even though one process +// now hosts every facade. Only the health probe and the listen-port lookup are +// table-driven so far; folding the loops themselves into the engine table is +// deliberately left as follow-up rather than done alongside adding an engine. +func (b *Broker) runAutoAdvertiseLlamaCpp(ctx context.Context) { + client := &http.Client{Timeout: 2 * time.Second} + ticker := time.NewTicker(autoAdvertiseInterval) + defer ticker.Stop() + + b.reconcileAdvertiseLlamaCpp(client) + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + b.reconcileAdvertiseLlamaCpp(client) + } + } +} + +// reconcileAdvertiseLlamaCpp brings this node's lc registration into line with +// the local llama.cpp server, mirroring reconcileAdvertiseLMStudio: it +// advertises the promoted proxy port (never the engine) and hands the engine's +// loopback port to the facade via node/set-local-backend. +// +// This engine has a managed facade like the other two, but it needs neither +// sibling's recovery path. The hazard both guard against is a stale +// engine:status still naming the stock port after the facade claimed it, which +// would advertise the facade as the engine and forward it to itself; the equal +// proxy/engine refuse below already drops exactly that reading. And where the +// siblings fall back to their stock port when engine-manager is unavailable, +// this one falls back to no port at all, so an unreachable manager cannot be +// read as permission to adopt whatever answers on 8080. +func (b *Broker) reconcileAdvertiseLlamaCpp(client *http.Client) { + // An unavailable manager is unknown ownership, never authority to adopt a + // process answering on the stock port. + enginePort, probe := b.localEnginePort("llamacpp", 0) + probe = probe && enginePort > 0 + proxyPort := b.llamaCppProxyListenPort() + if proxyPort != 0 && enginePort == proxyPort { + enginePort = 0 + probe = false + } + up := probe && proxyPort != 0 && enginePort != proxyPort && checkEngineHealth(llamacppProxyProfile, client, enginePort) + if up { + b.registerService(noderec.RegisterParams{Service: noderec.ServiceLlamaCpp, Port: proxyPort}) + b.setProxyLocalBackend(b.getLlamaCppProxy(), "llamacpp", enginePort, true) + } else { + b.unregisterService(noderec.ServiceLlamaCpp) + b.setProxyLocalBackend(b.getLlamaCppProxy(), "llamacpp", enginePort, false) + } +} + // proxyLocalBackend is the node/set-local-backend payload: the loopback engine // the proxy's cluster mTLS ingress forwards to, and the proxy's own self // candidate on the local routing path. @@ -265,6 +329,10 @@ func (b *Broker) lmstudioProxyListenPort() int { return b.engineProxyListenPort(lmstudioProxyProfile) } +func (b *Broker) llamaCppProxyListenPort() int { + return b.engineProxyListenPort(llamacppProxyProfile) +} + // checkEngineHealth reports whether a local engine is answering on the given // port, by probing the path its own liveness convention uses. The port is // resolved per poll (see localEnginePort) rather than hardcoded, so the proxy diff --git a/services/nvpair-ui-broker/advertiser_test.go b/services/nvpair-ui-broker/advertiser_test.go index a32c2cbe..00bb8ff2 100644 --- a/services/nvpair-ui-broker/advertiser_test.go +++ b/services/nvpair-ui-broker/advertiser_test.go @@ -23,6 +23,9 @@ func TestLocalEnginePortFallback(t *testing.T) { if got, ok := b.localEnginePort("lmstudio", defaultLMStudioPort); !ok || got != defaultLMStudioPort { t.Errorf("no engine-manager: localEnginePort = (%d, %v), want (%d, true)", got, ok, defaultLMStudioPort) } + if got, ok := b.localEnginePort("llamacpp", defaultLlamaCppPort); !ok || got != defaultLlamaCppPort { + t.Errorf("no engine-manager: localEnginePort = (%d, %v), want (%d, true)", got, ok, defaultLlamaCppPort) + } } func TestRunningEnginePort(t *testing.T) { diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go index c5da7fd1..90f7654c 100644 --- a/services/nvpair-ui-broker/broker.go +++ b/services/nvpair-ui-broker/broker.go @@ -203,6 +203,14 @@ type Broker struct { lmstudioPortReady chan struct{} lmstudioPortReadyOnce sync.Once lmstudioReadyMu sync.Mutex + // llamaCppProxyGeneration is a plain atomic like LM Studio's rather than + // alias-guarded like Ollama's: llama.cpp claims no host alias, so nothing + // has to be committed alongside the bump. + llamaCppProxyGeneration atomic.Uint64 + llamaCppProxyPublishedGeneration atomic.Uint64 + llamacppPortReady chan struct{} + llamacppPortReadyOnce sync.Once + llamacppReadyMu sync.Mutex store *discoveryStore telemetry *telemetryCache // relayDir is the discovery directory, fed by the promoted daemon's @@ -417,6 +425,7 @@ func NewBroker(codec *Codec, paths workerPaths) *Broker { workloads: workloadstore.New(), ollamaPortReady: make(chan struct{}), lmstudioPortReady: make(chan struct{}), + llamacppPortReady: make(chan struct{}), } } @@ -535,15 +544,20 @@ func (b *Broker) restoreEnabledEnginesAfterPortGate(ctx context.Context) bool { return true } +// runEngineAvailabilityAfterPortGates starts every engine's advertise loop once +// managed port ownership has settled. One loop runs on this goroutine and the +// rest get their own, so the caller is not left holding a goroutine per engine. func (b *Broker) runEngineAvailabilityAfterPortGates( ctx context.Context, runOllama func(context.Context), runLMStudio func(context.Context), + runLlamaCpp func(context.Context), ) bool { if !b.restoreEnabledEnginesAfterPortGate(ctx) { return false } go runOllama(ctx) + go runLlamaCpp(ctx) runLMStudio(ctx) return true } @@ -760,6 +774,8 @@ func (b *Broker) enableEngineFacade( return b.enableProxyFacadeWithFallback(ctx, pp, b.ollamaFacadeSpec(alias), b.ollamaFallbackPort) case lmstudioProxyProfile.Name: return b.enableProxyFacadeWithFallback(ctx, pp, b.lmstudioFacadeSpec(), b.lmstudioFallbackPort) + case llamacppProxyProfile.Name: + return b.enableProxyFacadeWithFallback(ctx, pp, b.llamacppFacadeSpec(), b.llamacppFallbackPort) default: return fmt.Errorf("no facade spec for engine %q", profile.Name) } @@ -786,6 +802,11 @@ func (b *Broker) blockAndFinishEngineProxy(profile engineProxyProfile) { _, _ = b.blockManagedLMStudioFacade("the LM Studio proxy facade could not be brought up", nil) } b.finishLMStudioProxyTerminal() + case llamacppProxyProfile.Name: + if b.llamacppState().managedFacade.Load() { + _, _ = b.blockManagedLlamaCppFacade("the llama.cpp proxy facade could not be brought up", nil) + } + b.finishLlamaCppProxyTerminal() default: slog.Warn("no terminal handling for engine", "engine", profile.Name) } @@ -871,9 +892,11 @@ func (b *Broker) spawnProxy() (supervisedHandle, error) { // lmstudioProxyGenerationIsCurrent is what drops the stale run. lmstudioGeneration := b.lmstudioProxyGeneration.Add(1) + llamacppGeneration := b.llamaCppProxyGeneration.Add(1) + pp, err := startProxy(engines.ProxyComponent, b.proxyPath, applog.LevelString(), b.relayDir, func(method string, params json.RawMessage) { - b.forwardProxyProcessNotification(ollamaGeneration, lmstudioGeneration, method, params) + b.forwardProxyProcessNotification(ollamaGeneration, lmstudioGeneration, llamacppGeneration, method, params) }, b.proxyArgs()...) if err != nil { return nil, err @@ -886,6 +909,7 @@ func (b *Broker) spawnProxy() (supervisedHandle, error) { } } b.lmstudioProxyPublishedGeneration.Store(lmstudioGeneration) + b.llamaCppProxyPublishedGeneration.Store(llamacppGeneration) slog.Info("proxy started", "path", b.proxyPath, "pid", pp.cmd.Process.Pid) // The child is serving its JSON-RPC channel but has no listeners yet, so @@ -936,6 +960,7 @@ func (b *Broker) spawnProxy() (supervisedHandle, error) { // exists to prevent. A later successful spawn re-derives the real state. b.markOllamaPortReady() b.markLMStudioPortReady() + b.markLlamaCppPortReady() return nil, fmt.Errorf("no engine facade could be brought up in %s", b.proxyPath) } @@ -957,6 +982,9 @@ func (b *Broker) spawnProxy() (supervisedHandle, error) { if ready, port := pp.Status(lmstudioProxyProfile.Name); ready && port > 0 { go b.reconcileLMStudioProxyPortOnReadyForGeneration(lmstudioGeneration, port) } + if ready, port := pp.Status(llamacppProxyProfile.Name); ready && port > 0 { + go b.reconcileLlamaCppProxyPortOnReadyForGeneration(llamacppGeneration, port) + } return pp, nil } @@ -968,7 +996,7 @@ func (b *Broker) spawnProxy() (supervisedHandle, error) { // and must be handled exactly once, not once per engine — routing a workload // record through both handlers would record every job twice. func (b *Broker) forwardProxyProcessNotification( - ollamaGeneration, lmstudioGeneration uint64, method string, params json.RawMessage, + ollamaGeneration, lmstudioGeneration, llamacppGeneration uint64, method string, params json.RawMessage, ) { engine, bare := engines.SplitAddressedMethod(method) switch engine { @@ -976,6 +1004,8 @@ func (b *Broker) forwardProxyProcessNotification( b.forwardProxyNotificationForGeneration(ollamaGeneration, method, params) case lmstudioProxyProfile.Name: b.forwardLMStudioProxyNotificationForGeneration(lmstudioGeneration, method, params) + case llamacppProxyProfile.Name: + b.forwardLlamaCppProxyNotificationForGeneration(llamacppGeneration, method, params) case "": if !b.routeProcessScopedProxyNotification(bare, params) { slog.Debug("ignoring unaddressed proxy notification", "method", bare) @@ -1119,7 +1149,8 @@ func (b *Broker) spawnEngineMgr() (supervisedHandle, error) { // adopt or start a backend on the port the proxy alias owns. b.syncCurrentEngineOllamaHostAliasReservation() b.reconcileLMStudioProxyAfterEngineManagerReady() - // Initial restore waits for both managed compatibility ports below. A later + b.reconcileLlamaCppProxyAfterEngineManagerReady() + // Initial restore waits for the managed compatibility ports below. A later // engine-manager respawn sees the already-open gates and restores here. if b.managedPortOwnershipReady() { b.restoreEnabledEngines(w) @@ -1487,6 +1518,7 @@ func (b *Broker) forwardEngineNotification(method string, params json.RawMessage } if method == "engine:ready" { b.reconcileLMStudioProxyAfterEngineManagerReady() + b.reconcileLlamaCppProxyAfterEngineManagerReady() go func() { b.engineConfigMu.Lock() defer b.engineConfigMu.Unlock() @@ -2180,9 +2212,11 @@ func (b *Broker) Serve(ctx context.Context) error { slog.Warn("proxy failed to start; continuing without local engine proxies", "path", b.proxyPath, "err", err) b.proxySup = nil - // No facade will come up, so both engines take the terminal path: + // No facade will come up, so every engine takes the terminal path: // report the blocked claim where one was made, then finish, which - // is what returns the OLLAMA_HOST alias reservation. + // is what returns the OLLAMA_HOST alias reservation and opens each + // ownership gate. An engine missed here keeps its gate shut for the + // life of the process. if b.ollamaState().managedFacade.Load() { b.blockManagedOllamaFacade("the proxy could not be started") } @@ -2191,6 +2225,10 @@ func (b *Broker) Serve(ctx context.Context) error { _, _ = b.blockManagedLMStudioFacade("the proxy could not be started", nil) } b.finishLMStudioProxyTerminal() + if b.llamacppState().managedFacade.Load() { + _, _ = b.blockManagedLlamaCppFacade("the proxy could not be started", nil) + } + b.finishLlamaCppProxyTerminal() } else { defer b.proxySup.Stop() } @@ -2200,7 +2238,7 @@ func (b *Broker) Serve(ctx context.Context) error { // startup attempts have established either readiness or a terminal outcome. // This prevents a restored engine from taking a persisted proxy port before // the broker can resolve ownership. - go b.runEngineAvailabilityAfterPortGates(ctx, b.runAutoAdvertise, b.runAutoAdvertiseLMStudio) + go b.runEngineAvailabilityAfterPortGates(ctx, b.runAutoAdvertise, b.runAutoAdvertiseLMStudio, b.runAutoAdvertiseLlamaCpp) // nvpair-workload-manager is another auxiliary worker: it relays local // workload lifecycle events to peer nodes and surfaces peer events @@ -2410,22 +2448,16 @@ func (b *Broker) forwardProxyNotificationForGeneration(generation uint64, method // enable retries. Deciding it here rather than there is deliberate: only // this path knows whether the managed facade has to be given up, and it // owns telling the user. - if method == "error" { - var ep struct { - Code string `json:"code"` - Port int `json:"port"` - } - if json.Unmarshal(params, &ep) == nil && ep.Code == "bind-failed" && !b.ollamaState().explicitSettings.Load() { - switch { - case b.ollamaState().managedFacade.Load() && ep.Port == managedOllamaFacadePort: - b.blockManagedOllamaFacade("another process acquired the compatibility port during startup") - default: - // Without this the retry asks for the same port and the enable - // fails identically, leaving the facade down for the life of - // the process. - fallback := b.setOllamaProxyFallback(ep.Port) - slog.Warn("Ollama proxy bind failed; retrying on fallback", "port", ep.Port, "fallback", fallback) - } + if port, recover := b.facadeBindFailure(ollamaProxyProfile, method, params); recover { + switch { + case b.ollamaState().managedFacade.Load() && port == managedOllamaFacadePort: + b.blockManagedOllamaFacade("another process acquired the compatibility port during startup") + default: + // Without this the retry asks for the same port and the enable + // fails identically, leaving the facade down for the life of + // the process. + fallback := b.setOllamaProxyFallback(port) + slog.Warn("Ollama proxy bind failed; retrying on fallback", "port", port, "fallback", fallback) } } // When the proxy announces a (re)bound port — notably its restored port @@ -3236,9 +3268,11 @@ func (b *Broker) handleMessage(msg *Message) { case "engine:set-port": go b.handleSettingsPortRPC(msg, "") case "ollama-proxy:set-port": - go b.handleSettingsPortRPC(msg, "ollama") + go b.handleSettingsPortRPC(msg, ollamaProxyProfile.Name) case "lmstudio-proxy:set-port": - go b.handleSettingsPortRPC(msg, "lmstudio") + go b.handleSettingsPortRPC(msg, lmstudioProxyProfile.Name) + case "llamacpp-proxy:set-port": + go b.handleSettingsPortRPC(msg, llamacppProxyProfile.Name) case "lmstudio-proxy:get-status": // Answered locally from the lmstudio-proxy handle's captured state, @@ -3278,6 +3312,45 @@ func (b *Broker) handleMessage(msg *Message) { log.Printf("failed to respond to lmstudio-proxy:unsubscribe: %v", err) } + case "llamacpp-proxy:get-status": + var result ProxyStatusResult + if p := b.getLlamaCppProxy(); p != nil { + ready, port := p.Status(llamacppProxyProfile.Name) + result.Ready = ready + result.Port = port + } + if err := b.codec.Respond(msg.ID, result); err != nil { + log.Printf("failed to respond to llamacpp-proxy:get-status: %v", err) + } + + case "llamacpp-proxy:subscribe": + b.proxyMu.Lock() + wasSubscribed := b.setEngineProxySubscribed(llamacppProxyProfile, true) + b.proxyMu.Unlock() + if err := b.codec.Respond(msg.ID, SubscriptionResult{Subscribed: true}); err != nil { + log.Printf("failed to respond to llamacpp-proxy:subscribe: %v", err) + } + if !wasSubscribed { + if p := b.getLlamaCppProxy(); p != nil { + if rp := p.ReadyParams(llamacppProxyProfile.Name); rp != nil { + if err := b.codec.Notify("llamacpp-proxy:ready", rp); err != nil { + slog.Warn("emit baseline llamacpp-proxy:ready failed", "err", err) + } + } + } + } + + case "llamacpp-proxy:unsubscribe": + b.proxyMu.Lock() + b.setEngineProxySubscribed(llamacppProxyProfile, false) + b.proxyMu.Unlock() + if err := b.codec.Respond(msg.ID, SubscriptionResult{Subscribed: false}); err != nil { + log.Printf("failed to respond to llamacpp-proxy:unsubscribe: %v", err) + } + + case "workloads:cancel": + b.cancelLlamaWorkload(msg) + case "workloads:subscribe": b.workloadsMu.Lock() b.workloadsSubscribed = true @@ -3435,6 +3508,17 @@ func (b *Broker) relayToEngineNow(msg *Message) { }() return } + if needsLlamaCppPortGate(msg.Method, msg.Params) && b.llamacppPortOwnershipPending() { + go func() { + select { + case <-b.llamacppPortReady: + b.relayToEngine(msg) + case <-time.After(rpcWorkerCallTimeout): + _ = b.codec.RespondError(msg.ID, -32000, "llama.cpp port setup did not finish; retry") + } + }() + return + } requestedEngine, requestedPort, isEnginePortAssignment := enginePortAssignmentRequest(msg.Method, msg.Params) if isEnginePortAssignment || msg.Method == "engine:stop" || msg.Method == "engine:start" || msg.Method == "engine:restart" { @@ -3459,6 +3543,13 @@ func (b *Broker) relayToEngineNow(msg *Message) { } return } + requestedLlamaCppPort, isLlamaCppSetPort := llamacppSetPortRequest(msg.Method, msg.Params) + if isLlamaCppSetPort && requestedLlamaCppPort == llamacppEffectiveProfile().FacadePort && b.llamacppState().managedFacade.Load() { + if err := b.codec.RespondError(msg.ID, -32000, fmt.Sprintf("port %d is reserved by the managed llama.cpp proxy; choose another backend port or disable managed port ownership and restart NVPAIR", llamacppEffectiveProfile().FacadePort)); err != nil { + log.Printf("failed to reject conflicting llama.cpp engine:set-port: %v", err) + } + return + } em := b.getEngineMgr() if em == nil { diff --git a/services/nvpair-ui-broker/broker_lifecycle_test.go b/services/nvpair-ui-broker/broker_lifecycle_test.go index f60fcbe5..40a9dccf 100644 --- a/services/nvpair-ui-broker/broker_lifecycle_test.go +++ b/services/nvpair-ui-broker/broker_lifecycle_test.go @@ -35,7 +35,12 @@ func TestClusterManagerConfigDirTracksBrokerClusterDir(t *testing.T) { } } -func TestEngineAvailabilityWaitsForBothProxyOutcomes(t *testing.T) { +// Every managed engine has its own ownership gate, and restoration must wait +// for all of them: restoring desired state while any engine's port is still +// changing hands is what assigns an engine the port its proxy is about to take. +// The gates are released one at a time here so a missing one cannot pass by +// being masked by the others. +func TestEngineAvailabilityWaitsForEveryProxyOutcome(t *testing.T) { engineClient, engineServer := net.Pipe() defer engineClient.Close() defer engineServer.Close() @@ -44,6 +49,7 @@ func TestEngineAvailabilityWaitsForBothProxyOutcomes(t *testing.T) { b := &Broker{ ollamaPortReady: make(chan struct{}), lmstudioPortReady: make(chan struct{}), + llamacppPortReady: make(chan struct{}), } b.setEngineMgr(engine) @@ -54,7 +60,7 @@ func TestEngineAvailabilityWaitsForBothProxyOutcomes(t *testing.T) { restore <- msg.Method } }() - advertised := make(chan string, 2) + advertised := make(chan string, 3) ctx, cancel := context.WithCancel(context.Background()) defer cancel() done := make(chan bool, 1) @@ -63,25 +69,28 @@ func TestEngineAvailabilityWaitsForBothProxyOutcomes(t *testing.T) { ctx, func(context.Context) { advertised <- "ollama" }, func(context.Context) { advertised <- "lmstudio" }, + func(context.Context) { advertised <- "llamacpp" }, ) }() - select { - case got := <-restore: - t.Fatalf("restore %q ran before either proxy outcome", got) - case got := <-advertised: - t.Fatalf("%s advertising ran before either proxy outcome", got) - case <-time.After(100 * time.Millisecond): + gates := []struct { + name string + open chan struct{} + }{ + {"ollama", b.ollamaPortReady}, + {"lmstudio", b.lmstudioPortReady}, + {"llamacpp", b.llamacppPortReady}, } - close(b.ollamaPortReady) - select { - case got := <-restore: - t.Fatalf("restore %q ran before LM Studio proxy outcome", got) - case got := <-advertised: - t.Fatalf("%s advertising ran before LM Studio proxy outcome", got) - case <-time.After(100 * time.Millisecond): + for _, gate := range gates { + select { + case got := <-restore: + t.Fatalf("restore %q ran before the %s proxy outcome", got, gate.name) + case got := <-advertised: + t.Fatalf("%s advertising ran before the %s proxy outcome", got, gate.name) + case <-time.After(100 * time.Millisecond): + } + close(gate.open) } - close(b.lmstudioPortReady) select { case got := <-restore: @@ -89,15 +98,15 @@ func TestEngineAvailabilityWaitsForBothProxyOutcomes(t *testing.T) { t.Fatalf("restore method = %q, want %q", got, restoreEnabledEnginesMethod) } case <-time.After(2 * time.Second): - t.Fatal("enabled-engine restore did not run after both proxy outcomes") + t.Fatal("enabled-engine restore did not run after every proxy outcome") } seen := map[string]bool{} - for len(seen) < 2 { + for len(seen) < len(gates) { select { case got := <-advertised: seen[got] = true case <-time.After(2 * time.Second): - t.Fatalf("advertising did not start for both engines: %v", seen) + t.Fatalf("advertising did not start for every engine: %v", seen) } } if !<-done { diff --git a/services/nvpair-ui-broker/engineproxy.go b/services/nvpair-ui-broker/engineproxy.go index 1254d08e..b2e2fd5d 100644 --- a/services/nvpair-ui-broker/engineproxy.go +++ b/services/nvpair-ui-broker/engineproxy.go @@ -167,6 +167,14 @@ func buildEngineProxyProfiles() []engineProxyProfile { // LM Studio is the one engine engine-manager may move while running: // its identified command-mode runtime has an official stop command. "lmstudio": {Ownership: managedEngine, HealthProbePath: "/v1/models"}, + // llama.cpp is managed — engine-manager installs it and owns its + // lifecycle — but it is deliberately never facade-promoted. PAIR + // assigns both of its ports, so there is no stock port to take over + // and nothing to relocate the engine off of; prepareEnabledFacades + // omits it for that reason. Ownership still reads managedEngine + // because the question that enum answers — may the broker reposition + // this engine while it runs — is yes. + "llamacpp": {Ownership: managedEngine, HealthProbePath: "/v1/models"}, } out := make([]engineProxyProfile, 0, len(engines.All())) for _, e := range engines.All() { @@ -379,6 +387,9 @@ func (b *Broker) prepareEnabledFacades() { if b.proxyEnabled(lmstudioProxyProfile) { b.prepareManagedLMStudioFacade() } + if b.proxyEnabled(llamacppProxyProfile) { + b.prepareManagedLlamaCppFacade() + } } // proxyDisabledReason explains why an engine has no proxy, and reports whether diff --git a/services/nvpair-ui-broker/enginesettings.go b/services/nvpair-ui-broker/enginesettings.go index 9f65e88b..f5a95301 100644 --- a/services/nvpair-ui-broker/enginesettings.go +++ b/services/nvpair-ui-broker/enginesettings.go @@ -159,18 +159,30 @@ func (b *Broker) settingsWorkerCall(ctx context.Context, method string, params a return nil } +// settingsProxy is the live proxy process fronting an engine's facade, or nil +// when the engine has no proxy profile or its proxy is not running. func (b *Broker) settingsProxy(engine string) *proxyProcess { - if engine == "ollama" { - return b.getProxy() + profile, ok := engineProxyProfileFor(engine) + if !ok { + return nil } - if engine == "lmstudio" { - return b.getLMStudioProxy() + return b.engineProxyHandle(profile) +} + +// isEngineDiscoveryService reports whether a registered service key is one of +// the engine facades this broker advertises. Their ports are accounted for by +// engine below, so they are not treated as opaque reserved PAIR listeners. +func isEngineDiscoveryService(key noderec.ServiceKey) bool { + for _, profile := range engineProxyProfiles { + if profile.DiscoveryService == key { + return true + } } - return nil + return false } func (b *Broker) settingsSnapshotLocked(ctx context.Context, engine string) (settings.Snapshot, error) { - if engine != "ollama" && engine != "lmstudio" { + if _, ok := engineProxyProfileFor(engine); !ok { return settings.Snapshot{}, fmt.Errorf("this engine does not support settings") } if err := b.loadEngineSettingsLocked(); err != nil { @@ -233,8 +245,8 @@ func (b *Broker) settingsSnapshotLocked(ctx context.Context, engine string) (set func (b *Broker) publishSettingsLocked() { all := make([]settings.Snapshot, 0, len(b.engineSettings)) - for _, engine := range []string{"ollama", "lmstudio"} { - if record := b.engineSettings[engine]; record != nil { + for _, profile := range engineProxyProfiles { + if record := b.engineSettings[profile.Name]; record != nil { record.Snapshot.Sequence++ all = append(all, record.Snapshot) if b.codec != nil { @@ -261,7 +273,7 @@ func (b *Broker) validateSettingsPortsLocked(ctx context.Context, engine string, // Include every registered PAIR listener, including services added later. if b.regCache != nil { for _, service := range b.regCache.Snapshot() { - if service.Port > 0 && service.Service != noderec.ServiceOllama && service.Service != noderec.ServiceLMStudio { + if service.Port > 0 && !isEngineDiscoveryService(service.Service) { reserved[service.Port] = true } } @@ -283,7 +295,8 @@ func (b *Broker) validateSettingsPortsLocked(ctx context.Context, engine string, return fmt.Errorf("a selected port is reserved by another configured engine") } } - for _, other := range []string{"ollama", "lmstudio"} { + for _, otherProfile := range engineProxyProfiles { + other := otherProfile.Name if other == engine { continue } @@ -347,13 +360,14 @@ func (b *Broker) previewSettingsLocked(ctx context.Context, p settings.Request) // restart. settingsApplyMu, which the caller also holds, is what keeps a // second apply out of this window. func (b *Broker) runSettingsOperationLocked(ctx context.Context, engine string, record *engineSettingsRecord) error { + profile, ok := engineProxyProfileFor(engine) + if !ok { + return fmt.Errorf("this engine does not support settings") + } if err := b.migrateSettingsArgumentsLocked(ctx, engine, record); err != nil { return err } - service := noderec.ServiceOllama - if engine == "lmstudio" { - service = noderec.ServiceLMStudio - } + service := profile.DiscoveryService if b.regCache != nil { b.unregisterService(service) } @@ -398,11 +412,7 @@ func (b *Broker) runSettingsOperationLocked(ctx context.Context, engine string, record.Snapshot.EffectiveServerPort = launch.EffectivePort record.Snapshot.Editable = launch.Editable record.Snapshot.Reason = launch.Reason - if engine == "ollama" { - b.ollamaState().backendPort.Store(int32(launch.EffectivePort)) - } else { - b.lmstudioState().backendPort.Store(int32(launch.EffectivePort)) - } + b.engineProxy(profile).backendPort.Store(int32(launch.EffectivePort)) } proxyReady := false if proxy := b.settingsProxy(engine); proxy != nil { @@ -676,33 +686,28 @@ func (b *Broker) handleSettingsRelay(raw json.RawMessage) { } func (b *Broker) rebindSettingsProxy(engine string, port int) error { + profile, ok := engineProxyProfileFor(engine) + if !ok { + return fmt.Errorf("this engine does not support settings") + } p := b.settingsProxy(engine) if p == nil { return fmt.Errorf("proxy unavailable") } // Disable automatic facade takeover before the ready event can race the // explicit rebind. The accepted journal restores these choices after restart. - profile, _ := engineProxyProfileFor(engine) - b.engineProxy(profile).explicitSettings.Store(true) - if engine == "ollama" { - b.ollamaState().managedFacade.Store(false) - } else { - b.lmstudioState().managedFacade.Store(false) - } - _, rpcErr, err := p.Call(context.Background(), engine+":set-port", settingsJSON(map[string]int{"port": port})) + rt := b.engineProxy(profile) + rt.explicitSettings.Store(true) + rt.managedFacade.Store(false) + _, rpcErr, err := p.Call(context.Background(), profile.addressed("set-port"), settingsJSON(map[string]int{"port": port})) if err != nil { return err } if rpcErr != nil { return fmt.Errorf("%s", rpcErr.Message) } - if engine == "ollama" { - b.ollamaState().managedFacade.Store(false) - b.ollamaState().startupPort.Store(int32(port)) - } else { - b.lmstudioState().managedFacade.Store(false) - b.lmstudioState().startupPort.Store(int32(port)) - } + rt.managedFacade.Store(false) + rt.startupPort.Store(int32(port)) return nil } @@ -723,12 +728,12 @@ func (b *Broker) refreshEngineSettings(ctx context.Context) { readCtx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() changed := false - for _, engine := range []string{"ollama", "lmstudio"} { + for _, profile := range engineProxyProfiles { var before settings.Snapshot - if r := b.engineSettings[engine]; r != nil { + if r := b.engineSettings[profile.Name]; r != nil { before = r.Snapshot } - after, err := b.settingsSnapshotLocked(readCtx, engine) + after, err := b.settingsSnapshotLocked(readCtx, profile.Name) if err == nil && before != after { changed = true } diff --git a/services/nvpair-ui-broker/enginesettings_recovery.go b/services/nvpair-ui-broker/enginesettings_recovery.go index ec444c39..ffb4cbba 100644 --- a/services/nvpair-ui-broker/enginesettings_recovery.go +++ b/services/nvpair-ui-broker/enginesettings_recovery.go @@ -61,7 +61,16 @@ func (b *Broker) explicitEngineSettingsLocked(engine string) (settings.Config, b return settings.Config{}, false } +// prepareExplicitEngineSettings restores a saved explicit choice onto an +// engine's proxy runtime before its facade is prepared, and reports whether +// automatic facade planning must stand down for it. It is the startup half of +// the contract rebindSettingsProxy keeps at run time: an explicit choice turns +// managed takeover off and fixes both the backend and the startup port. func (b *Broker) prepareExplicitEngineSettings(engine string) bool { + profile, ok := engineProxyProfileFor(engine) + if !ok { + return false + } b.engineConfigMu.Lock() loadErr := b.loadEngineSettingsLocked() b.engineConfigMu.Unlock() @@ -72,25 +81,68 @@ func (b *Broker) prepareExplicitEngineSettings(engine string) bool { if !ok { return false } - profile, _ := engineProxyProfileFor(engine) - b.engineProxy(profile).explicitSettings.Store(true) - if engine == "ollama" { - b.ollamaState().managedFacade.Store(false) + rt := b.engineProxy(profile) + rt.explicitSettings.Store(true) + rt.managedFacade.Store(false) + rt.backendPort.Store(int32(config.ServerPort)) + rt.startupPort.Store(int32(config.ProxyPort)) + if engine == ollamaProxyProfile.Name { + // Only Ollama carries a pending backend move and an inherited + // OLLAMA_HOST alias reservation that have to follow the explicit choice. b.managedOllamaBackend.Store(0) - b.ollamaState().backendPort.Store(int32(config.ServerPort)) - b.ollamaState().startupPort.Store(int32(config.ProxyPort)) b.syncCurrentEngineOllamaHostAliasReservation() - } else { - b.lmstudioState().managedFacade.Store(false) - b.lmstudioState().backendPort.Store(int32(config.ServerPort)) - b.lmstudioState().startupPort.Store(int32(config.ProxyPort)) } return true } +// facadeBindFailure decodes a facade's bind-failed error notification and +// reports whether automatic recovery may act on it. A facade whose port is an +// explicit settings choice is never moved by that path: the choice stands, and +// the failure reaches the user through the settings operation that made it. +// This runs on the proxy reader goroutine, so it reads the atomic and takes no +// lock — a settings operation may hold engineConfigMu while waiting on that +// very reader. +func (b *Broker) facadeBindFailure(profile engineProxyProfile, method string, params json.RawMessage) (port int, recover bool) { + if method != "error" { + return 0, false + } + var ep struct { + Code string `json:"code"` + Port int `json:"port"` + } + if json.Unmarshal(params, &ep) != nil || ep.Code != "bind-failed" { + return 0, false + } + return ep.Port, !b.engineProxy(profile).explicitSettings.Load() +} + +// settingsGovernFacadeLocked reports whether the engine-settings journal, not +// automatic ownership planning, decides where this engine's facade listens: +// when the user has made an explicit choice, and when the journal cannot be +// read, which suppresses every automatic component rewrite. Readiness +// reconciliation stands down in both cases and opens the engine's gate instead. +// Caller holds engineConfigMu. +func (b *Broker) settingsGovernFacadeLocked(profile engineProxyProfile) bool { + if b.loadEngineSettingsLocked() != nil { + return true + } + _, explicit := b.explicitEngineSettingsLocked(profile.Name) + return explicit +} + +// obsoleteLegacyProxyDefault reports a saved proxy port that an older proxy +// wrote as its own default rather than as a record of a user's choice. Only LM +// Studio has one: its standalone proxy defaulted to 1235, the port its managed +// backend is now relocated onto, and its store migration already discards it. +// Ollama's saved port has always been preserved, and llama.cpp's store is newer +// than the journal, so neither has a default to exclude. +func obsoleteLegacyProxyDefault(profile engineProxyProfile, port int) bool { + return profile.Name == lmstudioProxyProfile.Name && port == managedLMStudioBackendStart +} + // Legacy proxy stores did not distinguish user choices from automatic moves. // Preserve a valid non-colliding saved choice as explicit on first upgrade. -// LM Studio's old 1235 default is excluded, matching its existing migration. +// Every engine's proxy keeps such a store, named by its profile's PortFile. func (b *Broker) migrateLegacyEngineSettings() { b.engineConfigMu.Lock() defer b.engineConfigMu.Unlock() @@ -101,22 +153,19 @@ func (b *Broker) migrateLegacyEngineSettings() { if err != nil { return } - for _, engine := range []string{"ollama", "lmstudio"} { - name := "proxy-port.json" - if engine == "lmstudio" { - name = "lmstudio-proxy-port.json" - } + for _, profile := range engineProxyProfiles { + engine := profile.Name if b.engineSettings[engine] != nil { continue } - data, err := os.ReadFile(filepath.Join(filepath.Dir(journal), name)) + data, err := os.ReadFile(filepath.Join(filepath.Dir(journal), profile.PortFile)) if err != nil { continue } var saved struct { Port int `json:"port"` } - if json.Unmarshal(data, &saved) != nil || saved.Port < 1 || saved.Port > 65535 || (engine == "lmstudio" && saved.Port == 1235) { + if json.Unmarshal(data, &saved) != nil || saved.Port < 1 || saved.Port > 65535 || obsoleteLegacyProxyDefault(profile, saved.Port) { continue } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) diff --git a/services/nvpair-ui-broker/enginesettings_review_test.go b/services/nvpair-ui-broker/enginesettings_review_test.go index 50cb11be..cf737262 100644 --- a/services/nvpair-ui-broker/enginesettings_review_test.go +++ b/services/nvpair-ui-broker/enginesettings_review_test.go @@ -20,9 +20,14 @@ func TestSettingsRebindAddressesOnlyRequestedFacade(t *testing.T) { for _, profile := range engineProxyProfiles { t.Run(profile.Name, func(t *testing.T) { h := newSettingsHarness(t) - p := h.b.getProxy() - _, ollamaBefore := p.Status("ollama") - _, lmstudioBefore := p.Status("lmstudio") + p := h.b.settingsProxy(profile.Name) + if p == nil { + t.Fatal("settings proxy unavailable") + } + before := make(map[string]int, len(engineProxyProfiles)) + for _, other := range engineProxyProfiles { + _, before[other.Name] = p.Status(other.Name) + } ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) @@ -32,22 +37,40 @@ func TestSettingsRebindAddressesOnlyRequestedFacade(t *testing.T) { if err := h.b.rebindSettingsProxy(profile.Name, port); err != nil { t.Fatal(err) } - wantOllama, wantLMStudio := ollamaBefore, lmstudioBefore - if profile.Name == "ollama" { - wantOllama = port - } else { - wantLMStudio = port - } - for engine, want := range map[string]int{"ollama": wantOllama, "lmstudio": wantLMStudio} { - ready, got := p.Status(engine) + for _, other := range engineProxyProfiles { + want := before[other.Name] + if other.Name == profile.Name { + want = port + } + ready, got := p.Status(other.Name) if !ready || got != want { - t.Fatalf("%s ready=%v port=%d, want %d", engine, ready, got, want) + t.Fatalf("%s ready=%v port=%d, want %d", other.Name, ready, got, want) + } + } + // The rebind also disarms automatic takeover, for this facade alone. + for _, other := range engineProxyProfiles { + rt := h.b.engineProxy(other) + if other.Name == profile.Name { + if !rt.explicitSettings.Load() || rt.managedFacade.Load() || int(rt.startupPort.Load()) != port { + t.Fatalf("%s runtime did not record the explicit rebind: explicit=%v managed=%v startup=%d", + other.Name, rt.explicitSettings.Load(), rt.managedFacade.Load(), rt.startupPort.Load()) + } + } else if rt.explicitSettings.Load() || rt.startupPort.Load() != 0 { + t.Fatalf("%s runtime changed by another engine's rebind", other.Name) } } }) } } +// forwardFacadeNotification delivers one facade-addressed notification through +// the routing the live proxy reader uses, so a test reaches whichever engine +// handler the table maps the profile to without naming it. +func forwardFacadeNotification(b *Broker, profile engineProxyProfile, method string, params json.RawMessage) { + b.forwardProxyProcessNotification(b.currentOllamaProxyGeneration(), b.lmstudioProxyGeneration.Load(), + b.llamaCppProxyGeneration.Load(), profile.addressed(method), params) +} + func TestExplicitSettingsBindFailurePreservesChosenPort(t *testing.T) { for _, profile := range engineProxyProfiles { t.Run(profile.Name, func(t *testing.T) { @@ -61,11 +84,7 @@ func TestExplicitSettingsBindFailurePreservesChosenPort(t *testing.T) { t.Fatal("explicit settings were not restored") } failure := settingsJSON(map[string]any{"code": "bind-failed", "port": requested}) - if profile.Name == "ollama" { - b.forwardProxyNotification("error", failure) - } else { - b.forwardLMStudioProxyNotification("error", failure) - } + forwardFacadeNotification(b, profile, "error", failure) if got := b.engineProxy(profile).startupPort.Load(); got != requested { t.Fatalf("bind notification changed chosen port to %d", got) } @@ -89,14 +108,16 @@ func TestExplicitSettingsBindFailurePreservesChosenPort(t *testing.T) { } } +// The request targets Ollama; every other engine in turn is the stopped sibling +// whose saved proxy port must stay reserved. func TestSettingsReservesStoppedProxySavedPort(t *testing.T) { - test := func(name string, serverPort bool) { - t.Run(name, func(t *testing.T) { + test := func(other engineProxyProfile, name string, serverPort bool) { + t.Run(other.Name+"/"+name, func(t *testing.T) { h := newSettingsHarness(t) request := h.request(t) - _, reserved := h.b.getLMStudioProxy().Status("lmstudio") - h.b.setLMStudioProxy(nil) - h.b.engineSettings["lmstudio"] = &engineSettingsRecord{ + _, reserved := h.b.settingsProxy(other.Name).Status(other.Name) + h.b.setEngineProxyHandle(other, nil) + h.b.engineSettings[other.Name] = &engineSettingsRecord{ Explicit: true, Snapshot: settings.Snapshot{Settings: settings.Config{ProxyPort: reserved}}, } @@ -114,8 +135,13 @@ func TestSettingsReservesStoppedProxySavedPort(t *testing.T) { } }) } - test("server port cannot reuse saved proxy port", true) - test("proxy port cannot reuse saved proxy port", false) + for _, other := range engineProxyProfiles { + if other.Name == ollamaProxyProfile.Name { + continue + } + test(other, "server port cannot reuse saved proxy port", true) + test(other, "proxy port cannot reuse saved proxy port", false) + } } func TestSettingsMigrationRejectsReservedPorts(t *testing.T) { @@ -151,15 +177,20 @@ func TestSettingsMigrationRejectsReservedPorts(t *testing.T) { h.b.ollamaHostAliasMu.Unlock() return request.Settings.ProxyPort }) - test("stopped proxy saved port", func(h *settingsHarness, request settings.Request) int { - _, port := h.b.getLMStudioProxy().Status("lmstudio") - h.b.setLMStudioProxy(nil) - h.b.engineSettings["lmstudio"] = &engineSettingsRecord{ - Explicit: true, - Snapshot: settings.Snapshot{Settings: settings.Config{ProxyPort: port}}, + for _, other := range engineProxyProfiles { + if other.Name == ollamaProxyProfile.Name { + continue } - return port - }) + test("stopped "+other.Name+" proxy saved port", func(h *settingsHarness, request settings.Request) int { + _, port := h.b.settingsProxy(other.Name).Status(other.Name) + h.b.setEngineProxyHandle(other, nil) + h.b.engineSettings[other.Name] = &engineSettingsRecord{ + Explicit: true, + Snapshot: settings.Snapshot{Settings: settings.Config{ProxyPort: port}}, + } + return port + }) + } } func TestEnabledEngineRestorationSurvivesInvalidSettingsJournal(t *testing.T) { diff --git a/services/nvpair-ui-broker/enginesettings_test.go b/services/nvpair-ui-broker/enginesettings_test.go index e8d487b6..37b87995 100644 --- a/services/nvpair-ui-broker/enginesettings_test.go +++ b/services/nvpair-ui-broker/enginesettings_test.go @@ -16,6 +16,7 @@ import ( "testing" "time" + "nvpair-shared/engines" settings "nvpair-shared/enginesettings" "nvpair-shared/noderec" "nvpair-ui-broker/relay" @@ -36,9 +37,15 @@ type settingsHarness struct { launchMu sync.Mutex } +// newSettingsHarness stands up a broker with a fixture engine manager and one +// fixture proxy process hosting a ready facade for every engine in the table, +// as production does. The fixture engine manager serves a single launch state +// for whichever engine is asked about. func newSettingsHarness(t *testing.T) *settingsHarness { t.Helper() - ports := make([]int, 3) + // One ephemeral port for the fixture engine's server, then one facade port + // per engine. + ports := make([]int, 1+len(engineProxyProfiles)) listeners := []net.Listener{} for i := range ports { ln, err := net.Listen("tcp", "127.0.0.1:0") @@ -55,12 +62,11 @@ func newSettingsHarness(t *testing.T) *settingsHarness { worker, codec := newTestRPCWorkerPipe(t) h.b.setEngineMgr(worker) proxyWorker, proxyCodec := newTestRPCWorkerPipe(t) - proxy := &proxyProcess{peer: proxyWorker.peer, facadeState: map[string]proxyFacadeState{ - "ollama": {ready: true, port: ports[1]}, - "lmstudio": {ready: true, port: ports[2]}, - }} - h.b.setProxy(proxy) - h.b.setLMStudioProxy(proxy) + proxy := &proxyProcess{peer: proxyWorker.peer, facadeState: make(map[string]proxyFacadeState, len(engineProxyProfiles))} + for i, profile := range engineProxyProfiles { + proxy.facadeState[profile.Name] = proxyFacadeState{ready: true, port: ports[1+i]} + h.b.setEngineProxyHandle(profile, proxy) + } go func() { for { msg, err := proxyCodec.Read() @@ -70,7 +76,10 @@ func newSettingsHarness(t *testing.T) *settingsHarness { if !msg.IsRequest() { continue } - if msg.Method != "ollama:set-port" && msg.Method != "lmstudio:set-port" { + // Only a facade-addressed set-port changes fixture state; every + // other request (node/set-local-backend and the like) is acknowledged. + engine, bare := engines.SplitAddressedMethod(msg.Method) + if engine == "" || bare != "set-port" { _ = proxyCodec.Respond(msg.ID, map[string]bool{"ok": true}) continue } @@ -79,16 +88,12 @@ func newSettingsHarness(t *testing.T) *settingsHarness { } _ = json.Unmarshal(msg.Params, &p) proxy.readyMu.Lock() - engine := "ollama" - if msg.Method == "lmstudio:set-port" { - engine = "lmstudio" - } proxy.facadeState[engine] = proxyFacadeState{ready: true, port: p.Port} proxy.readyMu.Unlock() _ = proxyCodec.Respond(msg.ID, map[string]int{"port": p.Port}) } }() - launch := settings.LaunchState{Engine: "ollama", ServerPort: ports[0], EffectivePort: ports[0], LaunchText: "--fixture-option", Running: true, Editable: true, Format: "pair-arguments-v1"} + launch := settings.LaunchState{ServerPort: ports[0], EffectivePort: ports[0], LaunchText: "--fixture-option", Running: true, Editable: true, Format: "pair-arguments-v1"} go func() { for { msg, err := codec.Read() @@ -100,12 +105,22 @@ func newSettingsHarness(t *testing.T) *settingsHarness { } switch msg.Method { case "engine:get-launch": + var p settings.Request + _ = json.Unmarshal(msg.Params, &p) h.launchMu.Lock() current := launch h.launchMu.Unlock() + current.Engine = p.Engine _ = codec.Respond(msg.ID, current) case "engine:configured-ports": - _ = codec.Respond(msg.ID, map[string]any{"engines": []any{map[string]any{"engine": "lmstudio", "port": h.otherEnginePort.Load()}}}) + // Every engine reports the fixture's "other engine" port; the + // broker skips the entry for the engine being configured, so + // whichever engine a test targets, its siblings hold this port. + configured := make([]any, 0, len(engineProxyProfiles)) + for _, profile := range engineProxyProfiles { + configured = append(configured, map[string]any{"engine": profile.Name, "port": h.otherEnginePort.Load()}) + } + _ = codec.Respond(msg.ID, map[string]any{"engines": configured}) case "engine:preview-launch": var p settings.Request _ = json.Unmarshal(msg.Params, &p) @@ -123,6 +138,8 @@ func newSettingsHarness(t *testing.T) *settingsHarness { // exactly what the broker's lock split makes possible. go func(msg *Message) { h.applies.Add(1) + var p settings.Configure + _ = json.Unmarshal(msg.Params, &p) if h.entered != nil { h.entered <- struct{}{} <-h.release @@ -130,16 +147,14 @@ func newSettingsHarness(t *testing.T) *settingsHarness { if h.failBeforeStop.Load() { if h.loseProxyOnStop.Load() { proxy.readyMu.Lock() - state := proxy.facadeState["ollama"] + state := proxy.facadeState[p.Engine] state.ready = false - proxy.facadeState["ollama"] = state + proxy.facadeState[p.Engine] = state proxy.readyMu.Unlock() } _ = codec.RespondError(msg.ID, -32000, "stop failure") return } - var p settings.Configure - _ = json.Unmarshal(msg.Params, &p) h.launchMu.Lock() launch.ServerPort = p.Settings.ServerPort launch.LaunchText = p.Settings.LaunchText @@ -158,6 +173,7 @@ func newSettingsHarness(t *testing.T) *settingsHarness { launch.Running = p.Resume current := launch h.launchMu.Unlock() + current.Engine = p.Engine if h.failResultSave.Load() { journal, _ := h.b.engineSettingsPath() if err := os.Remove(journal); err != nil { @@ -422,18 +438,148 @@ func TestSettingsPortValidationIncludesStoppedEnginesAndAliases(t *testing.T) { } } +// Every engine's proxy keeps a saved-port store under its own file name; a valid +// choice in any of them survives as an explicit setting and disarms automatic +// takeover for that engine alone. func TestSettingsMigratesLegacyProxyChoiceBeforeManagedDefaults(t *testing.T) { h := newSettingsHarness(t) path, _ := h.b.engineSettingsPath() - if err := os.WriteFile(filepath.Join(filepath.Dir(path), "proxy-port.json"), []byte(`{"port":26080}`), 0600); err != nil { + saved := make(map[string]int, len(engineProxyProfiles)) + for i, profile := range engineProxyProfiles { + saved[profile.Name] = 26080 + i + data, err := json.Marshal(map[string]int{"port": saved[profile.Name]}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(filepath.Dir(path), profile.PortFile), data, 0600); err != nil { + t.Fatal(err) + } + } + h.b.migrateLegacyEngineSettings() + for _, profile := range engineProxyProfiles { + config, ok := h.b.explicitEngineSettings(profile.Name) + if !ok || config.ProxyPort != saved[profile.Name] { + t.Fatalf("%s legacy choice lost: %+v %v", profile.Name, config, ok) + } + rt := h.b.engineProxy(profile) + if !h.b.prepareExplicitEngineSettings(profile.Name) || int(rt.startupPort.Load()) != saved[profile.Name] || rt.managedFacade.Load() || !rt.explicitSettings.Load() { + t.Fatalf("%s: automatic startup overrode saved proxy choice", profile.Name) + } + } +} + +// LM Studio's standalone proxy once wrote 1235 as its own default; that value +// carries no choice and must not become an explicit setting. The rule is LM +// Studio's alone. +func TestSettingsMigrationSkipsLMStudioObsoleteDefault(t *testing.T) { + h := newSettingsHarness(t) + path, _ := h.b.engineSettingsPath() + data, err := json.Marshal(map[string]int{"port": managedLMStudioBackendStart}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(filepath.Dir(path), lmstudioProxyProfile.PortFile), data, 0600); err != nil { t.Fatal(err) } h.b.migrateLegacyEngineSettings() - config, ok := h.b.explicitEngineSettings("ollama") - if !ok || config.ProxyPort != 26080 { - t.Fatalf("legacy choice lost: %+v %v", config, ok) + if _, explicit := h.b.explicitEngineSettings(lmstudioProxyProfile.Name); explicit { + t.Fatal("LM Studio's obsolete 1235 proxy default became an explicit setting") + } +} + +// lastResponse returns the final id-bearing response the broker wrote to its +// client codec, skipping the engine:settings-changed notifications a settings +// operation publishes on the way. +func lastResponse(t *testing.T, output *bytes.Buffer) Message { + t.Helper() + var last *Message + for _, line := range strings.Split(strings.TrimSpace(output.String()), "\n") { + if line == "" { + continue + } + var msg Message + if err := json.Unmarshal([]byte(line), &msg); err != nil { + t.Fatalf("unparseable frame %q: %v", line, err) + } + if msg.IsResponse() { + last = &msg + } + } + if last == nil { + t.Fatalf("no response written: %q", output.String()) } - if !h.b.prepareExplicitEngineSettings("ollama") || h.b.ollamaState().startupPort.Load() != 26080 || h.b.ollamaState().managedFacade.Load() { - t.Fatal("automatic startup overrode saved proxy choice") + return *last +} + +// Every engine's -proxy:set-port is the same intercepted settings +// operation: refused without an engine manager to vouch for port ownership, +// refused for a port another configured engine holds, and otherwise moving only +// the addressed facade while recording the port as an explicit choice. +func TestSettingsPortRPCServesEveryEngineProxy(t *testing.T) { + for _, profile := range engineProxyProfiles { + t.Run(profile.Name, func(t *testing.T) { + method := profile.ComponentName() + ":set-port" + id := json.RawMessage(`1`) + var output bytes.Buffer + bare := &Broker{codec: NewCodec(readWriter{Reader: bytes.NewReader(nil), Writer: &output}), clusterDir: filepath.Join(t.TempDir(), "cluster")} + bare.handleSettingsPortRPC(&Message{ID: &id, Method: method, Params: settingsJSON(map[string]int{"port": 8082})}, profile.Name) + if response := lastResponse(t, &output); response.Error == nil { + t.Fatalf("port moved without an engine manager to vouch for ownership: %s", output.String()) + } + + h := newSettingsHarness(t) + h.b.codec = NewCodec(readWriter{Reader: bytes.NewReader(nil), Writer: &output}) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := ln.Addr().(*net.TCPAddr).Port + _ = ln.Close() + request := &Message{ID: &id, Method: method, Params: settingsJSON(map[string]int{"port": port})} + + output.Reset() + h.otherEnginePort.Store(int32(port)) + h.b.handleSettingsPortRPC(request, profile.Name) + if response := lastResponse(t, &output); response.Error == nil { + t.Fatalf("took a port configured for another engine: %s", output.String()) + } + + h.otherEnginePort.Store(0) + output.Reset() + p := h.b.settingsProxy(profile.Name) + if p == nil { + t.Fatal("settings proxy unavailable") + } + before := make(map[string]int, len(engineProxyProfiles)) + for _, other := range engineProxyProfiles { + _, before[other.Name] = p.Status(other.Name) + } + h.b.handleSettingsPortRPC(request, profile.Name) + response := lastResponse(t, &output) + if response.Error != nil { + t.Fatalf("port move refused: %s", response.Error.Message) + } + var moved struct { + Port int `json:"port"` + } + if json.Unmarshal(response.Result, &moved) != nil || moved.Port != port { + t.Fatalf("response %s, want port %d", response.Result, port) + } + for _, other := range engineProxyProfiles { + want := before[other.Name] + if other.Name == profile.Name { + want = port + } + if ready, got := p.Status(other.Name); !ready || got != want { + t.Fatalf("%s ready=%v port=%d, want %d", other.Name, ready, got, want) + } + } + if s := h.b.engineSettings[profile.Name].Snapshot; s.Settings.ProxyPort != port || s.Phase != "succeeded" { + t.Fatalf("journal did not record the move: %+v", s) + } + if rt := h.b.engineProxy(profile); !rt.explicitSettings.Load() || rt.managedFacade.Load() { + t.Fatal("a port set from the terminal was not recorded as an explicit choice") + } + }) } } diff --git a/services/nvpair-ui-broker/llamacpp_advertise_test.go b/services/nvpair-ui-broker/llamacpp_advertise_test.go new file mode 100644 index 00000000..6e12655c --- /dev/null +++ b/services/nvpair-ui-broker/llamacpp_advertise_test.go @@ -0,0 +1,215 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "nvpair-shared/noderec" + "nvpair-ui-broker/relay" +) + +// llamacpp-proxy:set-port is served by the shared settings operation; see +// TestSettingsPortRPCServesEveryEngineProxy for its ownership refusals. + +func TestLlamaWorkloadCancelRejectsForeignOriginAndOtherEngine(t *testing.T) { + for _, body := range []string{ + `{"id":"1","runId":"a","engine":"llamacpp","originatedFrom":"peer"}`, + `{"id":"1","runId":"a","engine":"ollama","originatedFrom":"self"}`, + `{"id":"1","engine":"llamacpp","originatedFrom":"self"}`, + } { + var output bytes.Buffer + b := &Broker{nodeID: "self", codec: NewCodec(readWriter{Reader: bytes.NewReader(nil), Writer: &output})} + id := json.RawMessage(`1`) + b.cancelLlamaWorkload(&Message{ID: &id, Method: "workloads:cancel", Params: json.RawMessage(body)}) + var response Message + if json.Unmarshal(output.Bytes(), &response) != nil || response.Error == nil { + t.Fatalf("unsafe cancellation accepted: %s", output.String()) + } + } +} + +func TestReconcileAdvertiseLlamaCppRegistersProxyPort(t *testing.T) { + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/models" { + http.NotFound(w, r) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"data":[]}`)) + })) + t.Cleanup(engine.Close) + enginePort := engine.Listener.Addr().(*net.TCPAddr).Port + if enginePort == defaultLlamaCppProxyPort { + t.Fatal("httptest bound the llama.cpp proxy port; cannot distinguish engine from proxy") + } + + proxy, localBackend := llamaCppProxyPipe(t, defaultLlamaCppProxyPort) + b := &Broker{regCache: relay.NewRegistrationCache()} + b.setLlamaCppProxy(proxy) + b.setEngineMgr(serveEngineStatus(t, enginePort)) + + b.reconcileAdvertiseLlamaCpp(&http.Client{Timeout: 2 * time.Second}) + + got, ok := registrationFor(b, noderec.ServiceLlamaCpp) + if !ok || got.Port != defaultLlamaCppProxyPort { + t.Fatalf("lc registration = (%+v, %v), want port %d", got, ok, defaultLlamaCppProxyPort) + } + select { + case backend := <-localBackend: + if backend.Engine != "llamacpp" || backend.Port != enginePort || !backend.Healthy { + t.Fatalf("local backend = %+v, want healthy llamacpp:%d", backend, enginePort) + } + case <-time.After(2 * time.Second): + t.Fatal("llama.cpp proxy did not receive a healthy local backend") + } +} + +// TestStaleEngineStatusOnTheFacadePortIsRefused covers the reading the managed +// facade makes possible: the facade has claimed the stock port and the engine +// has been relocated above it, but engine:status still answers with the port it +// had before the move. Believing it would advertise the facade as the engine and +// then hand the facade its own listener as a local backend, so every request it +// admitted would forward to itself. +// +// This is what reconcileAdvertiseLlamaCpp relies on instead of the +// pending-backend guard the Ollama loop carries. +func TestStaleEngineStatusOnTheFacadePortIsRefused(t *testing.T) { + proxy, localBackend := llamaCppProxyPipe(t, defaultLlamaCppProxyPort) + b := &Broker{regCache: relay.NewRegistrationCache()} + b.setLlamaCppProxy(proxy) + b.setEngineMgr(serveEngineStatus(t, defaultLlamaCppProxyPort)) + + // A nil client, so a health probe that got as far as the network would be a + // panic rather than a pass: the refusal has to happen on the port reading + // alone, before anything asks the facade whether it is alive. + b.reconcileAdvertiseLlamaCpp(nil) + + if got, ok := registrationFor(b, noderec.ServiceLlamaCpp); ok { + t.Fatalf("stale status advertised the facade as the engine: %+v", got) + } + select { + case got := <-localBackend: + if got.Port != 0 || got.Healthy { + t.Fatalf("facade was handed its own listener as a backend: %+v", got) + } + case <-time.After(2 * time.Second): + t.Fatal("llama.cpp facade did not receive a cleared local backend") + } +} + +func TestLlamaCppFallbackNeverAdvertisesItsProxy(t *testing.T) { + proxy, localBackend := llamaCppProxyPipe(t, defaultLlamaCppPort) + b := &Broker{regCache: relay.NewRegistrationCache()} + b.setLlamaCppProxy(proxy) + + // No engine manager, so there is no port worth probing at all. A nil client + // asserts that: an unreachable manager must not fall back to the stock port + // the way the Ollama and LM Studio loops do. + b.reconcileAdvertiseLlamaCpp(nil) + if got := b.regCache.Snapshot(); len(got) != 0 { + t.Fatalf("llama.cpp proxy was advertised as an engine: %+v", got) + } + select { + case got := <-localBackend: + if got.Port != 0 || got.Healthy { + t.Fatalf("proxy listener was retained as the local backend: %+v", got) + } + case <-time.After(2 * time.Second): + t.Fatal("llama.cpp proxy did not receive a cleared local backend") + } +} + +func TestReconcileAdvertiseLlamaCppUnregistersWhenEngineDown(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + deadPort := ln.Addr().(*net.TCPAddr).Port + _ = ln.Close() + + proxy, localBackend := llamaCppProxyPipe(t, defaultLlamaCppProxyPort) + b := &Broker{regCache: relay.NewRegistrationCache()} + b.setLlamaCppProxy(proxy) + b.setEngineMgr(serveEngineStatus(t, deadPort)) + b.registerService(noderec.RegisterParams{Service: noderec.ServiceLlamaCpp, Port: defaultLlamaCppProxyPort}) + + b.reconcileAdvertiseLlamaCpp(&http.Client{Timeout: 2 * time.Second}) + + if _, ok := registrationFor(b, noderec.ServiceLlamaCpp); ok { + t.Fatal("lc stayed registered while the engine was down") + } + select { + case got := <-localBackend: + if got.Healthy { + t.Fatalf("local backend stayed healthy while the engine was down: %+v", got) + } + case <-time.After(2 * time.Second): + t.Fatal("llama.cpp proxy did not receive an unhealthy local backend") + } +} + +func llamaCppProxyPipe(t *testing.T, listenPort int) (*proxyProcess, <-chan proxyLocalBackend) { + t.Helper() + proxyClient, proxyServer := net.Pipe() + t.Cleanup(func() { + _ = proxyClient.Close() + _ = proxyServer.Close() + }) + proxy := &proxyProcess{ + peer: NewPeer(NewCodec(proxyClient)), + facadeState: readyFacade(llamacppProxyProfile.Name, listenPort), + } + go proxy.peer.Serve(nil, nil) + + localBackend := make(chan proxyLocalBackend, 1) + go func() { + codec := NewCodec(proxyServer) + msg, err := codec.Read() + if err != nil { + return + } + var got proxyLocalBackend + if json.Unmarshal(msg.Params, &got) == nil { + localBackend <- got + } + _ = codec.Respond(msg.ID, map[string]bool{"ok": true}) + }() + return proxy, localBackend +} + +func serveEngineStatus(t *testing.T, port int) *rpcWorker { + t.Helper() + engineClient, engineServer := net.Pipe() + t.Cleanup(func() { + _ = engineClient.Close() + _ = engineServer.Close() + }) + engine := &rpcWorker{peer: NewPeer(NewCodec(engineClient))} + go engine.peer.Serve(nil, nil) + go func() { + codec := NewCodec(engineServer) + msg, err := codec.Read() + if err != nil { + return + } + _ = codec.Respond(msg.ID, map[string]any{"running": true, "port": port}) + }() + return engine +} + +func registrationFor(b *Broker, svc noderec.ServiceKey) (noderec.RegisterParams, bool) { + for _, p := range b.regCache.Snapshot() { + if p.Service == svc { + return p, true + } + } + return noderec.RegisterParams{}, false +} diff --git a/services/nvpair-ui-broker/llamacppport.go b/services/nvpair-ui-broker/llamacppport.go new file mode 100644 index 00000000..9d6b4f90 --- /dev/null +++ b/services/nvpair-ui-broker/llamacppport.go @@ -0,0 +1,458 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +// llama.cpp's managed-port choreography, the counterpart of lmstudioport.go. +// +// llama.cpp is a managed engine in the same sense LM Studio is — engine-manager +// installed it and owns its lifecycle, so it may be stopped and repositioned — +// which means it gets the same treatment: move the backend first, verify the +// compatibility port is free, then force the proxy onto it. The shared policy +// in planManagedEnginePorts decides all of that; what is here is the +// orchestration around it. +// +// The one thing llama.cpp has that neither of the others does is a documented +// environment override for its own listen port. Upstream reads LLAMA_ARG_PORT +// for --port (common/arg.cpp), so a user who has set it has told us where their +// llama.cpp listens and therefore which port their clients already point at. +// That is what the facade should claim, so the effective profile resolves it +// rather than assuming the 8080 default. It is the same reasoning that makes an +// inherited OLLAMA_HOST supersede 11434 for Ollama, without any of the alias +// machinery: llama.cpp's variable names a port, not a whole endpoint PAIR has +// to keep answering on. + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + "strconv" + + "nvpair-shared/errors" +) + +const llamacppPortOwnershipBlockedID = "llamacpp-proxy:port-ownership-blocked" + +// llamacppProxyProfile is the descriptor entry this file's callers plan +// against. See ollamaProxyProfile. +var llamacppProxyProfile = mustEngineProxyProfile("llamacpp") + +// llamacppEnvPort reads LLAMA_ARG_PORT, the variable upstream llama.cpp reads +// for --port. Zero means unset or unusable, in which case the table's default +// stands. +func llamacppEnvPort() int { + raw := os.Getenv("LLAMA_ARG_PORT") + if raw == "" { + return 0 + } + port, err := strconv.Atoi(raw) + // The ceiling is 65534, not 65535: the backend is relocated to one above + // whatever the facade claims, so accepting the top port would plan a move + // onto 65536, which is not a port at all. + if err != nil || port < 1 || port > 65534 { + slog.Warn("ignoring unusable LLAMA_ARG_PORT", "value", raw) + return 0 + } + return port +} + +// llamacppEffectiveProfile is llamacppProxyProfile with the environment's port +// applied, and is what every port decision in this file plans against. +// +// The backend moves to one above whatever the facade claims, so the pair keeps +// the table's relationship rather than stranding the engine on 8081 while the +// facade sits somewhere else entirely. +func llamacppEffectiveProfile() engineProxyProfile { + profile := llamacppProxyProfile + if port := llamacppEnvPort(); port > 0 { + profile.FacadePort = port + profile.EnginePortBase = port + 1 + } + return profile +} + +// planManagedLlamaCppPorts plans llama.cpp's managed ports under the shared +// policy for an engine the broker may reposition while it runs. +func planManagedLlamaCppPorts(enabled bool, st ollamaPortStatus, available func(int) bool) managedPortPlan { + return planManagedEnginePorts(llamacppEffectiveProfile(), enabled, st, available) +} + +func (b *Broker) markLlamaCppPortReady() { + if b.llamacppPortReady != nil { + b.llamacppPortReadyOnce.Do(func() { close(b.llamacppPortReady) }) + } +} + +func (b *Broker) llamacppPortOwnershipPending() bool { + if b.llamacppPortReady == nil { + return false + } + select { + case <-b.llamacppPortReady: + return false + default: + return true + } +} + +// needsLlamaCppPortGate mirrors needsLMStudioPortGate: the client requests that +// can probe and adopt the configured port, so they must wait while it is +// changing hands. +func needsLlamaCppPortGate(method string, params json.RawMessage) bool { + if method == "engine:get-installed" { + return true + } + if method != "engine:status" && method != "engine:install" && method != "engine:start" && method != "engine:restart" { + return false + } + var request struct { + Engine string `json:"engine"` + } + return json.Unmarshal(params, &request) == nil && request.Engine == llamacppProxyProfile.Name +} + +func llamacppSetPortRequest(method string, params json.RawMessage) (int, bool) { + if method != "engine:set-port" { + return 0, false + } + var request struct { + Engine string `json:"engine"` + Port int `json:"port"` + } + if json.Unmarshal(params, &request) != nil || request.Engine != llamacppProxyProfile.Name || request.Port <= 0 { + return 0, false + } + return request.Port, true +} + +func (b *Broker) reportLlamaCppPortOwnershipBlocked(reason string) { + b.forwardErrorsReport(errors.ServiceError{ + ID: llamacppPortOwnershipBlockedID, + Message: fmt.Sprintf("NVPAIR could not safely reserve llama.cpp port %d: %s. No unknown process was stopped.", + llamacppEffectiveProfile().FacadePort, reason), + Timestamp: nowMillis(), + NodeID: b.nodeID, + Severity: "warning", + Action: "none", + }) +} + +func (b *Broker) rebindLlamaCppProxy(p *proxyProcess, port int) bool { + if p == nil || port == 0 { + return false + } + body, _ := json.Marshal(map[string]int{"port": port}) + result, rpcErr, err := p.Call(context.Background(), llamacppProxyProfile.addressed("set-port"), body) + if err != nil || rpcErr != nil { + slog.Warn("failed to rebind llama.cpp proxy", "port", port, "err", err, "rpcErr", rpcErr) + return false + } + var ready proxyReadyParams + return json.Unmarshal(result, &ready) == nil && ready.Port == port +} + +// blockManagedLlamaCppFacade records an explicit fallback and optionally +// rebinds a live proxy. It does not open the ownership gate: only a confirmed +// bound proxy generation or an exhausted supervisor may do that. +func (b *Broker) blockManagedLlamaCppFacade(reason string, p *proxyProcess, excludedPorts ...int) (int, bool) { + b.llamacppState().managedFacade.Store(false) + fallback := b.setLlamaCppProxyFallback(excludedPorts...) + b.reportLlamaCppPortOwnershipBlocked(reason) + return fallback, b.rebindLlamaCppProxy(p, fallback) +} + +func (b *Broker) cacheLlamaCppPortStatus() (ollamaPortStatus, bool) { + em := b.getEngineMgr() + if em == nil { + return ollamaPortStatus{}, false + } + params, _ := json.Marshal(map[string]string{"engine": llamacppProxyProfile.Name}) + result, rpcErr, err := em.Call(context.Background(), "engine:status", params) + if err != nil || rpcErr != nil { + return ollamaPortStatus{}, false + } + var st ollamaPortStatus + if json.Unmarshal(result, &st) != nil { + return ollamaPortStatus{}, false + } + if st.Port <= 0 { + return st, false + } + b.llamacppState().backendPort.Store(int32(st.Port)) + return st, true +} + +func (b *Broker) configureUnmanagedLlamaCppFacade() { + b.llamacppState().managedFacade.Store(false) + b.llamacppState().startupPort.Store(0) + b.forwardErrorsClear(llamacppPortOwnershipBlockedID) +} + +// prepareManagedLlamaCppFacade runs after engine-manager starts and before the +// proxy is spawned, exactly as LM Studio's does: engine-manager is the +// authority that may safely stop and reposition an engine it installed, so the +// backend moves first and the compatibility port is judged afterwards. +func (b *Broker) prepareManagedLlamaCppFacade() { + b.prepareManagedLlamaCppFacadeWithPortCheck(tcpPortAvailable) +} + +func (b *Broker) prepareManagedLlamaCppFacadeWithPortCheck(portAvailable func(int) bool) { + // An explicit settings choice fixes both ports and turns takeover off, for + // this engine exactly as for the other two. + if b.prepareExplicitEngineSettings(llamacppProxyProfile.Name) { + return + } + profile := llamacppEffectiveProfile() + // Ollama's facade is prepared first, so an inherited OLLAMA_HOST alias is + // already reserved and must stay out of llama.cpp's backend search. + portAvailable = b.availableOffOllamaHostAlias(portAvailable) + + settings := b.getSettings() + if settings == nil { + b.cacheLlamaCppPortStatus() + _, _ = b.blockManagedLlamaCppFacade("managed-port policy is unavailable", nil) + return + } + result, rpcErr, err := settings.Call(context.Background(), "settings/get-force-ports", nil) + if err != nil || rpcErr != nil { + slog.Warn("failed to read managed llama.cpp port setting", "err", err, "rpcErr", rpcErr) + b.cacheLlamaCppPortStatus() + _, _ = b.blockManagedLlamaCppFacade("managed-port policy could not be verified", nil) + return + } + var policy struct { + Value bool `json:"value"` + } + if json.Unmarshal(result, &policy) != nil { + b.cacheLlamaCppPortStatus() + _, _ = b.blockManagedLlamaCppFacade("managed-port policy could not be decoded", nil) + return + } + + em := b.getEngineMgr() + if em == nil { + if !policy.Value { + b.configureUnmanagedLlamaCppFacade() + return + } + _, _ = b.blockManagedLlamaCppFacade("engine manager is unavailable", nil) + return + } + + params, _ := json.Marshal(map[string]any{"engine": profile.Name, "port": profile.FacadePort}) + result, rpcErr, err = em.Call(context.Background(), "engine:status", params) + if err != nil || rpcErr != nil { + if !policy.Value { + b.configureUnmanagedLlamaCppFacade() + return + } + _, _ = b.blockManagedLlamaCppFacade("llama.cpp status could not be verified", nil) + return + } + var st ollamaPortStatus + if json.Unmarshal(result, &st) != nil { + if !policy.Value { + b.configureUnmanagedLlamaCppFacade() + return + } + _, _ = b.blockManagedLlamaCppFacade("llama.cpp status could not be decoded", nil) + return + } + if st.Port > 0 { + b.llamacppState().backendPort.Store(int32(st.Port)) + } + if !policy.Value { + b.configureUnmanagedLlamaCppFacade() + return + } + + plan := planManagedLlamaCppPorts(true, st, portAvailable) + if plan.Blocked != "" { + _, _ = b.blockManagedLlamaCppFacade(plan.Blocked, nil, st.Port) + return + } + if plan.BackendPort != 0 { + params, _ = json.Marshal(map[string]any{"engine": profile.Name, "port": plan.BackendPort}) + result, rpcErr, err = em.CallNoTimeout(context.Background(), "engine:set-port", params) + if err != nil || rpcErr != nil { + b.cacheLlamaCppPortStatus() + _, _ = b.blockManagedLlamaCppFacade("the llama.cpp backend could not be moved", nil, st.Port, plan.BackendPort) + return + } + b.llamacppState().backendPort.Store(int32(plan.BackendPort)) + var moved ollamaPortStatus + if json.Unmarshal(result, &moved) == nil && moved.Port > 0 { + b.llamacppState().backendPort.Store(int32(moved.Port)) + } + } + if !portAvailable(profile.FacadePort) { + _, _ = b.blockManagedLlamaCppFacade("the compatibility port is already in use", nil, st.Port) + return + } + + b.llamacppState().managedFacade.Store(plan.Enabled) + b.llamacppState().startupPort.Store(int32(profile.FacadePort)) + b.forwardErrorsClear(llamacppPortOwnershipBlockedID) +} + +func (b *Broker) llamacppProxyGenerationIsCurrent(generation uint64, p *proxyProcess) bool { + return b.llamaCppProxyGeneration.Load() == generation && + b.llamaCppProxyPublishedGeneration.Load() == generation && + b.getLlamaCppProxy() == p +} + +// rebindLlamaCppFacadeOrFinish moves the facade to another port when the one +// just asked for could not be bound, and gives up on this engine alone if the +// second attempt fails too. See rebindLMStudioFacadeOrFinish for why this +// replaces a process restart. +// +// Caller holds llamacppReadyMu. +func (b *Broker) rebindLlamaCppFacadeOrFinish(p *proxyProcess, generation uint64, avoid ...int) (int, bool) { + if b.llamaCppProxyGeneration.Load() != generation { + return 0, false + } + fallback := b.setLlamaCppProxyFallback(avoid...) + if fallback != 0 && b.rebindLlamaCppProxy(p, fallback) { + return fallback, true + } + slog.Warn("llama.cpp facade could not be rebound; releasing its ownership gate", + "attempted", fallback, "avoided", avoid) + b.finishLlamaCppProxyTerminal() + return 0, false +} + +func (b *Broker) reconcileLlamaCppProxyAfterEngineManagerReady() { + if !b.llamacppPortOwnershipPending() { + return + } + p := b.getLlamaCppProxy() + if p == nil { + return + } + ready, port := p.Status(llamacppProxyProfile.Name) + generation := b.llamaCppProxyPublishedGeneration.Load() + if !ready || port <= 0 || generation != b.llamaCppProxyGeneration.Load() { + return + } + go b.reconcileLlamaCppProxyPortOnReadyForGeneration(generation, port) +} + +// reconcileLlamaCppProxyPortOnReadyForGeneration runs off the proxy reader +// goroutine because both set-port and node/set-local-backend round-trip through +// that reader. +// +// It holds the node configuration lock throughout, as LM Studio's does: +// automatic port reconciliation and a settings operation both move this facade, +// and the journal decides which of them is in charge. +func (b *Broker) reconcileLlamaCppProxyPortOnReadyForGeneration(generation uint64, boundPort int) { + b.engineConfigMu.Lock() + defer b.engineConfigMu.Unlock() + if b.settingsGovernFacadeLocked(llamacppProxyProfile) { + if b.llamaCppProxyGeneration.Load() == generation { + b.markLlamaCppPortReady() + } + return + } + if b.llamaCppProxyGeneration.Load() != generation { + return + } + // Serialize the ownership transition, including fallback rebind: a set-port + // emits another ready before its response, so without this guard that + // second callback could release the gate while the first was still moving + // the proxy. + b.llamacppReadyMu.Lock() + defer b.llamacppReadyMu.Unlock() + + profile := llamacppEffectiveProfile() + p := b.getLlamaCppProxy() + if p == nil || !b.llamacppProxyGenerationIsCurrent(generation, p) { + return + } + if b.llamacppState().managedFacade.Load() && boundPort != profile.FacadePort { + if b.rebindLlamaCppProxy(p, profile.FacadePort) { + boundPort = profile.FacadePort + } else { + fallback, rebound := b.blockManagedLlamaCppFacade("the proxy could not bind the compatibility port", p, boundPort) + if !rebound { + next, ok := b.rebindLlamaCppFacadeOrFinish(p, generation, boundPort, fallback) + if !ok { + return + } + fallback = next + } + boundPort = fallback + } + } + + backend := int(b.llamacppState().backendPort.Load()) + if backend == 0 { + // Fail closed on the relocated default before asking engine-manager + // for status, so the proxy is never transiently sitting on the port + // the engine is configured for. + if boundPort == profile.EnginePortBase { + fallback := b.setLlamaCppProxyFallback(boundPort) + if !b.rebindLlamaCppProxy(p, fallback) { + next, ok := b.rebindLlamaCppFacadeOrFinish(p, generation, boundPort, fallback) + if !ok { + return + } + fallback = next + } + boundPort = fallback + } + backend = profile.EnginePortBase + } + avoidBackendCollision := func() bool { + if backend != boundPort { + return true + } + var fallback int + var rebound bool + if b.llamacppState().managedFacade.Load() { + fallback, rebound = b.blockManagedLlamaCppFacade("the proxy bound the configured llama.cpp backend port", p, boundPort) + } else { + fallback = b.setLlamaCppProxyFallback(boundPort) + rebound = b.rebindLlamaCppProxy(p, fallback) + } + if !rebound { + next, ok := b.rebindLlamaCppFacadeOrFinish(p, generation, boundPort, fallback, backend) + if !ok { + return false + } + fallback = next + } + boundPort = fallback + return true + } + if !avoidBackendCollision() { + return + } + + st, current := b.cacheLlamaCppPortStatus() + if !b.llamacppProxyGenerationIsCurrent(generation, p) { + return + } + cached := int(b.llamacppState().backendPort.Load()) + if cached <= 0 { + // A bound proxy plus an unknown configured backend is not a terminal + // ownership result. Keep restoration gated until engine-manager + // supplies the authoritative port. + slog.Warn("llama.cpp backend port remains unknown; keeping ownership gate closed") + return + } + backend = cached + if !avoidBackendCollision() { + return + } + healthy := current && st.Running && st.Port == backend + if !b.llamacppProxyGenerationIsCurrent(generation, p) { + return + } + b.setProxyLocalBackend(p, llamacppProxyProfile.Name, backend, healthy) + if !b.llamacppProxyGenerationIsCurrent(generation, p) { + return + } + b.markLlamaCppPortReady() +} diff --git a/services/nvpair-ui-broker/llamacppport_test.go b/services/nvpair-ui-broker/llamacppport_test.go new file mode 100644 index 00000000..076dc9c0 --- /dev/null +++ b/services/nvpair-ui-broker/llamacppport_test.go @@ -0,0 +1,284 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +// llama.cpp's managed-port behaviour: which port the facade claims, where the +// engine is moved to, and how a user's own configuration overrides both. + +import ( + "encoding/json" + "path/filepath" + "testing" + "time" +) + +// The facade claims llama.cpp's own default and the engine is relocated +// directly above it, the same relationship Ollama and LM Studio have. 8080 is +// upstream's default (common/common.h), so a llama.cpp client that was already +// pointed at this machine keeps working through the proxy. +func TestLlamaCppClaimsItsStockPortAndMovesTheEngineAbove(t *testing.T) { + profile := llamacppEffectiveProfile() + if profile.FacadePort != 8080 { + t.Errorf("facade port = %d, want llama.cpp's own default 8080", profile.FacadePort) + } + if profile.EnginePortBase != profile.FacadePort+1 { + t.Errorf("engine port base = %d, want one above the facade (%d)", + profile.EnginePortBase, profile.FacadePort+1) + } +} + +// 11434 and 1234 belong to Ollama and LM Studio. One process hosts all three +// facades, so a shared port would not be a config mistake to discover at +// runtime — it would be one facade losing the bind and landing somewhere +// arbitrary, with the OLLAMA_HOST reservation built around 11434 being +// Ollama's alone. +func TestLlamaCppPortsDoNotCollideWithSiblingEngines(t *testing.T) { + llamacpp := llamacppEffectiveProfile() + for _, sibling := range []engineProxyProfile{ollamaProxyProfile, lmstudioProxyProfile} { + for _, taken := range []int{sibling.FacadePort, sibling.EnginePortBase} { + if llamacpp.FacadePort == taken || llamacpp.EnginePortBase == taken { + t.Errorf("llama.cpp (%d/%d) collides with %s port %d", + llamacpp.FacadePort, llamacpp.EnginePortBase, sibling.Name, taken) + } + } + } +} + +// A user who set LLAMA_ARG_PORT has told us where their llama.cpp listens, so +// that is the port their clients use and the one the facade must claim. The +// backend follows it rather than staying on the table's default, which would +// otherwise strand the engine away from its own facade. +func TestLlamaArgPortOverridesTheStockPort(t *testing.T) { + t.Setenv("LLAMA_ARG_PORT", "9000") + profile := llamacppEffectiveProfile() + if profile.FacadePort != 9000 { + t.Errorf("facade port = %d, want the configured 9000", profile.FacadePort) + } + if profile.EnginePortBase != 9001 { + t.Errorf("engine port base = %d, want 9001", profile.EnginePortBase) + } +} + +// Whatever the environment says, the pair it produces has to be two real +// ports. The backend sits one above the facade, so the facade can never be the +// last port. +func TestEffectiveLlamaCppPortsAreAlwaysValid(t *testing.T) { + for _, value := range []string{"", "1", "8080", "65534", "65535", "99999"} { + t.Run("LLAMA_ARG_PORT="+value, func(t *testing.T) { + t.Setenv("LLAMA_ARG_PORT", value) + profile := llamacppEffectiveProfile() + for name, port := range map[string]int{ + "facade": profile.FacadePort, + "engine": profile.EnginePortBase, + } { + if port < 1 || port > 65535 { + t.Errorf("%s port = %d, which is not a usable TCP port", name, port) + } + } + }) + } +} + +// An unusable value is not a reason to plan against a nonsense port; the +// documented default stands. +// +// 65535 is unusable here even though it is a valid port: the backend goes one +// above the facade, so honouring it would plan a move onto 65536. +func TestUnusableLlamaArgPortFallsBackToTheDefault(t *testing.T) { + for _, value := range []string{"", "not-a-port", "0", "-1", "70000", "65535"} { + t.Run("value="+value, func(t *testing.T) { + t.Setenv("LLAMA_ARG_PORT", value) + if got := llamacppEffectiveProfile().FacadePort; got != 8080 { + t.Errorf("facade port = %d with LLAMA_ARG_PORT=%q, want 8080", got, value) + } + }) + } +} + +// The policy that decides whether to move the engine or leave it alone is the +// shared one; these pin that llama.cpp is routed through it as a managed engine +// rather than being special-cased. +func TestLlamaCppPortPlanMovesTheEngineOffTheFacade(t *testing.T) { + free := func(int) bool { return true } + profile := llamacppEffectiveProfile() + + // Engine sitting on the port the facade wants: it is managed, so it moves. + plan := planManagedLlamaCppPorts(true, ollamaPortStatus{Running: true, Port: profile.FacadePort}, free) + if plan.Blocked != "" { + t.Fatalf("a managed engine on the facade port was blocked: %q", plan.Blocked) + } + if !plan.Enabled || plan.BackendPort != profile.EnginePortBase { + t.Fatalf("plan = %+v, want the backend moved to %d", plan, profile.EnginePortBase) + } + + // Already clear of the facade: nothing to move. + plan = planManagedLlamaCppPorts(true, ollamaPortStatus{Running: true, Port: profile.EnginePortBase}, free) + if !plan.Enabled || plan.BackendPort != 0 { + t.Fatalf("plan = %+v, want the facade claimed with no backend move", plan) + } + + // Policy off: nothing is claimed and nothing is moved. + if plan = planManagedLlamaCppPorts(false, ollamaPortStatus{}, free); plan.Enabled || plan.BackendPort != 0 { + t.Fatalf("plan = %+v, want an untouched engine when managed ports are off", plan) + } +} + +// A busy facade port is refused rather than displacing whatever holds it. +func TestLlamaCppPortPlanRefusesAnOccupiedFacade(t *testing.T) { + profile := llamacppEffectiveProfile() + occupied := func(port int) bool { return port != profile.FacadePort } + plan := planManagedLlamaCppPorts(true, ollamaPortStatus{Running: true, Port: profile.EnginePortBase}, occupied) + if plan.Blocked == "" { + t.Fatalf("plan = %+v, want a block while the compatibility port is in use", plan) + } +} + +// The gate and the set-port guard are scoped to this engine: another engine's +// request must not be held behind llama.cpp's port transition, and must not be +// rejected by llama.cpp's reservation. +func TestLlamaCppPortGuardsAreScopedToThisEngine(t *testing.T) { + mine, _ := json.Marshal(map[string]any{"engine": "llamacpp", "port": 8080}) + theirs, _ := json.Marshal(map[string]any{"engine": "lmstudio", "port": 1234}) + + if !needsLlamaCppPortGate("engine:status", mine) { + t.Error("llama.cpp engine:status is not gated") + } + if needsLlamaCppPortGate("engine:status", theirs) { + t.Error("another engine's engine:status is gated behind llama.cpp") + } + if _, ok := llamacppSetPortRequest("engine:set-port", mine); !ok { + t.Error("llama.cpp engine:set-port was not recognised") + } + if _, ok := llamacppSetPortRequest("engine:set-port", theirs); ok { + t.Error("another engine's engine:set-port was claimed by llama.cpp") + } + if _, ok := llamacppSetPortRequest("engine:start", mine); ok { + t.Error("a non set-port method was treated as one") + } +} + +// The ownership gate is what holds engine requests while the port changes +// hands, so a facade that never comes up has to release it or those requests +// wait out their call timeout for the life of the process. +func TestLlamaCppTerminalFacadeReleasesItsGate(t *testing.T) { + b := &Broker{llamacppPortReady: make(chan struct{})} + b.llamacppState().managedFacade.Store(true) + + if !b.llamacppPortOwnershipPending() { + t.Fatal("a fresh gate is not pending") + } + b.finishLlamaCppProxyTerminal() + if b.llamacppPortOwnershipPending() { + t.Error("the gate is still pending after the facade went terminal") + } + if b.llamacppState().managedFacade.Load() { + t.Error("the managed claim survived a terminal facade") + } + // Idempotent: the supervisor and the spawn path can both reach this. + b.finishLlamaCppProxyTerminal() +} + +// A llama.cpp facade can announce ready before engine-manager is up. Nothing +// can finish reconciling then — the configured backend port is unknown, so the +// gate deliberately stays closed — and it is engine-manager's own ready +// notification that has to replay the reconcile for the published generation, +// exactly as it does for LM Studio. Without that hook every gated llama.cpp +// request waits out its call timeout for the life of the process. +func TestLlamaCppGateOpensWhenEngineManagerReadiesAfterTheFacade(t *testing.T) { + t.Setenv("LLAMA_ARG_PORT", "") + proxy, localBackend := llamaCppProxyPipe(t, defaultLlamaCppProxyPort) + b := &Broker{ + clusterDir: filepath.Join(t.TempDir(), "cluster"), + llamacppPortReady: make(chan struct{}), + } + b.setLlamaCppProxy(proxy) + // spawnProxy publishes the generation it brought the facade up under; the + // facade's own ready ran before any engine-manager existed. + generation := b.llamaCppProxyGeneration.Add(1) + b.llamaCppProxyPublishedGeneration.Store(generation) + if !b.llamacppPortOwnershipPending() { + t.Fatal("a fresh gate is not pending") + } + + b.setEngineMgr(serveEngineStatus(t, defaultLlamaCppPort)) + b.forwardEngineNotification("engine:ready", nil) + + select { + case <-b.llamacppPortReady: + case <-time.After(5 * time.Second): + t.Fatal("llama.cpp gate stayed closed after engine-manager readied") + } + select { + case backend := <-localBackend: + if backend.Engine != llamacppProxyProfile.Name || backend.Port != defaultLlamaCppPort || !backend.Healthy { + t.Fatalf("local backend = %+v, want healthy %s:%d", backend, llamacppProxyProfile.Name, defaultLlamaCppPort) + } + case <-time.After(2 * time.Second): + t.Fatal("the replayed reconcile did not hand the facade its engine backend") + } + if got := int(b.llamacppState().backendPort.Load()); got != defaultLlamaCppPort { + t.Fatalf("cached backend port = %d, want %d", got, defaultLlamaCppPort) + } + if b.llamacppPortOwnershipPending() { + t.Fatal("the gate reports pending after it was released") + } +} + +// Every engine needs a startup-gate finisher, because the gate is what holds +// client requests while a port changes hands. An engine missing from +// finishEngineProxyStartup keeps its gate shut for the life of the process, so +// every gated request — engine:get-installed among them — waits out its call +// timeout and answers "retry" forever. That is what a broker running without a +// resolvable proxy binary does, and it is not a test-only condition. +func TestEveryEngineHasAStartupGateFinisher(t *testing.T) { + for _, profile := range engineProxyProfiles { + t.Run(profile.Name, func(t *testing.T) { + b := &Broker{ + ollamaPortReady: make(chan struct{}), + lmstudioPortReady: make(chan struct{}), + llamacppPortReady: make(chan struct{}), + } + // Only this engine's gate, not managedPortOwnershipReady: that + // wants every gate open, and finishing one engine is not supposed + // to release the others. + gates := map[string]chan struct{}{ + ollamaProxyProfile.Name: b.ollamaPortReady, + lmstudioProxyProfile.Name: b.lmstudioPortReady, + llamacppProxyProfile.Name: b.llamacppPortReady, + } + gate, ok := gates[profile.Name] + if !ok { + t.Fatalf("%s has no ownership gate in this test's table", profile.Name) + } + + b.finishEngineProxyStartup(profile) + + select { + case <-gate: + default: + t.Fatalf("%s has no startup-gate finisher, so its gated requests would time out", + profile.Name) + } + }) + } +} + +// A nil gate means "not configured", which is how a bare &Broker{} drives one +// behaviour in isolation. Waiting on it would block that caller forever. +func TestUnconfiguredPortGatesAreNotWaitedOn(t *testing.T) { + if got := len((&Broker{}).managedPortGates()); got != 0 { + t.Fatalf("managedPortGates on a bare broker = %d, want none", got) + } + b := &Broker{llamacppPortReady: make(chan struct{})} + if got := len(b.managedPortGates()); got != 1 { + t.Fatalf("managedPortGates = %d, want just the configured one", got) + } + if b.managedPortOwnershipReady() { + t.Error("ownership reported ready while the one configured gate is shut") + } + close(b.llamacppPortReady) + if !b.managedPortOwnershipReady() { + t.Error("ownership not ready after the only configured gate opened") + } +} diff --git a/services/nvpair-ui-broker/llamacppproxy.go b/services/nvpair-ui-broker/llamacppproxy.go new file mode 100644 index 00000000..413d73fe --- /dev/null +++ b/services/nvpair-ui-broker/llamacppproxy.go @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "log/slog" +) + +// llamacppproxy.go is the broker's llama.cpp counterpart to lmstudioproxy.go: +// the engine-specific head of the shared proxy wiring. The facade lives in the +// same nvpair-proxy process as every other engine's and is brought up with a +// facade/enable after spawn, so there is no separate binary, supervisor, or +// argv here. +// +// llama.cpp is treated exactly as LM Studio is: its stock port is claimed for +// the facade and the engine is relocated above it. The port choreography that +// makes that safe lives in llamacppport.go; this file is the process-facing +// head — the enable spec, the fallback port, and the notification reader. +// +// llamacppProxyProfile is declared in llamacppport.go, next to the effective +// profile that applies LLAMA_ARG_PORT to it. + +// llamacppState is shorthand for this engine's runtime row, matching +// ollamaState / lmstudioState. +func (b *Broker) llamacppState() *engineProxyRuntime { return b.engineProxy(llamacppProxyProfile) } + +func (b *Broker) setLlamaCppProxy(p *proxyProcess) { + b.setEngineProxyHandle(llamacppProxyProfile, p) +} + +func (b *Broker) getLlamaCppProxy() *proxyProcess { + return b.engineProxyHandle(llamacppProxyProfile) +} + +// llamacppFacadeSpec is the enable request for llama.cpp's facade. Like LM +// Studio's it carries no alias addresses — the alias stands in for an +// inherited host variable, which only Ollama has. +func (b *Broker) llamacppFacadeSpec() enableFacadeRequest { + spec := enableFacadeRequest{Engine: llamacppProxyProfile.Name} + if port := int(b.llamacppState().startupPort.Load()); port != 0 { + spec.Port = port + spec.IgnorePersistedPort = true + } + return spec +} + +// setLlamaCppProxyFallback picks a port for this facade to retry on and records +// it, so a later enable asks for the same one. +// +// Mirrors setLMStudioProxyFallback: search from the backend base, and with no +// authoritative backend port yet treat both the compatibility port and the +// relocated default as unsafe rather than as evidence they are free. +func (b *Broker) setLlamaCppProxyFallback(excludedPorts ...int) int { + profile := llamacppEffectiveProfile() + if aliasPort := b.currentOllamaHostAlias().Port; aliasPort > 0 { + excludedPorts = append(excludedPorts, aliasPort) + } + for port := range b.siblingEngineProxyPorts(llamacppProxyProfile) { + excludedPorts = append(excludedPorts, port) + } + if backend := int(b.llamacppState().backendPort.Load()); backend > 0 { + excludedPorts = append(excludedPorts, backend) + } else { + excludedPorts = append(excludedPorts, profile.FacadePort, profile.EnginePortBase) + } + fallback := nextAvailablePortExcluding(profile.EnginePortBase, excludedPorts, tcpPortAvailable) + b.llamacppState().startupPort.Store(int32(fallback)) + return fallback +} + +// finishLlamaCppProxyTerminal gives up this engine's managed claim and releases +// its startup gate, so client requests stop waiting on an engine that is not +// coming up. +func (b *Broker) finishLlamaCppProxyTerminal() { + b.llamacppState().managedFacade.Store(false) + b.markLlamaCppPortReady() +} + +// llamacppFallbackPort mirrors lmstudioFallbackPort: prefer the port the +// bind-failed notification already chose, and recompute only if it did not run. +func (b *Broker) llamacppFallbackPort(failed int) int { + if planned := int(b.llamacppState().startupPort.Load()); planned != 0 && planned != failed { + return planned + } + return b.setLlamaCppProxyFallback(failed) +} + +// forwardLlamaCppProxyNotificationForGeneration is the hook spawnProxy invokes +// for this facade's notifications, mirroring its LM Studio counterpart: +// errors:report / errors:clear go into the nvpair-errors pipeline; a lost bind +// chooses a fallback without calling back into this reader goroutine; a ready +// notification starts the ownership reconcile on its own goroutine, because +// that path's set-port and node/set-local-backend both round-trip through here. +func (b *Broker) forwardLlamaCppProxyNotificationForGeneration(generation uint64, method string, params json.RawMessage) { + if b.llamaCppProxyGeneration.Load() != generation { + return + } + // Strip the facade address before anything dispatches on the method, + // starting with the errors relay below: it matches bare names. + method, addressed := facadeMethodFor(llamacppProxyProfile, method) + if !addressed { + return + } + if b.dispatchErrorsNotif(llamacppProxyProfile.ComponentName(), method, params) { + return + } + // A process can win the compatibility port after preparation's free-port + // check but before the facade binds. Choose an explicit fallback here; the + // failing enable retries in-process and llamacppFallbackPort finds it. A + // port the user chose through settings is left alone, as for every engine. + if port, recover := b.facadeBindFailure(llamacppProxyProfile, method, params); recover { + if b.llamacppState().managedFacade.Load() && port == llamacppEffectiveProfile().FacadePort { + _, _ = b.blockManagedLlamaCppFacade("another process acquired the compatibility port during startup", nil) + } else { + fallback := b.setLlamaCppProxyFallback(port) + slog.Warn("llama.cpp proxy bind failed; retrying on fallback", "port", port, "fallback", fallback) + } + } + if method == "ready" { + var rp proxyReadyParams + if err := json.Unmarshal(params, &rp); err == nil && rp.Port > 0 { + go b.reconcileLlamaCppProxyPortOnReadyForGeneration(generation, rp.Port) + } + } + b.forwardEngineProxyNotification(llamacppProxyProfile, method, params) +} + +// cancelLlamaWorkload forwards a headless cancel to the exact request that +// produced it. Only llama.cpp requests that originated on this node are +// accepted: the run id identifies one proxy lifetime, so a stale run or a +// foreign origin cannot cancel an unrelated request that reused an id. +func (b *Broker) cancelLlamaWorkload(msg *Message) { + var params struct { + ID string `json:"id"` + RunID string `json:"runId"` + Engine string `json:"engine"` + Origin string `json:"originatedFrom"` + } + if json.Unmarshal(msg.Params, ¶ms) != nil || params.ID == "" || params.RunID == "" { + _ = b.codec.RespondError(msg.ID, -32602, "exact workload id and runId are required") + return + } + if params.Engine != llamacppProxyProfile.Name || params.Origin == "" || params.Origin != b.nodeID { + _ = b.codec.RespondError(msg.ID, -32000, "only llama.cpp requests originating on this node can be cancelled here") + return + } + forward := *msg + forward.Method = llamacppProxyProfile.ComponentName() + ":workload/cancel" + b.relayToEngineProxy(llamacppProxyProfile, &forward) +} diff --git a/services/nvpair-ui-broker/lmstudioport.go b/services/nvpair-ui-broker/lmstudioport.go index a7366627..4b9cbde7 100644 --- a/services/nvpair-ui-broker/lmstudioport.go +++ b/services/nvpair-ui-broker/lmstudioport.go @@ -38,8 +38,24 @@ func (b *Broker) markLMStudioPortReady() { } } +// managedPortGates are the per-engine ownership gates, in table order. +// +// A nil gate is skipped rather than waited on, which is the convention the rest +// of this file already follows: nil means "no gate configured", as it is for a +// test that drives one behaviour off a bare &Broker{}. Selecting on a nil +// channel would block that caller forever instead. +func (b *Broker) managedPortGates() []<-chan struct{} { + gates := make([]<-chan struct{}, 0, 3) + for _, gate := range []chan struct{}{b.ollamaPortReady, b.lmstudioPortReady, b.llamacppPortReady} { + if gate != nil { + gates = append(gates, gate) + } + } + return gates +} + func (b *Broker) waitForManagedPortOwnership(ctx context.Context) bool { - for _, ready := range []<-chan struct{}{b.ollamaPortReady, b.lmstudioPortReady} { + for _, ready := range b.managedPortGates() { select { case <-ctx.Done(): return false @@ -50,7 +66,7 @@ func (b *Broker) waitForManagedPortOwnership(ctx context.Context) bool { } func (b *Broker) managedPortOwnershipReady() bool { - for _, ready := range []<-chan struct{}{b.ollamaPortReady, b.lmstudioPortReady} { + for _, ready := range b.managedPortGates() { select { case <-ready: default: @@ -354,7 +370,7 @@ func (b *Broker) reconcileLMStudioProxyPortOnReady(boundPort int) { func (b *Broker) reconcileLMStudioProxyPortOnReadyForGeneration(generation uint64, boundPort int) { b.engineConfigMu.Lock() defer b.engineConfigMu.Unlock() - if b.loadEngineSettingsLocked() != nil { + if b.settingsGovernFacadeLocked(lmstudioProxyProfile) { if b.lmstudioProxyGeneration.Load() == generation { b.markLMStudioPortReady() } @@ -363,13 +379,6 @@ func (b *Broker) reconcileLMStudioProxyPortOnReadyForGeneration(generation uint6 if b.lmstudioProxyGeneration.Load() != generation { return } - if _, explicit := b.explicitEngineSettingsLocked("lmstudio"); explicit { - b.markLMStudioPortReady() - return - } - if b.lmstudioProxyGeneration.Load() != generation { - return - } // Serialize the ownership transition, including fallback rebind. A set-port // emits another ready before its response, so without this guard that second // callback could release the gate while the first callback was still moving diff --git a/services/nvpair-ui-broker/lmstudioproxy.go b/services/nvpair-ui-broker/lmstudioproxy.go index 28f1afc0..201d8dd1 100644 --- a/services/nvpair-ui-broker/lmstudioproxy.go +++ b/services/nvpair-ui-broker/lmstudioproxy.go @@ -84,18 +84,12 @@ func (b *Broker) forwardLMStudioProxyNotificationForGeneration(generation uint64 // The failed process is not exiting any more: this notification precedes // the failing enable's response, so the port chosen here is what // lmstudioFallbackPort finds when that enable retries in-process. - if method == "error" { - var ep struct { - Code string `json:"code"` - Port int `json:"port"` - } - if json.Unmarshal(params, &ep) == nil && ep.Code == "bind-failed" && !b.lmstudioState().explicitSettings.Load() { - if b.lmstudioState().managedFacade.Load() && ep.Port == managedLMStudioFacadePort { - _, _ = b.blockManagedLMStudioFacade("another process acquired the compatibility port during startup", nil) - } else { - fallback := b.setLMStudioProxyFallback(ep.Port) - slog.Warn("LM Studio proxy bind failed; retrying on fallback", "port", ep.Port, "fallback", fallback) - } + if port, recover := b.facadeBindFailure(lmstudioProxyProfile, method, params); recover { + if b.lmstudioState().managedFacade.Load() && port == managedLMStudioFacadePort { + _, _ = b.blockManagedLMStudioFacade("another process acquired the compatibility port during startup", nil) + } else { + fallback := b.setLMStudioProxyFallback(port) + slog.Warn("LM Studio proxy bind failed; retrying on fallback", "port", port, "fallback", fallback) } } if method == "ready" { diff --git a/services/nvpair-ui-broker/manualnodes.go b/services/nvpair-ui-broker/manualnodes.go index e0786874..04dbf930 100644 --- a/services/nvpair-ui-broker/manualnodes.go +++ b/services/nvpair-ui-broker/manualnodes.go @@ -17,9 +17,9 @@ import ( // broker needs. Its JSON tags match the producer's so node/discovered| // updated|removed payloads unmarshal straight into it (the GPU/CPU/memory // sub-objects reuse the broker's discovery types, whose tags are identical). -// The ollama_* / lmstudio_* fields drive the per-engine manual→proxy bridge -// (bridgeManualNode); the rest project into the discovery store via -// manualToEnriched. +// The ollama_* / lmstudio_* / llamacpp_* fields drive the per-engine +// manual→proxy bridge (bridgeManualNode); the rest project into the discovery +// store via manualToEnriched. type manualNodeStatus struct { ID string `json:"id"` Address string `json:"address"` @@ -29,6 +29,9 @@ type manualNodeStatus struct { LMStudioUp bool `json:"lmstudio_up"` LMStudioPort int `json:"lmstudio_port"` LMStudioModels []string `json:"lmstudio_models,omitempty"` + LlamaCppUp bool `json:"llamacpp_up"` + LlamaCppPort int `json:"llamacpp_port"` + LlamaCppModels []string `json:"llamacpp_models,omitempty"` NodeInfoPort int `json:"node_info_port"` GPUs []GPUInfo `json:"gpus"` CPU *CPUInfo `json:"cpu"` @@ -88,7 +91,7 @@ func manualToEnriched(s manualNodeStatus) EnrichedNode { GPUs: s.GPUs, CPU: s.CPU, Memory: s.Memory, - Models: mergeModels(s.OllamaModels, s.LMStudioModels), + Models: mergeModels(s.OllamaModels, s.LMStudioModels, s.LlamaCppModels), ModelsByEngine: manualModelsByEngine(s), } if s.Address != "" { @@ -99,9 +102,9 @@ func manualToEnriched(s manualNodeStatus) EnrichedNode { // manualModelsByEngine builds the per-engine attribution for a manual node from // the per-engine lists the prober already collected, keyed by the same -// engine-manager engine names discovered nodes use ("ollama", "lmstudio") so the -// two discovery sources present ModelsByEngine identically. An engine with no -// models adds no key; returns nil when neither engine reports any. +// engine-manager engine names discovered nodes use ("ollama", "lmstudio", +// "llamacpp") so the two discovery sources present ModelsByEngine identically. +// An engine with no models adds no key; returns nil when no engine reports any. func manualModelsByEngine(s manualNodeStatus) map[string][]string { byEngine := map[string][]string{} if len(s.OllamaModels) > 0 { @@ -110,6 +113,9 @@ func manualModelsByEngine(s manualNodeStatus) map[string][]string { if len(s.LMStudioModels) > 0 { byEngine["lmstudio"] = s.LMStudioModels } + if len(s.LlamaCppModels) > 0 { + byEngine["llamacpp"] = s.LlamaCppModels + } if len(byEngine) == 0 { return nil } @@ -150,11 +156,11 @@ type proxyManualNode struct { // bridgeManualNode keeps every supervised proxy's manual-node set in step with // a manual node's per-engine reachability: a node whose Ollama is up is bridged -// into ollama-proxy and one whose LM Studio is up into lmstudio-proxy -// (idempotent — each proxy upserts on a repeat), while an engine that is not -// (or no longer) reachable is removed from its proxy. Each leg is a no-op when -// that proxy isn't supervised — the bridge only applies when the broker owns -// both ends. +// into ollama-proxy, one whose LM Studio is up into lmstudio-proxy, and one +// whose llama.cpp is up into llamacpp-proxy (idempotent — each proxy upserts on +// a repeat), while an engine that is not (or no longer) reachable is removed +// from its proxy. Each leg is a no-op when that proxy isn't supervised — the +// bridge only applies when the broker owns both ends. // // Manual nodes are, by definition, the nodes that never appear in the discovery // relay's snapshots — they advertise no _nvpair-node record for the scanner @@ -163,6 +169,7 @@ type proxyManualNode struct { func (b *Broker) bridgeManualNode(s manualNodeStatus, key string) { b.bridgeToProxy(b.getProxy(), "ollama", s, key, s.OllamaUp, s.OllamaPort, s.OllamaModels) b.bridgeToProxy(b.getLMStudioProxy(), "lmstudio", s, key, s.LMStudioUp, s.LMStudioPort, s.LMStudioModels) + b.bridgeToProxy(b.getLlamaCppProxy(), "llamacpp", s, key, s.LlamaCppUp, s.LlamaCppPort, s.LlamaCppModels) } // bridgeToProxy adds the node to p when its engine is reachable, or removes it @@ -197,6 +204,7 @@ func (b *Broker) bridgeToProxy(p *proxyProcess, engine string, s manualNodeStatu func (b *Broker) removeManualNodeFromProxies(id string) { b.callProxyManual(b.getProxy(), "ollama", "node/remove-manual", map[string]string{"id": id}, id) b.callProxyManual(b.getLMStudioProxy(), "lmstudio", "node/remove-manual", map[string]string{"id": id}, id) + b.callProxyManual(b.getLlamaCppProxy(), "llamacpp", "node/remove-manual", map[string]string{"id": id}, id) } // callProxyManual issues a best-effort node/add-manual|remove-manual to one diff --git a/services/nvpair-ui-broker/manualnodes_test.go b/services/nvpair-ui-broker/manualnodes_test.go new file mode 100644 index 00000000..d2b5c033 --- /dev/null +++ b/services/nvpair-ui-broker/manualnodes_test.go @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "net" + "testing" + "time" +) + +func TestManualModelsByEngineIncludesLlamaCpp(t *testing.T) { + s := manualNodeStatus{ + ID: "lab", + OllamaModels: []string{"llama3"}, + LMStudioModels: []string{"qwen"}, + LlamaCppModels: []string{"loaded-one"}, + } + got := manualModelsByEngine(s) + if len(got["llamacpp"]) != 1 || got["llamacpp"][0] != "loaded-one" { + t.Fatalf("llamacpp = %#v, want [loaded-one]", got["llamacpp"]) + } + en := manualToEnriched(s) + found := false + for _, m := range en.Models { + if m == "loaded-one" { + found = true + break + } + } + if !found { + t.Fatalf("Models missing loaded-one: %#v", en.Models) + } +} + +func TestManualNodeBridgesLlamaCppIntoProxy(t *testing.T) { + proxy, calls := llamaCppManualProxyPipe(t) + b := newManualTestBroker() + b.setLlamaCppProxy(proxy) + + b.bridgeManualNode(manualNodeStatus{ + ID: "lab", + Address: "10.0.0.5", + LlamaCppUp: true, + LlamaCppPort: 8082, + LlamaCppModels: []string{"loaded-one"}, + }, "lab") + + call := readProxyManualCall(t, calls) + // Addressed to the facade, not bare: one process hosts every engine, so + // the engine has to travel with the message. + if want := llamacppProxyProfile.addressed("node/add-manual"); call.method != want { + t.Fatalf("method = %q, want %q", call.method, want) + } + var node proxyManualNode + if err := json.Unmarshal(call.params, &node); err != nil { + t.Fatalf("decode add-manual: %v", err) + } + if node.ID != "lab" || node.Host != "10.0.0.5" || node.Port != 8082 { + t.Fatalf("bridged node = %+v", node) + } + if len(node.Models) != 1 || node.Models[0] != "loaded-one" { + t.Fatalf("models = %#v, want [loaded-one]", node.Models) + } +} + +func TestManualNodeRemovesLlamaCppFromProxyWhenDown(t *testing.T) { + proxy, calls := llamaCppManualProxyPipe(t) + b := newManualTestBroker() + b.setLlamaCppProxy(proxy) + + b.bridgeManualNode(manualNodeStatus{ + ID: "lab", + Address: "10.0.0.5", + LlamaCppUp: false, + LlamaCppPort: 8082, + }, "lab") + + call := readProxyManualCall(t, calls) + if want := llamacppProxyProfile.addressed("node/remove-manual"); call.method != want { + t.Fatalf("method = %q, want %q", call.method, want) + } + var params map[string]string + if err := json.Unmarshal(call.params, ¶ms); err != nil { + t.Fatalf("decode remove-manual: %v", err) + } + if params["id"] != "lab" { + t.Fatalf("remove id = %q, want lab", params["id"]) + } +} + +func TestRemoveManualNodeFromProxiesDropsLlamaCpp(t *testing.T) { + proxy, calls := llamaCppManualProxyPipe(t) + b := newManualTestBroker() + b.setLlamaCppProxy(proxy) + + b.removeManualNodeFromProxies("lab") + + call := readProxyManualCall(t, calls) + if want := llamacppProxyProfile.addressed("node/remove-manual"); call.method != want { + t.Fatalf("method = %q, want %q", call.method, want) + } + var params map[string]string + if err := json.Unmarshal(call.params, ¶ms); err != nil { + t.Fatalf("decode remove-manual: %v", err) + } + if params["id"] != "lab" { + t.Fatalf("remove id = %q, want lab", params["id"]) + } +} + +type proxyManualCall struct { + method string + params json.RawMessage +} + +func llamaCppManualProxyPipe(t *testing.T) (*proxyProcess, <-chan proxyManualCall) { + t.Helper() + proxyClient, proxyServer := net.Pipe() + t.Cleanup(func() { + _ = proxyClient.Close() + _ = proxyServer.Close() + }) + proxy := &proxyProcess{peer: NewPeer(NewCodec(proxyClient))} + go proxy.peer.Serve(nil, nil) + + calls := make(chan proxyManualCall, 1) + go func() { + codec := NewCodec(proxyServer) + msg, err := codec.Read() + if err != nil { + return + } + calls <- proxyManualCall{method: msg.Method, params: msg.Params} + _ = codec.Respond(msg.ID, map[string]bool{"ok": true}) + }() + return proxy, calls +} + +func readProxyManualCall(t *testing.T, calls <-chan proxyManualCall) proxyManualCall { + t.Helper() + select { + case call := <-calls: + return call + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for llamacpp-proxy manual-node call") + return proxyManualCall{} + } +} diff --git a/services/nvpair-ui-broker/proxyport.go b/services/nvpair-ui-broker/proxyport.go index 7ecfb7b5..017f9cbe 100644 --- a/services/nvpair-ui-broker/proxyport.go +++ b/services/nvpair-ui-broker/proxyport.go @@ -182,8 +182,16 @@ func (b *Broker) finishEngineProxyStartup(profile engineProxyProfile) { b.finishOllamaProxyTerminal() case lmstudioProxyProfile.Name: b.finishLMStudioProxyTerminal() + case llamacppProxyProfile.Name: + b.finishLlamaCppProxyTerminal() default: - slog.Warn("no startup-gate finisher for engine", "engine", profile.Name) + // Reaching this leaves that engine's gate shut for the life of the + // process, so every gated client request waits out its call timeout + // and answers "retry" forever. A warning is the most this can do from + // here, but an engine added without a finisher is a startup hang, not + // a logging gap. + slog.Error("no startup-gate finisher for engine; its gated requests will time out", + "engine", profile.Name) } } @@ -417,11 +425,7 @@ func needsOllamaPortGate(method string, params json.RawMessage) bool { func (b *Broker) reconcileProxyPortOnReady(boundPort int) { b.engineConfigMu.Lock() defer b.engineConfigMu.Unlock() - if b.loadEngineSettingsLocked() != nil { - b.markOllamaPortReady() - return - } - if _, explicit := b.explicitEngineSettingsLocked("ollama"); explicit { + if b.settingsGovernFacadeLocked(ollamaProxyProfile) { b.markOllamaPortReady() return } diff --git a/services/nvpair-ui-broker/settingsport.go b/services/nvpair-ui-broker/settingsport.go index c1d11105..de20068a 100644 --- a/services/nvpair-ui-broker/settingsport.go +++ b/services/nvpair-ui-broker/settingsport.go @@ -11,17 +11,18 @@ import ( settings "nvpair-shared/enginesettings" ) -// handleSettingsPortRPC serves the port-only RPCs — engine:set-port, -// proxy:set-port, and lmstudio-proxy:set-port — which nvpair-tui calls to move -// a single port without rendering the full launch settings form. They keep -// their own narrow request and response shapes, but run through the same -// authoritative settings operation as the desktop editor, so a port change -// made from the terminal cannot diverge from one made from the UI. In -// particular a proxy change now fails on a busy port instead of silently -// binding a different one. +// handleSettingsPortRPC serves the port-only RPCs — engine:set-port and every +// engine's -proxy:set-port — which nvpair-tui calls to move a single +// port without rendering the full launch settings form. They keep their own +// narrow request and response shapes, but run through the same authoritative +// settings operation as the desktop editor, so a port change made from the +// terminal cannot diverge from one made from the UI. In particular a proxy +// change now fails on a busy port instead of silently binding a different one. // // proxyEngine selects which port the request addresses: "" is the engine's own -// server port, and a non-empty value names the proxy in front of that engine. +// server port, and a non-empty value names the engine whose proxy facade moves. +// Every engine in engineProxyProfiles is served the same way; an engine without +// a profile is refused by the settings operation itself. func (b *Broker) handleSettingsPortRPC(msg *Message, proxyEngine string) { var p struct { Engine string `json:"engine"` diff --git a/services/readme.md b/services/readme.md index a701adc7..d4567b35 100644 --- a/services/readme.md +++ b/services/readme.md @@ -11,8 +11,9 @@ local network: each node advertises itself over mDNS as one consolidated node offers and where to reach them. What a discovered node can actually serve is a separate question, answered after -discovery. A node may be running [Ollama](https://ollama.com/), LM Studio, both, -or neither, and its model inventory is fetched over HTTP from its engine-manager +discovery. A node may be running [Ollama](https://ollama.com/), LM Studio, +llama.cpp, any combination, or none, and its model inventory is fetched over HTTP +from its engine-manager rather than crammed into mDNS TXT records, which are too small to carry it. Locally, each node exposes compatibility proxies — Ollama-compatible and @@ -40,7 +41,7 @@ This tree builds twelve Go binaries. `nvpair-ui-broker` is the parent service an | Binary | Role | | --- | --- | | `nvpair-ui-broker` | Parent service and JSON-RPC API surface used by the bundled UI and other clients. Supervises workers, relays consolidated discovery, and coordinates routing and scheduling. | -| `nvpair-proxy` | The engine HTTP reverse proxy: one process hosting a facade per enabled engine, each enabled over `facade/enable` after spawn. Serves each engine's own dialect (Ollama-native and OpenAI-compatible), routes only to advertised model owners, and applies owner failover and scheduler priorities. | +| `nvpair-proxy` | The engine HTTP reverse proxy: one process hosting a facade per enabled engine, each enabled over `facade/enable` after spawn. Serves each engine's own dialect (Ollama-native and OpenAI-compatible), routes only to advertised model owners, and applies owner failover and scheduler priorities. llama.cpp's facade is the one whose eligibility is **loaded** models only, because PAIR runs it with `--no-models-autoload`. | | `nvpair-node-info` | Local HTTP service on `:14318` exposing GPU, CPU, and memory inventory at `/v1/node-info`. | | `nvpair-node-scanner` | Consolidated discovery daemon. Advertises and browses `_nvpair-node._tcp`, maintains the node directory, and enriches peers with hardware and model information over HTTP. | | `nvpair-manual-nodes` | Manages user-added nodes that don't appear via mDNS; probes them every 10 s. | @@ -58,7 +59,7 @@ The mDNS responder is our own rather than the host's, because Windows ships none The broker feeds every accepted local or peer workload transition plus compact GPU telemetry to the scheduler. Queued and running work is counted by destination -node across Ollama and LM Studio together. Fresh maximum-GPU utilization is +node across Ollama, LM Studio, and llama.cpp together. Fresh maximum-GPU utilization is smoothed into pressure 0–3; missing or stale telemetry is neutral. Rankings use `pending + gpuPressure`, and each proxy adds local reservations before choosing, so bursts spread without waiting for workload feedback. diff --git a/services/shared/appdir/appdir.go b/services/shared/appdir/appdir.go index ced1358c..4bd8a414 100644 --- a/services/shared/appdir/appdir.go +++ b/services/shared/appdir/appdir.go @@ -33,6 +33,16 @@ func Dir() (string, error) { return filepath.Join(base, orgDir, appDir), nil } +// ModelsDir is persistent user content, deliberately outside the application +// directory removed by reset and uninstall. +func ModelsDir() (string, error) { + base, err := baseDir() + if err != nil { + return "", err + } + return filepath.Join(base, orgDir, "Personal AI Router Models"), nil +} + // Path joins elems onto Dir(). func Path(elems ...string) (string, error) { d, err := Dir() diff --git a/services/shared/appdir/appdir_test.go b/services/shared/appdir/appdir_test.go new file mode 100644 index 00000000..639ac279 --- /dev/null +++ b/services/shared/appdir/appdir_test.go @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package appdir + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestPersistentModelsSurviveApplicationDirectoryRemoval(t *testing.T) { + root := t.TempDir() + for _, key := range []string{"HOME", "XDG_CONFIG_HOME", "LOCALAPPDATA"} { + t.Setenv(key, root) + } + app, err := Dir() + if err != nil { + t.Fatal(err) + } + models, err := ModelsDir() + if err != nil { + t.Fatal(err) + } + for _, path := range []string{app, models} { + if !strings.HasPrefix(path, root+string(filepath.Separator)) { + t.Fatal("fixture escaped its isolated root") + } + if err := os.MkdirAll(path, 0700); err != nil { + t.Fatal(err) + } + } + file := filepath.Join(models, "retained.gguf") + if err := os.WriteFile(file, []byte("GGUFretained"), 0600); err != nil { + t.Fatal(err) + } + if err := os.RemoveAll(app); err != nil { + t.Fatal(err) + } + if bytes, err := os.ReadFile(file); err != nil || string(bytes) != "GGUFretained" { + t.Fatalf("application removal deleted persistent model data: %v", err) + } +} diff --git a/services/shared/engines/engines.go b/services/shared/engines/engines.go index b7f1e8c0..b3842ac2 100644 --- a/services/shared/engines/engines.go +++ b/services/shared/engines/engines.go @@ -61,6 +61,11 @@ // relocates the engine to free that port, and the base of the next-free-port // search. The two must differ. // +// Where an engine lets the user move that default through the environment, +// the value read at preparation time wins over the constant here. The +// constant is what a stock install uses, not an assertion about this +// machine. +// // - All is ordered, and Ollama is first. The broker prepares managed ports in // this order, and Ollama's preparation reserves any inherited OLLAMA_HOST // alias that later engines must route around. Iterating a map here would @@ -89,7 +94,9 @@ type Engine struct { DiscoveryService noderec.ServiceKey // FacadePort is the engine's stock client-facing port, which PAIR's proxy - // claims in managed mode. + // claims in managed mode. An engine whose port the user can move through + // the environment supersedes this at preparation time; see the package + // comment. FacadePort int // EnginePortBase is where PAIR relocates the engine so the proxy can take @@ -143,6 +150,23 @@ var all = []Engine{ EnginePortBase: 1235, PortFile: "lmstudio-proxy-port.json", }, + { + // 8080 is llama.cpp's own default: common/common.h declares + // `int32_t port = 8080` and the server documents "listens on + // 127.0.0.1:8080". So it is treated exactly like the other two — the + // facade claims the port a llama.cpp client already points at, and + // the engine is relocated to EnginePortBase directly above it. + // + // A user who has set LLAMA_ARG_PORT has told us where their llama.cpp + // listens; that is read at preparation time and supersedes this + // default, the same way an inherited OLLAMA_HOST supersedes 11434. + Name: "llamacpp", + DisplayName: "llama.cpp", + DiscoveryService: noderec.ServiceLlamaCpp, + FacadePort: 8080, + EnginePortBase: 8081, + PortFile: "llamacpp-proxy-port.json", + }, } // All returns the engine set in preparation order. The result is a copy, so a diff --git a/services/shared/engines/engines_test.go b/services/shared/engines/engines_test.go index aa609be8..1cc0af55 100644 --- a/services/shared/engines/engines_test.go +++ b/services/shared/engines/engines_test.go @@ -23,7 +23,7 @@ func TestOllamaIsPreparedFirst(t *testing.T) { func TestNames(t *testing.T) { got := Names() - want := []string{"ollama", "lmstudio"} + want := []string{"ollama", "lmstudio", "llamacpp"} if len(got) != len(want) { t.Fatalf("Names() = %v, want %v", got, want) } diff --git a/services/shared/noderec/noderec.go b/services/shared/noderec/noderec.go index 112e7fe3..d1e58189 100644 --- a/services/shared/noderec/noderec.go +++ b/services/shared/noderec/noderec.go @@ -87,6 +87,7 @@ const ( ServiceNodeInfo ServiceKey = "ni" ServiceOllama ServiceKey = "ol" ServiceLMStudio ServiceKey = "lm" + ServiceLlamaCpp ServiceKey = "lc" ServiceErrors ServiceKey = "er" ServiceWorkload ServiceKey = "wl" ServiceCluster ServiceKey = "cl" @@ -104,7 +105,7 @@ const ( // serviceKeyOrder is the deterministic emit order for service ports in TXT. var serviceKeyOrder = []ServiceKey{ - ServiceNodeInfo, ServiceOllama, ServiceLMStudio, + ServiceNodeInfo, ServiceOllama, ServiceLMStudio, ServiceLlamaCpp, ServiceErrors, ServiceWorkload, ServiceCluster, ServiceEngineManager, ServiceEngineControl, } @@ -577,6 +578,17 @@ func (n DirectoryNode) EngineModels(engine string) []string { return n.Models } +// EngineLoadedModels returns models currently resident in memory for one +// engine. It never falls back to Models or ModelsByEngine: a missing +// LoadedByEngine report means nothing is loaded, so a router cannot treat +// catalog ids as eligible. +func (n DirectoryNode) EngineLoadedModels(engine string) []string { + if n.LoadedByEngine == nil { + return nil + } + return n.LoadedByEngine[engine] +} + // SubscribeParams filters a subscription to nodes advertising any of the listed // services; an empty list subscribes to all nodes. type SubscribeParams struct { diff --git a/services/shared/noderec/noderec_test.go b/services/shared/noderec/noderec_test.go index 8b0757e5..a02aedf8 100644 --- a/services/shared/noderec/noderec_test.go +++ b/services/shared/noderec/noderec_test.go @@ -40,6 +40,48 @@ func TestEngineModels(t *testing.T) { } } +func TestEngineLoadedModels(t *testing.T) { + n := DirectoryNode{ + Models: []string{"catalog-a", "catalog-b"}, + ModelsByEngine: map[string][]string{ + "llamacpp": {"catalog-a", "catalog-b"}, + }, + LoadedByEngine: map[string][]string{ + "llamacpp": {"catalog-a"}, + }, + } + got := n.EngineLoadedModels("llamacpp") + if !reflect.DeepEqual(got, []string{"catalog-a"}) { + t.Fatalf("EngineLoadedModels(llamacpp) = %v, want [catalog-a]", got) + } + if got := n.EngineLoadedModels("ollama"); len(got) != 0 { + t.Fatalf("EngineLoadedModels(missing) = %v, want empty", got) + } + legacy := DirectoryNode{Models: []string{"catalog-a"}} + if got := legacy.EngineLoadedModels("llamacpp"); len(got) != 0 { + t.Fatalf("nil LoadedByEngine must not fall back to catalog, got %v", got) + } +} + +func TestServiceLlamaCppKey(t *testing.T) { + if ServiceLlamaCpp != "lc" { + t.Fatalf("ServiceLlamaCpp = %q, want lc", ServiceLlamaCpp) + } + found := false + for _, k := range serviceKeyOrder { + if k == ServiceLlamaCpp { + found = true + break + } + } + if !found { + t.Fatal("ServiceLlamaCpp missing from serviceKeyOrder") + } + if ServiceLlamaCpp.Transport() != TransportPlain { + t.Fatal("lc transport must be TransportPlain (same as ol/lm)") + } +} + func TestParseTXT(t *testing.T) { txt := []string{ "v=1", "uuid=host-abc", "cluster-uuid=clu-xyz", "ip=192.168.1.10", diff --git a/services/shared/splitlisten/splitlisten_test.go b/services/shared/splitlisten/splitlisten_test.go index 52e0412a..758a9be4 100644 --- a/services/shared/splitlisten/splitlisten_test.go +++ b/services/shared/splitlisten/splitlisten_test.go @@ -237,11 +237,11 @@ func testServerTLSConfig(t *testing.T) *tls.Config { // scriptedListener is a net.Listener whose Accept behavior is driven by acceptFn. type scriptedListener struct { - addr net.Addr - acceptFn func(call int) (net.Conn, error) - mu sync.Mutex - calls int - closed chan struct{} + addr net.Addr + acceptFn func(call int) (net.Conn, error) + mu sync.Mutex + calls int + closed chan struct{} closeOnce sync.Once } diff --git a/services/tests/llama_headless_test.go b/services/tests/llama_headless_test.go new file mode 100644 index 00000000..4e9a48c1 --- /dev/null +++ b/services/tests/llama_headless_test.go @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestFreshLlamaManagerRefusesExternalListenerMutation(t *testing.T) { + var reads, mutations atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + reads.Add(1) + } else { + mutations.Add(1) + } + w.Header().Set("Server", "llama.cpp") + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":[]}`) + })) + defer server.Close() + home := t.TempDir() + base := home + if runtime.GOOS == "darwin" { + base = filepath.Join(home, "Library", "Application Support") + } + app := filepath.Join(base, "Nvidia Corporation", "Personal AI Router") + name := "llama" + if runtime.GOOS == "windows" { + name += ".exe" + } + bin := filepath.Join(app, "engine-bin", "llamacpp", "runtime", name) + if err := os.MkdirAll(filepath.Dir(bin), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(bin, []byte("managed image fixture"), 0600); err != nil { + t.Fatal(err) + } + config := filepath.Join(app, "engines", "llamacpp.json") + if err := os.MkdirAll(filepath.Dir(config), 0700); err != nil { + t.Fatal(err) + } + data, _ := json.Marshal(map[string]any{"engine": "llamacpp", "runtime": map[string]int{"port": portOfURL(t, server.URL)}}) + if err := os.WriteFile(config, data, 0600); err != nil { + t.Fatal(err) + } + stdin, msgs, stop := startEngineManagerStdio(t, home) + t.Cleanup(stop) + waitForMethod(t, msgs, "engine:ready", 10*time.Second) + writeRawFrame(t, stdin, `{"jsonrpc":"2.0","id":1,"method":"engine:action","params":{"engine":"llamacpp","action":"load_model","params":{"model":"not-owned"}}}`) + response := waitForResponse(t, msgs, 10*time.Second) + if response.Error == nil || !strings.Contains(response.Error.Message, "external") { + t.Fatalf("fresh manager failed to refuse the external owner: %+v", response) + } + if reads.Load() == 0 || mutations.Load() != 0 { + t.Fatalf("identity reads=%d mutation requests=%d; want observed identity and no mutation", reads.Load(), mutations.Load()) + } +} diff --git a/services/tests/main_test.go b/services/tests/main_test.go index a30b1198..eff3e1d6 100644 --- a/services/tests/main_test.go +++ b/services/tests/main_test.go @@ -187,7 +187,6 @@ func waitForMethod(t *testing.T, ch <-chan jsonrpc.Message, method string, timeo t.Fatalf("timed out (%s) waiting for method %q", timeout, method) } } - return jsonrpc.Message{} } func waitForResponse(t *testing.T, ch <-chan jsonrpc.Message, timeout time.Duration) jsonrpc.Message { @@ -207,5 +206,4 @@ func waitForResponse(t *testing.T, ch <-chan jsonrpc.Message, timeout time.Durat t.Fatal("timed out waiting for JSON-RPC response") } } - return jsonrpc.Message{} } diff --git a/services/tests/remote_engine_test.go b/services/tests/remote_engine_test.go index 32351496..2ebac9d7 100644 --- a/services/tests/remote_engine_test.go +++ b/services/tests/remote_engine_test.go @@ -25,6 +25,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "testing" "time" @@ -192,12 +193,17 @@ func TestRemoteEngineRejectsUntrusted(t *testing.T) { // controlPort with the given cluster dir. It keeps stdin open (so the process // stays alive) and drains stdout. Returns stdin and a cleanup. func startEngineManagerServer(t *testing.T, clusterDir string, controlPort int) (io.WriteCloser, func()) { + return startEngineManagerServerBinary(t, engineMgrBin, clusterDir, controlPort) +} + +func startEngineManagerServerBinary(t *testing.T, binary, clusterDir string, controlPort int) (io.WriteCloser, func()) { t.Helper() - cmd := exec.Command(engineMgrBin, + cmd := exec.Command(binary, "--control-port", fmt.Sprintf("%d", controlPort), "--cluster-dir", clusterDir, "--log-level", "warn", ) + cmd.Env = append(os.Environ(), "HOME="+clusterDir, "XDG_CONFIG_HOME="+clusterDir, "APPDATA="+clusterDir, "LOCALAPPDATA="+clusterDir) cmd.Stderr = os.Stderr stdin, err := cmd.StdinPipe() if err != nil { @@ -227,8 +233,13 @@ func startEngineManagerServer(t *testing.T, clusterDir string, controlPort int) // startEngineManagerStdio launches an engine-manager as a JSON-RPC-over-stdio // client. Returns its stdin, a reader over its stdout frames, and a cleanup. func startEngineManagerStdio(t *testing.T, clusterDir string) (io.WriteCloser, <-chan jsonrpc.Message, func()) { + return startEngineManagerStdioBinary(t, engineMgrBin, clusterDir) +} + +func startEngineManagerStdioBinary(t *testing.T, binary, clusterDir string) (io.WriteCloser, <-chan jsonrpc.Message, func()) { t.Helper() - cmd := exec.Command(engineMgrBin, "--cluster-dir", clusterDir, "--log-level", "warn") + cmd := exec.Command(binary, "--cluster-dir", clusterDir, "--log-level", "warn") + cmd.Env = append(os.Environ(), "HOME="+clusterDir, "XDG_CONFIG_HOME="+clusterDir, "APPDATA="+clusterDir, "LOCALAPPDATA="+clusterDir) cmd.Stderr = os.Stderr stdin, err := cmd.StdinPipe() if err != nil { @@ -254,3 +265,50 @@ func startEngineManagerStdio(t *testing.T, clusterDir string) (io.WriteCloser, < } } } + +func TestRemoteLlamaControlsRefusedByOldPeer(t *testing.T) { + old := os.Getenv("NVPAIR_TEST_OLD_ENGINE_MANAGER") + if old == "" { + t.Skip("set NVPAIR_TEST_OLD_ENGINE_MANAGER to a pre-llama engine-manager binary") + } + dirA, dirB := t.TempDir(), t.TempDir() + const uuidA, uuidB = "new-manager", "old-manager" + certA, certB := mintClusterIdentity(t, dirA, uuidA), mintClusterIdentity(t, dirB, uuidB) + writePin(t, dirA, uuidB, certB) + writePin(t, dirB, uuidA, certA) + port := freePort(t) + _, stopB := startEngineManagerServerBinary(t, old, dirB, port) + t.Cleanup(stopB) + waitForPort(t, "127.0.0.1", port, 10*time.Second) + stdin, msgs, stopA := startEngineManagerStdio(t, dirA) + t.Cleanup(stopA) + waitForMethod(t, msgs, "engine:ready", 10*time.Second) + writeRawFrame(t, stdin, fmt.Sprintf(`{"jsonrpc":"2.0","method":"discovery:nodes","params":{"nodes":[{"hostUuid":"old","name":"old","ip":"127.0.0.1","clusterUuid":%q,"trusted":true,"services":{"ec":{"port":%d}}}]}}`, uuidB, port)) + writeRawFrame(t, stdin, `{"jsonrpc":"2.0","id":1,"method":"engine:remote-get-installed","params":{"node":"old"}}`) + listed := waitForResponse(t, msgs, 10*time.Second) + if listed.Error != nil || strings.Contains(string(listed.Result), `"llamacpp"`) { + t.Fatalf("old peer must explicitly omit unsupported llama: %s %+v", listed.Result, listed.Error) + } + for i, method := range []string{"engine:remote-load-model", "engine:remote-cancel-pull"} { + writeRawFrame(t, stdin, fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":%q,"params":{"node":"old","engine":"llamacpp","model":"unsupported-model"}}`, i+2, method)) + resp := waitForResponse(t, msgs, 10*time.Second) + if resp.Error == nil || (!strings.Contains(resp.Error.Message, "404") && !strings.Contains(resp.Error.Message, "unknown engine")) { + t.Fatalf("%s must return the old peer's explicit capability refusal: %+v", method, resp) + } + } + // Reverse direction: the old client preserves the new inventory record + // verbatim; it must not relabel the unknown engine as a familiar one. + newPort := freePort(t) + _, stopNew := startEngineManagerServer(t, dirA, newPort) + t.Cleanup(stopNew) + waitForPort(t, "127.0.0.1", newPort, 10*time.Second) + oldIn, oldMsgs, stopOld := startEngineManagerStdioBinary(t, old, dirB) + t.Cleanup(stopOld) + waitForMethod(t, oldMsgs, "engine:ready", 10*time.Second) + writeRawFrame(t, oldIn, fmt.Sprintf(`{"jsonrpc":"2.0","method":"discovery:nodes","params":{"nodes":[{"hostUuid":"new","name":"new","ip":"127.0.0.1","clusterUuid":%q,"trusted":true,"services":{"ec":{"port":%d}}}]}}`, uuidA, newPort)) + writeRawFrame(t, oldIn, `{"jsonrpc":"2.0","id":1,"method":"engine:remote-get-installed","params":{"node":"new"}}`) + reverse := waitForResponse(t, oldMsgs, 10*time.Second) + if reverse.Error != nil || !strings.Contains(string(reverse.Result), `"engine":"llamacpp"`) { + t.Fatalf("old client lost the new engine identity: %s %+v", reverse.Result, reverse.Error) + } +} diff --git a/services/tests/scheduler_interop_test.go b/services/tests/scheduler_interop_test.go index d494fce9..daa9c7f3 100644 --- a/services/tests/scheduler_interop_test.go +++ b/services/tests/scheduler_interop_test.go @@ -28,6 +28,7 @@ import ( "testing" "time" + "nvpair-shared/engines" "nvpair-shared/jsonrpc" "nvpair-shared/schedulerwire" ) @@ -85,14 +86,15 @@ func waitForPriorityPair(t *testing.T, ch <-chan jsonrpc.Message, timeout time.D if json.Unmarshal(msg.Params, &p) != nil { continue } - if p.Engine == "ollama" || p.Engine == "lmstudio" { + if _, ok := engines.ByName(p.Engine); ok { got[p.Engine] = p } - if len(got) == 2 { + if len(got) == len(engines.Names()) { return got } case <-to: - t.Fatalf("timed out (%s) waiting for both schedule:priority outputs; got %v", timeout, got) + t.Fatalf("timed out (%s) waiting for a schedule:priority output per engine (%v); got %v", + timeout, engines.Names(), got) } } } @@ -100,15 +102,16 @@ func waitForPriorityPair(t *testing.T, ch <-chan jsonrpc.Message, timeout time.D func waitForSchedulePair(t *testing.T, ch <-chan jsonrpc.Message, timeout time.Duration) map[string][]string { t.Helper() priorities := waitForPriorityPair(t, ch, timeout) - return map[string][]string{ - "ollama": priorities["ollama"].Nodes, - "lmstudio": priorities["lmstudio"].Nodes, + out := make(map[string][]string, len(priorities)) + for _, engine := range engines.Names() { + out[engine] = priorities[engine].Nodes } + return out } func assertSchedulePair(t *testing.T, got map[string][]string, want []string) { t.Helper() - for _, engine := range []string{"ollama", "lmstudio"} { + for _, engine := range engines.Names() { assertScheduleOrder(t, engine, got[engine], want) } } @@ -216,7 +219,7 @@ func assertPriorityPair( wantPressure map[string]int, ) { t.Helper() - for _, engine := range []string{"ollama", "lmstudio"} { + for _, engine := range engines.Names() { priority := got[engine] assertScheduleOrder(t, engine, priority.Nodes, wantOrder) if len(priority.Ranks) != len(wantPressure) { @@ -251,10 +254,10 @@ func TestSchedulerSyntheticMixedEngineBurst(t *testing.T) { current := initial for i := 0; i < 50; i++ { target := current[0] - engine := "ollama" - if i%2 == 1 { - engine = "lmstudio" - } + // Rotate through every engine: the ranking is node-global, so load + // attributed to one engine must move the others' order too. + names := engines.Names() + engine := names[i%len(names)] depths[target]++ want := rankSyntheticDepths(depths) writeRawFrame(t, stdin, fmt.Sprintf( From 85fb8b4dfca854cbc9b673875a2b1c89223c193c Mon Sep 17 00:00:00 2001 From: pgoode41 Date: Tue, 22 Sep 2026 03:44:37 -0400 Subject: [PATCH 02/13] Walk ancestors in the Unix owned-path guard validateLlamaPath on Unix resolved the whole path with EvalSymlinks and returned nil as soon as the leaf did not exist, so a symlinked ancestor of a not-yet-created runtime slot, model file or archive entry was never inspected, and a dangling link at the leaf passed. The Windows guard already walks every component up to the volume root and refuses any reparse point regardless of whether the leaf exists. Make the Unix guard do the same with Lstat per component: skip missing components, refuse a symlink anywhere in the chain, return other errors. Existing real paths keep the strictness they had. The fixture covers an ordinary tree, a plain missing leaf, a models slot redirected outside the managed root, missing children beneath it, and a dangling leaf, and checks the outside marker is untouched. It carries the !windows build tag; on this Windows host it was compile-checked with GOOS=linux and GOOS=darwin vet and test builds, not executed. Co-Authored-By: Claude Fable 5.1 Signed-off-by: pgoode41 --- .../llamapath_unix_test.go | 81 +++++++++++++++++++ services/nvpair-engine-manager/proc_unix.go | 23 ++++++ 2 files changed, 104 insertions(+) create mode 100644 services/nvpair-engine-manager/llamapath_unix_test.go diff --git a/services/nvpair-engine-manager/llamapath_unix_test.go b/services/nvpair-engine-manager/llamapath_unix_test.go new file mode 100644 index 00000000..866e86c0 --- /dev/null +++ b/services/nvpair-engine-manager/llamapath_unix_test.go @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build !windows + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// The Unix guard must match the Windows reparse-point walk: a symbolic link is +// refused whether it sits at the leaf, at an ancestor of a leaf that does not +// exist yet, or dangles. Ordinary directories and plain missing leaves pass. +func TestLlamaUnixPathGuardWalksAncestors(t *testing.T) { + // Resolve the fixture roots first: on macOS t.TempDir() lives under /var, + // itself a symlink to /private/var, which must not count as a redirect the + // fixture introduced. + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + outside, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + marker := filepath.Join(outside, "keep.txt") + if err := os.WriteFile(marker, []byte("outside"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "runtime"), 0o700); err != nil { + t.Fatal(err) + } + + // (a) An ordinary managed tree is accepted, including its missing slots. + if err := validateLlamaOwnedPaths(root); err != nil { + t.Fatalf("ordinary tree rejected: %v", err) + } + // (e) A plain missing leaf beneath a real ancestor is accepted. + if err := validateLlamaPath(filepath.Join(root, "runtime", "missing.bin")); err != nil { + t.Fatalf("plain missing leaf rejected: %v", err) + } + + // (b) A models slot redirected to an existing outside directory is refused. + models := filepath.Join(root, "models") + if err := os.Symlink(outside, models); err != nil { + t.Skipf("cannot create symlink fixture: %v", err) + } + if err := validateLlamaOwnedPaths(root); err == nil { + t.Fatal("symlinked models slot accepted") + } + // (d) A child that does not exist yet must not hide the symlinked ancestor. + if err := validateLlamaPath(filepath.Join(models, "new.gguf")); err == nil { + t.Fatal("missing leaf hid ancestor symlink") + } + if err := validateLlamaPath(filepath.Join(models, "nested", "deeper", "new.gguf")); err == nil { + t.Fatal("deep missing leaf hid ancestor symlink") + } + if err := validateLlamaOwnedPaths(filepath.Join(models, "missing-child")); err == nil { + t.Fatal("missing root beneath a symlink accepted") + } + + // (c) A dangling link at the leaf is still a redirect. + previous := filepath.Join(root, "previous") + if err := os.Symlink(filepath.Join(outside, "gone"), previous); err != nil { + t.Fatal(err) + } + if err := validateLlamaPath(previous); err == nil { + t.Fatal("dangling symlink leaf accepted") + } + if err := validateLlamaPath(filepath.Join(previous, "llama")); err == nil { + t.Fatal("missing leaf under dangling symlink accepted") + } + + // The guard only inspects; nothing outside the managed tree may change. + if data, err := os.ReadFile(marker); err != nil || string(data) != "outside" { + t.Fatalf("external marker changed: %q %v", data, err) + } +} diff --git a/services/nvpair-engine-manager/proc_unix.go b/services/nvpair-engine-manager/proc_unix.go index 876e2b69..25e3f3ce 100644 --- a/services/nvpair-engine-manager/proc_unix.go +++ b/services/nvpair-engine-manager/proc_unix.go @@ -8,8 +8,10 @@ package main import ( "context" "errors" + "fmt" "os" "os/exec" + "path/filepath" "strconv" "strings" "syscall" @@ -21,6 +23,27 @@ import ( // lookup that hung would wedge StopAll (and the whole app shutdown). const portLookupTimeout = 2 * time.Second +// validateLlamaPath refuses a managed path when the leaf or any existing +// ancestor is a symbolic link. Missing components are skipped and the walk +// continues upward, so a symlinked ancestor cannot hide behind a child that +// does not exist yet (a fresh runtime slot, a model about to be written) and a +// dangling link at the leaf is still seen. This mirrors the Windows +// reparse-point walk in proc_windows.go. +func validateLlamaPath(path string) error { + for current := filepath.Clean(path); ; current = filepath.Dir(current) { + info, err := os.Lstat(current) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil && info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("llama managed directory contains a redirected path; external data is left untouched") + } + if parent := filepath.Dir(current); parent == current { + return nil + } + } +} + // configureSysProcAttr puts the child in its own process group so a // terminate signals the whole group — engines that fork helper // processes (model runners, etc.) get cleaned up too. From ffc7d5df41d200d8362931fa4a02a5b855950ff2 Mon Sep 17 00:00:00 2001 From: pgoode41 Date: Tue, 22 Sep 2026 03:44:37 -0400 Subject: [PATCH 03/13] Add llama.cpp to the desktop Register the engine type, display name and links; map the llamacpp-proxy relay source; carry the engine manager's install-support, install-reason and managed facts through the bridge so an external llama.cpp runtime is shown read-only and the install decision follows the manager rather than the displayed OS; keep cached models distinct from loaded ones; and route model import, load, delete and download cancellation through the owning engine manager, locally and on peers. The settings editor develop introduced is reused for llama.cpp and hidden for an external runtime PAIR does not own. The bridge refuses the update command for llama.cpp instead of letting it fall into the generic uninstall-then-install pair; remote update is refused for every engine as before. Workload identity carries the engine and proxy run so two engines' equal request ids stay distinct. The service contract inventory is regenerated for the new methods. Co-authored-by: Terve Co-Authored-By: Claude Fable 5.1 Signed-off-by: pgoode41 --- desktop/docs/architecture.md | 20 +- desktop/docs/service-contract-exceptions.json | 1 + desktop/docs/services-api.md | 13 + desktop/docs/services-backend.md | 18 +- desktop/docs/services-parity.md | 36 ++- desktop/electron-builder.config.ts | 15 ++ desktop/scripts/build/installer.nsh | 20 +- desktop/scripts/build/linux/after-remove.sh | 9 +- desktop/scripts/build/macos/uninstall.sh | 9 + desktop/scripts/generate-licenses.ts | 5 +- desktop/src/electron/model-hub/index.ts | 2 +- .../electron/service-bridge/empty-handlers.ts | 88 ++++++- .../electron/service-bridge/modular-state.ts | 153 +++++++++--- .../service-bridge/modular-supervisor.ts | 75 ++++-- desktop/src/shared/constants/engines.ts | 26 +- desktop/src/shared/types/engine-api.ts | 2 + desktop/src/shared/types/engines.ts | 3 + desktop/src/shared/types/inference-demo.ts | 5 +- .../src/shared/types/inference-dispatcher.ts | 2 +- desktop/src/shared/types/workloads.ts | 8 + desktop/src/shared/types/ws-channels.ts | 8 +- desktop/src/shared/utils/engine-progress.ts | 7 + desktop/src/shared/utils/workloads.ts | 22 +- desktop/src/ui/api/engine-api.ts | 6 + desktop/src/ui/api/pair-api.ts | 14 +- .../components/BackendRow/BackendFooter.tsx | 5 +- .../components/BackendRow/BackendHeader.tsx | 19 +- .../ui/components/BackendRow/BackendRow.tsx | 22 +- .../components/BackendRow/InstallButton.tsx | 13 +- desktop/src/ui/components/EngineIcon.tsx | 52 ++-- .../components/ModelManager/ModelManager.tsx | 131 ++++++++++- .../NodeList/NodeEngineSettings.tsx | 5 +- .../components/Welcome/WelcomeEngineRow.tsx | 17 +- .../ui/components/Welcome/WelcomeModal.tsx | 2 + .../components/Workloads/WorkloadItemCard.tsx | 26 +- .../components/Workloads/WorkloadListView.tsx | 9 +- .../Workloads/WorkloadNodeConnections.tsx | 27 ++- .../src/ui/constants/engine-capabilities.ts | 13 + desktop/src/ui/constants/welcome.ts | 3 +- desktop/src/ui/stores/workloads.store.ts | 36 ++- desktop/src/ui/types/engine-info.ts | 3 + .../src/ui/utils/format-model-display-name.ts | 1 + desktop/src/ui/utils/formatters.ts | 7 +- .../src/ui/utils/gateway-inference-paths.ts | 10 +- desktop/tests/modular/app-data-wipe.test.ts | 90 +++++++ .../modular/deb-runtime-dependencies.test.ts | 27 +++ .../tests/modular/engine-command-load.test.ts | 119 +++++++++- .../modular/inference-demo-lifecycle.test.ts | 24 +- desktop/tests/modular/llamacpp-engine.test.ts | 34 +++ desktop/tests/modular/llamacpp-state.test.ts | 222 ++++++++++++++++++ .../modular/lmstudio-stale-model.test.ts | 45 ++++ .../tests/modular/uuid-node-keying.test.ts | 4 +- 52 files changed, 1345 insertions(+), 188 deletions(-) create mode 100644 desktop/tests/modular/deb-runtime-dependencies.test.ts create mode 100644 desktop/tests/modular/llamacpp-engine.test.ts create mode 100644 desktop/tests/modular/llamacpp-state.test.ts diff --git a/desktop/docs/architecture.md b/desktop/docs/architecture.md index 0874b5ff..2370592f 100644 --- a/desktop/docs/architecture.md +++ b/desktop/docs/architecture.md @@ -109,7 +109,7 @@ subscribes to broker relays after `app:ready`, and converts backend responses into stable UI contracts. Electron reports the service connected after broker `app:ready`. The -broker-owned Ollama and LM Studio proxies remain asynchronous capabilities; a +broker-owned Ollama, LM Studio, and llama.cpp proxies remain asynchronous capabilities; a late or failed proxy does not misreport the broker startup as failed. If `app:ready` does not arrive within the startup deadline, Overview opens Settings @@ -227,16 +227,18 @@ ordinary environment assignments can be edited locally or by a pinned peer. authoritative settings operation rather than forwarding to the engine manager, so both entry points validate, restart, and persist identically. -The Ollama and LM Studio proxies are cluster-aware. For model-bearing inference, -each proxy first keeps only nodes whose per-engine discovery inventory advertises -the requested model. Empty and non-matching inventories are excluded; an empty -owner set returns a local `502`. Routing precedence within the eligible set is: +The Ollama, LM Studio, and llama.cpp proxies are cluster-aware. For +model-bearing inference, each proxy first keeps only nodes whose per-engine +discovery inventory advertises the requested model. llama.cpp uses the **loaded** +set, not the on-disk catalog. Empty and non-matching inventories are excluded; an +empty owner set returns a local `502`. Routing precedence within the eligible set +is: 1. a user-selected manual node; 2. the priority list emitted by `nvpair-job-scheduler`; 3. the proxy's deterministic default ordering. -The scheduler combines total pending (queued and running) workload across both +The scheduler combines total pending (queued and running) workload across all engines with a smoothed 0–3 pressure derived from the busiest GPU. Missing, invalid, or older-than-10-second telemetry has neutral pressure. It emits the order, pending count, and pressure, reranking on meaningful workload, discovery, @@ -298,6 +300,12 @@ cannot yet be reported are centralized in `src/shared/constants/modular-runtime.ts`. - Ollama-compatible clients use the proxy port reported by the broker. +- llama.cpp clients use the OpenAI-compatible proxy at `http://127.0.0.1:8080/v1` + by default. Engine Manager installs the official `llama` app, serves its + managed router on the separately reported engine port (default `8082`), and + owns model download, load/unload and removal. The proxy routes only to models + observed as loaded. Existing external listeners remain externally owned; + their presence does not authorize PAIR to mutate them. - Cluster pairing currently uses port `14321`. - Node telemetry is read from `/v1/node-info` at each discovered node's advertised port. diff --git a/desktop/docs/service-contract-exceptions.json b/desktop/docs/service-contract-exceptions.json index 90ae4eec..523b9040 100644 --- a/desktop/docs/service-contract-exceptions.json +++ b/desktop/docs/service-contract-exceptions.json @@ -9,6 +9,7 @@ "engine:restore-enabled": "Broker-internal startup restoration. nvpair-ui-broker emits engine:restore-enabled directly to its supervised engine-manager after the managed Ollama port gate and on manager respawn; it is not a renderer/UI notification.", "ollama-proxy:ready": "Consumed, not missing: the broker relays it and normalizeBrokerProxy (modular-supervisor.ts) strips the `ollama-proxy:` prefix, so the bridge handles the de-prefixed `ready` (sets proxyPort). The literal `ollama-proxy:ready` is intentionally absent from our TS — extractor limitation, not a gap.", "lmstudio-proxy:ready": "Consumed, not missing: the LM Studio counterpart of ollama-proxy:ready, de-prefixed by the same normalizeBrokerProxy loop. It only became visible to the checker when METHOD_RE started accepting hyphens in a namespace segment; before that the whole lmstudio-proxy:* surface was silently unmatched.", + "llamacpp-proxy:ready": "Consumed, not missing: the llama.cpp counterpart of the two above, de-prefixed by the same normalizeBrokerProxy loop, which iterates PROXY_NODE_SOURCES and so covers every engine without naming one. llama.cpp ships no proxy binary — one nvpair-proxy process hosts a facade per engine — but `llamacpp-proxy` remains the facade's relay namespace, which is what the broker emits and this entry names.", "node/selection-changed": "Automatic routing has no selected-node UI, so PAIR deliberately does not consume proxy selection changes.", "proxy/request": "Per-request proxy telemetry is not rendered; workload lifecycle uses the broker workloads stream.", "proxy/request-started": "Per-request proxy telemetry is not rendered; see proxy/request.", diff --git a/desktop/docs/services-api.md b/desktop/docs/services-api.md index 7aea40f9..5ddb43c4 100644 --- a/desktop/docs/services-api.md +++ b/desktop/docs/services-api.md @@ -41,12 +41,16 @@ - ⚠️ nvpair-proxy → facade/enable - ⚠️ nvpair-proxy → node/selected - ⚠️ nvpair-proxy → node/set-local-backend +- ⚠️ nvpair-proxy → workload/cancel - ⚠️ nvpair-ui-broker → discovery:unsubscribe - ⚠️ nvpair-ui-broker → engine:configure-launch - ⚠️ nvpair-ui-broker → engine:set-port - ⚠️ nvpair-ui-broker → engine:set-reserved-port - ⚠️ nvpair-ui-broker → engine:unsubscribe - ⚠️ nvpair-ui-broker → internal:set-reserved-port +- ⚠️ nvpair-ui-broker → llamacpp-proxy:get-status +- ⚠️ nvpair-ui-broker → llamacpp-proxy:set-port +- ⚠️ nvpair-ui-broker → llamacpp-proxy:unsubscribe - ⚠️ nvpair-ui-broker → lmstudio-proxy:get-status - ⚠️ nvpair-ui-broker → lmstudio-proxy:set-port - ⚠️ nvpair-ui-broker → lmstudio-proxy:unsubscribe @@ -109,6 +113,7 @@ | `engine:prepare-shutdown` | request (we call) | ✅ yes | | `engine:preview-launch` | request (we call) | ⚠️ not called | | `engine:remote-apply-settings` | request (we call) | ⚠️ not called | +| `engine:remote-cancel-pull` | request (we call) | ✅ yes | | `engine:remote-delete-model` | request (we call) | ✅ yes | | `engine:remote-get-installed` | request (we call) | ✅ yes | | `engine:remote-get-settings` | request (we call) | ⚠️ not called | @@ -222,6 +227,7 @@ | `node/set-local-backend` | request (we call) | ⚠️ not called | | `node/set-priority` | request (we call) | ✅ yes | | `nodes/list` | request (we call) | ✅ yes | +| `workload/cancel` | request (we call) | ⚠️ not called | **Dynamic / unresolved notify sites (verify by hand — `npm run service-contracts` prints the line numbers):** - `method (var) (proxy.go)` @@ -233,6 +239,7 @@ | `cluster:identity-changed` | request (we call) | ✅ yes | | `cluster:invite-received` | request (we call) | ✅ yes | | `engine:install-progress` | request (we call) | ✅ yes | +| `engine:models-changed` | request (we call) | ✅ yes | | `engine:pull-progress` | request (we call) | ✅ yes | | `engine:state-changed` | request (we call) | ✅ yes | | `error` | request (we call) | ✅ yes | @@ -253,6 +260,7 @@ | `errors:clear` | notification (we consume) | ✅ yes | | `errors:report` | notification (we consume) | ✅ yes | | `errors:update` | notification (we consume) | ✅ yes | +| `llamacpp-proxy:ready` | notification (we consume) | ➖ ignored | | `lmstudio-proxy:ready` | notification (we consume) | ➖ ignored | | `ollama-proxy:ready` | notification (we consume) | ➖ ignored | | `workloads:upsert` | notification (we consume) | ✅ yes | @@ -273,6 +281,10 @@ | `engine:unsubscribe` | request (we call) | ⚠️ not called | | `errors:get-initial` | request (we call) | ✅ yes | | `internal:set-reserved-port` | request (we call) | ⚠️ not called | +| `llamacpp-proxy:get-status` | request (we call) | ⚠️ not called | +| `llamacpp-proxy:set-port` | request (we call) | ⚠️ not called | +| `llamacpp-proxy:subscribe` | request (we call) | ✅ yes | +| `llamacpp-proxy:unsubscribe` | request (we call) | ⚠️ not called | | `lmstudio-proxy:get-status` | request (we call) | ⚠️ not called | | `lmstudio-proxy:set-port` | request (we call) | ⚠️ not called | | `lmstudio-proxy:subscribe` | request (we call) | ✅ yes | @@ -288,6 +300,7 @@ | `ollama-proxy:subscribe` | request (we call) | ✅ yes | | `ollama-proxy:unsubscribe` | request (we call) | ⚠️ not called | | `ready` | request (we call) | ✅ yes | +| `workloads:cancel` | request (we call) | ✅ yes | | `workloads:get-initial` | request (we call) | ✅ yes | | `workloads:remove` | request (we call) | ✅ yes | | `workloads:subscribe` | request (we call) | ✅ yes | diff --git a/desktop/docs/services-backend.md b/desktop/docs/services-backend.md index 25d8328c..5933880f 100644 --- a/desktop/docs/services-backend.md +++ b/desktop/docs/services-backend.md @@ -89,8 +89,8 @@ engine, workload, cluster, and error relays. The bridge then emits renderer push events from backend notifications. Connector readiness follows the broker contract: `app:ready` establishes the -service connection, while Ollama and LM Studio proxy readiness remains an -asynchronous capability signal. Personal AI Router waits up to the canonical +service connection, while Ollama, LM Studio, and llama.cpp proxy readiness +remains an asynchronous capability signal. Personal AI Router waits up to the canonical startup deadline in `src/shared/constants/modular-runtime.ts` for `app:ready`; an outright failure or stalled broker startup is surfaced in Settings > Service with retry and log access. If a stalled broker reports ready @@ -116,7 +116,7 @@ reserved for inference clients. | ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------- | | `app:ready` | Complete broker startup and refresh snapshots | `state:request-refresh` | | `discovery:nodes-changed` | Replace discovery snapshot and diff nodes | `discovery:nodes-changed`, `nodes:upsert`, `nodes:remove` | -| `ollama-proxy:ready` / `lmstudio-proxy:ready` | Record engine proxy port | `engines:state-changed` | +| `ollama-proxy:ready` / `lmstudio-proxy:ready` / `llamacpp-proxy:ready` | Record engine proxy port | `engines:state-changed` | | proxy `node/*` | Update per-engine node presence; the advertised port is the peer's promoted proxy port (not the engine's private loopback port) | node and engine pushes | | `engine:ready` / `engine:state-changed` | Update engine facts and models | `engines:state-changed` | | `engine:settings-changed` | Validate and republish the owning node's settings snapshot | `engines:settings-changed` | @@ -128,7 +128,7 @@ reserved for inference clients. | `nodes:changed` | Replace membership snapshot | `nodes:changed` | | `workloads:upsert` / `workloads:remove` | Update workload catalog | workload pushes | -`nvpair-job-scheduler` combines queued and running work across both engines with +`nvpair-job-scheduler` combines queued and running work across all engines with a smoothed 0–3 pressure from the busiest GPU. Invalid, missing, or older-than-10-second telemetry receives neutral pressure. It emits `schedule:priority` with order, pending count, and pressure; the broker applies @@ -150,8 +150,14 @@ waiting for authoritative state. Pending state clears on matching engine state, progress, or error pushes. Local engine operations include install, start, stop, uninstall, update, port -changes, and model actions. Remote cluster operations use the engine manager's -remote control surface where supported. +changes, and model actions. Engine Manager owns the official llama app install, +router process, managed cache, downloads, load/unload and delete. llama.cpp has +no managed update: the bridge refuses `update` for it rather than substituting +an uninstall and reinstall. The desktop uses reported install support and +ownership; external runtimes remain read-only. Downloaded cache entries remain +distinct from runtime residency. The app endpoint is `http://127.0.0.1:8080/v1`; +routing requires the model **loaded** on the serving node. Remote cluster +operations use the engine manager's remote control surface where supported. ### Engine settings diff --git a/desktop/docs/services-parity.md b/desktop/docs/services-parity.md index c0b8cbc5..9870945c 100644 --- a/desktop/docs/services-parity.md +++ b/desktop/docs/services-parity.md @@ -24,6 +24,7 @@ history. | Manual nodes | Complete with local persistence | Broker owns probing and proxy registration; Electron persists entries for replay | | Ollama routing | Complete | Broker relay and backend scheduler drive proxy routing | | LM Studio routing | Complete | Parallel broker relay and scheduler path | +| llama.cpp routing | Source integration; native validation pending | `nvpair-proxy` facade on llama.cpp's own `8080`, relayed as `llamacpp-proxy:`; eligibility is loaded models only | | Local engine lifecycle | Complete | Install, start, stop, uninstall, update, and port configuration | | Remote engine lifecycle | Partial | Remote install, start, stop, status, and model pull are supported | | Engine models | Partial | Core list, pull, load, unload, and supported delete actions are wired | @@ -97,23 +98,44 @@ Manual nodes use the broker's `node/add`, `node/remove`, and `nodes/list` surface. Electron persists user entries and replays them after broker startup so they survive worker restarts. +### Multi-node UI acceptance + +Engine integration must preserve each participating desktop's view of the +cluster, not only the request origin's view. During the same bounded inference +run, verify every available participating desktop independently: + +- Cluster and member UUIDs, engine availability and loaded-model ownership agree + after discovery converges; offline members do not look live or routable. +- New workloads agree by their full `(originatedFrom, engine, runId, id)` + identity on model, destination and terminal state. Record propagation delay; + do not require identical historical catalog totals or instantaneous equality + during a state transition. +- Capture each desktop's original Performance view during actual work. A remote + backend response or the origin's aggregate UI does not prove another native UI. + +Record unavailable or untested desktop cells explicitly. This checklist states +the acceptance requirement; it does not assert that every platform has passed. + ## Routing and inference -Both text-engine facades are broker-owned and cluster-aware. They live in one +Every text-engine facade is broker-owned and cluster-aware. They live in one `nvpair-proxy` process, each enabled after spawn on its own port, and each serves its engine's dialect: - the Ollama facade serves the Ollama-compatible surface; -- the LM Studio facade serves the LM Studio/OpenAI-compatible surface. +- the LM Studio facade serves the LM Studio/OpenAI-compatible surface; +- the llama.cpp facade serves its OpenAI-compatible surface on `8080`, and is + the one facade whose routing eligibility is loaded models rather than the + on-disk catalog. Sharing a process is what lets them share the burst reservations the scheduler -depends on: two facades bursting at once compete for the same node's GPU, so a -dispatch through either has to be visible to the other. +depends on: facades bursting at once compete for the same node's GPU, so a +dispatch through any of them has to be visible to the others. Routing precedence is manual selection, scheduler priority, then deterministic proxy ordering. Personal AI Router leaves proxies in automatic mode. -`nvpair-job-scheduler` combines total queued and running workload across both +`nvpair-job-scheduler` combines total queued and running workload across all engines with a smoothed 0–3 GPU-pressure signal. The backend scanner and manual node worker provide maximum-GPU utilization, while invalid, missing, or older-than-10-second samples receive neutral pressure. The scheduler emits order, @@ -159,6 +181,10 @@ Personal AI Router supports local: - desired-state restoration across app restarts; - engine and model progress. +Managed update covers Ollama and LM Studio. llama.cpp has no managed update; +the bridge refuses `update` for it instead of substituting an uninstall and +reinstall. + Before shutdown, Personal AI Router calls `engine:prepare-shutdown`. This stops managed engine processes without changing the persisted desired state; the broker restores enabled engines on the next launch. The broker also self-initiates diff --git a/desktop/electron-builder.config.ts b/desktop/electron-builder.config.ts index 1f713d47..957ea06e 100644 --- a/desktop/electron-builder.config.ts +++ b/desktop/electron-builder.config.ts @@ -389,6 +389,21 @@ const config: Configuration = { } }, deb: { + // Explicit depends replace electron-builder defaults. Preserve its current + // runtime set, adding GBM and the ALSA SONAME on both pre/post-t64 distros. + depends: [ + 'libgtk-3-0', + 'libnotify4', + 'libnss3', + 'libxss1', + 'libxtst6', + 'xdg-utils', + 'libatspi2.0-0', + 'libuuid1', + 'libsecret-1-0', + 'libgbm1', + 'libasound2t64 | libasound2' + ], afterInstall: 'scripts/build/linux/after-install.sh', afterRemove: 'scripts/build/linux/after-remove.sh' }, diff --git a/desktop/scripts/build/installer.nsh b/desktop/scripts/build/installer.nsh index 1517393e..1914da07 100644 --- a/desktop/scripts/build/installer.nsh +++ b/desktop/scripts/build/installer.nsh @@ -71,8 +71,20 @@ DetailPrint "Removing Personal AI Router user data..." ClearErrors ReadEnvStr $0 LOCALAPPDATA - RMDir /r "$0\Nvidia Corporation\Personal AI Router" - RMDir /r "$0\NVIDIA Corporation\PAIR" + ; Engine Manager migrates old caches outside app data. An uninstall before + ; first launch must preserve unmigrated weights, not recursively delete them. + IfFileExists "$0\Nvidia Corporation\Personal AI Router\engine-bin\llamacpp\models\*.*" pairKeepCurrentModels + RMDir /r "$0\Nvidia Corporation\Personal AI Router" + Goto pairCurrentDataDone + pairKeepCurrentModels: + DetailPrint "Preserving app data containing unmigrated llama models. Reinstall and open PAIR to migrate the library." + pairCurrentDataDone: + IfFileExists "$0\NVIDIA Corporation\PAIR\engine-bin\llamacpp\models\*.*" pairKeepLegacyModels + RMDir /r "$0\NVIDIA Corporation\PAIR" + Goto pairLegacyDataDone + pairKeepLegacyModels: + DetailPrint "Preserving legacy app data containing unmigrated llama models." + pairLegacyDataDone: RMDir "$0\NVIDIA Corporation" RMDir /r "$0\nvpair-updater" ClearErrors @@ -108,7 +120,7 @@ IfFileExists "$0\nvpair-updater\*.*" 0 +2 StrCpy $9 "$9$\n$0\nvpair-updater" StrCmp $9 "" pairNoLeftover - MessageBox MB_OK|MB_ICONEXCLAMATION "Some Personal AI Router data could not be removed because files were still in use (for example, a running engine such as Ollama).$\n$\nClose those programs, then delete these folders manually:$9" + MessageBox MB_OK|MB_ICONEXCLAMATION "Some Personal AI Router data was retained because it contains unmigrated llama models or files still in use.$\n$\nDo not delete model folders manually. Reinstall and open the updated app to migrate the library before removing app data.$\n$9" pairNoLeftover: !macroend @@ -152,6 +164,7 @@ ; leaves rules pointing at binaries this version no longer ships. nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router Ollama Proxy"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router LM Studio Proxy"' + nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router llama.cpp Proxy"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router Node Info"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router Node Scanner"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router Workload Manager"' @@ -161,6 +174,7 @@ nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router mDNS (UDP 5353)"' ; Pre-unification; see above. nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router mDNS LM Studio Proxy (UDP 5353)"' + nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router mDNS llama.cpp Proxy (UDP 5353)"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router mDNS Node Info (UDP 5353)"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router mDNS Node Scanner (UDP 5353)"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router mDNS Workload Manager (UDP 5353)"' diff --git a/desktop/scripts/build/linux/after-remove.sh b/desktop/scripts/build/linux/after-remove.sh index f506a05d..d06449ae 100644 --- a/desktop/scripts/build/linux/after-remove.sh +++ b/desktop/scripts/build/linux/after-remove.sh @@ -69,8 +69,13 @@ case "${1:-}" in # AI Router" (backend base = $XDG_CONFIG_HOME or ~/.config). The living, # append-only inventory is scripts/wipe-app-data.sh — do not silently diverge. # Every delete is best-effort so a locked or missing path never aborts removal. - rm -rf "$user_home/.config/Nvidia Corporation/Personal AI Router" 2>/dev/null || true - rm -rf "$user_home/.config/NVIDIA Corporation/PAIR" 2>/dev/null || true + for data_root in "$user_home/.config/Nvidia Corporation/Personal AI Router" "$user_home/.config/NVIDIA Corporation/PAIR"; do + if [ -e "$data_root/engine-bin/llamacpp/models" ] || [ -L "$data_root/engine-bin/llamacpp/models" ]; then + echo "Preserving unmigrated llama models in $data_root; reinstall and open PAIR before purging app data." >&2 + else + rm -rf "$data_root" 2>/dev/null || true + fi + done # Remove the current and previous parents only when empty so other NVIDIA # applications survive. rmdir "$user_home/.config/Nvidia Corporation" 2>/dev/null || true diff --git a/desktop/scripts/build/macos/uninstall.sh b/desktop/scripts/build/macos/uninstall.sh index ec948cc8..1bc6148e 100644 --- a/desktop/scripts/build/macos/uninstall.sh +++ b/desktop/scripts/build/macos/uninstall.sh @@ -49,6 +49,15 @@ fi APP_SUPPORT="$target_home/Library/Application Support" +if [ "$PURGE_DATA" = "1" ]; then + for root in "$APP_SUPPORT/Nvidia Corporation/Personal AI Router" "$APP_SUPPORT/NVIDIA Corporation/PAIR"; do + if [ -e "$root/engine-bin/llamacpp/models" ] || [ -L "$root/engine-bin/llamacpp/models" ]; then + echo "Llama models remain under app data. Open the updated app to migrate them before purging: $root" >&2 + exit 1 + fi + done +fi + echo "Stopping Personal AI Router processes..." # Keep this list in sync with MODULAR_RUNTIME_BINARIES and # MODULAR_BUNDLED_BINARIES in src/shared/constants/modular-binaries.ts, plus the diff --git a/desktop/scripts/generate-licenses.ts b/desktop/scripts/generate-licenses.ts index eb1ab9c1..1b61ef69 100644 --- a/desktop/scripts/generate-licenses.ts +++ b/desktop/scripts/generate-licenses.ts @@ -167,7 +167,10 @@ function renderMarkdown(entries: Entry[]): string { parts.push('```text', e.licenseText, '```', '') } - return `${parts.join('\n').trimEnd()}\n` + return `${parts + .join('\n') + .replace(/[\t ]+$/gm, '') + .trimEnd()}\n` } async function serviceComponentNames(): Promise { diff --git a/desktop/src/electron/model-hub/index.ts b/desktop/src/electron/model-hub/index.ts index d1fda727..570923bf 100644 --- a/desktop/src/electron/model-hub/index.ts +++ b/desktop/src/electron/model-hub/index.ts @@ -52,7 +52,7 @@ export async function getEngineHubModels(engineType: EngineType): Promise): ) break case 'update': + if (engine === 'llamacpp') { + // llama.cpp has no managed update, and the generic + // uninstall-then-install pair below must not stand in for one. + // Refuse without touching the runtime or beginning a pending op; + // the engineType context lets the renderer drop its optimistic + // lifecycle entry. + supervisor.reportError( + 'llama.cpp has no managed update; uninstall and reinstall the managed runtime instead.', + 'warning', + `engine-cmd:${payload.command}:${engine}`, + { engineType: payload.engineType, operation: 'update' } + ) + break + } // The engine-manager serializes per-engine ops via its lifecycle // lock, so the queued install waits for the uninstall to finish. The // uninstall's engine:state-changed briefly clears this, then the @@ -255,6 +269,29 @@ function routeEngineManagerCommand(payload: WsInvokeRequest<'engine:command'>): void supervisor.deleteModel(engine, payload.engineType, payload.model) } break + case 'cancelPull': + if (payload.model) { + supervisor.sendProcess( + 'broker', + 'engine:action', + { + engine, + action: 'cancel_pull', + params: { model: payload.model } + }, + failAction('cancel model download'), + // Observe the response like every other long-running engine + // action here: without it a backend refusal is indistinguishable + // from a cancel that worked. + true + ) + } + break + case 'importModel': + if (payload.model && engine === 'llamacpp') { + void supervisor.importLlamaModel(payload.model) + } + break case 'loadModel': // "Load" warms a model into the engine's memory/VRAM. Ollama has no // first-class load action, so we POST its `run_model` HTTP action @@ -282,6 +319,23 @@ function routeEngineManagerCommand(payload: WsInvokeRequest<'engine:command'>): }), true ) + } else if (payload.engineType === 'llamacpp') { + supervisor.sendProcess( + 'broker', + 'engine:action', + { + engine, + action: 'load_model', + params: { model: payload.model } + }, + failAction('load model', { + nodeId: payload.nodeId, + engineType: payload.engineType, + operation: 'load', + modelName: payload.model + }), + true + ) } else { supervisor.sendProcess( 'broker', @@ -371,6 +425,21 @@ function routeRemoteEngineCommand(payload: WsInvokeRequest<'engine:command'>): v `${payload.command} is only available on the local node — remote uninstall/update is not supported yet.` ) break + case 'cancelPull': + if (payload.model && engine === 'llamacpp') { + supervisor.sendProcess( + 'broker', + 'engine:remote-cancel-pull', + { + node: nodeId, + engine, + model: payload.model + }, + detail => refuseRemote(`Cancel download failed: ${detail}`), + true + ) + } + break case 'deleteModel': case 'loadModel': case 'unloadModel': @@ -718,7 +787,24 @@ const EMPTY_SERVICE_BRIDGE_HANDLERS: BridgeHandlerMap = { 'errors:get-initial': () => handleErrorsGetInitial(), 'errors:clear': payload => (payload ? handleErrorsClear(payload) : null), - 'workloads:get-initial': () => handleWorkloadsGetInitial() + 'workloads:get-initial': () => handleWorkloadsGetInitial(), + 'workloads:cancel': async payload => { + const supervisor = getModularSupervisor() + if (!payload) return { accepted: false } + try { + const result = objectValue( + await supervisor.callProcess('broker', 'workloads:cancel', payload) + ) + return { accepted: booleanValue(result?.accepted) } + } catch (err) { + supervisor.reportError( + `Cancel request failed: ${getErrorString(err)}`, + 'error', + 'llamacpp-cancel' + ) + return { accepted: false } + } + } } export function handleServiceBridgeInvoke( diff --git a/desktop/src/electron/service-bridge/modular-state.ts b/desktop/src/electron/service-bridge/modular-state.ts index 151f1745..b3e240d0 100644 --- a/desktop/src/electron/service-bridge/modular-state.ts +++ b/desktop/src/electron/service-bridge/modular-state.ts @@ -35,10 +35,10 @@ import { emitBridgePush } from './broadcaster' import { mergePullProgressPercent } from './pull-error-handling' import type { JsonObject, JsonRpcNotification, JsonValue } from './json-rpc-subprocess' import { serviceLogLevel } from './service-log-level' -// Live node sources are the two reverse proxies, relayed through the broker, +// Live node sources are the reverse proxies, relayed through the broker, // and the broker's consolidated discovery snapshot. Electron does not consume // worker discovery protocols directly. -type ProxyNodeSource = 'ollama-proxy' | 'lmstudio-proxy' +type ProxyNodeSource = 'ollama-proxy' | 'lmstudio-proxy' | 'llamacpp-proxy' type BrokerNodeSource = ProxyNodeSource | 'broker' /** @@ -53,13 +53,20 @@ export const PROXY_NODE_SOURCES: readonly ProxyNodeSource[] = ['ollama-proxy', ' * Engines surfaced by the broker's proxy plane. Other engine-manager engines * are not currently routed across nodes. */ -export type ProxyEngine = Extract -export const PROXY_ENGINES: readonly ProxyEngine[] = ['ollama', 'lm-studio'] +export type ProxyEngine = Extract +export const PROXY_ENGINES: readonly ProxyEngine[] = ['ollama', 'lm-studio', 'llamacpp'] /** Map a proxy node source onto the engine it describes. */ const PROXY_SOURCE_ENGINE: Record = { 'ollama-proxy': 'ollama', - 'lmstudio-proxy': 'lm-studio' + 'lmstudio-proxy': 'lm-studio', + 'llamacpp-proxy': 'llamacpp' +} + +function proxySourceForEngine(engine: EngineType): ProxyNodeSource { + if (engine === 'ollama') return 'ollama-proxy' + if (engine === 'lm-studio') return 'lmstudio-proxy' + return 'llamacpp-proxy' } /** Per-engine presence on a node — each proxy reports its own engine. */ @@ -68,7 +75,7 @@ interface EnginePresence { /** * The node's promoted inference **proxy** port for this engine, as * advertised in discovery. Under secure inference the broker registers the - * `ol`/`lm` service at the proxy's port — never the engine's own port, which + * `ol`/`lm`/`lc` service at the proxy's port — never the engine's own port, which * is loopback-private and reachable by peers only through that proxy's * cluster-mTLS ingress. The engine's real server port is not in discovery; * it comes from `engine:remote-get-installed` facts (a peer) or @@ -92,6 +99,9 @@ interface RemoteEngineFacts { running: boolean healthy: boolean port: number + installSupported?: boolean + installReason?: string + managed?: boolean } interface ModularNode { @@ -177,19 +187,16 @@ function emptyPresence(): EnginePresence { } function emptyEngines(): Record { - return { ollama: emptyPresence(), 'lm-studio': emptyPresence() } + return { ollama: emptyPresence(), 'lm-studio': emptyPresence(), llamacpp: emptyPresence() } } -/** Immutably set one engine's presence, preserving the other. */ +/** Immutably set one engine's presence, preserving the others. */ function setEngine( engines: Record, engine: ProxyEngine, presence: EnginePresence ): Record { - return { - ollama: engine === 'ollama' ? presence : engines.ollama, - 'lm-studio': engine === 'lm-studio' ? presence : engines['lm-studio'] - } + return { ...engines, [engine]: presence } } /** @@ -369,6 +376,7 @@ function parseWorkload(value: JsonValue | undefined): Workload | null { const workload: Workload = { id, + runId: stringValue(obj.runId) || undefined, model: stringValue(obj.model), engine, state: stateValue, @@ -403,7 +411,7 @@ export function parseWorkloadsInitial(value: JsonValue | undefined): Workload[] /** True for an engine fronted by a broker-supervised reverse proxy. */ export function isProxyEngine(engine: EngineType): engine is ProxyEngine { - return engine === 'ollama' || engine === 'lm-studio' + return engine === 'ollama' || engine === 'lm-studio' || engine === 'llamacpp' } const PENDING_OP_IDLE_TIMEOUT_MS = 90_000 @@ -582,11 +590,11 @@ function sameTelemetry( ) } -function modelItem(name: string, loaded = false): ModelItem { +function modelItem(name: string, loaded = false, downloaded = true): ModelItem { return { name, size: 0, - downloaded: true, + downloaded, status: loaded ? 'loaded' : 'idle', parameterSize: '', quantization: '', @@ -721,7 +729,7 @@ function parseProxyNode(params: JsonValue | undefined, engine: ProxyEngine): Mod } return { id, - sources: [engine === 'ollama' ? 'ollama-proxy' : 'lmstudio-proxy'], + sources: [proxySourceForEngine(engine)], // `Node.Host` is the hostname; empty for the self-bridge manual node, // in which case the broker discovery entry supplies the display name on // merge (see mergeNode). Never fall back to the UUID id here. @@ -892,8 +900,9 @@ class ModularBridgeState { private logs: LogEntry[] = [] // Per-engine bound proxy port reported by the broker. 0 = not reported yet; // we never fabricate a default — an unknown port surfaces as null, not a - // guess. `ollama` is the `ollama-proxy`, `lm-studio` is the `lmstudio-proxy`. - private proxyPorts: Record = { ollama: 0, 'lm-studio': 0 } + // guess. `ollama` is the `ollama-proxy`, `lm-studio` is the `lmstudio-proxy`, + // `llamacpp` is the `llamacpp-proxy`. + private proxyPorts: Record = { ollama: 0, 'lm-studio': 0, llamacpp: 0 } private selfId: string | null = null /** * Authoritative local-engine facts from `nvpair-engine-manager`, keyed by @@ -903,7 +912,14 @@ class ModularBridgeState { */ private engineManagerFacts = new Map< EngineType, - { installed: boolean; running: boolean; port: number } + { + installed: boolean + running: boolean + port: number + installSupported?: boolean + installReason?: string + managed?: boolean + } >() /** * Local model lists pulled from `nvpair-engine-manager`'s `list_models` action by @@ -1287,7 +1303,12 @@ class ModularBridgeState { */ seedWorkloads(workloads: Workload[]): WsInvokeResponse<'workloads:get-initial'> { for (const workload of workloads) { - const key = workloadKey(workload.originatedFrom, workload.id) + const key = workloadKey( + workload.originatedFrom, + workload.id, + workload.engine, + workload.runId + ) if (!this.workloads.has(key)) this.workloads.set(key, workload) } return this.getWorkloads() @@ -1298,7 +1319,10 @@ class ModularBridgeState { const obj = objectValue(params) const workload = parseWorkload(obj?.workloadInfo) if (!workload) return - this.workloads.set(workloadKey(workload.originatedFrom, workload.id), workload) + this.workloads.set( + workloadKey(workload.originatedFrom, workload.id, workload.engine, workload.runId), + workload + ) emitBridgePush('workloads:upsert', workload) } @@ -1308,8 +1332,18 @@ class ModularBridgeState { const workloadId = stringValue(obj?.workloadId) if (!workloadId) return const originatedFrom = nullableStringValue(obj?.originatedFrom) - this.workloads.delete(workloadKey(originatedFrom, workloadId)) - emitBridgePush('workloads:remove', { workloadId, originatedFrom }) + const engine = engineTypeFromManagerName(stringValue(obj?.engine)) ?? undefined + const runId = stringValue(obj?.runId) || undefined + for (const [key, workload] of this.workloads) { + if ( + workload.originatedFrom === originatedFrom && + workload.id === workloadId && + (!engine || workload.engine === engine) && + (!runId || workload.runId === runId) + ) + this.workloads.delete(key) + } + emitBridgePush('workloads:remove', { workloadId, originatedFrom, engine, runId }) } /** Push a `workloads:remove` for an entry and drop it from the catalog. */ @@ -1317,7 +1351,9 @@ class ModularBridgeState { this.workloads.delete(key) emitBridgePush('workloads:remove', { workloadId: workload.id, - originatedFrom: workload.originatedFrom + originatedFrom: workload.originatedFrom, + engine: workload.engine, + runId: workload.runId }) } @@ -1368,7 +1404,11 @@ class ModularBridgeState { this.engineManagerFacts.set(engineType, { installed: booleanValue(obj.installed), running: booleanValue(obj.running), - port: numberValue(obj.port) + port: numberValue(obj.port), + installSupported: + typeof obj.install_supported === 'boolean' ? obj.install_supported : undefined, + installReason: stringValue(obj.install_reason), + managed: typeof obj.managed === 'boolean' ? obj.managed : undefined }) // A fresh authoritative state is the resolution of whatever op was in // flight (start/stop done, install `done`+installed, uninstall removed). @@ -1705,7 +1745,7 @@ class ModularBridgeState { * the renderer renders the peer engine as unavailable rather than as an * installed-but-off toggle. * - * The peer's promoted **proxy** port IS carried in discovery (the `ol`/`lm` + * The peer's promoted **proxy** port IS carried in discovery (the `ol`/`lm`/`lc` * advertisement points at the proxy), so it is surfaced as `proxyPort` from * that per-engine presence regardless of facts. * The peer's engine port stays private (loopback) and comes only from facts; @@ -1725,6 +1765,9 @@ class ModularBridgeState { base = { engineType: engine, nodeId, + installSupported: facts.installSupported, + installReason: facts.installReason, + managed: facts.managed, processStatus: facts.running ? 'running' : facts.installed @@ -1782,7 +1825,13 @@ class ModularBridgeState { installed: booleanValue(engineObj.installed), running: booleanValue(engineObj.running), healthy: booleanValue(engineObj.healthy), - port: numberValue(engineObj.port) + port: numberValue(engineObj.port), + installSupported: + typeof engineObj.install_supported === 'boolean' + ? engineObj.install_supported + : undefined, + installReason: stringValue(engineObj.install_reason), + managed: typeof engineObj.managed === 'boolean' ? engineObj.managed : undefined }) seen.add(engineType) } @@ -1811,7 +1860,11 @@ class ModularBridgeState { installed: booleanValue(obj.installed), running: booleanValue(obj.running), healthy: booleanValue(obj.healthy), - port: numberValue(obj.port) + port: numberValue(obj.port), + installSupported: + typeof obj.install_supported === 'boolean' ? obj.install_supported : undefined, + installReason: stringValue(obj.install_reason), + managed: typeof obj.managed === 'boolean' ? obj.managed : undefined }) this.emitRemoteEngineStatus(nodeId, engineType) } @@ -1854,14 +1907,17 @@ class ModularBridgeState { * Called by the supervisor after a `list_models` pull (or with an empty list * when the engine is stopped, since its HTTP `list_models` is unreachable). */ - setLocalEngineModels(engineType: EngineType, modelNames: string[]): void { + setLocalEngineModels(engineType: EngineType, modelNames: string[], downloaded = true): void { const nodeId = this.selfId // Stamp `'loaded'` from the self node's loaded set so a `list_models` // refresh (pull/lifecycle) preserves residency instead of resetting every // row to idle. The loaded set is seeded by discovery self-enrichment and // kept fresh by {@link applyLocalLoadedModels}. - const loaded = loadedNamesForEngine(nodeId ? this.nodes.get(nodeId) : undefined, engineType) - const items = modelNames.map(name => modelItem(name, loaded.has(name))) + const loaded = + this.engineManagerFacts.get(engineType)?.running === false + ? new Set() + : loadedNamesForEngine(nodeId ? this.nodes.get(nodeId) : undefined, engineType) + const items = modelNames.map(name => modelItem(name, loaded.has(name), downloaded)) this.localManagerModels.set(engineType, items) if (!nodeId) return emitBridgePush('engines:state-changed', { @@ -1989,11 +2045,21 @@ class ModularBridgeState { private toEngineModels(node: ModularNode, engine: ProxyEngine): EngineModels { const loaded = loadedNamesForEngine(node, engine) + // Authenticated managed-peer facts and its attributed inventory prove + // downloaded weights; an external router catalogue alone does not. + const managedPeerInventory = + node.id !== this.selfId && + this.remoteEngineFacts.get(this.remoteOpKey(node.id, engine))?.managed === true && + Object.hasOwn(node.modelsByEngine, engineManagerName(engine)) return { engineType: engine, nodeId: node.id, models: this.modelsForEngine(node, engine).map(name => - modelItem(name, loaded.has(name)) + modelItem( + name, + loaded.has(name), + engine !== 'llamacpp' || managedPeerInventory || loaded.has(name) + ) ) } } @@ -2024,7 +2090,10 @@ class ModularBridgeState { // The cached list is the authoritative `list_models` set (names // only); stamp `'loaded'` from the self node's discovery/push // loaded set so the local card reflects in-memory residency too. - const loaded = loadedNamesForEngine(node, engine) + const loaded = + this.engineManagerFacts.get(engine)?.running === false + ? new Set() + : loadedNamesForEngine(node, engine) return { engineType: engine, nodeId, @@ -2066,6 +2135,9 @@ class ModularBridgeState { return { engineType, nodeId, + installSupported: facts?.installSupported, + installReason: facts?.installReason, + managed: facts?.managed, processStatus: pending, enginePort: facts && facts.running && facts.port > 0 ? facts.port : null, proxyPort: isProxyEngine(engineType) ? this.getProxyPort(engineType) : null @@ -2076,6 +2148,9 @@ class ModularBridgeState { return { engineType, nodeId, + installSupported: facts.installSupported, + installReason: facts.installReason, + managed: facts.managed, processStatus: facts.running ? 'running' : facts.installed @@ -2282,6 +2357,10 @@ class ModularBridgeState { this.handleProxyNotification(notification, 'lm-studio') return } + if (notification.source === 'llamacpp-proxy') { + this.handleProxyNotification(notification, 'llamacpp') + return + } if (notification.source === 'broker') { this.handleBrokerNotification(notification) } @@ -2327,7 +2406,7 @@ class ModularBridgeState { if (notification.method === 'node/discovered' || notification.method === 'node/updated') { const node = parseProxyNode(notification.params, engine) if (!node) return - this.upsertNode(node, engine === 'ollama' ? 'ollama-proxy' : 'lmstudio-proxy') + this.upsertNode(node, proxySourceForEngine(engine)) } } @@ -2339,7 +2418,7 @@ class ModularBridgeState { private clearNodeEngine(nodeId: string, engine: ProxyEngine): void { const existing = this.nodes.get(nodeId) if (!existing) return - const source: BrokerNodeSource = engine === 'ollama' ? 'ollama-proxy' : 'lmstudio-proxy' + const source: BrokerNodeSource = proxySourceForEngine(engine) const sources = removeSource(existing.sources, source) if (sources.length === 0 && !existing.nodeInfoUp) { this.removeNodeEntry(nodeId) @@ -2532,7 +2611,7 @@ class ModularBridgeState { // install/running state; discovery only fills in models. A remote node // has no local engine-manager, so its status comes from authoritative // peer facts or its advertisement, and is omitted when neither is known. - // Push per proxy-engine (Ollama + LM Studio) so both light up per node. + // Push per proxy-engine (Ollama, LM Studio, llama.cpp) so each lights up per node. const isSelf = merged.id === this.selfId for (const engine of PROXY_ENGINES) { if (!isSelf) { @@ -2587,7 +2666,7 @@ class ModularBridgeState { } } - // A proxy source (ollama-proxy / lmstudio-proxy): refresh only that + // A proxy source (ollama-proxy / lmstudio-proxy / llamacpp-proxy): refresh only that // engine's presence; keep the other engine, telemetry, and node-info. const engine = PROXY_SOURCE_ENGINE[source] return { diff --git a/desktop/src/electron/service-bridge/modular-supervisor.ts b/desktop/src/electron/service-bridge/modular-supervisor.ts index 8cd1012b..4a0f226e 100644 --- a/desktop/src/electron/service-bridge/modular-supervisor.ts +++ b/desktop/src/electron/service-bridge/modular-supervisor.ts @@ -163,13 +163,14 @@ export function parseListModelNames(result: JsonValue | undefined): string[] { const obj = objectValue(result) if (!obj) throw new Error('list_models returned a non-object response') const names: string[] = [] - if (Array.isArray(obj.models)) { - for (const entry of obj.models) { + const rows = Array.isArray(obj.models) ? obj.models : Array.isArray(obj.data) ? obj.data : null + if (rows) { + for (const entry of rows) { const row = objectValue(entry) - const name = stringValue(row?.name) || stringValue(row?.key) + const name = stringValue(row?.name) || stringValue(row?.key) || stringValue(row?.id) if (name) names.push(name) } - if (obj.models.length > 0 && names.length === 0) { + if (rows.length > 0 && names.length === 0) { throw new Error('list_models returned no usable model names') } return names @@ -300,7 +301,16 @@ function proxyEngineFromManagerId(id: string): ProxyEngine | null { /** The broker relay namespace fronting an engine's reverse proxy. */ function proxyRelayPrefix(engine: ProxyEngine): string { - return engine === 'ollama' ? 'ollama-proxy' : 'lmstudio-proxy' + if (engine === 'ollama') return 'ollama-proxy' + if (engine === 'lm-studio') return 'lmstudio-proxy' + return 'llamacpp-proxy' +} + +function proxyEngineFromRelaySource(source: string): ProxyEngine | null { + if (source === 'ollama-proxy') return 'ollama' + if (source === 'lmstudio-proxy') return 'lm-studio' + if (source === 'llamacpp-proxy') return 'llamacpp' + return null } /** @@ -310,12 +320,13 @@ function proxyRelayPrefix(engine: ProxyEngine): string { * `docs/services-backend.md`): * * - The `nvpair-ui-broker` is the **only** Electron-spawned binary and is itself the - * parent of every broker-owned worker (`ollama-proxy`, `lmstudio-proxy`, + * parent of every broker-owned worker (`nvpair-proxy`, which hosts a facade + * per engine rather than shipping one binary each, * `nvpair-node-scanner`, `nvpair-node-info`, `nvpair-workload-manager`, * `nvpair-cluster-manager`, `nvpair-node-settings`, `nvpair-manual-nodes`, * `nvpair-engine-manager`, `nvpair-errors`, `nvpair-job-scheduler`). Electron passes their resolved paths to * the broker (see `brokerStartupArgs`) and reaches each through a broker relay: - * `ollama-proxy:` / `lmstudio-proxy:` for the two engine proxies, `engine:` for the + * `ollama-proxy:` / `lmstudio-proxy:` / `llamacpp-proxy:` for the engine facades, `engine:` for the * engine-manager, `errors:` for the error pipeline, `node/*` for manual nodes, * `settings/*` and `cluster:` for the rest. Local inference jobs arrive on the * broker's `workloads:subscribe` stream. @@ -338,6 +349,7 @@ class ModularSupervisor { // last-known-good result (including an authoritative empty list). private stoppedModelSentinels = new Set() private stoppedModelEngines = new Set() + private managedLlama = false private successfulModelInventories = new Set() private modelRefreshGenerations = new Map() private discoveryModelRetryTimers = new Map>() @@ -875,6 +887,7 @@ class ModularSupervisor { await subscribe('discovery:subscribe', 'subscribe to broker discovery') await subscribe('ollama-proxy:subscribe', 'subscribe to broker ollama-proxy relay') await subscribe('lmstudio-proxy:subscribe', 'subscribe to broker lmstudio-proxy relay') + await subscribe('llamacpp-proxy:subscribe', 'subscribe to broker llamacpp-proxy relay') // Engine events are opt-in and replay no baseline — subscribe then hydrate. await subscribe('engine:subscribe', 'subscribe to broker engine relay') await subscribe('workloads:subscribe', 'subscribe to broker workloads stream') @@ -1076,7 +1089,7 @@ class ModularSupervisor { const obj = objectValue(result) if (obj && booleanValue(obj.ready)) { getModularBridgeState().handleNotification({ - source: engine === 'ollama' ? 'ollama-proxy' : 'lmstudio-proxy', + source: proxyRelayPrefix(engine), method: 'ready', params: { port: numberValue(obj.port) } }) @@ -1097,7 +1110,7 @@ class ModularSupervisor { if (!obj || !Array.isArray(obj.nodes)) return for (const node of obj.nodes) { getModularBridgeState().handleNotification({ - source: engine === 'ollama' ? 'ollama-proxy' : 'lmstudio-proxy', + source: proxyRelayPrefix(engine), method: 'node/discovered', params: node }) @@ -1263,12 +1276,7 @@ class ModularSupervisor { this.scheduleRemoteEngineStatusRefresh() } - const proxyEngine: ProxyEngine | null = - event.source === 'ollama-proxy' - ? 'ollama' - : event.source === 'lmstudio-proxy' - ? 'lm-studio' - : null + const proxyEngine = proxyEngineFromRelaySource(event.source) if (proxyEngine && event.method === 'ready') { // A (re)bound proxy starts with an empty manual-node set, so forget // what we think we bridged and re-push the local node if applicable. @@ -1906,6 +1914,24 @@ class ModularSupervisor { } } + async importLlamaModel(path: string): Promise { + try { + await this.callProcess( + 'broker', + 'engine:action', + { engine: 'llamacpp', action: 'import_model', params: { path } }, + PULL_TIMEOUT_MS + ) + await this.refreshEngineModels('llamacpp', 'llamacpp') + } catch (err) { + this.reportError( + `Model import failed: ${getErrorString(err)}`, + 'error', + 'llamacpp-import' + ) + } + } + /** * Load, unload (eject), or delete a model on a remote peer via the ec surface. * @@ -1965,7 +1991,10 @@ class ModularSupervisor { try { const result = await this.callProcess('broker', 'engine:action', { engine, - action: 'list_models' + action: + engineType === 'llamacpp' && this.managedLlama + ? 'list_downloaded' + : 'list_models' }) const models = parseListModelNames(result) this.commitModelInventory(engineType, models, generation) @@ -1994,8 +2023,9 @@ class ModularSupervisor { const engine = stringValue(obj.engine) const engineType = getModularBridgeState().modelPullTarget(engine) if (!engineType) return + if (engineType === 'llamacpp') this.managedLlama = booleanValue(obj.managed) - if (!booleanValue(obj.running)) { + if (!booleanValue(obj.running) && !(engineType === 'llamacpp' && this.managedLlama)) { this.beginModelRefresh(engineType) this.stoppedModelEngines.add(engineType) this.stoppedModelSentinels.add(engineType) @@ -2016,7 +2046,8 @@ class ModularSupervisor { const generation = this.beginModelRefresh(engineType) this.callProcess('broker', 'engine:action', { engine, - action: 'list_models' + action: + engineType === 'llamacpp' && this.managedLlama ? 'list_downloaded' : 'list_models' }) .then(result => { const models = parseListModelNames(result) @@ -2050,7 +2081,7 @@ class ModularSupervisor { const generation = this.beginModelRefresh(engine) this.callProcess('broker', 'engine:action', { engine: engineManagerName(engine), - action: 'list_models' + action: engine === 'llamacpp' && this.managedLlama ? 'list_downloaded' : 'list_models' }) .then(result => { const models = parseListModelNames(result) @@ -2077,7 +2108,11 @@ class ModularSupervisor { if (isProxyEngine(engine)) { this.cancelDiscoveryModelRefreshRetry(engine) } - getModularBridgeState().setLocalEngineModels(engine, models) + getModularBridgeState().setLocalEngineModels( + engine, + models, + engine !== 'llamacpp' || this.managedLlama + ) } private scheduleDiscoveryModelRefreshRetry(engine: ProxyEngine): void { diff --git a/desktop/src/shared/constants/engines.ts b/desktop/src/shared/constants/engines.ts index 5904ae61..d924c17e 100644 --- a/desktop/src/shared/constants/engines.ts +++ b/desktop/src/shared/constants/engines.ts @@ -4,16 +4,16 @@ import { EngineType, ModelExpiry } from '@/shared/types/engines' // The engines `nvpair-engine-manager` ships a manifest for, and therefore the -// only ones PAIR can install, run or route to. llama-cpp, whisper-cpp, -// piper-tts, sherpa-onnx-tts and stable-diffusion-cpp were carried here as -// never-enabled placeholders; they were removed with the chat window, which was -// their only in-app consumer. Adding an engine back means shipping its manifest -// first -- an engine row without one renders commands that fail with `-32000`. -export const EngineTypes = ['ollama', 'lm-studio'] as const +// only ones PAIR can install, run or route to. whisper-cpp, piper-tts, +// sherpa-onnx-tts and stable-diffusion-cpp were carried here as never-enabled +// placeholders; they were removed with the chat window, which was their only +// in-app consumer. Adding an engine back means shipping its manifest first -- +// an engine row without one renders commands that fail with `-32000`. +export const EngineTypes = ['ollama', 'lm-studio', 'llamacpp'] as const // Kept as a distinct export so a future engine can ship behind it rather than // appearing the moment its type exists. -export const EnabledEngineTypes: EngineType[] = ['ollama', 'lm-studio'] as const +export const EnabledEngineTypes: EngineType[] = ['ollama', 'lm-studio', 'llamacpp'] as const /** * How `nvpair-engine-manager` spells each engine on the wire. Only LM Studio @@ -23,20 +23,26 @@ export const EnabledEngineTypes: EngineType[] = ['ollama', 'lm-studio'] as const */ export const EngineManagerNames = { ollama: 'ollama', - 'lm-studio': 'lmstudio' + 'lm-studio': 'lmstudio', + llamacpp: 'llamacpp' } as const satisfies Record export const EngineSources = ['bundled', 'detected', 'installed'] as const export const EngineDisplayNames: Record = { ollama: 'Ollama', - 'lm-studio': 'LM Studio' + 'lm-studio': 'LM Studio', + llamacpp: 'llama.cpp' } as const /** Default docs/install URLs for built-in backends. Single source of truth for UI and adapter buildInfo(). */ export const EngineDefaultLinks: Record = { ollama: { docsUrl: 'https://docs.ollama.com/', installUrl: 'https://ollama.com/download' }, - 'lm-studio': { docsUrl: 'https://lmstudio.ai/docs', installUrl: 'https://lmstudio.ai/' } + 'lm-studio': { docsUrl: 'https://lmstudio.ai/docs', installUrl: 'https://lmstudio.ai/' }, + llamacpp: { + docsUrl: 'https://github.com/ggml-org/llama.cpp', + installUrl: 'https://github.com/ggml-org/llama.cpp' + } } as const export const ModelItemStatuses = ['idle', 'loading', 'loaded', 'ejecting', 'pulling'] as const diff --git a/desktop/src/shared/types/engine-api.ts b/desktop/src/shared/types/engine-api.ts index 9187e658..58467cff 100644 --- a/desktop/src/shared/types/engine-api.ts +++ b/desktop/src/shared/types/engine-api.ts @@ -45,6 +45,8 @@ export type EngineCommandType = | 'uninstall' | 'update' | 'pullModel' + | 'cancelPull' + | 'importModel' | 'loadModel' | 'unloadModel' | 'deleteModel' diff --git a/desktop/src/shared/types/engines.ts b/desktop/src/shared/types/engines.ts index 863c415c..de934f61 100644 --- a/desktop/src/shared/types/engines.ts +++ b/desktop/src/shared/types/engines.ts @@ -49,6 +49,9 @@ export interface EngineStatusData { * reported version data or engines that are not installed. */ installedVersion?: string + installSupported?: boolean + installReason?: string + managed?: boolean } export type ModelItemStatus = (typeof ModelItemStatuses)[number] diff --git a/desktop/src/shared/types/inference-demo.ts b/desktop/src/shared/types/inference-demo.ts index 4af2f35a..adccd928 100644 --- a/desktop/src/shared/types/inference-demo.ts +++ b/desktop/src/shared/types/inference-demo.ts @@ -54,10 +54,11 @@ export const DEMO_REQUEST_TIMEOUT_SECONDS = 120 export const DEMO_ENGINE_PROBES: readonly { backend: DispatcherBackend /** Engine key used by the broker's proxy port registry. */ - proxyEngine: 'ollama' | 'lm-studio' + proxyEngine: 'ollama' | 'lm-studio' | 'llamacpp' }[] = [ { backend: 'ollama', proxyEngine: 'ollama' }, - { backend: 'lmstudio', proxyEngine: 'lm-studio' } + { backend: 'lmstudio', proxyEngine: 'lm-studio' }, + { backend: 'llamacpp', proxyEngine: 'llamacpp' } ] /** diff --git a/desktop/src/shared/types/inference-dispatcher.ts b/desktop/src/shared/types/inference-dispatcher.ts index 2abe35ae..9c6d7a45 100644 --- a/desktop/src/shared/types/inference-dispatcher.ts +++ b/desktop/src/shared/types/inference-dispatcher.ts @@ -10,7 +10,7 @@ * reads its `--list-models` inventory. */ -export type DispatcherBackend = 'ollama' | 'lmstudio' +export type DispatcherBackend = 'ollama' | 'lmstudio' | 'llamacpp' /** One entry from the binary's `--list-models` JSON inventory. */ export interface DispatcherModel { diff --git a/desktop/src/shared/types/workloads.ts b/desktop/src/shared/types/workloads.ts index 71340812..21b9be45 100644 --- a/desktop/src/shared/types/workloads.ts +++ b/desktop/src/shared/types/workloads.ts @@ -6,8 +6,16 @@ import { EngineType } from '@/shared/types/engines' export type WorkloadState = (typeof WorkloadStates)[number] +export interface WorkloadRemoval { + workloadId: string + originatedFrom: string | null + engine?: EngineType + runId?: string +} + export interface Workload { id: string + runId?: string model: string engine: EngineType state: WorkloadState diff --git a/desktop/src/shared/types/ws-channels.ts b/desktop/src/shared/types/ws-channels.ts index 15923765..b26705f0 100644 --- a/desktop/src/shared/types/ws-channels.ts +++ b/desktop/src/shared/types/ws-channels.ts @@ -41,7 +41,7 @@ import type { AppInitialSnapshot, ClusterInitialSnapshot } from '@/shared/types/ import type { ServiceError } from '@/shared/types/errors' import type { NodeItem } from '@/shared/types/nodes' import type { NodeItemMetrics } from '@/shared/types/metrics' -import type { Workload } from '@/shared/types/workloads' +import type { Workload, WorkloadRemoval } from '@/shared/types/workloads' import type { AvailableNode, ClusterIdentityPayload, @@ -110,6 +110,10 @@ export interface WsInvokeChannelMap { // Workloads 'workloads:get-initial': { request: void; response: Record } + 'workloads:cancel': { + request: { id: string; runId: string; engine: EngineType; originatedFrom: string } + response: { accepted: boolean } + } } export type WsInvokeChannel = keyof WsInvokeChannelMap @@ -141,7 +145,7 @@ export interface WsPushChannelMap { // workload ids are a per-node proxy counter (the catalog is keyed by the // (originatedFrom, id) pair). 'workloads:upsert': Workload - 'workloads:remove': { workloadId: string; originatedFrom: string | null } + 'workloads:remove': WorkloadRemoval // Errors 'errors:update': ServiceError[] diff --git a/desktop/src/shared/utils/engine-progress.ts b/desktop/src/shared/utils/engine-progress.ts index 123919a8..3e7cea2b 100644 --- a/desktop/src/shared/utils/engine-progress.ts +++ b/desktop/src/shared/utils/engine-progress.ts @@ -9,6 +9,13 @@ import { EngineOperationType, EngineProgress, EngineType } from '@/shared/types/engines' +/** Unknown vendor progress (including -1) must not become a numeric percentage. */ +export function roundedProgressPercent(percent: number | undefined): number | null { + return percent !== undefined && Number.isFinite(percent) && percent >= 0 && percent <= 100 + ? Math.round(percent) + : null +} + /** Build the map key for an EngineProgress entry. */ export function engineProgressKey(p: { nodeId: string diff --git a/desktop/src/shared/utils/workloads.ts b/desktop/src/shared/utils/workloads.ts index 0cee3b6d..a4216023 100644 --- a/desktop/src/shared/utils/workloads.ts +++ b/desktop/src/shared/utils/workloads.ts @@ -7,17 +7,19 @@ import type { Workload } from '@/shared/types/workloads' * Stable catalog key for a workload. * * The backend's catalog is keyed by `(originatedFrom, engine, runId, id)`, but - * the `workloads:remove` push carries only `(workloadId, originatedFrom)` — and - * the broker's own `Store.Remove` drops every record matching that pair — so - * `(originatedFrom, id)` is the only key a subscribe client can maintain - * consistently across upsert and remove. Each node's proxy assigns workload ids - * from its own monotonic counter, so ids collide across nodes; `originatedFrom` - * (the origin node) disambiguates. Mirror that here so a remote node's job never - * overwrites a local one that happens to share an id. The `\u0000` separator - * cannot appear in a host id or proxy counter, so the key is unambiguous. + * clients retain all four fields so equal counters from different engines or + * proxy runs never overwrite one another. A legacy removal without engine/run + * identity removes the matching origin/id prefix, as the broker does. New + * targeted removals and cancellation retain exact identity. */ -export function workloadKey(originatedFrom: string | null, id: string): string { - return `${originatedFrom ?? ''}\u0000${id}` +export function workloadKey( + originatedFrom: string | null, + id: string, + engine?: string, + runId?: string +): string { + const prefix = `${originatedFrom ?? ''}\u0000${id}` + return engine === undefined ? prefix : `${prefix}\u0000${engine}\u0000${runId ?? ''}` } /** diff --git a/desktop/src/ui/api/engine-api.ts b/desktop/src/ui/api/engine-api.ts index 71d8f163..20202016 100644 --- a/desktop/src/ui/api/engine-api.ts +++ b/desktop/src/ui/api/engine-api.ts @@ -54,6 +54,8 @@ export interface IEngineApi { uninstall(engineType: EngineType, nodeId: string): void /** Pull (download) a model on a node. */ pullModel(engineType: EngineType, nodeId: string, model: string): void + cancelPull(engineType: EngineType, nodeId: string, model: string): void + importModel(engineType: EngineType, nodeId: string, path: string): void /** Load a model into memory on a node. */ loadModel(engineType: EngineType, nodeId: string, model: string): void /** Unload a model from memory on a node. */ @@ -100,6 +102,10 @@ export function createEngineApi(transport: ServiceTransport): IEngineApi { fireCommand(transport, { command: 'uninstall', engineType, nodeId }), pullModel: (engineType, nodeId, model) => fireCommand(transport, { command: 'pullModel', engineType, nodeId, model }), + cancelPull: (engineType, nodeId, model) => + fireCommand(transport, { command: 'cancelPull', engineType, nodeId, model }), + importModel: (engineType, nodeId, path) => + fireCommand(transport, { command: 'importModel', engineType, nodeId, model: path }), loadModel: (engineType, nodeId, model) => fireCommand(transport, { command: 'loadModel', engineType, nodeId, model }), unloadModel: (engineType, nodeId, model) => diff --git a/desktop/src/ui/api/pair-api.ts b/desktop/src/ui/api/pair-api.ts index a0a34155..344d0742 100644 --- a/desktop/src/ui/api/pair-api.ts +++ b/desktop/src/ui/api/pair-api.ts @@ -12,7 +12,8 @@ import type { import type { NodeItem } from '@/shared/types/nodes' import type { ServiceError } from '@/shared/types/errors' import type { NodeItemMetrics } from '@/shared/types/metrics' -import type { Workload } from '@/shared/types/workloads' +import type { Workload, WorkloadRemoval } from '@/shared/types/workloads' +import type { EngineType } from '@/shared/types/engines' import type { AppInitialSnapshot, ClusterInitialSnapshot } from '@/shared/types/bootstrap' // --------------------------------------------------------------------------- @@ -80,14 +81,18 @@ export interface IDiscoveryApi { } export interface IWorkloadsApi { + cancel(request: { + id: string + runId: string + engine: EngineType + originatedFrom: string + }): Promise<{ accepted: boolean }> /** Fetch all active workloads (inference jobs). */ getInitial(): Promise> /** A workload was created or updated. */ onUpsert(callback: (workload: Workload) => void): () => void /** A workload was completed and removed. */ - onRemove( - callback: (removal: { workloadId: string; originatedFrom: string | null }) => void - ): () => void + onRemove(callback: (removal: WorkloadRemoval) => void): () => void } export interface IErrorsApi { @@ -171,6 +176,7 @@ export function createPairApi(transport: ServiceTransport): IPairApi { engines: createEngineApi(transport), workloads: { getInitial: () => transport.invoke('workloads:get-initial'), + cancel: request => transport.invoke('workloads:cancel', request), onUpsert: cb => transport.subscribePush('workloads:upsert', cb), onRemove: cb => transport.subscribePush('workloads:remove', cb) }, diff --git a/desktop/src/ui/components/BackendRow/BackendFooter.tsx b/desktop/src/ui/components/BackendRow/BackendFooter.tsx index fc7ece39..8aa56d5b 100644 --- a/desktop/src/ui/components/BackendRow/BackendFooter.tsx +++ b/desktop/src/ui/components/BackendRow/BackendFooter.tsx @@ -33,7 +33,10 @@ export function BackendFooter({ disabled: boolean onUninstall: () => void }) { - const autoInstall = canAutoInstallBackendForOs(backend.type, targetOs) + const autoInstall = + backend.type === 'llamacpp' + ? backend.installSupported === true + : canAutoInstallBackendForOs(backend.type, targetOs) const isTransitioning = backend.processStatus === 'installing' || backend.processStatus === 'uninstalling' const isNotInstalled = backend.processStatus === 'not-installed' diff --git a/desktop/src/ui/components/BackendRow/BackendHeader.tsx b/desktop/src/ui/components/BackendRow/BackendHeader.tsx index d54bd629..05473127 100644 --- a/desktop/src/ui/components/BackendRow/BackendHeader.tsx +++ b/desktop/src/ui/components/BackendRow/BackendHeader.tsx @@ -11,6 +11,7 @@ import { DismissibleTooltip } from '@/ui/components/DismissibleTooltip/Dismissib import { gatewayEndpointDisplayUrl } from '@/ui/utils/gateway-inference-paths' import { EngineCapabilities } from '@/ui/constants/engine-capabilities' import { statusLabel } from '@/ui/utils/status' +import { roundedProgressPercent } from '@/shared/utils/engine-progress' /** Install/uninstall lines include asset names + percentages — allow more room than generic status. */ const INSTALL_STATUS_MAX_LEN = 52 @@ -91,7 +92,17 @@ export function BackendHeader({ className={`${isUnavailable ? 'cursor-default' : 'cursor-pointer'} p-4 -m-4`} > - + {isLocalNode && @@ -114,7 +125,7 @@ export function BackendHeader({ e.stopPropagation() handleCopy() }} - title={`Copy ${backend.displayName} API http://127.0.0.1:${backend.proxyPort}`} + title={`Copy ${backend.displayName} API ${proxyUrl}`} style={{ padding: '2px 6px', minWidth: 'auto' }} aria-label={`Copy ${backend.displayName} API URL`} > @@ -160,7 +171,7 @@ export function BackendHeader({ const baseStatus = backend.installProgress?.status ?? statusLabel[backend.processStatus] const pct = backend.installProgress?.percent - const pctRounded = pct != null && Number.isFinite(pct) ? Math.round(pct) : null + const pctRounded = roundedProgressPercent(pct) const pctSuffix = pctRounded != null ? ` · ${pctRounded}%` : '' const baseWithoutDuplicatePercent = pctRounded != null diff --git a/desktop/src/ui/components/BackendRow/BackendRow.tsx b/desktop/src/ui/components/BackendRow/BackendRow.tsx index f3c16238..2f96121b 100644 --- a/desktop/src/ui/components/BackendRow/BackendRow.tsx +++ b/desktop/src/ui/components/BackendRow/BackendRow.tsx @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { useCallback, useEffect, useMemo, useState } from 'react' -import { Divider, Stack } from '@nvidia/foundations-react-core' +import { Divider, Stack, Text } from '@nvidia/foundations-react-core' import type { BackendInfo } from '@/ui/types/engine-info' import type { EngineProcessStatus } from '@/shared/types/engines' @@ -173,11 +173,23 @@ export function BackendRow({ }, [isUnavailable]) // Install/start/stop, model pull, and the settings editor work on clustered - // peers; uninstall, update, and model load/delete remain local-only. - const controlsDisabled = isTransitioning + // peers; uninstall, update, and model load/delete remain local-only. A + // llama.cpp runtime PAIR detected but does not manage is observe-only: its + // owner keeps lifecycle, settings, and model changes. + const externalLlama = + backend.type === 'llamacpp' && + backend.processStatus !== 'not-installed' && + backend.managed !== true + const controlsDisabled = isTransitioning || externalLlama const content = expanded ? ( + {externalLlama && ( + + External llama.cpp runtime. PAIR observes it; lifecycle and model changes remain + with its owner. + + )} )} - {canShowAccordions && ( + {canShowAccordions && !externalLlama && ( diff --git a/desktop/src/ui/components/BackendRow/InstallButton.tsx b/desktop/src/ui/components/BackendRow/InstallButton.tsx index 88127754..cadca10b 100644 --- a/desktop/src/ui/components/BackendRow/InstallButton.tsx +++ b/desktop/src/ui/components/BackendRow/InstallButton.tsx @@ -23,7 +23,10 @@ export function InstallButton({ disabled: boolean onInstall: () => void }) { - const autoInstall = canAutoInstallBackendForOs(backend.type, targetOs) + const autoInstall = + backend.type === 'llamacpp' + ? backend.installSupported === true + : canAutoInstallBackendForOs(backend.type, targetOs) const isNotInstalled = backend.processStatus === 'not-installed' const missingPrereqs = (backend.prerequisites ?? []).filter(p => !p.installed) const prereqsMet = missingPrereqs.length === 0 @@ -36,6 +39,14 @@ export function InstallButton({ const onInstallAllClick = useDismissibleTooltipTrigger(onInstall) + if (backend.type === 'llamacpp' && backend.installSupported !== true && isNotInstalled) { + return ( + + {backend.installReason || 'Install support not reported by this node.'} + + ) + } + return ( <> {!autoInstall && isLocalNode && isNotInstalled && backend.installUrl && ( diff --git a/desktop/src/ui/components/EngineIcon.tsx b/desktop/src/ui/components/EngineIcon.tsx index 4ee81367..a5f71755 100644 --- a/desktop/src/ui/components/EngineIcon.tsx +++ b/desktop/src/ui/components/EngineIcon.tsx @@ -21,23 +21,39 @@ export default function EngineIcon({ type, size = 32 }: { type: EngineType; size overflow: 'hidden' } - if (type === 'ollama') { - return ( -
- Ollama -
- ) + switch (type) { + case 'ollama': + return ( +
+ Ollama +
+ ) + case 'lm-studio': + imgStyle.objectFit = 'cover' + return ( +
+ LM Studio +
+ ) + case 'llamacpp': + return ( +
+ + cpp + +
+ ) } - - if (type === 'lm-studio') { - imgStyle.objectFit = 'cover' - - return ( -
- LM Studio -
- ) - } - - return null } diff --git a/desktop/src/ui/components/ModelManager/ModelManager.tsx b/desktop/src/ui/components/ModelManager/ModelManager.tsx index e63dacf2..91c03113 100644 --- a/desktop/src/ui/components/ModelManager/ModelManager.tsx +++ b/desktop/src/ui/components/ModelManager/ModelManager.tsx @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { useCallback, useMemo, useState } from 'react' -import { Button, Flex, Stack, Text } from '@nvidia/foundations-react-core' +import { Button, Flex, FormField, Stack, Text, TextInput } from '@nvidia/foundations-react-core' import type { BackendInfo } from '@/ui/types/engine-info' import { EngineCapabilities } from '@/ui/constants/engine-capabilities' import { formatModelDisplayName } from '@/ui/utils/format-model-display-name' @@ -13,6 +13,7 @@ import { ModelHubModal } from '@/ui/components/ModelHub/ModelHubModal' import { useEngineProgressStore } from '@/ui/stores/engine-progress.store' import { usePendingActionsStore } from '@/ui/stores/pending-actions.store' import { isEnginePullInProgress } from '@/shared/utils/engine-progress' +import { useConnectionStore } from '@/ui/stores/connection.store' import ModelRow from './ModelRow' import { IncomingSyncPullRow } from './IncomingSyncPullRow' @@ -21,6 +22,9 @@ import type { IncomingSyncRow } from '@/ui/types/model-manager' import type { ModelEntry } from '@/ui/types/model-hub' export function ModelManager({ backend, nodeId }: { backend: BackendInfo; nodeId: string }) { + const selfId = useConnectionStore(state => state.selfId) + const [importPath, setImportPath] = useState('') + const [llamaRepo, setLlamaRepo] = useState('') const [openModelHubModal, setOpenModelHubModal] = useState(false) const [modelPendingDelete, setModelPendingDelete] = useState(null) const models = (backend.models ?? []).sort((a, b) => @@ -28,7 +32,10 @@ export function ModelManager({ backend, nodeId }: { backend: BackendInfo; nodeId formatModelDisplayName(b.name, backend.type) ) ) - const caps = EngineCapabilities[backend.type] + const externalLlama = backend.type === 'llamacpp' && backend.managed !== true + const caps = externalLlama + ? { ...EngineCapabilities[backend.type], hasEject: false, hasDeleteModel: false } + : EngineCapabilities[backend.type] const backendType = backend.type const getProgress = useEngineProgressStore(state => state.getProgress) @@ -195,9 +202,34 @@ export function ModelManager({ backend, nodeId }: { backend: BackendInfo; nodeId )} {incomingSyncs.map(p => ( - + + + {backendType === 'llamacpp' && !externalLlama && ( + + )} + ))} + {backendType === 'llamacpp' && transientModel && transientPullProgress && ( + + )} + {!isBusy && ( <> {(isRunning || modelOpsWhenStopped) && @@ -222,7 +254,7 @@ export function ModelManager({ backend, nodeId }: { backend: BackendInfo; nodeId - {supportsSearch && ( + {/* + llama.cpp has no browsable catalog, so where the other engines + offer "Add model" and a searchable list, it takes the model's + identifier directly. These are the same brand-coloured download + action the hub uses once a row is picked, so the two routes to a + model read as the same operation. + */} + {backendType === 'llamacpp' && !externalLlama && ( + + + { + if (e.key === 'Enter' && llamaRepo.trim()) { + window.pairApi.engines.pullModel( + backendType, + nodeId, + llamaRepo.trim() + ) + } + }} + className="min-w-0" + /> + + + + )} + {/* + Importing is deliberately the quieter of the two: it only works + on this machine, because the path is resolved where the engine + runs rather than where the window is. + */} + {backendType === 'llamacpp' && !externalLlama && nodeId === selfId && ( + + + { + if (e.key === 'Enter' && importPath.trim()) { + window.pairApi.engines.importModel( + backendType, + nodeId, + importPath.trim() + ) + } + }} + className="min-w-0" + /> + + + + )} + {supportsSearch && backendType !== 'llamacpp' && ( + )} diff --git a/desktop/src/ui/components/Workloads/WorkloadListView.tsx b/desktop/src/ui/components/Workloads/WorkloadListView.tsx index 28323190..f6da8827 100644 --- a/desktop/src/ui/components/Workloads/WorkloadListView.tsx +++ b/desktop/src/ui/components/Workloads/WorkloadListView.tsx @@ -82,7 +82,7 @@ function WorkloadListView({ filter, setFilter }: WorkloadListViewProps) { .map(([_, value]) => `${value}`) .join('-') const workloadsKey = currentWorkloads - .map(w => workloadKey(w.originatedFrom, w.id)) + .map(w => workloadKey(w.originatedFrom, w.id, w.engine, w.runId)) .join('-') return `${filterKey}-${workloadsKey}` }, [filter, currentWorkloads]) @@ -128,7 +128,12 @@ function WorkloadListView({ filter, setFilter }: WorkloadListViewProps) { data-workload-list-content > {currentWorkloads.map(workload => { - const key = workloadKey(workload.originatedFrom, workload.id) + const key = workloadKey( + workload.originatedFrom, + workload.id, + workload.engine, + workload.runId + ) return (
diff --git a/desktop/src/ui/components/Workloads/WorkloadNodeConnections.tsx b/desktop/src/ui/components/Workloads/WorkloadNodeConnections.tsx index 60d88a87..76c6f481 100644 --- a/desktop/src/ui/components/Workloads/WorkloadNodeConnections.tsx +++ b/desktop/src/ui/components/Workloads/WorkloadNodeConnections.tsx @@ -50,14 +50,6 @@ function connectionsEqual(a: readonly Connection[], b: readonly Connection[]): b return true } -// Build a key that uniquely identifies a workload's DOM anchor. Workload ids are -// a per-node proxy counter and collide across nodes, so we key on the -// (origin, id) pair the same way `workloadKey` does. The null byte can't appear -// in either value, so it is a safe separator. -function anchorKey(origin: string, id: string): string { - return `${origin}\u0000${id}` -} - function computeConnections(svg: SVGSVGElement, workloads: readonly Workload[]): Connection[] { const svgRect = svg.getBoundingClientRect() const result: Connection[] = [] @@ -69,7 +61,15 @@ function computeConnections(svg: SVGSVGElement, workloads: readonly Workload[]): for (const el of document.querySelectorAll('[data-workload-id]')) { const id = el.getAttribute('data-workload-id') if (id === null) continue - workloadElByKey.set(anchorKey(el.getAttribute('data-workload-origin') ?? '', id), el) + workloadElByKey.set( + workloadKey( + el.getAttribute('data-workload-origin') ?? '', + id, + el.getAttribute('data-workload-engine') ?? '', + el.getAttribute('data-workload-run') ?? '' + ), + el + ) } const nodeElById = new Map() for (const el of document.querySelectorAll('[data-node-id]')) { @@ -82,7 +82,7 @@ function computeConnections(svg: SVGSVGElement, workloads: readonly Workload[]): if (!executionNodeId) continue const workloadEl = workloadElByKey.get( - anchorKey(workload.originatedFrom ?? '', workload.id) + workloadKey(workload.originatedFrom, workload.id, workload.engine, workload.runId) ) const nodeEl = nodeElById.get(executionNodeId) if (!workloadEl || !nodeEl) continue @@ -126,7 +126,12 @@ function computeConnections(svg: SVGSVGElement, workloads: readonly Workload[]): const color = getWorkloadStateColor(workload.state) result.push({ - workloadId: workloadKey(workload.originatedFrom, workload.id), + workloadId: workloadKey( + workload.originatedFrom, + workload.id, + workload.engine, + workload.runId + ), nodeId: executionNodeId, color: `${WORKLOAD_COLOR_MAP[color]}${workload.state === 'running' ? 'FF' : '88'}`, path, diff --git a/desktop/src/ui/constants/engine-capabilities.ts b/desktop/src/ui/constants/engine-capabilities.ts index 77e6f2bf..013249f8 100644 --- a/desktop/src/ui/constants/engine-capabilities.ts +++ b/desktop/src/ui/constants/engine-capabilities.ts @@ -43,5 +43,18 @@ export const EngineCapabilities: Record = { // server. Deleting therefore interrupts inference and needs a warning. restartsOnModelDelete: true, engineHub: { label: 'LM Studio', url: 'https://lmstudio.ai/models' } + }, + llamacpp: { + hasExpiry: false, + hasEject: true, + hasInstall: ['win32', 'darwin', 'linux'], + hasEnginePort: true, + hasInstallPath: false, + hasProxyWebUI: false, + hasPreferredNode: false, + hasCrashAlert: false, + hasModelSearchOnlyWhenRunning: false, + modelOpsWhenStopped: true, + hasDeleteModel: true } } diff --git a/desktop/src/ui/constants/welcome.ts b/desktop/src/ui/constants/welcome.ts index 4f9fa04a..71755562 100644 --- a/desktop/src/ui/constants/welcome.ts +++ b/desktop/src/ui/constants/welcome.ts @@ -13,7 +13,8 @@ export const WELCOME_STEP_SUB_HEADINGS = ['', 'You can update later by clicking export const WELCOME_ENGINE_DEFAULT_SELECTED: Record = { ollama: true, - 'lm-studio': true + 'lm-studio': true, + llamacpp: false } export function getWelcomeEngineCandidates(os: PlatformDisplayName): EngineType[] { diff --git a/desktop/src/ui/stores/workloads.store.ts b/desktop/src/ui/stores/workloads.store.ts index 7bb90822..2d961ea5 100644 --- a/desktop/src/ui/stores/workloads.store.ts +++ b/desktop/src/ui/stores/workloads.store.ts @@ -45,15 +45,23 @@ export const useWorkloadsStore = create((set, get) => ({ for (const op of ops) { const base = updated ?? prev if (op.kind === 'upsert') { - const key = workloadKey(op.workload.originatedFrom, op.workload.id) + const key = workloadKey( + op.workload.originatedFrom, + op.workload.id, + op.workload.engine, + op.workload.runId + ) const existing = base.get(key) if (existing && deepEqual(existing, op.workload)) continue if (!updated) updated = new Map(prev) updated.set(key, op.workload) } else { - if (!base.has(op.key)) continue - if (!updated) updated = new Map(prev) - updated.delete(op.key) + for (const key of base.keys()) { + if (key === op.key || key.startsWith(op.key + '\u0000')) { + if (!updated) updated = new Map(prev) + updated.delete(key) + } + } } } @@ -83,12 +91,14 @@ export const useWorkloadsStore = create((set, get) => ({ pendingOps.push({ kind: 'upsert', workload }) schedule() }), - window.pairApi.workloads.onRemove(({ workloadId, originatedFrom }) => { - const key = workloadKey(originatedFrom, workloadId) - if (initializing) removedDuringInit.add(key) - pendingOps.push({ kind: 'remove', key }) - schedule() - }) + window.pairApi.workloads.onRemove( + ({ workloadId, originatedFrom, engine, runId }) => { + const key = workloadKey(originatedFrom, workloadId, engine, runId) + if (initializing) removedDuringInit.add(key) + pendingOps.push({ kind: 'remove', key }) + schedule() + } + ) ) } @@ -101,7 +111,11 @@ export const useWorkloadsStore = create((set, get) => ({ // Subtract jobs a `workloads:remove` retired during the fetch (that // delta is newer than the snapshot), before overlaying upserts — so a // remove-then-readd of the same key still surfaces the re-add. - for (const key of removedDuringInit) map.delete(key) + for (const prefix of removedDuringInit) { + for (const key of map.keys()) { + if (key === prefix || key.startsWith(prefix + '\u0000')) map.delete(key) + } + } // Overlay upserts that already landed during the fetch (also newer than // the baseline), so seeding never regresses a live transition. for (const [key, workload] of get().workloads) { diff --git a/desktop/src/ui/types/engine-info.ts b/desktop/src/ui/types/engine-info.ts index 3c4cb780..cf1d4250 100644 --- a/desktop/src/ui/types/engine-info.ts +++ b/desktop/src/ui/types/engine-info.ts @@ -95,6 +95,9 @@ export interface BackendInfo { proxyPort: number | null /** Installed engine binary version reported by the owning node */ installedVersion?: string + installSupported?: boolean + installReason?: string + managed?: boolean /** Models on this backend with per-model status */ models: ModelItem[] /** System-level dependencies required before install/run (local node only) */ diff --git a/desktop/src/ui/utils/format-model-display-name.ts b/desktop/src/ui/utils/format-model-display-name.ts index e16282ed..dc1cd1e7 100644 --- a/desktop/src/ui/utils/format-model-display-name.ts +++ b/desktop/src/ui/utils/format-model-display-name.ts @@ -40,6 +40,7 @@ export function formatModelDisplayName(name: string, engineType?: string | null) switch (engineType) { case 'lm-studio': + case 'llamacpp': return isHfModel ? formatted : formatLmStudioModelName(formatted) case 'ollama': diff --git a/desktop/src/ui/utils/formatters.ts b/desktop/src/ui/utils/formatters.ts index 799a2762..7090dc47 100644 --- a/desktop/src/ui/utils/formatters.ts +++ b/desktop/src/ui/utils/formatters.ts @@ -5,6 +5,8 @@ * Utility functions for formatting data */ +import { roundedProgressPercent } from '@/shared/utils/engine-progress' + /** * Format a timestamp as a relative time distance (e.g., "2 minutes ago") */ @@ -87,9 +89,8 @@ type PullProgressFields = { * Percent or byte-derived percent for file-style model pulls (Whisper, Piper, Stable-Diffusion, etc.). */ function formatPullProgressDetail(p: PullProgressFields): string { - if (p.percent != null && Number.isFinite(p.percent)) { - return `${Math.round(Math.min(100, Math.max(0, p.percent)))}%` - } + const percent = roundedProgressPercent(p.percent) + if (percent !== null) return percent + '%' return '' } diff --git a/desktop/src/ui/utils/gateway-inference-paths.ts b/desktop/src/ui/utils/gateway-inference-paths.ts index 688dd9ec..4e67e517 100644 --- a/desktop/src/ui/utils/gateway-inference-paths.ts +++ b/desktop/src/ui/utils/gateway-inference-paths.ts @@ -8,7 +8,13 @@ export function gatewayEndpointDisplayUrl( proxyPort: number, inferenceType: EngineType ): string | null { - void inferenceType if (proxyPort <= 0) return null - return `http://127.0.0.1:${proxyPort}` + const base = `http://127.0.0.1:${proxyPort}` + switch (inferenceType) { + case 'ollama': + return base + case 'lm-studio': + case 'llamacpp': + return `${base}/v1` + } } diff --git a/desktop/tests/modular/app-data-wipe.test.ts b/desktop/tests/modular/app-data-wipe.test.ts index 12336080..439dbf57 100644 --- a/desktop/tests/modular/app-data-wipe.test.ts +++ b/desktop/tests/modular/app-data-wipe.test.ts @@ -4,6 +4,7 @@ import { spawnSync } from 'child_process' import fs from 'fs' import path from 'path' +import os from 'os' import { describe, expect, it } from 'vitest' import { currentPlatform } from '@/shared/utils/platform' @@ -21,6 +22,95 @@ describe('repo-root wipe scripts', () => { // the Windows twin, so only its static inventory is observable there. const itUnix = it.skipIf(currentPlatform() === 'win32') + it.skipIf(currentPlatform() !== 'win32')( + 'Windows reset removes app data but retains the sibling model library', + () => { + const isolated = fs.mkdtempSync(path.join(os.tmpdir(), 'pair-model-retention-')) + const config = path.join(isolated, 'config') + const app = path.join(config, 'Nvidia Corporation', 'Personal AI Router') + const model = path.join( + config, + 'Nvidia Corporation', + 'Personal AI Router Models', + 'llamacpp', + 'retained.gguf' + ) + fs.mkdirSync(app, { recursive: true }) + fs.mkdirSync(path.dirname(model), { recursive: true }) + fs.writeFileSync(path.join(app, 'settings.json'), '{}') + fs.writeFileSync(model, 'retained-model') + try { + // Only process enumeration is stubbed: never stop a real application. + // The exact script still performs its real filesystem removal. + const quotedScript = ps1.replaceAll("'", "''") + const result = spawnSync( + 'powershell.exe', + [ + '-NoProfile', + '-Command', + `function tasklist {}; & '${quotedScript}' --confirm` + ], + { + encoding: 'utf8', + env: { + ...process.env, + USERPROFILE: isolated, + LOCALAPPDATA: config, + TEMP: isolated, + TMP: isolated + } + } + ) + expect(result.status).toBe(0) + expect(fs.existsSync(app)).toBe(false) + expect(fs.readFileSync(model, 'utf8')).toBe('retained-model') + } finally { + fs.rmSync(isolated, { recursive: true, force: true }) + } + } + ) + + it('refuses reset before deleting unmigrated llama models', () => { + const isolated = fs.mkdtempSync(path.join(os.tmpdir(), 'pair-legacy-models-')) + const config = + currentPlatform() === 'darwin' + ? path.join(isolated, 'Library', 'Application Support') + : path.join(isolated, 'config') + const model = path.join( + config, + 'Nvidia Corporation', + 'Personal AI Router', + 'engine-bin', + 'llamacpp', + 'models', + 'retained.gguf' + ) + fs.mkdirSync(path.dirname(model), { recursive: true }) + fs.writeFileSync(model, 'retained-model') + try { + const windows = currentPlatform() === 'win32' + const result = spawnSync( + windows ? 'powershell.exe' : 'bash', + windows ? ['-NoProfile', '-File', ps1, '--confirm'] : [sh, '--confirm'], + { + encoding: 'utf8', + env: { + ...process.env, + HOME: isolated, + USERPROFILE: isolated, + LOCALAPPDATA: config, + XDG_CONFIG_HOME: config + } + } + ) + expect(result.status).toBe(1) + expect(result.stdout + result.stderr).toContain('Open the updated app to migrate') + expect(fs.readFileSync(model, 'utf8')).toBe('retained-model') + } finally { + fs.rmSync(isolated, { recursive: true, force: true }) + } + }) + it('ships unix and windows entrypoints', () => { expect(fs.existsSync(sh), sh).toBe(true) expect(fs.existsSync(ps1), ps1).toBe(true) diff --git a/desktop/tests/modular/deb-runtime-dependencies.test.ts b/desktop/tests/modular/deb-runtime-dependencies.test.ts new file mode 100644 index 00000000..fa612bfe --- /dev/null +++ b/desktop/tests/modular/deb-runtime-dependencies.test.ts @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import { describe, expect, it } from 'vitest' + +describe('Debian runtime dependencies', () => { + it('preserves the installed builder defaults and supplies GBM and ALSA runtimes', () => { + const require = createRequire(import.meta.url) + const builder = readFileSync( + require.resolve('app-builder-lib/out/targets/FpmTarget.js'), + 'utf8' + ) + const defaults = builder.match(/case "deb":\s*return (\[[^\n]+\]);/) + expect(defaults).not.toBeNull() + const config = readFileSync('electron-builder.config.ts', 'utf8') + const dependencyBlock = config.match(/\bdeb:\s*\{[\s\S]*?depends:\s*\[([^\]]+)\]/) + expect(dependencyBlock).not.toBeNull() + const dependencies = [...dependencyBlock![1].matchAll(/'([^']+)'/g)].map(m => m[1]) + expect(dependencies).toEqual([ + ...JSON.parse(defaults![1]), + 'libgbm1', + 'libasound2t64 | libasound2' + ]) + }) +}) diff --git a/desktop/tests/modular/engine-command-load.test.ts b/desktop/tests/modular/engine-command-load.test.ts index 1689cc1d..41cf0ce4 100644 --- a/desktop/tests/modular/engine-command-load.test.ts +++ b/desktop/tests/modular/engine-command-load.test.ts @@ -5,10 +5,12 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ state: { - getSelfId: vi.fn(() => 'local-node') + getSelfId: vi.fn(() => 'local-node'), + beginLocalEngineOp: vi.fn() }, supervisor: { hasProcess: vi.fn(() => true), + callProcess: vi.fn().mockResolvedValue({ accepted: true }), sendProcess: vi.fn(), reportError: vi.fn() } @@ -32,6 +34,121 @@ describe('local model load command', () => { vi.clearAllMocks() mocks.state.getSelfId.mockReturnValue('local-node') mocks.supervisor.hasProcess.mockReturnValue(true) + mocks.supervisor.callProcess.mockResolvedValue({ accepted: true }) + }) + + it('forwards the exact workload identity and reports only cancellation acceptance', async () => { + const request = { + id: '1', + engine: 'llamacpp', + runId: 'current', + originatedFrom: 'local-node' + } as const + expect(await handleServiceBridgeInvoke('workloads:cancel', request)).toEqual({ + accepted: true + }) + expect(mocks.supervisor.callProcess).toHaveBeenCalledWith( + 'broker', + 'workloads:cancel', + request + ) + }) + + it('refuses a local llama update without uninstalling, installing, or acting', async () => { + await handleServiceBridgeInvoke('engine:command', { + command: 'update', + engineType: 'llamacpp', + nodeId: 'local-node' + }) + // No engine:uninstall, engine:install, or engine:action leaves the + // bridge, and no optimistic lifecycle op is begun: the refusal is the + // whole response. + expect(mocks.supervisor.sendProcess).not.toHaveBeenCalled() + expect(mocks.supervisor.callProcess).not.toHaveBeenCalled() + expect(mocks.state.beginLocalEngineOp).not.toHaveBeenCalled() + expect(mocks.supervisor.reportError).toHaveBeenCalledExactlyOnceWith( + 'llama.cpp has no managed update; uninstall and reinstall the managed runtime instead.', + 'warning', + 'engine-cmd:update:llamacpp', + { engineType: 'llamacpp', operation: 'update' } + ) + }) + + it('refuses a remote llama update like every other remote update', async () => { + mocks.state.getSelfId.mockReturnValue('other-node') + await handleServiceBridgeInvoke('engine:command', { + command: 'update', + engineType: 'llamacpp', + nodeId: 'peer-node' + }) + expect(mocks.supervisor.sendProcess).not.toHaveBeenCalled() + expect(mocks.supervisor.callProcess).not.toHaveBeenCalled() + expect(mocks.supervisor.reportError).toHaveBeenCalledExactlyOnceWith( + 'update is only available on the local node — remote uninstall/update is not supported yet.', + 'warning', + 'engine-cmd:remote:update' + ) + }) + + it('keeps the uninstall-then-install update pair for the other managed engines', async () => { + await handleServiceBridgeInvoke('engine:command', { + command: 'update', + engineType: 'ollama', + nodeId: 'local-node' + }) + expect(mocks.state.beginLocalEngineOp).toHaveBeenCalledExactlyOnceWith( + 'ollama', + 'installing' + ) + expect(mocks.supervisor.sendProcess).toHaveBeenCalledTimes(2) + expect(mocks.supervisor.sendProcess).toHaveBeenNthCalledWith( + 1, + 'broker', + 'engine:uninstall', + { engine: 'ollama' }, + expect.any(Function), + true + ) + expect(mocks.supervisor.sendProcess).toHaveBeenNthCalledWith( + 2, + 'broker', + 'engine:install', + { engine: 'ollama', start: true }, + expect.any(Function), + true + ) + expect(mocks.supervisor.reportError).not.toHaveBeenCalled() + }) + + it('routes llama load and cancel to the owning engine manager', async () => { + await handleServiceBridgeInvoke('engine:command', { + command: 'loadModel', + engineType: 'llamacpp', + nodeId: 'local-node', + model: 'owner/model:Q4' + }) + expect(mocks.supervisor.sendProcess).toHaveBeenCalledWith( + 'broker', + 'engine:action', + { engine: 'llamacpp', action: 'load_model', params: { model: 'owner/model:Q4' } }, + expect.any(Function), + true + ) + await handleServiceBridgeInvoke('engine:command', { + command: 'cancelPull', + engineType: 'llamacpp', + nodeId: 'local-node', + model: 'owner/model:Q4' + }) + expect(mocks.supervisor.sendProcess).toHaveBeenLastCalledWith( + 'broker', + 'engine:action', + { engine: 'llamacpp', action: 'cancel_pull', params: { model: 'owner/model:Q4' } }, + expect.any(Function), + // Observed like the load above: a cancel the backend refuses has to + // clear the optimistic row rather than look like it succeeded. + true + ) }) it('observes and attributes an Ollama load rejection to the pending model row', async () => { diff --git a/desktop/tests/modular/inference-demo-lifecycle.test.ts b/desktop/tests/modular/inference-demo-lifecycle.test.ts index d9c4b95e..08a46cd9 100644 --- a/desktop/tests/modular/inference-demo-lifecycle.test.ts +++ b/desktop/tests/modular/inference-demo-lifecycle.test.ts @@ -45,9 +45,10 @@ const POISONED_ENV: Record = { } /** Proxy ports the fake broker reports. Mutable so a test can withhold one. */ -const proxyPorts: Record<'ollama' | 'lm-studio', number | null> = { +const proxyPorts: Record<'ollama' | 'lm-studio' | 'llamacpp', number | null> = { ollama: 11434, - 'lm-studio': 1234 + 'lm-studio': 1234, + llamacpp: null } /** Model inventory each probe returns. Mutable so a test can return none. */ @@ -114,7 +115,7 @@ vi.mock('electron', () => ({ vi.mock('@/electron/service-bridge/modular-state', () => ({ getModularBridgeState: () => ({ - getProxyPort: (engine: 'ollama' | 'lm-studio') => proxyPorts[engine] + getProxyPort: (engine: 'ollama' | 'lm-studio' | 'llamacpp') => proxyPorts[engine] }) })) @@ -138,6 +139,7 @@ beforeEach(() => { inventory = [{ name: 'demo-model', type: 'llm' }] proxyPorts.ollama = 11434 proxyPorts['lm-studio'] = 1234 + proxyPorts.llamacpp = null Object.assign(process.env, POISONED_ENV) vi.useFakeTimers() }) @@ -151,6 +153,22 @@ afterEach(() => { }) describe('inference demo lifecycle', () => { + it('drives llama.cpp through its reported proxy when it is the only engine available', async () => { + proxyPorts.ollama = null + proxyPorts['lm-studio'] = null + proxyPorts.llamacpp = 19084 + const state = await startInferenceDemo() + expect(state.engineCount).toBe(1) + expect(state.targetCount).toBe(1) + await vi.advanceTimersByTimeAsync(70_000) + expect(spawned.length).toBeGreaterThan(0) + for (const child of spawned) { + expect(child.args[child.args.indexOf('--backend') + 1]).toBe('llamacpp') + expect(portOf(child)).toBe(19084) + } + expect(getInferenceDemoState().status).toBe('idle') + }) + it('returns to idle immediately on stop, with no draining tail', async () => { await startInferenceDemo() expect(getInferenceDemoState().status).toBe('running') diff --git a/desktop/tests/modular/llamacpp-engine.test.ts b/desktop/tests/modular/llamacpp-engine.test.ts new file mode 100644 index 00000000..301cc087 --- /dev/null +++ b/desktop/tests/modular/llamacpp-engine.test.ts @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from 'vitest' +import { EngineDisplayNames, EnabledEngineTypes, EngineTypes } from '@/shared/constants/engines' +import { EngineCapabilities } from '@/ui/constants/engine-capabilities' +import { MODULAR_RUNTIME_BINARIES } from '@/shared/constants/modular-binaries' + +describe('llamacpp engine', () => { + it('is a enabled engine type', () => { + expect(EngineTypes).toContain('llamacpp') + expect(EnabledEngineTypes).toContain('llamacpp') + expect(EngineDisplayNames.llamacpp).toBe('llama.cpp') + }) + it('exposes managed install and model actions', () => { + const caps = EngineCapabilities.llamacpp + expect(caps.hasInstall).toEqual(['win32', 'darwin', 'linux']) + expect(caps.hasEject).toBe(true) + expect(caps.hasDeleteModel).toBe(true) + expect(caps.modelOpsWhenStopped).toBe(true) + expect(caps.hasEnginePort).toBe(true) + expect(caps.engineHub).toBeUndefined() + }) + // llama.cpp ships no binary of its own: one nvpair-proxy process hosts a + // facade per engine, so adding an engine adds no runtime executable. This + // asserts the absence, because a stray entry would mean packaging looking + // for a file the build never produces. + it('adds no runtime binary of its own', () => { + const names = MODULAR_RUNTIME_BINARIES.map(b => b.processName) + expect(names).not.toContain('llamacpp-proxy') + expect(names).toContain('nvpair-proxy') + expect(names.filter(n => n.endsWith('-proxy'))).toEqual(['nvpair-proxy']) + }) +}) diff --git a/desktop/tests/modular/llamacpp-state.test.ts b/desktop/tests/modular/llamacpp-state.test.ts new file mode 100644 index 00000000..0c89fbbd --- /dev/null +++ b/desktop/tests/modular/llamacpp-state.test.ts @@ -0,0 +1,222 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from 'vitest' +vi.mock('electron', () => ({ BrowserWindow: { getAllWindows: () => [] } })) +vi.mock('@/electron/window', () => ({ createOverviewWindow: vi.fn() })) +import { getModularBridgeState } from '@/electron/service-bridge/modular-state' +import { roundedProgressPercent } from '@/shared/utils/engine-progress' +import { formatPullProgressLabel } from '@/ui/utils/formatters' +import { subscribePush } from '@/electron/service-bridge/push-bus' + +describe('managed llama state', () => { + it.each([true, false])( + 'settles an already-installed no-op without changing running=%s facts', + running => { + const state = getModularBridgeState() + const self = 'llama-noop-self' + const peer = 'llama-noop-peer' + const facts = { + engine: 'llamacpp', + installed: true, + running, + port: 8082, + managed: true, + install_supported: true, + install_reason: 'Supported' + } + state.setSelfId(self) + for (const id of [self, peer]) { + state.handleNotification({ + source: 'llamacpp-proxy', + method: 'node/discovered', + params: { id, port: 8080 } + }) + } + state.applyEngineManagerStatus(facts) + state.applyRemoteEngineFacts(peer, { engines: [facts] }) + state.setLocalEngineModels('llamacpp', ['cached']) + state.applyLocalLoadedModels({ llamacpp: running ? ['cached'] : [] }) + const cleared: string[] = [] + const unsubscribe = subscribePush(event => { + if (event.channel === 'engines:progress-cleared') cleared.push(event.payload.key) + }) + try { + state.beginLocalEngineOp('llamacpp', 'installing') + state.beginRemoteEngineOp(peer, 'llamacpp', 'installing') + state.applyEngineManagerProgress({ + engine: 'llamacpp', + stage: 'already-installed', + percent: 100 + }) + state.applyRemoteEngineProgress({ + node: peer, + engine: 'llamacpp', + op: 'install', + stage: 'already-installed', + percent: 100 + }) + for (const nodeId of [self, peer]) { + expect( + state + .getEngineInitialState() + .statuses.find(s => s.nodeId === nodeId && s.engineType === 'llamacpp') + ).toMatchObject({ + processStatus: running ? 'running' : 'stopped', + managed: true, + installSupported: true, + installReason: 'Supported', + enginePort: 8082 + }) + expect(cleared).toContain(nodeId + ':llamacpp:install') + } + expect( + state + .getEngineInitialState() + .models.find(m => m.nodeId === self && m.engineType === 'llamacpp') + ?.models[0] + ).toMatchObject({ downloaded: true, status: running ? 'loaded' : 'idle' }) + } finally { + unsubscribe() + state.applyEngineManagerStatus(facts) + state.clearPendingRemoteEngineOp(peer, 'llamacpp') + } + } + ) + it('renders unknown progress without a false numeric percentage', () => { + for (const value of [undefined, -1, 101, NaN, Infinity]) { + expect(roundedProgressPercent(value)).toBeNull() + expect(formatPullProgressLabel({ status: 'Installing', percent: value })).toBe( + 'Installing' + ) + } + expect(roundedProgressPercent(0)).toBe(0) + expect(roundedProgressPercent(12.5)).toBe(13) + expect(roundedProgressPercent(100)).toBe(100) + }) + it('preserves authoritative ownership/support through local and remote pending operations', () => { + const state = getModularBridgeState() + const self = 'llama-pending-self' + const peer = 'llama-pending-peer' + const facts = { + engine: 'llamacpp', + installed: true, + running: false, + port: 8082, + managed: true, + install_supported: true, + install_reason: 'Supported inventory' + } + state.setSelfId(self) + state.handleNotification({ + source: 'broker', + method: 'discovery:nodes-changed', + params: { + nodes: [ + { + hostUuid: peer, + name: 'peer', + port: 14318, + modelsByEngine: { llamacpp: ['cached-peer-model'] } + } + ] + } + }) + state.applyEngineManagerStatus(facts) + state.applyRemoteEngineFacts(peer, { engines: [facts] }) + state.beginLocalEngineOp('llamacpp', 'installing') + state.beginRemoteEngineOp(peer, 'llamacpp', 'installing') + try { + for (const nodeId of [self, peer]) { + expect( + state + .getEngineInitialState() + .statuses.find(s => s.nodeId === nodeId && s.engineType === 'llamacpp') + ).toMatchObject({ + processStatus: 'installing', + managed: true, + installSupported: true, + installReason: 'Supported inventory' + }) + } + const peerModels = () => + state + .getEngineInitialState() + .models.find(m => m.nodeId === peer && m.engineType === 'llamacpp')?.models + expect(peerModels()?.[0]).toMatchObject({ downloaded: true, status: 'idle' }) + state.applyRemoteEngineFacts(peer, { engines: [{ ...facts, managed: false }] }) + expect(peerModels()?.[0]).toMatchObject({ downloaded: false, status: 'idle' }) + } finally { + state.applyEngineManagerStatus(facts) + state.clearPendingRemoteEngineOp(peer, 'llamacpp') + } + }) + it('keeps equal request counters distinct across engines and proxy runs', () => { + const state = getModularBridgeState() + state.clearWorkloads() + const base = { + id: '1', + model: 'model', + state: 'running', + originatedFrom: 'self', + createdAt: 1 + } + state.upsertWorkloadFromInfo({ workloadInfo: { ...base, engine: 'ollama', runId: 'a' } }) + state.upsertWorkloadFromInfo({ workloadInfo: { ...base, engine: 'llamacpp', runId: 'a' } }) + state.upsertWorkloadFromInfo({ workloadInfo: { ...base, engine: 'llamacpp', runId: 'b' } }) + expect(Object.values(state.getWorkloads())).toHaveLength(3) + state.removeWorkloadFromParams({ + workloadId: '1', + originatedFrom: 'self', + engine: 'llamacpp', + runId: 'a' + }) + expect(Object.values(state.getWorkloads())).toHaveLength(2) + }) + it('preserves installation support and cached downloads while stopped without stale loaded state', () => { + const state = getModularBridgeState() + const nodeId = 'llama-state-self' + state.setSelfId(nodeId) + state.handleNotification({ + source: 'llamacpp-proxy', + method: 'node/discovered', + params: { id: nodeId, port: 8080 } + }) + state.applyEngineManagerStatus({ + engine: 'llamacpp', + installed: true, + running: true, + port: 8082, + managed: true, + install_supported: true + }) + state.applyLocalLoadedModels({ llamacpp: ['owner/model:Q4'] }) + state.setLocalEngineModels('llamacpp', ['owner/model:Q4']) + const models = () => + state + .getEngineInitialState() + .models.find(m => m.nodeId === nodeId && m.engineType === 'llamacpp')?.models + expect(models()?.[0]).toMatchObject({ downloaded: true, status: 'loaded' }) + state.applyEngineManagerStatus({ + engine: 'llamacpp', + installed: true, + running: false, + port: 8082, + managed: true, + install_supported: false, + install_reason: 'Unsupported platform' + }) + expect(models()?.[0]).toMatchObject({ downloaded: true, status: 'idle' }) + expect( + state + .getEngineInitialState() + .statuses.find(s => s.nodeId === nodeId && s.engineType === 'llamacpp') + ).toMatchObject({ + managed: true, + installSupported: false, + installReason: 'Unsupported platform' + }) + state.setLocalEngineModels('llamacpp', ['catalogue-only'], false) + expect(models()?.[0]).toMatchObject({ downloaded: false, status: 'idle' }) + }) +}) diff --git a/desktop/tests/modular/lmstudio-stale-model.test.ts b/desktop/tests/modular/lmstudio-stale-model.test.ts index 8b37e862..d22e7440 100644 --- a/desktop/tests/modular/lmstudio-stale-model.test.ts +++ b/desktop/tests/modular/lmstudio-stale-model.test.ts @@ -16,6 +16,51 @@ import { } from '@/electron/service-bridge/modular-supervisor' describe('LM Studio model reconciliation', () => { + it('loads the native llama downloaded inventory into actionable local model rows', async () => { + const state = getModularBridgeState() + const supervisor = getModularSupervisor() + const nodeId = 'llama-native-inventory-self' + const facts = { + engine: 'llamacpp', + installed: true, + running: true, + port: 8082, + managed: true, + install_supported: true + } + state.setSelfId(nodeId) + state.applyEngineManagerStatus(facts) + const hasProcess = vi.spyOn(supervisor, 'hasProcess').mockReturnValue(true) + const call = vi + .spyOn(supervisor, 'callProcess') + .mockResolvedValue({ data: [{ id: 'owner/cached:Q4', status: { value: 'unloaded' } }] }) + try { + supervisor.refreshManagedEngineModels(facts) + await vi.waitFor(() => + expect( + state + .getEngineInitialState() + .models.find(m => m.nodeId === nodeId && m.engineType === 'llamacpp') + ?.models[0] + ).toMatchObject({ name: 'owner/cached:Q4', downloaded: true, status: 'idle' }) + ) + expect(call).toHaveBeenCalledWith('broker', 'engine:action', { + engine: 'llamacpp', + action: 'list_downloaded' + }) + expect(parseListModelNames({ data: [] })).toEqual([]) + expect(() => parseListModelNames({ data: [{}] })).toThrow('no usable model names') + } finally { + supervisor.refreshManagedEngineModels({ + engine: 'llamacpp', + installed: false, + running: false, + managed: false + }) + call.mockRestore() + hasProcess.mockRestore() + } + }) it('parses the native inventory and distinguishes explicit empty from unknown', () => { expect( parseListModelNames({ diff --git a/desktop/tests/modular/uuid-node-keying.test.ts b/desktop/tests/modular/uuid-node-keying.test.ts index 70092971..176d843e 100644 --- a/desktop/tests/modular/uuid-node-keying.test.ts +++ b/desktop/tests/modular/uuid-node-keying.test.ts @@ -206,8 +206,8 @@ describe('UUID node keying', () => { }) ) - const liveKey = 'uuid-wl-seed\u0000job-live' - const newKey = 'uuid-wl-seed\u0000job-new' + const liveKey = 'uuid-wl-seed\u0000job-live\u0000ollama\u0000' + const newKey = 'uuid-wl-seed\u0000job-new\u0000ollama\u0000' // The live entry is preserved, not clobbered by the older baseline row. expect(seeded[liveKey]).toMatchObject({ state: 'running', model: 'live-model' }) // A baseline job the stream had not delivered is filled in. From 7a178ae6eb86efc5f0bcfe4ada3f1bcfda5ac658 Mon Sep 17 00:00:00 2001 From: pgoode41 Date: Tue, 22 Sep 2026 03:44:38 -0400 Subject: [PATCH 04/13] Document llama.cpp and teach the tooling about it Describe the managed llama.cpp engine in the product docs: install, lifecycle, model actions, loaded-only routing, launch settings, data retention on uninstall, and the terminal interface keys. The inference dispatcher recognises the engine, the app-data wipe scripts preserve an unmigrated llama model library instead of deleting it, and shell scripts are checked out with LF endings so Windows-built packages ship executable Unix scripts. Co-authored-by: Terve Co-Authored-By: Claude Fable 5.1 Signed-off-by: pgoode41 --- .gitattributes | 5 + README.md | 7 +- docs/architecture.mdx | 24 ++--- docs/engine-lifecycle.mdx | 100 ++++++++++++++++-- docs/engine-settings.mdx | 7 +- docs/terminal-interface.mdx | 30 ++++-- scripts/inference-dispatcher/client.go | 12 ++- scripts/inference-dispatcher/config.go | 14 ++- .../inference-dispatcher/dispatcher_test.go | 46 ++++++++ scripts/wipe-app-data.ps1 | 17 ++- scripts/wipe-app-data.sh | 17 ++- 11 files changed, 233 insertions(+), 46 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..a727c7f1 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Windows-built packages must ship executable Unix shell scripts. +*.sh text eol=lf diff --git a/README.md b/README.md index 757c3f39..0e2930ad 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ one, and both report live GPU and memory use throughout. | **Architectures** | x64 and arm64 on all three. Windows on ARM is experimental. | | **Installers** | Windows `.exe`; Linux `.deb`; macOS `.dmg`. On other Linux distributions, [build from source](docs/building.mdx). | | **Mixing nodes** | Windows, Linux, and macOS nodes can all be paired with each other | -| **Inference engines** | Ollama and LM Studio | +| **Inference engines** | Ollama, LM Studio and managed llama.cpp | **PAIR running on a machine does not mean an engine will.** PAIR itself runs on any supported Windows, Linux, or macOS machine. Each engine sets its own requirements @@ -49,6 +49,11 @@ before assuming a node can serve a model. A node only becomes a candidate for a request once it is actually running a compatible engine, and PAIR prefers the nodes it already knows hold the model. +Managed llama.cpp provides an official CPU app for Intel Macs and confirmed +non-NVIDIA Windows ARM hardware. NVIDIA Windows ARM remains CUDA-required; +a failed driver or hardware query does not silently select CPU. Apple Silicon +uses the official Metal app. See the [managed engine requirements and policies](services/nvpair-engine-manager/README.md#managed-llama-app). + ## Quick start Download a released build and use the desktop application. That is the path we diff --git a/docs/architecture.mdx b/docs/architecture.mdx index d49e4a3e..5ad71feb 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -40,8 +40,8 @@ flowchart TB one cluster. - A **node** is one machine. Nodes are peers where each runs the same services, and each can both serve requests and route them elsewhere. -- An **engine** is an inference server on a node, Ollama or LM Studio. A node can - run both, one, or neither, and a node with no running engine is not eligible to +- An **engine** is an inference server on a node: Ollama, LM Studio or llama.cpp. + A node can run several, one, or none, and a node with no running engine is not eligible to serve. - **Models** belong to an engine on a specific node. Nothing is shared. The same model on two nodes is two independent copies and that duplication is what makes @@ -418,17 +418,16 @@ the same node. are the same unit of pending work, so "fewest jobs" is not "least busy." A node running one enormous request looks more idle than a node running two trivial ones. -**Model load state is not considered.** Eligibility asks whether a node *has* the -model, not whether it is already loaded in memory. PAIR knows which models are -loaded, and the interface shows it, but routing does not use it, so a request can -be sent to a node that must cold-load the model while a node holding it warm sits -one place lower in the order. +**Load eligibility depends on the engine.** llama.cpp routing requires the model +to be reported loaded; downloaded-only or unknown residency is ineligible. +Ollama and LM Studio can cold-load an advertised model, so their routing does +not prefer a warm copy over a cold copy. **Only work PAIR routed contributes to pending counts.** Inference sent straight to an engine's own port is absent from workload events. GPU-heavy external work can still raise pressure, but CPU-only work and queued demand remain invisible. -**Both engines are counted as one pool.** Ollama and LM Studio load is summed, +**All engines are counted as one pool.** Ollama, LM Studio and llama.cpp load is summed, and maximum GPU pressure applies to the whole node. That is conservative on a typical single-GPU machine and can underuse a multi-GPU node where the engines occupy different devices. @@ -712,7 +711,7 @@ desktop build compiles the sibling tree directly. | Build In | Output | Contents | | --- | --- | --- | | `desktop/` | `desktop/cli-bin/` | What the app supervises, one OS/arch | -| `services/` | `services/build/bin/` | All 13 executables, for standalone use | +| `services/` | `services/build/bin/` | All 14 executables, for standalone use | `desktop/scripts/build-modular-binaries.ts` compiles the runtime inventory for a selected target and writes a manifest recording the source identity, versions, @@ -723,9 +722,10 @@ stale or hand-placed binary is caught rather than silently supervised. stamped from `versions.json`. Building one component by hand without restaging is the one thing to avoid: the broker keeps running whatever is in `build/bin/`. -Both paths produce binaries you run on the machine that built them. This tree -provides build materials only. There is no packaging, installer, or signing step -here, and installable builds come from the +Both paths support target-specific binaries. The desktop build scripts can also +produce unsigned reference installers with the public electron-builder config. +Signing and publication belong to NVIDIA's release pipeline; supported released +installers come from the [releases page](https://github.com/NVIDIA/Personal-AI-Router/releases). Refer to [Building PAIR](building.mdx). diff --git a/docs/engine-lifecycle.mdx b/docs/engine-lifecycle.mdx index 0d492277..a5014e05 100644 --- a/docs/engine-lifecycle.mdx +++ b/docs/engine-lifecycle.mdx @@ -6,7 +6,7 @@ SPDX-License-Identifier: Apache-2.0 # Managing Engines in NVIDIA Personal AI Router An **engine** is the local inference runtime Personal AI Router (PAIR) uses to -run models. Today that means Ollama or LM Studio on a given machine. PAIR can +run models. That includes Ollama, LM Studio and llama.cpp on a given machine. PAIR can install and run those engines for you, or work with a copy you already have. This page explains what you can expect when you install, start, stop, update, or remove an engine. @@ -18,6 +18,73 @@ an engine's ports or launch command, refer to ## Engine Requirements +### Managed llama app + +Engine Manager installs the official llama app into its own runtime directory +and starts its router one port above the endpoint your clients use +(`http://127.0.0.1:8080/v1`), so llama.cpp's usual address keeps working and +requests go through routing. If you have set `LLAMA_ARG_PORT`, that port is used +instead. Install eligibility is reported by the backend for the current +platform. A detected external runtime remains read-only. + +On NVIDIA Windows ARM64, Install first tries the current official upstream +installer and release in a separate staging directory. The candidate must report +the selected version, readable licenses and a usable CUDA device. If that bounded +attempt fails, PAIR uses its checksum-pinned b10826 app and CUDA 13.4 runtime +archives. Cancel stops the operation instead of starting a fallback attempt. +Install on an already-installed managed runtime reports it as already installed +and changes nothing. There is no update action for managed llama, and PAIR does +not uninstall and reinstall it as a substitute for one. + +This primary path trusts current official upstream downloads; recording their +hashes is not prior PAIR qualification or a guarantee about future releases. +The fallback remains the same official app and CLI, not a separate legacy engine. +Its ARM64 CUDA support is an upstream preview and requires a compatible NVIDIA +driver; PAIR does not install a toolkit or change drivers. Windows x64, Linux +and Apple Silicon retain the pinned official script. + +Confirmed non-NVIDIA Windows ARM64 hardware uses the pinned official CPU installer. +Failed or incomplete hardware detection does not select CPU. Intel Macs use the +official pinned CPU app archive; this does not provide Radeon acceleration. +Both CPU paths use the same managed model, retention and uninstall controls. +Native hardware qualification is separate from having an installation recipe. + +Download by Hugging Face reference (`owner/repository:QUANT`) or import a local +GGUF in the engine's model controls. The managed cache survives engine uninstall, +application uninstall and reset. It lives in `Nvidia Corporation/Personal AI +Router Models/llamacpp` under the platform configuration base: `%LOCALAPPDATA%` +on Windows, `~/Library/Application Support` on macOS, or `$XDG_CONFIG_HOME` +(default `~/.config`) on Linux. This library is separate from PAIR application +data. Engine Manager migrates an older in-app cache before using it; if migration +cannot safely finish, reset refuses and uninstall preserves the old data root. +Open the updated app and resolve the migration error before retrying data removal. +Download completion does not mean a model is loaded: load it explicitly before +sending inference through PAIR. + +The desktop workload card and TUI can cancel a local-origin llama request. +Cancellation stops its forwarding context and the workload reports its final +state; the response accepting cancellation is not a vendor stop receipt. +PAIR routes each complete request to one eligible node and does not distribute +a model across several machines. Native platform and packaged UI validation of +this integration must be recorded separately from passing source tests. + +Use a PAIR build with managed llama support on each node you intend to manage. +Older Engine Managers reject unknown llama commands; older desktop and terminal +builds do not gain the new controls from upgrading only a peer. Upgrade the +application and its bundled services together. Existing model files remain +separate from that application update. + +The managed default serves chat and text completions. The embeddings route can +forward a request, but a runtime not configured for embeddings returns the +vendor's error; route availability is not a promise of that model capability. +Likewise, a detected GPU is not proof that the vendor selected it for inference. +Use the actual runtime/device observation when reporting acceleration. + +PAIR discovery carries configured engine ports. The generic manual-node prober +uses each engine's documented default port (llama: `8082`), like the other +engines; the proxy's direct `node/add-manual` API accepts an explicit port. +Do not treat a manual default-port probe as discovery of an arbitrary port. + PAIR does not run models by itself. Each participating machine needs at least one compatible engine that is: @@ -44,8 +111,8 @@ operations run. These actions need the UI: -- Listing, loading, ejecting, and deleting models -- Updating an engine +- Changing an engine's port +- Updating an engine (Ollama or LM Studio; managed llama has no update action) - Managing another node's engines In the terminal: @@ -59,6 +126,8 @@ handles these engine lifecycle actions: - Restart an engine. - Uninstall an engine. - Pull a model. +- List models and use supported load, eject, delete, and cancel controls. +- Import a local GGUF into managed llama's model cache. It does not do everything that the application does. @@ -114,10 +183,20 @@ Both actions apply only to an engine PAIR installed: - **Uninstall** removes the NVPAIR-installed copy of that engine. Use it when you no longer want PAIR to own that engine on the machine. -Uninstalling an engine does **not** remove its models. Downloaded model files stay -on disk, in the engine's own storage such as `~/.ollama`, so uninstalling and -reinstalling does not cost you re-downloading them. To reclaim that space, delete -the models through the engine, or remove its data directory yourself. +The shared managed-engine standard is **runtime-only removal**: stop the correct +PAIR-owned instance and remove its runtime, keeping normal separate models, +settings and user data. Unknown external/shared ownership must result in refusal, +not a broad cleanup. The exact recipe's native evidence establishes compliance; +this standard is not a promise about unqualified legacy installations. + +Managed llama follows that standard: uninstall removes its runtime slots but +keeps its model library and separate settings, while saving Off. Reinstall can +reuse the retained models. Profile reset and model deletion are different actions +and are not part of runtime uninstall. Update is not offered for managed llama, +and PAIR does not run uninstall and reinstall in its place. The backend +[managed install/uninstall contract](../services/nvpair-engine-manager/README.md#managed-installuninstall-contract) +defines the same requirement for Ollama and LM Studio recipes. It does not change +their engine-specific start/stop, port or update behavior. Port changes for an NVPAIR-installed engine also live in **Engine settings**. Prefer the controls in PAIR over editing the engine's own config when PAIR is @@ -188,9 +267,10 @@ brings it back without fetching it again. ![An engine's model list showing the Load, Eject, and Delete actions against a downloaded model.](assets/onboarding/engine-lifecycle/04-model-actions.png) On a headless machine the terminal interface covers installing, starting, -stopping, restarting, and uninstalling an engine, and pulling a model. Operations -it does not have, such as deleting a model or updating an engine, need the -desktop application on that machine. +stopping, restarting, and uninstalling an engine, plus supported model inventory, +download, load, unload, and delete actions. Managed llama also supports download +cancellation and local GGUF import. See the +[terminal controls](terminal-interface.mdx) for exact keys and limitations. PAIR does **not** warm models. Starting an engine does not pre-load models into memory, and it holds nothing ready in advance. Depending on the engine, a model diff --git a/docs/engine-settings.mdx b/docs/engine-settings.mdx index 61ceb7e1..6235207a 100644 --- a/docs/engine-settings.mdx +++ b/docs/engine-settings.mdx @@ -5,7 +5,7 @@ SPDX-License-Identifier: Apache-2.0 # Engine Settings in NVIDIA Personal AI Router -Open a device's Ollama or LM Studio row and expand **Settings**. Edit the server +Open a device's Ollama, LM Studio or llama.cpp row and expand **Settings**. Edit the server port, proxy port and engine arguments, then select the green **Apply** button. A paired device that supports settings can be edited from another device in the cluster. @@ -78,6 +78,7 @@ configuration consistent: | --- | --- | --- | --- | | Ollama | `OLLAMA_HOST=127.0.0.1:` | The host in `OLLAMA_HOST` | `OLLAMA_ORIGINS` | | LM Studio | `--port`, `-p` | `--bind`, `LMS_SERVER_HOST` | `--cors` | +| llama.cpp | `--port` | `--host` | none; PAIR follows the engine's own responses | For LM Studio, `--port 1235`, `--port=1235`, `-p 1235`, `-p1235` and `-p=1235` all synchronize the same server-port field. Conflicting repetitions and invalid @@ -86,6 +87,10 @@ them are ambiguous and rejected. CORS switches do not take `=true` or `=false`: add or remove the switch locally. Origin lists are normalized before validation; conflicting repeated lists are rejected. Other vendor options remain opaque. +For managed llama.cpp, `serve --no-models-autoload` stays fixed so routing keeps +treating only loaded models as eligible, and PAIR owns the model cache location: +`LLAMA_CACHE` and `HF_HUB_CACHE` assignments are rejected in launch settings. + Changing the server-port field updates the port in the command. Editing the command's port updates the numeric field. If both are edited before validation finishes, the numeric field takes precedence. The server and proxy must use diff --git a/docs/terminal-interface.mdx b/docs/terminal-interface.mdx index 1eb99b5f..21427768 100644 --- a/docs/terminal-interface.mdx +++ b/docs/terminal-interface.mdx @@ -13,8 +13,8 @@ desktop environment, or over SSH, where the desktop application cannot run. If a desktop is available, use the desktop application. **It has known limitations.** It is an operations tool, not a full replacement. -It cannot list or delete models, change an engine's port, update an engine, -control engines on other nodes, or show which node served a workload. The full +It cannot change an engine's port, update an engine, or control engines on +other nodes. Managed llama supports model inventory and actions. The full list is in [What the Terminal Interface Cannot Do](#what-the-terminal-interface-cannot-do), and it is worth reading before you depend on it. @@ -130,7 +130,7 @@ Tab switching and `q` do not work until you do. | 1 | **Overview** | Service uptime and version, and an `ok` / `DOWN` table for each worker | | 2 | **Errors** | Active service errors by severity, age, node, and message | | 3 | **Nodes** | Nodes discovered on the network, with `Connected` or `In cluster` status | -| 4 | **Proxies** | Both compatible proxies: listening port, discovered upstreams, and which node is selected | +| 4 | **Proxies** | Ollama, LM Studio and llama.cpp proxies: listening port, discovered upstreams, and which node is selected | | 5 | **Workloads** | Live inference workloads: ID, model, engine, state, and age | | 6 | **Engines** | Local engines: installed, running, healthy, and port | | 7 | **Cluster** | This node's identity, cluster membership, and pairing | @@ -197,10 +197,20 @@ From the **Engines** tab (6), select an engine with `j` / `k`, then: | `r` | Restart | | `u` | Uninstall | | `p` | Download a model | +| `m` | Show model inventory; arrows select a model, `esc` returns | +| `L` / `e` / `d` | Load / unload / delete the selected model where supported | +| `c` | Cancel the selected model download | +| `I` | Import a local single-file GGUF into managed llama's cache | Pressing `p` opens a prompt. Type the model name, for example `qwen4:12b`, and press `enter`. Progress appears on the status line. +For llama, enter `owner/repository:QUANT` for a Hugging Face download, or use +`I` with a local GGUF path. Model actions prefill the selected model's exact ID; +press `enter` to submit. Start llama and explicitly load a downloaded model +before sending inference. Downloaded weights alone do not make it routable. +External llama runtimes remain under their owner's lifecycle control. + A node can serve a request only when it is online, a compatible engine is running, and the requested model is present on that node. To route across several machines, download the same model on each. @@ -208,13 +218,16 @@ machines, download the same model on each. ## Check Routing and Health The **Workloads** tab (5) lists inference work as it runs, with its model, -engine, and state. +engine, and state. Select a row to see its origin and reported serving node; +an absent target remains unknown. Press `c` to request cancellation of an active +local-origin llama request. The terminal workload event determines the outcome; +acceptance of cancellation does not prove a vendor stop acknowledgement. ![The terminal interface Workloads tab listing live inference workloads with model, engine, state, and age.](assets/onboarding/terminal-interface/03-tui-workloads.png) The **Proxies** tab (4) shows each proxy's listening port and whether it is routing automatically (`selected=auto`) or pinned to one node. Press `g` to -switch between the two engines, `enter` to pin the highlighted upstream, and `a` +switch between the three engines, `enter` to pin the highlighted upstream, and `a` to return to automatic routing. Leave it on automatic unless you are deliberately testing one node. @@ -243,12 +256,11 @@ for how PAIR arranges ports. It is an operations tool, not a full replacement for the desktop application: -- It cannot list or delete models. You can download one, but the interface shows - no model inventory. - It cannot change an engine's port. The port column is read-only. Use the desktop application to change it. -- It cannot update an engine or control engines on other cluster nodes. -- It does not show which node served a particular workload. +- It cannot update an engine or control engines on other cluster nodes. Managed + llama has no update action anywhere in PAIR; uninstall and reinstall are + separate, explicit steps and are never run on your behalf as a substitute. - It has no built-in way to send an inference request. Use `curl` or another client against the proxy port, as in [Getting Started](getting-started.mdx#6-run-your-first-inference). diff --git a/scripts/inference-dispatcher/client.go b/scripts/inference-dispatcher/client.go index a759e973..da585444 100644 --- a/scripts/inference-dispatcher/client.go +++ b/scripts/inference-dispatcher/client.go @@ -111,7 +111,7 @@ func decodeObject(data []byte, target any) error { func (c *backendClient) listModels(ctx context.Context) ([]RegisteredModel, error) { var models []RegisteredModel var err error - if c.cfg.Backend == "lmstudio" { + if usesOpenAIAPI(c.cfg.Backend) { models, err = c.listLMStudioModels(ctx) } else { models, err = c.listOllamaModels(ctx) @@ -340,8 +340,12 @@ func (c *backendClient) resolveModel(ctx context.Context) (string, []RegisteredM return "", models, errors.New("no available model advertises text-generation support") } +func usesOpenAIAPI(backend string) bool { + return backend == "lmstudio" || backend == "llamacpp" +} + func (c *backendClient) inferencePath() string { - if c.cfg.Backend == "lmstudio" { + if usesOpenAIAPI(c.cfg.Backend) { return "/v1/chat/completions" } return "/api/generate" @@ -349,7 +353,7 @@ func (c *backendClient) inferencePath() string { func (c *backendClient) infer(ctx context.Context, model, prompt string) (string, error) { var payload map[string]any - if c.cfg.Backend == "lmstudio" { + if usesOpenAIAPI(c.cfg.Backend) { payload = map[string]any{ "model": model, "messages": []map[string]string{{"role": "user", "content": prompt}}, @@ -386,7 +390,7 @@ func (c *backendClient) infer(ctx context.Context, model, prompt string) (string if err != nil { return "", err } - if c.cfg.Backend == "lmstudio" { + if usesOpenAIAPI(c.cfg.Backend) { return parseLMStudioResponse(data) } var response struct { diff --git a/scripts/inference-dispatcher/config.go b/scripts/inference-dispatcher/config.go index 2598dcbc..543b5fef 100644 --- a/scripts/inference-dispatcher/config.go +++ b/scripts/inference-dispatcher/config.go @@ -23,6 +23,11 @@ const ( // PAIR's managed LM Studio backend is moved behind it starting at 1235; // pass --port explicitly to reach that directly, which bypasses routing. defaultLMStudioPort = 1234 + // llama.cpp's own default, which is also the port PAIR's llama.cpp proxy + // facade claims (the FacadePort in services/shared/engines). + // PAIR's managed llama.cpp backend is moved behind it starting at 8081; + // pass --port explicitly to reach that directly, which bypasses routing. + defaultLlamaCppPort = 8080 defaultErrorLog = "inference_errors.txt" maxPromptsPerBatch = 100 ) @@ -289,7 +294,7 @@ func parseConfig(args []string, stderr io.Writer) (Config, error) { fs.SetOutput(stderr) var parsedConfigPath string fs.StringVar(&parsedConfigPath, "config", configPath, "JSON configuration file") - fs.StringVar(&cfg.Backend, "backend", cfg.Backend, "backend: ollama or lmstudio") + fs.StringVar(&cfg.Backend, "backend", cfg.Backend, "backend: ollama, lmstudio, or llamacpp") fs.StringVar(&cfg.Backend, "provider", cfg.Backend, "alias for --backend") fs.IntVar(&cfg.Port, "port", cfg.Port, "server port (backend default when omitted)") fs.StringVar(&cfg.Model, "model", cfg.Model, "model name; omitted or auto selects an available model") @@ -356,8 +361,8 @@ func parseConfig(args []string, stderr io.Writer) (Config, error) { } func validateConfig(cfg Config) error { - if cfg.Backend != "ollama" && cfg.Backend != "lmstudio" { - return errors.New("--backend must be ollama or lmstudio") + if cfg.Backend != "ollama" && cfg.Backend != "lmstudio" && cfg.Backend != "llamacpp" { + return errors.New("--backend must be ollama, lmstudio, or llamacpp") } if cfg.Port < 0 || cfg.Port > 65535 { return errors.New("--port must be between 1 and 65535") @@ -417,5 +422,8 @@ func effectivePort(cfg Config) int { if cfg.Backend == "lmstudio" { return defaultLMStudioPort } + if cfg.Backend == "llamacpp" { + return defaultLlamaCppPort + } return defaultOllamaPort } diff --git a/scripts/inference-dispatcher/dispatcher_test.go b/scripts/inference-dispatcher/dispatcher_test.go index c858fcc7..9ddcc06d 100644 --- a/scripts/inference-dispatcher/dispatcher_test.go +++ b/scripts/inference-dispatcher/dispatcher_test.go @@ -334,6 +334,52 @@ func TestLMStudioDefaultPort(t *testing.T) { } } +func TestBackendLlamaCpp(t *testing.T) { + var stderr bytes.Buffer + cfg, err := parseConfig([]string{"--backend", "llamacpp", "--prompt", "hi"}, &stderr) + if err != nil { + t.Fatal(err) + } + if cfg.Backend != "llamacpp" { + t.Fatalf("backend = %q", cfg.Backend) + } + if port := effectivePort(cfg); port != defaultLlamaCppPort { + t.Fatalf("port = %d, want %d", port, defaultLlamaCppPort) + } +} + +func TestLlamaCppUsesOpenAIInventory(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/models": + http.Error(w, "not found", http.StatusNotFound) + case "/v1/models": + _, _ = w.Write([]byte(`{"data":[{"id":"gguf-model"}]}`)) + case "/v1/chat/completions": + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"done"}}]}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + var stdout, stderr bytes.Buffer + exit := runAgainstServer( + t, + context.Background(), + []string{"--backend", "llamacpp", "--prompt", "test"}, + server.URL, + &stdout, + &stderr, + ) + if exit != 0 { + t.Fatalf("exit=%d stderr=%s", exit, stderr.String()) + } + if !strings.Contains(stdout.String(), "gguf-model") { + t.Fatalf("selected model missing from output: %s", stdout.String()) + } +} + func TestResponseTextNeverReachesStdout(t *testing.T) { const secret = "the capital of France is Paris" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/scripts/wipe-app-data.ps1 b/scripts/wipe-app-data.ps1 index 4f2caeb7..11d3172a 100644 --- a/scripts/wipe-app-data.ps1 +++ b/scripts/wipe-app-data.ps1 @@ -23,7 +23,8 @@ # - desktop/scripts/build/{installer.nsh,linux/after-remove.sh,macos/uninstall.sh} # - scripts/wipe-app-data.sh (Unix twin — update both in the same change) # -# Explicit exclusions (never add): %USERPROFILE%\.ollama, .lmstudio, external +# Explicit exclusions (never add): the sibling Personal AI Router Models directory, +# %USERPROFILE%\.ollama, .lmstudio, external # engine installs, and the application install tree (Program Files\PAIR). # --------------------------------------------------------------------------- @@ -51,7 +52,7 @@ Usage: scripts\wipe-app-data.cmd [options] Delete all Personal AI Router-owned application data (settings, logs, cluster identity, chat history, PAIR-managed engines under the app data root). -Does NOT delete third-party model libraries (e.g. %USERPROFILE%\.ollama). +Does NOT delete model libraries, including Personal AI Router Models\llamacpp. Does NOT uninstall the application binary. Options: @@ -168,6 +169,16 @@ if ($DryRun) { exit 0 } +# Engine Manager alone migrates legacy llama weights to the retained library. +# Never wipe an unmigrated cache, including a redirected models directory. +foreach ($root in @($CurrentRoot, $LegacyRoot)) { + $legacyModels = Join-Path $root 'engine-bin\llamacpp\models' + if (Get-Item -LiteralPath $legacyModels -Force -ErrorAction SilentlyContinue) { + Write-Error "Llama models remain under app data. Open the updated app to migrate them before resetting: $legacyModels" + exit 1 + } +} + # The app spawns this script detached and then exits, so the wipe must happen # after its process is gone. Otherwise Chromium flushes session/cache files back # into the directory we just deleted and the "clean" relaunch is not clean. @@ -225,7 +236,7 @@ if (-not $Confirmed) { } Write-Host '' Write-Host 'WARNING: This permanently deletes all Personal AI Router app data.' - Write-Host 'Third-party model libraries (e.g. %USERPROFILE%\.ollama) are NOT removed.' + Write-Host 'Model libraries, including Personal AI Router Models\llamacpp, are NOT removed.' Write-Host '' Write-Host 'Paths to remove:' foreach ($t in $Targets) { Write-Host (" - {0}" -f $t.Path) } diff --git a/scripts/wipe-app-data.sh b/scripts/wipe-app-data.sh index 659393d2..04c66d6e 100755 --- a/scripts/wipe-app-data.sh +++ b/scripts/wipe-app-data.sh @@ -24,7 +24,8 @@ # - desktop/scripts/build/{installer.nsh,linux/after-remove.sh,macos/uninstall.sh} # - scripts/wipe-app-data.ps1 (Windows twin — update both in the same change) # -# Explicit exclusions (never add): ~/.ollama, ~/.lmstudio, external engine +# Explicit exclusions (never add): the sibling Personal AI Router Models directory, +# ~/.ollama, ~/.lmstudio, external engine # installs, and the application install tree (Program Files / /opt/PAIR / # PAIR.app). # --------------------------------------------------------------------------- @@ -45,7 +46,7 @@ Usage: scripts/wipe-app-data.sh [options] Delete all Personal AI Router-owned application data (settings, logs, cluster identity, chat history, PAIR-managed engines under the app data root). -Does NOT delete third-party model libraries (e.g. ~/.ollama, ~/.lmstudio). +Does NOT delete model libraries, including Personal AI Router Models/llamacpp. Does NOT uninstall the application binary. Options: @@ -172,6 +173,16 @@ if [[ "$DRY_RUN" -eq 1 ]]; then exit 0 fi +# Engine Manager owns cache migration. Refuse even a dangling link rather than +# deleting a legacy library when the updated app has not migrated it yet. +for root in "$CURRENT_ROOT" "$LEGACY_ROOT"; do + legacy_models="$root/engine-bin/llamacpp/models" + if [[ -e "$legacy_models" || -L "$legacy_models" ]]; then + echo "Llama models remain under app data. Open the updated app to migrate them before resetting: $legacy_models" >&2 + exit 1 + fi +done + # The app spawns this script detached and then exits, so the wipe must happen # after its process is gone. Otherwise Chromium flushes session/cache files back # into the directory we just deleted and the "clean" relaunch is not clean. @@ -215,7 +226,7 @@ if [[ "$CONFIRM" -eq 0 ]]; then fi echo "" echo "WARNING: This permanently deletes all Personal AI Router app data." - echo "Third-party model libraries (e.g. ~/.ollama, ~/.lmstudio) are NOT removed." + echo "Model libraries, including Personal AI Router Models/llamacpp, are NOT removed." echo "" echo "Paths to remove:" for entry in "${TARGETS[@]}"; do From 56e837d924323441f8891caa65ca286124fbeb6b Mon Sep 17 00:00:00 2001 From: pgoode41 Date: Tue, 22 Sep 2026 04:39:45 -0400 Subject: [PATCH 05/13] Focus workload display and repair engine event consumers Defer optional desktop/TUI inference-cancel controls and expanded workload browsing while retaining full workload identity and engine/model controls. Restore the public-base live workload table. Consume llama proxy relay events, nested Engine Manager residency events, and LM Studio model keys. Cover the real consumer boundaries with socket-free regressions for all three engines. Signed-off-by: pgoode41 --- desktop/docs/services-api.md | 3 - desktop/docs/services-parity.md | 8 + .../electron/service-bridge/empty-handlers.ts | 19 +-- .../electron/service-bridge/modular-state.ts | 6 +- desktop/src/shared/types/workloads.ts | 6 +- desktop/src/shared/types/ws-channels.ts | 4 - desktop/src/shared/utils/workloads.ts | 4 +- desktop/src/ui/api/pair-api.ts | 8 - .../components/Workloads/WorkloadItemCard.tsx | 24 +-- .../tests/modular/engine-command-load.test.ts | 17 --- .../modular/llamacpp-proxy-events.test.ts | 75 ++++++++++ .../tests/modular/workloads-display.test.ts | 120 +++++++++++++++ services/nvpair-tui/README.md | 10 +- services/nvpair-tui/ui/engine_models_test.go | 141 ++++++++++++++++++ services/nvpair-tui/ui/engines.go | 10 +- services/nvpair-tui/ui/engines_test.go | 4 +- services/nvpair-tui/ui/workloads.go | 90 ++--------- services/nvpair-tui/ui/workloads_test.go | 102 +++++++++++-- 18 files changed, 473 insertions(+), 178 deletions(-) create mode 100644 desktop/tests/modular/llamacpp-proxy-events.test.ts create mode 100644 desktop/tests/modular/workloads-display.test.ts create mode 100644 services/nvpair-tui/ui/engine_models_test.go diff --git a/desktop/docs/services-api.md b/desktop/docs/services-api.md index 5ddb43c4..4c2e914f 100644 --- a/desktop/docs/services-api.md +++ b/desktop/docs/services-api.md @@ -41,7 +41,6 @@ - ⚠️ nvpair-proxy → facade/enable - ⚠️ nvpair-proxy → node/selected - ⚠️ nvpair-proxy → node/set-local-backend -- ⚠️ nvpair-proxy → workload/cancel - ⚠️ nvpair-ui-broker → discovery:unsubscribe - ⚠️ nvpair-ui-broker → engine:configure-launch - ⚠️ nvpair-ui-broker → engine:set-port @@ -227,7 +226,6 @@ | `node/set-local-backend` | request (we call) | ⚠️ not called | | `node/set-priority` | request (we call) | ✅ yes | | `nodes/list` | request (we call) | ✅ yes | -| `workload/cancel` | request (we call) | ⚠️ not called | **Dynamic / unresolved notify sites (verify by hand — `npm run service-contracts` prints the line numbers):** - `method (var) (proxy.go)` @@ -300,7 +298,6 @@ | `ollama-proxy:subscribe` | request (we call) | ✅ yes | | `ollama-proxy:unsubscribe` | request (we call) | ⚠️ not called | | `ready` | request (we call) | ✅ yes | -| `workloads:cancel` | request (we call) | ✅ yes | | `workloads:get-initial` | request (we call) | ✅ yes | | `workloads:remove` | request (we call) | ✅ yes | | `workloads:subscribe` | request (we call) | ✅ yes | diff --git a/desktop/docs/services-parity.md b/desktop/docs/services-parity.md index 9870945c..a6288c02 100644 --- a/desktop/docs/services-parity.md +++ b/desktop/docs/services-parity.md @@ -98,6 +98,14 @@ Manual nodes use the broker's `node/add`, `node/remove`, and `nodes/list` surface. Electron persists user entries and replays them after broker startup so they survive worker restarts. +### Workload display + +The desktop displays workload snapshots and live updates, retaining origin, +engine, proxy run and request identity. Execution labels and connection lines +use the reported destination, not the request origin. Workload cancellation is +not exposed by the desktop API or UI. Engine lifecycle and model-download +cancellation are separate controls and remain supported. + ### Multi-node UI acceptance Engine integration must preserve each participating desktop's view of the diff --git a/desktop/src/electron/service-bridge/empty-handlers.ts b/desktop/src/electron/service-bridge/empty-handlers.ts index f65b6d95..465832f4 100644 --- a/desktop/src/electron/service-bridge/empty-handlers.ts +++ b/desktop/src/electron/service-bridge/empty-handlers.ts @@ -787,24 +787,7 @@ const EMPTY_SERVICE_BRIDGE_HANDLERS: BridgeHandlerMap = { 'errors:get-initial': () => handleErrorsGetInitial(), 'errors:clear': payload => (payload ? handleErrorsClear(payload) : null), - 'workloads:get-initial': () => handleWorkloadsGetInitial(), - 'workloads:cancel': async payload => { - const supervisor = getModularSupervisor() - if (!payload) return { accepted: false } - try { - const result = objectValue( - await supervisor.callProcess('broker', 'workloads:cancel', payload) - ) - return { accepted: booleanValue(result?.accepted) } - } catch (err) { - supervisor.reportError( - `Cancel request failed: ${getErrorString(err)}`, - 'error', - 'llamacpp-cancel' - ) - return { accepted: false } - } - } + 'workloads:get-initial': () => handleWorkloadsGetInitial() } export function handleServiceBridgeInvoke( diff --git a/desktop/src/electron/service-bridge/modular-state.ts b/desktop/src/electron/service-bridge/modular-state.ts index b3e240d0..918a9019 100644 --- a/desktop/src/electron/service-bridge/modular-state.ts +++ b/desktop/src/electron/service-bridge/modular-state.ts @@ -47,7 +47,11 @@ type BrokerNodeSource = ProxyNodeSource | 'broker' * recorded here. That is `ComponentName` in `services/shared/engines`, always * `-proxy`. */ -export const PROXY_NODE_SOURCES: readonly ProxyNodeSource[] = ['ollama-proxy', 'lmstudio-proxy'] +export const PROXY_NODE_SOURCES: readonly ProxyNodeSource[] = [ + 'ollama-proxy', + 'lmstudio-proxy', + 'llamacpp-proxy' +] /** * Engines surfaced by the broker's proxy plane. Other engine-manager engines diff --git a/desktop/src/shared/types/workloads.ts b/desktop/src/shared/types/workloads.ts index 21b9be45..effc2584 100644 --- a/desktop/src/shared/types/workloads.ts +++ b/desktop/src/shared/types/workloads.ts @@ -21,9 +21,9 @@ export interface Workload { state: WorkloadState /** * Owner/origin node of the workload — the node whose proxy received the - * request. This is the identity half of the backend's `(originatedFrom, id)` - * global catalog key (workload ids are a per-node proxy counter, so they - * collide across nodes; `originatedFrom` disambiguates). + * request. Together with engine, runId and id, this identifies one request + * in the global catalog; proxy counters can repeat across nodes, engines + * and proxy runs. */ originatedFrom: string | null /** diff --git a/desktop/src/shared/types/ws-channels.ts b/desktop/src/shared/types/ws-channels.ts index b26705f0..8484982e 100644 --- a/desktop/src/shared/types/ws-channels.ts +++ b/desktop/src/shared/types/ws-channels.ts @@ -110,10 +110,6 @@ export interface WsInvokeChannelMap { // Workloads 'workloads:get-initial': { request: void; response: Record } - 'workloads:cancel': { - request: { id: string; runId: string; engine: EngineType; originatedFrom: string } - response: { accepted: boolean } - } } export type WsInvokeChannel = keyof WsInvokeChannelMap diff --git a/desktop/src/shared/utils/workloads.ts b/desktop/src/shared/utils/workloads.ts index a4216023..9ed23b46 100644 --- a/desktop/src/shared/utils/workloads.ts +++ b/desktop/src/shared/utils/workloads.ts @@ -9,8 +9,8 @@ import type { Workload } from '@/shared/types/workloads' * The backend's catalog is keyed by `(originatedFrom, engine, runId, id)`, but * clients retain all four fields so equal counters from different engines or * proxy runs never overwrite one another. A legacy removal without engine/run - * identity removes the matching origin/id prefix, as the broker does. New - * targeted removals and cancellation retain exact identity. + * identity removes the matching origin/id prefix, as the broker does. Targeted + * removals retain exact identity. */ export function workloadKey( originatedFrom: string | null, diff --git a/desktop/src/ui/api/pair-api.ts b/desktop/src/ui/api/pair-api.ts index 344d0742..a01e0180 100644 --- a/desktop/src/ui/api/pair-api.ts +++ b/desktop/src/ui/api/pair-api.ts @@ -13,7 +13,6 @@ import type { NodeItem } from '@/shared/types/nodes' import type { ServiceError } from '@/shared/types/errors' import type { NodeItemMetrics } from '@/shared/types/metrics' import type { Workload, WorkloadRemoval } from '@/shared/types/workloads' -import type { EngineType } from '@/shared/types/engines' import type { AppInitialSnapshot, ClusterInitialSnapshot } from '@/shared/types/bootstrap' // --------------------------------------------------------------------------- @@ -81,12 +80,6 @@ export interface IDiscoveryApi { } export interface IWorkloadsApi { - cancel(request: { - id: string - runId: string - engine: EngineType - originatedFrom: string - }): Promise<{ accepted: boolean }> /** Fetch all active workloads (inference jobs). */ getInitial(): Promise> /** A workload was created or updated. */ @@ -176,7 +169,6 @@ export function createPairApi(transport: ServiceTransport): IPairApi { engines: createEngineApi(transport), workloads: { getInitial: () => transport.invoke('workloads:get-initial'), - cancel: request => transport.invoke('workloads:cancel', request), onUpsert: cb => transport.subscribePush('workloads:upsert', cb), onRemove: cb => transport.subscribePush('workloads:remove', cb) }, diff --git a/desktop/src/ui/components/Workloads/WorkloadItemCard.tsx b/desktop/src/ui/components/Workloads/WorkloadItemCard.tsx index 25f7eeea..f7a32039 100644 --- a/desktop/src/ui/components/Workloads/WorkloadItemCard.tsx +++ b/desktop/src/ui/components/Workloads/WorkloadItemCard.tsx @@ -2,14 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 import { useMemo, memo } from 'react' -import { Button, Card, Flex, Stack, Text } from '@nvidia/foundations-react-core' +import { Card, Flex, Stack, Text } from '@nvidia/foundations-react-core' import type { Workload } from '@/shared/types/workloads' import { workloadExecutionNodeId } from '@/shared/utils/workloads' import { useNodesStore } from '@/ui/stores/nodes.store' import { formatModelDisplayName } from '@/ui/utils/format-model-display-name' import { getWorkloadColorBar } from '@/ui/utils/colors' import EngineIcon from '@/ui/components/EngineIcon' -import { useConnectionStore } from '@/ui/stores/connection.store' const formatDate = (timestamp: number) => { const date = new Date(timestamp) @@ -42,7 +41,6 @@ const formatDate = (timestamp: number) => { } function WorkloadItemCard({ workload }: { workload: Workload }) { - const selfId = useConnectionStore(state => state.selfId) // Subscribe to only this workload's execution node name. Selecting the whole // nodes array re-rendered every job card on any node/metrics update. const ranOnNodeText = useNodesStore(state => { @@ -131,26 +129,6 @@ function WorkloadItemCard({ workload }: { workload: Workload }) { >
- {workload.engine === 'llamacpp' && - workload.state === 'running' && - workload.originatedFrom === selfId && - workload.runId && ( - - )} diff --git a/desktop/tests/modular/engine-command-load.test.ts b/desktop/tests/modular/engine-command-load.test.ts index 41cf0ce4..b229342a 100644 --- a/desktop/tests/modular/engine-command-load.test.ts +++ b/desktop/tests/modular/engine-command-load.test.ts @@ -37,23 +37,6 @@ describe('local model load command', () => { mocks.supervisor.callProcess.mockResolvedValue({ accepted: true }) }) - it('forwards the exact workload identity and reports only cancellation acceptance', async () => { - const request = { - id: '1', - engine: 'llamacpp', - runId: 'current', - originatedFrom: 'local-node' - } as const - expect(await handleServiceBridgeInvoke('workloads:cancel', request)).toEqual({ - accepted: true - }) - expect(mocks.supervisor.callProcess).toHaveBeenCalledWith( - 'broker', - 'workloads:cancel', - request - ) - }) - it('refuses a local llama update without uninstalling, installing, or acting', async () => { await handleServiceBridgeInvoke('engine:command', { command: 'update', diff --git a/desktop/tests/modular/llamacpp-proxy-events.test.ts b/desktop/tests/modular/llamacpp-proxy-events.test.ts new file mode 100644 index 00000000..4bf0c39e --- /dev/null +++ b/desktop/tests/modular/llamacpp-proxy-events.test.ts @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from 'vitest' + +vi.mock('electron', () => ({ + app: { isPackaged: false, getAppPath: () => process.cwd() }, + BrowserWindow: { getAllWindows: () => [] } +})) +vi.mock('@/electron/window', () => ({ createOverviewWindow: vi.fn() })) +vi.mock('@/shared/utils/log', () => ({ + createStructuredLogger: () => ({ + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + verbose: vi.fn() + }) +})) +vi.mock('@/electron/config/ui-config', () => ({ isFirstRun: () => false })) +vi.mock('@/electron/service-bridge/manual-nodes-store', () => ({ listManualNodeEntries: () => [] })) +vi.mock('@/electron/service-bridge/node-info-poller', () => ({ + startNodeInfoPoller: vi.fn(), + stopNodeInfoPoller: vi.fn() +})) + +import { getModularSupervisor } from '@/electron/service-bridge/modular-supervisor' +import { getModularBridgeState, type ProxyEngine } from '@/electron/service-bridge/modular-state' + +const engines = [ + { engine: 'ollama', source: 'ollama-proxy', port: 11434 }, + { engine: 'lm-studio', source: 'lmstudio-proxy', port: 1234 }, + { engine: 'llamacpp', source: 'llamacpp-proxy', port: 8080 } +] satisfies { engine: ProxyEngine; source: string; port: number }[] + +describe.each(engines)('$engine broker proxy events', ({ engine, source, port }) => { + it('updates the reported port on live ready and rebind frames', () => { + const supervisor = getModularSupervisor() + const state = getModularBridgeState() + // Exercise production dispatch and normalization without spawning a broker. + supervisor['handleNotification']({ + source: 'broker', + method: `${source}:ready`, + params: { port } + }) + expect(state.getProxyPort(engine)).toBe(port) + supervisor['handleNotification']({ + source: 'broker', + method: `${source}:ready`, + params: { port: port + 1 } + }) + expect(state.getProxyPort(engine)).toBe(port + 1) + }) + + it('attributes live discovered nodes to the reporting engine', () => { + const id = `reported-${engine}` + getModularSupervisor()['handleNotification']({ + source: 'broker', + method: `${source}:node/discovered`, + params: { id, host: id, port, addresses: ['192.0.2.1'] } + }) + const state = getModularBridgeState() + expect(state.getNodesInitial().nodes[id]).toBeDefined() + expect( + state + .getEngineInitialState() + .statuses.find(s => s.nodeId === id && s.engineType === engine) + ).toBeDefined() + getModularSupervisor()['handleNotification']({ + source: 'broker', + method: `${source}:node/removed`, + params: { id } + }) + expect(state.getNodesInitial().nodes[id]).toBeUndefined() + }) +}) diff --git a/desktop/tests/modular/workloads-display.test.ts b/desktop/tests/modular/workloads-display.test.ts new file mode 100644 index 00000000..ccec8dc5 --- /dev/null +++ b/desktop/tests/modular/workloads-display.test.ts @@ -0,0 +1,120 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { PreloadServiceTransport } from '@/shared/types/service-bridge' +import type { Workload } from '@/shared/types/workloads' +import { workloadExecutionNodeId, workloadKey } from '@/shared/utils/workloads' +import { createPairApi, type IWorkloadsApi } from '@/ui/api/pair-api' +import { useWorkloadsStore } from '@/ui/stores/workloads.store' + +const sample: Workload = { + id: '1', + runId: 'a', + model: 'owner/model:Q4', + engine: 'llamacpp', + state: 'running', + originatedFrom: 'origin', + scheduledOn: 'serving-node', + createdAt: 100, + startedAt: 101, + completedAt: null, + error: null, + requesterId: null +} + +it('exposes workload display without a cancellation command', () => { + const transport: PreloadServiceTransport = { + connected: true, + invoke: vi.fn(), + subscribePush: vi.fn(), + onConnect: vi.fn(), + onDisconnect: vi.fn(), + onAuthFailure: vi.fn(), + destroy: vi.fn() + } + expect(Object.keys(createPairApi(transport).workloads).sort()).toEqual([ + 'getInitial', + 'onRemove', + 'onUpsert' + ]) +}) + +describe('workload display identity', () => { + const getInitial = vi.fn() + const onUpsert = vi.fn() + const onRemove = vi.fn() + const frames = vi.fn() + + beforeEach(() => { + getInitial.mockResolvedValue({}) + onUpsert.mockReturnValue(() => {}) + onRemove.mockReturnValue(() => {}) + frames.mockReturnValue(1) + vi.stubGlobal('requestAnimationFrame', frames) + vi.stubGlobal('cancelAnimationFrame', vi.fn()) + vi.stubGlobal('window', { + pairApi: { workloads: { getInitial, onUpsert, onRemove } satisfies IWorkloadsApi } + }) + useWorkloadsStore.setState({ workloads: new Map() }) + }) + + afterEach(() => { + useWorkloadsStore.getState().cleanup() + useWorkloadsStore.setState({ workloads: new Map() }) + vi.unstubAllGlobals() + }) + + it('keeps all engines and proxy runs distinct and removes only the reported identity', async () => { + await useWorkloadsStore.getState().initialize() + const upsert = onUpsert.mock.calls[0][0] + for (const workload of [ + { ...sample, engine: 'ollama' }, + { ...sample, engine: 'lm-studio' }, + sample, + { ...sample, runId: 'b' } + ] satisfies Workload[]) { + upsert(workload) + } + frames.mock.calls[0][0](0) + expect(useWorkloadsStore.getState().workloads.size).toBe(4) + + onRemove.mock.calls[0][0]({ + workloadId: '1', + originatedFrom: 'origin', + engine: 'llamacpp', + runId: 'a' + }) + frames.mock.calls[1][0](0) + const workloads = useWorkloadsStore.getState().workloads + expect(workloads.size).toBe(3) + expect(workloads.has(workloadKey('origin', '1', 'llamacpp', 'a'))).toBe(false) + expect(workloads.get(workloadKey('origin', '1', 'llamacpp', 'b'))).toMatchObject({ + scheduledOn: 'serving-node', + model: sample.model + }) + }) + + it('does not resurrect an exact removal from a pending desktop snapshot', async () => { + const key = workloadKey('origin', '1', 'llamacpp', 'a') + const pending = Promise.withResolvers>() + getInitial.mockReturnValue(pending.promise) + const initializing = useWorkloadsStore.getState().initialize() + onRemove.mock.calls[0][0]({ + workloadId: '1', + originatedFrom: 'origin', + engine: 'llamacpp', + runId: 'a' + }) + frames.mock.calls[0][0](0) + pending.resolve({ [key]: sample }) + await initializing + expect(useWorkloadsStore.getState().workloads.has(key)).toBe(false) + }) + + it('attributes execution only to the reported destination', () => { + expect(workloadExecutionNodeId(sample)).toBe('serving-node') + expect(workloadExecutionNodeId({ ...sample, scheduledOn: null })).toBeNull() + expect(workloadExecutionNodeId({ ...sample, scheduledOn: undefined })).toBeNull() + }) +}) diff --git a/services/nvpair-tui/README.md b/services/nvpair-tui/README.md index d91979ce..a7184120 100644 --- a/services/nvpair-tui/README.md +++ b/services/nvpair-tui/README.md @@ -31,7 +31,7 @@ Tabs: | **Errors** | The service-error datastore (`errors:get-initial` + live `errors:update`); `c` clears the selected entry. | | **Nodes** | mDNS-discovered Ollama nodes (`discovery:subscribe` / `discovery:nodes-changed`). | | **Proxies** | Ollama, LM Studio and llama.cpp reverse proxies: status, upstream selection and listen port. | -| **Workloads** | Baseline plus live workloads keyed by origin/engine/run/id. `c` requests cancellation of one local-origin llama request; other origins/engines are refused. | +| **Workloads** | Live cluster workload table for Ollama, LM Studio and llama.cpp, keyed by origin/engine/run/id. No workload cancellation key or selected-row detail pane. | | **Engines** | Install (`i`), start (`s`), stop (`x`), restart (`r`), uninstall (`u`); model inventory (`m`), pull (`p`), load (`L`), unload (`e`), delete (`d`), cancel pull (`c`), local GGUF import (`I`). No engine update key: managed llama has no update action, and the TUI never substitutes uninstall plus reinstall for one. | | **Cluster** | Pairing + membership: invite by address (`i`, shows the six-digit PIN — the first invite auto-founds a cluster of one), accept (`a`) / decline (`d`) an inbound invite, remove a member (`r`), leave (`L`). | | **Manual** | User-added nodes: add by address (`a`), remove (`r`). | @@ -44,8 +44,12 @@ The model view uses arrow keys to scroll and `Esc` to return. Model actions prefill the selected exact model ID and require Enter. llama downloads accept `owner/repository:QUANT`; imports take a local GGUF path. Install support and external ownership come from Engine Manager. Downloaded models are not loaded -until the runtime reports them resident. Cancellation acceptance is not a vendor -stop acknowledgement; the workload stream supplies the terminal outcome. +until the runtime reports them resident. + +The workload table consumes live `workloads:upsert` and `workloads:remove` +events after subscribing. It does not fetch a startup snapshot: requests already +in flight appear on their next event. Arrow keys scroll the table; `c` remains +the cancel-pull command in the Engines tab, not a workload action. - `tab` / `shift+tab` (or `→` / `←`, `l` / `h`) — switch tabs - `?` — toggle full help diff --git a/services/nvpair-tui/ui/engine_models_test.go b/services/nvpair-tui/ui/engine_models_test.go new file mode 100644 index 00000000..cc244766 --- /dev/null +++ b/services/nvpair-tui/ui/engine_models_test.go @@ -0,0 +1,141 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package ui + +import ( + "context" + "encoding/json" + "fmt" + "io" + "reflect" + "testing" + + "nvpair-tui/rpc" +) + +func TestModelInventoryRPCShapes(t *testing.T) { + for _, tc := range []struct { + name, engine, inventory, action string + managed bool + want []string + }{ + {"LM Studio native keys", "lmstudio", `{"models":[{"key":"phi-3","loaded_instances":[]},{"key":"gemma-2b","loaded_instances":[]}]}`, "list_models", false, []string{"phi-3", "gemma-2b"}}, + {"Ollama names", "ollama", `{"models":[{"name":"llama3:8b","model":"llama3:8b"},{"name":"qwen:0.5b"}]}`, "list_models", false, []string{"llama3:8b", "qwen:0.5b"}}, + {"Ollama model field", "ollama", `{"models":[{"model":"llama3:8b"}]}`, "list_models", false, []string{"llama3:8b"}}, + {"managed llama IDs", "llamacpp", `{"data":[{"id":"owner/model:Q4"}]}`, "list_downloaded", true, []string{"owner/model:Q4"}}, + {"external llama IDs", "llamacpp", `{"data":[{"id":"owner/model:Q4"}]}`, "list_models", false, []string{"owner/model:Q4"}}, + } { + t.Run(tc.name, func(t *testing.T) { + // The real RPC client and decoder run over in-memory streams, not sockets. + clientIn, serverOut := io.Pipe() + serverIn, clientOut := io.Pipe() + client := rpc.NewClient(clientIn, clientOut) + server := rpc.NewCodec(serverIn, serverOut) + ctx, cancel := context.WithCancel(context.Background()) + clientDone := make(chan struct{}) + go func() { + defer close(clientDone) + _ = client.Run(ctx) + }() + t.Cleanup(func() { + cancel() + _ = clientOut.Close() + _ = serverOut.Close() + _ = clientIn.Close() + _ = serverIn.Close() + <-clientDone + }) + loaded, err := json.Marshal(map[string]map[string][]string{ + "loadedByEngine": {tc.engine: {tc.want[0]}}, + }) + if err != nil { + t.Fatal(err) + } + serve := func() error { + for _, response := range []struct { + method string + result json.RawMessage + }{{"engine:action", json.RawMessage(tc.inventory)}, {"engine:models", loaded}} { + req, err := server.Read() + if err != nil { + return err + } + if req.Method != response.method { + return fmt.Errorf("method = %q, want %q", req.Method, response.method) + } + if req.Method == "engine:action" { + var params map[string]string + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + return err + } + if params["engine"] != tc.engine || params["action"] != tc.action { + return fmt.Errorf("wrong inventory action: %v", params) + } + } + if err := server.Write(&rpc.Message{JSONRPC: "2.0", ID: req.ID, Result: response.result}); err != nil { + return err + } + } + return nil + } + serverDone := make(chan error, 1) + go func() { + err := serve() + serverDone <- err + if err != nil { + _ = serverOut.CloseWithError(err) + } + }() + v := newEnginesView(client) + v.SetSize(90, 20) + v.merge(engineStatus{Engine: tc.engine, Installed: true, Managed: tc.managed}) + cmd := v.loadModelsCmd() + if cmd == nil { + t.Fatal("selected engine has no model inventory command") + } + msg := cmd() + if err := <-serverDone; err != nil { + t.Fatal(err) + } + result, ok := msg.(engineModelsMsg) + if !ok { + t.Fatalf("inventory result = %#v", msg) + } + if result.err != nil { + t.Fatal(result.err) + } + if !reflect.DeepEqual(result.names, tc.want) || !result.loaded[tc.want[0]] { + t.Fatalf("inventory = %+v, want names %v with first model loaded", result, tc.want) + } + v.Update(result) + if rows := v.models.Rows(); len(rows) != len(tc.want) || rows[0][0] != tc.want[0] || rows[0][1] != "loaded" { + t.Fatalf("model rows lost identity/residency: %v", rows) + } + }) + } +} + +func TestResidencyNotificationUsesEngineIdentity(t *testing.T) { + for _, engine := range []string{"ollama", "lmstudio", "llamacpp"} { + t.Run(engine, func(t *testing.T) { + v := newEnginesView(nil) + v.pendingLoadEngine, v.pendingLoadModel = engine, "model" + v.Update(NotificationMsg{Msg: &rpc.Message{Method: "engine:models-changed", Params: json.RawMessage(`{"engine":"other","models":{"loadedByEngine":{"other":["model"]}}}`)}}) + if v.pendingLoadModel == "" { + t.Fatal("another engine's residency settled this load") + } + params, err := json.Marshal(map[string]any{ + "engine": engine, + "models": map[string]map[string][]string{"loadedByEngine": {engine: {"model"}}}, + }) + if err != nil { + t.Fatal(err) + } + v.Update(NotificationMsg{Msg: &rpc.Message{Method: "engine:models-changed", Params: params}}) + if v.pendingLoadModel != "" || v.status != "Model loaded: model" || v.showModels { + t.Fatalf("real residency envelope did not settle without opening model browser: %s", v.status) + } + }) + } +} diff --git a/services/nvpair-tui/ui/engines.go b/services/nvpair-tui/ui/engines.go index f963015e..43517256 100644 --- a/services/nvpair-tui/ui/engines.go +++ b/services/nvpair-tui/ui/engines.go @@ -207,10 +207,12 @@ func (v *enginesView) Update(msg tea.Msg) tea.Cmd { switch msg.Msg.Method { case "engine:models-changed": var snapshot struct { - Loaded map[string][]string `json:"loadedByEngine"` + Models struct { + Loaded map[string][]string `json:"loadedByEngine"` + } `json:"models"` } if decodeParams(msg.Msg.Params, &snapshot) == nil { - for _, name := range snapshot.Loaded[v.pendingLoadEngine] { + for _, name := range snapshot.Models.Loaded[v.pendingLoadEngine] { if name == v.pendingLoadModel && name != "" { v.status = "Model loaded: " + name v.pendingLoadEngine, v.pendingLoadModel = "", "" @@ -368,6 +370,7 @@ func (v *enginesView) loadModelsCmd() tea.Cmd { Models []struct { Name string `json:"name"` Model string `json:"model"` + Key string `json:"key"` } `json:"models"` } if err := decodeParams(msg.Result, &inventory); err != nil { @@ -384,6 +387,9 @@ func (v *enginesView) loadModelsCmd() tea.Cmd { if name == "" { name = model.Model } + if name == "" { + name = model.Key + } if name != "" { result.names = append(result.names, name) } diff --git a/services/nvpair-tui/ui/engines_test.go b/services/nvpair-tui/ui/engines_test.go index 60cf7a3b..dabb55b8 100644 --- a/services/nvpair-tui/ui/engines_test.go +++ b/services/nvpair-tui/ui/engines_test.go @@ -25,7 +25,7 @@ func TestLongEngineCallKeepsObservedLoadTruth(t *testing.T) { if v.pendingLoadModel == "" || !strings.Contains(v.status, "outcome unknown") || strings.Contains(v.status, "failed") { t.Fatalf("client timeout invented backend failure: %s", v.status) } - v.Update(NotificationMsg{Msg: &rpc.Message{Method: "engine:models-changed", Params: json.RawMessage(`{"loadedByEngine":{"llamacpp":["owner/model:Q4"]}}`)}}) + v.Update(NotificationMsg{Msg: &rpc.Message{Method: "engine:models-changed", Params: json.RawMessage(`{"engine":"llamacpp","models":{"models":["owner/model:Q4"],"modelsByEngine":{"llamacpp":["owner/model:Q4"]},"loadedByEngine":{"llamacpp":["owner/model:Q4"]}}}`)}}) v.Update(timedOut) if v.pendingLoadModel != "" || v.status != "Model loaded: owner/model:Q4" { t.Fatalf("late client timeout erased observed success: %s", v.status) @@ -80,7 +80,7 @@ func TestLlamaLoadAcceptanceWaitsForObservation(t *testing.T) { if v.pendingLoadModel == "" || !strings.Contains(v.status, "waiting") { t.Fatal("RPC acceptance completed the load") } - v.Update(NotificationMsg{Msg: &rpc.Message{Method: "engine:models-changed", Params: json.RawMessage(`{"loadedByEngine":{"llamacpp":["owner/model:Q4"]}}`)}}) + v.Update(NotificationMsg{Msg: &rpc.Message{Method: "engine:models-changed", Params: json.RawMessage(`{"engine":"llamacpp","models":{"models":["owner/model:Q4"],"modelsByEngine":{"llamacpp":["owner/model:Q4"]},"loadedByEngine":{"llamacpp":["owner/model:Q4"]}}}`)}}) if v.pendingLoadModel != "" || !strings.Contains(v.status, "Model loaded") { t.Fatal("loaded observation did not settle action") } diff --git a/services/nvpair-tui/ui/workloads.go b/services/nvpair-tui/ui/workloads.go index f09c9af8..29b17f53 100644 --- a/services/nvpair-tui/ui/workloads.go +++ b/services/nvpair-tui/ui/workloads.go @@ -19,11 +19,11 @@ type workload struct { RunID string `json:"runId"` State string `json:"state"` OriginatedFrom string `json:"originatedFrom"` - ScheduledOn string `json:"scheduledOn"` CreatedAt int64 `json:"createdAt"` // Unix millis } -// workloadsView combines the initial snapshot with live workload events. +// workloadsView shows live cluster workloads after subscribing. Requests already +// in flight appear on their next event; this table does not fetch a baseline. type workloadsView struct { client *rpc.Client table table.Model @@ -35,16 +35,6 @@ type workloadsView struct { } type workloadsSubscribedMsg struct{ err error } -type workloadsInitialMsg struct { - workloads []workload - err error -} -type workloadCancelMsg struct { - accepted bool - err error -} - -var workloadCancelKey = key.NewBinding(key.WithKeys("c"), key.WithHelp("c", "cancel local llama request")) func newWorkloadsView(client *rpc.Client) *workloadsView { v := &workloadsView{client: client, byKey: map[string]workload{}} @@ -73,42 +63,14 @@ func (v *workloadsView) SetSize(w, h int) { {Title: "AGE", Width: age}, }) v.table.SetWidth(w) - v.table.SetHeight(clampWidth(h-3, 1)) + v.table.SetHeight(clampWidth(h-1, 1)) } func (v *workloadsView) Update(msg tea.Msg) tea.Cmd { switch msg := msg.(type) { - case workloadCancelMsg: - if msg.err != nil { - v.status = "Cancel failed: " + msg.err.Error() - } else if msg.accepted { - v.status = "Cancellation requested; awaiting terminal workload event." - } else { - v.status = "Request is no longer active in that proxy run." - } - return nil case workloadsSubscribedMsg: if msg.err != nil { v.status = "workloads subscribe failed: " + msg.err.Error() - return nil - } - return call(v.client, "workloads:get-initial", nil, func(msg *rpc.Message, err error) tea.Msg { - if err != nil { - return workloadsInitialMsg{err: err} - } - var result struct { - Workloads []workload `json:"workloads"` - } - err = decodeParams(msg.Result, &result) - return workloadsInitialMsg{workloads: result.Workloads, err: err} - }) - case workloadsInitialMsg: - if msg.err != nil { - v.status = "Workload baseline unavailable: " + msg.err.Error() - return nil - } - for _, w := range msg.workloads { - v.upsert(w) } return nil @@ -137,27 +99,6 @@ func (v *workloadsView) Update(msg tea.Msg) tea.Cmd { return nil case tea.KeyMsg: - if key.Matches(msg, workloadCancelKey) { - index := v.table.Cursor() - if index < 0 || index >= len(v.order) { - return nil - } - w := v.byKey[v.order[index]] - if w.Engine != "llamacpp" || w.RunID == "" || w.State != "running" { - v.status = "Only active llama requests with a run identity can be cancelled." - return nil - } - return call(v.client, "workloads:cancel", map[string]string{"id": w.ID, "runId": w.RunID, "engine": w.Engine, "originatedFrom": w.OriginatedFrom}, func(msg *rpc.Message, err error) tea.Msg { - if err != nil { - return workloadCancelMsg{err: err} - } - var result struct { - Accepted bool `json:"accepted"` - } - err = decodeParams(msg.Result, &result) - return workloadCancelMsg{accepted: result.Accepted, err: err} - }) - } var cmd tea.Cmd v.table, cmd = v.table.Update(msg) return cmd @@ -207,29 +148,16 @@ func (v *workloadsView) refreshRows() { } func (v *workloadsView) View() string { - if len(v.order) == 0 { - empty := "No active workloads. Live cluster workloads will appear here as they run." - // An empty list and a failed baseline fetch look identical otherwise, - // so a subscribe or get-initial error would be written to v.status and - // never rendered. - if v.status != "" { - return footerStyle.Render(empty) + "\n" + footerStyle.Render(v.status) - } - return footerStyle.Render(empty) + if v.status != "" { + return statusErrStyle.Render(v.status) } - detail := "" - if index := v.table.Cursor(); index >= 0 && index < len(v.order) { - w := v.byKey[v.order[index]] - target := w.ScheduledOn - if target == "" { - target = "unknown (not reported)" - } - detail = "Origin: " + w.OriginatedFrom + " | Runs on: " + target + if len(v.order) == 0 { + return footerStyle.Render("No active workloads. Live cluster workloads will appear here as they run.") } - return v.table.View() + "\n" + footerStyle.Width(v.width).Render(detail) + "\n" + footerStyle.Render(v.status) + return v.table.View() } -func (v *workloadsView) Help() []key.Binding { return []key.Binding{workloadCancelKey} } +func (v *workloadsView) Help() []key.Binding { return nil } func workloadKey(origin, id string, identity ...string) string { key := origin + "/" + id diff --git a/services/nvpair-tui/ui/workloads_test.go b/services/nvpair-tui/ui/workloads_test.go index 1aa3fdc8..b7194767 100644 --- a/services/nvpair-tui/ui/workloads_test.go +++ b/services/nvpair-tui/ui/workloads_test.go @@ -4,22 +4,68 @@ package ui import ( + "encoding/json" + "errors" "strings" "testing" + + "nvpair-tui/rpc" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" ) -func TestWorkloadsShowReportedTargetWithoutInferringOrigin(t *testing.T) { +func TestWorkloadsRemainALiveTable(t *testing.T) { v := newWorkloadsView(nil) v.SetSize(100, 20) - w := workload{ID: "1", Engine: "llamacpp", OriginatedFrom: "origin-node", ScheduledOn: "serving-node", State: "running"} - v.upsert(w) - if !strings.Contains(v.View(), "Runs on: serving-node") { - t.Fatal("reported serving node missing from selected workload") + if cmd := v.Update(workloadsSubscribedMsg{}); cmd != nil { + t.Fatal("live table must not request an expanded startup snapshot") + } + for _, engine := range []string{"ollama", "lmstudio", "llamacpp"} { + w := workload{ID: "1", Model: "model", Engine: engine, RunID: "a", OriginatedFrom: "origin-node", State: "running"} + params, err := json.Marshal(map[string]workload{"workloadInfo": w}) + if err != nil { + t.Fatal(err) + } + v.Update(NotificationMsg{Msg: &rpc.Message{Method: "workloads:upsert", Params: params}}) + } + rows := v.table.Rows() + if len(rows) != 3 { + t.Fatalf("workload rows = %v, want all three engines", rows) + } + for i, engine := range []string{"ollama", "lmstudio", "llamacpp"} { + if rows[i][1] != "model" || rows[i][2] != engine || rows[i][3] != "running" { + t.Fatalf("row %d lost workload attribution: %v", i, rows[i]) + } + } + if v.View() != v.table.View() { + t.Fatal("workload view must not add a detail/status pane") } - w.ScheduledOn = "" - v.upsert(w) - if !strings.Contains(v.View(), "Runs on: unknown (not reported)") { - t.Fatal("missing target must remain unknown") + if height := lipgloss.Height(v.View()); height != 19 { + t.Fatalf("table height = %d, want 19 including headers", height) + } +} + +func TestWorkloadsHaveNoCancellationControl(t *testing.T) { + v := newWorkloadsView(nil) + v.SetSize(100, 20) + v.upsert(workload{ID: "1", Engine: "llamacpp", RunID: "a", OriginatedFrom: "self", State: "running"}) + before := v.View() + if cmd := v.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{'c'}}); cmd != nil { + t.Fatal("c must not dispatch workload cancellation") + } + if len(v.Help()) != 0 || v.View() != before { + t.Fatal("workload cancellation changed view or help") + } +} + +func TestWorkloadsShowSubscriptionFailure(t *testing.T) { + v := newWorkloadsView(nil) + if cmd := v.Update(workloadsSubscribedMsg{err: errors.New("subscription unavailable")}); cmd != nil { + t.Fatal("failed subscription must not request a snapshot") + } + if !strings.Contains(v.View(), "subscription unavailable") { + t.Fatal("subscription error hidden by empty workload table") } } @@ -33,11 +79,45 @@ func TestWorkloadsKeepEngineRunIdentityAndTerminalTruth(t *testing.T) { } { v.upsert(w) } - v.Update(workloadsInitialMsg{workloads: []workload{{ID: "1", Engine: "llamacpp", RunID: "a", OriginatedFrom: "self", State: "running"}}}) + v.upsert(workload{ID: "1", Engine: "llamacpp", RunID: "a", OriginatedFrom: "self", State: "running"}) if len(v.byKey) != 3 { t.Fatalf("identity collision: %v", v.byKey) } if v.byKey[workloadKey("self", "1", "llamacpp", "a")].State != "completed" { - t.Fatal("late baseline regressed completed request") + t.Fatal("late event regressed completed request") + } +} + +func TestWorkloadsRemovalIdentity(t *testing.T) { + for _, tc := range []struct { + name string + params string + want int + }{ + {"exact", `{"workloadId":"1","originatedFrom":"self","engine":"llamacpp","runId":"a"}`, 3}, + {"origin and id", `{"workloadId":"1","originatedFrom":"self"}`, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + v := newWorkloadsView(nil) + v.SetSize(100, 20) + for _, w := range []workload{ + {ID: "1", Engine: "ollama", RunID: "a", OriginatedFrom: "self"}, + {ID: "1", Engine: "llamacpp", RunID: "a", OriginatedFrom: "self"}, + {ID: "1", Engine: "llamacpp", RunID: "b", OriginatedFrom: "self"}, + {ID: "1", Engine: "llamacpp", RunID: "a", OriginatedFrom: "peer"}, + } { + v.upsert(w) + } + v.Update(NotificationMsg{Msg: &rpc.Message{Method: "workloads:remove", Params: json.RawMessage(tc.params)}}) + if len(v.byKey) != tc.want || len(v.table.Rows()) != tc.want { + t.Fatalf("removal changed other identities: %v", v.byKey) + } + if _, exists := v.byKey[workloadKey("self", "1", "llamacpp", "a")]; exists { + t.Fatal("removed workload still present") + } + if _, exists := v.byKey[workloadKey("peer", "1", "llamacpp", "a")]; !exists { + t.Fatal("removal crossed the origin boundary") + } + }) } } From dd958ba69efa41fe4d939963a943498054edff3c Mon Sep 17 00:00:00 2001 From: pgoode41 Date: Tue, 22 Sep 2026 04:50:37 -0400 Subject: [PATCH 06/13] Defer explicit workload cancellation from the llama.cpp integration Signed-off-by: pgoode41 --- services/nvpair-proxy/cancel_test.go | 205 ++---------------- services/nvpair-proxy/facade.go | 79 ------- services/nvpair-proxy/proxy.go | 67 +----- services/nvpair-ui-broker/README.md | 7 - services/nvpair-ui-broker/broker.go | 3 - .../llamacpp_advertise_test.go | 9 +- services/nvpair-ui-broker/llamacppproxy.go | 24 -- 7 files changed, 32 insertions(+), 362 deletions(-) diff --git a/services/nvpair-proxy/cancel_test.go b/services/nvpair-proxy/cancel_test.go index eac41c40..7079a347 100644 --- a/services/nvpair-proxy/cancel_test.go +++ b/services/nvpair-proxy/cancel_test.go @@ -3,196 +3,33 @@ package main -// Coverage for cancelling one in-flight request by id. -// -// The registry this drives is per facade rather than per process, and the -// reason is only visible with more than one facade enabled: each facade mints -// request ids from its own counter starting at 1, so "cancel request 1" is -// ambiguous across engines and the runId guard cannot disambiguate it — -// runId names the process, which every facade shares. - import ( - "context" "encoding/json" "testing" "nvpair-shared/engines" ) -// cancelResult drives workload/cancel through the control plane and returns -// what the caller was told. -func cancelResult(t *testing.T, p *Proxy, rec *recordingWriter, engine, id, runID string) bool { - t.Helper() - before := len(rec.lines()) - params, err := json.Marshal(map[string]string{"id": id, "runId": runID}) - if err != nil { - t.Fatalf("marshal cancel params: %v", err) - } - msgID := json.RawMessage(`1`) - p.handleMessage(&Message{ - Method: engines.AddressMethod(engine, "workload/cancel"), - Params: params, - ID: &msgID, - }) - - lines := rec.lines() - if len(lines) <= before { - t.Fatalf("workload/cancel for %s/%s produced no response", engine, id) - } - var reply struct { - Result struct { - Accepted bool `json:"accepted"` - } `json:"result"` - Error *struct { - Message string `json:"message"` - } `json:"error"` - } - if err := json.Unmarshal(lines[len(lines)-1], &reply); err != nil { - t.Fatalf("decode cancel reply: %v", err) - } - if reply.Error != nil { - t.Fatalf("workload/cancel for %s/%s errored: %s", engine, id, reply.Error.Message) - } - return reply.Result.Accepted -} - -// twoFacadeRecordingProxy is twoFacadeProxy with its upward frames captured, so -// a test can read the reply to a request rather than only observe side effects. -func twoFacadeRecordingProxy(t *testing.T) (*Proxy, *recordingWriter) { - t.Helper() - redirectConfigDir(t) - - rec := &recordingWriter{} - p := NewProxy(NewCodec(rec)) - p.serveCtx = t.Context() - for _, e := range engines.All() { - port := freeTCPPort(t) - if _, err := p.enableFacade(enableFacadeParams{ - Engine: e.Name, - Port: port, - IgnorePersistedPort: true, - }); err != nil { - t.Fatalf("enable %s facade on :%d: %v", e.Name, port, err) - } - } - t.Cleanup(func() { p.shutdown(t.Context()) }) - return p, rec -} - -// A cancel names one request of one engine. Accepting it must cancel exactly -// that request's context and nothing else. -func TestCancelAbortsOnlyTheNamedRequest(t *testing.T) { - p, rec := twoFacadeRecordingProxy(t) - f := p.facadeFor(engines.All()[0].Name) - - first, cancelFirst := context.WithCancel(context.Background()) - second, cancelSecond := context.WithCancel(context.Background()) - _, forgetFirst := f.trackInflight("1", cancelFirst) - _, forgetSecond := f.trackInflight("2", cancelSecond) - defer forgetFirst() - defer forgetSecond() - - if cancelResult(t, p, rec, f.profile.Name, "missing", p.runID) { - t.Error("a request id that was never registered was accepted") - } - if first.Err() != nil || second.Err() != nil { - t.Fatal("an unmatched cancel aborted a live request") - } - - if !cancelResult(t, p, rec, f.profile.Name, "1", p.runID) { - t.Fatal("the named request was not accepted") - } - if first.Err() != context.Canceled { - t.Errorf("named request context = %v, want cancelled", first.Err()) - } - if second.Err() != nil { - t.Errorf("unrelated request was cancelled: %v", second.Err()) - } -} - -// runId names the proxy process, and request ids restart with it. A cancel -// carrying a previous run's id refers to a request that no longer exists, and -// the id it names may now belong to an unrelated one. -func TestCancelRefusesAStaleRun(t *testing.T) { - p, rec := twoFacadeRecordingProxy(t) - f := p.facadeFor(engines.All()[0].Name) - - ctx, cancel := context.WithCancel(context.Background()) - _, forget := f.trackInflight("1", cancel) - defer forget() - - if cancelResult(t, p, rec, f.profile.Name, "1", p.runID+"-previous") { - t.Error("a cancel from a previous run was accepted") - } - if ctx.Err() != nil { - t.Errorf("a stale run cancelled a live request: %v", ctx.Err()) - } -} - -// The property a process-wide registry would break. Both facades have a -// request numbered "1", because each counts from 1; a cancel addressed to one -// engine must leave the other engine's request of the same number running. -func TestCancelIsScopedToTheAddressedFacade(t *testing.T) { - p, rec := twoFacadeRecordingProxy(t) - all := engines.All() - target := p.facadeFor(all[0].Name) - bystander := p.facadeFor(all[1].Name) - - targetCtx, cancelTarget := context.WithCancel(context.Background()) - bystanderCtx, cancelBystander := context.WithCancel(context.Background()) - _, forgetTarget := target.trackInflight("1", cancelTarget) - _, forgetBystander := bystander.trackInflight("1", cancelBystander) - defer forgetTarget() - defer forgetBystander() - - if !cancelResult(t, p, rec, target.profile.Name, "1", p.runID) { - t.Fatalf("%s request 1 was not accepted", target.profile.Name) - } - if targetCtx.Err() != context.Canceled { - t.Errorf("%s request 1 = %v, want cancelled", target.profile.Name, targetCtx.Err()) - } - if bystanderCtx.Err() != nil { - t.Fatalf("cancelling %s request 1 also aborted %s request 1", - target.profile.Name, bystander.profile.Name) - } -} - -// An asked-for cancel and a client hanging up both surface as a cancelled -// request context. The workload's error text is user-visible, so the two must -// not be reported the same way. -func TestCancelIsDistinguishedFromAClientDisconnect(t *testing.T) { - p, _ := twoFacadeRecordingProxy(t) - f := p.facadeFor(engines.All()[0].Name) - - cancelled, cancelOne := context.WithCancel(context.Background()) - defer cancelOne() - req, forget := f.trackInflight("1", cancelOne) - defer forget() - if req.cancelled.Load() { - t.Fatal("a freshly tracked request already reports being cancelled") - } - if !f.cancelInflight("1") { - t.Fatal("cancel was not accepted") - } - if !req.cancelled.Load() { - t.Error("an asked-for cancel was not distinguished from a disconnect") - } - if cancelled.Err() != context.Canceled { - t.Errorf("request context = %v, want cancelled", cancelled.Err()) - } -} - -// Forgetting a finished request keeps a later cancel of the same id from -// reaching a request that has since reused it. -func TestCancelFindsNothingAfterTheRequestFinishes(t *testing.T) { - p, rec := twoFacadeRecordingProxy(t) - f := p.facadeFor(engines.All()[0].Name) - - _, cancel := context.WithCancel(context.Background()) - _, forget := f.trackInflight("1", cancel) - forget() - - if cancelResult(t, p, rec, f.profile.Name, "1", p.runID) { - t.Error("a finished request was still cancellable") +// Exercise only the control dispatcher; no facade or listener is started. +func TestWorkloadCancelIsNotAProxyAPI(t *testing.T) { + methods := []string{"workload/cancel"} + for _, engine := range engines.All() { + methods = append(methods, engines.AddressMethod(engine.Name, "workload/cancel")) + } + for _, method := range methods { + t.Run(method, func(t *testing.T) { + rec := &recordingWriter{} + p := NewProxy(NewCodec(rec)) + id := json.RawMessage(`1`) + p.handleMessage(&Message{ID: &id, Method: method, Params: json.RawMessage(`{"id":"1","runId":"a"}`)}) + lines := rec.lines() + if len(lines) != 1 { + t.Fatalf("responses = %d, want one method-not-found response", len(lines)) + } + var response Message + if json.Unmarshal(lines[0], &response) != nil || response.Error == nil || response.Error.Code != -32601 { + t.Fatalf("removed cancellation API did not return method not found: %s", lines[0]) + } + }) } } diff --git a/services/nvpair-proxy/facade.go b/services/nvpair-proxy/facade.go index 0f34b0cb..b53cfa3c 100644 --- a/services/nvpair-proxy/facade.go +++ b/services/nvpair-proxy/facade.go @@ -105,85 +105,6 @@ type facade struct { // separates them. A shared counter would hand the second facade "2" and // leave that guarantee untested. nextRequestID atomic.Uint64 - - // inflightMu guards inflight, the cancel handle for every cancellable - // request this facade is currently serving. - // - // Per facade for exactly the reason nextRequestID is: ids restart at 1 in - // each facade and therefore collide across them, so a process-wide map - // keyed by id would let a cancel aimed at one engine abort another - // engine's request of the same number. The runId guard cannot catch that, - // because runId names the process and every facade shares it. - inflightMu sync.Mutex - inflight map[string]*inflightRequest -} - -// inflightRequest is one cancellable request in progress. -type inflightRequest struct { - // cancel aborts the origin request's context, which propagates to the - // upstream connection so the engine stops generating rather than finishing - // a result nobody will read. - cancel context.CancelFunc - - // cancelled records that the abort was asked for, rather than the client - // hanging up. Both reach the disconnect watcher as a cancelled context, - // and the workload's error text is user-visible, so reporting an operator - // cancel as a client disconnect would be a lie about what happened. - cancelled atomic.Bool -} - -// trackInflight registers a request's cancel handle for the life of the -// request. The returned cleanup forgets it and releases the context. -// -// Every section holding inflightMu releases it by defer, for the reason the -// httpMu accessors below do: cancelInflight is reached from handleMessage, -// which recovers panics, and a mutex stranded by a recovered panic would trade -// a contained crash for a facade that can never track or cancel a request -// again. -func (f *facade) trackInflight(id string, cancel context.CancelFunc) (*inflightRequest, func()) { - req := &inflightRequest{cancel: cancel} - f.putInflight(id, req) - return req, func() { - f.forgetInflight(id) - cancel() - } -} - -func (f *facade) putInflight(id string, req *inflightRequest) { - f.inflightMu.Lock() - defer f.inflightMu.Unlock() - if f.inflight == nil { - f.inflight = make(map[string]*inflightRequest) - } - f.inflight[id] = req -} - -func (f *facade) forgetInflight(id string) { - f.inflightMu.Lock() - defer f.inflightMu.Unlock() - delete(f.inflight, id) -} - -func (f *facade) lookupInflight(id string) *inflightRequest { - f.inflightMu.Lock() - defer f.inflightMu.Unlock() - return f.inflight[id] -} - -// cancelInflight aborts one request by id and reports whether there was a live -// request to abort. -// -// Acceptance means that request's context was cancelled, not that the engine -// has acknowledged anything: the request's ordinary terminal workload event -// still reports the outcome. -func (f *facade) cancelInflight(id string) bool { - req := f.lookupInflight(id) - if req == nil { - return false - } - req.cancelled.Store(true) - req.cancel() - return true } func newFacade(host *Proxy, profile engineProfile, discovery *Discovery, port int) *facade { diff --git a/services/nvpair-proxy/proxy.go b/services/nvpair-proxy/proxy.go index 2735b759..13bdde16 100644 --- a/services/nvpair-proxy/proxy.go +++ b/services/nvpair-proxy/proxy.go @@ -1291,16 +1291,6 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) { var wlSeq int64 nextWlSeq := func() int64 { wlSeq++; return wlSeq } - // inflight is the handle workload/cancel reaches this request through, and - // is nil for anything that is not a cancellable workload. The request is - // re-pointed at a context derived from the origin's, so cancelling it does - // everything a client hangup does: the retry loop below reads r.Context() - // before every dispatch and every backoff, each attempt's context is a - // child of it, and the exhaustion response is suppressed on it. A cancel - // therefore tears down the attempt in flight, stops further retries, and - // is classified by the same reporters — once, through terminalOnce. - var inflight *inflightRequest - // Emit workload:submitted the moment the request is admitted, before any // dispatch. A burst of concurrent inference requests must surface as job // cards immediately — the upstream engine serializes work on a single GPU @@ -1314,12 +1304,6 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) { // not tried would inflate its load. Each dispatch and each gap between // attempts re-points it. if isInf && model != "" { - requestCtx, cancelRequest := context.WithCancel(r.Context()) - r = r.WithContext(requestCtx) - var forget func() - inflight, forget = f.trackInflight(reqID, cancelRequest) - defer forget() - createdMs := start.UnixMilli() wl = &Workload{ ID: reqID, @@ -1373,26 +1357,12 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) { }) } - // cancelReason names why the request context ended. An asked-for cancel - // (workload/cancel) arrives as the same cancelled context a vanished client - // or our own shutdown does, and the workload's error text is user-visible, - // so reporting an operator's cancel as a disconnect would misstate what - // happened. Both reporters below go through this so the answer does not - // depend on which of them wins the race to emit. - cancelReason := func(otherwise string) string { - if inflight != nil && inflight.cancelled.Load() { - return "cancelled before completion" - } - return otherwise - } - // Watch for the client going away while the request is in flight. The // terminal event is otherwise emitted only after the stream copy returns; // a client that disconnects mid-stream can leave the copy blocked, so we // emit the terminal here the moment r.Context() is cancelled instead of - // waiting for the unwind. Cancelling r.Context() (client close, a - // workload/cancel, or our own shutdown) also propagates to the - // ReverseProxy's upstream request, so the engine stops generating. + // waiting for the unwind. Cancelling r.Context() (client close or our own + // shutdown) also propagates to the ReverseProxy's upstream request. // terminalOnce keeps this from double-emitting with the normal path. The // half-open case (no FIN, r.Context() never fires) is caught instead by // statusCapture's write deadline below. @@ -1403,7 +1373,7 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) { go func() { select { case <-reqCtx.Done(): - emitTerminal("cancelled", cancelReason("client disconnected before completion")) + emitTerminal("cancelled", "client disconnected before completion") case <-finished: } }() @@ -1494,16 +1464,15 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) { if wl != nil { switch { case r.Context().Err() != nil: - // The request was cancelled before it finished — the client - // disconnected, a workload/cancel asked for it, or, on - // shutdown, we cancelled it to stop the in-flight inference. A + // The request was cancelled before it finished: the client + // disconnected or shutdown cancelled the in-flight inference. A // mid-stream cancel never reaches ErrorHandler (the 200 headers // are already sent), so without this branch it would be // misreported as completed. (The watcher above usually beats us // to it; emitTerminal makes that a no-op.) Cancelled rather // than failed: nothing went wrong here, the requester stopped // waiting. - emitTerminal("cancelled", cancelReason("request cancelled before completion")) + emitTerminal("cancelled", "request cancelled before completion") case committedSC != nil && committedSC.wroteErr != nil: // The response committed but a write to (or flush toward) the // client failed — typically the idle deadline tripping on a @@ -2734,30 +2703,6 @@ func (p *Proxy) handleMessage(msg *Message) { log.Printf("failed to respond to facade/enable: %v", err) } - case "workload/cancel": - f, ok := p.requireFacade(msg, engine) - if !ok { - return - } - var params struct { - ID string `json:"id"` - RunID string `json:"runId"` - } - if json.Unmarshal(msg.Params, ¶ms) != nil || params.ID == "" || params.RunID == "" { - p.codec.RespondError(msg.ID, -32602, "invalid params: expected {\"id\",\"runId\"}") - return - } - // runId names this process, and request ids restart with it. A stale - // runId therefore identifies a request from a previous proxy lifetime - // whose id may now belong to an unrelated request, so it is refused - // rather than matched. The facade is addressed, which is what keeps - // one engine's cancel off another engine's identically numbered - // request. - accepted := params.RunID == p.runID && f.cancelInflight(params.ID) - if err := p.codec.Respond(msg.ID, map[string]bool{"accepted": accepted}); err != nil { - log.Printf("failed to respond to workload/cancel: %v", err) - } - case "nodes/list": f, ok := p.requireFacade(msg, engine) if !ok { diff --git a/services/nvpair-ui-broker/README.md b/services/nvpair-ui-broker/README.md index 20c37010..bc1351bf 100644 --- a/services/nvpair-ui-broker/README.md +++ b/services/nvpair-ui-broker/README.md @@ -39,13 +39,6 @@ lifecycle, and relay rules. Two responsibilities live in the broker itself rather than in a worker: -The headless `workloads:cancel` method takes `{id, runId, engine, -originatedFrom}`. Only `engine: "llamacpp"` and this broker's exact local origin -UUID are accepted. It returns `{accepted}` after cancelling that active proxy -request context; the ordinary workload event reports its terminal outcome. -Foreign-origin requests and other engines are unsupported. A stale proxy run or -finished request returns `accepted: false`. Desktop and TUI use this same API. - `llamacpp-proxy:set-port` is intercepted exactly as `ollama-proxy:set-port` and `lmstudio-proxy:set-port` are: it runs through the authoritative engine-settings operation (see [`ENGINE_SETTINGS.md`](ENGINE_SETTINGS.md)), which refuses a port diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go index 90f7654c..cf890d15 100644 --- a/services/nvpair-ui-broker/broker.go +++ b/services/nvpair-ui-broker/broker.go @@ -3348,9 +3348,6 @@ func (b *Broker) handleMessage(msg *Message) { log.Printf("failed to respond to llamacpp-proxy:unsubscribe: %v", err) } - case "workloads:cancel": - b.cancelLlamaWorkload(msg) - case "workloads:subscribe": b.workloadsMu.Lock() b.workloadsSubscribed = true diff --git a/services/nvpair-ui-broker/llamacpp_advertise_test.go b/services/nvpair-ui-broker/llamacpp_advertise_test.go index 6e12655c..c5648f99 100644 --- a/services/nvpair-ui-broker/llamacpp_advertise_test.go +++ b/services/nvpair-ui-broker/llamacpp_advertise_test.go @@ -19,8 +19,9 @@ import ( // llamacpp-proxy:set-port is served by the shared settings operation; see // TestSettingsPortRPCServesEveryEngineProxy for its ownership refusals. -func TestLlamaWorkloadCancelRejectsForeignOriginAndOtherEngine(t *testing.T) { +func TestWorkloadCancelIsNotABrokerAPI(t *testing.T) { for _, body := range []string{ + `{"id":"1","runId":"a","engine":"llamacpp","originatedFrom":"self"}`, `{"id":"1","runId":"a","engine":"llamacpp","originatedFrom":"peer"}`, `{"id":"1","runId":"a","engine":"ollama","originatedFrom":"self"}`, `{"id":"1","engine":"llamacpp","originatedFrom":"self"}`, @@ -28,10 +29,10 @@ func TestLlamaWorkloadCancelRejectsForeignOriginAndOtherEngine(t *testing.T) { var output bytes.Buffer b := &Broker{nodeID: "self", codec: NewCodec(readWriter{Reader: bytes.NewReader(nil), Writer: &output})} id := json.RawMessage(`1`) - b.cancelLlamaWorkload(&Message{ID: &id, Method: "workloads:cancel", Params: json.RawMessage(body)}) + b.handleMessage(&Message{ID: &id, Method: "workloads:cancel", Params: json.RawMessage(body)}) var response Message - if json.Unmarshal(output.Bytes(), &response) != nil || response.Error == nil { - t.Fatalf("unsafe cancellation accepted: %s", output.String()) + if json.Unmarshal(output.Bytes(), &response) != nil || response.Error == nil || response.Error.Code != -32601 { + t.Fatalf("removed cancellation API did not return method not found: %s", output.String()) } } } diff --git a/services/nvpair-ui-broker/llamacppproxy.go b/services/nvpair-ui-broker/llamacppproxy.go index 413d73fe..d4e65b2f 100644 --- a/services/nvpair-ui-broker/llamacppproxy.go +++ b/services/nvpair-ui-broker/llamacppproxy.go @@ -126,27 +126,3 @@ func (b *Broker) forwardLlamaCppProxyNotificationForGeneration(generation uint64 } b.forwardEngineProxyNotification(llamacppProxyProfile, method, params) } - -// cancelLlamaWorkload forwards a headless cancel to the exact request that -// produced it. Only llama.cpp requests that originated on this node are -// accepted: the run id identifies one proxy lifetime, so a stale run or a -// foreign origin cannot cancel an unrelated request that reused an id. -func (b *Broker) cancelLlamaWorkload(msg *Message) { - var params struct { - ID string `json:"id"` - RunID string `json:"runId"` - Engine string `json:"engine"` - Origin string `json:"originatedFrom"` - } - if json.Unmarshal(msg.Params, ¶ms) != nil || params.ID == "" || params.RunID == "" { - _ = b.codec.RespondError(msg.ID, -32602, "exact workload id and runId are required") - return - } - if params.Engine != llamacppProxyProfile.Name || params.Origin == "" || params.Origin != b.nodeID { - _ = b.codec.RespondError(msg.ID, -32000, "only llama.cpp requests originating on this node can be cancelled here") - return - } - forward := *msg - forward.Method = llamacppProxyProfile.ComponentName() + ":workload/cancel" - b.relayToEngineProxy(llamacppProxyProfile, &forward) -} From 66eb1098f9bd370ad6ccd22b05538548909e1624 Mon Sep 17 00:00:00 2001 From: pgoode41 Date: Tue, 22 Sep 2026 10:15:46 -0400 Subject: [PATCH 07/13] Include embedded engine manifests in the service build fingerprint The desktop's service-binary cache keys on a content hash of the Go sources and module files, but nvpair-engine-manager compiles manifests/*.json into the binary (`//go:embed manifests/*.json`). A manifest-only change, such as a pin bump or a launch default, therefore produced a different binary while the cache still reported cli-bin as current, so a developer or packaging run could ship stale engine behaviour without noticing. Hash the embedded manifests with the sources. Verified by building twice (rebuild, then skip), appending one byte to llamacpp.json (rebuild), and restoring it (rebuild back to the earlier fingerprint). Co-Authored-By: Claude Fable 5.1 Signed-off-by: pgoode41 (cherry picked from commit 2dae370af14c1662d0f05c6f3ba63b4e030396bd) --- desktop/scripts/build-modular-binaries.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/desktop/scripts/build-modular-binaries.ts b/desktop/scripts/build-modular-binaries.ts index e84b4995..ed3167c0 100644 --- a/desktop/scripts/build-modular-binaries.ts +++ b/desktop/scripts/build-modular-binaries.ts @@ -211,6 +211,11 @@ function listFingerprintFiles(repo: string): string[] { out.push(full) } else if (entry === 'go.mod' || entry === 'go.sum') { out.push(full) + } else if (entry.endsWith('.json') && path.basename(dir) === 'manifests') { + // Engine manifests are compiled into nvpair-engine-manager + // (`//go:embed manifests/*.json`), so a manifest-only change + // produces a different binary and must miss the cache. + out.push(full) } } } @@ -218,7 +223,7 @@ function listFingerprintFiles(repo: string): string[] { return out.sort() } -/** Content hash of services Go sources + module files — not monorepo git HEAD. */ +/** Content hash of services Go sources, module files and embedded engine manifests — not monorepo git HEAD. */ function servicesSourceFingerprint(repo: string): string { const hash = createHash('sha256') for (const file of listFingerprintFiles(repo)) { From 21873ca6464becb6daec28a87124d6101921b3cb Mon Sep 17 00:00:00 2001 From: pgoode41 Date: Tue, 22 Sep 2026 09:02:10 -0400 Subject: [PATCH 08/13] Repair llama.cpp integration findings from the whole-engine assessment Each fix was verified against source and carries a regression test. - The paired-node settings relay dropped every llama.cpp snapshot, so a peer's llama editor never received live updates; relay every engine in the shared table. - The llama.cpp advertiser polled without the node configuration lock its siblings hold, so a tick could re-register lc between a settings apply's withdraw and restart. - The copied engine:set-port reservation guard was unreachable because the settings path intercepts the method before the relay; removed with its parser. - The registry pinned one vendor build's literals (archive root name, archive count, platform keys); validate the recipe shape and leave the exact pins to the tests so a pin bump is a manifest change. - Install and uninstall discarded the Detect result after promoting or removing the runtime; both now fail like the generic paths. - The fake engine gained the vendor's `download` shape so pull_model, preset rejection and cancel_pull are exercised; public Uninstall for llama.cpp is covered for runtime-only removal, model retention, saved Off and refusal while another process serves the port. - The broker engine-table tests pin llama.cpp's ownership, probe path, occupied-facade outcome and the third restore gate. - The node list offered Start/Stop for an observe-only external runtime and Install where the node reported no recipe; one shared predicate now gates the engine row, model manager and node list, and an unreachable Cancel button is gone. - Documentation drift: manual-probe port, engine default port, removed workload cancellation, executable count, facade relay list, profile and advertiser comments. spec.md and MANIFEST.md now name the bundled driver, builtins, identity probe, archive recipe fields, {model_dir} and dotted result fields as the fail-closed exception to manifest-only onboarding, and EngineStatus's install and managed fields. Co-Authored-By: Claude Fable 5.1 Signed-off-by: pgoode41 (cherry picked from commit 9845b82963d827e3abddb3668edf673ab5c770ce) --- .cursor/rules/proxy-inference-routing.mdc | 2 +- .cursor/rules/system-architecture.mdc | 2 +- desktop/docs/services-parity.md | 7 +- .../ui/components/BackendRow/BackendRow.tsx | 6 +- .../components/ModelManager/ModelManager.tsx | 15 +- .../components/NodeList/NodeEnginesInline.tsx | 31 +++- desktop/src/ui/utils/engine-ownership.ts | 19 ++ desktop/tests/modular/llamacpp-engine.test.ts | 10 + docs/architecture.mdx | 2 +- docs/engine-lifecycle.mdx | 2 +- docs/terminal-interface.mdx | 4 +- services/nvpair-engine-manager/MANIFEST.md | 41 ++++- services/nvpair-engine-manager/install.go | 6 +- .../nvpair-engine-manager/llamainstall.go | 6 +- .../nvpair-engine-manager/llamamodels_test.go | 172 ++++++++++++++++++ .../llamaupstream_test.go | 6 +- services/nvpair-engine-manager/registry.go | 15 +- .../nvpair-engine-manager/settingsremote.go | 29 ++- .../settingsremote_test.go | 39 ++++ services/nvpair-engine-manager/spec.md | 12 +- .../testdata/fakeengine/main.go | 55 ++++++ services/nvpair-ui-broker/advertiser.go | 12 +- services/nvpair-ui-broker/broker.go | 7 - services/nvpair-ui-broker/engineproxy.go | 13 +- services/nvpair-ui-broker/engineproxy_test.go | 12 +- services/nvpair-ui-broker/llamacppport.go | 14 -- .../nvpair-ui-broker/llamacppport_test.go | 9 - services/nvpair-ui-broker/proxyport_test.go | 11 +- 28 files changed, 450 insertions(+), 109 deletions(-) create mode 100644 desktop/src/ui/utils/engine-ownership.ts diff --git a/.cursor/rules/proxy-inference-routing.mdc b/.cursor/rules/proxy-inference-routing.mdc index 9dabf40b..87d9697e 100644 --- a/.cursor/rules/proxy-inference-routing.mdc +++ b/.cursor/rules/proxy-inference-routing.mdc @@ -10,7 +10,7 @@ SPDX-License-Identifier: Apache-2.0 # Proxy and Inference Routing Routing is owned by `nvpair-proxy` (one process hosting a facade per engine, -addressed by clients as `ollama-proxy:` / `lmstudio-proxy:`), +addressed by clients as `ollama-proxy:` / `lmstudio-proxy:` / `llamacpp-proxy:`), `nvpair-job-scheduler`, and `nvpair-ui-broker`. For model-bearing inference, each facade first filters a request-local discovery diff --git a/.cursor/rules/system-architecture.mdc b/.cursor/rules/system-architecture.mdc index 7b3b0f3b..45e4ab0b 100644 --- a/.cursor/rules/system-architecture.mdc +++ b/.cursor/rules/system-architecture.mdc @@ -28,7 +28,7 @@ Broker-owned workers: - `nvpair-proxy`, one process hosting a facade per enabled engine. It starts with no engine and no listener; the broker sends a `facade/enable` per engine carrying that engine's port. Clients still address each facade as - `ollama-proxy:` / `lmstudio-proxy:`, and one supervisor covers them all, so a + `ollama-proxy:` / `lmstudio-proxy:` / `llamacpp-proxy:`, and one supervisor covers them all, so a crash is reported against `nvpair-proxy` and restarts every facade together; - `nvpair-node-scanner`; - `nvpair-node-info`; diff --git a/desktop/docs/services-parity.md b/desktop/docs/services-parity.md index a6288c02..8f32de55 100644 --- a/desktop/docs/services-parity.md +++ b/desktop/docs/services-parity.md @@ -243,9 +243,12 @@ Personal AI Router uses: - `list_models`; - `pull_model`; - Ollama `run_model`, `unload_model` (`keep_alive: 0`), and `delete_model`; -- LM Studio `load_model`, `unload_model`, and `delete_model` (`remove_path`). +- LM Studio `load_model`, `unload_model`, and `delete_model` (`remove_path`); +- llama.cpp `load_model`, `unload_model` (router `/models/load` and + `/models/unload`, settled on observed residency), `delete_model`, + `pull_model` and `import_model` (managed cache builtin), and `cancel_pull`. -Both engines expose Load, Eject, and Delete in the model manager when the +All three engines expose Load, Eject, and Delete in the model manager when the backend action exists. Keep-alive / expiry controls remain unsupported. LM Studio's `delete_model` declares `restart_after`, so the engine manager diff --git a/desktop/src/ui/components/BackendRow/BackendRow.tsx b/desktop/src/ui/components/BackendRow/BackendRow.tsx index 2f96121b..db5dcf92 100644 --- a/desktop/src/ui/components/BackendRow/BackendRow.tsx +++ b/desktop/src/ui/components/BackendRow/BackendRow.tsx @@ -23,6 +23,7 @@ import { BackendFooter } from './BackendFooter' import { BackendUpdateBanner } from './BackendUpdateBanner' import { EngineSettingsSection } from './EngineSettingsSection' +import { isExternalRuntime } from '@/ui/utils/engine-ownership' /** * The transitional status to display while an optimistic lifecycle command is @@ -176,10 +177,7 @@ export function BackendRow({ // peers; uninstall, update, and model load/delete remain local-only. A // llama.cpp runtime PAIR detected but does not manage is observe-only: its // owner keeps lifecycle, settings, and model changes. - const externalLlama = - backend.type === 'llamacpp' && - backend.processStatus !== 'not-installed' && - backend.managed !== true + const externalLlama = isExternalRuntime(backend.type, backend.processStatus, backend.managed) const controlsDisabled = isTransitioning || externalLlama const content = expanded ? ( diff --git a/desktop/src/ui/components/ModelManager/ModelManager.tsx b/desktop/src/ui/components/ModelManager/ModelManager.tsx index 91c03113..ee45d5c1 100644 --- a/desktop/src/ui/components/ModelManager/ModelManager.tsx +++ b/desktop/src/ui/components/ModelManager/ModelManager.tsx @@ -20,6 +20,7 @@ import { IncomingSyncPullRow } from './IncomingSyncPullRow' import { TransientModelStatusRow } from './TransientModelStatusRow' import type { IncomingSyncRow } from '@/ui/types/model-manager' import type { ModelEntry } from '@/ui/types/model-hub' +import { isExternalRuntime } from '@/ui/utils/engine-ownership' export function ModelManager({ backend, nodeId }: { backend: BackendInfo; nodeId: string }) { const selfId = useConnectionStore(state => state.selfId) @@ -32,7 +33,7 @@ export function ModelManager({ backend, nodeId }: { backend: BackendInfo; nodeId formatModelDisplayName(b.name, backend.type) ) ) - const externalLlama = backend.type === 'llamacpp' && backend.managed !== true + const externalLlama = isExternalRuntime(backend.type, backend.processStatus, backend.managed) const caps = externalLlama ? { ...EngineCapabilities[backend.type], hasEject: false, hasDeleteModel: false } : EngineCapabilities[backend.type] @@ -218,18 +219,6 @@ export function ModelManager({ backend, nodeId }: { backend: BackendInfo; nodeId ))} - {backendType === 'llamacpp' && transientModel && transientPullProgress && ( - - )} - {!isBusy && ( <> {(isRunning || modelOpsWhenStopped) && diff --git a/desktop/src/ui/components/NodeList/NodeEnginesInline.tsx b/desktop/src/ui/components/NodeList/NodeEnginesInline.tsx index 8dfafe4f..3fb6fb10 100644 --- a/desktop/src/ui/components/NodeList/NodeEnginesInline.tsx +++ b/desktop/src/ui/components/NodeList/NodeEnginesInline.tsx @@ -8,6 +8,7 @@ import { useConnectionStore } from '@/ui/stores/connection.store' import { useNodesStore } from '@/ui/stores/nodes.store' import { usePendingActionsStore } from '@/ui/stores/pending-actions.store' import { getEnginesForNode } from '@/ui/utils/get-engines-for-node' +import { isExternalRuntime } from '@/ui/utils/engine-ownership' import { Button, Flex, Switch, Text } from '@nvidia/foundations-react-core' import { Download } from '@/ui/components/icons' import { useCallback, useMemo } from 'react' @@ -51,7 +52,12 @@ export default function NodeEnginesInline({ nodeId }: { nodeId: string }) { { type, name: EngineDisplayNames[type], - status: backend.processStatus + status: backend.processStatus, + // The same facts the engine row uses: a runtime PAIR only + // observes gets no lifecycle control, and an engine the + // node reports as not installable gets no Install button. + external: isExternalRuntime(type, backend.processStatus, backend.managed), + installable: type !== 'llamacpp' || backend.installSupported === true } ] }) @@ -85,6 +91,7 @@ export default function NodeEnginesInline({ nodeId }: { nodeId: string }) { b.status === 'stopping' if (b.status === 'not-installed' && !pending) { + if (!b.installable) return null return (