From 9ddfb2e7d4f7da650cf66ac82fee0f8dff5c3e9a Mon Sep 17 00:00:00 2001 From: Rayees Date: Sat, 5 Sep 2026 23:20:57 +0530 Subject: [PATCH 1/5] Add Anthropic Messages API Routing Signed-off-by: Rayees --- docs/getting-started.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.mdx b/docs/getting-started.mdx index cb7e7e34..b17af1bb 100644 --- a/docs/getting-started.mdx +++ b/docs/getting-started.mdx @@ -406,8 +406,8 @@ What that base URL serves depends on the engine behind it: | Endpoint | Default base URL | Paths it serves | | --- | --- | --- | -| Ollama | `http://127.0.0.1:11434` | Ollama's own API — `/api/chat`, `/api/generate`, `/api/embed`, `/api/tags` — and the OpenAI-compatible `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/models` | -| LM Studio | `http://127.0.0.1:1234` | The OpenAI-compatible paths only | +| Ollama | `http://127.0.0.1:11434` | Ollama's own API — `/api/chat`, `/api/generate`, `/api/embed`, `/api/tags` — and the OpenAI-compatible `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/models`, plus the Anthropic Messages API `/v1/messages` | +| LM Studio | `http://127.0.0.1:1234` | The OpenAI-compatible paths (`/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/models`) and the Anthropic Messages API `/v1/messages` | The distinction matters when you fill in an application's settings. A client written against the OpenAI API usually wants the `/v1` included, as in From 6a916d18d1f0a24ab649758eb94ebcecf98538bc Mon Sep 17 00:00:00 2001 From: Rayees Date: Thu, 10 Sep 2026 01:28:28 +0530 Subject: [PATCH 2/5] Add Anthropic messages routing integration test Signed-off-by: Rayees --- services/tests/model_routing_interop_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/services/tests/model_routing_interop_test.go b/services/tests/model_routing_interop_test.go index 8c81a276..13247197 100644 --- a/services/tests/model_routing_interop_test.go +++ b/services/tests/model_routing_interop_test.go @@ -71,6 +71,7 @@ func TestStrictModelRoutingAcrossProcesses(t *testing.T) { cases := []proxyCase{ {name: "ollama", rpcPrefix: "ollama-proxy", path: "/api/chat", port: ollamaPort}, {name: "lmstudio", rpcPrefix: "lmstudio-proxy", path: "/v1/chat/completions", port: lmstudioPort}, + {name: "lmstudio-anthropic", rpcPrefix: "lmstudio-proxy", path: "/v1/messages", port: lmstudioPort}, } client := &http.Client{Timeout: 5 * time.Second} t.Cleanup(client.CloseIdleConnections) From 7749b5401f73974394f56863c77ee61d724c8595 Mon Sep 17 00:00:00 2001 From: Rayees Date: Tue, 22 Sep 2026 20:43:49 +0530 Subject: [PATCH 3/5] Add Anthropic Messages API routing to unified proxy Signed-off-by: Rayees --- services/nvpair-proxy/README.md | 2 +- services/nvpair-proxy/engines.go | 3 +- services/nvpair-proxy/engines_test.go | 2 + services/nvpair-proxy/failover_test.go | 73 ++++++++++++++++++-- services/tests/model_routing_interop_test.go | 1 + 5 files changed, 73 insertions(+), 8 deletions(-) diff --git a/services/nvpair-proxy/README.md b/services/nvpair-proxy/README.md index c36fb0c2..cccd8b0c 100644 --- a/services/nvpair-proxy/README.md +++ b/services/nvpair-proxy/README.md @@ -102,7 +102,7 @@ and the health crash key are matched against each other, so they move together | Standalone port, used when `port` is omitted | 11435 | 1234 | | Persisted-port file (declared, not derived) | `proxy-port.json` | `lmstudio-proxy-port.json` | | Model-list routes | `GET /api/tags` (native), `GET /v1/models` (OpenAI) | `GET /v1/models` (OpenAI) | -| Inference routes | `/api/generate`, `/api/chat`, `/api/embeddings`, `/api/embed`, plus the OpenAI set | `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings` | +| Inference routes | `/api/generate`, `/api/chat`, `/api/embeddings`, `/api/embed`, plus the OpenAI set and `/v1/messages` | `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/messages` | | Model naming | untagged means `:latest`, so `llama3` and `llama3:latest` are one model | identifiers compared byte for byte | The route table is a **classifier, not an allowlist**. An unlisted path is diff --git a/services/nvpair-proxy/engines.go b/services/nvpair-proxy/engines.go index 06d1a54f..d50e489c 100644 --- a/services/nvpair-proxy/engines.go +++ b/services/nvpair-proxy/engines.go @@ -30,7 +30,7 @@ import ( // POST to /v1/models is not a model list, and a GET to /api/chat is not // inference. Folding them into one constant keeps the two from disagreeing. // -// The dialect distinction is meaningful only for model-list roles. All seven +// The dialect distinction is meaningful only for model-list roles. All eight // of Ollama's inference paths are handled identically — no envelope, identity // field or response shape is selected — so there is deliberately no // per-dialect inference role. @@ -122,6 +122,7 @@ var openAIInferenceRoutes = []route{ {Path: "/v1/chat/completions", Role: roleInferencePOST}, {Path: "/v1/completions", Role: roleInferencePOST}, {Path: "/v1/embeddings", Role: roleInferencePOST}, + {Path: "/v1/messages", Role: roleInferencePOST}, } var profiles = buildProfiles() diff --git a/services/nvpair-proxy/engines_test.go b/services/nvpair-proxy/engines_test.go index 4b149e3f..422159c7 100644 --- a/services/nvpair-proxy/engines_test.go +++ b/services/nvpair-proxy/engines_test.go @@ -29,12 +29,14 @@ func TestRoleForClassifiesOnlyDeclaredRoutes(t *testing.T) { }{ {"ollama native chat", ollama, "POST", "/api/chat", roleInferencePOST, true}, {"ollama openai chat", ollama, "POST", "/v1/chat/completions", roleInferencePOST, true}, + {"ollama anthropic messages", ollama, "POST", "/v1/messages", roleInferencePOST, true}, {"ollama native list", ollama, "GET", "/api/tags", roleModelListNativeGET, true}, {"ollama openai list", ollama, "GET", "/v1/models", roleModelListOpenAIGET, true}, {"ollama passthrough", ollama, "POST", "/api/pull", 0, false}, {"ollama version passthrough", ollama, "GET", "/api/version", 0, false}, {"lmstudio chat", lmstudio, "POST", "/v1/chat/completions", roleInferencePOST, true}, + {"lmstudio anthropic messages", lmstudio, "POST", "/v1/messages", roleInferencePOST, true}, {"lmstudio list", lmstudio, "GET", "/v1/models", roleModelListOpenAIGET, true}, // LM Studio serves no native Ollama routes, so /api/chat is not // inference for it — it is forwarded verbatim like any other path. diff --git a/services/nvpair-proxy/failover_test.go b/services/nvpair-proxy/failover_test.go index e84429ba..631180b3 100644 --- a/services/nvpair-proxy/failover_test.go +++ b/services/nvpair-proxy/failover_test.go @@ -354,6 +354,58 @@ func TestHandleHTTP_AllNodesDownReturnsError(t *testing.T) { }) } +// TestHandleHTTP_InferenceRouting proves every path classified as inference +// applies model-based candidate filtering and is forwarded unchanged — not +// only the single engineCase.inferencePath most bodies use. That includes +// shared paths such as /v1/messages. +func TestHandleHTTP_InferenceRouting(t *testing.T) { + forEachEngine(t, func(t *testing.T, tc engineCase) { + for _, r := range tc.profile.Routes { + if r.Role != roleInferencePOST { + continue + } + path := r.Path + t.Run(path, func(t *testing.T) { + var gotBody string + var gotPath string + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + gotBody = string(b) + gotPath = r.URL.Path + w.WriteHeader(http.StatusOK) + })) + defer good.Close() + + wrongModel := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + t.Error("wrong-model node should not receive request") + w.WriteHeader(http.StatusOK) + })) + defer wrongModel.Close() + + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "wrong", wrongModel.URL, "different-model")) + disc.AddManual(nodeForModel(t, "good", good.URL, tc.advertisedModel)) + p := testProxy(tc.profile, disc, tc.profile.FacadePort) + p.soleFacade().SetSelected("wrong") + + body := tc.inferenceBody() + rec := httptest.NewRecorder() + p.soleFacade().handleHTTP(rec, httptest.NewRequest(http.MethodPost, path, strings.NewReader(body))) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + if gotBody != body { + t.Errorf("node got body %q, want %q", gotBody, body) + } + if gotPath != path { + t.Errorf("path = %q, want %q", gotPath, path) + } + }) + } + }) +} + // TestHandleHTTP_404FailoverInferenceOnly: a 404 (model-not-found) on an // inference call fails over to the next advertised owner, but a 404 on a // non-inference path is returned as-is. @@ -379,15 +431,24 @@ func TestHandleHTTP_404FailoverInferenceOnly(t *testing.T) { return p } - // Inference POST: 404 on first → fail over → 200. - rec := httptest.NewRecorder() - newProxy().soleFacade().handleHTTP(rec, tc.inferenceRequest()) - if rec.Code != http.StatusOK { - t.Fatalf("inference 404: status = %d, want 200 (should fail over)", rec.Code) + // Inference POST: 404 on first → fail over → 200, on every classified + // inference path (including shared routes such as /v1/messages). + for _, r := range tc.profile.Routes { + if r.Role != roleInferencePOST { + continue + } + path := r.Path + t.Run(path, func(t *testing.T) { + rec := httptest.NewRecorder() + newProxy().soleFacade().handleHTTP(rec, httptest.NewRequest(http.MethodPost, path, strings.NewReader(tc.inferenceBody()))) + if rec.Code != http.StatusOK { + t.Fatalf("inference 404: status = %d, want 200 (should fail over)", rec.Code) + } + }) } // An ordinary non-inference GET still returns the first node's 404. - rec = httptest.NewRecorder() + rec := httptest.NewRecorder() newProxy().soleFacade().handleHTTP(rec, httptest.NewRequest(http.MethodGet, tc.nonInferencePath, nil)) if rec.Code != http.StatusNotFound { t.Fatalf("non-inference 404: status = %d, want 404 (must NOT fail over)", rec.Code) diff --git a/services/tests/model_routing_interop_test.go b/services/tests/model_routing_interop_test.go index 13247197..e7c77475 100644 --- a/services/tests/model_routing_interop_test.go +++ b/services/tests/model_routing_interop_test.go @@ -70,6 +70,7 @@ func TestStrictModelRoutingAcrossProcesses(t *testing.T) { } cases := []proxyCase{ {name: "ollama", rpcPrefix: "ollama-proxy", path: "/api/chat", port: ollamaPort}, + {name: "ollama-anthropic", rpcPrefix: "ollama-proxy", path: "/v1/messages", port: ollamaPort}, {name: "lmstudio", rpcPrefix: "lmstudio-proxy", path: "/v1/chat/completions", port: lmstudioPort}, {name: "lmstudio-anthropic", rpcPrefix: "lmstudio-proxy", path: "/v1/messages", port: lmstudioPort}, } From 97b21edc87aa82a7e38d8280a7abcd27d09b47b5 Mon Sep 17 00:00:00 2001 From: Kaylee Lubick Date: Tue, 22 Sep 2026 11:57:45 -0400 Subject: [PATCH 4/5] rebase onto develop Signed-off-by: Kaylee Lubick --- README.md | 8 +- docs/overview.mdx | 13 +- services/nvpair-proxy/README.md | 2 +- services/nvpair-proxy/engines.go | 45 +++--- services/nvpair-proxy/failover_test.go | 154 +++++++++++-------- services/tests/model_routing_interop_test.go | 4 +- 6 files changed, 129 insertions(+), 97 deletions(-) diff --git a/README.md b/README.md index 757c3f39..7ffc8a2d 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,10 @@ SPDX-License-Identifier: Apache-2.0 NVIDIA Personal AI Router (PAIR) is a local inference router for a group of compatible computers on the same network. It discovers participating nodes, -manages supported inference engines, and presents Ollama-compatible and -OpenAI-compatible proxy endpoints to applications and agents. Independent -requests can be routed to eligible nodes according to engine availability, -model availability, and current workload. +manages supported inference engines, and presents local proxy endpoints for +Ollama-compatible, OpenAI-compatible, and Anthropic Messages API requests. +Independent requests can be routed to eligible nodes according to engine +availability, model availability, and current workload. PAIR is useful for concurrent local workloads such as multi-agent applications. Prompts and responses are intended to remain on the local network when every diff --git a/docs/overview.mdx b/docs/overview.mdx index fb519d50..0efbb5ab 100644 --- a/docs/overview.mdx +++ b/docs/overview.mdx @@ -76,14 +76,14 @@ you form a cluster, only the last step repeats. ## Request Model -An application sends an ordinary Ollama-compatible or OpenAI-compatible HTTP -request to a local proxy. The proxy selects one eligible node and forwards the -whole request. That node's engine performs the inference, and the response -streams back through the proxy. +An application sends an ordinary Ollama-compatible or OpenAI-compatible +request, or an Anthropic Messages API request, to a local proxy. The proxy +selects one eligible node and forwards the whole request. That node's engine +performs the inference, and the response streams back through the proxy. ```mermaid flowchart LR - Client["Your AI app or agent"] -->|"Ollama- or OpenAI-compatible request"| Endpoint["PAIR endpoint
on your machine"] + Client["Your AI app or agent"] -->|"Ollama, OpenAI, or Anthropic Messages request"| Endpoint["PAIR endpoint
on your machine"] Endpoint <-->|"encrypted both ways"| Node["A paired node
with the model"] Node --> Engine["Inference engine"] ``` @@ -136,7 +136,8 @@ PAIR provides these capabilities: - A local endpoint for compatible AI applications and development tools. - LAN discovery plus manually configured nodes. -- Ollama-compatible and LM Studio/OpenAI-compatible routing proxies. +- Routing proxies for Ollama-compatible, OpenAI-compatible, and Anthropic + Messages API requests. - Pairing and cluster membership managed by the background services. - Model-aware, workload-informed routing of independent requests. - Encrypted routing between machines: a request sent to another node travels over diff --git a/services/nvpair-proxy/README.md b/services/nvpair-proxy/README.md index cccd8b0c..20c753e1 100644 --- a/services/nvpair-proxy/README.md +++ b/services/nvpair-proxy/README.md @@ -102,7 +102,7 @@ and the health crash key are matched against each other, so they move together | Standalone port, used when `port` is omitted | 11435 | 1234 | | Persisted-port file (declared, not derived) | `proxy-port.json` | `lmstudio-proxy-port.json` | | Model-list routes | `GET /api/tags` (native), `GET /v1/models` (OpenAI) | `GET /v1/models` (OpenAI) | -| Inference routes | `/api/generate`, `/api/chat`, `/api/embeddings`, `/api/embed`, plus the OpenAI set and `/v1/messages` | `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/messages` | +| Inference routes | `/api/generate`, `/api/chat`, `/api/embeddings`, `/api/embed`, plus the OpenAI and Anthropic Messages sets | `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/messages` | | Model naming | untagged means `:latest`, so `llama3` and `llama3:latest` are one model | identifiers compared byte for byte | The route table is a **classifier, not an allowlist**. An unlisted path is diff --git a/services/nvpair-proxy/engines.go b/services/nvpair-proxy/engines.go index d50e489c..d76f7127 100644 --- a/services/nvpair-proxy/engines.go +++ b/services/nvpair-proxy/engines.go @@ -30,8 +30,8 @@ import ( // POST to /v1/models is not a model list, and a GET to /api/chat is not // inference. Folding them into one constant keeps the two from disagreeing. // -// The dialect distinction is meaningful only for model-list roles. All eight -// of Ollama's inference paths are handled identically — no envelope, identity +// The dialect distinction is meaningful only for model-list roles. All of +// Ollama's inference paths are handled identically — no envelope, identity // field or response shape is selected — so there is deliberately no // per-dialect inference role. type routeRole int @@ -116,12 +116,15 @@ type engineProfile struct { SupportsHostAlias bool } -// openAIInferenceRoutes is the inference surface every OpenAI-compatible -// engine exposes. Ollama serves these alongside its native routes. +// openAIInferenceRoutes is the OpenAI-compatible inference surface. var openAIInferenceRoutes = []route{ {Path: "/v1/chat/completions", Role: roleInferencePOST}, {Path: "/v1/completions", Role: roleInferencePOST}, {Path: "/v1/embeddings", Role: roleInferencePOST}, +} + +// anthropicInferenceRoutes is the Anthropic-compatible inference surface. +var anthropicInferenceRoutes = []route{ {Path: "/v1/messages", Role: roleInferencePOST}, } @@ -130,19 +133,27 @@ var profiles = buildProfiles() func buildProfiles() []engineProfile { ollama, _ := engines.ByName("ollama") lmstudio, _ := engines.ByName("lmstudio") + ollamaRoutes := []route{ + {Path: "/api/generate", Role: roleInferencePOST}, + {Path: "/api/chat", Role: roleInferencePOST}, + {Path: "/api/embeddings", Role: roleInferencePOST}, + {Path: "/api/embed", Role: roleInferencePOST}, + {Path: "/api/tags", Role: roleModelListNativeGET}, + {Path: "/v1/models", Role: roleModelListOpenAIGET}, + } + ollamaRoutes = append(ollamaRoutes, openAIInferenceRoutes...) + ollamaRoutes = append(ollamaRoutes, anthropicInferenceRoutes...) + lmstudioRoutes := []route{ + {Path: "/v1/models", Role: roleModelListOpenAIGET}, + } + lmstudioRoutes = append(lmstudioRoutes, openAIInferenceRoutes...) + lmstudioRoutes = append(lmstudioRoutes, anthropicInferenceRoutes...) return []engineProfile{ { - Engine: ollama, - StandalonePort: 11435, - Routes: append([]route{ - {Path: "/api/generate", Role: roleInferencePOST}, - {Path: "/api/chat", Role: roleInferencePOST}, - {Path: "/api/embeddings", Role: roleInferencePOST}, - {Path: "/api/embed", Role: roleInferencePOST}, - {Path: "/api/tags", Role: roleModelListNativeGET}, - {Path: "/v1/models", Role: roleModelListOpenAIGET}, - }, openAIInferenceRoutes...), + Engine: ollama, + StandalonePort: 11435, + Routes: ollamaRoutes, ModelNaming: impliedLatestTag, ReservedPersistedPort: 0, SupportsHostAlias: true, @@ -150,10 +161,8 @@ func buildProfiles() []engineProfile { { Engine: lmstudio, StandalonePort: 1234, - Routes: append([]route{ - {Path: "/v1/models", Role: roleModelListOpenAIGET}, - }, openAIInferenceRoutes...), - ModelNaming: exactID, + Routes: lmstudioRoutes, + ModelNaming: exactID, // 1235 is where engine-manager runs a managed LM Studio, so a // proxy that restored it would sit on the engine's own port. The // stored value predates the current default of 1234. diff --git a/services/nvpair-proxy/failover_test.go b/services/nvpair-proxy/failover_test.go index 631180b3..0d54b6fe 100644 --- a/services/nvpair-proxy/failover_test.go +++ b/services/nvpair-proxy/failover_test.go @@ -354,58 +354,6 @@ func TestHandleHTTP_AllNodesDownReturnsError(t *testing.T) { }) } -// TestHandleHTTP_InferenceRouting proves every path classified as inference -// applies model-based candidate filtering and is forwarded unchanged — not -// only the single engineCase.inferencePath most bodies use. That includes -// shared paths such as /v1/messages. -func TestHandleHTTP_InferenceRouting(t *testing.T) { - forEachEngine(t, func(t *testing.T, tc engineCase) { - for _, r := range tc.profile.Routes { - if r.Role != roleInferencePOST { - continue - } - path := r.Path - t.Run(path, func(t *testing.T) { - var gotBody string - var gotPath string - good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - b, _ := io.ReadAll(r.Body) - gotBody = string(b) - gotPath = r.URL.Path - w.WriteHeader(http.StatusOK) - })) - defer good.Close() - - wrongModel := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - t.Error("wrong-model node should not receive request") - w.WriteHeader(http.StatusOK) - })) - defer wrongModel.Close() - - disc := NewDiscovery() - disc.AddManual(nodeForModel(t, "wrong", wrongModel.URL, "different-model")) - disc.AddManual(nodeForModel(t, "good", good.URL, tc.advertisedModel)) - p := testProxy(tc.profile, disc, tc.profile.FacadePort) - p.soleFacade().SetSelected("wrong") - - body := tc.inferenceBody() - rec := httptest.NewRecorder() - p.soleFacade().handleHTTP(rec, httptest.NewRequest(http.MethodPost, path, strings.NewReader(body))) - - if rec.Code != http.StatusOK { - t.Fatalf("status = %d, want 200", rec.Code) - } - if gotBody != body { - t.Errorf("node got body %q, want %q", gotBody, body) - } - if gotPath != path { - t.Errorf("path = %q, want %q", gotPath, path) - } - }) - } - }) -} - // TestHandleHTTP_404FailoverInferenceOnly: a 404 (model-not-found) on an // inference call fails over to the next advertised owner, but a 404 on a // non-inference path is returned as-is. @@ -431,24 +379,15 @@ func TestHandleHTTP_404FailoverInferenceOnly(t *testing.T) { return p } - // Inference POST: 404 on first → fail over → 200, on every classified - // inference path (including shared routes such as /v1/messages). - for _, r := range tc.profile.Routes { - if r.Role != roleInferencePOST { - continue - } - path := r.Path - t.Run(path, func(t *testing.T) { - rec := httptest.NewRecorder() - newProxy().soleFacade().handleHTTP(rec, httptest.NewRequest(http.MethodPost, path, strings.NewReader(tc.inferenceBody()))) - if rec.Code != http.StatusOK { - t.Fatalf("inference 404: status = %d, want 200 (should fail over)", rec.Code) - } - }) + // Inference POST: 404 on first → fail over → 200. + rec := httptest.NewRecorder() + newProxy().soleFacade().handleHTTP(rec, tc.inferenceRequest()) + if rec.Code != http.StatusOK { + t.Fatalf("inference 404: status = %d, want 200 (should fail over)", rec.Code) } // An ordinary non-inference GET still returns the first node's 404. - rec := httptest.NewRecorder() + rec = httptest.NewRecorder() newProxy().soleFacade().handleHTTP(rec, httptest.NewRequest(http.MethodGet, tc.nonInferencePath, nil)) if rec.Code != http.StatusNotFound { t.Fatalf("non-inference 404: status = %d, want 404 (must NOT fail over)", rec.Code) @@ -702,6 +641,87 @@ func TestHandleHTTP_StrictModelRouting(t *testing.T) { }) } +// TestHandleHTTP_InferenceRouting proves each engine inference route is +// model-routed, retries a model-not-found response, and +// forwards the request path and body unchanged. +func TestHandleHTTP_InferenceRouting(t *testing.T) { + test := func(name, path string, profiles ...engineProfile) { + t.Run(name, func(t *testing.T) { + for _, profile := range profiles { + t.Run(profile.Name, func(t *testing.T) { + requestedModel := "requested-model" + advertisedModel := profile.normalizeModel(requestedModel) + + wrongModel := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + t.Error("wrong-model node should not receive request") + w.WriteHeader(http.StatusOK) + })) + defer wrongModel.Close() + + missing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + if _, err := io.WriteString(w, `{"error":"model not found"}`); err != nil { + t.Errorf("write missing-model response: %v", err) + } + })) + defer missing.Close() + + var gotBody string + var gotPath string + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("read forwarded request body: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + gotBody = string(body) + gotPath = r.URL.Path + w.WriteHeader(http.StatusOK) + })) + defer good.Close() + + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "wrong", wrongModel.URL, "different-model")) + disc.AddManual(nodeForModel(t, "missing", missing.URL, advertisedModel)) + disc.AddManual(nodeForModel(t, "good", good.URL, advertisedModel)) + p := testProxy(profile, disc, profile.FacadePort) + p.soleFacade().SetSelected("wrong") + // Stable node ordering would send this request to "good" first and + // never exercise 404 failover. Prioritize "missing" so the test + // independently proves both model filtering and retry behavior. + p.SetPriority([]string{"missing", "good"}) + + body := `{"model":"requested-model","max_tokens":1024,"messages":[{"role":"user","content":"Hello"}]}` + rec := httptest.NewRecorder() + p.soleFacade().handleHTTP(rec, httptest.NewRequest(http.MethodPost, path, strings.NewReader(body))) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 after 404 failover", rec.Code) + } + if gotBody != body { + t.Errorf("node got body %q, want %q", gotBody, body) + } + if gotPath != path { + t.Errorf("path = %q, want %q", gotPath, path) + } + }) + } + }) + } + + ollama := ollamaCase(t).profile + lmstudio := lmstudioCase(t).profile + test("native Ollama generate", "/api/generate", ollama) + test("native Ollama chat", "/api/chat", ollama) + test("native Ollama embeddings", "/api/embeddings", ollama) + test("native Ollama embed", "/api/embed", ollama) + test("OpenAI chat completions", "/v1/chat/completions", ollama, lmstudio) + test("OpenAI completions", "/v1/completions", ollama, lmstudio) + test("OpenAI embeddings", "/v1/embeddings", ollama, lmstudio) + test("Anthropic messages", "/v1/messages", ollama, lmstudio) +} + func TestHandleHTTP_NoAdvertisedModelRejectsLocally(t *testing.T) { forEachEngine(t, func(t *testing.T, tc engineCase) { hits := 0 diff --git a/services/tests/model_routing_interop_test.go b/services/tests/model_routing_interop_test.go index e7c77475..58ef22ea 100644 --- a/services/tests/model_routing_interop_test.go +++ b/services/tests/model_routing_interop_test.go @@ -113,7 +113,9 @@ func TestStrictModelRoutingAcrossProcesses(t *testing.T) { requestID++ } callBrokerRPC(t, stdin, msgs, requestID, tc.rpcPrefix+":node/set-priority", map[string]any{ - "generation": 1, + // Both facades share one proxy process, so every snapshot must + // advance the process-wide generation. + "generation": caseIndex + 1, "nodes": []string{missingID, unknownID, owner404ID, ownerOKID}, }) From 1a9847e4ffb7278a6a4d5e9ae5dc6ab676d0cd50 Mon Sep 17 00:00:00 2001 From: Kaylee Lubick Date: Wed, 23 Sep 2026 08:36:13 -0400 Subject: [PATCH 5/5] refactor: extract engine base routes Signed-off-by: Kaylee Lubick --- services/nvpair-proxy/engines.go | 37 ++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/services/nvpair-proxy/engines.go b/services/nvpair-proxy/engines.go index d76f7127..71201a30 100644 --- a/services/nvpair-proxy/engines.go +++ b/services/nvpair-proxy/engines.go @@ -20,6 +20,7 @@ package main // nvpair-shared/engines. import ( + "slices" "strings" "nvpair-shared/engines" @@ -116,6 +117,23 @@ type engineProfile struct { SupportsHostAlias bool } +// ollamaBaseRoutes is the engine-specific surface that Ollama exposes before +// the shared compatibility routes are added. +var ollamaBaseRoutes = []route{ + {Path: "/api/generate", Role: roleInferencePOST}, + {Path: "/api/chat", Role: roleInferencePOST}, + {Path: "/api/embeddings", Role: roleInferencePOST}, + {Path: "/api/embed", Role: roleInferencePOST}, + {Path: "/api/tags", Role: roleModelListNativeGET}, + {Path: "/v1/models", Role: roleModelListOpenAIGET}, +} + +// lmStudioBaseRoutes is the engine-specific surface that LM Studio exposes +// before the shared compatibility routes are added. +var lmStudioBaseRoutes = []route{ + {Path: "/v1/models", Role: roleModelListOpenAIGET}, +} + // openAIInferenceRoutes is the OpenAI-compatible inference surface. var openAIInferenceRoutes = []route{ {Path: "/v1/chat/completions", Role: roleInferencePOST}, @@ -133,21 +151,8 @@ var profiles = buildProfiles() func buildProfiles() []engineProfile { ollama, _ := engines.ByName("ollama") lmstudio, _ := engines.ByName("lmstudio") - ollamaRoutes := []route{ - {Path: "/api/generate", Role: roleInferencePOST}, - {Path: "/api/chat", Role: roleInferencePOST}, - {Path: "/api/embeddings", Role: roleInferencePOST}, - {Path: "/api/embed", Role: roleInferencePOST}, - {Path: "/api/tags", Role: roleModelListNativeGET}, - {Path: "/v1/models", Role: roleModelListOpenAIGET}, - } - ollamaRoutes = append(ollamaRoutes, openAIInferenceRoutes...) - ollamaRoutes = append(ollamaRoutes, anthropicInferenceRoutes...) - lmstudioRoutes := []route{ - {Path: "/v1/models", Role: roleModelListOpenAIGET}, - } - lmstudioRoutes = append(lmstudioRoutes, openAIInferenceRoutes...) - lmstudioRoutes = append(lmstudioRoutes, anthropicInferenceRoutes...) + ollamaRoutes := slices.Concat(ollamaBaseRoutes, openAIInferenceRoutes, anthropicInferenceRoutes) + lmStudioRoutes := slices.Concat(lmStudioBaseRoutes, openAIInferenceRoutes, anthropicInferenceRoutes) return []engineProfile{ { @@ -161,7 +166,7 @@ func buildProfiles() []engineProfile { { Engine: lmstudio, StandalonePort: 1234, - Routes: lmstudioRoutes, + Routes: lmStudioRoutes, ModelNaming: exactID, // 1235 is where engine-manager runs a managed LM Studio, so a // proxy that restored it would sit on the engine's own port. The