diff --git a/README.md b/README.md index ec1aa02..3de818e 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,8 @@ backends: type: gcp_openai project: your-google-cloud-project location: global + # Vertex AI's OpenAI-compatible endpoint does not provide /v1/models. + model_discovery: false auth: type: google_adc models: diff --git a/docs/EXAMPLES.md b/docs/EXAMPLES.md index 4c3b4eb..ec9ac1d 100644 --- a/docs/EXAMPLES.md +++ b/docs/EXAMPLES.md @@ -41,6 +41,8 @@ backends: type: gcp_openai project: example location: global + # Vertex AI's OpenAI-compatible endpoint does not provide /v1/models. + model_discovery: false auth: type: google_adc models: @@ -202,6 +204,8 @@ backends: type: gcp_openai project: example location: global + # Vertex AI's OpenAI-compatible endpoint does not provide /v1/models. + model_discovery: false auth: type: google_adc models: diff --git a/docs/USAGE.md b/docs/USAGE.md index 7b9e447..e0b5ad4 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -95,14 +95,14 @@ The app uses the TUI when stdout is an interactive terminal and `--headless` is The TUI has the following views, accessible with **Tab**, **←/→**, or **h/l**: - **Stats** – Displays per-model statistics and recent requests (the default view). -- **Models** – Queries every upstream backend's `/models` endpoint and compares the response with the configured models. +- **Models** – Queries each enabled upstream backend's `/models` endpoint and compares the response with the configured models. - **Test** – Lists all configured models. Use **↑/↓** to select a model and **Enter** to send a test request. The response is displayed inline. Test requests go through the proxy like any other request, so they count towards stats and token usage. ### Model Discovery Diagnostics -Opening the **Models** tab calls the OpenAI-compatible `/models` endpoint on each configured upstream backend. The request uses the backend's configured base URL, authentication, HTTP client, and TLS settings. It does not compare against the proxy's own `GET /v1/models` endpoint, because that endpoint is generated from the config itself. +Opening the **Models** tab calls the OpenAI-compatible `/models` endpoint on each enabled upstream backend. The request uses the backend's configured base URL, authentication, HTTP client, and TLS settings. It does not compare against the proxy's own `GET /v1/models` endpoint, because that endpoint is generated from the config itself. Set a backend's `model_discovery: false` when its provider does not support the endpoint; the TUI shows it as disabled instead of sending a request. The combined list distinguishes the source and consistency of every model: @@ -130,6 +130,7 @@ Each backend declares where requests go, how they authenticate, and which models | `project` | yes* | Google Cloud project for `gcp_openai` when `base_url` is omitted. | | `location` | yes* | Google Cloud location for `gcp_openai` when `base_url` is omitted. Defaults to `global` when project is configured. | | `insecure_skip_verify` | no | Disables TLS certificate verification for backend API calls. Defaults to `false`. | +| `model_discovery` | no | Whether the Models TUI tab queries this backend's upstream `/v1/models` endpoint. Defaults to `true`; set to `false` for providers that do not support it. | | `auth` | yes | Auth configuration. | | `models` | yes | `all` or a list of model entries. | diff --git a/internal/config/config.go b/internal/config/config.go index f000d18..0da8e65 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -45,10 +45,17 @@ type BackendConfig struct { Project string `yaml:"project"` Location string `yaml:"location"` InsecureSkipVerify bool `yaml:"insecure_skip_verify"` + ModelDiscovery *bool `yaml:"model_discovery"` Auth AuthConfig `yaml:"auth"` Models BackendModels `yaml:"models"` } +// ModelDiscoveryEnabled returns whether the Models TUI tab should query this +// backend's upstream /v1/models endpoint. It defaults to true. +func (bc *BackendConfig) ModelDiscoveryEnabled() bool { + return bc.ModelDiscovery == nil || *bc.ModelDiscovery +} + // AuthConfig describes how to authenticate against a backend. type AuthConfig struct { Type string `yaml:"type"` // none, bearer, google_adc, oauth_client_credentials diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 345883a..1708a3d 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -79,6 +79,34 @@ ui: } } +func TestLoadBackendModelDiscovery(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(` +backends: + - name: google + type: gcp_openai + project: example + model_discovery: false + auth: + type: google_adc + models: + - id: gemini +`), 0600); err != nil { + t.Fatal(err) + } + + cfg, err := Load(Flags{ConfigPath: path}) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.Backends[0].ModelDiscoveryEnabled() { + t.Fatal("expected model discovery to be disabled") + } + if (&BackendConfig{}).ModelDiscoveryEnabled() == false { + t.Fatal("expected model discovery to default to enabled") + } +} + func TestRejectsInvalidUIRecentRequests(t *testing.T) { cfg := Config{ Server: ServerConfig{Host: DefaultHost, Port: DefaultPort}, diff --git a/internal/proxy/models.go b/internal/proxy/models.go index eea047c..75a6d7b 100644 --- a/internal/proxy/models.go +++ b/internal/proxy/models.go @@ -26,6 +26,7 @@ type ModelDiscovery struct { StatusCode int Models []DiscoveredModel Err error + Skipped bool } // DiscoverModels queries every configured backend's OpenAI-compatible /models @@ -36,6 +37,10 @@ func (p *Proxy) DiscoverModels(ctx context.Context) []ModelDiscovery { results := make([]ModelDiscovery, len(p.backends)) var wg sync.WaitGroup for i, backend := range p.backends { + if !backend.cfg.ModelDiscoveryEnabled() { + results[i] = ModelDiscovery{Backend: backend.cfg.Name, Skipped: true} + continue + } wg.Add(1) go func() { defer wg.Done() @@ -87,7 +92,7 @@ func (p *Proxy) discoverBackendModels(ctx context.Context, backend *resolvedBack return result } if resp.StatusCode < 200 || resp.StatusCode >= 300 { - result.Err = fmt.Errorf("HTTP %d: %s", resp.StatusCode, truncateForError(string(body), 200)) + result.Err = fmt.Errorf("HTTP %d %s", resp.StatusCode, http.StatusText(resp.StatusCode)) return result } @@ -115,10 +120,3 @@ func (p *Proxy) discoverBackendModels(ctx context.Context, backend *resolvedBack sort.Slice(result.Models, func(i, j int) bool { return result.Models[i].ID < result.Models[j].ID }) return result } - -func truncateForError(value string, limit int) string { - if len(value) <= limit { - return value - } - return value[:limit] + "..." -} diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index aed76a1..3fa24fc 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -142,11 +142,30 @@ func TestDiscoverModelsReportsBackendHTTPError(t *testing.T) { handler := mustNew(t, Options{Config: cfg, TokenProvider: StaticTokenProvider("token"), Metrics: NewMetrics()}) results := handler.DiscoverModels(context.Background()) - if len(results) != 1 || results[0].Err == nil || !strings.Contains(results[0].Err.Error(), "HTTP 503") { + if len(results) != 1 || results[0].Err == nil || results[0].Err.Error() != "HTTP 503 Service Unavailable" { t.Fatalf("expected per-backend HTTP error, got %#v", results) } } +func TestDiscoverModelsSkipsDisabledBackends(t *testing.T) { + called := false + upstream := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + called = true + })) + defer upstream.Close() + + disabled := false + cfg := testConfig([]config.Model{{ID: "model"}}) + cfg.Backends[0].BaseURL = upstream.URL + "/v1" + cfg.Backends[0].ModelDiscovery = &disabled + handler := mustNew(t, Options{Config: cfg, TokenProvider: StaticTokenProvider("token"), Metrics: NewMetrics()}) + + results := handler.DiscoverModels(context.Background()) + if len(results) != 1 || !results[0].Skipped || results[0].Err != nil || called { + t.Fatalf("expected disabled discovery without an upstream request, got %#v (called=%t)", results, called) + } +} + func TestChatCompletionsMissingModel(t *testing.T) { handler := mustNew(t, Options{ Config: testConfig([]config.Model{{ID: "google/gemini-2.5-flash"}}), diff --git a/internal/ui/ui.go b/internal/ui/ui.go index 8e42338..27d83b6 100644 --- a/internal/ui/ui.go +++ b/internal/ui/ui.go @@ -438,7 +438,7 @@ func reconcileModels(cfg *config.Config, results []proxy.ModelDiscovery) []diagn row.status, row.rowStyle = "MATCH", "green" row.details = combinedModelDetails(backend, configured, found) matched[upstreamID] = true - } else if !queried || result.Err != nil { + } else if !queried || result.Err != nil || result.Skipped { row.status = "UNKNOWN" row.rowStyle = "yellow" } @@ -659,7 +659,9 @@ func (m tuiModel) viewModels(tableWidth int) string { } if m.discoveryLoaded { for _, result := range m.discoveryResults { - if result.Err != nil { + if result.Skipped { + fmt.Fprintf(&b, "%s %s\n", style(m.color, "muted").Render(result.Backend+":"), style(m.color, "muted").Render("model discovery disabled by configuration")) + } else if result.Err != nil { fmt.Fprintf(&b, "%s %s\n", style(m.color, "red").Render(result.Backend+":"), style(m.color, "red").Render(result.Err.Error())) } else { fmt.Fprintf(&b, "%s %s\n", style(m.color, "green").Render(result.Backend+":"), style(m.color, "muted").Render(fmt.Sprintf("HTTP %d, %d models from %s", result.StatusCode, len(result.Models), result.URL))) diff --git a/internal/ui/ui_test.go b/internal/ui/ui_test.go index 2387b12..7e4bcb2 100644 --- a/internal/ui/ui_test.go +++ b/internal/ui/ui_test.go @@ -98,6 +98,19 @@ func TestReconcileModelsUsesUnknownWhenDiscoveryFails(t *testing.T) { } } +func TestModelsViewShowsDisabledDiscoveryWithoutError(t *testing.T) { + m := testTUIModel() + m.activeTab = tabModels + m.discoveryLoaded = true + m.discoveryResults = []proxy.ModelDiscovery{{Backend: "google", Skipped: true}} + m.diagnosticModels = reconcileModels(m.cfg, m.discoveryResults) + + view := m.viewModels(m.tableWidth()) + if !strings.Contains(view, "model discovery disabled by configuration") { + t.Fatalf("expected disabled-discovery message, got:\n%s", view) + } +} + func TestModelsTabOpensDetailsAndReturns(t *testing.T) { m := testTUIModel() m.activeTab = tabModels