diff --git a/README.md b/README.md index 0055f49..2be276f 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ - Browser OAuth helper for obtaining a Command Code API key (CLI or WebUI); each OAuth run adds an account - Local bearer-token auth for clients, separate admin password for the WebUI - Usage counters (global, per-account, per-client-key) and cached quota snapshots persisted to `usage.json` -- Base64 `image_url` conversion to Command Code image blocks; CORS enabled for local UI clients +- Multimodal image input: base64 `image_url`, string-form `image_url`, `input_image`, and Anthropic `image` blocks, converted to Command Code image blocks; CORS enabled for local UI clients - `GET /health` and `GET /usage` endpoints ## Quick start @@ -293,7 +293,14 @@ deepseek-ai/deepseek-v4-flash ✗ wrong provider prefix **Supported request styles:** plain text messages, multimodal content arrays, `stream: true` server-sent events, and `stream: false` JSON responses. -**Images:** multimodal `image_url` values must be base64 `data:image/...;base64,...` URLs. Remote HTTP(S) image URLs are rejected with `400 invalid_request_error` — the gateway never downloads remote images. +**Images:** the gateway accepts several image shapes and converts them to Command Code image blocks: + +- `{"type":"image_url","image_url":{"url":"data:image/...;base64,..."}}` — standard OpenAI +- `{"type":"image_url","image_url":"data:image/...;base64,..."}` — string form +- `{"type":"input_image","image_url":"data:image/...;base64,..."}` +- `{"type":"image","source":{"type":"base64","media_type":"...","data":"..."}}` — Anthropic style + +Image data must be inline base64 (`data:image/...;base64,...`). Remote HTTP(S) image URLs and any other URL scheme are rejected with `400 invalid_request_error` — the gateway never downloads remote images. Upstream `inputTokenDetails.cacheReadTokens` is exposed in the response as `usage.prompt_tokens_details.cached_tokens`. `cacheWriteTokens` is available in `/usage` but not in the Chat Completions response, because OpenAI's standard usage schema has no cache-write field. diff --git a/README.zh-CN.md b/README.zh-CN.md index 0b8a494..854e586 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -14,7 +14,7 @@ - 浏览器 OAuth 助手获取 Command Code API Key(CLI 或 WebUI),每次授权添加一个账号 - 客户端 Bearer Token 鉴权,WebUI 使用独立管理密码 - 用量计数(全局、按账号、按密钥)与额度快照持久化在 `usage.json` -- base64 `image_url` 转 Command Code 图片块;为本地 UI 客户端开启 CORS +- 多模态图片输入:支持 base64 `image_url`、字符串形式 `image_url`、`input_image` 以及 Anthropic `image` 块,并转换为 Command Code 图片块;为本地 UI 客户端开启 CORS - `GET /health` 与 `GET /usage` 端点 ## 快速开始 @@ -293,7 +293,14 @@ deepseek-ai/deepseek-v4-flash ✗ provider 前缀错误 **支持的请求形式**:纯文本消息、多模态 content 数组、`stream: true` 的 SSE 流式响应、`stream: false` 的 JSON 响应。 -**图片**:多模态 `image_url` 必须使用 `data:image/...;base64,...` 形式;远程 HTTP(S) 图片地址返回 `400 invalid_request_error`,服务不会主动下载远程图片。 +**图片**:网关接受多种图片形态,并转换为 Command Code 图片块: + +- `{"type":"image_url","image_url":{"url":"data:image/...;base64,..."}}` —— 标准 OpenAI +- `{"type":"image_url","image_url":"data:image/...;base64,..."}` —— 字符串形式 +- `{"type":"input_image","image_url":"data:image/...;base64,..."}` +- `{"type":"image","source":{"type":"base64","media_type":"...","data":"..."}}` —— Anthropic 风格 + +图片数据必须是内联 base64(`data:image/...;base64,...`)。远程 HTTP(S) 图片地址及其它 URL scheme 返回 `400 invalid_request_error`,服务不会主动下载远程图片。 上游 `inputTokenDetails.cacheReadTokens` 在响应中对应 `usage.prompt_tokens_details.cached_tokens`。`cacheWriteTokens` 可在 `/usage` 查看,但不出现在 Chat Completions 响应中——OpenAI 的标准 usage 结构没有 cache-write 字段。 diff --git a/internal/app/cc_test.go b/internal/app/cc_test.go index 2bf8ba9..406d435 100644 --- a/internal/app/cc_test.go +++ b/internal/app/cc_test.go @@ -413,3 +413,27 @@ func TestUpstreamClientIdentityHeaders(t *testing.T) { t.Errorf("ccCLIVersion = %q, want a dotted version", ccCLIVersion) } } + +func TestContentPartNormalizesImageShapes(t *testing.T) { + const dataURL = "data:image/png;base64,QUJD" + cases := []struct { + name string + raw string + }{ + {"openai object", `{"type":"image_url","image_url":{"url":"` + dataURL + `","detail":"high"}}`}, + {"openai string", `{"type":"image_url","image_url":"` + dataURL + `"}`}, + {"responses input_image", `{"type":"input_image","image_url":"` + dataURL + `"}`}, + {"anthropic source", `{"type":"image","source":{"type":"base64","media_type":"image/png","data":"QUJD"}}`}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + var part ContentPart + if err := json.Unmarshal([]byte(c.raw), &part); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if part.Type != "image_url" || part.ImageURL == nil || part.ImageURL.URL != dataURL { + t.Fatalf("normalized part = %#v, want type=image_url url=%q", part, dataURL) + } + }) + } +} diff --git a/internal/app/types.go b/internal/app/types.go index 74c9440..9970903 100644 --- a/internal/app/types.go +++ b/internal/app/types.go @@ -137,6 +137,106 @@ type ImageURL struct { Detail string `json:"detail,omitempty"` } +// rawContentPart is the wire shape of one content part as sent by clients. Its +// accessors normalize the image variants clients actually use. +type rawContentPart struct { + Type string `json:"type"` + Text string `json:"text"` + ImageURL json.RawMessage `json:"image_url"` + Source json.RawMessage `json:"source"` +} + +// UnmarshalJSON accepts every image shape common OpenAI-compatible clients +// emit and normalizes it to the canonical +// {"type":"image_url","image_url":{"url":...}} form so the conversion layer +// only ever sees one representation. It also accepts the Anthropic +// {"type":"image","source":{...}} block. Unknown types are kept verbatim so +// contentToCC can reject them with a precise message. +func (p *ContentPart) UnmarshalJSON(data []byte) error { + var raw rawContentPart + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + p.Type = raw.Type + p.Text = raw.Text + if raw.Type == "text" { + return nil + } + + switch raw.Type { + case "image_url", "input_image": + imageURL, detail, err := raw.imageURL() + if err != nil { + return err + } + p.Type = "image_url" + p.ImageURL = &ImageURL{URL: imageURL, Detail: detail} + case "image": + // Anthropic-style block. Fall back to a plain image_url if a client + // sends {"type":"image","image_url":...} instead of a source object. + if imageURL, err := raw.anthropicSource(); err != nil { + return err + } else if imageURL != "" { + p.Type = "image_url" + p.ImageURL = &ImageURL{URL: imageURL} + return nil + } + imageURL, detail, err := raw.imageURL() + if err != nil { + return err + } + p.Type = "image_url" + p.ImageURL = &ImageURL{URL: imageURL, Detail: detail} + } + return nil +} + +// imageURL accepts both the standard object form {"url":...,"detail":...} and +// the bare string form "data:...". +func (r rawContentPart) imageURL() (url, detail string, err error) { + if len(r.ImageURL) == 0 || string(r.ImageURL) == "null" { + return "", "", nil + } + var asString string + if err := json.Unmarshal(r.ImageURL, &asString); err == nil { + return asString, "", nil + } + var obj ImageURL + if err := json.Unmarshal(r.ImageURL, &obj); err != nil { + return "", "", fmt.Errorf("image_url must be a string or an object with a url") + } + return obj.URL, obj.Detail, nil +} + +// anthropicSource converts {"type":"base64","media_type":...,"data":...} (or a +// url source) into a data URL the rest of the pipeline understands. It returns +// an empty string when no source is present. +func (r rawContentPart) anthropicSource() (string, error) { + if len(r.Source) == 0 || string(r.Source) == "null" { + return "", nil + } + var src struct { + Type string `json:"type"` + MediaType string `json:"media_type"` + Data string `json:"data"` + URL string `json:"url"` + } + if err := json.Unmarshal(r.Source, &src); err != nil { + return "", fmt.Errorf("image source must be an object") + } + if src.URL != "" { + return src.URL, nil + } + if src.Data == "" { + return "", nil + } + mediaType := src.MediaType + if mediaType == "" { + mediaType = "image/png" + } + return "data:" + mediaType + ";base64," + src.Data, nil +} + type Tool struct { Type string `json:"type"` // "function" Function ToolFunction `json:"function"`