diff --git a/src/config.test.ts b/src/config.test.ts index 51246a2..74a9360 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -4,7 +4,8 @@ import os from "node:os"; import path from "node:path"; import test from "node:test"; import YAML from "yaml"; -import { configPath, initializeWorkspace, isOnboardingComplete, loadConfig, markOnboardingComplete, oobeProgressPath, readOobeProgress, readSettings, saveOobeProgress, saveSettings, type OobeProgress } from "./config.js"; +import { OFFICIAL_VISION_MODEL, configPath, initializeWorkspace, isOnboardingComplete, loadConfig, markOnboardingComplete, oobeProgressPath, readOobeProgress, readSettings, resolveVisionAgentConfig, saveOobeProgress, saveSettings, type OobeProgress } from "./config.js"; +import type { SecAgentConfig } from "./types.js"; import { SYSTEM_PROMPT } from "./system-prompt.js"; function temporaryWorkspace(): string { @@ -121,3 +122,87 @@ test("defaults and persists Windows update preferences", () => { fs.rmSync(workspace, { recursive: true, force: true }); } }); + +test("persists the vision model id with the settings", () => { + const workspace = temporaryWorkspace(); + try { + initializeWorkspace(workspace); + const settings = readSettings(workspace); + const saved = saveSettings(workspace, { ...settings, visionModelId: "custom-provider:vision" }); + assert.equal(saved.visionModelId, "custom-provider:vision"); + assert.equal(readSettings(workspace).visionModelId, "custom-provider:vision"); + // Clearing the value removes the persisted key. + const cleared = saveSettings(workspace, { ...settings, visionModelId: undefined }); + assert.equal(cleared.visionModelId, undefined); + assert.equal(readSettings(workspace).visionModelId, undefined); + } finally { + fs.rmSync(workspace, { recursive: true, force: true }); + } +}); + +function multiModelConfig(overrides: Partial = {}): SecAgentConfig { + return { + version: 1, + workspace: "/tmp/secagent-test-ws", + agent: { + provider: "openai-compatible", + model: "main", + apiKeyEnv: "MAIN_KEY", + baseUrl: "https://main.test/v1", + endpoint: "/chat/completions", + maxTokens: 100, + systemPrompt: "main", + models: [ + { id: "main", provider: "openai-compatible", model: "main", apiKeyEnv: "MAIN_KEY", baseUrl: "https://main.test/v1", endpoint: "/chat/completions", maxTokens: 100 }, + { id: "vision", provider: "openai-compatible", model: "vision-model", apiKeyEnv: "VISION_KEY", baseUrl: "https://vision.test/v1", endpoint: "/chat/completions", maxTokens: 100 }, + { id: "sectl-official:deepseek-v4-flash", provider: "openai-responses", model: "deepseek-v4-flash", apiKeyEnv: "SECTL_OFFICIAL_TOKEN", baseUrl: "https://relay.test/v1", endpoint: "/responses", maxTokens: 100 } + ] + }, + mcp: { servers: {} }, + ...overrides + } as SecAgentConfig; +} + +test("resolveVisionAgentConfig honors an explicit vision model id without mutating the source config", () => { + const config = multiModelConfig({ defaults: { modelId: "main", visionModelId: "vision" } }); + const vision = resolveVisionAgentConfig(config); + assert.ok(vision); + assert.equal(vision.agent.model, "vision-model"); + assert.equal(vision.agent.apiKeyEnv, "VISION_KEY"); + assert.equal(vision.agent.baseUrl, "https://vision.test/v1"); + // The caller's config keeps its own model. + assert.equal(config.agent.model, "main"); +}); + +test("resolveVisionAgentConfig returns undefined for a stale vision model id", () => { + const config = multiModelConfig({ defaults: { visionModelId: "does-not-exist" } }); + assert.equal(resolveVisionAgentConfig(config), undefined); +}); + +test("resolveVisionAgentConfig falls back to the official virtual-vision model in official mode", () => { + const previous = process.env.SECTL_OFFICIAL_TOKEN; + process.env.SECTL_OFFICIAL_TOKEN = "test-token"; + try { + const config = multiModelConfig({ defaults: { customModelMode: false } }); + const vision = resolveVisionAgentConfig(config); + assert.ok(vision); + assert.equal(vision.agent.model, OFFICIAL_VISION_MODEL); + assert.equal(vision.agent.apiKeyEnv, "SECTL_OFFICIAL_TOKEN"); + assert.equal(vision.agent.endpoint, "/responses"); + } finally { + if (previous === undefined) delete process.env.SECTL_OFFICIAL_TOKEN; + else process.env.SECTL_OFFICIAL_TOKEN = previous; + } +}); + +test("resolveVisionAgentConfig does not fall back in custom model mode", () => { + const previous = process.env.SECTL_OFFICIAL_TOKEN; + process.env.SECTL_OFFICIAL_TOKEN = "test-token"; + try { + const config = multiModelConfig({ defaults: { customModelMode: true } }); + assert.equal(resolveVisionAgentConfig(config), undefined); + } finally { + if (previous === undefined) delete process.env.SECTL_OFFICIAL_TOKEN; + else process.env.SECTL_OFFICIAL_TOKEN = previous; + } +}); diff --git a/src/config.ts b/src/config.ts index 4a989a1..9f046fc 100644 --- a/src/config.ts +++ b/src/config.ts @@ -9,6 +9,13 @@ import { SYSTEM_PROMPT } from "./system-prompt.js"; export const DEFAULT_GOOGLE_MODEL = "gemini-2.5-flash"; export const DEFAULT_MAX_TOKENS = 16_384; +/** + * Client-facing virtual vision model served by the official relay. The relay routes this + * model id to a vision-capable upstream, so the client does not need to know which concrete + * model is behind it. Only the client part lives in this repository; the relay contract is + * documented in the README/settings help text. + */ +export const OFFICIAL_VISION_MODEL = "virtual-vision"; const ONBOARDING_MARKER = ".oobe-complete"; const OOBE_PROGRESS_FILE = ".oobe-progress.json"; const LEGACY_AGENT_MODEL_FIELDS = ["provider", "model", "apiKeyEnv", "baseUrl", "endpoint", "anthropicVersion", "maxTokens"] as const; @@ -303,6 +310,41 @@ export function useConfiguredModel(config: SecAgentConfig, id?: string): void { config.agent = { ...config.agent, ...selected, model: dynamicModel || selectedModels[profileIndex] || selectedModels[0] || DEFAULT_GOOGLE_MODEL, maxTokens: selected.maxTokens || config.agent.maxTokens, systemPrompt: config.agent.systemPrompt, models: config.agent.models }; } +/** + * Return a copy of the config with `agent` resolved to the given model id, without + * mutating the caller's config. Unlike `useConfiguredModel`, this is safe to call for + * a secondary (e.g. vision) model while the main session keeps using its own model. + */ +export function resolveModelConfig(config: SecAgentConfig, modelId: string): SecAgentConfig { + const next = { ...config, agent: { ...config.agent } }; + useConfiguredModel(next, modelId); + if (!next.agent.models?.length) throw new Error(`未找到配置模型：${modelId}`); + return next; +} + +/** + * Resolve the dedicated image-recognition model config, if any. + * 1. An explicitly configured `defaults.visionModelId` wins. + * 2. In official (non-custom) mode, fall back to the relay's virtual-vision model so + * the feature works without any manual setup. + * Returns undefined when no vision model is configured or the id is stale — the vision + * tool is then simply not exposed to the main agent. + */ +export function resolveVisionAgentConfig(config: SecAgentConfig): SecAgentConfig | undefined { + const id = config.defaults?.visionModelId; + if (id) { + try { return resolveModelConfig(config, id); } + catch { return undefined; } + } + if (config.defaults?.customModelMode === false + && process.env.SECTL_OFFICIAL_TOKEN + && config.agent.models?.some((model) => model.id.startsWith("sectl-official:"))) { + try { return resolveModelConfig(config, `official:sectl-official:${OFFICIAL_VISION_MODEL}`); } + catch { return undefined; } + } + return undefined; +} + export interface SettingsPayload { providers: Array; /** Compatibility field for older IPC callers; the settings UI uses providers. */ @@ -315,6 +357,8 @@ export interface SettingsPayload { mcp: { servers: Record }; defaultModelId?: string; defaultReasoningEffort?: ReasoningEffort; + /** Model used by the dedicated image-recognition tool when the main model has no vision input. */ + visionModelId?: string; autostart?: boolean; /** On by default: an autostart launch stays in the tray instead of opening the main window. */ autostartHidden?: boolean; @@ -338,7 +382,7 @@ export function readSettings(workspaceInput: string): SettingsPayload { maxTokens: config.agent.maxTokens }]; const providers = config.agent.providers?.length ? config.agent.providers : groupLegacyModels(configured); - return { providers: providers.map((provider) => ({ ...provider, apiKeyConfigured: Boolean(process.env[provider.apiKeyEnv]) })), models: configured.map((model) => ({ ...model, apiKeyConfigured: Boolean(process.env[model.apiKeyEnv]) })), tts: { voice: config.tts?.voice || DEFAULT_TTS_VOICE, rate: config.tts?.rate || DEFAULT_TTS_RATE }, wake: { hotkey: config.wake?.hotkey || DEFAULT_WAKE_HOTKEY, ...(config.wake?.modelId ? { modelId: config.wake.modelId } : {}), voiceEnabled: config.wake?.voiceEnabled === true, voicePhrase: config.wake?.voicePhrase || DEFAULT_WAKE_PHRASE }, speech: { betterRecognition: config.speech?.betterRecognition === true }, updates: { ...(config.updates || DEFAULT_UPDATE_PREFERENCES) }, telemetry: { enabled: config.telemetry?.enabled !== false }, mcp: config.mcp, defaultModelId: config.defaults?.modelId, defaultReasoningEffort: config.defaults?.reasoningEffort, autostart: config.defaults?.autostart === true, autostartHidden: config.defaults?.autostartHidden !== false, customModelMode: config.defaults?.customModelMode ?? false }; + return { providers: providers.map((provider) => ({ ...provider, apiKeyConfigured: Boolean(process.env[provider.apiKeyEnv]) })), models: configured.map((model) => ({ ...model, apiKeyConfigured: Boolean(process.env[model.apiKeyEnv]) })), tts: { voice: config.tts?.voice || DEFAULT_TTS_VOICE, rate: config.tts?.rate || DEFAULT_TTS_RATE }, wake: { hotkey: config.wake?.hotkey || DEFAULT_WAKE_HOTKEY, ...(config.wake?.modelId ? { modelId: config.wake.modelId } : {}), voiceEnabled: config.wake?.voiceEnabled === true, voicePhrase: config.wake?.voicePhrase || DEFAULT_WAKE_PHRASE }, speech: { betterRecognition: config.speech?.betterRecognition === true }, updates: { ...(config.updates || DEFAULT_UPDATE_PREFERENCES) }, telemetry: { enabled: config.telemetry?.enabled !== false }, mcp: config.mcp, defaultModelId: config.defaults?.modelId, defaultReasoningEffort: config.defaults?.reasoningEffort, visionModelId: config.defaults?.visionModelId, autostart: config.defaults?.autostart === true, autostartHidden: config.defaults?.autostartHidden !== false, customModelMode: config.defaults?.customModelMode ?? false }; } function groupLegacyModels(models: ModelProfile[]): ProviderConfig[] { @@ -388,7 +432,7 @@ export function saveSettings(workspaceInput: string, payload: SettingsPayload): raw.updates = nextUpdates; raw.telemetry = nextTelemetry; raw.mcp = payload.mcp; - raw.defaults = { modelId: payload.defaultModelId || undefined, reasoningEffort: payload.defaultReasoningEffort || undefined, customModelMode: Boolean(payload.customModelMode), autostart: payload.autostart === true, autostartHidden: payload.autostartHidden !== false }; + raw.defaults = { modelId: payload.defaultModelId || undefined, reasoningEffort: payload.defaultReasoningEffort || undefined, customModelMode: Boolean(payload.customModelMode), autostart: payload.autostart === true, autostartHidden: payload.autostartHidden !== false, visionModelId: payload.visionModelId || undefined }; delete (raw as SecAgentConfig & { policy?: unknown }).policy; fs.writeFileSync(file, YAML.stringify(raw), "utf8"); return readSettings(workspace); diff --git a/src/electron/main.ts b/src/electron/main.ts index 1a95bde..9df35b2 100644 --- a/src/electron/main.ts +++ b/src/electron/main.ts @@ -10,7 +10,7 @@ import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; import { DEFAULT_WORKSPACE } from "../paths.js"; -import { configuredModels, configPath, DEFAULT_TELEMETRY_SETTINGS, initializeWorkspace, isOnboardingComplete, loadConfig, markOnboardingComplete, readOobeProgress, readSettings, saveOobeProgress, saveSettings, useConfiguredModel, writeWorkspaceEnv, type OobeProgress, type SettingsPayload } from "../config.js"; +import { configuredModels, configPath, DEFAULT_TELEMETRY_SETTINGS, initializeWorkspace, isOnboardingComplete, loadConfig, markOnboardingComplete, OFFICIAL_VISION_MODEL, readOobeProgress, readSettings, resolveVisionAgentConfig, saveOobeProgress, saveSettings, useConfiguredModel, writeWorkspaceEnv, type OobeProgress, type SettingsPayload } from "../config.js"; import { loadEnabledSkills } from "../skills.js"; import { AuditStore } from "../audit.js"; import { SecAgentRuntime, type TraceEvent } from "../runtime.js"; @@ -861,8 +861,11 @@ ipcMain.handle("models:list", async () => { // 自定义模型模式开启：只加入后台允许的官方真实模型与本地自定义模型。 return [...visibleRemote, ...options]; } - // 关闭：官方档位模式 —— 下拉只有快速/标准/深度三个虚拟档位，看不到具体模型。 - return visibleRemote.filter((model) => (OFFICIAL_TIER_IDS as readonly string[]).includes(model.model)); + // 关闭：官方档位模式 —— 下拉只有快速/标准/深度三个虚拟档位，看不到具体模型； + // 另外提供一个识图虚拟模型（virtual-vision），它只作为识图工具的后端模型， + // 不作为主 Agent 模型出现在前端下拉中（前端按 vision 标记过滤）。 + return visibleRemote.filter((model) => (OFFICIAL_TIER_IDS as readonly string[]).includes(model.model) || model.model === OFFICIAL_VISION_MODEL) + .map((model) => ({ ...model, vision: model.model === OFFICIAL_VISION_MODEL })); } catch { return customModelMode ? options : []; } }); ipcMain.handle("providers:list", async () => { @@ -1526,10 +1529,12 @@ ipcMain.handle("sessions:send", async (_event, id: string, text: string, modelId logMain("ipc.sessions.send", { sessionId: id, text }); trace({ stage: "user.request", data: { text } }); const skills = [...loadEnabledSkills(config), ...(pluginManager?.getSkills() || [])]; + const visionConfig = resolveVisionAgentConfig(config); + if (visionConfig) logMain("session.vision-model", { model: visionConfig.agent.model, provider: visionConfig.agent.provider, baseUrl: visionConfig.agent.baseUrl }); const runtimeConfig = isWakeRequest ? { ...config, agent: { ...config.agent, systemPrompt: `${config.agent.systemPrompt}\n\n## 快速唤起输出协议\n${QUICK_WAKE_OUTPUT_PROMPT}` } } : config; - runtime = new SecAgentRuntime(runtimeConfig, audit, skills, trace, pluginManager); + runtime = new SecAgentRuntime(runtimeConfig, audit, skills, trace, pluginManager, visionConfig); const previousReadSkillNames = before.messages.flatMap((message) => message.toolCalls || []).filter((call) => call.name === "secagent__read_skill" || call.name === "read_skill").map((call) => typeof (call.arguments as { name?: unknown })?.name === "string" ? (call.arguments as { name: string }).name : ""); const result = await runtime.run(historyInput(before, text), selectedReasoningEffort, conversationInput(before, text, attachments), abortController.signal, { previousAutoLoadedSkills: before.autoLoadedSkills, previousReadSkillNames, preRule }); if (result.autoLoadedSkills?.length) { diff --git a/src/index.ts b/src/index.ts index 6e73be2..0430f3f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,7 @@ #!/usr/bin/env node import readline from "node:readline/promises"; import { stdin as input, stdout as output } from "node:process"; -import { initializeWorkspace, loadConfig, normalizeAndValidate, useConfiguredModel } from "./config.js"; +import { initializeWorkspace, loadConfig, normalizeAndValidate, resolveVisionAgentConfig, useConfiguredModel } from "./config.js"; import { DEFAULT_WORKSPACE, expandPath } from "./paths.js"; import { loadEnabledSkills } from "./skills.js"; import { AuditStore } from "./audit.js"; @@ -160,7 +160,8 @@ async function openRuntime(workspace: string, modelId: string | undefined, trace const plugins = new PluginManager(workspace); await plugins.initialize(); const skills = [...loadEnabledSkills(config), ...plugins.getSkills()]; - return { runtime: new SecAgentRuntime(config, audit, skills, trace, plugins), audit, plugins, config }; + const visionConfig = resolveVisionAgentConfig(config); + return { runtime: new SecAgentRuntime(config, audit, skills, trace, plugins, visionConfig), audit, plugins, config }; } async function closeRuntime(handle: RuntimeHandle | undefined): Promise { diff --git a/src/model-provider.test.ts b/src/model-provider.test.ts index d9c59a2..150becd 100644 --- a/src/model-provider.test.ts +++ b/src/model-provider.test.ts @@ -224,3 +224,36 @@ test("persisted tool calls and results are restored to the next OpenAI request", delete process.env.TEST_MODEL_KEY; } }); + +test("a single-turn sub-agent encodes user attachments as OpenAI image parts", async () => { + const originalFetch = globalThis.fetch; + process.env.TEST_MODEL_KEY = "test-key"; + let requestBody: Record | undefined; + globalThis.fetch = async (_url, init) => { + requestBody = JSON.parse(String(init?.body || "{}")) as Record; + return response('data: {"choices":[{"delta":{"content":"red"}}]}\n\ndata: [DONE]\n\n'); + }; + try { + // Tool-less vision sub-agent: allowEmptyTools and no runtime prompts. + const agent = new ModelToolAgent(config(), [], undefined, undefined, false, true); + const conversation: ConversationMessage[] = [ + { + role: "user", + content: "图中是什么颜色", + attachments: [{ id: "a1", name: "test.png", mimeType: "image/png", dataUrl: "data:image/png;base64,iVBORw0KGgo=", size: 6 }] + } + ]; + const result = await agent.run("图中是什么颜色", [], async () => { + throw new Error("识图模型不应调用工具"); + }, "low", conversation); + assert.equal(result, "red"); + const messages = requestBody?.messages as Array>; + const content = messages[1]?.content as Array>; + assert.ok(Array.isArray(content)); + assert.deepEqual(content[0], { type: "text", text: "图中是什么颜色" }); + assert.deepEqual(content[1], { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw0KGgo=" } }); + } finally { + globalThis.fetch = originalFetch; + delete process.env.TEST_MODEL_KEY; + } +}); diff --git a/src/pi-tools.ts b/src/pi-tools.ts index 41ef92a..29e3828 100644 --- a/src/pi-tools.ts +++ b/src/pi-tools.ts @@ -7,6 +7,36 @@ import type { ToolImageContent } from "./tool-content.js"; const execAsync = promisify(exec); +/** Supported local image formats shared by `look_at` and the vision sub-model tool. */ +const IMAGE_MEDIA_TYPES: Record = { ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".webp": "image/webp", ".gif": "image/gif" }; +export const MAX_IMAGE_BYTES = 12 * 1024 * 1024; + +export interface ReadImageResult { + filePath: string; + name: string; + mimeType: string; + base64: string; +} + +export function resolveWorkspacePath(workspace: string, filePath: string): string { + return path.isAbsolute(filePath) ? filePath : path.resolve(workspace, filePath); +} + +/** + * Shared image read + validation used both by the `look_at` Pi tool (returns the image to a + * vision-capable main model) and by `secagent__look_at_image` (feeds the image to a dedicated + * vision sub-model and returns text). + */ +export async function readImageFile(workspace: string, filePath: string): Promise { + const resolved = resolveWorkspacePath(workspace, filePath); + const mediaType = IMAGE_MEDIA_TYPES[path.extname(resolved).toLowerCase()]; + if (!mediaType) throw new Error("仅支持 png、jpg、jpeg、webp、gif 图片"); + const stat = await fs.stat(resolved); + if (!stat.isFile()) throw new Error("path 不是文件"); + if (stat.size > MAX_IMAGE_BYTES) throw new Error("图片不能超过 12 MB"); + return { filePath: resolved, name: path.basename(resolved), mimeType: mediaType, base64: (await fs.readFile(resolved)).toString("base64") }; +} + export const piTools: AgentTool[] = [ { key: "look_at", description: "查看本地图片并把图片内容直接提供给模型。path 可使用绝对路径或相对于工作区的路径；仅支持 png、jpg、jpeg、webp、gif 图片。需要理解图片内容时必须调用此工具，不要只读取图片文件的二进制内容。", inputSchema: { type: "object", additionalProperties: false, required: ["path"], properties: { path: { type: "string", description: "图片的绝对路径或相对于工作区的路径" } } } }, { key: "read", description: "读取文件内容。path 可使用绝对路径或相对于工作区的路径。", inputSchema: { type: "object", additionalProperties: false, required: ["path"], properties: { path: { type: "string" }, offset: { type: "integer", minimum: 1 }, limit: { type: "integer", minimum: 1 } } } }, @@ -15,19 +45,13 @@ export const piTools: AgentTool[] = [ { key: "bash", description: "在工作区目录执行 shell 命令并返回标准输出和错误输出。", inputSchema: { type: "object", additionalProperties: false, required: ["command"], properties: { command: { type: "string" }, timeout: { type: "integer", minimum: 1 } } } } ]; -function resolvePath(workspace: string, filePath: string): string { return path.isAbsolute(filePath) ? filePath : path.resolve(workspace, filePath); } +function resolvePath(workspace: string, filePath: string): string { return resolveWorkspacePath(workspace, filePath); } export async function callPiTool(workspace: string, key: string, args: Record): Promise { if (key === "look_at") { if (typeof args.path !== "string" || !args.path.trim()) throw new Error("look_at 需要非空 path"); - const filePath = resolvePath(workspace, args.path); - const mediaTypes: Record = { ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".webp": "image/webp", ".gif": "image/gif" }; - const mediaType = mediaTypes[path.extname(filePath).toLowerCase()]; - if (!mediaType) throw new Error("look_at 仅支持 png、jpg、jpeg、webp、gif 图片"); - const stat = await fs.stat(filePath); - if (!stat.isFile()) throw new Error("look_at 的 path 不是文件"); - if (stat.size > 12 * 1024 * 1024) throw new Error("look_at 图片不能超过 12 MB"); - const result: ToolImageContent = { type: "image", data: (await fs.readFile(filePath)).toString("base64"), mimeType: mediaType, name: path.basename(filePath), path: filePath }; + const image = await readImageFile(workspace, args.path); + const result: ToolImageContent = { type: "image", data: image.base64, mimeType: image.mimeType, name: image.name, path: image.filePath }; return result; } if (key === "read") { diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index f517dca..aed5a35 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -11,7 +11,7 @@ import { WorkspaceFileStrip } from "./components/WorkspaceFileStrip.js"; import { stripWorkspaceFilesMarkup } from "../../workspace-file-contract.js"; import { reasoningEffortLabels, traceLabel } from "./constants.js"; import type { TraceEvent } from "./constants.js"; -import { isOfficialModel, isOfficialTierModel, reasoningEffortsForModel } from "./utils.js"; +import { isOfficialModel, isOfficialTierModel, isOfficialVisionModel, reasoningEffortsForModel } from "./utils.js"; import { officialTiers, tierDefaultId } from "./constants.js"; import { buildQuotedUserMessage, parseQuotedUserMessage, webSearchUrl } from "../../quoted-message.js"; @@ -91,8 +91,11 @@ export function App() { const answerScrollLockTimer = useRef(undefined); const answerStartScrollPending = useRef(false); const modelMenuEnd = useRef(null); - const orderedModels = useMemo(() => [...models.filter(isOfficialModel), ...models.filter((model) => !isOfficialModel(model))], [models]); - const selectedModel = models.find((model) => model.id === selectedModelId); + const orderedModels = useMemo(() => { + const visible = models.filter((model) => !isOfficialVisionModel(model)); + return [...visible.filter(isOfficialModel), ...visible.filter((model) => !isOfficialModel(model))]; + }, [models]); + const selectedModel = models.find((model) => model.id === selectedModelId && !isOfficialVisionModel(model)); const reasoningEfforts = useMemo(() => reasoningEffortsForModel(selectedModel), [selectedModel]); useEffect(() => { bridge.setWakeContext({ sessionId: session?.meta.id, modelId: selectedModelId || undefined, reasoningEffort: customModelMode ? reasoningEffort : defaultEffort }); @@ -187,9 +190,10 @@ export function App() { setSession(active); requestAnimationFrame(() => textareaRef.current?.focus()); const configured = await modelsPromise; - const preferred = configured.find((model) => model.id === savedSettings.defaultModelId) - || configured.find((model) => isOfficialTierModel(model) && model.model === tierDefaultId) - || configured[0]; + const mainModels = configured.filter((model) => !isOfficialVisionModel(model)); + const preferred = mainModels.find((model) => model.id === savedSettings.defaultModelId) + || mainModels.find((model) => isOfficialTierModel(model) && model.model === tierDefaultId) + || mainModels[0]; setSelectedModelId(preferred?.id || ""); })(); }, [bridge]); @@ -204,10 +208,10 @@ export function App() { setCustomModelMode(customMode); setDefaultEffort((settings.defaultReasoningEffort || "high") as ReasoningEffort); setSelectedModelId((current) => { - if (models.some((model) => model.id === (settings.defaultModelId || current))) return settings.defaultModelId || current; + if (models.some((model) => !isOfficialVisionModel(model) && model.id === (settings.defaultModelId || current))) return settings.defaultModelId || current; const tier = models.find((model) => isOfficialTierModel(model) && model.model === tierDefaultId); if (tier) return tier.id; - return models[0]?.id || ""; + return models.find((model) => !isOfficialVisionModel(model))?.id || ""; }); setReasoningEffort(settings.defaultReasoningEffort || "high"); }); diff --git a/src/renderer/src/components/SettingsApp.tsx b/src/renderer/src/components/SettingsApp.tsx index b3e00b9..11bb2af 100644 --- a/src/renderer/src/components/SettingsApp.tsx +++ b/src/renderer/src/components/SettingsApp.tsx @@ -5,7 +5,7 @@ import { PresetCombobox } from "./PresetCombobox.js"; import { SelectCombobox } from "./SelectCombobox.js"; import { OobeWizard } from "./OobeWizard.js"; import { reasoningEffortLabels, ttsRates, ttsVoices } from "../constants.js"; -import { emptyMcp, emptyProvider, reasoningEffortsForModel } from "../utils.js"; +import { emptyMcp, emptyProvider, isOfficialVisionModel, reasoningEffortsForModel } from "../utils.js"; import { formatOfficialBalanceExpiry, formatOfficialPoints } from "../official-balance.js"; import { DEFAULT_WAKE_HOTKEY, displayWakeHotkey, wakeHotkeyFromKeyboardEvent } from "../../../wake-hotkey.js"; @@ -273,7 +273,8 @@ export function SettingsApp() { setError(reason instanceof Error ? reason.message : String(reason)); } finally { setDiagnosticBusy(false); } }; - const defaultModel = availableModels.find((model) => model.id === settings.defaultModelId) || availableModels.find((model) => model.id === "sectl-official") || availableModels[0]; + const mainModelOptions = availableModels.filter((model) => !isOfficialVisionModel(model)); + const defaultModel = mainModelOptions.find((model) => model.id === settings.defaultModelId) || mainModelOptions.find((model) => model.id === "sectl-official") || mainModelOptions[0]; const defaultReasoningEfforts = reasoningEffortsForModel(defaultModel); const defaultReasoningEffort = defaultReasoningEfforts.includes(settings.defaultReasoningEffort || "high") ? (settings.defaultReasoningEffort || "high") : defaultReasoningEfforts.includes("high") ? "high" : defaultReasoningEfforts[0]; const updateSupported = bridge.platform === "win32"; @@ -285,7 +286,7 @@ export function SettingsApp() { {error &&
{error}
} {success &&
{success}
}

随时唤醒

按下全局快捷键后，在当前显示器工作区唤起语音 Agent。窗口不会覆盖任务栏。

-
setSettings((current) => current && { ...current, wake: { ...current.wake, hotkey } })} />

Windows/Linux 默认 Ctrl Alt A；macOS 默认 Ctrl Option A。语音唤醒始终使用随安装包提供的本地模型，无需网络。

+
setSettings((current) => current && { ...current, wake: { ...current.wake, hotkey } })} />

Windows/Linux 默认 Ctrl Alt A；macOS 默认 Ctrl Option A。语音唤醒始终使用随安装包提供的本地模型，无需网络。

更新

从 GitHub Release 获取 SecAgent 更新。当前仅支持 Windows 安装包更新。

@@ -309,7 +310,7 @@ export function SettingsApp() {

模型提供商

每个提供商可以包含多个模型；预设信息在启动时从 models.dev 更新。

-
SecAgent 官方服务{officialLoggedIn && }
{!officialLoggedIn &&

使用浏览器打开 SECTL 授权页登录，登录完成后自动返回 SecAgent。

}{!officialLoggedIn && }{officialLoggedIn &&
账户余额{officialPointsBusy ? "读取中…" : officialPoints === null ? "暂不可用" : `${officialPoints.toFixed(6)} Points`}
}{officialLoggedIn &&
setRedeemCode(event.target.value)} placeholder="输入兑换码" aria-label="兑换码" onKeyDown={(event) => { if (event.key === "Enter") { event.preventDefault(); void redeemOfficialCode(); } }} />
}
+
SecAgent 官方服务{officialLoggedIn && }
{!officialLoggedIn &&

使用浏览器打开 SECTL 授权页登录，登录完成后自动返回 SecAgent。

}{!officialLoggedIn && }{officialLoggedIn &&
账户余额{officialPointsBusy ? "读取中…" : officialPoints === null ? "暂不可用" : `${officialPoints.toFixed(6)} Points`}
}{officialLoggedIn &&
setRedeemCode(event.target.value)} placeholder="输入兑换码" aria-label="兑换码" onKeyDown={(event) => { if (event.key === "Enter") { event.preventDefault(); void redeemOfficialCode(); } }} />
}

当主模型不支持直接查看图片时，Agent 会自动调用识图工具，把本地图片发给此处选择的识图模型并返回文字结果。官方模式默认使用识图虚拟模型，无需手动选择。

{providerModalOpen && editingProvider &&
{ if (event.target === event.currentTarget) { setProviderModalOpen(false); setEditingProvider(null); } }}>
{settings.providers.some((provider) => provider.id === editingProvider.id) ? "编辑提供商" : "添加提供商"}
模型列表
{editingProvider.models.map((model, index) =>
updateProvider({ models: editingProvider.models.map((item, itemIndex) => itemIndex === index ? { ...item, enabled: item.enabled === false } : item) })} /> updateProvider({ models: editingProvider.models.map((item, itemIndex) => itemIndex === index ? { ...item, name: event.target.value } : item) })} />{model.id}
)}
} {settings.customModelMode ? <>

自定义提供商

{settings.providers.filter((provider) => provider.id !== "sectl-official" && provider.name !== "SecAgent 官方服务").map((provider) =>
{provider.name}{provider.models.length} 个模型 · {provider.preset && provider.preset !== "custom" ? `预设：${provider.preset}` : "自定义"}
)}
: null} diff --git a/src/renderer/src/utils.ts b/src/renderer/src/utils.ts index f5c647b..a2c7657 100644 --- a/src/renderer/src/utils.ts +++ b/src/renderer/src/utils.ts @@ -29,6 +29,12 @@ export function isOfficialTierModel(model?: ModelOption | null): boolean { return Boolean(model && model.id.startsWith("official:") && (OFFICIAL_TIER_IDS as readonly string[]).includes(model.model)); } +/** The relay's virtual vision model is only a backend for the image-recognition tool, + * never a main agent model. It is marked `vision: true` by `models:list`. */ +export function isOfficialVisionModel(model?: ModelOption | null): boolean { + return Boolean(model && model.id.startsWith("official:") && (model.vision === true || model.model === "virtual-vision")); +} + export function toolTitle(name: string): string { return name.replace(/__/g, " · ").replace(/_/g, " "); } diff --git a/src/renderer/src/vite-env.d.ts b/src/renderer/src/vite-env.d.ts index 3cbf702..778802d 100644 --- a/src/renderer/src/vite-env.d.ts +++ b/src/renderer/src/vite-env.d.ts @@ -12,14 +12,14 @@ interface UpdatePreferences { channel: UpdateChannel; autoCheck: boolean; autoDo interface UpdateRelease { version: string; tag: string; releaseType?: "alpha" | "beta"; channel: UpdateChannel; htmlUrl: string; body: string; publishedAt?: string; assetName: string; assetUrl: string; checksumUrl?: string; sha256?: string; size?: number } interface UpdateRequestAttempt { phase: "metadata" | "release-api" | "checksum" | "asset"; route: "proxy" | "direct"; url: string; ok: boolean; status?: number; contentType?: string; responseBytes?: number; durationMs: number; error?: string } interface UpdateState { currentVersion: string; channel: UpdateChannel; status: UpdateStatus; release?: UpdateRelease; downloadedVersion?: string; downloadedBytes: number; totalBytes?: number; checkedAt?: string; error?: string; operationId?: string; attempts?: UpdateRequestAttempt[]; supportReason?: string } -interface ModelOption { id: string; name: string; model: string; provider: string; virtual?: boolean } +interface ModelOption { id: string; name: string; model: string; provider: string; virtual?: boolean; vision?: boolean } interface ModelProfile { id: string; name?: string; enabled?: boolean; provider: "openai-compatible" | "openai-responses" | "anthropic" | "google"; model: string; apiKeyEnv: string; apiKey?: string; apiKeyConfigured?: boolean; baseUrl: string; endpoint?: string; anthropicVersion?: string; maxTokens?: number } interface McpServerConfig { transport: "stdio" | "http"; command?: string; args?: string[]; url?: string; enabled: boolean } interface ProviderModel { id: string; name?: string; enabled?: boolean } interface ProviderConfig { id: string; name: string; preset?: string; provider: ModelProfile["provider"]; apiKeyEnv: string; apiKey?: string; apiKeyConfigured?: boolean; baseUrl: string; endpoint?: string; anthropicVersion?: string; maxTokens?: number; models: ProviderModel[] } interface ProviderPreset { id: string; name: string; env: string[]; api: string; models: ProviderModel[] } interface TelemetrySettings { enabled: boolean } -interface SettingsPayload { providers: ProviderConfig[]; models: ModelProfile[]; tts: { voice: string; rate: string }; wake: { hotkey: string; modelId?: string; voiceEnabled?: boolean; voicePhrase?: string }; speech: { betterRecognition?: boolean }; updates: UpdatePreferences; telemetry: TelemetrySettings; mcp: { servers: Record }; defaultModelId?: string; defaultReasoningEffort?: ReasoningEffort; autostart?: boolean; autostartHidden?: boolean; customModelMode?: boolean } +interface SettingsPayload { providers: ProviderConfig[]; models: ModelProfile[]; tts: { voice: string; rate: string }; wake: { hotkey: string; modelId?: string; voiceEnabled?: boolean; voicePhrase?: string }; speech: { betterRecognition?: boolean }; updates: UpdatePreferences; telemetry: TelemetrySettings; mcp: { servers: Record }; defaultModelId?: string; defaultReasoningEffort?: ReasoningEffort; visionModelId?: string; autostart?: boolean; autostartHidden?: boolean; customModelMode?: boolean } interface SkillSummary { name: string; description: string; path: string } interface PluginStatus { id: string; format?: "secagent" | "agent"; name: string; version: string; icon?: string; enabled: boolean; state: "inactive" | "starting" | "error" | "ready"; message?: string; description?: string; author?: string; repository?: string; permissions?: string[]; readme?: string; settingsPages: Array<{ id: string; title: string; description?: string }> } interface MarketplaceVersion { version: string; minHostApiVersion: number; assetUrl: string; sha256: string; permissions: string[]; platforms: string[] } diff --git a/src/runtime.test.ts b/src/runtime.test.ts index 0e39336..2f55212 100644 --- a/src/runtime.test.ts +++ b/src/runtime.test.ts @@ -9,6 +9,7 @@ import path from "node:path"; import type { LoadedSkill } from "./skills.js"; import { AuditStore } from "./audit.js"; import { PluginManager } from "./plugin-manager.js"; +import { resolveVisionAgentConfig } from "./config.js"; import { SecAgentRuntime } from "./runtime.js"; import AdmZip from "adm-zip"; @@ -129,3 +130,166 @@ export function activate(api) { fs.rmSync(workspace, { recursive: true, force: true }); } }); + +/** 1x1 transparent PNG used to exercise the vision tool path. */ +const TEST_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; + +function visionTestConfig(workspace: string, defaults: { visionModelId?: string; customModelMode?: boolean }): SecAgentConfig { + return { + workspace, + agent: { + provider: "openai-compatible", + model: "main", + apiKeyEnv: "TEST_MODEL_KEY", + baseUrl: "https://main.test/v1", + endpoint: "/chat/completions", + maxTokens: 100, + systemPrompt: "unused", + models: [ + { id: "main", provider: "openai-compatible", model: "main", apiKeyEnv: "TEST_MODEL_KEY", baseUrl: "https://main.test/v1", endpoint: "/chat/completions", maxTokens: 100 }, + { id: "vision", provider: "openai-compatible", model: "vision-model", apiKeyEnv: "VISION_MODEL_KEY", baseUrl: "https://vision.test/v1", endpoint: "/chat/completions", maxTokens: 100 } + ] + }, + mcp: { servers: {} }, + version: 1, + defaults + } as SecAgentConfig; +} + +function sse(body: string): Response { + return new Response(body, { status: 200, headers: { "Content-Type": "text/event-stream" } }); +} + +test("secagent__look_at_image sends the image to the vision model and returns its text", async () => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "secagent-vision-ok-")); + const originalFetch = globalThis.fetch; + const previousMain = process.env.TEST_MODEL_KEY; + const previousVision = process.env.VISION_MODEL_KEY; + const requestBodies: Array<{ messages?: Array> }> = []; + let requestCount = 0; + try { + fs.writeFileSync(path.join(workspace, "test.png"), Buffer.from(TEST_PNG_BASE64, "base64")); + process.env.TEST_MODEL_KEY = "main-key"; + process.env.VISION_MODEL_KEY = "vision-key"; + globalThis.fetch = async (_url, init) => { + requestBodies.push(JSON.parse(String(init?.body || "{}")) as { messages?: Array> }); + requestCount += 1; + if (requestCount === 1) return sse('data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call-1","function":{"name":"secagent__look_at_image","arguments":"{\\"path\\":\\"test.png\\",\\"prompt\\":\\"图中是什么颜色\\"}"}}]}}]}\n\ndata: [DONE]\n\n'); + if (requestCount === 2) return sse('data: {"choices":[{"delta":{"content":"红色的圆。"}}]}\n\ndata: [DONE]\n\n'); + return sse('data: {"choices":[{"delta":{"content":"图片内容：红色的圆。"}}]}\n\ndata: [DONE]\n\n'); + }; + const audit = new AuditStore(workspace); + const traces: string[] = []; + const config = visionTestConfig(workspace, { visionModelId: "vision" }); + const runtime = new SecAgentRuntime(config, audit, [], (event) => traces.push(event.stage), undefined, resolveVisionAgentConfig(config)); + try { + const result = await runtime.run("看看这张图片", "high", [{ role: "user", content: "看看这张图片" }]); + assert.equal(result.message, "图片内容：红色的圆。"); + assert.equal(requestCount, 3); + // Vision sub-model request carries the prompt and the image dataUrl. + const visionBody = requestBodies[1]?.messages || []; + const userContent = visionBody[1]?.content as Array<{ type?: string; text?: string; image_url?: { url?: string } }>; + assert.ok(Array.isArray(userContent)); + assert.equal(userContent[0]?.type, "text"); + assert.equal(userContent[0]?.text, "图中是什么颜色"); + assert.equal(userContent[1]?.type, "image_url"); + assert.match(userContent[1]?.image_url?.url || "", /^data:image\/png;base64,/); + assert.equal(traces.includes("secagent.tools/call"), true); + assert.equal(traces.includes("vision.model.request"), true); + } finally { + await runtime.close(); + audit.close(); + } + } finally { + globalThis.fetch = originalFetch; + if (previousMain === undefined) delete process.env.TEST_MODEL_KEY; + else process.env.TEST_MODEL_KEY = previousMain; + if (previousVision === undefined) delete process.env.VISION_MODEL_KEY; + else process.env.VISION_MODEL_KEY = previousVision; + fs.rmSync(workspace, { recursive: true, force: true }); + } +}); + +test("secagent__look_at_image reports a clear error when no vision model is configured", async () => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "secagent-vision-none-")); + const originalFetch = globalThis.fetch; + const previousMain = process.env.TEST_MODEL_KEY; + let requestCount = 0; + let secondBody = ""; + try { + fs.writeFileSync(path.join(workspace, "test.png"), Buffer.from(TEST_PNG_BASE64, "base64")); + process.env.TEST_MODEL_KEY = "main-key"; + globalThis.fetch = async (_url, init) => { + requestCount += 1; + if (requestCount === 2) secondBody = String(init?.body || ""); + return requestCount === 1 + ? sse('data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call-1","function":{"name":"secagent__look_at_image","arguments":"{\\"path\\":\\"test.png\\",\\"prompt\\":\\"图中是什么\\"}"}}]}}]}\n\ndata: [DONE]\n\n') + : sse('data: {"choices":[{"delta":{"content":"识图功能当前不可用。"}}]}\n\ndata: [DONE]\n\n'); + }; + const audit = new AuditStore(workspace); + const traces: string[] = []; + // No visionModelId and customModelMode undefined → no fallback, no vision agent. + const config = visionTestConfig(workspace, { visionModelId: undefined }); + assert.equal(resolveVisionAgentConfig(config), undefined); + const runtime = new SecAgentRuntime(config, audit, [], (event) => traces.push(event.stage), undefined, resolveVisionAgentConfig(config)); + try { + const result = await runtime.run("看看这张图片", "high", [{ role: "user", content: "看看这张图片" }]); + assert.equal(result.message, "识图功能当前不可用。"); + // Only the main agent requests happened; no vision sub-request. + assert.equal(requestCount, 2); + assert.equal(traces.includes("vision.model.request"), false); + assert.match(secondBody, /未配置识图模型/); + } finally { + await runtime.close(); + audit.close(); + } + } finally { + globalThis.fetch = originalFetch; + if (previousMain === undefined) delete process.env.TEST_MODEL_KEY; + else process.env.TEST_MODEL_KEY = previousMain; + fs.rmSync(workspace, { recursive: true, force: true }); + } +}); + +test("secagent__look_at_image validates image input before any vision request", async () => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "secagent-vision-bad-")); + const originalFetch = globalThis.fetch; + const previousMain = process.env.TEST_MODEL_KEY; + const previousVision = process.env.VISION_MODEL_KEY; + let requestCount = 0; + let secondBody = ""; + try { + fs.writeFileSync(path.join(workspace, "notes.txt"), "not an image"); + process.env.TEST_MODEL_KEY = "main-key"; + process.env.VISION_MODEL_KEY = "vision-key"; + globalThis.fetch = async (_url, init) => { + requestCount += 1; + if (requestCount === 2) secondBody = String(init?.body || ""); + return requestCount === 1 + ? sse('data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call-1","function":{"name":"secagent__look_at_image","arguments":"{\\"path\\":\\"notes.txt\\",\\"prompt\\":\\"图里有什么\\"}"}}]}}]}\n\ndata: [DONE]\n\n') + : sse('data: {"choices":[{"delta":{"content":"无法识别。"}}]}\n\ndata: [DONE]\n\n'); + }; + const audit = new AuditStore(workspace); + const traces: string[] = []; + const config = visionTestConfig(workspace, { visionModelId: "vision" }); + const runtime = new SecAgentRuntime(config, audit, [], (event) => traces.push(event.stage), undefined, resolveVisionAgentConfig(config)); + try { + const result = await runtime.run("看看这个文件", "high", [{ role: "user", content: "看看这个文件" }]); + assert.equal(result.message, "无法识别。"); + // The vision sub-model was never called (2 = main turn 1 + main turn 2). + assert.equal(requestCount, 2); + assert.equal(traces.includes("vision.model.request"), false); + assert.match(secondBody, /仅支持 png、jpg、jpeg、webp、gif 图片/); + } finally { + await runtime.close(); + audit.close(); + } + } finally { + globalThis.fetch = originalFetch; + if (previousMain === undefined) delete process.env.TEST_MODEL_KEY; + else process.env.TEST_MODEL_KEY = previousMain; + if (previousVision === undefined) delete process.env.VISION_MODEL_KEY; + else process.env.VISION_MODEL_KEY = previousVision; + fs.rmSync(workspace, { recursive: true, force: true }); + } +}); diff --git a/src/runtime.ts b/src/runtime.ts index 4e4baf6..81361a4 100644 --- a/src/runtime.ts +++ b/src/runtime.ts @@ -1,16 +1,17 @@ import { randomUUID } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; -import type { ReasoningEffort, SecAgentConfig } from "./types.js"; +import type { ChatAttachment, ReasoningEffort, SecAgentConfig } from "./types.js"; import { AuditStore } from "./audit.js"; import { McpRegistry } from "./mcp-adapter.js"; import { ModelToolAgent } from "./model-provider.js"; import type { ConversationMessage } from "./model-provider.js"; import type { LoadedSkill } from "./skills.js"; -import { callPiTool, piTools } from "./pi-tools.js"; +import { callPiTool, piTools, readImageFile } from "./pi-tools.js"; import { PluginManager } from "./plugin-manager.js"; import type { ResolvedPluginPreRule } from "./plugin-manager.js"; import { summarizeToolResult } from "./tool-content.js"; +import { VISION_SYSTEM_PROMPT } from "./system-prompt.js"; export type RunResult = | { kind: "completed"; message: string; actionId?: string; autoLoadedSkills?: string[] } @@ -44,10 +45,24 @@ export function selectAutoLoadedSkills(skills: LoadedSkill[], content: string, p export class SecAgentRuntime { private registry: McpRegistry; private agent: ModelToolAgent; + private visionAgent: ModelToolAgent | undefined; private sequence = 0; - constructor(private config: SecAgentConfig, private audit: AuditStore, private skills: LoadedSkill[], private trace?: (event: TraceEvent) => void, private plugins?: PluginManager) { + constructor(private config: SecAgentConfig, private audit: AuditStore, private skills: LoadedSkill[], private trace?: (event: TraceEvent) => void, private plugins?: PluginManager, visionConfig?: SecAgentConfig) { this.registry = new McpRegistry(config, plugins?.getMcpServers()); this.agent = new ModelToolAgent(config, skills, (stage, data) => this.emit(stage, data), () => this.plugins?.getPromptContributions() ?? Promise.resolve([])); + // A dedicated image-recognition sub-agent. Only present when a vision model is + // configured; its tool (`secagent__look_at_image`) is then exposed to the main agent + // and its trace events are isolated under the `vision.*` namespace. + this.visionAgent = visionConfig + ? new ModelToolAgent( + { ...visionConfig, agent: { ...visionConfig.agent, systemPrompt: VISION_SYSTEM_PROMPT } }, + [], + (stage, data) => this.emit(`vision.${stage}`, data), + undefined, + false, // includeRuntimePrompts: keep only the dedicated vision system prompt + true // allowEmptyTools: single-turn text-only sub-agent + ) + : undefined; } async run(input: string, reasoningEffort: ReasoningEffort = "high", conversation?: ConversationMessage[], signal?: AbortSignal, state: { previousAutoLoadedSkills?: string[]; previousReadSkillNames?: string[]; preRule?: ResolvedPluginPreRule } = {}): Promise { signal?.throwIfAborted(); @@ -95,6 +110,7 @@ export class SecAgentRuntime { ...mcpTools.filter((tool) => !hiddenTools.has(tool.key)), ...pluginTools.filter((tool) => !hiddenTools.has(tool.key)), ...piTools, + ...(this.visionAgent ? [{ key: "secagent__look_at_image", description: "使用独立的识图模型查看本地图片并返回文字结果。当你的模型不支持直接查看图片（无法通过 look_at 查看图片内容）时，必须调用此工具代替 look_at。path 可使用绝对路径或相对于工作区的路径；仅支持 png、jpg、jpeg、webp、gif 图片。prompt 为需要基于图片内容回答的问题。", inputSchema: { type: "object", additionalProperties: false, required: ["path", "prompt"], properties: { path: { type: "string", description: "图片的绝对路径或相对于工作区的路径" }, prompt: { type: "string", description: "需要基于图片内容回答的问题" } } } }] : []), { key: "secagent__read_skill", description: "读取指定 Skill 或其 Skill 目录内专题 Markdown 的完整操作说明。仅当需要该 Skill 的详细流程、约束或示例时调用。", inputSchema: { type: "object", additionalProperties: false, required: ["name"], properties: { name: { type: "string", description: "Skill 名称，必须来自系统提示词中的可用 Skills 目录。" }, file: { type: "string", description: "可选；Skill 目录内的相对 Markdown 文件名，例如 components.md。" } } } }, { key: "secagent__call_hidden_tool", description: "调用 Skill 约定的隐藏 MCP 工具。工具名称和参数格式应严格遵循 Skill 正文或模型已知的其他契约。", inputSchema: { type: "object", additionalProperties: false, required: ["name", "arguments"], properties: { name: { type: "string", description: "隐藏工具的完整 key，例如 secscore-connector__list_students。" }, arguments: { type: "object", description: "按照工具契约填写的参数。" } } } } ]; @@ -103,7 +119,7 @@ export class SecAgentRuntime { const prepared = this.prepareAutoLoadedSkills(conversation, state); this.emit("secagent.skills/auto-load", prepared.loaded.map((skill) => ({ name: skill.name, path: skill.path }))); this.emit("model.agent.request", { provider: this.config.agent.provider, model: this.config.agent.model, baseUrl: this.config.agent.baseUrl, instruction: input }); - const message = await this.agent.run(input, tools, async (key, args) => this.callTool(input, key, args, hiddenTools), reasoningEffort, prepared.conversation, signal); + const message = await this.agent.run(input, tools, async (key, args) => this.callTool(input, key, args, hiddenTools, signal), reasoningEffort, prepared.conversation, signal); this.emit("model.agent.result", { message }); return { kind: "completed", message, autoLoadedSkills: prepared.loaded.map((skill) => skill.name) }; } @@ -137,7 +153,8 @@ export class SecAgentRuntime { const response = await this.callTool(`undo ${actionId}`, connectorUndoKey || "secscore__undo_score", { event_uuid: result.event_uuid, student_id: result.student_id }); return { kind: "completed", message: `已请求撤销 ${actionId}：${JSON.stringify(response)}` }; } - private async callTool(request: string, key: string, args: Record, hiddenTools?: Set): Promise { + private async callTool(request: string, key: string, args: Record, hiddenTools?: Set, signal?: AbortSignal): Promise { + if (key === "secagent__look_at_image") return this.callVision(request, args, signal); if (piTools.some((tool) => tool.key === key)) { this.emit("secagent.tools/call", { name: key, arguments: args }); try { @@ -156,6 +173,45 @@ export class SecAgentRuntime { if (key === "secagent__call_hidden_tool") return this.callHiddenTool(request, args, hiddenTools); return this.executeTool(request, key, args); } + /** + * `secagent__look_at_image`: read a local image and send it to the dedicated vision + * sub-model, returning the sub-model's text answer to the (possibly non-vision) main + * agent. The image is only attached to the sub-model request, never fed back to the + * main model, and the audit/trace result is summarized without binary data. + */ + private async callVision(request: string, args: Record, signal?: AbortSignal): Promise { + if (!this.visionAgent) throw new Error("未配置识图模型。请在「设置 → 模型」中选择识图模型后再调用识图工具。"); + this.emit("secagent.tools/call", { name: "secagent__look_at_image", arguments: args }); + try { + if (typeof args.path !== "string" || !args.path.trim()) throw new Error("secagent__look_at_image 需要非空 path"); + if (typeof args.prompt !== "string" || !args.prompt.trim()) throw new Error("secagent__look_at_image 需要非空 prompt"); + const image = await readImageFile(this.config.workspace, args.path); + const attachment: ChatAttachment = { + id: randomUUID(), + name: image.name, + mimeType: image.mimeType, + dataUrl: `data:${image.mimeType};base64,${image.base64}`, + size: Math.floor(image.base64.length * 3 / 4) - (image.base64.endsWith("==") ? 2 : image.base64.endsWith("=") ? 1 : 0) + }; + const text = await this.visionAgent.run( + args.prompt, + [], + async () => { throw new Error("识图模型不应调用工具"); }, + "low", + [{ role: "user", content: args.prompt, attachments: [attachment] }], + signal + ); + const result = { path: image.filePath, name: image.name, text }; + const summary = summarizeToolResult(result); + this.emit("secagent.tools/result", { name: "secagent__look_at_image", result: summary }); + this.audit.log({ id: randomUUID(), status: "completed", tool: "secagent.look_at_image", request, params: args, result: summary }); + return result; + } catch (error) { + const result = { error: error instanceof Error ? error.message : String(error) }; + this.emit("secagent.tools/result", { name: "secagent__look_at_image", result }); + throw error; + } + } private async callHiddenTool(request: string, args: Record, hiddenTools?: Set): Promise { const key = typeof args.name === "string" ? args.name : ""; const toolArgs = args.arguments; diff --git a/src/system-prompt.ts b/src/system-prompt.ts index 2e38c4f..1fc1767 100644 --- a/src/system-prompt.ts +++ b/src/system-prompt.ts @@ -25,3 +25,11 @@ export const SYSTEM_PROMPT = `你是 SecAgent，一个智慧教育 AI Agent， - 下午第一节是啥 - 明天这节是啥课 这类指令你需要调用课表软件（如ClassIsland），如果没有，请引导用户安装对应课表软件的联动适配插件`; + +/** + * System prompt for the dedicated image-recognition sub-model. The sub-agent is called by + * the `secagent__look_at_image` tool when the main agent cannot ingest images itself. It is + * a single-turn, tool-less assistant that must only return the answer as text. + */ +export const VISION_SYSTEM_PROMPT = `你是 SecAgent 的图片识别助手。用户会发送一张图片和一个问题，你需要仔细观察图片后直接回答该问题。 +只输出回答内容本身，不要添加任何多余的说明、前缀或 Markdown 包装。如果图片内容与问题无关或无法识别，请如实说明。`; diff --git a/src/types.ts b/src/types.ts index 0b160eb..a231f67 100644 --- a/src/types.ts +++ b/src/types.ts @@ -125,7 +125,7 @@ export interface SecAgentConfig { updates?: UpdatePreferences; telemetry?: TelemetrySettings; mcp: { servers: Record }; - defaults?: { modelId?: string; reasoningEffort?: ReasoningEffort; customModelMode?: boolean; autostart?: boolean; autostartHidden?: boolean }; + defaults?: { modelId?: string; reasoningEffort?: ReasoningEffort; customModelMode?: boolean; autostart?: boolean; autostartHidden?: boolean; visionModelId?: string }; } /** A tool supplied by a locally installed SecAgent plugin. */