From 53b0271f497b52603ef9f981d1276fef4325e26d Mon Sep 17 00:00:00 2001 From: BeiLuoqu Date: Sat, 24 Oct 2026 22:24:15 +0800 Subject: [PATCH 01/15] fix: show local images that DeepSeek Harness replies reference Codex Desktop loads a Markdown image from disk only when its destination is an absolute path or file: URL. DSH replies usually reference files relative to the working directory, so Desktop resolved them against its own page and every image showed the "Image unavailable" placeholder. - Move Grok's local media rewrite to the shared @codexhost/harness-adapter/local-media-markdown export and apply it to DSH live output and history, resolving against the agent's current working directory (working-directory/change), then the Session's. - Stream through LocalMediaMarkdownStream: hold back only a trailing construct that later text can still change, so a malformed image or a stray backtick no longer withholds the rest of a reply; remember file checks per message so published text never needs retracting. - Write Windows drive paths with forward slashes; CommonMark reads a backslash before punctuation (\.codex) as an escape. - Leave unresolved references byte-for-byte unchanged. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/external-thread-media.md | 29 ++ docs/index.md | 1 + .../deepseek-harness/src/modern/history.ts | 47 ++- .../deepseek-harness/src/modern/session.ts | 55 ++- .../test/modern/history.test.ts | 50 +++ .../test/modern/session.test.ts | 111 +++++ packages/adapters/grok/src/grok-adapter.ts | 3 +- packages/adapters/grok/src/grok-history.ts | 3 +- .../adapters/grok/src/local-media-markdown.ts | 308 -------------- .../adapters/grok/test/grok-history.test.ts | 2 +- .../grok/test/local-media-markdown.test.ts | 17 +- packages/harness-adapter/package.json | 4 + .../src/local-media-markdown.ts | 385 ++++++++++++++++++ .../test/local-media-markdown.test.ts | 152 +++++++ 14 files changed, 835 insertions(+), 332 deletions(-) create mode 100644 docs/architecture/external-thread-media.md create mode 100644 packages/harness-adapter/src/local-media-markdown.ts create mode 100644 packages/harness-adapter/test/local-media-markdown.test.ts diff --git a/docs/architecture/external-thread-media.md b/docs/architecture/external-thread-media.md new file mode 100644 index 000000000..61bda0685 --- /dev/null +++ b/docs/architecture/external-thread-media.md @@ -0,0 +1,29 @@ +# 外部 Thread 的本地图片显示 + +外部 Harness 的消息由 codexhost 投影成 Codex Desktop 原生条目后显示。本地图片能否显示，取决于投影出的条目是否满足 Desktop 的加载规则。本文记录当前做法和边界，依据 Codex Desktop 26.1021.3705.0 的渲染代码核对；Desktop 更新后需重新核实。 + +## Desktop 的加载规则 + +Agent 消息中的 Markdown 图片（`![alt](目标)`）只有在目标是绝对路径（`/…`、`C:\…`、`C:/…`、UNC）或 `file:` URL 时，才会被当作本地文件，经 app-server 读取后显示。其他目标按 Desktop 页面自身地址解析，加载失败后显示“Image unavailable”占位框。只有预览 `.md` 文件时，Desktop 才会按该文件所在目录补全相对路径；对话消息没有这个基准目录。 + +Harness 常按工作目录写相对路径，因此需要由 Adapter 改写。 + +## Agent 消息：改写本地媒体路径 + +`@codexhost/harness-adapter/local-media-markdown` 提供共用的改写： + +- 改写图片目标中的本地媒体路径，以及只包含一个媒体路径的行内代码（转成图片）。相对路径按 Adapter 给出的根目录依次查找，只有找到实际存在的文件才改写。 +- 远程 URL、`data:` 等地址、找不到的文件、无法解析的写法和围栏代码块中的内容保持原样，不改动一个字节。 +- Windows 盘符路径写成正斜杠（`C:/…`）。CommonMark 会把反斜杠加标点当作转义，原样输出会让 `C:\Users\me\.codex\…` 丢掉分隔符。 +- 支持的扩展名：图片 `png`、`jpg`、`jpeg`、`gif`、`webp`、`svg`，音视频 `mp4`、`m4v`、`mov`、`webm`、`mp3`、`m4a`、`aac`、`ogg`、`wav`。 + +实时输出只能追加，Host 要求条目完成时的全文等于此前所有增量之和。`LocalMediaMarkdownStream` 因此只暂缓结尾仍可能被后续文本改变的写法（未闭合的图片、未闭合的行内代码），写错的图片或单个反引号不会扣住后面的文字；Item 结束前再补发暂缓部分。同一条消息内，每个路径只检查一次文件，保证前后改写一致。 + +接入的 Harness： + +| Harness | 根目录 | 范围 | +| --- | --- | --- | +| Grok | 工作目录、Grok 会话目录及其 `videos/`、`images/`、用户 `Downloads/` | 实时输出与历史 | +| DeepSeek Harness | Agent 当前工作目录（随 `working-directory/change` 更新），再到 Session 工作目录 | 实时输出与历史（`readSnapshot`）；用量刷新不做改写 | + +其他 Harness 的 Agent 文本仍原样转发；它们输出相对路径时，图片仍显示为占位框。 diff --git a/docs/index.md b/docs/index.md index a4c0cbc1d..357f37910 100644 --- a/docs/index.md +++ b/docs/index.md @@ -23,6 +23,7 @@ | [`architecture/harness-questions.md`](architecture/harness-questions.md) | 外部 Harness 多选提问卡片、原生单选分流、回答归属与生命周期；修改提问交互时阅读。 | | [`architecture/external-thread-steering.md`](architecture/external-thread-steering.md) | 外部 Thread 按 Session 能力选择原生同轮插入或停止后重发；修改 steering 时阅读。 | | [`architecture/external-thread-queue.md`](architecture/external-thread-queue.md) | 外部 Thread 跟进消息的 Host 原生 `thread/queue/*`、自动推进与中断暂停；修改排队或 Turn 启动时阅读。 | +| [`architecture/external-thread-media.md`](architecture/external-thread-media.md) | 外部 Thread 中本地图片的 Desktop 加载条件，以及 Agent 消息媒体路径的改写；修改媒体显示或排查图片占位框时阅读。 | | [`architecture/delegation-titles.md`](architecture/delegation-titles.md) | 委派子会话标题、子任务显示名、创建快照与重试边界。 | | [`architecture/thread-watch.md`](architecture/thread-watch.md) | 一次性 Thread 停下通知（`thread watch`）的模型、结果、送达与边界；修改委派跟进或通知时阅读。 | | [`architecture/official-traffic-ownership.md`](architecture/official-traffic-ownership.md) | Host 只截获自身流量、其余原样转发官方的归属判别与例外；修改请求路由或参数校验时阅读。 | diff --git a/packages/adapters/deepseek-harness/src/modern/history.ts b/packages/adapters/deepseek-harness/src/modern/history.ts index c1bc40250..96ccbd177 100644 --- a/packages/adapters/deepseek-harness/src/modern/history.ts +++ b/packages/adapters/deepseek-harness/src/modern/history.ts @@ -1,3 +1,4 @@ +import path from "node:path"; import { isDeepStrictEqual } from "node:util"; import type { @@ -16,6 +17,7 @@ import type { HostUsage, TurnOutcome, } from "@codexhost/harness-adapter"; +import { rewriteLocalMediaMarkdown } from "@codexhost/harness-adapter/local-media-markdown"; import { harnessIdSchema, hostItemIdSchema, @@ -781,6 +783,11 @@ export interface ProjectModernHistoryInput { readonly toolOutputLimit?: number; readonly maxEvents?: number; readonly profile?: DeepSeekModernProfile; + /** + * Session working directory from the journal header. When present, local media that agent + * messages reference by relative path becomes absolute, so Codex Desktop can display it. + */ + readonly cwd?: string; } /** Strictly validate and project one complete authoritative Modern journal prefix. */ @@ -800,6 +807,7 @@ export function projectModernHistory(input: ProjectModernHistoryInput): ModernHi let effectiveThinkingOptionId = input.fallbackThinkingOptionId; let contextWindowTokens: number | undefined; let usage: HostUsage | null = null; + let workingDirectory = input.cwd; const rawUsageByStep = new Map(); const rebuildUsage = (): void => { @@ -820,6 +828,7 @@ export function projectModernHistory(input: ProjectModernHistoryInput): ModernHi validator.accept(event); if (!KNOWN_EVENT_TYPES.has(event.type) || !isRecord(event.data)) continue; const data = event.data; + workingDirectory = modernWorkingDirectory(workingDirectory, event); switch (event.type) { case "turn/start": active = { @@ -867,7 +876,14 @@ export function projectModernHistory(input: ProjectModernHistoryInput): ModernHi case "assistant/message": if (event.surfaceOp === "append") { recordSettlementUsage(data, event.seq); - if (active) projectAssistantMessage(active, input.sessionId, data); + if (active) { + projectAssistantMessage( + active, + input.sessionId, + data, + modernMediaRoots(workingDirectory, input.cwd), + ); + } } break; case "compaction/start": @@ -1002,10 +1018,37 @@ export function modernItemId(sessionId: string, key: string): HostItemId { return hostItemIdSchema.parse(`dsh-modern:${sessionId}:${key}`); } +/** Working directory after `event`; DSH records each change its `working_directory` tool makes. */ +export function modernWorkingDirectory( + current: string | undefined, + event: ModernJournalEvent, +): string | undefined { + if ( + event.type !== "working-directory/change" || + !isRecord(event.data) || + typeof event.data.cwd !== "string" + ) { + return current; + } + const next = event.data.cwd; + if (path.isAbsolute(next)) return path.resolve(next); + return current === undefined ? undefined : path.resolve(current, next); +} + +/** Directories that resolve a relative media path: the current, then the Session's. */ +export function modernMediaRoots( + workingDirectory: string | undefined, + sessionDirectory: string | undefined, +): string[] | undefined { + if (sessionDirectory === undefined) return undefined; + return [...new Set([workingDirectory ?? sessionDirectory, sessionDirectory])]; +} + function projectAssistantMessage( turn: HistoryTurn, sessionId: string, data: Record, + mediaRoots: readonly string[] | undefined, ): void { const message = data.message as Record; if (Array.isArray(message.content)) { @@ -1035,7 +1078,7 @@ function projectAssistantMessage( const item: HostAgentMessageItem = { type: "agentMessage", itemId: modernItemId(sessionId, `turn:${turn.turn}:step:${String(data.step)}:assistant`), - text, + text: mediaRoots ? rewriteLocalMediaMarkdown(text, mediaRoots) : text, }; turn.items.push({ item, outcome: { status: "succeeded" } }); } diff --git a/packages/adapters/deepseek-harness/src/modern/session.ts b/packages/adapters/deepseek-harness/src/modern/session.ts index 942f9a551..41a09b3e9 100644 --- a/packages/adapters/deepseek-harness/src/modern/session.ts +++ b/packages/adapters/deepseek-harness/src/modern/session.ts @@ -46,6 +46,7 @@ import { type TurnSteerAccepted, type TurnSteerCommand, } from "@codexhost/harness-adapter"; +import { LocalMediaMarkdownStream } from "@codexhost/harness-adapter/local-media-markdown"; import { harnessIdSchema, hostInteractionIdSchema, @@ -101,7 +102,9 @@ import { isNotStartedToolResult, modernCheckpointRef, modernItemId, + modernMediaRoots, modernNativeTurnRef, + modernWorkingDirectory, projectModernHistory, } from "./history.js"; import { @@ -214,6 +217,11 @@ interface LiveTextItem { text: string; } +/** `text` is native; `item.text` is what `media` has published from it. */ +interface LiveAgentItem extends LiveTextItem { + readonly media: LocalMediaMarkdownStream; +} + interface LiveReasoningItem extends LiveTextItem { readonly attemptId?: string; readonly provisional: boolean; @@ -231,7 +239,7 @@ interface ActiveHostTurn { readonly nativeTurn: number; readonly input: HostTextInput[]; readonly autonomous: boolean; - agent?: LiveTextItem; + agent?: LiveAgentItem; reasoning?: LiveReasoningItem; reasoningOrdinal: number; compaction?: { nativeId: string; item: HostContextCompactionItem }; @@ -361,6 +369,8 @@ export class ModernHarnessSession implements HarnessSession, ModernEventSink { readonly #flushSession: (() => Promise) | undefined; readonly #fallbackModel: HarnessModelRef; readonly #fallbackThinkingOptionId: HarnessThinkingOptionId | undefined; + readonly #sessionDirectory: string | undefined; + #workingDirectory: string | undefined; readonly #channel = new HarnessOutputChannel(); readonly #validator: ModernEventValidator; readonly #events: ModernJournalEvent[]; @@ -449,7 +459,12 @@ export class ModernHarnessSession implements HarnessSession, ModernEventSink { this.#events = [...options.journal.events]; this.#historyBytes = journalHistoryBytes(this.#events, this.#maxHistoryBytes); this.#validator = new ModernEventValidator(this.#maxEvents, this.#profile); - for (const event of this.#events) this.#validator.accept(event); + this.#sessionDirectory = options.journal.header.cwd; + this.#workingDirectory = this.#sessionDirectory; + for (const event of this.#events) { + this.#validator.accept(event); + this.#workingDirectory = modernWorkingDirectory(this.#workingDirectory, event); + } const projection = projectModernHistory({ harnessId: this.harnessId, @@ -544,7 +559,7 @@ export class ModernHarnessSession implements HarnessSession, ModernEventSink { if (this.#reading) return Promise.resolve({ ok: false, error: busyError("read history") }); this.#reading = true; try { - const projection = this.#project(); + const projection = this.#project(true); const configuration = this.#readConfiguration(); return Promise.resolve({ ok: true, @@ -1743,6 +1758,7 @@ export class ModernHarnessSession implements HarnessSession, ModernEventSink { this.#validator.accept(event); this.#observeAssistantSettlement(event); this.#events.push(event); + this.#workingDirectory = modernWorkingDirectory(this.#workingDirectory, event); this.#meterUsage(event); this.#historyBytes += bytes; this.#receive(event); @@ -1896,8 +1912,8 @@ export class ModernHarnessSession implements HarnessSession, ModernEventSink { #cancelAgentItem(active: ActiveHostTurn): void { const agent = active.agent; if (!agent) return; - agent.text += CANCELLED_ASSISTANT_ATTEMPT_SUFFIX; - agent.item = { ...agent.item, text: agent.text }; + this.#flushAgent(active, agent); + agent.item = { ...agent.item, text: agent.item.text + CANCELLED_ASSISTANT_ATTEMPT_SUFFIX }; this.#emit({ type: "item.updated", turnId: active.turnId, @@ -2436,16 +2452,27 @@ export class ModernHarnessSession implements HarnessSession, ModernEventSink { ), text: "", }; - active.agent = { item, text: "" }; + const roots = modernMediaRoots(this.#workingDirectory, this.#sessionDirectory) ?? []; + active.agent = { item, text: "", media: new LocalMediaMarkdownStream(roots) }; this.#emit({ type: "item.started", turnId: active.turnId, item }); } active.agent.text += text; - active.agent.item = { ...active.agent.item, text: active.agent.text }; + this.#publishAgent(active, active.agent, active.agent.media.append(text)); + } + + /** Publish text the media projection held for a trailing construct before the Item ends. */ + #flushAgent(active: ActiveHostTurn, agent: LiveAgentItem): void { + this.#publishAgent(active, agent, agent.media.flush()); + } + + #publishAgent(active: ActiveHostTurn, agent: LiveAgentItem, delta: string): void { + if (!delta) return; + agent.item = { ...agent.item, text: agent.media.text }; this.#emit({ type: "item.updated", turnId: active.turnId, - itemId: active.agent.item.itemId, - update: { type: "text.append", text }, + itemId: agent.item.itemId, + update: { type: "text.append", text: delta }, }); } @@ -2554,6 +2581,7 @@ export class ModernHarnessSession implements HarnessSession, ModernEventSink { current = active.agent; } if (!current) return; + this.#flushAgent(active, current); delete active.agent; this.#completeItem(active, current.item, { status: "succeeded" }); } @@ -2731,9 +2759,10 @@ export class ModernHarnessSession implements HarnessSession, ModernEventSink { this.#completeItem(active, item, outcome); } if (active.agent) { - const item = active.agent.item; + const agent = active.agent; + this.#flushAgent(active, agent); delete active.agent; - this.#completeItem(active, item, outcome); + this.#completeItem(active, agent.item, outcome); } for (const tool of active.tools.values()) this.#completeItem(active, tool.item, outcome); active.tools.clear(); @@ -2766,12 +2795,14 @@ export class ModernHarnessSession implements HarnessSession, ModernEventSink { } } - #project(): ReturnType { + /** `media` also resolves local media paths, which only displayed history needs. */ + #project(media = false): ReturnType { return projectModernHistory({ harnessId: this.harnessId, sessionId: this.#sessionId, events: this.#events, profile: this.#profile, + ...(media && this.#sessionDirectory !== undefined ? { cwd: this.#sessionDirectory } : {}), ...(this.#fallbackModel ? { fallbackModel: this.#fallbackModel } : {}), ...(this.#fallbackThinkingOptionId ? { fallbackThinkingOptionId: this.#fallbackThinkingOptionId } diff --git a/packages/adapters/deepseek-harness/test/modern/history.test.ts b/packages/adapters/deepseek-harness/test/modern/history.test.ts index ab7cfa6ea..4fb910849 100644 --- a/packages/adapters/deepseek-harness/test/modern/history.test.ts +++ b/packages/adapters/deepseek-harness/test/modern/history.test.ts @@ -1,3 +1,7 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + import { describe, expect, it } from "vitest"; import { @@ -243,6 +247,52 @@ describe("DeepSeek Harness Modern history projection", () => { expect(projection.lastSeq).toBe(10); }); + it("makes local media in agent text absolute, following working-directory changes", async () => { + const workspace = await mkdtemp(path.join(os.tmpdir(), "codexhost-dsh-history-media-")); + const shots = path.join(workspace, "shots"); + try { + await mkdir(shots); + await writeFile(path.join(shots, "a.png"), "png"); + await writeFile(path.join(shots, "b.png"), "png"); + const events = [ + event(0, "turn/start", { turn: 1 }), + event(1, "step/start", { turn: 1, step: 1 }), + userMessage(2, ["show"]), + assistantMessage(3, 1, 1, "![a](shots/a.png) `shots/b.png` ![b](b.png)", "think"), + event(4, "step/end", { turn: 1, step: 1 }), + event(5, "turn/end", { turn: 1, reason: { kind: "completed" } }), + event(6, "working-directory/change", { cwd: "shots" }), + event(7, "turn/start", { turn: 2 }), + event(8, "step/start", { turn: 2, step: 1 }), + userMessage(9, ["again"], "request-2"), + assistantMessage(10, 2, 1, "![b](b.png) ![a](shots/a.png)", "think"), + event(11, "step/end", { turn: 2, step: 1 }), + event(12, "turn/end", { turn: 2, reason: { kind: "completed" } }), + ]; + const agentTexts = (cwd?: string) => + projectModernHistory({ + sessionId: SESSION_ID, + events, + ...(cwd === undefined ? {} : { cwd }), + }).snapshot.turns.flatMap(({ items }) => + items.flatMap(({ item }) => (item.type === "agentMessage" ? [item.text] : [])), + ); + const a = path.join(shots, "a.png").replaceAll("\\", "/"); + const b = path.join(shots, "b.png").replaceAll("\\", "/"); + + expect(agentTexts(workspace)).toEqual([ + `![a](${a}) ![b.png](${b}) ![b](b.png)`, + `![b](${b}) ![a](${a})`, + ]); + expect(agentTexts()).toEqual([ + "![a](shots/a.png) `shots/b.png` ![b](b.png)", + "![b](b.png) ![a](shots/a.png)", + ]); + } finally { + await rm(workspace, { recursive: true, force: true }); + } + }); + it("keeps surface replacement copies out of the human transcript and Usage", () => { const replacementUser = { ...userMessage(3, ["model-only replacement"]), diff --git a/packages/adapters/deepseek-harness/test/modern/session.test.ts b/packages/adapters/deepseek-harness/test/modern/session.test.ts index a51ea9ae3..e7f0e8cbc 100644 --- a/packages/adapters/deepseek-harness/test/modern/session.test.ts +++ b/packages/adapters/deepseek-harness/test/modern/session.test.ts @@ -1,3 +1,7 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + import { afterEach, describe, expect, it, vi } from "vitest"; import { CodexTurnProjector } from "@codexhost/protocol-core"; @@ -1367,6 +1371,113 @@ describe("DeepSeek Harness Modern Session", () => { await test.session.close(); }); + it("streams relative local media as absolute paths, live and in history", async () => { + const workspace = await mkdtemp(path.join(os.tmpdir(), "codexhost-dsh-media-")); + const shots = path.join(workspace, "shots"); + try { + await mkdir(shots); + await writeFile(path.join(shots, "a.png"), "png"); + await writeFile(path.join(shots, "b.png"), "png"); + const feed = new EventFeed(); + const remote = new FakeRemote([() => accepted()]); + const session = new ModernHarnessSession({ + remote, + journal: { + header: { version: 4, id: SESSION_ID, createdAt: 1, isSeeded: false, cwd: workspace }, + cursor: -1, + projections: { asOfSeq: -1, values: {} }, + events: [], + live: feed, + close: () => { + feed.finish(); + return Promise.resolve(); + }, + }, + control: new FakeControl(), + eventGateway: new ModernEventGateway(remote), + modelCatalog: MODEL_CATALOG, + permissionModes: null, + sessionId: SESSION_ID, + randomUUID: () => "request-1", + now: () => 10, + }); + const outputs = lifecycleOutputs(session); + const id = turnId("host-turn-1"); + await session.execute({ + type: "turn.start", + turnId: id, + input: [{ type: "text", text: "show" }], + }); + feed.push(event(0, "turn/start", { turn: 1 })); + feed.push(event(1, "step/start", { turn: 1, step: 1 })); + feed.push(userMessage(2, "show", "request-1")); + expect(await nextEvent(outputs)).toEqual({ type: "turn.started", turnId: id }); + // Relative paths resolve against the agent's current directory, then the Session's. + feed.push(event(3, "working-directory/change", { cwd: shots })); + const stream = new LiveAttempts(feed); + stream.start("attempt-1", 3); + stream.chunk({ type: "text-delta", index: 0, text: "![a](shots/" }); + stream.chunk({ type: "text-delta", index: 0, text: "a.png) and ![b](b.p" }); + stream.chunk({ type: "text-delta", index: 0, text: "ng)" }); + feed.push(finalAssistantMessage(4, 1, "![a](shots/a.png) and ![b](b.png)", "think")); + stream.commit(4); + feed.push(event(5, "step/end", { turn: 1, step: 1 })); + feed.push(event(6, "turn/end", { turn: 1, reason: { kind: "completed" } })); + const emitted = await eventsThrough(outputs, "turn.completed"); + + const a = path.join(shots, "a.png").replaceAll("\\", "/"); + const b = path.join(shots, "b.png").replaceAll("\\", "/"); + const agent = emitted.find( + (entry) => entry.type === "item.started" && entry.item.type === "agentMessage", + ); + const agentId = agent?.type === "item.started" ? agent.item.itemId : undefined; + expect( + emitted.flatMap((entry) => + entry.type === "item.updated" && + entry.itemId === agentId && + entry.update.type === "text.append" + ? [entry.update.text] + : [], + ), + ).toEqual(["![a](", `${a}) and ![b](`, `${b})`]); + const text = `![a](${a}) and ![b](${b})`; + expect(emitted).toContainEqual( + expect.objectContaining({ + type: "item.completed", + snapshot: { + item: expect.objectContaining({ itemId: agentId, text }), + outcome: { status: "succeeded" }, + }, + }), + ); + // The Host accepts a completion only when it matches the appended text. + const ui = new CodexTurnProjector({ + threadId: "dsh-media", + turnId: id, + cwd: workspace, + startedAtMs: 1_000, + }); + ui.project({ type: "turn.started", turnId: id }); + for (const entry of emitted) { + if ( + entry.type === "item.started" || + entry.type === "item.updated" || + entry.type === "item.completed" + ) { + ui.project(entry); + } + } + + const snapshot = await session.readSnapshot(); + expect(snapshot.ok && snapshot.value.turns[0]?.items.map(({ item }) => item)).toContainEqual( + expect.objectContaining({ type: "agentMessage", text }), + ); + await session.close(); + } finally { + await rm(workspace, { recursive: true, force: true }); + } + }); + it("keeps a live Turn healthy through the DSH 0.2.1-rc.2 search request with disabled thinking", async () => { const test = setup([() => accepted()], [], ["request-1"]); const outputs = lifecycleOutputs(test.session); diff --git a/packages/adapters/grok/src/grok-adapter.ts b/packages/adapters/grok/src/grok-adapter.ts index fee878b61..049aa87f5 100644 --- a/packages/adapters/grok/src/grok-adapter.ts +++ b/packages/adapters/grok/src/grok-adapter.ts @@ -57,6 +57,7 @@ import { type TurnSteerAccepted, type TurnSteerCommand, } from "@codexhost/harness-adapter"; +import { rewriteLocalMediaMarkdown } from "@codexhost/harness-adapter/local-media-markdown"; import { harnessCommandCatalogSchema, type HarnessCommandCatalog, @@ -94,7 +95,7 @@ import { type GrokQuestionRequest, type GrokQuestionResponse, } from "./grok-questions.js"; -import { grokMediaResolveRoots, rewriteLocalMediaMarkdown } from "./local-media-markdown.js"; +import { grokMediaResolveRoots } from "./local-media-markdown.js"; import { GrokSubagentLifecycle } from "./grok-subagent-lifecycle.js"; import { grokNativeSubagentId, diff --git a/packages/adapters/grok/src/grok-history.ts b/packages/adapters/grok/src/grok-history.ts index 6b4596354..79b3576de 100644 --- a/packages/adapters/grok/src/grok-history.ts +++ b/packages/adapters/grok/src/grok-history.ts @@ -8,6 +8,7 @@ import type { HostThreadSnapshot, HostTurnSnapshot, } from "@codexhost/harness-adapter"; +import { rewriteLocalMediaMarkdown } from "@codexhost/harness-adapter/local-media-markdown"; import { hostItemIdSchema, nativeCheckpointRefSchema, @@ -18,7 +19,7 @@ import { import type { GrokTransportEvent } from "./acp-transport.js"; import { projectGrokFileChanges } from "./grok-file-change.js"; -import { grokMediaResolveRoots, rewriteLocalMediaMarkdown } from "./local-media-markdown.js"; +import { grokMediaResolveRoots } from "./local-media-markdown.js"; import { grokNativeSubagentId, grokSubagentBackground, diff --git a/packages/adapters/grok/src/local-media-markdown.ts b/packages/adapters/grok/src/local-media-markdown.ts index bbbd6918b..90e344c46 100644 --- a/packages/adapters/grok/src/local-media-markdown.ts +++ b/packages/adapters/grok/src/local-media-markdown.ts @@ -1,31 +1,6 @@ -import { existsSync, statSync } from "node:fs"; import os from "node:os"; import path from "node:path"; -const MEDIA_EXTENSIONS = new Set([ - ".aac", - ".gif", - ".jpeg", - ".jpg", - ".m4a", - ".m4v", - ".mov", - ".mp3", - ".mp4", - ".ogg", - ".png", - ".svg", - ".wav", - ".webm", - ".webp", -]); - -export interface RewriteLocalMediaMarkdownOptions { - exists?: (absolutePath: string) => boolean; - /** When true, omit a trailing unfinished `![alt](dest` so later rewrite stays prefix-stable. */ - holdIncomplete?: boolean; -} - export function grokMediaResolveRoots(cwd: string, sessionDirectory?: string): string[] { const roots = [path.resolve(cwd)]; if (sessionDirectory) { @@ -36,286 +11,3 @@ export function grokMediaResolveRoots(cwd: string, sessionDirectory?: string): s if (home) roots.push(path.join(home, "Downloads")); return roots; } - -export function rewriteLocalMediaMarkdown( - text: string, - roots: readonly string[], - options: RewriteLocalMediaMarkdownOptions = {}, -): string { - const exists = options.exists ?? mediaFileExists; - const holdIncomplete = options.holdIncomplete === true; - let output = ""; - let index = 0; - let fence = false; - while (index < text.length) { - if (fence) { - const fenceEnd = text.indexOf("```", index); - if (fenceEnd === -1) { - output += text.slice(index); - break; - } - output += text.slice(index, fenceEnd + 3); - index = fenceEnd + 3; - fence = false; - continue; - } - const fenceStart = text.indexOf("```", index); - const imageStart = text.indexOf("![", index); - const tickStart = indexOfInlineTick(text, index); - const next = earliestIndex([fenceStart, imageStart, tickStart]); - if (next === -1) { - output += text.slice(index); - break; - } - output += text.slice(index, next); - if (next === fenceStart) { - output += "```"; - index = fenceStart + 3; - fence = true; - continue; - } - if (next === imageStart) { - const parsed = parseMarkdownImage(text, imageStart); - if (!parsed) { - output += "!"; - index = imageStart + 1; - continue; - } - if (!parsed.closed) { - output += holdIncomplete - ? text.slice(imageStart, parsed.destinationStart) - : text.slice(imageStart); - break; - } - const resolved = resolveLocalMediaPath(parsed.destination, roots, exists); - output += - text.slice(imageStart, parsed.destinationStart) + - formatMarkdownDestination(resolved ?? parsed.destination, parsed.bracketed) + - text.slice(parsed.destinationEnd, parsed.end); - index = parsed.end; - continue; - } - const inline = parseInlineCode(text, tickStart); - if (!inline.closed) { - if (!holdIncomplete) output += text.slice(tickStart); - break; - } - const resolved = resolveLocalMediaPath(inline.content, roots, exists); - output += resolved ? mediaMarkdownImage(resolved) : text.slice(tickStart, inline.end); - index = inline.end; - } - return output; -} - -function indexOfInlineTick(text: string, start: number): number { - let index = start; - while (index < text.length) { - const tick = text.indexOf("`", index); - if (tick === -1) return -1; - if (text.startsWith("```", tick)) { - index = tick + 3; - continue; - } - return tick; - } - return -1; -} - -function earliestIndex(candidates: readonly number[]): number { - let next = -1; - for (const candidate of candidates) { - if (candidate === -1) continue; - if (next === -1 || candidate < next) next = candidate; - } - return next; -} - -function parseInlineCode( - text: string, - start: number, -): { closed: boolean; content: string; end: number } { - let index = start + 1; - while (index < text.length) { - const character = text[index]; - if (character === "\n") - return { closed: false, content: text.slice(start + 1, index), end: index }; - if (character === "`") - return { closed: true, content: text.slice(start + 1, index), end: index + 1 }; - index += 1; - } - return { closed: false, content: text.slice(start + 1), end: text.length }; -} - -function mediaMarkdownImage(absolutePath: string): string { - const name = path.basename(absolutePath).replaceAll("]", "\\]"); - return `![${name}](${formatMarkdownDestination(absolutePath, false)})`; -} - -function parseMarkdownImage( - text: string, - start: number, -): { - closed: boolean; - destination: string; - destinationStart: number; - destinationEnd: number; - end: number; - bracketed: boolean; -} | null { - if (text.slice(start, start + 2) !== "![") return null; - let index = start + 2; - while (index < text.length) { - const character = text[index]; - if (character === "\n") return null; - if (character === "\\" && index + 1 < text.length) { - index += 2; - continue; - } - if (character === "]") break; - index += 1; - } - if (text[index] !== "]" || text[index + 1] !== "(") return null; - const destinationStart = index + 2; - index = destinationStart; - while (text[index] === " " || text[index] === "\t") index += 1; - if (index >= text.length) { - return { - closed: false, - destination: "", - destinationStart, - destinationEnd: index, - end: text.length, - bracketed: false, - }; - } - let destination: string; - let destinationEnd: number; - let bracketed = false; - if (text[index] === "<") { - bracketed = true; - const close = text.indexOf(">", index + 1); - if (close === -1 || text.slice(index, close).includes("\n")) { - return { - closed: false, - destination: text.slice(index + 1), - destinationStart, - destinationEnd: text.length, - end: text.length, - bracketed: true, - }; - } - destination = text.slice(index + 1, close); - destinationEnd = close + 1; - index = destinationEnd; - } else { - const begin = index; - while (index < text.length) { - const character = text[index]; - if (character === "\n" || character === " " || character === "\t" || character === ")") break; - if (character === "\\" && index + 1 < text.length) { - index += 2; - continue; - } - index += 1; - } - destination = unescapeMarkdown(text.slice(begin, index)); - destinationEnd = index; - } - while (text[index] === " " || text[index] === "\t") index += 1; - if (text[index] === '"' || text[index] === "'" || text[index] === "(") { - const closer = text[index] === "(" ? ")" : text[index]; - index += 1; - while (index < text.length && text[index] !== closer && text[index] !== "\n") index += 1; - if (text[index] !== closer) { - return { - closed: false, - destination, - destinationStart, - destinationEnd, - end: text.length, - bracketed, - }; - } - index += 1; - while (text[index] === " " || text[index] === "\t") index += 1; - } - if (text[index] !== ")") { - return { - closed: false, - destination, - destinationStart, - destinationEnd, - end: text.length, - bracketed, - }; - } - return { - closed: true, - destination, - destinationStart, - destinationEnd, - end: index + 1, - bracketed, - }; -} - -function formatMarkdownDestination(destination: string, bracketed: boolean): string { - if (bracketed || /[\s()]/.test(destination)) return `<${destination}>`; - return destination.replaceAll(" ", "%20"); -} - -function unescapeMarkdown(value: string): string { - return value.replace(/\\([\\()])/g, "$1"); -} - -export function resolveLocalMediaPath( - raw: string, - roots: readonly string[], - exists: (absolutePath: string) => boolean = mediaFileExists, -): string | null { - const trimmed = raw.trim(); - if (trimmed.length === 0 || isRemoteMediaUrl(trimmed)) return null; - const stripped = stripFileUrl(trimmed); - if (!hasMediaExtension(stripped)) return null; - if (isAbsoluteMediaPath(stripped)) { - const absolute = path.resolve(stripped); - return exists(absolute) ? absolute : null; - } - for (const root of roots) { - const absolute = path.resolve(root, stripped); - if (exists(absolute)) return absolute; - } - return null; -} - -function isRemoteMediaUrl(value: string): boolean { - return /^(?:https?:|data:|app:|blob:)/i.test(value); -} - -function stripFileUrl(value: string): string { - if (!value.toLowerCase().startsWith("file:")) return value; - try { - const url = new URL(value); - if (url.protocol !== "file:") return value; - return decodeURIComponent(url.pathname); - } catch { - return value.replace(/^file:\/\//i, ""); - } -} - -function isAbsoluteMediaPath(value: string): boolean { - return path.isAbsolute(value) || /^[a-zA-Z]:[\\/]/.test(value); -} - -function hasMediaExtension(value: string): boolean { - const pathname = value.split(/[?#]/, 1)[0] ?? value; - return MEDIA_EXTENSIONS.has(path.extname(pathname).toLowerCase()); -} - -function mediaFileExists(absolutePath: string): boolean { - try { - return existsSync(absolutePath) && statSync(absolutePath).isFile(); - } catch { - return false; - } -} diff --git a/packages/adapters/grok/test/grok-history.test.ts b/packages/adapters/grok/test/grok-history.test.ts index a3230ebe8..98b337105 100644 --- a/packages/adapters/grok/test/grok-history.test.ts +++ b/packages/adapters/grok/test/grok-history.test.ts @@ -388,7 +388,7 @@ describe("Grok history local media Markdown", () => { ); expect(snapshot.turns[0]?.items).toMatchObject([ { - item: { type: "agentMessage", text: `![clip](${videoPath})` }, + item: { type: "agentMessage", text: `![clip](${videoPath.replaceAll("\\", "/")})` }, outcome: { status: "succeeded" }, }, ]); diff --git a/packages/adapters/grok/test/local-media-markdown.test.ts b/packages/adapters/grok/test/local-media-markdown.test.ts index b34fbe616..6f5cb332b 100644 --- a/packages/adapters/grok/test/local-media-markdown.test.ts +++ b/packages/adapters/grok/test/local-media-markdown.test.ts @@ -1,8 +1,9 @@ import path from "node:path"; +import { rewriteLocalMediaMarkdown } from "@codexhost/harness-adapter/local-media-markdown"; import { describe, expect, it } from "vitest"; -import { grokMediaResolveRoots, rewriteLocalMediaMarkdown } from "../src/local-media-markdown.js"; +import { grokMediaResolveRoots } from "../src/local-media-markdown.js"; const workspace = path.resolve("test-fixtures/workspace"); const home = path.resolve("test-fixtures/home"); @@ -15,6 +16,8 @@ const files = new Set([ ]); const roots = grokMediaResolveRoots(workspace, session); const exists = (absolutePath: string) => files.has(path.resolve(absolutePath)); +/** Markdown spelling of an absolute path: Windows drive paths use forward slashes. */ +const md = (absolutePath: string) => absolutePath.replaceAll("\\", "/"); describe("Grok local media Markdown", () => { it("rewrites session-relative video images to absolute paths Codex can play", () => { @@ -34,9 +37,9 @@ describe("Grok local media Markdown", () => { [ "10 秒宣傳片做好了。", "", - `![usegrokbot-templates-promo-10s.mp4](${path.join(downloads, "usegrokbot-templates-promo-10s.mp4")})`, + `![usegrokbot-templates-promo-10s.mp4](${md(path.join(downloads, "usegrokbot-templates-promo-10s.mp4"))})`, "", - `![UseGrokBot Templates 10s promo](${path.join(session, "videos/usegrokbot-templates-promo-10s.mp4")})`, + `![UseGrokBot Templates 10s promo](${md(path.join(session, "videos/usegrokbot-templates-promo-10s.mp4"))})`, ].join("\n"), ); }); @@ -65,11 +68,11 @@ describe("Grok local media Markdown", () => { [ "影片在這裡：", "", - `![aura-10s.mp4](${path.join(session, "videos/aura-10s.mp4")})`, + `![aura-10s.mp4](${md(path.join(session, "videos/aura-10s.mp4"))})`, "", "打開這份：", "", - `![aura-product-film-10s.mp4](${path.join(downloads, "aura-product-film-10s.mp4")})`, + `![aura-product-film-10s.mp4](${md(path.join(downloads, "aura-product-film-10s.mp4"))})`, ].join("\n"), ); }); @@ -89,7 +92,7 @@ describe("Grok local media Markdown", () => { emitted = projected; } expect(emitted).toBe( - `影片：\n\n![clip](${path.join(session, "videos/usegrokbot-templates-promo-10s.mp4")})\n完成`, + `影片：\n\n![clip](${md(path.join(session, "videos/usegrokbot-templates-promo-10s.mp4"))})\n完成`, ); }); @@ -126,7 +129,7 @@ describe("Grok local media Markdown", () => { emitted = projected; } expect(emitted).toBe( - `影片：\n\n![usegrokbot-templates-promo-10s.mp4](${path.join(session, "videos/usegrokbot-templates-promo-10s.mp4")})\n完成`, + `影片：\n\n![usegrokbot-templates-promo-10s.mp4](${md(path.join(session, "videos/usegrokbot-templates-promo-10s.mp4"))})\n完成`, ); }); diff --git a/packages/harness-adapter/package.json b/packages/harness-adapter/package.json index f403bcbe8..7ed5f1acb 100644 --- a/packages/harness-adapter/package.json +++ b/packages/harness-adapter/package.json @@ -15,6 +15,10 @@ "types": "./dist/plugin.d.ts", "import": "./dist/plugin.js" }, + "./local-media-markdown": { + "types": "./dist/local-media-markdown.d.ts", + "import": "./dist/local-media-markdown.js" + }, "./session-import": { "types": "./dist/session-import.d.ts", "import": "./dist/session-import.js" diff --git a/packages/harness-adapter/src/local-media-markdown.ts b/packages/harness-adapter/src/local-media-markdown.ts new file mode 100644 index 000000000..ed6a68d50 --- /dev/null +++ b/packages/harness-adapter/src/local-media-markdown.ts @@ -0,0 +1,385 @@ +import { existsSync, statSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const MEDIA_EXTENSIONS = new Set([ + ".aac", + ".gif", + ".jpeg", + ".jpg", + ".m4a", + ".m4v", + ".mov", + ".mp3", + ".mp4", + ".ogg", + ".png", + ".svg", + ".wav", + ".webm", + ".webp", +]); + +/** Whether an absolute candidate path is a media file Codex Desktop can load. */ +export type LocalMediaExists = (absolutePath: string) => boolean; + +export interface RewriteLocalMediaMarkdownOptions { + exists?: LocalMediaExists; + /** + * Hold back a trailing construct that later text could still complete or invalidate + * (an open image or inline code span), so a growing text projects prefix-stably. + */ + holdIncomplete?: boolean; +} + +/** + * Codex Desktop loads a Markdown image from local disk only when its destination is an absolute + * path or `file:` URL; it resolves any other destination against its own page. Rewrite local + * media in image destinations, and inline code holding only a media path, to absolute paths + * found under `roots` in order. Remote URLs, missing files, and fenced code stay unchanged. + */ +export function rewriteLocalMediaMarkdown( + text: string, + roots: readonly string[], + options: RewriteLocalMediaMarkdownOptions = {}, +): string { + return projectLocalMedia( + text, + roots, + options.exists ?? mediaFileExists, + options.holdIncomplete === true, + ).text; +} + +/** + * Streams native agent text through {@link rewriteLocalMediaMarkdown} as append-only deltas. + * Published text never changes: a trailing construct waits until later text decides it, `flush` + * releases it, and each file check is remembered so one message resolves a path one way. + */ +export class LocalMediaMarkdownStream { + readonly #roots: readonly string[]; + readonly #exists: LocalMediaExists; + #native = ""; + #text = ""; + /** Native text length represented by `#text`; the rest is held. */ + #consumed = 0; + #verbatim = false; + + constructor(roots: readonly string[], exists: LocalMediaExists = mediaFileExists) { + this.#roots = [...roots]; + const known = new Map(); + this.#exists = (absolutePath) => { + let found = known.get(absolutePath); + if (found === undefined) { + found = exists(absolutePath); + known.set(absolutePath, found); + } + return found; + }; + } + + /** Projected text published so far: every returned delta, concatenated. */ + get text(): string { + return this.#text; + } + + /** Add native text and return the projected text it makes publishable, possibly empty. */ + append(text: string): string { + this.#native += text; + return this.#advance(true); + } + + /** Return the projected text still held at the end of the stream. */ + flush(): string { + return this.#advance(false); + } + + #advance(holdIncomplete: boolean): string { + const next = this.#verbatim + ? undefined + : projectLocalMedia(this.#native, this.#roots, this.#exists, holdIncomplete); + let delta: string; + if (next?.text.startsWith(this.#text)) { + delta = next.text.slice(this.#text.length); + this.#consumed = next.consumed; + } else { + // Published text cannot be retracted, so continue with the native text verbatim. + this.#verbatim = true; + delta = this.#native.slice(this.#consumed); + this.#consumed = this.#native.length; + } + this.#text += delta; + return delta; + } +} + +interface LocalMediaProjection { + readonly text: string; + /** Length of the native text the projection covers. */ + readonly consumed: number; +} + +function projectLocalMedia( + text: string, + roots: readonly string[], + exists: LocalMediaExists, + holdIncomplete: boolean, +): LocalMediaProjection { + let output = ""; + let index = 0; + let fence = false; + while (index < text.length) { + if (fence) { + const fenceEnd = text.indexOf("```", index); + if (fenceEnd === -1) break; + output += text.slice(index, fenceEnd + 3); + index = fenceEnd + 3; + fence = false; + continue; + } + const next = earliestIndex([ + text.indexOf("```", index), + text.indexOf("![", index), + indexOfInlineTick(text, index), + ]); + if (next === -1) break; + output += text.slice(index, next); + index = next; + if (text.startsWith("```", next)) { + output += "```"; + index += 3; + fence = true; + continue; + } + if (text.startsWith("![", next)) { + const image = parseMarkdownImage(text, next); + if (image.kind === "text") { + output += text.slice(next, image.end); + index = image.end; + continue; + } + if (image.kind === "open") { + if (!holdIncomplete) break; + return { text: output + text.slice(next, image.holdFrom), consumed: image.holdFrom }; + } + const resolved = resolveLocalMediaPath(image.destination, roots, exists); + output += + resolved === null + ? text.slice(next, image.end) + : text.slice(next, image.destinationStart) + + markdownDestination(resolved, image.bracketed) + + text.slice(image.destinationEnd, image.end); + index = image.end; + continue; + } + const code = parseInlineCode(text, next); + if (code.kind === "text") { + output += "`"; + index = next + 1; + continue; + } + if (code.kind === "open") { + if (!holdIncomplete) break; + return { text: output, consumed: next }; + } + const resolved = resolveLocalMediaPath(code.content, roots, exists); + output += resolved === null ? text.slice(next, code.end) : mediaMarkdownImage(resolved); + index = code.end; + } + return { text: output + text.slice(index), consumed: text.length }; +} + +function indexOfInlineTick(text: string, start: number): number { + let index = start; + while (index < text.length) { + const tick = text.indexOf("`", index); + if (tick === -1) return -1; + if (text.startsWith("```", tick)) { + index = tick + 3; + continue; + } + return tick; + } + return -1; +} + +function earliestIndex(candidates: readonly number[]): number { + let next = -1; + for (const candidate of candidates) { + if (candidate === -1) continue; + if (next === -1 || candidate < next) next = candidate; + } + return next; +} + +/** `text`: ordinary text through `end`. `open`: the text ends before the construct is decided. */ +type ParsedInlineCode = + { kind: "code"; content: string; end: number } | { kind: "open" } | { kind: "text" }; + +function parseInlineCode(text: string, start: number): ParsedInlineCode { + for (let index = start + 1; index < text.length; index += 1) { + const character = text[index]; + // A path never spans lines, so leave the tick as text rather than hold everything after it. + if (character === "\n") return { kind: "text" }; + if (character === "`") { + return { kind: "code", content: text.slice(start + 1, index), end: index + 1 }; + } + } + return { kind: "open" }; +} + +type ParsedMarkdownImage = + | { + kind: "image"; + destination: string; + destinationStart: number; + destinationEnd: number; + end: number; + bracketed: boolean; + } + /** Published text may end at `holdFrom`; only later text can decide the rest. */ + | { kind: "open"; holdFrom: number } + | { kind: "text"; end: number }; + +function parseMarkdownImage(text: string, start: number): ParsedMarkdownImage { + // Until the label closes, only `![` is settled: `!` was publishable before `[` arrived. + const openLabel = { kind: "open", holdFrom: start + 2 } as const; + let index = start + 2; + while (index < text.length) { + const character = text[index]; + if (character === "\n") return { kind: "text", end: start + 1 }; + if (character === "\\") { + index += 2; + continue; + } + if (character === "]") break; + index += 1; + } + if (index + 1 >= text.length) return openLabel; + if (text[index + 1] !== "(") return { kind: "text", end: start + 1 }; + const destinationStart = index + 2; + // From here `![label](` is settled; a malformed rest leaves it, and the label, as text. + const open = { kind: "open", holdFrom: destinationStart } as const; + const malformed = { kind: "text", end: destinationStart } as const; + index = skipBlanks(text, destinationStart); + if (index >= text.length) return open; + let destination: string; + let destinationEnd: number; + let bracketed = false; + if (text[index] === "<") { + bracketed = true; + let close = index + 1; + while (close < text.length && text[close] !== ">" && text[close] !== "\n") close += 1; + if (close >= text.length) return open; + if (text[close] === "\n") return malformed; + destination = text.slice(index + 1, close); + destinationEnd = close + 1; + index = destinationEnd; + } else { + const begin = index; + while (index < text.length) { + const character = text[index]; + if (character === "\n" || character === " " || character === "\t" || character === ")") break; + index += character === "\\" ? 2 : 1; + } + if (index >= text.length) return open; + destination = unescapeMarkdown(text.slice(begin, index)); + destinationEnd = index; + } + index = skipBlanks(text, index); + if (index >= text.length) return open; + const opener = text[index]; + if (opener === '"' || opener === "'" || opener === "(") { + const closer = opener === "(" ? ")" : opener; + index += 1; + while (index < text.length && text[index] !== closer && text[index] !== "\n") index += 1; + if (index >= text.length) return open; + if (text[index] === "\n") return malformed; + index = skipBlanks(text, index + 1); + if (index >= text.length) return open; + } + if (text[index] !== ")") return malformed; + return { + kind: "image", + destination, + destinationStart, + destinationEnd, + end: index + 1, + bracketed, + }; +} + +function skipBlanks(text: string, start: number): number { + let index = start; + while (text[index] === " " || text[index] === "\t") index += 1; + return index; +} + +function mediaMarkdownImage(absolutePath: string): string { + const name = path.basename(absolutePath).replaceAll("]", "\\]"); + return `![${name}](${markdownDestination(absolutePath, false)})`; +} + +function markdownDestination(absolutePath: string, bracketed: boolean): string { + // CommonMark drops a backslash before punctuation (`C:\Users\me\.codex`), and Codex reads + // `C:/…` as absolute; other paths keep their backslashes escaped. + const destination = /^[A-Za-z]:\\/.test(absolutePath) + ? absolutePath.replaceAll("\\", "/") + : absolutePath.replaceAll("\\", "\\\\"); + return bracketed || /[\s()]/.test(destination) ? `<${destination}>` : destination; +} + +function unescapeMarkdown(value: string): string { + return value.replace(/\\([\\()])/g, "$1"); +} + +/** Resolve a media reference to an existing absolute file, trying relative paths under `roots`. */ +export function resolveLocalMediaPath( + raw: string, + roots: readonly string[], + exists: LocalMediaExists = mediaFileExists, +): string | null { + const trimmed = raw.trim(); + if (trimmed.length === 0 || isRemoteMediaUrl(trimmed)) return null; + const stripped = stripFileUrl(trimmed); + if (stripped === null || !hasMediaExtension(stripped)) return null; + if (isAbsoluteMediaPath(stripped)) { + const absolute = path.resolve(stripped); + return exists(absolute) ? absolute : null; + } + for (const root of roots) { + const absolute = path.resolve(root, stripped); + if (exists(absolute)) return absolute; + } + return null; +} + +function isRemoteMediaUrl(value: string): boolean { + return /^(?:https?:|data:|app:|blob:)/i.test(value); +} + +function stripFileUrl(value: string): string | null { + if (!value.toLowerCase().startsWith("file:")) return value; + try { + return fileURLToPath(value); + } catch { + return null; + } +} + +function isAbsoluteMediaPath(value: string): boolean { + return path.isAbsolute(value) || /^[a-zA-Z]:[\\/]/.test(value); +} + +function hasMediaExtension(value: string): boolean { + const pathname = value.split(/[?#]/, 1)[0] ?? value; + return MEDIA_EXTENSIONS.has(path.extname(pathname).toLowerCase()); +} + +function mediaFileExists(absolutePath: string): boolean { + try { + return existsSync(absolutePath) && statSync(absolutePath).isFile(); + } catch { + return false; + } +} diff --git a/packages/harness-adapter/test/local-media-markdown.test.ts b/packages/harness-adapter/test/local-media-markdown.test.ts new file mode 100644 index 000000000..119f9cc4d --- /dev/null +++ b/packages/harness-adapter/test/local-media-markdown.test.ts @@ -0,0 +1,152 @@ +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { describe, expect, it, vi } from "vitest"; + +import { + LocalMediaMarkdownStream, + resolveLocalMediaPath, + rewriteLocalMediaMarkdown, +} from "../src/local-media-markdown.js"; + +const workspace = path.resolve("test-fixtures/media-workspace"); +const files = new Set( + [ + "shot.png", + "a.png", + "x.png", + "sub/b.png", + "dir with space/c.png", + ".codex/d.png", + "截图/e.png", + ].map((file) => path.join(workspace, file)), +); +const exists = (absolutePath: string) => files.has(path.resolve(absolutePath)); +const roots = [workspace]; +/** Markdown spelling of an absolute path: Windows drive paths use forward slashes. */ +const md = (absolutePath: string) => absolutePath.replaceAll("\\", "/"); +const at = (file: string) => md(path.join(workspace, file)); +const rewrite = (text: string, holdIncomplete = false) => + rewriteLocalMediaMarkdown(text, roots, { exists, holdIncomplete }); + +describe("rewriteLocalMediaMarkdown", () => { + it("makes relative image destinations absolute", () => { + expect(rewrite("![shot](shot.png)\n![b](./sub/b.png)")).toBe( + `![shot](${at("shot.png")})\n![b](${at("sub/b.png")})`, + ); + }); + + it("keeps non-ASCII paths and brackets paths with spaces", () => { + expect(rewrite('![e](截图/e.png) ![c]( "title")')).toBe( + `![e](${at("截图/e.png")}) ![c](<${at("dir with space/c.png")}> "title")`, + ); + }); + + it("writes Windows paths so Markdown escapes cannot drop a separator", () => { + const rewritten = rewrite("![d](.codex/d.png)"); + expect(rewritten).toBe(`![d](${at(".codex/d.png")})`); + expect(rewritten).not.toMatch(/\\[!-/:-@[-`{-~]/u); + }); + + it("resolves file URLs and absolute paths", () => { + const absolute = path.join(workspace, "a.png"); + expect(rewrite(`![a](${pathToFileURL(absolute).href}) and \`${absolute}\``)).toBe( + `![a](${at("a.png")}) and ![a.png](${at("a.png")})`, + ); + }); + + it("leaves remote, missing, and unparsed references byte-for-byte unchanged", () => { + const text = [ + "![remote](https://example.com/a.png)", + "![missing](missing.png)", + "![escaped](missing\\(1\\).png)", + "`npm test` and `a.txt`", + "```md\n![a](a.png)\n`a.png`\n```", + ].join("\n"); + expect(rewrite(text)).toBe(text); + }); + + it("does not let a malformed image or stray backtick stop later rewrites", () => { + expect(rewrite("![a](a.png bad) then `\nand ![x](x.png)")).toBe( + `![a](a.png bad) then \`\nand ![x](${at("x.png")})`, + ); + }); + + it("holds only a trailing construct that later text could still change", () => { + expect(rewrite("see ![a](a.p", true)).toBe("see ![a]("); + expect(rewrite("see `a.p", true)).toBe("see "); + expect(rewrite("see ![a", true)).toBe("see !["); + expect(rewrite("![a](a.png bad) then ` stray\nmore", true)).toBe( + "![a](a.png bad) then ` stray\nmore", + ); + }); + + it("resolves relative paths against roots in order", () => { + const first = path.join(workspace, "sub"); + expect(resolveLocalMediaPath("b.png", [first, workspace], exists)).toBe( + path.join(first, "b.png"), + ); + expect(resolveLocalMediaPath("notes.txt", roots, () => true)).toBeNull(); + }); +}); + +describe("LocalMediaMarkdownStream", () => { + const samples = [ + 'see `a.png` and ![b](sub/b.png "t") then ![c](missing.png)', + "![a`x.png`](a.png) and `x.png`", + "```md\n![a](a.png)\n```\nafter `a.png`", + "x ` y\n![a](a.png)", + "![a](a.png bad) then ![b](sub/b.png)", + "![a]\n![c]()", + "a!![b](sub/b.png)!", + "``a.png`` and `a.png`` and ```", + "![e](截图/e.png)。", + ]; + + function streamed(text: string, cuts: readonly number[]): { deltas: string[]; text: string } { + const stream = new LocalMediaMarkdownStream(roots, exists); + const deltas: string[] = []; + let start = 0; + for (const end of [...cuts, text.length]) { + deltas.push(stream.append(text.slice(start, end))); + start = end; + } + deltas.push(stream.flush()); + expect(stream.text).toBe(deltas.join("")); + return { deltas, text: stream.text }; + } + + it("publishes the same text as a one-shot rewrite for every split", () => { + for (const sample of samples) { + const expected = rewrite(sample); + for (let first = 0; first <= sample.length; first += 1) { + for (let second = first; second <= sample.length; second += 1) { + expect(streamed(sample, [first, second]).text, `${sample} @${first},${second}`).toBe( + expected, + ); + } + } + const characters = Array.from({ length: sample.length }, (_, index) => index); + expect(streamed(sample, characters).text).toBe(expected); + } + }); + + it("does not hold settled text while the stream is open", () => { + const stream = new LocalMediaMarkdownStream(roots, exists); + expect(stream.append("x ` stray\nthen ![a](a.png) and ![b](sub/")).toBe( + `x \` stray\nthen ![a](${at("a.png")}) and ![b](`, + ); + expect(stream.append("b.png)")).toBe(`${at("sub/b.png")})`); + expect(stream.flush()).toBe(""); + }); + + it("checks each path once for the lifetime of a stream", () => { + const check = vi.fn(exists); + const stream = new LocalMediaMarkdownStream(roots, check); + stream.append("![a](a.png) "); + stream.append("more "); + stream.append("text"); + stream.flush(); + expect(check).toHaveBeenCalledTimes(1); + }); +}); From a53c6a3356f8c5bd3d4b375256674db613899961 Mon Sep 17 00:00:00 2001 From: BeiLuoqu Date: Sat, 24 Oct 2026 22:34:16 +0800 Subject: [PATCH 02/15] fix: show images users attach in external Harness threads Desktop sends an attached image both as a localImage input item and as an absolute path in the prompt's attachment context. Harness input is text only, so the Host kept the text and Native Session history has no image items either. Desktop draws user-bubble thumbnails only from image items; without them every attached image degraded to a file-name chip. - Restore localImage items from the attachment context when projecting a user message at Turn start, for steered input, and in history. - Read the paths with desktopImageAttachmentPaths in shared-contracts, which the Renderer's remote attachment sender now reuses; paths typed into the request are never treated as attachments. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/external-thread-media.md | 14 +++++ docs/index.md | 2 +- .../protocol-core/src/codex-ui-projector.ts | 25 ++++++-- .../test/codex-ui-projector.test.ts | 59 ++++++++++++++++++ .../src/renderer-remote-attachments.ts | 20 +----- .../src/desktop-attachments.ts | 31 ++++++++++ packages/shared-contracts/src/index.ts | 2 + .../test/desktop-attachments.test.ts | 62 +++++++++++++++++++ 8 files changed, 193 insertions(+), 22 deletions(-) create mode 100644 packages/shared-contracts/src/desktop-attachments.ts create mode 100644 packages/shared-contracts/test/desktop-attachments.test.ts diff --git a/docs/architecture/external-thread-media.md b/docs/architecture/external-thread-media.md index 61bda0685..ae6d372fc 100644 --- a/docs/architecture/external-thread-media.md +++ b/docs/architecture/external-thread-media.md @@ -8,6 +8,8 @@ Agent 消息中的 Markdown 图片（`![alt](目标)`）只有在目标是绝对 Harness 常按工作目录写相对路径，因此需要由 Adapter 改写。 +用户消息气泡中的缩略图只来自消息 `content` 里的 `image` / `localImage` 条目。Desktop 同时把附件清单（`# Files mentioned by the user:`）解析成文件标签；只有消息带图片条目时，才把对应图片从标签中移除。消息没有图片条目时，所贴图片只显示为文件名标签。 + ## Agent 消息：改写本地媒体路径 `@codexhost/harness-adapter/local-media-markdown` 提供共用的改写： @@ -27,3 +29,15 @@ Harness 常按工作目录写相对路径，因此需要由 Adapter 改写。 | DeepSeek Harness | Agent 当前工作目录（随 `working-directory/change` 更新），再到 Session 工作目录 | 实时输出与历史（`readSnapshot`）；用量刷新不做改写 | 其他 Harness 的 Agent 文本仍原样转发；它们输出相对路径时，图片仍显示为占位框。 + +## 用户消息：还原图片附件 + +Desktop 发送带图片的消息时，请求里既有 `localImage` 条目，也在文字的附件清单中写下每张图片的绝对路径，并标注 `Image attachment: true`。公共 Harness 输入只有文字，Host 转发时只保留文字，Harness 按路径读取图片；Native Session 历史同样只有文字。 + +`protocol-core` 投影用户消息时，从文字中 `## My request:` 之前的附件清单读出这些路径，补回 `localImage` 条目，放在文字之后。开始 Turn、运行中插入和历史回放三处投影一致，因此新旧会话都显示缩略图，`imageCount` 也与 Desktop 的乐观消息一致。解析规则在 `@codexhost/shared-contracts` 的 `desktopImageAttachmentPaths`，Renderer 的远程附件补存使用同一规则。 + +边界： + +- 只认附件清单中的绝对路径；用户在请求正文里写的路径不算附件。 +- 不读取、不复制图片。剪贴板截图通常保存在系统临时目录，文件被清理后缩略图无法加载。 +- 远程外部会话的图片路径位于远程主机，其缩略图显示尚未单独验证。 diff --git a/docs/index.md b/docs/index.md index 357f37910..1680ebf7f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -23,7 +23,7 @@ | [`architecture/harness-questions.md`](architecture/harness-questions.md) | 外部 Harness 多选提问卡片、原生单选分流、回答归属与生命周期；修改提问交互时阅读。 | | [`architecture/external-thread-steering.md`](architecture/external-thread-steering.md) | 外部 Thread 按 Session 能力选择原生同轮插入或停止后重发；修改 steering 时阅读。 | | [`architecture/external-thread-queue.md`](architecture/external-thread-queue.md) | 外部 Thread 跟进消息的 Host 原生 `thread/queue/*`、自动推进与中断暂停；修改排队或 Turn 启动时阅读。 | -| [`architecture/external-thread-media.md`](architecture/external-thread-media.md) | 外部 Thread 中本地图片的 Desktop 加载条件，以及 Agent 消息媒体路径的改写；修改媒体显示或排查图片占位框时阅读。 | +| [`architecture/external-thread-media.md`](architecture/external-thread-media.md) | 外部 Thread 中本地图片的 Desktop 加载条件、Agent 消息媒体路径改写和用户消息图片附件还原；修改媒体显示或排查图片不显示时阅读。 | | [`architecture/delegation-titles.md`](architecture/delegation-titles.md) | 委派子会话标题、子任务显示名、创建快照与重试边界。 | | [`architecture/thread-watch.md`](architecture/thread-watch.md) | 一次性 Thread 停下通知（`thread watch`）的模型、结果、送达与边界；修改委派跟进或通知时阅读。 | | [`architecture/official-traffic-ownership.md`](architecture/official-traffic-ownership.md) | Host 只截获自身流量、其余原样转发官方的归属判别与例外；修改请求路由或参数校验时阅读。 | diff --git a/packages/protocol-core/src/codex-ui-projector.ts b/packages/protocol-core/src/codex-ui-projector.ts index c2925005c..e9ac5f43a 100644 --- a/packages/protocol-core/src/codex-ui-projector.ts +++ b/packages/protocol-core/src/codex-ui-projector.ts @@ -7,6 +7,7 @@ import type { HostItemSnapshot, HostItemUpdate, HostQuestionInteraction, + HostTextInput, HostTurnSnapshot, HistoricalTurnOutcome, InteractionClosedEvent, @@ -24,7 +25,10 @@ import type { JsonObject, JsonValue, } from "@codexhost/shared-contracts"; -import { REASONING_TRANSCRIPT_COMMAND } from "@codexhost/shared-contracts"; +import { + desktopImageAttachmentPaths, + REASONING_TRANSCRIPT_COMMAND, +} from "@codexhost/shared-contracts"; import { createTwoFilesPatch } from "diff"; import path from "node:path"; import { summarizeFileChanges } from "./file-change-summary.js"; @@ -467,6 +471,19 @@ function collabAgentStatus( } } +/** + * A Harness receives a turn's images only as the attachment paths in its text. Restore them as + * native `localImage` input, from which Desktop draws thumbnails instead of file chips. + */ +function projectUserInput(input: readonly HostTextInput[]): JsonObject[] { + return [ + ...input.map(({ text }) => ({ type: "text", text, text_elements: [] })), + ...input.flatMap(({ text }) => + desktopImageAttachmentPaths(text).map((path) => ({ type: "localImage", path })), + ), + ]; +} + /** `clientId` lets Desktop settle its optimistic steering message for this input. */ function projectUserMessage( item: Extract, @@ -476,7 +493,7 @@ function projectUserMessage( id: item.itemId, type: "userMessage", clientId, - content: item.input.map(({ text }) => ({ type: "text", text, text_elements: [] })), + content: projectUserInput(item.input), }; } @@ -705,7 +722,7 @@ export function projectHistoricalTurn(input: HistoricalTurnProjectionInput): Jso id: `${turnId}-user`, type: "userMessage", clientId: null, - content: snapshot.input.map(({ text }) => ({ type: "text", text, text_elements: [] })), + content: projectUserInput(snapshot.input), }, ...entries.flatMap(({ entry, projected }) => finalAnswer && entry === lastVisible @@ -1355,7 +1372,7 @@ export class CodexTurnProjector { id: `${this.#turnId}-user`, type: "userMessage", clientId: this.#clientUserMessageId, - content: this.#input.map(({ text }) => ({ type: "text", text, text_elements: [] })), + content: projectUserInput(this.#input), }, ]; } diff --git a/packages/protocol-core/test/codex-ui-projector.test.ts b/packages/protocol-core/test/codex-ui-projector.test.ts index 904469463..599bc554c 100644 --- a/packages/protocol-core/test/codex-ui-projector.test.ts +++ b/packages/protocol-core/test/codex-ui-projector.test.ts @@ -186,6 +186,65 @@ describe("Codex UI projector", () => { ]); }); + it("restores Desktop image attachments as native localImage input", () => { + const text = [ + "", + "# Files mentioned by the user:", + "", + "## codex-clipboard-1.png: C:/Users/me/AppData/Local/Temp/codex-clipboard-1.png", + "Image attachment: true", + "", + "## notes.md: /work/notes.md", + "", + "## My request:", + "Compare /work/typed.png with the screenshot", + ].join("\n"); + const input = [{ type: "text" as const, text }]; + const content = [ + { type: "text", text, text_elements: [] }, + { type: "localImage", path: "C:/Users/me/AppData/Local/Temp/codex-clipboard-1.png" }, + ]; + + const live = new CodexTurnProjector({ + threadId: "thread-1", + turnId, + cwd: "/workspace", + startedAtMs: 1_000, + initialInput: input, + }); + expect(live.project({ type: "turn.started", turnId }).messages).toContainEqual( + expect.objectContaining({ + method: "item/started", + params: expect.objectContaining({ + item: { id: "turn-1-user", type: "userMessage", clientId: null, content }, + }), + }), + ); + const steered = { type: "userMessage" as const, itemId: itemId("steer-1"), input }; + expect(live.project({ type: "item.started", turnId, item: steered }).messages).toMatchObject([ + { method: "item/started", params: { item: { id: "steer-1", content } } }, + ]); + + const historical = projectHistoricalTurn({ + turnId, + cwd: "/workspace", + snapshot: { + nativeTurnRef: nativeTurnRefSchema.parse({ + harnessId: "claude-code", + nativeSessionId: "native", + nativeTurnKey: "user", + formatVersion: 1, + }), + input, + items: [], + outcome: { status: "succeeded" }, + }, + }); + expect(historical.items).toEqual([ + { id: "turn-1-user", type: "userMessage", clientId: null, content }, + ]); + }); + it("does not invent historical duration from invalid native timing", () => { const snapshot: HostThreadSnapshot["turns"][number] = { nativeTurnRef: nativeTurnRefSchema.parse({ diff --git a/packages/renderer-extension/src/renderer-remote-attachments.ts b/packages/renderer-extension/src/renderer-remote-attachments.ts index 60dc0b919..cbb970903 100644 --- a/packages/renderer-extension/src/renderer-remote-attachments.ts +++ b/packages/renderer-extension/src/renderer-remote-attachments.ts @@ -1,3 +1,5 @@ +import { desktopImageAttachmentPaths } from "@codexhost/shared-contracts"; + function isRecord(value: unknown): value is Record { return value !== null && typeof value === "object" && !Array.isArray(value); } @@ -49,22 +51,6 @@ function imageFile(value: Record): { contents: Blob; extension: return { contents: new Blob([bytes], { type: mimeType }), extension }; } -/** Only native serialized attachment context, not arbitrary paths in the user's request. */ -function nativeImagePaths(text: string): string[] { - const context = text.split("## My request:")[0] ?? ""; - const start = context.indexOf("# Files mentioned by the user:"); - if (start < 0) return []; - return [ - ...context - .slice(start) - .matchAll( - /(?:^|\n)## [^\r\n]+: ((?:\/|[A-Za-z]:[\\/]|\\\\)[^\r\n]+)\r?\nImage attachment: true(?=\r?\n|$)/gu, - ), - ] - .map((match) => match[1]) - .filter(filePath); -} - /** * Desktop 26.930 carries remote images inline and fills file paths asynchronously. * Materialize missing paths through the same Manager's native attachment store, @@ -116,7 +102,7 @@ export function createRemoteAttachmentSender( .filter((item) => isRecord(item) && item.type === "text") .map((item) => (isRecord(item) && typeof item.text === "string" ? item.text : "")) .join("\n"); - const existingPaths = nativeImagePaths(text); + const existingPaths = desktopImageAttachmentPaths(text); const prepare = async () => { // Reuse native completed uploads. With partial context there is no reliable // image-to-path association, so do not guess which image is missing. diff --git a/packages/shared-contracts/src/desktop-attachments.ts b/packages/shared-contracts/src/desktop-attachments.ts new file mode 100644 index 000000000..9beef7327 --- /dev/null +++ b/packages/shared-contracts/src/desktop-attachments.ts @@ -0,0 +1,31 @@ +/** + * Attachment context Codex Desktop serializes ahead of the user's request: + * + * ```text + * # Files mentioned by the user: + * + * ## shot.png: /absolute/path/shot.png + * Image attachment: true + * + * ## My request: + * … + * ``` + * + * A Harness receives a turn's images only as these paths, so the Host reads them back from the + * same text. Paths the user typed into the request are never attachments. + */ +const ATTACHED_FILES_HEADING = "# Files mentioned by the user:"; +const REQUEST_HEADING = "## My request:"; +const IMAGE_ATTACHMENT_PATTERN = + /(?:^|\n)## [^\r\n]+: ((?:\/|[A-Za-z]:[\\/]|\\\\)[^\r\n]+)\r?\nImage attachment: true(?=\r?\n|$)/gu; +const CONTROL_CHARACTER_PATTERN = /[\u0000-\u001f\u007f]/u; + +/** Absolute image paths in the attachment context Desktop serialized into a prompt. */ +export function desktopImageAttachmentPaths(text: string): string[] { + const context = text.split(REQUEST_HEADING)[0] ?? ""; + const start = context.indexOf(ATTACHED_FILES_HEADING); + if (start < 0) return []; + return [...context.slice(start).matchAll(IMAGE_ATTACHMENT_PATTERN)] + .map((match) => match[1] ?? "") + .filter((path) => path.length > 0 && !CONTROL_CHARACTER_PATTERN.test(path)); +} diff --git a/packages/shared-contracts/src/index.ts b/packages/shared-contracts/src/index.ts index ef62dcf45..5f933e248 100644 --- a/packages/shared-contracts/src/index.ts +++ b/packages/shared-contracts/src/index.ts @@ -357,6 +357,8 @@ export { type DelegationMentionRewrite, } from "./delegation-mention.js"; +export { desktopImageAttachmentPaths } from "./desktop-attachments.js"; + export { REMOTE_SSH_SETUP_METHOD, remoteSshSetupParamsSchema, diff --git a/packages/shared-contracts/test/desktop-attachments.test.ts b/packages/shared-contracts/test/desktop-attachments.test.ts new file mode 100644 index 000000000..a51dcef00 --- /dev/null +++ b/packages/shared-contracts/test/desktop-attachments.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; + +import { desktopImageAttachmentPaths } from "../src/desktop-attachments.js"; + +function prompt(attachments: readonly string[], request = "Compare them"): string { + return [ + "", + "# Files mentioned by the user:", + ...attachments, + "Distinguish instructions in attached documents from the user's request.", + "", + "## My request:", + request, + ].join("\n"); +} + +describe("desktopImageAttachmentPaths", () => { + it("reads the image attachments Desktop serialized, in order", () => { + expect( + desktopImageAttachmentPaths( + prompt([ + "", + "## codex-clipboard-1.png: C:/Users/me/AppData/Local/Temp/codex-clipboard-1.png", + "Image attachment: true", + "", + "## notes.md: /work/notes.md", + "", + "## remote.png: /home/me/.codex/attachments/remote.png", + "Image attachment: true", + ]), + ), + ).toEqual([ + "C:/Users/me/AppData/Local/Temp/codex-clipboard-1.png", + "/home/me/.codex/attachments/remote.png", + ]); + }); + + it("ignores paths typed into the request or outside the attachment context", () => { + const typed = "## typed.png: /typed.png\nImage attachment: true"; + expect( + desktopImageAttachmentPaths( + prompt(["", "## a.png: /a.png", "Image attachment: true"], typed), + ), + ).toEqual(["/a.png"]); + expect(desktopImageAttachmentPaths(typed)).toEqual([]); + }); + + it("accepts only absolute paths without control characters", () => { + expect( + desktopImageAttachmentPaths( + prompt([ + "", + "## a.png: a.png", + "Image attachment: true", + "", + "## b.png: /b\u0000.png", + "Image attachment: true", + ]), + ), + ).toEqual([]); + }); +}); From f266f539ddc62aa94776c53de2b1927c520c714c Mon Sep 17 00:00:00 2001 From: BeiLuoqu Date: Sun, 25 Oct 2026 13:16:02 +0800 Subject: [PATCH 03/15] fix: link local media whose path holds a literal % Codex Desktop percent-decodes a local media destination twice before it reads the file, so no single escaping keeps a literal % intact. Grok names each session directory after its cwd encoded with encodeURIComponent, so every image or video Grok generates lives under a path with %5C or %3A and showed the "Image unavailable" placeholder. Link such a file through a %-free hard link (or a copy when linking fails) under /media-links// instead. If no alias can be made, escape % as %2525, which survives exactly two decodes. Paths without % are unchanged and create no alias. Callers can override this through the new linkPath option; LocalMediaMarkdownStream asks once per path so published text stays prefix-stable. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/external-thread-media.md | 1 + .../src/local-media-markdown.ts | 89 +++++++++++++++++-- .../test/local-media-markdown.test.ts | 79 ++++++++++++++++ 3 files changed, 162 insertions(+), 7 deletions(-) diff --git a/docs/architecture/external-thread-media.md b/docs/architecture/external-thread-media.md index ae6d372fc..d0b940af4 100644 --- a/docs/architecture/external-thread-media.md +++ b/docs/architecture/external-thread-media.md @@ -17,6 +17,7 @@ Harness 常按工作目录写相对路径，因此需要由 Adapter 改写。 - 改写图片目标中的本地媒体路径，以及只包含一个媒体路径的行内代码（转成图片）。相对路径按 Adapter 给出的根目录依次查找，只有找到实际存在的文件才改写。 - 远程 URL、`data:` 等地址、找不到的文件、无法解析的写法和围栏代码块中的内容保持原样，不改动一个字节。 - Windows 盘符路径写成正斜杠（`C:/…`）。CommonMark 会把反斜杠加标点当作转义，原样输出会让 `C:\Users\me\.codex\…` 丢掉分隔符。 +- 路径中含有 `%` 时另行处理。Desktop 读取文件前会对目标做两次 `decodeURI`，任何一次性的转义都无法保住字面 `%`；Grok 的会话目录名是用 `encodeURIComponent` 编码的工作目录，总会包含 `%`。改写时为这类文件在 `<数据目录>/media-links/<路径哈希>/` 下建一个不含 `%` 的硬链接（失败则复制），消息中链接这个别名；硬链接和复制都失败时，把 `%` 转义成 `%2525`，正好经受两次解码。不含 `%` 的路径不受影响，也不创建别名。别名目录可以随时删除，下次改写时重新生成。 - 支持的扩展名：图片 `png`、`jpg`、`jpeg`、`gif`、`webp`、`svg`，音视频 `mp4`、`m4v`、`mov`、`webm`、`mp3`、`m4a`、`aac`、`ogg`、`wav`。 实时输出只能追加，Host 要求条目完成时的全文等于此前所有增量之和。`LocalMediaMarkdownStream` 因此只暂缓结尾仍可能被后续文本改变的写法（未闭合的图片、未闭合的行内代码），写错的图片或单个反引号不会扣住后面的文字；Item 结束前再补发暂缓部分。同一条消息内，每个路径只检查一次文件，保证前后改写一致。 diff --git a/packages/harness-adapter/src/local-media-markdown.ts b/packages/harness-adapter/src/local-media-markdown.ts index ed6a68d50..d036ccd76 100644 --- a/packages/harness-adapter/src/local-media-markdown.ts +++ b/packages/harness-adapter/src/local-media-markdown.ts @@ -1,4 +1,6 @@ -import { existsSync, statSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { copyFileSync, existsSync, linkSync, mkdirSync, rmSync, statSync } from "node:fs"; +import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -23,8 +25,13 @@ const MEDIA_EXTENSIONS = new Set([ /** Whether an absolute candidate path is a media file Codex Desktop can load. */ export type LocalMediaExists = (absolutePath: string) => boolean; +/** The path a Markdown destination should name for a resolved local media file. */ +export type LocalMediaLinkPath = (absolutePath: string) => string; + export interface RewriteLocalMediaMarkdownOptions { exists?: LocalMediaExists; + /** Defaults to {@link linkableLocalMediaPath}. */ + linkPath?: LocalMediaLinkPath; /** * Hold back a trailing construct that later text could still complete or invalidate * (an open image or inline code span), so a growing text projects prefix-stably. @@ -48,6 +55,7 @@ export function rewriteLocalMediaMarkdown( roots, options.exists ?? mediaFileExists, options.holdIncomplete === true, + options.linkPath ?? linkableLocalMediaPath, ).text; } @@ -59,14 +67,28 @@ export function rewriteLocalMediaMarkdown( export class LocalMediaMarkdownStream { readonly #roots: readonly string[]; readonly #exists: LocalMediaExists; + readonly #linkPath: LocalMediaLinkPath; #native = ""; #text = ""; /** Native text length represented by `#text`; the rest is held. */ #consumed = 0; #verbatim = false; - constructor(roots: readonly string[], exists: LocalMediaExists = mediaFileExists) { + constructor( + roots: readonly string[], + exists: LocalMediaExists = mediaFileExists, + linkPath: LocalMediaLinkPath = linkableLocalMediaPath, + ) { this.#roots = [...roots]; + const linked = new Map(); + this.#linkPath = (absolutePath) => { + let link = linked.get(absolutePath); + if (link === undefined) { + link = linkPath(absolutePath); + linked.set(absolutePath, link); + } + return link; + }; const known = new Map(); this.#exists = (absolutePath) => { let found = known.get(absolutePath); @@ -97,7 +119,7 @@ export class LocalMediaMarkdownStream { #advance(holdIncomplete: boolean): string { const next = this.#verbatim ? undefined - : projectLocalMedia(this.#native, this.#roots, this.#exists, holdIncomplete); + : projectLocalMedia(this.#native, this.#roots, this.#exists, holdIncomplete, this.#linkPath); let delta: string; if (next?.text.startsWith(this.#text)) { delta = next.text.slice(this.#text.length); @@ -124,6 +146,7 @@ function projectLocalMedia( roots: readonly string[], exists: LocalMediaExists, holdIncomplete: boolean, + linkPath: LocalMediaLinkPath, ): LocalMediaProjection { let output = ""; let index = 0; @@ -167,7 +190,7 @@ function projectLocalMedia( resolved === null ? text.slice(next, image.end) : text.slice(next, image.destinationStart) + - markdownDestination(resolved, image.bracketed) + + markdownDestination(linkPath(resolved), image.bracketed) + text.slice(image.destinationEnd, image.end); index = image.end; continue; @@ -183,7 +206,10 @@ function projectLocalMedia( return { text: output, consumed: next }; } const resolved = resolveLocalMediaPath(code.content, roots, exists); - output += resolved === null ? text.slice(next, code.end) : mediaMarkdownImage(resolved); + output += + resolved === null + ? text.slice(next, code.end) + : mediaMarkdownImage(resolved, linkPath(resolved)); index = code.end; } return { text: output + text.slice(index), consumed: text.length }; @@ -315,9 +341,58 @@ function skipBlanks(text: string, start: number): number { return index; } -function mediaMarkdownImage(absolutePath: string): string { +function mediaMarkdownImage(absolutePath: string, link: string): string { const name = path.basename(absolutePath).replaceAll("]", "\\]"); - return `![${name}](${markdownDestination(absolutePath, false)})`; + return `![${name}](${markdownDestination(link, false)})`; +} + +/** + * Codex Desktop percent-decodes a local media destination twice before reading the file, so no + * single escaping keeps a literal `%` intact; Grok's session directories are the cwd encoded + * with encodeURIComponent and always contain one. Such a file is hard-linked (or copied) to a + * `%`-free alias under `/media-links` and the alias is linked instead; if that + * fails, the path is escaped for exactly two decodes. Other paths are returned unchanged. + */ +export function linkableLocalMediaPath( + absolutePath: string, + aliasRoot: string = defaultMediaAliasRoot(), +): string { + if (!absolutePath.includes("%")) return absolutePath; + return mediaAlias(absolutePath, aliasRoot) ?? absolutePath.replaceAll("%", "%2525"); +} + +function defaultMediaAliasRoot(): string { + const dataDirectory = process.env.CODEXHOST_DATA_DIR; + return path.join( + dataDirectory ? path.resolve(dataDirectory) : path.join(os.homedir(), ".codexhost"), + "media-links", + ); +} + +function mediaAlias(absolutePath: string, aliasRoot: string): string | null { + try { + const digest = createHash("sha256").update(absolutePath).digest("hex").slice(0, 16); + const directory = path.join(aliasRoot, digest); + const alias = path.join(directory, path.basename(absolutePath).replaceAll("%", "_")); + if (alias.includes("%")) return null; + const source = statSync(absolutePath); + if (!source.isFile()) return null; + if (existsSync(alias)) { + const current = statSync(alias); + // A hard link shares the source's inode; a copy is current when it is at least as new. + if (current.size === source.size && current.mtimeMs >= source.mtimeMs) return alias; + rmSync(alias, { force: true }); + } + mkdirSync(directory, { recursive: true }); + try { + linkSync(absolutePath, alias); + } catch { + copyFileSync(absolutePath, alias); + } + return alias; + } catch { + return null; + } } function markdownDestination(absolutePath: string, bracketed: boolean): string { diff --git a/packages/harness-adapter/test/local-media-markdown.test.ts b/packages/harness-adapter/test/local-media-markdown.test.ts index 119f9cc4d..be7de0170 100644 --- a/packages/harness-adapter/test/local-media-markdown.test.ts +++ b/packages/harness-adapter/test/local-media-markdown.test.ts @@ -1,3 +1,5 @@ +import { mkdtemp, readFile, rm, stat, writeFile, mkdir } from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; @@ -5,6 +7,7 @@ import { describe, expect, it, vi } from "vitest"; import { LocalMediaMarkdownStream, + linkableLocalMediaPath, resolveLocalMediaPath, rewriteLocalMediaMarkdown, } from "../src/local-media-markdown.js"; @@ -150,3 +153,79 @@ describe("LocalMediaMarkdownStream", () => { expect(check).toHaveBeenCalledTimes(1); }); }); + +describe("links to media whose path holds a literal %", () => { + // Codex Desktop decodes a local media destination twice before it reads the file. + const desktopPath = (destination: string) => decodeURI(decodeURI(destination)); + + it("links a %-free hard link or copy of the file", async () => { + const temp = await mkdtemp(path.join(os.tmpdir(), "media-percent-")); + try { + // Grok names a session directory after its cwd, encoded with encodeURIComponent. + const session = path.join(temp, "sessions", "C%3A%5CUsers%5Cme%5C%E6%96%87"); + await mkdir(path.join(session, "images"), { recursive: true }); + const image = path.join(session, "images", "1%25.jpg"); + await writeFile(image, "jpeg bytes"); + const aliasRoot = path.join(temp, "media-links"); + const linkPath = (absolutePath: string) => linkableLocalMediaPath(absolutePath, aliasRoot); + + const output = rewriteLocalMediaMarkdown("![fish](images/1%25.jpg)", [session], { + linkPath, + }); + expect(output).toMatch(/^!\[fish\]\([^%<>]+\)$/); + const destination = output.slice("![fish](".length, -1); + const alias = desktopPath(destination); + expect(alias.startsWith(md(aliasRoot) + "/")).toBe(true); + expect(path.basename(alias)).toBe("1_25.jpg"); + expect(await readFile(alias, "utf8")).toBe("jpeg bytes"); + // The same file keeps the same alias. + expect(rewriteLocalMediaMarkdown("`images/1%25.jpg`", [session], { linkPath })).toBe( + `![1%25.jpg](${destination})`, + ); + } finally { + await rm(temp, { recursive: true, force: true }); + } + }); + + it("escapes % for both decodes when no alias can be made", async () => { + const temp = await mkdtemp(path.join(os.tmpdir(), "media-percent-")); + try { + const image = path.join(temp, "C%3A%5Cdir", "1.png"); + await mkdir(path.dirname(image)); + await writeFile(image, "png"); + // A file where the alias directory should be makes every alias fail. + const blocked = path.join(temp, "blocked"); + await writeFile(blocked, ""); + const destination = md(linkableLocalMediaPath(image, blocked)); + expect(destination).toContain("%2525"); + expect(path.resolve(desktopPath(destination))).toBe(image); + } finally { + await rm(temp, { recursive: true, force: true }); + } + }); + + it("leaves paths without % alone and creates no alias", async () => { + const temp = await mkdtemp(path.join(os.tmpdir(), "media-percent-")); + try { + const aliasRoot = path.join(temp, "media-links"); + const image = path.join(workspace, "shot.png"); + expect(linkableLocalMediaPath(image, aliasRoot)).toBe(image); + await expect(stat(aliasRoot)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await rm(temp, { recursive: true, force: true }); + } + }); + + it("asks the stream's link path once per file so published text stays stable", () => { + const linkPath = vi.fn((absolutePath: string) => absolutePath + ".alias"); + const stream = new LocalMediaMarkdownStream(roots, exists, linkPath); + let text = ""; + for (const piece of ["![a](sho", "t.png) and `shot.p", "ng` again ![b](shot.png)"]) + text += stream.append(piece); + text += stream.flush(); + expect(text).toBe( + `![a](${at("shot.png")}.alias) and ![shot.png](${at("shot.png")}.alias) again ![b](${at("shot.png")}.alias)`, + ); + expect(linkPath).toHaveBeenCalledTimes(1); + }); +}); From 220cb924808712cb363e3ff7fb056b68a4688ae6 Mon Sep 17 00:00:00 2001 From: BeiLuoqu Date: Sun, 25 Oct 2026 16:34:41 +0800 Subject: [PATCH 04/15] fix: put a media path written as inline code on its own line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grok usually ends a reply with the generated file as inline code, as in "…鱼：`images/1.jpg`。". The rewrite turned the code span into an image in place, so the image sat inside the sentence with the full stop dangling after it. Agents that write the Markdown image themselves put it on its own line. Give an image converted from inline code its own block: a blank line on each side, with sentence punctuation that directly follows moved before it in place of the colon that introduced it ("…鱼。", then the image). Punctuation stays after the image when its line has no text, and table rows and headings keep the image inline. Streaming waits for one more character to see the punctuation and holds a trailing colon or space, so published text stays prefix-stable for every split. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/external-thread-media.md | 1 + .../test/modern/history.test.ts | 2 +- .../grok/test/local-media-markdown.test.ts | 2 +- .../src/local-media-markdown.ts | 64 +++++++++++++++++-- .../test/local-media-markdown.test.ts | 28 +++++++- 5 files changed, 86 insertions(+), 11 deletions(-) diff --git a/docs/architecture/external-thread-media.md b/docs/architecture/external-thread-media.md index d0b940af4..4d5b9ecb2 100644 --- a/docs/architecture/external-thread-media.md +++ b/docs/architecture/external-thread-media.md @@ -17,6 +17,7 @@ Harness 常按工作目录写相对路径，因此需要由 Adapter 改写。 - 改写图片目标中的本地媒体路径，以及只包含一个媒体路径的行内代码（转成图片）。相对路径按 Adapter 给出的根目录依次查找，只有找到实际存在的文件才改写。 - 远程 URL、`data:` 等地址、找不到的文件、无法解析的写法和围栏代码块中的内容保持原样，不改动一个字节。 - Windows 盘符路径写成正斜杠（`C:/…`）。CommonMark 会把反斜杠加标点当作转义，原样输出会让 `C:\Users\me\.codex\…` 丢掉分隔符。 +- 行内代码转成的图片单独成段，与 Agent 自己写 Markdown 图片时的排版一致：前后各空一行，紧跟其后的句末标点（`。.!！?？;；,，`）移到图片之前，并替换引出图片的冒号，例如 `` …鱼：`images/1.jpg`。 `` 显示为“…鱼。”，下一段是图片。同一行没有文字时，标点留在图片后面。表格行和标题行无法放入独立段落，其中的图片保持行内。实时输出会多等一个字符，用来判断其后是否有标点，结尾的冒号和空格同样暂缓发出。 - 路径中含有 `%` 时另行处理。Desktop 读取文件前会对目标做两次 `decodeURI`，任何一次性的转义都无法保住字面 `%`；Grok 的会话目录名是用 `encodeURIComponent` 编码的工作目录，总会包含 `%`。改写时为这类文件在 `<数据目录>/media-links/<路径哈希>/` 下建一个不含 `%` 的硬链接（失败则复制），消息中链接这个别名；硬链接和复制都失败时，把 `%` 转义成 `%2525`，正好经受两次解码。不含 `%` 的路径不受影响，也不创建别名。别名目录可以随时删除，下次改写时重新生成。 - 支持的扩展名：图片 `png`、`jpg`、`jpeg`、`gif`、`webp`、`svg`，音视频 `mp4`、`m4v`、`mov`、`webm`、`mp3`、`m4a`、`aac`、`ogg`、`wav`。 diff --git a/packages/adapters/deepseek-harness/test/modern/history.test.ts b/packages/adapters/deepseek-harness/test/modern/history.test.ts index 4fb910849..b9cc9bf26 100644 --- a/packages/adapters/deepseek-harness/test/modern/history.test.ts +++ b/packages/adapters/deepseek-harness/test/modern/history.test.ts @@ -281,7 +281,7 @@ describe("DeepSeek Harness Modern history projection", () => { const b = path.join(shots, "b.png").replaceAll("\\", "/"); expect(agentTexts(workspace)).toEqual([ - `![a](${a}) ![b.png](${b}) ![b](b.png)`, + `![a](${a})\n\n![b.png](${b})\n\n![b](b.png)`, `![b](${b}) ![a](${a})`, ]); expect(agentTexts()).toEqual([ diff --git a/packages/adapters/grok/test/local-media-markdown.test.ts b/packages/adapters/grok/test/local-media-markdown.test.ts index 6f5cb332b..46b433e33 100644 --- a/packages/adapters/grok/test/local-media-markdown.test.ts +++ b/packages/adapters/grok/test/local-media-markdown.test.ts @@ -129,7 +129,7 @@ describe("Grok local media Markdown", () => { emitted = projected; } expect(emitted).toBe( - `影片：\n\n![usegrokbot-templates-promo-10s.mp4](${md(path.join(session, "videos/usegrokbot-templates-promo-10s.mp4"))})\n完成`, + `影片：\n\n![usegrokbot-templates-promo-10s.mp4](${md(path.join(session, "videos/usegrokbot-templates-promo-10s.mp4"))})\n\n完成`, ); }); diff --git a/packages/harness-adapter/src/local-media-markdown.ts b/packages/harness-adapter/src/local-media-markdown.ts index d036ccd76..49f614f4e 100644 --- a/packages/harness-adapter/src/local-media-markdown.ts +++ b/packages/harness-adapter/src/local-media-markdown.ts @@ -166,6 +166,7 @@ function projectLocalMedia( indexOfInlineTick(text, index), ]); if (next === -1) break; + const segmentStart = index; output += text.slice(index, next); index = next; if (text.startsWith("```", next)) { @@ -203,18 +204,69 @@ function projectLocalMedia( } if (code.kind === "open") { if (!holdIncomplete) break; - return { text: output, consumed: next }; + // The span may become media, which moves the colon introducing it. + const held = mediaIntroduction(text.slice(segmentStart, next)).length; + return { text: output.slice(0, output.length - held), consumed: next - held }; } const resolved = resolveLocalMediaPath(code.content, roots, exists); - output += - resolved === null - ? text.slice(next, code.end) - : mediaMarkdownImage(resolved, linkPath(resolved)); - index = code.end; + if (resolved === null) { + output += text.slice(next, code.end); + index = code.end; + continue; + } + const image = mediaMarkdownImage(resolved, linkPath(resolved)); + if (!canPlaceMediaBlock(output)) { + output += image; + index = code.end; + continue; + } + // A path the agent wrote inline becomes its own block, as when it writes the image itself: + // `…鱼：\`images/1.jpg\`。` reads `…鱼。` and then the image. + const punctuation = /^[。．.!！?？;；,，]*/u.exec(text.slice(code.end))?.[0] ?? ""; + const spacing = /^\s*/u.exec(text.slice(code.end + punctuation.length))?.[0] ?? ""; + const rest = code.end + punctuation.length + spacing.length; + const introduction = mediaIntroduction(text.slice(segmentStart, next)); + if (rest === text.length && holdIncomplete) { + // Later text may still add punctuation to move before the image. + return { + text: output.slice(0, output.length - introduction.length), + consumed: next - introduction.length, + }; + } + output = output.slice(0, output.length - introduction.length); + // Punctuation moves only onto a line that has text; otherwise it stays after the image. + const moved = /\S/u.test(output.slice(output.lastIndexOf("\n") + 1)) ? punctuation : ""; + output += moved || introduction.trim(); + if (output.length > 0 && !output.endsWith("\n\n")) + output += output.endsWith("\n") ? "\n" : "\n\n"; + output += image + (moved ? "" : punctuation); + if (rest < text.length) output += "\n\n"; + index = rest; + } + if (holdIncomplete && index < text.length) { + // A trailing colon or space may lead into a media path that has not arrived yet. + const held = mediaIntroduction(text.slice(index)).length; + if (held > 0) { + return { + text: output + text.slice(index, text.length - held), + consumed: text.length - held, + }; + } } return { text: output + text.slice(index), consumed: text.length }; } +/** Trailing spacing, optionally after a colon, that leads into an inline media path. */ +function mediaIntroduction(text: string): string { + return /[：:]?[ \t]*$/u.exec(text)?.[0] ?? ""; +} + +/** Table rows and headings cannot hold a block, so media written there stays inline. */ +function canPlaceMediaBlock(output: string): boolean { + const line = output.slice(output.lastIndexOf("\n") + 1); + return !/^\s*[|#]/u.test(line); +} + function indexOfInlineTick(text: string, start: number): number { let index = start; while (index < text.length) { diff --git a/packages/harness-adapter/test/local-media-markdown.test.ts b/packages/harness-adapter/test/local-media-markdown.test.ts index be7de0170..1e9b05567 100644 --- a/packages/harness-adapter/test/local-media-markdown.test.ts +++ b/packages/harness-adapter/test/local-media-markdown.test.ts @@ -54,7 +54,7 @@ describe("rewriteLocalMediaMarkdown", () => { it("resolves file URLs and absolute paths", () => { const absolute = path.join(workspace, "a.png"); expect(rewrite(`![a](${pathToFileURL(absolute).href}) and \`${absolute}\``)).toBe( - `![a](${at("a.png")}) and ![a.png](${at("a.png")})`, + `![a](${at("a.png")}) and\n\n![a.png](${at("a.png")})`, ); }); @@ -75,9 +75,27 @@ describe("rewriteLocalMediaMarkdown", () => { ); }); + it("puts a media path written as inline code on its own line, punctuation first", () => { + const a = at("a.png"); + // Grok's usual reply: the image follows the sentence instead of splitting it. + expect(rewrite("画好了，是一只大肥鱼：`a.png`。")).toBe( + `画好了，是一只大肥鱼。\n\n![a.png](${a})`, + ); + expect(rewrite("看图：`a.png`\n然后继续")).toBe(`看图：\n\n![a.png](${a})\n\n然后继续`); + expect(rewrite("see `a.png`. Next")).toBe(`see.\n\n![a.png](${a})\n\nNext`); + expect(rewrite("前文\n\n`a.png`")).toBe(`前文\n\n![a.png](${a})`); + // Nothing on the line to move punctuation onto. + expect(rewrite("`a.png`。")).toBe(`![a.png](${a})。`); + // A table row or heading cannot hold a block. + expect(rewrite("| 图 | `a.png` |")).toBe(`| 图 | ![a.png](${a}) |`); + expect(rewrite("# `a.png`")).toBe(`# ![a.png](${a})`); + }); + it("holds only a trailing construct that later text could still change", () => { + expect(rewrite("大肥鱼：`a.png`", true)).toBe("大肥鱼"); + expect(rewrite("大肥鱼：", true)).toBe("大肥鱼"); expect(rewrite("see ![a](a.p", true)).toBe("see ![a]("); - expect(rewrite("see `a.p", true)).toBe("see "); + expect(rewrite("see `a.p", true)).toBe("see"); expect(rewrite("see ![a", true)).toBe("see !["); expect(rewrite("![a](a.png bad) then ` stray\nmore", true)).toBe( "![a](a.png bad) then ` stray\nmore", @@ -104,6 +122,10 @@ describe("LocalMediaMarkdownStream", () => { "a!![b](sub/b.png)!", "``a.png`` and `a.png`` and ```", "![e](截图/e.png)。", + "画好了，是一只大肥鱼：`a.png`。后面", + "see `a.png`. Next `x.png`", + "| 图 | `a.png` |\n`sub/b.png`：", + "前文\n\n`a.png`", ]; function streamed(text: string, cuts: readonly number[]): { deltas: string[]; text: string } { @@ -224,7 +246,7 @@ describe("links to media whose path holds a literal %", () => { text += stream.append(piece); text += stream.flush(); expect(text).toBe( - `![a](${at("shot.png")}.alias) and ![shot.png](${at("shot.png")}.alias) again ![b](${at("shot.png")}.alias)`, + `![a](${at("shot.png")}.alias) and\n\n![shot.png](${at("shot.png")}.alias)\n\nagain ![b](${at("shot.png")}.alias)`, ); expect(linkPath).toHaveBeenCalledTimes(1); }); From f7fc29ac9e0666a11ac64a0930c9f2a0191361b0 Mon Sep 17 00:00:00 2001 From: BeiLuoqu Date: Sun, 25 Oct 2026 16:58:43 +0800 Subject: [PATCH 05/15] fix: show a link to local media as an image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grok sometimes links the file it generated instead of embedding it, as in "[Claude 御姐](images/1.jpg)". Desktop renders such a link as a file chip, so the picture never appeared. Treat a plain link whose destination resolves to local media like inline code naming it: show it as an image, with the link text as its alt text, placed as its own block. Links to web pages, other files, or missing files stay byte-for-byte, and so does a link whose text holds brackets, such as a badge wrapping an image. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/external-thread-media.md | 3 +- .../src/local-media-markdown.ts | 59 +++++++++++++------ .../test/local-media-markdown.test.ts | 19 ++++++ 3 files changed, 62 insertions(+), 19 deletions(-) diff --git a/docs/architecture/external-thread-media.md b/docs/architecture/external-thread-media.md index 4d5b9ecb2..2e7a53b7b 100644 --- a/docs/architecture/external-thread-media.md +++ b/docs/architecture/external-thread-media.md @@ -17,7 +17,8 @@ Harness 常按工作目录写相对路径，因此需要由 Adapter 改写。 - 改写图片目标中的本地媒体路径，以及只包含一个媒体路径的行内代码（转成图片）。相对路径按 Adapter 给出的根目录依次查找，只有找到实际存在的文件才改写。 - 远程 URL、`data:` 等地址、找不到的文件、无法解析的写法和围栏代码块中的内容保持原样，不改动一个字节。 - Windows 盘符路径写成正斜杠（`C:/…`）。CommonMark 会把反斜杠加标点当作转义，原样输出会让 `C:\Users\me\.codex\…` 丢掉分隔符。 -- 行内代码转成的图片单独成段，与 Agent 自己写 Markdown 图片时的排版一致：前后各空一行，紧跟其后的句末标点（`。.!！?？;；,，`）移到图片之前，并替换引出图片的冒号，例如 `` …鱼：`images/1.jpg`。 `` 显示为“…鱼。”，下一段是图片。同一行没有文字时，标点留在图片后面。表格行和标题行无法放入独立段落，其中的图片保持行内。实时输出会多等一个字符，用来判断其后是否有标点，结尾的冒号和空格同样暂缓发出。 +- 目标是本地媒体文件的普通链接（`[文字](images/1.jpg)`，没有 `!`）同样转成图片，链接文字作为图片说明；Desktop 对这种链接只显示文件标签，不显示图片。指向网页、其他文件或找不到的文件的链接保持原样；链接文字里嵌有方括号（如套着图片的徽章链接）时也不改。 +- 行内代码或链接转成的图片单独成段，与 Agent 自己写 Markdown 图片时的排版一致：前后各空一行，紧跟其后的句末标点（`。.!！?？;；,，`）移到图片之前，并替换引出图片的冒号，例如 `` …鱼：`images/1.jpg`。 `` 显示为“…鱼。”，下一段是图片。同一行没有文字时，标点留在图片后面。表格行和标题行无法放入独立段落，其中的图片保持行内。实时输出会多等一个字符，用来判断其后是否有标点，结尾的冒号和空格同样暂缓发出。 - 路径中含有 `%` 时另行处理。Desktop 读取文件前会对目标做两次 `decodeURI`，任何一次性的转义都无法保住字面 `%`；Grok 的会话目录名是用 `encodeURIComponent` 编码的工作目录，总会包含 `%`。改写时为这类文件在 `<数据目录>/media-links/<路径哈希>/` 下建一个不含 `%` 的硬链接（失败则复制），消息中链接这个别名；硬链接和复制都失败时，把 `%` 转义成 `%2525`，正好经受两次解码。不含 `%` 的路径不受影响，也不创建别名。别名目录可以随时删除，下次改写时重新生成。 - 支持的扩展名：图片 `png`、`jpg`、`jpeg`、`gif`、`webp`、`svg`，音视频 `mp4`、`m4v`、`mov`、`webm`、`mp3`、`m4a`、`aac`、`ogg`、`wav`。 diff --git a/packages/harness-adapter/src/local-media-markdown.ts b/packages/harness-adapter/src/local-media-markdown.ts index 49f614f4e..052fe79b1 100644 --- a/packages/harness-adapter/src/local-media-markdown.ts +++ b/packages/harness-adapter/src/local-media-markdown.ts @@ -163,6 +163,7 @@ function projectLocalMedia( const next = earliestIndex([ text.indexOf("```", index), text.indexOf("![", index), + text.indexOf("[", index), indexOfInlineTick(text, index), ]); if (next === -1) break; @@ -196,35 +197,50 @@ function projectLocalMedia( index = image.end; continue; } - const code = parseInlineCode(text, next); - if (code.kind === "text") { - output += "`"; - index = next + 1; + // A link or inline code naming local media becomes an image; anything else stays as is. + const construct = + text[next] === "[" + ? parseMarkdownImage(text, next, 1) + : (() => { + const code = parseInlineCode(text, next); + return code.kind === "code" + ? { kind: "code" as const, destination: code.content, end: code.end } + : code.kind === "text" + ? { kind: "text" as const, end: next + 1 } + : code; + })(); + if (construct.kind === "text") { + output += text.slice(next, construct.end); + index = construct.end; continue; } - if (code.kind === "open") { + if (construct.kind === "open") { if (!holdIncomplete) break; - // The span may become media, which moves the colon introducing it. + // The construct may become media, which moves the colon introducing it. const held = mediaIntroduction(text.slice(segmentStart, next)).length; return { text: output.slice(0, output.length - held), consumed: next - held }; } - const resolved = resolveLocalMediaPath(code.content, roots, exists); + const resolved = resolveLocalMediaPath(construct.destination, roots, exists); if (resolved === null) { - output += text.slice(next, code.end); - index = code.end; + output += text.slice(next, construct.end); + index = construct.end; continue; } - const image = mediaMarkdownImage(resolved, linkPath(resolved)); + const image = + construct.kind === "image" + ? `![${construct.label}](${markdownDestination(linkPath(resolved), construct.bracketed)})` + : mediaMarkdownImage(resolved, linkPath(resolved)); + const end = construct.end; if (!canPlaceMediaBlock(output)) { output += image; - index = code.end; + index = end; continue; } - // A path the agent wrote inline becomes its own block, as when it writes the image itself: + // Media the agent named inline becomes its own block, as when it writes the image itself: // `…鱼：\`images/1.jpg\`。` reads `…鱼。` and then the image. - const punctuation = /^[。．.!！?？;；,，]*/u.exec(text.slice(code.end))?.[0] ?? ""; - const spacing = /^\s*/u.exec(text.slice(code.end + punctuation.length))?.[0] ?? ""; - const rest = code.end + punctuation.length + spacing.length; + const punctuation = /^[。．.!！?？;；,，]*/u.exec(text.slice(end))?.[0] ?? ""; + const spacing = /^\s*/u.exec(text.slice(end + punctuation.length))?.[0] ?? ""; + const rest = end + punctuation.length + spacing.length; const introduction = mediaIntroduction(text.slice(segmentStart, next)); if (rest === text.length && holdIncomplete) { // Later text may still add punctuation to move before the image. @@ -309,6 +325,8 @@ function parseInlineCode(text: string, start: number): ParsedInlineCode { type ParsedMarkdownImage = | { kind: "image"; + /** Raw label text, without its brackets. */ + label: string; destination: string; destinationStart: number; destinationEnd: number; @@ -319,10 +337,11 @@ type ParsedMarkdownImage = | { kind: "open"; holdFrom: number } | { kind: "text"; end: number }; -function parseMarkdownImage(text: string, start: number): ParsedMarkdownImage { +/** Parses `![label](destination)`, or with `prefix` 1 a link `[label](destination)`. */ +function parseMarkdownImage(text: string, start: number, prefix = 2): ParsedMarkdownImage { // Until the label closes, only `![` is settled: `!` was publishable before `[` arrived. - const openLabel = { kind: "open", holdFrom: start + 2 } as const; - let index = start + 2; + const openLabel = { kind: "open", holdFrom: start + prefix } as const; + let index = start + prefix; while (index < text.length) { const character = text[index]; if (character === "\n") return { kind: "text", end: start + 1 }; @@ -331,8 +350,11 @@ function parseMarkdownImage(text: string, start: number): ParsedMarkdownImage { continue; } if (character === "]") break; + // A link around an image or another link is not a plain reference to media. + if (prefix === 1 && character === "[") return { kind: "text", end: start + 1 }; index += 1; } + const labelEnd = index; if (index + 1 >= text.length) return openLabel; if (text[index + 1] !== "(") return { kind: "text", end: start + 1 }; const destinationStart = index + 2; @@ -379,6 +401,7 @@ function parseMarkdownImage(text: string, start: number): ParsedMarkdownImage { if (text[index] !== ")") return malformed; return { kind: "image", + label: text.slice(start + prefix, labelEnd), destination, destinationStart, destinationEnd, diff --git a/packages/harness-adapter/test/local-media-markdown.test.ts b/packages/harness-adapter/test/local-media-markdown.test.ts index 1e9b05567..41f49d4c9 100644 --- a/packages/harness-adapter/test/local-media-markdown.test.ts +++ b/packages/harness-adapter/test/local-media-markdown.test.ts @@ -91,7 +91,24 @@ describe("rewriteLocalMediaMarkdown", () => { expect(rewrite("# `a.png`")).toBe(`# ![a.png](${a})`); }); + it("shows a link to local media as an image and leaves other links alone", () => { + const a = at("a.png"); + // Grok sometimes links the generated file instead of embedding it. + expect(rewrite("画好了：\n\n[Claude 御姐](a.png)\n\n想改再说。")).toBe( + `画好了：\n\n![Claude 御姐](${a})\n\n想改再说。`, + ); + expect(rewrite("见 [图](sub/b.png)。")).toBe(`见。\n\n![图](${at("sub/b.png")})`); + const unchanged = [ + "[site](https://example.com/a.png)", + "[notes](notes.txt) and [gone](missing.png)", + "- [ ] task\n- [x] done [1]", + `[![badge](${a})](a.png)`, + ].join("\n"); + expect(rewrite(unchanged)).toBe(unchanged); + }); + it("holds only a trailing construct that later text could still change", () => { + expect(rewrite("见 [图](sub/b", true)).toBe("见"); expect(rewrite("大肥鱼：`a.png`", true)).toBe("大肥鱼"); expect(rewrite("大肥鱼：", true)).toBe("大肥鱼"); expect(rewrite("see ![a](a.p", true)).toBe("see ![a]("); @@ -126,6 +143,8 @@ describe("LocalMediaMarkdownStream", () => { "see `a.png`. Next `x.png`", "| 图 | `a.png` |\n`sub/b.png`：", "前文\n\n`a.png`", + "画好了：\n\n[Claude 御姐](a.png)\n\n后面", + "[x](a.png) and [y](notes.txt) - [ ] t [z](https://e/a.png)", ]; function streamed(text: string, cuts: readonly number[]): { deltas: string[]; text: string } { From f29a5cd49543e41e7cc9fc297ba57d6c6a6c82f2 Mon Sep 17 00:00:00 2001 From: BeiLuoqu Date: Sun, 25 Oct 2026 17:40:53 +0800 Subject: [PATCH 06/15] fix(deepseek-harness): reload a Thread with the media live output showed Live output rewrites local media even when the journal header has no cwd: roots are then empty, yet absolute paths still get forward slashes, %-free aliases and their own block. History rewrote only when the header gave a cwd, so a reloaded message could fall back to text that Desktop cannot show. Add a media flag to projectModernHistory so readSnapshot always rewrites as live output does, with the same roots; it defaults to on when a cwd is given. modernMediaRoots now always returns the roots it has. The Usage projection still asks for no media. Co-Authored-By: Claude Opus 5.5 --- .../deepseek-harness/src/modern/history.ts | 21 ++++++++++----- .../deepseek-harness/src/modern/session.ts | 9 +++++-- .../test/modern/history.test.ts | 26 +++++++++++++++++++ 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/packages/adapters/deepseek-harness/src/modern/history.ts b/packages/adapters/deepseek-harness/src/modern/history.ts index 96ccbd177..268d5c026 100644 --- a/packages/adapters/deepseek-harness/src/modern/history.ts +++ b/packages/adapters/deepseek-harness/src/modern/history.ts @@ -784,10 +784,15 @@ export interface ProjectModernHistoryInput { readonly maxEvents?: number; readonly profile?: DeepSeekModernProfile; /** - * Session working directory from the journal header. When present, local media that agent - * messages reference by relative path becomes absolute, so Codex Desktop can display it. + * Session working directory from the journal header. Relative media paths in agent messages + * resolve against it when {@link media} is on. */ readonly cwd?: string; + /** + * Rewrite local media in agent messages exactly as live output does, so a reloaded Thread + * shows the same images. Defaults to on when `cwd` is given. + */ + readonly media?: boolean; } /** Strictly validate and project one complete authoritative Modern journal prefix. */ @@ -808,6 +813,7 @@ export function projectModernHistory(input: ProjectModernHistoryInput): ModernHi let contextWindowTokens: number | undefined; let usage: HostUsage | null = null; let workingDirectory = input.cwd; + const media = input.media ?? input.cwd !== undefined; const rawUsageByStep = new Map(); const rebuildUsage = (): void => { @@ -881,7 +887,7 @@ export function projectModernHistory(input: ProjectModernHistoryInput): ModernHi active, input.sessionId, data, - modernMediaRoots(workingDirectory, input.cwd), + media ? modernMediaRoots(workingDirectory, input.cwd) : undefined, ); } } @@ -1039,9 +1045,12 @@ export function modernWorkingDirectory( export function modernMediaRoots( workingDirectory: string | undefined, sessionDirectory: string | undefined, -): string[] | undefined { - if (sessionDirectory === undefined) return undefined; - return [...new Set([workingDirectory ?? sessionDirectory, sessionDirectory])]; +): string[] { + return [ + ...new Set( + [workingDirectory ?? sessionDirectory, sessionDirectory].filter((root) => root !== undefined), + ), + ]; } function projectAssistantMessage( diff --git a/packages/adapters/deepseek-harness/src/modern/session.ts b/packages/adapters/deepseek-harness/src/modern/session.ts index 41a09b3e9..f4e040ef0 100644 --- a/packages/adapters/deepseek-harness/src/modern/session.ts +++ b/packages/adapters/deepseek-harness/src/modern/session.ts @@ -2452,7 +2452,7 @@ export class ModernHarnessSession implements HarnessSession, ModernEventSink { ), text: "", }; - const roots = modernMediaRoots(this.#workingDirectory, this.#sessionDirectory) ?? []; + const roots = modernMediaRoots(this.#workingDirectory, this.#sessionDirectory); active.agent = { item, text: "", media: new LocalMediaMarkdownStream(roots) }; this.#emit({ type: "item.started", turnId: active.turnId, item }); } @@ -2802,7 +2802,12 @@ export class ModernHarnessSession implements HarnessSession, ModernEventSink { sessionId: this.#sessionId, events: this.#events, profile: this.#profile, - ...(media && this.#sessionDirectory !== undefined ? { cwd: this.#sessionDirectory } : {}), + ...(media + ? { + media: true, + ...(this.#sessionDirectory !== undefined ? { cwd: this.#sessionDirectory } : {}), + } + : {}), ...(this.#fallbackModel ? { fallbackModel: this.#fallbackModel } : {}), ...(this.#fallbackThinkingOptionId ? { fallbackThinkingOptionId: this.#fallbackThinkingOptionId } diff --git a/packages/adapters/deepseek-harness/test/modern/history.test.ts b/packages/adapters/deepseek-harness/test/modern/history.test.ts index b9cc9bf26..79ac07a5d 100644 --- a/packages/adapters/deepseek-harness/test/modern/history.test.ts +++ b/packages/adapters/deepseek-harness/test/modern/history.test.ts @@ -4,8 +4,10 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; +import { LocalMediaMarkdownStream } from "@codexhost/harness-adapter/local-media-markdown"; import { matchesModernForkHistory, + modernMediaRoots, ModernEventValidator, ModernHistoryError, resolveModernForkBoundary, @@ -288,6 +290,30 @@ describe("DeepSeek Harness Modern history projection", () => { "![a](shots/a.png) `shots/b.png` ![b](b.png)", "![b](b.png) ![a](shots/a.png)", ]); + + // A journal without a header cwd still reloads with the images live output showed. + const absolute = path.join(shots, "a.png"); + const live = new LocalMediaMarkdownStream(modernMediaRoots(undefined, undefined)); + const liveText = live.append(`see \`${absolute}\``) + live.flush(); + const reloaded = (media?: boolean) => + projectModernHistory({ + sessionId: SESSION_ID, + events: [ + event(0, "turn/start", { turn: 1 }), + event(1, "step/start", { turn: 1, step: 1 }), + userMessage(2, ["show"]), + assistantMessage(3, 1, 1, `see \`${absolute}\``, "think"), + event(4, "step/end", { turn: 1, step: 1 }), + event(5, "turn/end", { turn: 1, reason: { kind: "completed" } }), + ], + ...(media === undefined ? {} : { media }), + }).snapshot.turns.flatMap(({ items }) => + items.flatMap(({ item }) => (item.type === "agentMessage" ? [item.text] : [])), + ); + expect(liveText).toBe(`see\n\n![a.png](${a})`); + expect(reloaded(true)).toEqual([liveText]); + // Usage projections do not ask for media and keep the native text. + expect(reloaded()).toEqual([`see \`${absolute}\``]); } finally { await rm(workspace, { recursive: true, force: true }); } From 8c93119371384654aba01f1a992d8d52333c5275 Mon Sep 17 00:00:00 2001 From: BeiLuoqu Date: Sun, 25 Oct 2026 17:41:19 +0800 Subject: [PATCH 07/15] fix: recognize fenced code blocks as CommonMark does Any ``` opened a fence, wherever it sat on a line, and the next ``` anywhere closed it; ~~~ fences were not recognized. Media inside a ```` or ~~~ block could be rewritten, changing the code the user sees, and a literal ``` in a sentence stopped every later rewrite. Open a fence only with three or more backticks or tildes indented at most three spaces, and close it only with a line of the same marker, at least as long, followed by nothing but whitespace. A run still growing at the end of streamed text is held, so published text stays stable. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/external-thread-media.md | 2 +- .../src/local-media-markdown.ts | 81 +++++++++++++------ .../test/local-media-markdown.test.ts | 22 +++++ 3 files changed, 81 insertions(+), 24 deletions(-) diff --git a/docs/architecture/external-thread-media.md b/docs/architecture/external-thread-media.md index 2e7a53b7b..82c07a0d3 100644 --- a/docs/architecture/external-thread-media.md +++ b/docs/architecture/external-thread-media.md @@ -15,7 +15,7 @@ Harness 常按工作目录写相对路径，因此需要由 Adapter 改写。 `@codexhost/harness-adapter/local-media-markdown` 提供共用的改写： - 改写图片目标中的本地媒体路径，以及只包含一个媒体路径的行内代码（转成图片）。相对路径按 Adapter 给出的根目录依次查找，只有找到实际存在的文件才改写。 -- 远程 URL、`data:` 等地址、找不到的文件、无法解析的写法和围栏代码块中的内容保持原样，不改动一个字节。 +- 远程 URL、`data:` 等地址、找不到的文件、无法解析的写法和围栏代码块中的内容保持原样，不改动一个字节。围栏按 CommonMark 识别：开头是行首（最多缩进三个空格）三个或更多的 `` ` `` 或 `~`；结尾行用同一种字符，长度不短于开头，后面只能有空白。行中间出现的 ```` ``` ```` 不算围栏。 - Windows 盘符路径写成正斜杠（`C:/…`）。CommonMark 会把反斜杠加标点当作转义，原样输出会让 `C:\Users\me\.codex\…` 丢掉分隔符。 - 目标是本地媒体文件的普通链接（`[文字](images/1.jpg)`，没有 `!`）同样转成图片，链接文字作为图片说明；Desktop 对这种链接只显示文件标签，不显示图片。指向网页、其他文件或找不到的文件的链接保持原样；链接文字里嵌有方括号（如套着图片的徽章链接）时也不改。 - 行内代码或链接转成的图片单独成段，与 Agent 自己写 Markdown 图片时的排版一致：前后各空一行，紧跟其后的句末标点（`。.!！?？;；,，`）移到图片之前，并替换引出图片的冒号，例如 `` …鱼：`images/1.jpg`。 `` 显示为“…鱼。”，下一段是图片。同一行没有文字时，标点留在图片后面。表格行和标题行无法放入独立段落，其中的图片保持行内。实时输出会多等一个字符，用来判断其后是否有标点，结尾的冒号和空格同样暂缓发出。 diff --git a/packages/harness-adapter/src/local-media-markdown.ts b/packages/harness-adapter/src/local-media-markdown.ts index 052fe79b1..0f80f14de 100644 --- a/packages/harness-adapter/src/local-media-markdown.ts +++ b/packages/harness-adapter/src/local-media-markdown.ts @@ -150,31 +150,48 @@ function projectLocalMedia( ): LocalMediaProjection { let output = ""; let index = 0; - let fence = false; + let fence: { readonly marker: string; readonly length: number } | null = null; while (index < text.length) { if (fence) { - const fenceEnd = text.indexOf("```", index); - if (fenceEnd === -1) break; - output += text.slice(index, fenceEnd + 3); - index = fenceEnd + 3; - fence = false; + // Fenced code is copied verbatim through its closing fence line. + const close = closingFenceEnd(text, index, fence.marker, fence.length); + if (close === -1) break; + output += text.slice(index, close); + index = close; + fence = null; continue; } const next = earliestIndex([ - text.indexOf("```", index), text.indexOf("![", index), text.indexOf("[", index), - indexOfInlineTick(text, index), + text.indexOf("`", index), + indexOfLineStartTilde(text, index), ]); if (next === -1) break; const segmentStart = index; output += text.slice(index, next); index = next; - if (text.startsWith("```", next)) { - output += "```"; - index += 3; - fence = true; - continue; + const marker = text[next]; + if (marker === "`" || marker === "~") { + let runEnd = next; + while (text[runEnd] === marker) runEnd += 1; + if (runEnd === text.length && holdIncomplete) { + // The run may still grow into a fence, or a backtick may open a media path. + const held = mediaIntroduction(text.slice(segmentStart, next)).length; + return { text: output.slice(0, output.length - held), consumed: next - held }; + } + // CommonMark: a fence is three or more of one marker, indented at most three spaces. + if (runEnd - next >= 3 && isLineStart(text, next)) { + fence = { marker, length: runEnd - next }; + output += text.slice(next, runEnd); + index = runEnd; + continue; + } + if (marker === "~" || runEnd - next > 1) { + output += text.slice(next, runEnd); + index = runEnd; + continue; + } } if (text.startsWith("![", next)) { const image = parseMarkdownImage(text, next); @@ -283,16 +300,34 @@ function canPlaceMediaBlock(output: string): boolean { return !/^\s*[|#]/u.test(line); } -function indexOfInlineTick(text: string, start: number): number { - let index = start; - while (index < text.length) { - const tick = text.indexOf("`", index); - if (tick === -1) return -1; - if (text.startsWith("```", tick)) { - index = tick + 3; - continue; - } - return tick; +/** Whether only up to three spaces precede `index` on its line. */ +function isLineStart(text: string, index: number): boolean { + const lineStart = text.lastIndexOf("\n", index - 1) + 1; + return index - lineStart <= 3 && /^ *$/u.test(text.slice(lineStart, index)); +} + +/** A `~` that could open a fence; elsewhere tildes are ordinary text. */ +function indexOfLineStartTilde(text: string, start: number): number { + for (let tilde = text.indexOf("~", start); tilde !== -1; tilde = text.indexOf("~", tilde + 1)) { + if (isLineStart(text, tilde)) return tilde; + } + return -1; +} + +/** + * End of the line that closes a fence opened before `start`: the same marker, at least as long, + * indented at most three spaces and followed only by spaces or tabs. -1 while still inside. + */ +function closingFenceEnd(text: string, start: number, marker: string, length: number): number { + const closing = new RegExp(`^ {0,3}${marker === "`" ? "`" : "~"}{${length},}[ \\t]*$`, "u"); + // The opening line's info string is never a closing fence. + let lineStart = text.indexOf("\n", start); + while (lineStart !== -1) { + lineStart += 1; + const newline = text.indexOf("\n", lineStart); + const lineEnd = newline === -1 ? text.length : newline; + if (closing.test(text.slice(lineStart, lineEnd))) return lineEnd; + lineStart = newline; } return -1; } diff --git a/packages/harness-adapter/test/local-media-markdown.test.ts b/packages/harness-adapter/test/local-media-markdown.test.ts index 41f49d4c9..48b0b130a 100644 --- a/packages/harness-adapter/test/local-media-markdown.test.ts +++ b/packages/harness-adapter/test/local-media-markdown.test.ts @@ -107,6 +107,25 @@ describe("rewriteLocalMediaMarkdown", () => { expect(rewrite(unchanged)).toBe(unchanged); }); + it("follows CommonMark fences: line-start openers and matching closers only", () => { + const a = at("a.png"); + const fenced = [ + "````md\n```\n![a](a.png)\n```\n`a.png`\n````", + "~~~\n[x](a.png) `a.png`\n~~~", + " ```\n`a.png`\n ``` \t", + // Not closed: a shorter run, the other marker, or text after the run. + "```\n`a.png`\n``\n~~~\n``` x\n`a.png`", + ]; + for (const text of fenced) expect(rewrite(text), text).toBe(text); + // A fence ends at a matching line, and media after it is rewritten again. + expect(rewrite("~~~~\n`a.png`\n~~~~~\nafter `a.png`")).toBe( + `~~~~\n\`a.png\`\n~~~~~\nafter\n\n![a.png](${a})`, + ); + // Backtick runs inside a line are not fences, so later media still resolves. + expect(rewrite("use ``` here, then ![a](a.png)")).toBe(`use \`\`\` here, then ![a](${a})`); + expect(rewrite(" ```\n`a.png`")).toBe(` \`\`\`\n\n![a.png](${a})`); + }); + it("holds only a trailing construct that later text could still change", () => { expect(rewrite("见 [图](sub/b", true)).toBe("见"); expect(rewrite("大肥鱼：`a.png`", true)).toBe("大肥鱼"); @@ -145,6 +164,9 @@ describe("LocalMediaMarkdownStream", () => { "前文\n\n`a.png`", "画好了：\n\n[Claude 御姐](a.png)\n\n后面", "[x](a.png) and [y](notes.txt) - [ ] t [z](https://e/a.png)", + "````\n```\n`a.png`\n```\n````\nthen `a.png`", + "~~~\n![a](a.png)\n~~\n~~~ \nafter ![a](a.png)", + "use ``` here `a.png`", ]; function streamed(text: string, cuts: readonly number[]): { deltas: string[]; text: string } { From 440249137f4f3090760ef950bddeaf159eb9c435 Mon Sep 17 00:00:00 2001 From: BeiLuoqu Date: Sun, 25 Oct 2026 17:41:45 +0800 Subject: [PATCH 08/15] fix: read attachments only from a complete Desktop attachment context desktopImageAttachmentPaths accepted the attachment heading anywhere before the first "## My request:", or with no request heading at all, so a user who pasted text shaped like a context saw the paths they typed projected as localImage items. Read a context only when it opens the prompt, after blank lines, and is closed by a line that is exactly the request heading. The Renderer writes both headings when it adds remote attachments, so it is unaffected. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/external-thread-media.md | 2 +- .../src/desktop-attachments.ts | 21 ++++++++++++------- .../test/desktop-attachments.test.ts | 10 +++++++++ 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/docs/architecture/external-thread-media.md b/docs/architecture/external-thread-media.md index 82c07a0d3..53f2e09f3 100644 --- a/docs/architecture/external-thread-media.md +++ b/docs/architecture/external-thread-media.md @@ -41,6 +41,6 @@ Desktop 发送带图片的消息时，请求里既有 `localImage` 条目，也 边界： -- 只认附件清单中的绝对路径；用户在请求正文里写的路径不算附件。 +- 只认附件清单中的绝对路径；用户在请求正文里写的路径不算附件。附件清单必须位于消息开头（前面只允许空行），并以单独成行的 `## My request:` 结束；缺少结尾标题或前面还有其他文字时，视为用户自己输入或粘贴的文字，不读取附件。 - 不读取、不复制图片。剪贴板截图通常保存在系统临时目录，文件被清理后缩略图无法加载。 - 远程外部会话的图片路径位于远程主机，其缩略图显示尚未单独验证。 diff --git a/packages/shared-contracts/src/desktop-attachments.ts b/packages/shared-contracts/src/desktop-attachments.ts index 9beef7327..8c64ee567 100644 --- a/packages/shared-contracts/src/desktop-attachments.ts +++ b/packages/shared-contracts/src/desktop-attachments.ts @@ -14,18 +14,25 @@ * A Harness receives a turn's images only as these paths, so the Host reads them back from the * same text. Paths the user typed into the request are never attachments. */ -const ATTACHED_FILES_HEADING = "# Files mentioned by the user:"; -const REQUEST_HEADING = "## My request:"; +/** The context opens the prompt, after blank lines only. */ +const ATTACHED_FILES_HEADING = /^\s*# Files mentioned by the user:[ \t]*(?:\r?\n|$)/u; +/** The first line that is exactly the request heading ends the context. */ +const REQUEST_HEADING = /(?:^|\n)## My request:[ \t]*(?:\r?\n|$)/u; const IMAGE_ATTACHMENT_PATTERN = /(?:^|\n)## [^\r\n]+: ((?:\/|[A-Za-z]:[\\/]|\\\\)[^\r\n]+)\r?\nImage attachment: true(?=\r?\n|$)/gu; const CONTROL_CHARACTER_PATTERN = /[\u0000-\u001f\u007f]/u; -/** Absolute image paths in the attachment context Desktop serialized into a prompt. */ +/** + * Absolute image paths in the attachment context Desktop serialized into a prompt. Only a + * complete context counts: it opens the prompt and ends at the request heading, so text the + * user typed or pasted that merely looks like one is never read as attachments. + */ export function desktopImageAttachmentPaths(text: string): string[] { - const context = text.split(REQUEST_HEADING)[0] ?? ""; - const start = context.indexOf(ATTACHED_FILES_HEADING); - if (start < 0) return []; - return [...context.slice(start).matchAll(IMAGE_ATTACHMENT_PATTERN)] + const end = text.search(REQUEST_HEADING); + if (end < 0) return []; + const context = text.slice(0, end); + if (!ATTACHED_FILES_HEADING.test(context)) return []; + return [...context.matchAll(IMAGE_ATTACHMENT_PATTERN)] .map((match) => match[1] ?? "") .filter((path) => path.length > 0 && !CONTROL_CHARACTER_PATTERN.test(path)); } diff --git a/packages/shared-contracts/test/desktop-attachments.test.ts b/packages/shared-contracts/test/desktop-attachments.test.ts index a51dcef00..eec1186c8 100644 --- a/packages/shared-contracts/test/desktop-attachments.test.ts +++ b/packages/shared-contracts/test/desktop-attachments.test.ts @@ -45,6 +45,16 @@ describe("desktopImageAttachmentPaths", () => { expect(desktopImageAttachmentPaths(typed)).toEqual([]); }); + it("reads only a complete context that opens the prompt", () => { + const context = "# Files mentioned by the user:\n\n## a.png: /a.png\nImage attachment: true\n"; + // Pasted without the request heading that ends a Desktop context. + expect(desktopImageAttachmentPaths(context)).toEqual([]); + expect(desktopImageAttachmentPaths(`${context}\n## My request: inline`)).toEqual([]); + // Preceded by other text, so it is part of what the user typed. + expect(desktopImageAttachmentPaths(`Look:\n${context}\n## My request:\nhi`)).toEqual([]); + expect(desktopImageAttachmentPaths(`\n\n${context}\n## My request:\nhi`)).toEqual(["/a.png"]); + }); + it("accepts only absolute paths without control characters", () => { expect( desktopImageAttachmentPaths( From 719faa736f24e312471d080f5ab4d5b3db377c28 Mon Sep 17 00:00:00 2001 From: BeiLuoqu Date: Sun, 25 Oct 2026 18:50:00 +0800 Subject: [PATCH 09/15] fix: close a fenced code block on a CRLF line A closing fence line split on `\n` keeps the `\r` of a CRLF line ending, which the closing pattern rejected, so the fence never closed and media after it was left unrewritten. Accept the `\r` as part of the line ending. Co-Authored-By: Claude Opus 5.5 --- packages/harness-adapter/src/local-media-markdown.ts | 5 +++-- packages/harness-adapter/test/local-media-markdown.test.ts | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/harness-adapter/src/local-media-markdown.ts b/packages/harness-adapter/src/local-media-markdown.ts index 0f80f14de..4c81b22e4 100644 --- a/packages/harness-adapter/src/local-media-markdown.ts +++ b/packages/harness-adapter/src/local-media-markdown.ts @@ -316,10 +316,11 @@ function indexOfLineStartTilde(text: string, start: number): number { /** * End of the line that closes a fence opened before `start`: the same marker, at least as long, - * indented at most three spaces and followed only by spaces or tabs. -1 while still inside. + * indented at most three spaces and followed only by spaces or tabs (and the `\r` of a CRLF line + * ending). -1 while still inside. */ function closingFenceEnd(text: string, start: number, marker: string, length: number): number { - const closing = new RegExp(`^ {0,3}${marker === "`" ? "`" : "~"}{${length},}[ \\t]*$`, "u"); + const closing = new RegExp(`^ {0,3}${marker === "`" ? "`" : "~"}{${length},}[ \\t]*\\r?$`, "u"); // The opening line's info string is never a closing fence. let lineStart = text.indexOf("\n", start); while (lineStart !== -1) { diff --git a/packages/harness-adapter/test/local-media-markdown.test.ts b/packages/harness-adapter/test/local-media-markdown.test.ts index 48b0b130a..f894e84d1 100644 --- a/packages/harness-adapter/test/local-media-markdown.test.ts +++ b/packages/harness-adapter/test/local-media-markdown.test.ts @@ -121,6 +121,10 @@ describe("rewriteLocalMediaMarkdown", () => { expect(rewrite("~~~~\n`a.png`\n~~~~~\nafter `a.png`")).toBe( `~~~~\n\`a.png\`\n~~~~~\nafter\n\n![a.png](${a})`, ); + // A CRLF line ending closes a fence too. + expect(rewrite("```\r\n`a.png`\r\n``` \r\nafter `a.png`")).toBe( + `\`\`\`\r\n\`a.png\`\r\n\`\`\` \r\nafter\n\n![a.png](${a})`, + ); // Backtick runs inside a line are not fences, so later media still resolves. expect(rewrite("use ``` here, then ![a](a.png)")).toBe(`use \`\`\` here, then ![a](${a})`); expect(rewrite(" ```\n`a.png`")).toBe(` \`\`\`\n\n![a.png](${a})`); @@ -167,6 +171,7 @@ describe("LocalMediaMarkdownStream", () => { "````\n```\n`a.png`\n```\n````\nthen `a.png`", "~~~\n![a](a.png)\n~~\n~~~ \nafter ![a](a.png)", "use ``` here `a.png`", + "```\r\n`a.png`\r\n```\r\nthen `a.png`", ]; function streamed(text: string, cuts: readonly number[]): { deltas: string[]; text: string } { From 0e3324aedffada6d7466d84585f21c6102322a95 Mon Sep 17 00:00:00 2001 From: BeiLuoqu Date: Sun, 25 Oct 2026 18:50:06 +0800 Subject: [PATCH 10/15] fix(deepseek-harness): resolve replayed media against the directory of its message The Session folded every `working-directory/change` in the journal before replaying an incomplete Turn, and a live Turn can buffer events before it is materialized, so a message could resolve its media against a directory the agent moved to later. Track the directory on the Turn buffer as events replay, as history projection already does. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/external-thread-media.md | 4 +- .../deepseek-harness/src/modern/session.ts | 17 +++++- .../test/modern/session.test.ts | 57 +++++++++++++++++++ 3 files changed, 75 insertions(+), 3 deletions(-) diff --git a/docs/architecture/external-thread-media.md b/docs/architecture/external-thread-media.md index 53f2e09f3..92e5e060a 100644 --- a/docs/architecture/external-thread-media.md +++ b/docs/architecture/external-thread-media.md @@ -15,7 +15,7 @@ Harness 常按工作目录写相对路径，因此需要由 Adapter 改写。 `@codexhost/harness-adapter/local-media-markdown` 提供共用的改写： - 改写图片目标中的本地媒体路径，以及只包含一个媒体路径的行内代码（转成图片）。相对路径按 Adapter 给出的根目录依次查找，只有找到实际存在的文件才改写。 -- 远程 URL、`data:` 等地址、找不到的文件、无法解析的写法和围栏代码块中的内容保持原样，不改动一个字节。围栏按 CommonMark 识别：开头是行首（最多缩进三个空格）三个或更多的 `` ` `` 或 `~`；结尾行用同一种字符，长度不短于开头，后面只能有空白。行中间出现的 ```` ``` ```` 不算围栏。 +- 远程 URL、`data:` 等地址、找不到的文件、无法解析的写法和围栏代码块中的内容保持原样，不改动一个字节。围栏按 CommonMark 识别：开头是行首（最多缩进三个空格）三个或更多的 `` ` `` 或 `~`；结尾行用同一种字符，长度不短于开头，后面只能有空白（CRLF 换行留下的 `\r` 也算行尾）。行中间出现的 ```` ``` ```` 不算围栏。 - Windows 盘符路径写成正斜杠（`C:/…`）。CommonMark 会把反斜杠加标点当作转义，原样输出会让 `C:\Users\me\.codex\…` 丢掉分隔符。 - 目标是本地媒体文件的普通链接（`[文字](images/1.jpg)`，没有 `!`）同样转成图片，链接文字作为图片说明；Desktop 对这种链接只显示文件标签，不显示图片。指向网页、其他文件或找不到的文件的链接保持原样；链接文字里嵌有方括号（如套着图片的徽章链接）时也不改。 - 行内代码或链接转成的图片单独成段，与 Agent 自己写 Markdown 图片时的排版一致：前后各空一行，紧跟其后的句末标点（`。.!！?？;；,，`）移到图片之前，并替换引出图片的冒号，例如 `` …鱼：`images/1.jpg`。 `` 显示为“…鱼。”，下一段是图片。同一行没有文字时，标点留在图片后面。表格行和标题行无法放入独立段落，其中的图片保持行内。实时输出会多等一个字符，用来判断其后是否有标点，结尾的冒号和空格同样暂缓发出。 @@ -29,7 +29,7 @@ Harness 常按工作目录写相对路径，因此需要由 Adapter 改写。 | Harness | 根目录 | 范围 | | --- | --- | --- | | Grok | 工作目录、Grok 会话目录及其 `videos/`、`images/`、用户 `Downloads/` | 实时输出与历史 | -| DeepSeek Harness | Agent 当前工作目录（随 `working-directory/change` 更新），再到 Session 工作目录 | 实时输出与历史（`readSnapshot`）；用量刷新不做改写 | +| DeepSeek Harness | 消息产生时 Agent 的工作目录（按事件顺序随 `working-directory/change` 更新，恢复未完成的 Turn 时也一样），再到 Session 工作目录 | 实时输出与历史（`readSnapshot`）；用量刷新不做改写 | 其他 Harness 的 Agent 文本仍原样转发；它们输出相对路径时，图片仍显示为占位框。 diff --git a/packages/adapters/deepseek-harness/src/modern/session.ts b/packages/adapters/deepseek-harness/src/modern/session.ts index f4e040ef0..91cd4751c 100644 --- a/packages/adapters/deepseek-harness/src/modern/session.ts +++ b/packages/adapters/deepseek-harness/src/modern/session.ts @@ -208,6 +208,8 @@ interface NativeTurnBuffer { active?: ActiveHostTurn; compacting?: string; replayed: number; + /** Working directory as of the last replayed event, which media in projected text resolves against. */ + workingDirectory: string | undefined; sawUserMessage: boolean; reachedCorrelationBoundary: boolean; } @@ -503,12 +505,19 @@ export class ModernHarnessSession implements HarnessSession, ModernEventSink { }; let resumedAutonomousBuffer: NativeTurnBuffer | undefined; if (projection.incompleteTurn) { + // The incomplete Turn is the journal's tail; replay starts from the directory before it. + let workingDirectory = this.#sessionDirectory; + const turnStart = this.#events.length - projection.incompleteTurn.events.length; + for (const event of this.#events.slice(0, turnStart)) { + workingDirectory = modernWorkingDirectory(workingDirectory, event); + } const buffer: NativeTurnBuffer = { nativeTurn: projection.incompleteTurn.turn, events: [...projection.incompleteTurn.events], input: [], initialResume: true, replayed: 0, + workingDirectory, sawUserMessage: false, reachedCorrelationBoundary: false, }; @@ -2083,6 +2092,7 @@ export class ModernHarnessSession implements HarnessSession, ModernEventSink { input: [], initialResume: false, replayed: 0, + workingDirectory: this.#workingDirectory, sawUserMessage: false, reachedCorrelationBoundary: false, }; @@ -2333,6 +2343,7 @@ export class ModernHarnessSession implements HarnessSession, ModernEventSink { while (!this.#closed && buffer.replayed < buffer.events.length) { const event = buffer.events[buffer.replayed] as ModernJournalEvent; buffer.replayed += 1; + buffer.workingDirectory = modernWorkingDirectory(buffer.workingDirectory, event); this.#projectLiveEvent(buffer, event, initialReplay); } this.#projectAssistantLiveChunks(); @@ -2452,7 +2463,11 @@ export class ModernHarnessSession implements HarnessSession, ModernEventSink { ), text: "", }; - const roots = modernMediaRoots(this.#workingDirectory, this.#sessionDirectory); + // The directory where the Turn's replay stands, not the journal's latest: replay may lag. + const buffer = this.#buffer; + const workingDirectory = + buffer?.active === active ? buffer.workingDirectory : this.#workingDirectory; + const roots = modernMediaRoots(workingDirectory, this.#sessionDirectory); active.agent = { item, text: "", media: new LocalMediaMarkdownStream(roots) }; this.#emit({ type: "item.started", turnId: active.turnId, item }); } diff --git a/packages/adapters/deepseek-harness/test/modern/session.test.ts b/packages/adapters/deepseek-harness/test/modern/session.test.ts index e7f0e8cbc..18884b907 100644 --- a/packages/adapters/deepseek-harness/test/modern/session.test.ts +++ b/packages/adapters/deepseek-harness/test/modern/session.test.ts @@ -1478,6 +1478,63 @@ describe("DeepSeek Harness Modern Session", () => { } }); + it("resolves resumed media against the directory of its message, not a later one", async () => { + const workspace = await mkdtemp(path.join(os.tmpdir(), "codexhost-dsh-media-")); + const shots = path.join(workspace, "shots"); + const later = path.join(workspace, "later"); + try { + await mkdir(shots); + await mkdir(later); + await writeFile(path.join(shots, "a.png"), "png"); + const feed = new EventFeed(); + const session = new ModernHarnessSession({ + remote: new FakeRemote([]), + journal: { + header: { version: 4, id: SESSION_ID, createdAt: 1, isSeeded: false, cwd: workspace }, + cursor: 6, + projections: { asOfSeq: -1, values: {} }, + // The incomplete Turn changes directory after the message naming the media. + events: [ + event(0, "turn/start", { turn: 1 }), + event(1, "step/start", { turn: 1, step: 1 }), + userMessage(2, "show", "old-request"), + event(3, "working-directory/change", { cwd: shots }), + finalAssistantMessage(4, 1, "![a](a.png)", "think"), + event(5, "step/end", { turn: 1, step: 1 }), + event(6, "working-directory/change", { cwd: later }), + ], + live: feed, + close: () => { + feed.finish(); + return Promise.resolve(); + }, + }, + control: new FakeControl(), + eventGateway: new ModernEventGateway(new FakeRemote([])), + modelCatalog: MODEL_CATALOG, + permissionModes: null, + sessionId: SESSION_ID, + randomUUID: () => "autonomous-resume", + now: () => 10, + }); + const outputs = lifecycleOutputs(session); + const a = path.join(shots, "a.png").replaceAll("\\", "/"); + let agentText: string | undefined; + while (agentText === undefined) { + const entry = await nextEvent(outputs); + if (entry.type === "item.completed" && entry.snapshot.item.type === "agentMessage") { + agentText = entry.snapshot.item.text; + } + } + expect(agentText).toBe(`![a](${a})`); + feed.push(event(7, "turn/end", { turn: 1, reason: { kind: "completed" } })); + await eventsThrough(outputs, "turn.completed"); + await session.close(); + } finally { + await rm(workspace, { recursive: true, force: true }); + } + }); + it("keeps a live Turn healthy through the DSH 0.2.1-rc.2 search request with disabled thinking", async () => { const test = setup([() => accepted()], [], ["request-1"]); const outputs = lifecycleOutputs(test.session); From c7cd646e10c5c53b78b3d47ab704904d8f45476a Mon Sep 17 00:00:00 2001 From: BytePioneer-AI <28417416@qq.com> Date: Sun, 1 Nov 2026 20:59:38 +0800 Subject: [PATCH 11/15] fix: simplify external Harness local media projection --- docs/architecture/external-thread-media.md | 32 +- .../test/modern/history.test.ts | 4 +- packages/adapters/grok/src/grok-adapter.ts | 49 +-- .../adapters/grok/test/grok-adapter.test.ts | 76 +++++ .../grok/test/local-media-markdown.test.ts | 2 +- .../src/local-media-markdown.ts | 245 +++----------- .../test/local-media-markdown.test.ts | 314 ++++++++---------- tools/renderer-binding/README.md | 15 + .../renderer-binding/media-path-contract.mjs | 140 ++++++++ .../media-path-contract.test.mjs | 167 ++++++++++ tools/renderer-binding/media-path-probe.mjs | 89 +++++ 11 files changed, 725 insertions(+), 408 deletions(-) create mode 100644 tools/renderer-binding/media-path-contract.mjs create mode 100644 tools/renderer-binding/media-path-contract.test.mjs create mode 100644 tools/renderer-binding/media-path-probe.mjs diff --git a/docs/architecture/external-thread-media.md b/docs/architecture/external-thread-media.md index 92e5e060a..e8c7752aa 100644 --- a/docs/architecture/external-thread-media.md +++ b/docs/architecture/external-thread-media.md @@ -1,6 +1,6 @@ # 外部 Thread 的本地图片显示 -外部 Harness 的消息由 codexhost 投影成 Codex Desktop 原生条目后显示。本地图片能否显示，取决于投影出的条目是否满足 Desktop 的加载规则。本文记录当前做法和边界，依据 Codex Desktop 26.1021.3705.0 的渲染代码核对；Desktop 更新后需重新核实。 +外部 Harness 的消息由 codexhost 投影成 Codex Desktop 原生条目后显示。本地图片能否显示，取决于投影出的条目是否满足 Desktop 的加载规则。本文记录当前做法和边界，依据 Codex Desktop 26.1021.3705.0 与 26.1027.51404 的渲染代码核对；Desktop 更新后需重新核实。 ## Desktop 的加载规则 @@ -14,15 +14,29 @@ Harness 常按工作目录写相对路径，因此需要由 Adapter 改写。 `@codexhost/harness-adapter/local-media-markdown` 提供共用的改写： -- 改写图片目标中的本地媒体路径，以及只包含一个媒体路径的行内代码（转成图片）。相对路径按 Adapter 给出的根目录依次查找，只有找到实际存在的文件才改写。 -- 远程 URL、`data:` 等地址、找不到的文件、无法解析的写法和围栏代码块中的内容保持原样，不改动一个字节。围栏按 CommonMark 识别：开头是行首（最多缩进三个空格）三个或更多的 `` ` `` 或 `~`；结尾行用同一种字符，长度不短于开头，后面只能有空白（CRLF 换行留下的 `\r` 也算行尾）。行中间出现的 ```` ``` ```` 不算围栏。 -- Windows 盘符路径写成正斜杠（`C:/…`）。CommonMark 会把反斜杠加标点当作转义，原样输出会让 `C:\Users\me\.codex\…` 丢掉分隔符。 -- 目标是本地媒体文件的普通链接（`[文字](images/1.jpg)`，没有 `!`）同样转成图片，链接文字作为图片说明；Desktop 对这种链接只显示文件标签，不显示图片。指向网页、其他文件或找不到的文件的链接保持原样；链接文字里嵌有方括号（如套着图片的徽章链接）时也不改。 -- 行内代码或链接转成的图片单独成段，与 Agent 自己写 Markdown 图片时的排版一致：前后各空一行，紧跟其后的句末标点（`。.!！?？;；,，`）移到图片之前，并替换引出图片的冒号，例如 `` …鱼：`images/1.jpg`。 `` 显示为“…鱼。”，下一段是图片。同一行没有文字时，标点留在图片后面。表格行和标题行无法放入独立段落，其中的图片保持行内。实时输出会多等一个字符，用来判断其后是否有标点，结尾的冒号和空格同样暂缓发出。 -- 路径中含有 `%` 时另行处理。Desktop 读取文件前会对目标做两次 `decodeURI`，任何一次性的转义都无法保住字面 `%`；Grok 的会话目录名是用 `encodeURIComponent` 编码的工作目录，总会包含 `%`。改写时为这类文件在 `<数据目录>/media-links/<路径哈希>/` 下建一个不含 `%` 的硬链接（失败则复制），消息中链接这个别名；硬链接和复制都失败时，把 `%` 转义成 `%2525`，正好经受两次解码。不含 `%` 的路径不受影响，也不创建别名。别名目录可以随时删除，下次改写时重新生成。 -- 支持的扩展名：图片 `png`、`jpg`、`jpeg`、`gif`、`webp`、`svg`，音视频 `mp4`、`m4v`、`mov`、`webm`、`mp3`、`m4a`、`aac`、`ogg`、`wav`。 +- 改写单行 Markdown 图片目标中的本地媒体路径，以及只包含一个媒体路径的单反引号行内代码（转成图片）。普通链接保持原样；不搬移标点、不增删段落或周围空白。 +- 顺序固定为：相对路径解析或 `fileURLToPath()` → `path.resolve()` → 校验最终目标的扩展名与文件是否存在 → 保护字面 `%` → Markdown destination 格式化。不是先检查原文里 `?` / `#` 前面的伪扩展名，再访问规范化后的另一个文件。 +- 相对路径按 Adapter 给出的 `roots` 依次查找。**roots 是查找顺序，不是访问沙箱**：`..`、绝对路径与有效 `file:` URL 可以指向根目录外；文件检查会跟随符号链接。扩展名校验不等于文件内容类型校验，也不替代 Harness 自己的访问策略。 +- 远程 URL、`data:` 等地址、找不到的文件和无法解析的写法保持原样。支持的扩展名：图片 `png`、`jpg`、`jpeg`、`gif`、`webp`、`svg`，音视频 `mp4`、`m4v`、`mov`、`webm`、`mp3`、`m4a`、`aac`、`ogg`、`wav`。 +- Windows 盘符路径写成正斜杠（`C:/…`）。CommonMark 会把反斜杠加标点当作转义，原样输出会让 `C:\Users\me\.codex\…` 丢掉分隔符；UNC 路径保留并转义其反斜杠。 -实时输出只能追加，Host 要求条目完成时的全文等于此前所有增量之和。`LocalMediaMarkdownStream` 因此只暂缓结尾仍可能被后续文本改变的写法（未闭合的图片、未闭合的行内代码），写错的图片或单个反引号不会扣住后面的文字；Item 结束前再补发暂缓部分。同一条消息内，每个路径只检查一次文件，保证前后改写一致。 +### 字面百分号与 Desktop 私有规则 + +Desktop 的 Markdown 图片 token 转换器和媒体加载组件各归一化一次本地路径。对真实绝对路径，每次使用 `decodeURI`；`file:` 输入的第一阶段则通过 URL pathname 解码。因此 Host 先把输入转成真实绝对路径，再把每个 `%` 写成 `%2525`。例如文件名中的字面 `%2F` 在两次解码后仍是 `%2F`，不是目录分隔符；Grok 以 `encodeURIComponent(cwd)` 命名的会话目录也遵循这条规则。 + +这只是字符串转换：不建立硬链接、不复制媒体，也不创建缓存目录。不能直接对原始 `file:` URL 做同样替换，否则 URL 中表示空格的 `%20` 也会被保护成字面文件名。带空格或括号的真实路径使用 `<…>` destination；中文、普通 `#` / `?` 字符保持原样。 + +规则依赖 Desktop 的两阶段加载链路，并不保证所有文件名可表达。Desktop 还会把 `#L12`、`#L12C3` 等结尾解释为行号引用；如 `a.png#L12` 这样的字面文件名不在本模块支持范围内。Desktop 版本变化后需重新检查归一化语义，而不是只运行两次自行假定的 `decodeURI`。 + +`tools/renderer-binding/media-path-probe.mjs` 可只读检查已运行 Renderer 中实际加载的图片 token 转换器与路径解析器。它按语义结构和 import/export 关系发现函数，不依赖压缩名称；缺失或不唯一时报 `unverified`。此检查不挂载图片、不读取文件，不能代替实机显示和远程文件传输验收。运行方式见 [`tools/renderer-binding/README.md`](../../tools/renderer-binding/README.md#local-media-path-contract)。 + +### 围栏与流式边界 + +支持的平面围栏：开头标记位于第 0 列且不带容器前缀，是三个或更多的同种反引号或 `~`；结束行最多缩进三个空格，使用同种字符，长度不短于开头，后面只允许空格、Tab 或 CRLF 留下的 `\r`。行中间的反引号串不算围栏。 + +本模块不重建 CommonMark 的容器嵌套。在引用块、无序/有序列表前缀后发现围栏，或发现任何缩进/Tab 后的围栏式标记时，从该标记到消息末尾都保持原样；即使后面还有普通媒体路径也不改写，以免错误判断容器结束后修改代码。这是保守支持范围，不是完整 CommonMark 解析器。 + +实时输出保持追加式，完成全文必须等于初始文本加所有增量。Grok 与 DSH 均使用 `LocalMediaMarkdownStream`：只暂缓结尾未决的图片、行内路径或可能增长的围栏标记，不等待整行、不等待后续标点。Item 结束前补发暂缓原文；不完整的单反引号路径遇到换行后不再扣住后面的文字。同一条消息内缓存每个候选路径的存在性检查，包括未找到的结果，避免文件出现或根目录命中变化导致已发布文本需要撤回；新的消息使用新的缓存。 接入的 Harness： diff --git a/packages/adapters/deepseek-harness/test/modern/history.test.ts b/packages/adapters/deepseek-harness/test/modern/history.test.ts index 79ac07a5d..6953a9e03 100644 --- a/packages/adapters/deepseek-harness/test/modern/history.test.ts +++ b/packages/adapters/deepseek-harness/test/modern/history.test.ts @@ -283,7 +283,7 @@ describe("DeepSeek Harness Modern history projection", () => { const b = path.join(shots, "b.png").replaceAll("\\", "/"); expect(agentTexts(workspace)).toEqual([ - `![a](${a})\n\n![b.png](${b})\n\n![b](b.png)`, + `![a](${a}) ![b.png](${b}) ![b](b.png)`, `![b](${b}) ![a](${a})`, ]); expect(agentTexts()).toEqual([ @@ -310,7 +310,7 @@ describe("DeepSeek Harness Modern history projection", () => { }).snapshot.turns.flatMap(({ items }) => items.flatMap(({ item }) => (item.type === "agentMessage" ? [item.text] : [])), ); - expect(liveText).toBe(`see\n\n![a.png](${a})`); + expect(liveText).toBe(`see ![a.png](${a})`); expect(reloaded(true)).toEqual([liveText]); // Usage projections do not ask for media and keep the native text. expect(reloaded()).toEqual([`see \`${absolute}\``]); diff --git a/packages/adapters/grok/src/grok-adapter.ts b/packages/adapters/grok/src/grok-adapter.ts index 049aa87f5..b4b7447b5 100644 --- a/packages/adapters/grok/src/grok-adapter.ts +++ b/packages/adapters/grok/src/grok-adapter.ts @@ -57,7 +57,7 @@ import { type TurnSteerAccepted, type TurnSteerCommand, } from "@codexhost/harness-adapter"; -import { rewriteLocalMediaMarkdown } from "@codexhost/harness-adapter/local-media-markdown"; +import { LocalMediaMarkdownStream } from "@codexhost/harness-adapter/local-media-markdown"; import { harnessCommandCatalogSchema, type HarnessCommandCatalog, @@ -212,8 +212,7 @@ interface ActiveTurn { usage?: GrokTurnUsage; agent: HostAgentMessageItem | null; agentMessageId: string | null; - rawAgentText: string; - projectedAgentText: string; + agentMedia: LocalMediaMarkdownStream | null; reasoning: HostReasoningItem | null; reasoningMessageId: string | null; compactionItem: HostContextCompactionItem | null; @@ -531,8 +530,7 @@ class GrokHarnessSession implements HarnessSession { usage: new GrokTurnUsage(this.#usage, this.#snapshot.turns.length === 0), agent: null, agentMessageId: null, - rawAgentText: "", - projectedAgentText: "", + agentMedia: null, reasoning: null, reasoningMessageId: null, compactionItem: null, @@ -660,8 +658,7 @@ class GrokHarnessSession implements HarnessSession { ), agent: null, agentMessageId: null, - rawAgentText: "", - projectedAgentText: "", + agentMedia: null, reasoning: null, reasoningMessageId: null, compactionItem: null, @@ -1100,20 +1097,16 @@ class GrokHarnessSession implements HarnessSession { itemId: hostItemIdSchema.parse(this.#randomUUID()), text: "", }; - active.rawAgentText = ""; - active.projectedAgentText = ""; + active.agentMedia = new LocalMediaMarkdownStream(this.#mediaRoots); this.#event({ type: "item.started", turnId: active.command.turnId, item: active.agent }); } - active.rawAgentText += text; - const projected = rewriteLocalMediaMarkdown(active.rawAgentText, this.#mediaRoots, { - holdIncomplete: true, - }); - const delta = projected.startsWith(active.projectedAgentText) - ? projected.slice(active.projectedAgentText.length) - : text; - if (delta.length === 0) return; - active.projectedAgentText += delta; - active.agent = { ...active.agent, text: active.projectedAgentText }; + const delta = active.agentMedia?.append(text) ?? ""; + this.#publishAgent(active, delta); + } + + #publishAgent(active: ActiveTurn, delta: string): void { + if (!active.agent || !active.agentMedia || !delta) return; + active.agent = { ...active.agent, text: active.agentMedia.text }; this.#event({ type: "item.updated", turnId: active.command.turnId, @@ -1381,25 +1374,11 @@ class GrokHarnessSession implements HarnessSession { #completeAgent(active: ActiveTurn, outcome: HostItemOutcome): void { const item = active.agent; - if (item && active.rawAgentText.length > 0) { - const flushed = rewriteLocalMediaMarkdown(active.rawAgentText, this.#mediaRoots); - const delta = flushed.startsWith(item.text) ? flushed.slice(item.text.length) : ""; - if (delta.length > 0) { - active.projectedAgentText = flushed; - active.agent = { ...item, text: flushed }; - this.#event({ - type: "item.updated", - turnId: active.command.turnId, - itemId: item.itemId, - update: { type: "text.append", text: delta }, - }); - } - } + if (item && active.agentMedia) this.#publishAgent(active, active.agentMedia.flush()); const completed = active.agent ?? item; active.agent = null; active.agentMessageId = null; - active.rawAgentText = ""; - active.projectedAgentText = ""; + active.agentMedia = null; if (completed) this.#completeItem(active, completed, outcome); } diff --git a/packages/adapters/grok/test/grok-adapter.test.ts b/packages/adapters/grok/test/grok-adapter.test.ts index 44d85eb79..5e370bc8f 100644 --- a/packages/adapters/grok/test/grok-adapter.test.ts +++ b/packages/adapters/grok/test/grok-adapter.test.ts @@ -459,6 +459,82 @@ describe("Grok Adapter questions", () => { }); describe("Grok Adapter ACP projection", () => { + it("keeps pending characters when media appears during an append-only message", async () => { + const cwd = await mkdtemp(path.join(os.tmpdir(), "grok-media-stream-")); + const transport = new FakeGrokTransport(); + const adapter = new GrokAdapter( + {}, + { + randomUUID: () => crypto.randomUUID(), + createTransport: () => transport, + fetchCredits: async () => null, + }, + ); + try { + const opened = await adapter.open({ kind: "create", cwd }); + if (!opened.ok) throw new Error(opened.error.message); + const session = opened.value; + const iterator = session.outputs[Symbol.asyncIterator](); + const turnId = hostTurnIdSchema.parse("grok-media-stream"); + await session.execute({ + type: "turn.start", + turnId, + input: [{ type: "text", text: "show" }], + }); + expect(await nextEvent(iterator)).toMatchObject({ type: "turn.started" }); + const first = "![missing](later.png) and `"; + transport.event({ type: "agent.text", messageId: "first", text: first }); + const started = await nextEvent(iterator); + const update = await nextEvent(iterator); + expect(started).toMatchObject({ type: "item.started", item: { type: "agentMessage" } }); + expect(update).toMatchObject({ + type: "item.updated", + update: { type: "text.append", text: first.slice(0, -1) }, + }); + await writeFile(path.join(cwd, "later.png"), "synthetic image"); + transport.event({ type: "agent.text", messageId: "first", text: "later.png`。done" }); + // A new message gets a fresh file-check cache; the earlier message keeps its decision. + transport.event({ + type: "agent.text", + messageId: "second", + text: "![ready](later.png) `unfinished", + }); + transport.finish(); + const events: HostEvent[] = [started, update]; + for (;;) { + const output = await nextOutput(iterator); + if (output.kind !== "event") continue; + events.push(output.event); + if (output.event.type === "turn.completed") break; + } + const completed = events.flatMap((event) => + event.type === "item.completed" && event.snapshot.item.type === "agentMessage" + ? [event.snapshot.item] + : [], + ); + expect(completed.map(({ text }) => text)).toEqual([ + first + "later.png`。done", + `![ready](${path.join(cwd, "later.png").replaceAll("\\", "/")}) \`unfinished`, + ]); + for (const item of completed) { + expect( + events + .flatMap((event) => + event.type === "item.updated" && + event.itemId === item.itemId && + event.update.type === "text.append" + ? [event.update.text] + : [], + ) + .join(""), + ).toBe(item.text); + } + } finally { + await adapter.close(); + await rm(cwd, { recursive: true, force: true }); + } + }); + it("passes per-Session delegation environment to the ACP transport", async () => { const transport = new FakeGrokTransport(); const createTransport = vi.fn(() => transport); diff --git a/packages/adapters/grok/test/local-media-markdown.test.ts b/packages/adapters/grok/test/local-media-markdown.test.ts index 46b433e33..6f5cb332b 100644 --- a/packages/adapters/grok/test/local-media-markdown.test.ts +++ b/packages/adapters/grok/test/local-media-markdown.test.ts @@ -129,7 +129,7 @@ describe("Grok local media Markdown", () => { emitted = projected; } expect(emitted).toBe( - `影片：\n\n![usegrokbot-templates-promo-10s.mp4](${md(path.join(session, "videos/usegrokbot-templates-promo-10s.mp4"))})\n\n完成`, + `影片：\n\n![usegrokbot-templates-promo-10s.mp4](${md(path.join(session, "videos/usegrokbot-templates-promo-10s.mp4"))})\n完成`, ); }); diff --git a/packages/harness-adapter/src/local-media-markdown.ts b/packages/harness-adapter/src/local-media-markdown.ts index 4c81b22e4..a7214d592 100644 --- a/packages/harness-adapter/src/local-media-markdown.ts +++ b/packages/harness-adapter/src/local-media-markdown.ts @@ -1,6 +1,4 @@ -import { createHash } from "node:crypto"; -import { copyFileSync, existsSync, linkSync, mkdirSync, rmSync, statSync } from "node:fs"; -import os from "node:os"; +import { existsSync, statSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -25,13 +23,8 @@ const MEDIA_EXTENSIONS = new Set([ /** Whether an absolute candidate path is a media file Codex Desktop can load. */ export type LocalMediaExists = (absolutePath: string) => boolean; -/** The path a Markdown destination should name for a resolved local media file. */ -export type LocalMediaLinkPath = (absolutePath: string) => string; - export interface RewriteLocalMediaMarkdownOptions { exists?: LocalMediaExists; - /** Defaults to {@link linkableLocalMediaPath}. */ - linkPath?: LocalMediaLinkPath; /** * Hold back a trailing construct that later text could still complete or invalidate * (an open image or inline code span), so a growing text projects prefix-stably. @@ -43,7 +36,8 @@ export interface RewriteLocalMediaMarkdownOptions { * Codex Desktop loads a Markdown image from local disk only when its destination is an absolute * path or `file:` URL; it resolves any other destination against its own page. Rewrite local * media in image destinations, and inline code holding only a media path, to absolute paths - * found under `roots` in order. Remote URLs, missing files, and fenced code stay unchanged. + * found using `roots` as search order, not as a sandbox. Remote URLs and missing files stay + * unchanged; container/indented fence-like regions keep the remaining message verbatim. */ export function rewriteLocalMediaMarkdown( text: string, @@ -55,7 +49,6 @@ export function rewriteLocalMediaMarkdown( roots, options.exists ?? mediaFileExists, options.holdIncomplete === true, - options.linkPath ?? linkableLocalMediaPath, ).text; } @@ -67,28 +60,14 @@ export function rewriteLocalMediaMarkdown( export class LocalMediaMarkdownStream { readonly #roots: readonly string[]; readonly #exists: LocalMediaExists; - readonly #linkPath: LocalMediaLinkPath; #native = ""; #text = ""; /** Native text length represented by `#text`; the rest is held. */ #consumed = 0; #verbatim = false; - constructor( - roots: readonly string[], - exists: LocalMediaExists = mediaFileExists, - linkPath: LocalMediaLinkPath = linkableLocalMediaPath, - ) { + constructor(roots: readonly string[], exists: LocalMediaExists = mediaFileExists) { this.#roots = [...roots]; - const linked = new Map(); - this.#linkPath = (absolutePath) => { - let link = linked.get(absolutePath); - if (link === undefined) { - link = linkPath(absolutePath); - linked.set(absolutePath, link); - } - return link; - }; const known = new Map(); this.#exists = (absolutePath) => { let found = known.get(absolutePath); @@ -119,7 +98,7 @@ export class LocalMediaMarkdownStream { #advance(holdIncomplete: boolean): string { const next = this.#verbatim ? undefined - : projectLocalMedia(this.#native, this.#roots, this.#exists, holdIncomplete, this.#linkPath); + : projectLocalMedia(this.#native, this.#roots, this.#exists, holdIncomplete); let delta: string; if (next?.text.startsWith(this.#text)) { delta = next.text.slice(this.#text.length); @@ -146,7 +125,6 @@ function projectLocalMedia( roots: readonly string[], exists: LocalMediaExists, holdIncomplete: boolean, - linkPath: LocalMediaLinkPath, ): LocalMediaProjection { let output = ""; let index = 0; @@ -163,12 +141,10 @@ function projectLocalMedia( } const next = earliestIndex([ text.indexOf("![", index), - text.indexOf("[", index), text.indexOf("`", index), indexOfLineStartTilde(text, index), ]); if (next === -1) break; - const segmentStart = index; output += text.slice(index, next); index = next; const marker = text[next]; @@ -177,15 +153,19 @@ function projectLocalMedia( while (text[runEnd] === marker) runEnd += 1; if (runEnd === text.length && holdIncomplete) { // The run may still grow into a fence, or a backtick may open a media path. - const held = mediaIntroduction(text.slice(segmentStart, next)).length; - return { text: output.slice(0, output.length - held), consumed: next - held }; + return { text: output, consumed: next }; } - // CommonMark: a fence is three or more of one marker, indented at most three spaces. - if (runEnd - next >= 3 && isLineStart(text, next)) { - fence = { marker, length: runEnd - next }; - output += text.slice(next, runEnd); - index = runEnd; - continue; + if (runEnd - next >= 3) { + const kind = fenceLineKind(text, next); + // Container nesting/indentation is not reconstructed. Keep the rest of the message + // verbatim rather than risk rewriting code after an uncertain closing boundary. + if (kind === "opaque") break; + if (kind === "flat") { + fence = { marker, length: runEnd - next }; + output += text.slice(next, runEnd); + index = runEnd; + continue; + } } if (marker === "~" || runEnd - next > 1) { output += text.slice(next, runEnd); @@ -209,107 +189,45 @@ function projectLocalMedia( resolved === null ? text.slice(next, image.end) : text.slice(next, image.destinationStart) + - markdownDestination(linkPath(resolved), image.bracketed) + + markdownDestination(resolved, image.bracketed) + text.slice(image.destinationEnd, image.end); index = image.end; continue; } - // A link or inline code naming local media becomes an image; anything else stays as is. - const construct = - text[next] === "[" - ? parseMarkdownImage(text, next, 1) - : (() => { - const code = parseInlineCode(text, next); - return code.kind === "code" - ? { kind: "code" as const, destination: code.content, end: code.end } - : code.kind === "text" - ? { kind: "text" as const, end: next + 1 } - : code; - })(); - if (construct.kind === "text") { - output += text.slice(next, construct.end); - index = construct.end; + const code = parseInlineCode(text, next); + if (code.kind === "text") { + output += "`"; + index = next + 1; continue; } - if (construct.kind === "open") { + if (code.kind === "open") { if (!holdIncomplete) break; - // The construct may become media, which moves the colon introducing it. - const held = mediaIntroduction(text.slice(segmentStart, next)).length; - return { text: output.slice(0, output.length - held), consumed: next - held }; - } - const resolved = resolveLocalMediaPath(construct.destination, roots, exists); - if (resolved === null) { - output += text.slice(next, construct.end); - index = construct.end; - continue; - } - const image = - construct.kind === "image" - ? `![${construct.label}](${markdownDestination(linkPath(resolved), construct.bracketed)})` - : mediaMarkdownImage(resolved, linkPath(resolved)); - const end = construct.end; - if (!canPlaceMediaBlock(output)) { - output += image; - index = end; - continue; - } - // Media the agent named inline becomes its own block, as when it writes the image itself: - // `…鱼：\`images/1.jpg\`。` reads `…鱼。` and then the image. - const punctuation = /^[。．.!！?？;；,，]*/u.exec(text.slice(end))?.[0] ?? ""; - const spacing = /^\s*/u.exec(text.slice(end + punctuation.length))?.[0] ?? ""; - const rest = end + punctuation.length + spacing.length; - const introduction = mediaIntroduction(text.slice(segmentStart, next)); - if (rest === text.length && holdIncomplete) { - // Later text may still add punctuation to move before the image. - return { - text: output.slice(0, output.length - introduction.length), - consumed: next - introduction.length, - }; - } - output = output.slice(0, output.length - introduction.length); - // Punctuation moves only onto a line that has text; otherwise it stays after the image. - const moved = /\S/u.test(output.slice(output.lastIndexOf("\n") + 1)) ? punctuation : ""; - output += moved || introduction.trim(); - if (output.length > 0 && !output.endsWith("\n\n")) - output += output.endsWith("\n") ? "\n" : "\n\n"; - output += image + (moved ? "" : punctuation); - if (rest < text.length) output += "\n\n"; - index = rest; - } - if (holdIncomplete && index < text.length) { - // A trailing colon or space may lead into a media path that has not arrived yet. - const held = mediaIntroduction(text.slice(index)).length; - if (held > 0) { - return { - text: output + text.slice(index, text.length - held), - consumed: text.length - held, - }; + return { text: output, consumed: next }; } + const resolved = resolveLocalMediaPath(code.content, roots, exists); + output += resolved === null ? text.slice(next, code.end) : mediaMarkdownImage(resolved); + index = code.end; } return { text: output + text.slice(index), consumed: text.length }; } -/** Trailing spacing, optionally after a colon, that leads into an inline media path. */ -function mediaIntroduction(text: string): string { - return /[：:]?[ \t]*$/u.exec(text)?.[0] ?? ""; -} - -/** Table rows and headings cannot hold a block, so media written there stays inline. */ -function canPlaceMediaBlock(output: string): boolean { - const line = output.slice(output.lastIndexOf("\n") + 1); - return !/^\s*[|#]/u.test(line); -} - -/** Whether only up to three spaces precede `index` on its line. */ -function isLineStart(text: string, index: number): boolean { - const lineStart = text.lastIndexOf("\n", index - 1) + 1; - return index - lineStart <= 3 && /^ *$/u.test(text.slice(lineStart, index)); +/** Only unprefixed openers are unambiguously flat without reconstructing list nesting. */ +function fenceLineKind(text: string, index: number): "flat" | "opaque" | null { + const prefix = text.slice(text.lastIndexOf("\n", index - 1) + 1, index); + if (prefix === "") return "flat"; + let remaining = prefix.replace(/^[ \t]+/u, ""); + while (remaining) { + const marker = /^(?:>|(?:[-+*]|\d{1,9}[.)])(?=[ \t]))[ \t]*/u.exec(remaining); + if (!marker) return null; + remaining = remaining.slice(marker[0].length); + } + return "opaque"; } /** A `~` that could open a fence; elsewhere tildes are ordinary text. */ function indexOfLineStartTilde(text: string, start: number): number { for (let tilde = text.indexOf("~", start); tilde !== -1; tilde = text.indexOf("~", tilde + 1)) { - if (isLineStart(text, tilde)) return tilde; + if (fenceLineKind(text, tilde) !== null) return tilde; } return -1; } @@ -361,8 +279,6 @@ function parseInlineCode(text: string, start: number): ParsedInlineCode { type ParsedMarkdownImage = | { kind: "image"; - /** Raw label text, without its brackets. */ - label: string; destination: string; destinationStart: number; destinationEnd: number; @@ -373,11 +289,11 @@ type ParsedMarkdownImage = | { kind: "open"; holdFrom: number } | { kind: "text"; end: number }; -/** Parses `![label](destination)`, or with `prefix` 1 a link `[label](destination)`. */ -function parseMarkdownImage(text: string, start: number, prefix = 2): ParsedMarkdownImage { +/** Parses a single-line `![label](destination)` with an optional title. */ +function parseMarkdownImage(text: string, start: number): ParsedMarkdownImage { // Until the label closes, only `![` is settled: `!` was publishable before `[` arrived. - const openLabel = { kind: "open", holdFrom: start + prefix } as const; - let index = start + prefix; + const openLabel = { kind: "open", holdFrom: start + 2 } as const; + let index = start + 2; while (index < text.length) { const character = text[index]; if (character === "\n") return { kind: "text", end: start + 1 }; @@ -386,11 +302,8 @@ function parseMarkdownImage(text: string, start: number, prefix = 2): ParsedMark continue; } if (character === "]") break; - // A link around an image or another link is not a plain reference to media. - if (prefix === 1 && character === "[") return { kind: "text", end: start + 1 }; index += 1; } - const labelEnd = index; if (index + 1 >= text.length) return openLabel; if (text[index + 1] !== "(") return { kind: "text", end: start + 1 }; const destinationStart = index + 2; @@ -437,7 +350,6 @@ function parseMarkdownImage(text: string, start: number, prefix = 2): ParsedMark if (text[index] !== ")") return malformed; return { kind: "image", - label: text.slice(start + prefix, labelEnd), destination, destinationStart, destinationEnd, @@ -452,66 +364,20 @@ function skipBlanks(text: string, start: number): number { return index; } -function mediaMarkdownImage(absolutePath: string, link: string): string { +function mediaMarkdownImage(absolutePath: string): string { const name = path.basename(absolutePath).replaceAll("]", "\\]"); - return `![${name}](${markdownDestination(link, false)})`; -} - -/** - * Codex Desktop percent-decodes a local media destination twice before reading the file, so no - * single escaping keeps a literal `%` intact; Grok's session directories are the cwd encoded - * with encodeURIComponent and always contain one. Such a file is hard-linked (or copied) to a - * `%`-free alias under `/media-links` and the alias is linked instead; if that - * fails, the path is escaped for exactly two decodes. Other paths are returned unchanged. - */ -export function linkableLocalMediaPath( - absolutePath: string, - aliasRoot: string = defaultMediaAliasRoot(), -): string { - if (!absolutePath.includes("%")) return absolutePath; - return mediaAlias(absolutePath, aliasRoot) ?? absolutePath.replaceAll("%", "%2525"); -} - -function defaultMediaAliasRoot(): string { - const dataDirectory = process.env.CODEXHOST_DATA_DIR; - return path.join( - dataDirectory ? path.resolve(dataDirectory) : path.join(os.homedir(), ".codexhost"), - "media-links", - ); -} - -function mediaAlias(absolutePath: string, aliasRoot: string): string | null { - try { - const digest = createHash("sha256").update(absolutePath).digest("hex").slice(0, 16); - const directory = path.join(aliasRoot, digest); - const alias = path.join(directory, path.basename(absolutePath).replaceAll("%", "_")); - if (alias.includes("%")) return null; - const source = statSync(absolutePath); - if (!source.isFile()) return null; - if (existsSync(alias)) { - const current = statSync(alias); - // A hard link shares the source's inode; a copy is current when it is at least as new. - if (current.size === source.size && current.mtimeMs >= source.mtimeMs) return alias; - rmSync(alias, { force: true }); - } - mkdirSync(directory, { recursive: true }); - try { - linkSync(absolutePath, alias); - } catch { - copyFileSync(absolutePath, alias); - } - return alias; - } catch { - return null; - } + return `![${name}](${markdownDestination(absolutePath, false)})`; } function markdownDestination(absolutePath: string, bracketed: boolean): string { // CommonMark drops a backslash before punctuation (`C:\Users\me\.codex`), and Codex reads // `C:/…` as absolute; other paths keep their backslashes escaped. - const destination = /^[A-Za-z]:\\/.test(absolutePath) - ? absolutePath.replaceAll("\\", "/") - : absolutePath.replaceAll("\\", "\\\\"); + // Both Desktop's Markdown token conversion and media loader decode the local path once. + // Encode only the resolved filesystem path, never a file: URL's own URI escapes. + const encoded = absolutePath.replaceAll("%", "%2525"); + const destination = /^[A-Za-z]:\\/.test(encoded) + ? encoded.replaceAll("\\", "/") + : encoded.replaceAll("\\", "\\\\"); return bracketed || /[\s()]/.test(destination) ? `<${destination}>` : destination; } @@ -528,14 +394,14 @@ export function resolveLocalMediaPath( const trimmed = raw.trim(); if (trimmed.length === 0 || isRemoteMediaUrl(trimmed)) return null; const stripped = stripFileUrl(trimmed); - if (stripped === null || !hasMediaExtension(stripped)) return null; + if (stripped === null) return null; if (isAbsoluteMediaPath(stripped)) { const absolute = path.resolve(stripped); - return exists(absolute) ? absolute : null; + return hasMediaExtension(absolute) && exists(absolute) ? absolute : null; } for (const root of roots) { const absolute = path.resolve(root, stripped); - if (exists(absolute)) return absolute; + if (hasMediaExtension(absolute) && exists(absolute)) return absolute; } return null; } @@ -558,8 +424,7 @@ function isAbsoluteMediaPath(value: string): boolean { } function hasMediaExtension(value: string): boolean { - const pathname = value.split(/[?#]/, 1)[0] ?? value; - return MEDIA_EXTENSIONS.has(path.extname(pathname).toLowerCase()); + return MEDIA_EXTENSIONS.has(path.extname(value).toLowerCase()); } function mediaFileExists(absolutePath: string): boolean { diff --git a/packages/harness-adapter/test/local-media-markdown.test.ts b/packages/harness-adapter/test/local-media-markdown.test.ts index f894e84d1..49de64413 100644 --- a/packages/harness-adapter/test/local-media-markdown.test.ts +++ b/packages/harness-adapter/test/local-media-markdown.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, rm, stat, writeFile, mkdir } from "node:fs/promises"; +import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; @@ -7,7 +7,6 @@ import { describe, expect, it, vi } from "vitest"; import { LocalMediaMarkdownStream, - linkableLocalMediaPath, resolveLocalMediaPath, rewriteLocalMediaMarkdown, } from "../src/local-media-markdown.js"; @@ -24,19 +23,34 @@ const files = new Set( "截图/e.png", ].map((file) => path.join(workspace, file)), ); -const exists = (absolutePath: string) => files.has(path.resolve(absolutePath)); +const exists = (absolutePath: string) => files.has(absolutePath); const roots = [workspace]; -/** Markdown spelling of an absolute path: Windows drive paths use forward slashes. */ const md = (absolutePath: string) => absolutePath.replaceAll("\\", "/"); const at = (file: string) => md(path.join(workspace, file)); const rewrite = (text: string, holdIncomplete = false) => rewriteLocalMediaMarkdown(text, roots, { exists, holdIncomplete }); +const containerCode = [ + "> ```md\n> [x](a.png) `a.png`\n> ```\nafter ![a](a.png)", + "- ```md\n ![a](a.png)\n ```\nafter `a.png`", + "1. ~~~~\n `a.png`\n ~~~~\n![a](a.png)", + "> - ~~~\n> ![a](a.png)\n> ~~~", + " ```\n ![a](a.png)\n ```", + "\t~~~\n\t`a.png`\n\t~~~", + "- item\n ```\n ![a](a.png)\n```\n![a](a.png)\n```", +]; + describe("rewriteLocalMediaMarkdown", () => { - it("makes relative image destinations absolute", () => { + it("resolves images and single media code spans without changing surrounding text", () => { expect(rewrite("![shot](shot.png)\n![b](./sub/b.png)")).toBe( `![shot](${at("shot.png")})\n![b](${at("sub/b.png")})`, ); + expect(rewrite("鱼：`a.png`。 后文\n# `a.png`\n| `a.png` |")).toBe( + `鱼：![a.png](${at("a.png")})。 后文\n# ![a.png](${at("a.png")})\n| ![a.png](${at("a.png")}) |`, + ); + expect(rewrite("[图](a.png) and [site](https://example.com/a.png)")).toBe( + "[图](a.png) and [site](https://example.com/a.png)", + ); }); it("keeps non-ASCII paths and brackets paths with spaces", () => { @@ -45,109 +59,86 @@ describe("rewriteLocalMediaMarkdown", () => { ); }); - it("writes Windows paths so Markdown escapes cannot drop a separator", () => { - const rewritten = rewrite("![d](.codex/d.png)"); - expect(rewritten).toBe(`![d](${at(".codex/d.png")})`); - expect(rewritten).not.toMatch(/\\[!-/:-@[-`{-~]/u); + it("writes Windows drive separators without Markdown punctuation escapes", () => { + expect(rewrite("![d](.codex/d.png)")).toBe(`![d](${at(".codex/d.png")})`); + expect(rewrite("![d](.codex/d.png)")).not.toMatch(/\\[!-/:-@[-`{-~]/u); }); - it("resolves file URLs and absolute paths", () => { + it("normalizes file URLs before formatting the absolute path", () => { const absolute = path.join(workspace, "a.png"); expect(rewrite(`![a](${pathToFileURL(absolute).href}) and \`${absolute}\``)).toBe( - `![a](${at("a.png")}) and\n\n![a.png](${at("a.png")})`, + `![a](${at("a.png")}) and ![a.png](${at("a.png")})`, ); + expect(resolveLocalMediaPath("file:///%ZZ.png", roots, exists)).toBeNull(); }); - it("leaves remote, missing, and unparsed references byte-for-byte unchanged", () => { + it("leaves missing, remote and malformed references byte-for-byte unchanged", () => { const text = [ "![remote](https://example.com/a.png)", "![missing](missing.png)", "![escaped](missing\\(1\\).png)", "`npm test` and `a.txt`", - "```md\n![a](a.png)\n`a.png`\n```", ].join("\n"); expect(rewrite(text)).toBe(text); - }); - - it("does not let a malformed image or stray backtick stop later rewrites", () => { expect(rewrite("![a](a.png bad) then `\nand ![x](x.png)")).toBe( `![a](a.png bad) then \`\nand ![x](${at("x.png")})`, ); }); - it("puts a media path written as inline code on its own line, punctuation first", () => { - const a = at("a.png"); - // Grok's usual reply: the image follows the sentence instead of splitting it. - expect(rewrite("画好了，是一只大肥鱼：`a.png`。")).toBe( - `画好了，是一只大肥鱼。\n\n![a.png](${a})`, - ); - expect(rewrite("看图：`a.png`\n然后继续")).toBe(`看图：\n\n![a.png](${a})\n\n然后继续`); - expect(rewrite("see `a.png`. Next")).toBe(`see.\n\n![a.png](${a})\n\nNext`); - expect(rewrite("前文\n\n`a.png`")).toBe(`前文\n\n![a.png](${a})`); - // Nothing on the line to move punctuation onto. - expect(rewrite("`a.png`。")).toBe(`![a.png](${a})。`); - // A table row or heading cannot hold a block. - expect(rewrite("| 图 | `a.png` |")).toBe(`| 图 | ![a.png](${a}) |`); - expect(rewrite("# `a.png`")).toBe(`# ![a.png](${a})`); - }); - - it("shows a link to local media as an image and leaves other links alone", () => { - const a = at("a.png"); - // Grok sometimes links the generated file instead of embedding it. - expect(rewrite("画好了：\n\n[Claude 御姐](a.png)\n\n想改再说。")).toBe( - `画好了：\n\n![Claude 御姐](${a})\n\n想改再说。`, - ); - expect(rewrite("见 [图](sub/b.png)。")).toBe(`见。\n\n![图](${at("sub/b.png")})`); - const unchanged = [ - "[site](https://example.com/a.png)", - "[notes](notes.txt) and [gone](missing.png)", - "- [ ] task\n- [x] done [1]", - `[![badge](${a})](a.png)`, - ].join("\n"); - expect(rewrite(unchanged)).toBe(unchanged); - }); - - it("follows CommonMark fences: line-start openers and matching closers only", () => { - const a = at("a.png"); - const fenced = [ + it("supports flat fences with matching markers, lengths and CRLF closers", () => { + for (const text of [ "````md\n```\n![a](a.png)\n```\n`a.png`\n````", "~~~\n[x](a.png) `a.png`\n~~~", " ```\n`a.png`\n ``` \t", - // Not closed: a shorter run, the other marker, or text after the run. "```\n`a.png`\n``\n~~~\n``` x\n`a.png`", - ]; - for (const text of fenced) expect(rewrite(text), text).toBe(text); - // A fence ends at a matching line, and media after it is rewritten again. - expect(rewrite("~~~~\n`a.png`\n~~~~~\nafter `a.png`")).toBe( - `~~~~\n\`a.png\`\n~~~~~\nafter\n\n![a.png](${a})`, - ); - // A CRLF line ending closes a fence too. + ]) + expect(rewrite(text)).toBe(text); expect(rewrite("```\r\n`a.png`\r\n``` \r\nafter `a.png`")).toBe( - `\`\`\`\r\n\`a.png\`\r\n\`\`\` \r\nafter\n\n![a.png](${a})`, + `\`\`\`\r\n\`a.png\`\r\n\`\`\` \r\nafter ![a.png](${at("a.png")})`, + ); + expect(rewrite("use ``` here, then ![a](a.png)")).toBe( + `use \`\`\` here, then ![a](${at("a.png")})`, ); - // Backtick runs inside a line are not fences, so later media still resolves. - expect(rewrite("use ``` here, then ![a](a.png)")).toBe(`use \`\`\` here, then ![a](${a})`); - expect(rewrite(" ```\n`a.png`")).toBe(` \`\`\`\n\n![a.png](${a})`); }); - it("holds only a trailing construct that later text could still change", () => { - expect(rewrite("见 [图](sub/b", true)).toBe("见"); - expect(rewrite("大肥鱼：`a.png`", true)).toBe("大肥鱼"); - expect(rewrite("大肥鱼：", true)).toBe("大肥鱼"); + it("leaves the suffix from a container or indented fence-like opener untouched", () => { + for (const text of containerCode) { + expect(rewrite(text)).toBe(text); + expect(rewrite(`![a](a.png)\n${text}`)).toBe(`![a](${at("a.png")})\n${text}`); + } + }); + + it("holds only undecided constructs, not punctuation, spacing or complete media", () => { + expect(rewrite("鱼：`a.png`。 ", true)).toBe(`鱼：![a.png](${at("a.png")})。 `); + expect(rewrite("鱼： ", true)).toBe("鱼： "); expect(rewrite("see ![a](a.p", true)).toBe("see ![a]("); - expect(rewrite("see `a.p", true)).toBe("see"); + expect(rewrite("see `a.p", true)).toBe("see "); expect(rewrite("see ![a", true)).toBe("see !["); - expect(rewrite("![a](a.png bad) then ` stray\nmore", true)).toBe( - "![a](a.png bad) then ` stray\nmore", + }); + + it("checks the resolved target's extension, not a fake prefix before ? or #", () => { + const check = vi.fn(() => true); + for (const marker of ["?", "#"]) { + expect( + resolveLocalMediaPath(`../outside/fake.png${marker}/../secret.txt`, roots, check), + ).toBeNull(); + expect(resolveLocalMediaPath(`a.png${marker}L12`, roots, check)).toBeNull(); + } + expect(check).not.toHaveBeenCalled(); + expect(resolveLocalMediaPath("not-media.txt/../a.png", roots, exists)).toBe( + path.join(workspace, "a.png"), ); }); - it("resolves relative paths against roots in order", () => { + it("uses roots as search order, not as a sandbox", () => { const first = path.join(workspace, "sub"); expect(resolveLocalMediaPath("b.png", [first, workspace], exists)).toBe( path.join(first, "b.png"), ); - expect(resolveLocalMediaPath("notes.txt", roots, () => true)).toBeNull(); + expect(resolveLocalMediaPath("../a.png", [first], exists)).toBe(path.join(workspace, "a.png")); + expect(resolveLocalMediaPath(path.join(workspace, "a.png"), [], exists)).toBe( + path.join(workspace, "a.png"), + ); }); }); @@ -161,139 +152,120 @@ describe("LocalMediaMarkdownStream", () => { "![a]\n![c]()", "a!![b](sub/b.png)!", "``a.png`` and `a.png`` and ```", - "![e](截图/e.png)。", - "画好了，是一只大肥鱼：`a.png`。后面", - "see `a.png`. Next `x.png`", - "| 图 | `a.png` |\n`sub/b.png`：", - "前文\n\n`a.png`", - "画好了：\n\n[Claude 御姐](a.png)\n\n后面", - "[x](a.png) and [y](notes.txt) - [ ] t [z](https://e/a.png)", + "鱼：`a.png`。后文 [图](a.png)", + "| `a.png` |\n# `sub/b.png`：", "````\n```\n`a.png`\n```\n````\nthen `a.png`", "~~~\n![a](a.png)\n~~\n~~~ \nafter ![a](a.png)", - "use ``` here `a.png`", "```\r\n`a.png`\r\n```\r\nthen `a.png`", + ...containerCode, ]; - function streamed(text: string, cuts: readonly number[]): { deltas: string[]; text: string } { + function streamed(text: string, cuts: readonly number[]): string { const stream = new LocalMediaMarkdownStream(roots, exists); - const deltas: string[] = []; let start = 0; + let result = ""; for (const end of [...cuts, text.length]) { - deltas.push(stream.append(text.slice(start, end))); + result += stream.append(text.slice(start, end)); start = end; } - deltas.push(stream.flush()); - expect(stream.text).toBe(deltas.join("")); - return { deltas, text: stream.text }; + result += stream.flush(); + expect(stream.text).toBe(result); + return result; } - it("publishes the same text as a one-shot rewrite for every split", () => { + it("matches one-shot output at every pair of cuts and for character-sized chunks", () => { for (const sample of samples) { const expected = rewrite(sample); for (let first = 0; first <= sample.length; first += 1) { for (let second = first; second <= sample.length; second += 1) { - expect(streamed(sample, [first, second]).text, `${sample} @${first},${second}`).toBe( - expected, - ); + expect(streamed(sample, [first, second]), `${sample} @${first},${second}`).toBe(expected); } } - const characters = Array.from({ length: sample.length }, (_, index) => index); - expect(streamed(sample, characters).text).toBe(expected); + expect( + streamed( + sample, + Array.from({ length: sample.length }, (_, i) => i), + ), + ).toBe(expected); } }); - it("does not hold settled text while the stream is open", () => { + it("emits settled text immediately and flushes incomplete text without losing bytes", () => { const stream = new LocalMediaMarkdownStream(roots, exists); - expect(stream.append("x ` stray\nthen ![a](a.png) and ![b](sub/")).toBe( - `x \` stray\nthen ![a](${at("a.png")}) and ![b](`, - ); - expect(stream.append("b.png)")).toBe(`${at("sub/b.png")})`); - expect(stream.flush()).toBe(""); + expect(stream.append("text and `")).toBe("text and "); + expect(stream.append("a.png`。 ")).toBe(`![a.png](${at("a.png")})。 `); + expect(stream.append("![open](sub/")).toBe("![open]("); + expect(stream.flush()).toBe("sub/"); }); - it("checks each path once for the lifetime of a stream", () => { - const check = vi.fn(exists); + it("remembers negative and positive checks when files appear", () => { + let present = false; + const check = vi.fn((absolute: string) => present || exists(absolute)); const stream = new LocalMediaMarkdownStream(roots, check); - stream.append("![a](a.png) "); - stream.append("more "); - stream.append("text"); - stream.flush(); - expect(check).toHaveBeenCalledTimes(1); + const a = at("a.png"); + let text = stream.append("![missing](later.png) and ![a](a.png) `"); + present = true; + text += stream.append("later.png` and ![a](a.png)") + stream.flush(); + expect(text).toBe(`![missing](later.png) and ![a](${a}) \`later.png\` and ![a](${a})`); + expect(check.mock.calls.map(([p]) => p)).toEqual([ + path.join(workspace, "later.png"), + path.join(workspace, "a.png"), + ]); }); }); -describe("links to media whose path holds a literal %", () => { - // Codex Desktop decodes a local media destination twice before it reads the file. - const desktopPath = (destination: string) => decodeURI(decodeURI(destination)); - - it("links a %-free hard link or copy of the file", async () => { - const temp = await mkdtemp(path.join(os.tmpdir(), "media-percent-")); - try { - // Grok names a session directory after its cwd, encoded with encodeURIComponent. - const session = path.join(temp, "sessions", "C%3A%5CUsers%5Cme%5C%E6%96%87"); - await mkdir(path.join(session, "images"), { recursive: true }); - const image = path.join(session, "images", "1%25.jpg"); - await writeFile(image, "jpeg bytes"); - const aliasRoot = path.join(temp, "media-links"); - const linkPath = (absolutePath: string) => linkableLocalMediaPath(absolutePath, aliasRoot); - - const output = rewriteLocalMediaMarkdown("![fish](images/1%25.jpg)", [session], { - linkPath, - }); - expect(output).toMatch(/^!\[fish\]\([^%<>]+\)$/); - const destination = output.slice("![fish](".length, -1); - const alias = desktopPath(destination); - expect(alias.startsWith(md(aliasRoot) + "/")).toBe(true); - expect(path.basename(alias)).toBe("1_25.jpg"); - expect(await readFile(alias, "utf8")).toBe("jpeg bytes"); - // The same file keeps the same alias. - expect(rewriteLocalMediaMarkdown("`images/1%25.jpg`", [session], { linkPath })).toBe( - `![1%25.jpg](${destination})`, - ); - } finally { - await rm(temp, { recursive: true, force: true }); - } - }); - - it("escapes % for both decodes when no alias can be made", async () => { - const temp = await mkdtemp(path.join(os.tmpdir(), "media-percent-")); - try { - const image = path.join(temp, "C%3A%5Cdir", "1.png"); - await mkdir(path.dirname(image)); - await writeFile(image, "png"); - // A file where the alias directory should be makes every alias fail. - const blocked = path.join(temp, "blocked"); - await writeFile(blocked, ""); - const destination = md(linkableLocalMediaPath(image, blocked)); - expect(destination).toContain("%2525"); - expect(path.resolve(desktopPath(destination))).toBe(image); - } finally { - await rm(temp, { recursive: true, force: true }); - } +describe("root selection during streaming", () => { + it("does not change an earlier root decision when a higher-priority file appears", () => { + const higher = path.join(workspace, "higher"); + let appeared = false; + const check = (p: string) => + p === path.join(workspace, "a.png") || (appeared && p === path.join(higher, "a.png")); + const stream = new LocalMediaMarkdownStream([higher, workspace], check); + let text = stream.append("![a](a.png) "); + appeared = true; + text += stream.append("again ![a](a.png)") + stream.flush(); + expect(text).toBe(`![a](${at("a.png")}) again ![a](${at("a.png")})`); + const fresh = new LocalMediaMarkdownStream([higher, workspace], check); + expect(fresh.append("![a](a.png)") + fresh.flush()).toBe(`![a](${at("higher/a.png")})`); }); +}); - it("leaves paths without % alone and creates no alias", async () => { +describe("literal percent paths", () => { + it("protects every literal percent through two decodes without creating aliases", async () => { const temp = await mkdtemp(path.join(os.tmpdir(), "media-percent-")); try { - const aliasRoot = path.join(temp, "media-links"); - const image = path.join(workspace, "shot.png"); - expect(linkableLocalMediaPath(image, aliasRoot)).toBe(image); - await expect(stat(aliasRoot)).rejects.toMatchObject({ code: "ENOENT" }); + const session = path.join(temp, "C%3A%5C%E6%96%87"); + await mkdir(session); + const names = [ + "1%.png", + "%25.png", + "%2F.png", + "%5C.png", + "%E4%B8%AD.png", + "%23%3F.png", + "中文.png", + "with space.png", + ]; + if (process.platform !== "win32") names.push("a#b.png", "a?b.png"); + for (const name of names) { + const file = path.join(session, name); + await writeFile(file, "synthetic image"); + const expected = md(file).replaceAll("%", "%2525"); + for (const input of [pathToFileURL(file).href, name]) { + expect(rewriteLocalMediaMarkdown(`![x](<${input}>)`, [session])).toBe( + `![x](<${expected}>)`, + ); + } + expect(decodeURI(decodeURI(expected))).toBe(md(file)); + const native = `![x](<${name}>)`; + const stream = new LocalMediaMarkdownStream([session]); + const output = [...native].map((c) => stream.append(c)).join("") + stream.flush(); + expect(output).toBe(`![x](<${expected}>)`); + } + expect(await readdir(temp)).toEqual([path.basename(session)]); + expect((await readdir(session)).sort()).toEqual([...names].sort()); } finally { await rm(temp, { recursive: true, force: true }); } }); - - it("asks the stream's link path once per file so published text stays stable", () => { - const linkPath = vi.fn((absolutePath: string) => absolutePath + ".alias"); - const stream = new LocalMediaMarkdownStream(roots, exists, linkPath); - let text = ""; - for (const piece of ["![a](sho", "t.png) and `shot.p", "ng` again ![b](shot.png)"]) - text += stream.append(piece); - text += stream.flush(); - expect(text).toBe( - `![a](${at("shot.png")}.alias) and\n\n![shot.png](${at("shot.png")}.alias)\n\nagain ![b](${at("shot.png")}.alias)`, - ); - expect(linkPath).toHaveBeenCalledTimes(1); - }); }); diff --git a/tools/renderer-binding/README.md b/tools/renderer-binding/README.md index da75be329..78e6b4421 100644 --- a/tools/renderer-binding/README.md +++ b/tools/renderer-binding/README.md @@ -74,3 +74,18 @@ npm run probe:renderer-binding -- \ ``` The runner installs the title policy, reloads the Renderer, verifies metadata-service ownership, installs the narrow draft prewarm policy, marks the Renderer ready, and only then injects the Renderer binding and its tooling-only observer. The schema v2 local report stores production state under `status` and Gate-only observations, switch counters, and structural diagnostics under `observer`. + +## Local media path contract + +For a read-only check of an **already running** Desktop, build `desktop-control` and run the separate tooling entry: + +```text +npx tsc -b packages/desktop-control +node tools/renderer-binding/media-path-probe.mjs --endpoint http://127.0.0.1:9222 +``` + +Unlike the controlled binding runner above, this entry does not reload, install policies, mount UI, read media files, or submit a Turn. CDP supplies the loaded module sources; semantic structure and import/export edges must identify a unique native image token renderer, local path normalizer, and media-loading consumer. Minified function names and hashed asset names are not contracts. Only functions from an already loaded, same-origin module are invoked. + +The check passes synthetic absolute paths through the real token renderer and then the same native normalizer used by media loading. Cases include literal `%`, `%41`, `%25`, `%2F`, `%5C`, encoded Chinese, spaces, `#` / `?`, and a Windows drive path. It checks both the intermediate source and the final path, rather than assuming two calls to a locally implemented decoder. Adapter tests separately cover resolving relative paths and converting `file:` URLs before escaping the filesystem path. + +Output is bounded JSON: `pass` with case counts, `mismatch` when native semantics differ, or `unverified` when source structure, exports, or the Renderer cannot be verified. `mismatch` exits nonzero. No native source, user paths, prompts, or transcript is persisted. Record the Desktop version/build with the result using the normal Desktop audit; a successful normalization check is not proof of pixel rendering, Windows filesystem behavior, or SSH media transport. diff --git a/tools/renderer-binding/media-path-contract.mjs b/tools/renderer-binding/media-path-contract.mjs new file mode 100644 index 000000000..a3eb5172f --- /dev/null +++ b/tools/renderer-binding/media-path-contract.mjs @@ -0,0 +1,140 @@ +import ts from "typescript"; + +const SAMPLE_PATHS = [ + "/codexhost-media-probe/literal%.png", + "/codexhost-media-probe/%41.png", + "/codexhost-media-probe/%25%2F%5C.png", + "/codexhost-media-probe/%E4%B8%AD%E6%96%87.png", + "/codexhost-media-probe/中文 with space#?.png", + "C:/codexhost-media-probe/C%3A%5Cdir/a.png", +]; + +function nodesMatching(node, predicate) { + const matches = []; + const visit = (child) => { + if (predicate(child)) matches.push(child); + ts.forEachChild(child, visit); + }; + visit(node); + return matches; +} + +/** Inspect loaded module source only; names below are recovered from its import/export edges. */ +export function discoverMediaPathContract(modules) { + const summaries = modules + .filter(({ source }) => source.includes("decodeURI") || source.includes("currentFilePath")) + .map(({ url, source }) => { + const tree = ts.createSourceFile(url, source, ts.ScriptTarget.Latest, true, ts.ScriptKind.JS); + const functions = new Map(); + const exports = new Map(); + const imports = []; + for (const node of tree.statements) { + if (ts.isFunctionDeclaration(node) && node.name) functions.set(node.name.text, node); + if ( + ts.isExportDeclaration(node) && + node.exportClause && + ts.isNamedExports(node.exportClause) + ) { + for (const specifier of node.exportClause.elements) + exports.set((specifier.propertyName ?? specifier.name).text, specifier.name.text); + } + if ( + ts.isImportDeclaration(node) && + node.importClause?.namedBindings && + ts.isNamedImports(node.importClause.namedBindings) + ) { + for (const specifier of node.importClause.namedBindings.elements) + imports.push({ + local: specifier.name.text, + exported: (specifier.propertyName ?? specifier.name).text, + url: new URL(node.moduleSpecifier.text, url).href, + }); + } + } + const calls = (node) => + nodesMatching(node, ts.isCallExpression).flatMap(({ expression }) => + ts.isIdentifier(expression) ? [expression.text] : [], + ); + const hasProperty = (node, name) => + nodesMatching(node, ts.isPropertyAccessExpression).some( + (access) => access.name.text === name, + ); + const reaches = (name, target, depth = 2) => + name === target || + (depth > 0 && + functions.has(name) && + calls(functions.get(name)).some((next) => reaches(next, target, depth - 1))); + const normalizers = [...functions].filter( + ([name, node]) => + exports.has(name) && + calls(node).includes("decodeURI") && + nodesMatching(node, ts.isStringLiteralLike).some((literal) => literal.text === "file:"), + ); + const pairs = normalizers.flatMap(([name]) => + [...functions].flatMap(([renderer, node]) => + exports.has(renderer) && + hasProperty(node, "img") && + hasProperty(node, "components") && + reaches(renderer, name) + ? [{ url, normalizer: exports.get(name), imageRenderer: exports.get(renderer) }] + : [], + ), + ); + const consumers = [...functions.values()] + .filter( + (node) => hasProperty(node, "currentFilePath") && hasProperty(node, "contentsBase64"), + ) + .flatMap((node) => imports.filter(({ local }) => calls(node).includes(local))); + return { pairs, consumers }; + }); + const pairs = summaries.flatMap(({ pairs }) => pairs); + if (pairs.length !== 1) return null; + const pair = pairs[0]; + const consumers = summaries + .flatMap(({ consumers }) => consumers) + .filter(({ url, exported }) => url === pair.url && exported === pair.normalizer); + return consumers.length === 1 ? pair : null; +} + +/** No mounting, file reads or network requests: the native token renderer returns a React element. */ +export function replayMediaPaths(nativeModule, contract, paths) { + const unverified = { state: "unverified", reason: "native-media-functions-unavailable" }; + const render = nativeModule[contract.imageRenderer]; + const normalize = nativeModule[contract.normalizer]; + if (typeof render !== "function" || typeof normalize !== "function") return unverified; + try { + const results = paths.map((path) => { + const element = render( + { type: "image", text: "probe", href: path.replaceAll("%", "%2525") }, + "probe", + { components: { img: function ProbeImage() {} } }, + ); + const src = element?.props?.src; + return ( + typeof src === "string" && src === path.replaceAll("%", "%25") && normalize(src) === path + ); + }); + const matchingCases = results.filter(Boolean).length; + return { + state: matchingCases === paths.length ? "pass" : "mismatch", + checkedCases: paths.length, + matchingCases, + }; + } catch { + return unverified; + } +} + +export function mediaPathProbeExpression(contract) { + return `(async () => { + const url = new URL(${JSON.stringify(contract.url)}); + if (url.protocol !== location.protocol || url.host !== location.host) + return { state: 'unverified', reason: 'native-module-origin-mismatch' }; + try { + const native = await import(url.href); + return (${replayMediaPaths.toString()})(native, ${JSON.stringify(contract)}, ${JSON.stringify(SAMPLE_PATHS)}); + } catch { + return { state: 'unverified', reason: 'native-module-import-unavailable' }; + } + })()`; +} diff --git a/tools/renderer-binding/media-path-contract.test.mjs b/tools/renderer-binding/media-path-contract.test.mjs new file mode 100644 index 000000000..783e4a808 --- /dev/null +++ b/tools/renderer-binding/media-path-contract.test.mjs @@ -0,0 +1,167 @@ +import { describe, expect, it } from "vitest"; +import { discoverMediaPathContract, replayMediaPaths } from "./media-path-contract.mjs"; +import { inspectMediaPaths } from "./media-path-probe.mjs"; + +const url = "app://-/assets/shared.js"; +function modules(pathName = "pathParser", imageName = "imageToken") { + return [ + { + url, + source: ` + function ${pathName}(value) { if (value.startsWith('file:')) return new URL(value).pathname; return decodeURI(value); } + function imageSrc(value) { return ${pathName}(value); } + function ${imageName}(token, key, context) { const component = context.components?.img; return { type: component, props: { src: imageSrc(token.href) } }; } + export { ${pathName} as nativePath, ${imageName} as nativeImage }; + `, + }, + { + url: "app://-/assets/media.js", + source: ` + import { nativePath as resolveSource } from './shared.js'; + function loadMedia(props, context) { const path = context.currentFilePath || props.src; const data = context.contentsBase64; return { path: resolveSource(path), data }; } + `, + }, + ]; +} + +describe("native media path contract", () => { + it("discovers native import/export relationships without minified function names", () => { + for (const names of [ + ["pathParser", "imageToken"], + ["p7", "x0"], + ]) { + expect(discoverMediaPathContract(modules(...names))).toEqual({ + url, + normalizer: "nativePath", + imageRenderer: "nativeImage", + }); + } + }); + + it("requires both image token conversion and a unique media loader using the same parser", () => { + expect(discoverMediaPathContract([])).toBeNull(); + expect(discoverMediaPathContract(modules().slice(0, 1))).toBeNull(); + const duplicate = modules(); + duplicate.push({ ...duplicate[1], url: "app://-/assets/another-media.js" }); + expect(discoverMediaPathContract(duplicate)).toBeNull(); + const changed = modules(); + changed[0].source = changed[0].source.replace("return pathParser(value)", "return value"); + expect(discoverMediaPathContract(changed)).toBeNull(); + }); + + it("calls native functions and detects changed normalization stages", () => { + const contract = { normalizer: "path", imageRenderer: "image" }; + const image = (token) => ({ props: { src: decodeURI(token.href) } }); + const paths = ["/probe/%41.png", "/probe/%2F%5C.png", "C:/probe/中文 with space%.png"]; + expect(replayMediaPaths({ path: decodeURI, image }, contract, paths)).toEqual({ + state: "pass", + checkedCases: 3, + matchingCases: 3, + }); + expect(replayMediaPaths({ path: (p) => p, image }, contract, paths).state).toBe("mismatch"); + expect( + replayMediaPaths({ path: (p) => decodeURI(decodeURI(p)), image }, contract, paths.slice(0, 1)) + .state, + ).toBe("mismatch"); + expect(replayMediaPaths({}, contract, paths).state).toBe("unverified"); + expect( + replayMediaPaths( + { + path: decodeURI, + image: () => { + throw Error("changed API"); + }, + }, + contract, + paths, + ).state, + ).toBe("unverified"); + }); + + it("reports missing native structure as unverified without evaluating anything", async () => { + const calls = []; + let parsed; + const client = { + on: (_method, listener) => { + parsed = listener; + return () => {}; + }, + async command(method) { + calls.push(method); + if (method === "Debugger.enable") parsed({ isModule: true, url, scriptId: "1" }); + return { scriptSource: "export const unrelated = true;" }; + }, + }; + expect(await inspectMediaPaths(client, "app://-/index.html")).toEqual({ + state: "unverified", + reason: "native-media-structure-missing-or-ambiguous", + }); + expect(calls).toEqual(["Debugger.enable", "Debugger.getScriptSource"]); + }); + + it("does not evaluate a contract from a Renderer that reloaded during discovery", async () => { + const sources = modules(); + const listeners = new Map(); + let evaluated = false; + const client = { + on: (method, listener) => { + listeners.set(method, listener); + return () => listeners.delete(method); + }, + async command(method, params) { + if (method === "Debugger.enable") + sources.forEach(({ url }, i) => + listeners.get("Debugger.scriptParsed")({ isModule: true, url, scriptId: String(i) }), + ); + if (method === "Debugger.getScriptSource") { + listeners.get("Debugger.globalObjectCleared")(); + return { scriptSource: sources[Number(params.scriptId)].source }; + } + if (method === "Runtime.evaluate") evaluated = true; + return {}; + }, + }; + expect(await inspectMediaPaths(client, "app://-/index.html")).toEqual({ + state: "unverified", + reason: "renderer-changed-during-inspection", + }); + expect(evaluated).toBe(false); + expect(listeners.size).toBe(0); + }); + + it("evaluates only the verified native contract and returns bounded results", async () => { + const sources = modules(); + const calls = []; + let parsed; + const client = { + on: (_method, listener) => { + parsed = listener; + return () => {}; + }, + async command(method, params) { + calls.push(method); + if (method === "Debugger.enable") + sources.forEach(({ url }, i) => parsed({ isModule: true, url, scriptId: String(i) })); + if (method === "Debugger.getScriptSource") + return { scriptSource: sources[Number(params.scriptId)].source }; + if (method === "Runtime.evaluate") { + expect(params.expression).toContain("nativeImage"); + expect(params.expression).not.toMatch(/readBinary|createRoot|appendChild|sendRequest/); + return { result: { value: { state: "pass", checkedCases: 6, matchingCases: 6 } } }; + } + return {}; + }, + }; + expect(await inspectMediaPaths(client, "app://-/index.html")).toEqual({ + state: "pass", + checkedCases: 6, + matchingCases: 6, + }); + expect(calls).toEqual([ + "Debugger.enable", + "Debugger.getScriptSource", + "Debugger.getScriptSource", + "Runtime.evaluate", + ]); + }); +}); diff --git a/tools/renderer-binding/media-path-probe.mjs b/tools/renderer-binding/media-path-probe.mjs new file mode 100644 index 000000000..624b09695 --- /dev/null +++ b/tools/renderer-binding/media-path-probe.mjs @@ -0,0 +1,89 @@ +import { pathToFileURL } from "node:url"; +import { discoverMediaPathContract, mediaPathProbeExpression } from "./media-path-contract.mjs"; + +/** Read only an existing Renderer. Never install policies, reload, mount UI or submit a Turn. */ +export async function inspectMediaPaths(client, documentUrl) { + const page = new URL(documentUrl); + const scripts = new Map(); + let changed = false; + const removeReset = client.on("Debugger.globalObjectCleared", () => { + changed = true; + }); + const remove = client.on("Debugger.scriptParsed", (event) => { + if (event.isModule !== true || event.hasSourceURL === true || typeof event.url !== "string") + return; + try { + const url = new URL(event.url); + if ( + url.protocol === page.protocol && + url.host === page.host && + url.pathname.endsWith(".js") && + !url.search && + !url.hash + ) { + if (scripts.size >= 1024 && !scripts.has(url.href)) changed = true; + else scripts.set(url.href, event.scriptId); + } + } catch { + /* Non-module source URLs are not native candidates. */ + } + }); + try { + await client.command("Debugger.enable"); + if (scripts.size === 0 || scripts.size > 1024) + return { state: "unverified", reason: "loaded-module-catalog-unavailable" }; + const modules = []; + for (const [url, scriptId] of scripts) { + const { scriptSource } = await client.command("Debugger.getScriptSource", { scriptId }); + if (typeof scriptSource !== "string") + return { state: "unverified", reason: "loaded-module-source-unavailable" }; + modules.push({ url, source: scriptSource }); + } + const contract = discoverMediaPathContract(modules); + if (!contract) + return { state: "unverified", reason: "native-media-structure-missing-or-ambiguous" }; + if (changed) return { state: "unverified", reason: "renderer-changed-during-inspection" }; + const result = await client.command("Runtime.evaluate", { + expression: mediaPathProbeExpression(contract), + awaitPromise: true, + returnByValue: true, + }); + return changed || + result.exceptionDetails || + !["pass", "mismatch", "unverified"].includes(result.result?.value?.state) + ? { state: "unverified", reason: "native-media-evaluation-unavailable" } + : result.result.value; + } catch { + return { state: "unverified", reason: "native-media-inspection-unavailable" }; + } finally { + remove(); + removeReset(); + } +} + +async function main() { + const args = process.argv.slice(2); + if (args.length !== 2 || args[0] !== "--endpoint") + throw new Error( + "usage: node tools/renderer-binding/media-path-probe.mjs --endpoint ", + ); + const url = new URL(args[1]); + if (url.protocol !== "http:" || !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname)) + throw new Error("Media path probe requires a loopback HTTP endpoint"); + let client; + try { + const { CdpClient, waitForRendererTarget } = + await import("../../packages/desktop-control/dist/index.js"); + const target = await waitForRendererTarget(url.href, { timeoutMs: 5_000 }); + client = await CdpClient.connect(target.webSocketDebuggerUrl); + const result = await inspectMediaPaths(client, target.url); + console.log(JSON.stringify(result)); + if (result.state === "mismatch") process.exitCode = 1; + } catch { + console.log(JSON.stringify({ state: "unverified", reason: "renderer-unavailable" })); + } finally { + client?.close(); + } +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) await main(); From 17a7326146ed79764f3ffb885289c430746398ef Mon Sep 17 00:00:00 2001 From: BytePioneer-AI <28417416@qq.com> Date: Sun, 1 Nov 2026 21:28:38 +0800 Subject: [PATCH 12/15] docs: clarify the two-stage Desktop path decoding comment --- packages/harness-adapter/src/local-media-markdown.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/harness-adapter/src/local-media-markdown.ts b/packages/harness-adapter/src/local-media-markdown.ts index a7214d592..28f9c1961 100644 --- a/packages/harness-adapter/src/local-media-markdown.ts +++ b/packages/harness-adapter/src/local-media-markdown.ts @@ -372,7 +372,8 @@ function mediaMarkdownImage(absolutePath: string): string { function markdownDestination(absolutePath: string, bracketed: boolean): string { // CommonMark drops a backslash before punctuation (`C:\Users\me\.codex`), and Codex reads // `C:/…` as absolute; other paths keep their backslashes escaped. - // Both Desktop's Markdown token conversion and media loader decode the local path once. + // Desktop decodes the local path in two stages (Markdown token conversion, then media loading), + // so a literal `%` must survive both: `%2525` decodes to `%25`, then to `%`. // Encode only the resolved filesystem path, never a file: URL's own URI escapes. const encoded = absolutePath.replaceAll("%", "%2525"); const destination = /^[A-Za-z]:\\/.test(encoded) From 9be97087e16c9b48e9c843d098abf020508939d7 Mon Sep 17 00:00:00 2001 From: BytePioneer-AI <28417416@qq.com> Date: Sun, 1 Nov 2026 23:08:56 +0800 Subject: [PATCH 13/15] feat: project relative assistant images for external Harnesses --- docs/architecture/external-thread-media.md | 19 +- .../test/local-media-prototype.test.ts | 307 ++++++++++++++++++ .../src/local-media-markdown.ts | 27 +- .../test/local-media-markdown.test.ts | 104 +++++- .../test/delegation-snapshot.test.ts | 61 ++++ .../protocol-core/src/codex-agent-media.ts | 19 ++ .../protocol-core/src/codex-ui-projector.ts | 98 ++++-- 7 files changed, 589 insertions(+), 46 deletions(-) create mode 100644 packages/adapters/cursor-cli/test/local-media-prototype.test.ts create mode 100644 packages/protocol-core/src/codex-agent-media.ts diff --git a/docs/architecture/external-thread-media.md b/docs/architecture/external-thread-media.md index e8c7752aa..ea349ecbc 100644 --- a/docs/architecture/external-thread-media.md +++ b/docs/architecture/external-thread-media.md @@ -6,13 +6,20 @@ Agent 消息中的 Markdown 图片（`![alt](目标)`）只有在目标是绝对路径（`/…`、`C:\…`、`C:/…`、UNC）或 `file:` URL 时，才会被当作本地文件，经 app-server 读取后显示。其他目标按 Desktop 页面自身地址解析，加载失败后显示“Image unavailable”占位框。只有预览 `.md` 文件时，Desktop 才会按该文件所在目录补全相对路径；对话消息没有这个基准目录。 -Harness 常按工作目录写相对路径，因此需要由 Adapter 改写。 +Harness 常按工作目录写相对路径，因此由 Host 共享投影层补全为绝对路径。 用户消息气泡中的缩略图只来自消息 `content` 里的 `image` / `localImage` 条目。Desktop 同时把附件清单（`# Files mentioned by the user:`）解析成文件标签；只有消息带图片条目时，才把对应图片从标签中移除。消息没有图片条目时，所贴图片只显示为文件名标签。 ## Agent 消息：改写本地媒体路径 -`@codexhost/harness-adapter/local-media-markdown` 提供共用的改写： +共享投影层对**所有外部 Harness** 的 `agentMessage` 生效，覆盖实时输出与历史重放，不修改 Adapter 交付的原始文本： + +- 只处理单行显式 Markdown 图片（`![alt](目标)`）中的相对路径，不提升行内代码路径，普通链接保持原样。 +- 唯一基准是 **Thread cwd**，不猜子目录、会话目录或工具命令中的 `cd`。如果图片实际基于另一个目录，可能找不到；若 Thread cwd 下恰有同名文件，也可能显示错图。此规则不保证解析所有原生工作目录下的相对路径。 +- 解析出目的地后，在 resolve、文件检查及 percent 编码之前，跳过 POSIX 绝对路径、盘符、UNC、`file:` 及任何 URL，保持其原文。Grok/DSH Adapter 已改成绝对路径的目标不会再次编码。 +- reasoning、工具输出、官方 Codex Thread 不进入这项改写。delegation 读取既有的已投影可见消息，因此也会读到改写后的 Markdown。 + +`@codexhost/harness-adapter/local-media-markdown` 的 `relativeImagesOnly` 模式提供上述窄范围；默认行为保持不变，供 Grok/DSH Adapter 处理额外目录与行内路径： - 改写单行 Markdown 图片目标中的本地媒体路径，以及只包含一个媒体路径的单反引号行内代码（转成图片）。普通链接保持原样；不搬移标点、不增删段落或周围空白。 - 顺序固定为：相对路径解析或 `fileURLToPath()` → `path.resolve()` → 校验最终目标的扩展名与文件是否存在 → 保护字面 `%` → Markdown destination 格式化。不是先检查原文里 `?` / `#` 前面的伪扩展名，再访问规范化后的另一个文件。 @@ -36,16 +43,18 @@ Desktop 的 Markdown 图片 token 转换器和媒体加载组件各归一化一 本模块不重建 CommonMark 的容器嵌套。在引用块、无序/有序列表前缀后发现围栏，或发现任何缩进/Tab 后的围栏式标记时，从该标记到消息末尾都保持原样；即使后面还有普通媒体路径也不改写，以免错误判断容器结束后修改代码。这是保守支持范围，不是完整 CommonMark 解析器。 -实时输出保持追加式，完成全文必须等于初始文本加所有增量。Grok 与 DSH 均使用 `LocalMediaMarkdownStream`：只暂缓结尾未决的图片、行内路径或可能增长的围栏标记，不等待整行、不等待后续标点。Item 结束前补发暂缓原文；不完整的单反引号路径遇到换行后不再扣住后面的文字。同一条消息内缓存每个候选路径的存在性检查，包括未找到的结果，避免文件出现或根目录命中变化导致已发布文本需要撤回；新的消息使用新的缓存。 +实时输出保持追加式，原始完成全文必须等于原始初始文本加所有增量。共享投影层为每个 Agent Item 创建独立的 `LocalMediaMarkdownStream`，不把显示文本写回原始账本；显示完成文本同样等于已发布的初始显示文本加增量。pending、terminal 和 final-answer phase replay 读取缓存显示文本，不重新检查文件；历史用同一 stream 消费全文并 flush。 + +共享投影层、Grok 与 DSH 均使用 `LocalMediaMarkdownStream`：只暂缓结尾未决的图片、代码跨度或可能增长的围栏标记，不等待整行、不等待后续标点。Item 结束前补发暂缓原文；不完整的单反引号路径遇到换行后不再扣住后面的文字。同一条消息内缓存每个候选路径的存在性检查，包括未找到的结果，避免文件出现或根目录命中变化导致已发布文本需要撤回；新的消息使用新的缓存。文件变化后历史重载的显示结果可能不同。每次增量仍会重扫累计文本，超长回复与细粒度增量会增加 CPU 开销。 -接入的 Harness： +Grok/DSH 的额外目录仍由各自 Adapter 处理，优先于共享层： | Harness | 根目录 | 范围 | | --- | --- | --- | | Grok | 工作目录、Grok 会话目录及其 `videos/`、`images/`、用户 `Downloads/` | 实时输出与历史 | | DeepSeek Harness | 消息产生时 Agent 的工作目录（按事件顺序随 `working-directory/change` 更新，恢复未完成的 Turn 时也一样），再到 Session 工作目录 | 实时输出与历史（`readSnapshot`）；用量刷新不做改写 | -其他 Harness 的 Agent 文本仍原样转发；它们输出相对路径时，图片仍显示为占位框。 +其他外部 Harness 不需要专属媒体接入代码，统一使用 Thread cwd 的相对图片规则。远程或容器文件系统与 Host 不一致时，不保证可用；SSH 媒体读取链及 Windows 实际文件系统行为仍需平台验证。 ## 用户消息：还原图片附件 diff --git a/packages/adapters/cursor-cli/test/local-media-prototype.test.ts b/packages/adapters/cursor-cli/test/local-media-prototype.test.ts new file mode 100644 index 000000000..731a48c5c --- /dev/null +++ b/packages/adapters/cursor-cli/test/local-media-prototype.test.ts @@ -0,0 +1,307 @@ +import type { SessionNotification } from "@agentclientprotocol/sdk"; +import type { HostAgentMessageItem, HostEvent } from "@codexhost/harness-adapter"; +import { + CodexTurnProjector, + projectHistoricalTurn, + type ProjectableHostEvent, +} from "@codexhost/protocol-core"; +import { hostItemIdSchema, hostTurnIdSchema, type JsonObject } from "@codexhost/shared-contracts"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { CursorTurnOutput, cursorSnapshot } from "../src/projection.js"; + +describe("shared relative agent media projection", () => { + let cwd: string; + const turnId = hostTurnIdSchema.parse("media-prototype"); + const itemId = hostItemIdSchema.parse("message"); + const sessionId = "00000000-0000-4000-8000-000000000001"; + const notification = (text: string): SessionNotification => ({ + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text }, + }, + }); + const destination = (file: string) => + path.join(cwd, file).replaceAll("\\", "/").replaceAll("%", "%2525"); + const newProjector = () => + new CodexTurnProjector({ threadId: "external", turnId, cwd, startedAtMs: 1 }); + const item = (text: string): HostAgentMessageItem => ({ type: "agentMessage", itemId, text }); + const complete = (projector: CodexTurnProjector, message: HostAgentMessageItem) => + projector.project({ + type: "item.completed", + turnId, + snapshot: { item: message, outcome: { status: "succeeded" } }, + }); + + beforeEach(async () => { + cwd = await mkdtemp(path.join(os.tmpdir(), "host-media%41-")); + await writeFile(path.join(cwd, "shot.png"), "synthetic image"); + }); + afterEach(async () => { + await rm(cwd, { recursive: true, force: true }); + }); + + it("projects real Cursor text.append output and native replay without changing the native ledger", () => { + const projector = newProjector(); + const wire: JsonObject[] = []; + const native: HostEvent[] = []; + wire.push(...projector.project({ type: "turn.started", turnId }).messages); + const output = new CursorTurnOutput(turnId, (event) => { + native.push(structuredClone(event)); + wire.push(...projector.project(event as ProjectableHostEvent).messages); + }); + const first = notification("见 ![shot]()。"); + const chunks = [first, second]; + output.update(first); + expect(projector.pendingTurn().items).toMatchObject([ + { type: "agentMessage", text: "见 ![shot](" }, + ]); + output.update(second); + output.finish({ status: "succeeded" }); + const replay = cursorSnapshot( + sessionId, + [{ id: "native-turn", text: "show" }], + [ + { + sessionId, + update: { sessionUpdate: "user_message_chunk", content: { type: "text", text: "show" } }, + }, + ...chunks, + ], + ); + const snapshot = replay.turns[0]; + if (!snapshot) throw new Error("Expected a native Cursor Turn"); + const terminal = projector.project({ + type: "turn.completed", + turnId, + nativeTurnRef: snapshot.nativeTurnRef, + outcome: { status: "succeeded" }, + }); + wire.push(...terminal.messages); + const expected = `见 ![shot](<${destination("shot.png")}>)。`; + expect( + native + .filter((e) => e.type === "item.updated") + .map((e) => (e.update.type === "text.append" ? e.update.text : "")) + .join(""), + ).toBe("见 ![shot]()。"); + expect(snapshot.items[0]?.item).toMatchObject({ + type: "agentMessage", + text: "见 ![shot]()。", + }); + expect( + wire + .filter((e) => e.method === "item/agentMessage/delta") + .map((e) => (e.params as JsonObject).delta) + .join(""), + ).toBe(expected); + expect(terminal.completedTurn?.items).toMatchObject([ + { type: "agentMessage", text: expected, phase: "final_answer" }, + ]); + for (const message of wire.filter((e) => e.method === "item/completed")) { + expect((message.params as JsonObject).item).toMatchObject({ + type: "agentMessage", + text: expected, + }); + } + expect(projectHistoricalTurn({ turnId, cwd, snapshot }).items).toMatchObject([ + { type: "userMessage" }, + { type: "agentMessage", text: expected, phase: "final_answer" }, + ]); + }); + + it("covers full-text starts and caches the display through pending, terminal and phase replay", async () => { + const projector = newProjector(); + projector.project({ type: "turn.started", turnId }); + const native = item("![shot](shot.png)"); + const started = projector.project({ type: "item.started", turnId, item: native }); + expect((started.messages[0]?.params as JsonObject).item).toMatchObject({ + text: `![shot](${destination("shot.png")})`, + }); + expect(projector.pendingTurn().items).toMatchObject([ + { text: `![shot](${destination("shot.png")})` }, + ]); + const completed = complete(projector, native); + expect((completed.messages[0]?.params as JsonObject).item).toMatchObject({ + text: `![shot](${destination("shot.png")})`, + }); + await rm(path.join(cwd, "shot.png")); + expect(projector.pendingTurn().items).toMatchObject([ + { text: `![shot](${destination("shot.png")})` }, + ]); + const terminal = projector.project({ + type: "turn.completed", + turnId, + outcome: { status: "succeeded" }, + }); + expect(terminal.completedTurn?.items).toMatchObject([ + { text: `![shot](${destination("shot.png")})`, phase: "final_answer" }, + ]); + expect((terminal.messages[0]?.params as JsonObject).item).toMatchObject({ + text: `![shot](${destination("shot.png")})`, + }); + expect(native).toEqual(item("![shot](shot.png)")); + }); + + it("flushes unfinished native text before item/completed and final-answer replay", () => { + const projector = newProjector(); + projector.project({ type: "turn.started", turnId }); + projector.project({ type: "item.started", turnId, item: item("") }); + const raw = "![shot](shot.png) and `unfinished"; + const updates = projector.project({ + type: "item.updated", + turnId, + itemId, + update: { type: "text.append", text: raw }, + }); + const completed = complete(projector, item(raw)); + expect(completed.messages[0]).toMatchObject({ + method: "item/agentMessage/delta", + params: { delta: "`unfinished" }, + }); + const expected = `![shot](${destination("shot.png")}) and \`unfinished`; + expect( + [...updates.messages, ...completed.messages] + .filter((m) => m.method === "item/agentMessage/delta") + .map((m) => (m.params as JsonObject).delta) + .join(""), + ).toBe(expected); + const terminal = projector.project({ + type: "turn.completed", + turnId, + outcome: { status: "succeeded" }, + }); + expect((terminal.messages[0]?.params as JsonObject).item).toMatchObject({ + text: expected, + phase: "final_answer", + }); + }); + + it("still rejects native completion mismatches and recovers using native failure events", () => { + const projector = newProjector(); + projector.project({ type: "turn.started", turnId }); + projector.project({ type: "item.started", turnId, item: item("") }); + const raw = "![shot](shot.png) `unfinished"; + projector.project({ + type: "item.updated", + turnId, + itemId, + update: { type: "text.append", text: raw }, + }); + expect(() => complete(projector, item("different"))).toThrow( + "completion does not match its append updates", + ); + expect(() => + complete(projector, item(raw.replace("shot.png", destination("shot.png")))), + ).toThrow("completion does not match its append updates"); + const error = { code: "protocolError" as const, message: "fixture failed", retryable: false }; + const events = projector.failureEvents(error); + expect(events[0]).toMatchObject({ type: "item.completed", snapshot: { item: { text: raw } } }); + const wire = events.flatMap((event) => projector.project(event).messages); + expect(wire).toContainEqual( + expect.objectContaining({ + method: "item/completed", + params: expect.objectContaining({ + item: expect.objectContaining({ + text: `![shot](${destination("shot.png")}) \`unfinished`, + }), + }), + }), + ); + }); + + it("uses only Thread cwd, without guessing directories or promoting code and ordinary links", async () => { + const extra = path.join(cwd, "native-session"); + await mkdir(extra); + await writeFile(path.join(extra, "shot.png"), "different image outside the chosen cwd"); + await writeFile(path.join(extra, "nested.png"), "not searched recursively"); + const projector = newProjector(); + projector.project({ type: "turn.started", turnId }); + const native = item("![shot](shot.png) ![nested](nested.png) `shot.png` [link](shot.png)"); + projector.project({ type: "item.started", turnId, item: native }); + complete(projector, native); + expect(projector.pendingTurn().items).toMatchObject([ + { + text: `![shot](${destination("shot.png")}) ![nested](nested.png) \`shot.png\` [link](shot.png)`, + }, + ]); + }); + + it("keeps filesystem decisions stable within an Item, not across new Items or reloads", async () => { + const projector = newProjector(); + projector.project({ type: "turn.started", turnId }); + projector.project({ type: "item.started", turnId, item: item("") }); + const reference = "![later](later.png) "; + projector.project({ + type: "item.updated", + turnId, + itemId, + update: { type: "text.append", text: reference }, + }); + await writeFile(path.join(cwd, "later.png"), "appeared later"); + projector.project({ + type: "item.updated", + turnId, + itemId, + update: { type: "text.append", text: reference }, + }); + complete(projector, item(reference.repeat(2))); + const next = { ...item(reference), itemId: hostItemIdSchema.parse("next") }; + projector.project({ type: "item.started", turnId, item: next }); + complete(projector, next); + expect(projector.pendingTurn().items).toMatchObject([ + { text: reference.repeat(2) }, + { text: `![later](${destination("later.png")}) ` }, + ]); + const native = cursorSnapshot( + sessionId, + [{ id: "native-turn", text: "show" }], + [ + { + sessionId, + update: { sessionUpdate: "user_message_chunk", content: { type: "text", text: "show" } }, + }, + notification(reference.repeat(2)), + ], + ); + const snapshot = native.turns[0]; + if (!snapshot) throw new Error("Expected a native Cursor Turn"); + expect(projectHistoricalTurn({ turnId, cwd, snapshot }).items).toContainEqual( + expect.objectContaining({ text: `![later](${destination("later.png")}) `.repeat(2) }), + ); + }); + + it("does not rewrite reasoning previews or transcripts", () => { + const projector = newProjector(); + projector.project({ type: "turn.started", turnId }); + const text = "![shot](shot.png)"; + const reasoningId = hostItemIdSchema.parse("reasoning"); + projector.project({ + type: "item.started", + turnId, + item: { type: "reasoning", itemId: reasoningId, text: "" }, + }); + const update = projector.project({ + type: "item.updated", + turnId, + itemId: reasoningId, + update: { type: "text.append", text }, + }); + expect(update.messages).toContainEqual( + expect.objectContaining({ + method: "item/commandExecution/outputDelta", + params: expect.objectContaining({ delta: text }), + }), + ); + }); + + it("does not invent a start for an invalid bare item.completed event", () => { + const projector = newProjector(); + projector.project({ type: "turn.started", turnId }); + expect(() => complete(projector, item("![shot](shot.png)"))).toThrow("unknown Item"); + }); +}); diff --git a/packages/harness-adapter/src/local-media-markdown.ts b/packages/harness-adapter/src/local-media-markdown.ts index 28f9c1961..02dfc00b4 100644 --- a/packages/harness-adapter/src/local-media-markdown.ts +++ b/packages/harness-adapter/src/local-media-markdown.ts @@ -25,6 +25,8 @@ export type LocalMediaExists = (absolutePath: string) => boolean; export interface RewriteLocalMediaMarkdownOptions { exists?: LocalMediaExists; + /** Only rewrite relative Markdown image destinations; leave code spans and absolute/URL targets alone. */ + relativeImagesOnly?: boolean; /** * Hold back a trailing construct that later text could still complete or invalidate * (an open image or inline code span), so a growing text projects prefix-stably. @@ -49,6 +51,7 @@ export function rewriteLocalMediaMarkdown( roots, options.exists ?? mediaFileExists, options.holdIncomplete === true, + options.relativeImagesOnly === true, ).text; } @@ -60,14 +63,20 @@ export function rewriteLocalMediaMarkdown( export class LocalMediaMarkdownStream { readonly #roots: readonly string[]; readonly #exists: LocalMediaExists; + readonly #relativeImagesOnly: boolean; #native = ""; #text = ""; /** Native text length represented by `#text`; the rest is held. */ #consumed = 0; #verbatim = false; - constructor(roots: readonly string[], exists: LocalMediaExists = mediaFileExists) { + constructor( + roots: readonly string[], + exists: LocalMediaExists = mediaFileExists, + options: { relativeImagesOnly?: boolean } = {}, + ) { this.#roots = [...roots]; + this.#relativeImagesOnly = options.relativeImagesOnly === true; const known = new Map(); this.#exists = (absolutePath) => { let found = known.get(absolutePath); @@ -98,7 +107,13 @@ export class LocalMediaMarkdownStream { #advance(holdIncomplete: boolean): string { const next = this.#verbatim ? undefined - : projectLocalMedia(this.#native, this.#roots, this.#exists, holdIncomplete); + : projectLocalMedia( + this.#native, + this.#roots, + this.#exists, + holdIncomplete, + this.#relativeImagesOnly, + ); let delta: string; if (next?.text.startsWith(this.#text)) { delta = next.text.slice(this.#text.length); @@ -125,6 +140,7 @@ function projectLocalMedia( roots: readonly string[], exists: LocalMediaExists, holdIncomplete: boolean, + relativeImagesOnly: boolean, ): LocalMediaProjection { let output = ""; let index = 0; @@ -184,7 +200,10 @@ function projectLocalMedia( if (!holdIncomplete) break; return { text: output + text.slice(next, image.holdFrom), consumed: image.holdFrom }; } - const resolved = resolveLocalMediaPath(image.destination, roots, exists); + const resolved = + relativeImagesOnly && /^(?:[\\/]|[a-z][a-z\d+.-]*:)/i.test(image.destination.trim()) + ? null + : resolveLocalMediaPath(image.destination, roots, exists); output += resolved === null ? text.slice(next, image.end) @@ -204,7 +223,7 @@ function projectLocalMedia( if (!holdIncomplete) break; return { text: output, consumed: next }; } - const resolved = resolveLocalMediaPath(code.content, roots, exists); + const resolved = relativeImagesOnly ? null : resolveLocalMediaPath(code.content, roots, exists); output += resolved === null ? text.slice(next, code.end) : mediaMarkdownImage(resolved); index = code.end; } diff --git a/packages/harness-adapter/test/local-media-markdown.test.ts b/packages/harness-adapter/test/local-media-markdown.test.ts index 49de64413..1673931de 100644 --- a/packages/harness-adapter/test/local-media-markdown.test.ts +++ b/packages/harness-adapter/test/local-media-markdown.test.ts @@ -160,8 +160,8 @@ describe("LocalMediaMarkdownStream", () => { ...containerCode, ]; - function streamed(text: string, cuts: readonly number[]): string { - const stream = new LocalMediaMarkdownStream(roots, exists); + function streamed(text: string, cuts: readonly number[], relativeImagesOnly = false): string { + const stream = new LocalMediaMarkdownStream(roots, exists, { relativeImagesOnly }); let start = 0; let result = ""; for (const end of [...cuts, text.length]) { @@ -174,19 +174,25 @@ describe("LocalMediaMarkdownStream", () => { } it("matches one-shot output at every pair of cuts and for character-sized chunks", () => { - for (const sample of samples) { - const expected = rewrite(sample); - for (let first = 0; first <= sample.length; first += 1) { - for (let second = first; second <= sample.length; second += 1) { - expect(streamed(sample, [first, second]), `${sample} @${first},${second}`).toBe(expected); + for (const relativeImagesOnly of [false, true]) { + for (const sample of samples) { + const expected = rewriteLocalMediaMarkdown(sample, roots, { exists, relativeImagesOnly }); + for (let first = 0; first <= sample.length; first += 1) { + for (let second = first; second <= sample.length; second += 1) { + expect( + streamed(sample, [first, second], relativeImagesOnly), + `${sample} @${first},${second}`, + ).toBe(expected); + } } + expect( + streamed( + sample, + Array.from({ length: sample.length }, (_, i) => i), + relativeImagesOnly, + ), + ).toBe(expected); } - expect( - streamed( - sample, - Array.from({ length: sample.length }, (_, i) => i), - ), - ).toBe(expected); } }); @@ -214,6 +220,78 @@ describe("LocalMediaMarkdownStream", () => { }); }); +describe("relative Markdown images only", () => { + it("publishes ordinary text immediately and flushes an unfinished image without losing bytes", () => { + const stream = new LocalMediaMarkdownStream(roots, exists, { relativeImagesOnly: true }); + expect(stream.append("ordinary text ")).toBe("ordinary text "); + expect(stream.append("![shot](sho")).toBe("![shot]("); + expect(stream.flush()).toBe("sho"); + expect(stream.text).toBe("ordinary text ![shot](sho"); + }); + + it("rewrites only relative image destinations, leaving links, code and missing files alone", () => { + const text = "![a](./a.png) [link](a.png) `a.png` `![code](a.png)` ![missing](missing.png)"; + expect(rewriteLocalMediaMarkdown(text, roots, { exists, relativeImagesOnly: true })).toBe( + `![a](${at("a.png")}) [link](a.png) \`a.png\` \`![code](a.png)\` ![missing](missing.png)`, + ); + }); + + it("skips absolute, drive, UNC and all URI targets before checking the filesystem", () => { + const check = vi.fn(() => true); + const targets = [ + path.join(workspace, "a.png"), + "/var/x%41.png", + "C:/x%41.png", + String.raw`C:\images\a.png`, + String.raw`\\server\share\a.png`, + String.raw`\images\a.png`, + "//server/share/a.png", + "file:///tmp/a%20b.png", + "FILE:///%ZZ.png", + "https://example.com/a.png", + "ftp://host/a.png", + "data:image/png;base64,AA", + "blob:opaque.png", + "app:a.png", + "custom+v1://host/a.png", + "mailto:a.png", + ]; + for (const target of targets) { + const text = `![x](<${target}>)`; + expect( + rewriteLocalMediaMarkdown(text, roots, { exists: check, relativeImagesOnly: true }), + ).toBe(text); + const stream = new LocalMediaMarkdownStream(roots, check, { relativeImagesOnly: true }); + expect([...text].map((c) => stream.append(c)).join("") + stream.flush()).toBe(text); + } + expect(check).not.toHaveBeenCalled(); + }); + + it("does not encode an Adapter's absolute output again when a protected-name file also exists", () => { + const check = (p: string) => + ["x%41.png", "x%252541.png"].some((name) => p === path.join(workspace, name)); + const raw = "![x](x%41.png)"; + const expected = `![x](${at("x%252541.png")})`; + expect(rewriteLocalMediaMarkdown(raw, roots, { exists: check })).toBe(expected); + expect(rewriteLocalMediaMarkdown(raw, roots, { exists: check, relativeImagesOnly: true })).toBe( + expected, + ); + expect( + rewriteLocalMediaMarkdown(expected, roots, { exists: check, relativeImagesOnly: true }), + ).toBe(expected); + const adapter = new LocalMediaMarkdownStream(roots, check); + const outerCheck = vi.fn(check); + const projection = new LocalMediaMarkdownStream(roots, outerCheck, { + relativeImagesOnly: true, + }); + let text = ""; + for (const c of raw) text += projection.append(adapter.append(c)); + text += projection.append(adapter.flush()) + projection.flush(); + expect(text).toBe(expected); + expect(outerCheck).not.toHaveBeenCalled(); + }); +}); + describe("root selection during streaming", () => { it("does not change an earlier root decision when a higher-priority file appears", () => { const higher = path.join(workspace, "higher"); diff --git a/packages/host-runtime/test/delegation-snapshot.test.ts b/packages/host-runtime/test/delegation-snapshot.test.ts index 4291ade0d..50f668c0a 100644 --- a/packages/host-runtime/test/delegation-snapshot.test.ts +++ b/packages/host-runtime/test/delegation-snapshot.test.ts @@ -1,3 +1,12 @@ +import { projectHistoricalTurn } from "@codexhost/protocol-core"; +import { + hostItemIdSchema, + hostTurnIdSchema, + nativeTurnRefSchema, +} from "@codexhost/shared-contracts"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import { describe, expect, it } from "vitest"; import { projectDelegationThreadSnapshot } from "../src/delegation-snapshot.js"; @@ -27,6 +36,58 @@ const completedTurns = [ ]; describe("delegation snapshot", () => { + it("passes projected relative-image Markdown through result and message views", async () => { + const cwd = await mkdtemp(path.join(os.tmpdir(), "delegation-media%41-")); + try { + await writeFile(path.join(cwd, "shot.png"), "synthetic image"); + const turn = projectHistoricalTurn({ + turnId: hostTurnIdSchema.parse("turn-1"), + cwd, + snapshot: { + nativeTurnRef: nativeTurnRefSchema.parse({ + formatVersion: 1, + harnessId: "pi", + nativeSessionId: "session", + nativeTurnKey: "1", + }), + input: [], + outcome: { status: "succeeded" }, + items: [ + { + item: { + type: "agentMessage", + itemId: hostItemIdSchema.parse("answer"), + text: "![shot](shot.png)", + }, + outcome: { status: "succeeded" }, + }, + ], + }, + }); + const text = `![shot](${path.join(cwd, "shot.png").replaceAll("\\", "/").replaceAll("%", "%2525")})`; + for (const view of ["result", "messages"] as const) { + const snapshot = projectDelegationThreadSnapshot({ + threadId: "thread-1", + harnessId: "pi", + thread: {}, + turns: [turn], + running: false, + view, + }); + expect(snapshot).toMatchObject({ + status: "completed", + result: { availability: "available", text }, + }); + if (view === "messages") + expect(snapshot.messages).toEqual([ + { id: "answer", turnId: "turn-1", role: "agent", phase: "final", text }, + ]); + } + } finally { + await rm(cwd, { recursive: true, force: true }); + } + }); + it("returns only visible progress and the final Agent result by default", () => { const snapshot = projectDelegationThreadSnapshot({ threadId: "thread-1", diff --git a/packages/protocol-core/src/codex-agent-media.ts b/packages/protocol-core/src/codex-agent-media.ts new file mode 100644 index 000000000..a67556600 --- /dev/null +++ b/packages/protocol-core/src/codex-agent-media.ts @@ -0,0 +1,19 @@ +import type { HostAgentMessageItem } from "@codexhost/harness-adapter"; +import { LocalMediaMarkdownStream } from "@codexhost/harness-adapter/local-media-markdown"; + +/** Display-only state; the projector keeps native text separately for completion validation. */ +export function createAgentMedia(text: string, cwd: string): LocalMediaMarkdownStream { + const media = new LocalMediaMarkdownStream([cwd], undefined, { relativeImagesOnly: true }); + media.append(text); + return media; +} + +/** History uses the same rules and per-Item existence cache as live projection. */ +export function projectAgentMessageMedia( + item: HostAgentMessageItem, + cwd: string, +): HostAgentMessageItem { + const media = createAgentMedia(item.text, cwd); + media.flush(); + return { ...item, text: media.text }; +} diff --git a/packages/protocol-core/src/codex-ui-projector.ts b/packages/protocol-core/src/codex-ui-projector.ts index e9ac5f43a..03801e478 100644 --- a/packages/protocol-core/src/codex-ui-projector.ts +++ b/packages/protocol-core/src/codex-ui-projector.ts @@ -42,6 +42,7 @@ import { type CodexQuestionRequestProjection, } from "./codex-question.js"; import { inferredFinalAnswer } from "./final-answer-phase.js"; +import { createAgentMedia, projectAgentMessageMedia } from "./codex-agent-media.js"; export type ProjectableHostEvent = | TurnStartedEvent @@ -74,7 +75,9 @@ export interface HistoricalTurnProjectionInput { } interface ProjectedItem { + /** Native text is authoritative for validation and failure recovery. */ item: HostItem; + media?: ReturnType; outcome: HostItemOutcome | null; reasoningPartStarted: boolean; streamedCommandOutput: boolean; @@ -706,7 +709,15 @@ export function projectHistoricalTurn(input: HistoricalTurnProjectionInput): Jso projectItem(item, outcome, cwd, true, input.threadId ?? ""), projectReasoningTranscriptItem(item, outcome, cwd), ] - : [projectItem(item, outcome, cwd, true, input.threadId ?? "")]; + : [ + projectItem( + item.type === "agentMessage" ? projectAgentMessageMedia(item, cwd) : item, + outcome, + cwd, + true, + input.threadId ?? "", + ), + ]; }; const entries = snapshot.items.map((entry) => ({ entry, projected: projectEntry(entry) })); const lastVisible = entries.findLast(({ projected }) => projected.length > 0)?.entry; @@ -726,7 +737,9 @@ export function projectHistoricalTurn(input: HistoricalTurnProjectionInput): Jso }, ...entries.flatMap(({ entry, projected }) => finalAnswer && entry === lastVisible - ? projectEntry({ ...entry, item: finalAnswer }) + ? projected.map((item) => + item.type === "agentMessage" ? { ...item, phase: finalAnswer.phase ?? null } : item, + ) : projected, ), ], @@ -839,7 +852,7 @@ export class CodexTurnProjector { if (!projected?.wireStarted) return []; if (projected.item.type === "userMessage") return [this.#projectItem(projected.item)]; if (projected.item.type === "agentMessage") { - return [projectItem(projected.item, projected.outcome, this.#cwd)]; + return [this.#projectItem(projected.item, projected.outcome)]; } return itemId === this.#fileItemId ? [projectItem(this.#fileSummary(), null, this.#cwd)] @@ -1026,7 +1039,10 @@ export class CodexTurnProjector { this.#requireStarted(); if (this.#items.has(event.item.itemId)) throw new Error("Host Item started more than once"); const projected: ProjectedItem = { - item: event.item, + item: { ...event.item }, + ...(event.item.type === "agentMessage" + ? { media: createAgentMedia(event.item.text, this.#cwd) } + : {}), outcome: null, reasoningPartStarted: false, streamedCommandOutput: false, @@ -1038,7 +1054,7 @@ export class CodexTurnProjector { (event.item.type === "agentMessage" || event.item.type === "reasoning") && (event.item.type === "reasoning" ? reasoningDisplayText(event.item.text).length === 0 - : event.item.text.length === 0) + : this.#agentMedia(event.item.itemId).text.length === 0) ) { // An empty Reasoning Item would surface as a transcript card with no // content, so defer the wire Item until real summary text arrives. @@ -1056,7 +1072,8 @@ export class CodexTurnProjector { if (isFileMutatingTool(event.item.toolName)) return { messages: [] }; } if (event.item.type === "fileChange") return { messages: this.#fileChangeUpdates(startedAtMs) }; - const startedItem = event.item.type === "reasoning" ? { ...event.item, text: "" } : event.item; + const startedItem = + event.item.type === "reasoning" ? { ...event.item, text: "" } : this.#mediaItem(event.item); const messages = [this.#startWireItem(projected, startedItem, startedAtMs)]; if (event.item.type === "reasoning") { const text = reasoningDisplayText(event.item.text); @@ -1078,19 +1095,12 @@ export class CodexTurnProjector { if (event.update.type === "text.append") { if (event.update.text.length === 0) return { messages }; if (next.type === "agentMessage") { - if (!projected.wireStarted) { - messages.push(this.#startWireItem(projected, previous, emittedAtMs)); - } - messages.push({ - method: "item/agentMessage/delta", - emittedAtMs, - params: { - threadId: this.#threadId, - turnId: this.#turnId, - itemId: event.itemId, - delta: event.update.text, - }, - }); + const before = this.#mediaItem(previous); + const delta = this.#agentMedia(event.itemId).append(event.update.text); + if (!delta) return { messages }; + if (!projected.wireStarted) + messages.push(this.#startWireItem(projected, before, emittedAtMs)); + messages.push(this.#agentDelta(event.itemId, delta, emittedAtMs)); } else if (next.type === "reasoning") { const previousText = previous.type === "reasoning" ? reasoningDisplayText(previous.text) : ""; @@ -1180,7 +1190,17 @@ export class CodexTurnProjector { throw new Error("Host textual Item completion does not match its append updates"); } } - projected.item = event.snapshot.item; + // Validate against native text above before flushing the independent Desktop view. + const mediaMessages: JsonObject[] = []; + const before = this.#mediaItem(projected.item); + const delta = + projected.item.type === "agentMessage" ? this.#agentMedia(projected.item.itemId).flush() : ""; + if (delta) { + if (!projected.wireStarted) + mediaMessages.push(this.#startWireItem(projected, before, emittedAtMs)); + mediaMessages.push(this.#agentDelta(projected.item.itemId, delta, emittedAtMs)); + } + projected.item = { ...event.snapshot.item }; projected.outcome = event.snapshot.outcome; const startedAtMs = projected.startedAtMs; if (startedAtMs === undefined) throw new Error("Codex Item completed without a start time"); @@ -1212,6 +1232,7 @@ export class CodexTurnProjector { return { messages: [] }; } const messages = [ + ...mediaMessages, completedItem( this.#projectItem(projected.item, projected.outcome, !projected.streamedCommandOutput), ), @@ -1293,7 +1314,7 @@ export class CodexTurnProjector { } if (projected.item.type === "userMessage") return [this.#projectItem(projected.item)]; if (projected.item.type === "agentMessage") { - return [projectItem(projected.item, projected.outcome, this.#cwd)]; + return [this.#projectItem(projected.item, projected.outcome)]; } return itemId === this.#fileItemId ? [projectItem(this.#fileSummary(), { status: "succeeded" }, this.#cwd)] @@ -1359,7 +1380,7 @@ export class CodexTurnProjector { turnId: this.#turnId, startedAtMs, completedAtMs: startedAtMs + (projected.durationMs ?? 0), - item: projectItem(projected.item, projected.outcome, this.#cwd), + item: this.#projectItem(projected.item, projected.outcome), }, }; } @@ -1389,7 +1410,10 @@ export class CodexTurnProjector { threadId: this.#threadId, turnId: this.#turnId, startedAtMs, - item: this.#projectItem(item, null), + item: + item.type === "agentMessage" + ? projectItem(item, null, this.#cwd, true, this.#threadId) + : this.#projectItem(item, null), }, }; } @@ -1401,7 +1425,33 @@ export class CodexTurnProjector { ): JsonObject { return item.type === "userMessage" ? projectUserMessage(item, this.#userMessageClientIds.get(item.itemId) ?? null) - : projectItem(item, outcome, this.#cwd, includeCommandOutput, this.#threadId); + : projectItem( + this.#mediaItem(item), + outcome, + this.#cwd, + includeCommandOutput, + this.#threadId, + ); + } + + #agentMedia(itemId: HostItemId): ReturnType { + const media = this.#items.get(itemId)?.media; + if (!media) throw new Error("Agent Item media projection state is missing"); + return media; + } + + #mediaItem(item: HostItem): HostItem { + return item.type === "agentMessage" + ? { ...item, text: this.#agentMedia(item.itemId).text } + : item; + } + + #agentDelta(itemId: HostItemId, delta: string, emittedAtMs: number): JsonObject { + return { + method: "item/agentMessage/delta", + emittedAtMs, + params: { threadId: this.#threadId, turnId: this.#turnId, itemId, delta }, + }; } #startReasoningTranscript( From b79179c0c5b972e4f2eaf9cc03bafb5b790c727a Mon Sep 17 00:00:00 2001 From: BytePioneer-AI <28417416@qq.com> Date: Sun, 1 Nov 2026 23:46:24 +0800 Subject: [PATCH 14/15] fix: preserve code spans and image-only attachments --- docs/architecture/external-thread-media.md | 6 +- .../src/local-media-markdown.ts | 46 ++++---- .../test/local-media-markdown.test.ts | 26 +++++ .../test/codex-local-media.test.ts | 110 ++++++++++++++++++ .../test/codex-ui-projector.test.ts | 7 +- .../test/renderer-remote-attachments.test.ts | 25 ++++ .../src/desktop-attachments.ts | 9 +- .../test/desktop-attachments.test.ts | 20 +++- 8 files changed, 214 insertions(+), 35 deletions(-) create mode 100644 packages/protocol-core/test/codex-local-media.test.ts diff --git a/docs/architecture/external-thread-media.md b/docs/architecture/external-thread-media.md index ea349ecbc..94edca638 100644 --- a/docs/architecture/external-thread-media.md +++ b/docs/architecture/external-thread-media.md @@ -21,7 +21,7 @@ Harness 常按工作目录写相对路径，因此由 Host 共享投影层补全 `@codexhost/harness-adapter/local-media-markdown` 的 `relativeImagesOnly` 模式提供上述窄范围；默认行为保持不变，供 Grok/DSH Adapter 处理额外目录与行内路径： -- 改写单行 Markdown 图片目标中的本地媒体路径，以及只包含一个媒体路径的单反引号行内代码（转成图片）。普通链接保持原样；不搬移标点、不增删段落或周围空白。 +- 改写单行 Markdown 图片目标中的本地媒体路径，以及只包含一个媒体路径的单反引号单行代码（转成图片）。反引号代码跨度按等长反引号串闭合，多反引号或跨行代码跨度内的内容保持原样。普通链接保持原样；不搬移标点、不增删段落或周围空白。 - 顺序固定为：相对路径解析或 `fileURLToPath()` → `path.resolve()` → 校验最终目标的扩展名与文件是否存在 → 保护字面 `%` → Markdown destination 格式化。不是先检查原文里 `?` / `#` 前面的伪扩展名，再访问规范化后的另一个文件。 - 相对路径按 Adapter 给出的 `roots` 依次查找。**roots 是查找顺序，不是访问沙箱**：`..`、绝对路径与有效 `file:` URL 可以指向根目录外；文件检查会跟随符号链接。扩展名校验不等于文件内容类型校验，也不替代 Harness 自己的访问策略。 - 远程 URL、`data:` 等地址、找不到的文件和无法解析的写法保持原样。支持的扩展名：图片 `png`、`jpg`、`jpeg`、`gif`、`webp`、`svg`，音视频 `mp4`、`m4v`、`mov`、`webm`、`mp3`、`m4a`、`aac`、`ogg`、`wav`。 @@ -45,7 +45,7 @@ Desktop 的 Markdown 图片 token 转换器和媒体加载组件各归一化一 实时输出保持追加式，原始完成全文必须等于原始初始文本加所有增量。共享投影层为每个 Agent Item 创建独立的 `LocalMediaMarkdownStream`，不把显示文本写回原始账本；显示完成文本同样等于已发布的初始显示文本加增量。pending、terminal 和 final-answer phase replay 读取缓存显示文本，不重新检查文件；历史用同一 stream 消费全文并 flush。 -共享投影层、Grok 与 DSH 均使用 `LocalMediaMarkdownStream`：只暂缓结尾未决的图片、代码跨度或可能增长的围栏标记，不等待整行、不等待后续标点。Item 结束前补发暂缓原文；不完整的单反引号路径遇到换行后不再扣住后面的文字。同一条消息内缓存每个候选路径的存在性检查，包括未找到的结果，避免文件出现或根目录命中变化导致已发布文本需要撤回；新的消息使用新的缓存。文件变化后历史重载的显示结果可能不同。每次增量仍会重扫累计文本，超长回复与细粒度增量会增加 CPU 开销。 +共享投影层、Grok 与 DSH 均使用 `LocalMediaMarkdownStream`：只暂缓结尾未决的图片、代码跨度或可能增长的围栏标记，不等待整行、不等待后续标点。未闭合的反引号代码跨度可以跨行，暂缓到闭合或 Item 结束；末尾的闭合反引号串也须等待后续字符确认长度。Item 结束时将未匹配的反引号串作为普通文本处理，继续解析其后的图片。同一条消息内缓存每个候选路径的存在性检查，包括未找到的结果，避免文件出现或根目录命中变化导致已发布文本需要撤回；新的消息使用新的缓存。文件变化后历史重载的显示结果可能不同。每次增量仍会重扫累计文本，超长回复与细粒度增量会增加 CPU 开销。 Grok/DSH 的额外目录仍由各自 Adapter 处理，优先于共享层： @@ -64,6 +64,6 @@ Desktop 发送带图片的消息时，请求里既有 `localImage` 条目，也 边界： -- 只认附件清单中的绝对路径；用户在请求正文里写的路径不算附件。附件清单必须位于消息开头（前面只允许空行），并以单独成行的 `## My request:` 结束；缺少结尾标题或前面还有其他文字时，视为用户自己输入或粘贴的文字，不读取附件。 +- 只认附件清单中的绝对路径；用户在请求正文里写的路径不算附件。附件清单必须位于消息开头（前面只允许空行），遇到单独成行的 `## My request:` 时结束；纯图片消息没有该标题时，清单延伸到文本末尾。清单前有其他文字时不读取附件。 - 不读取、不复制图片。剪贴板截图通常保存在系统临时目录，文件被清理后缩略图无法加载。 - 远程外部会话的图片路径位于远程主机，其缩略图显示尚未单独验证。 diff --git a/packages/harness-adapter/src/local-media-markdown.ts b/packages/harness-adapter/src/local-media-markdown.ts index 02dfc00b4..b19f9dca0 100644 --- a/packages/harness-adapter/src/local-media-markdown.ts +++ b/packages/harness-adapter/src/local-media-markdown.ts @@ -164,9 +164,11 @@ function projectLocalMedia( output += text.slice(index, next); index = next; const marker = text[next]; + let markerLength = 1; if (marker === "`" || marker === "~") { let runEnd = next; while (text[runEnd] === marker) runEnd += 1; + markerLength = runEnd - next; if (runEnd === text.length && holdIncomplete) { // The run may still grow into a fence, or a backtick may open a media path. return { text: output, consumed: next }; @@ -183,7 +185,7 @@ function projectLocalMedia( continue; } } - if (marker === "~" || runEnd - next > 1) { + if (marker === "~") { output += text.slice(next, runEnd); index = runEnd; continue; @@ -213,17 +215,19 @@ function projectLocalMedia( index = image.end; continue; } - const code = parseInlineCode(text, next); - if (code.kind === "text") { - output += "`"; - index = next + 1; + const code = parseInlineCode(text, next, markerLength); + if (code.kind === "open" || (holdIncomplete && code.end === text.length)) { + // A trailing closing run can still grow into a different-length, non-closing run. + if (holdIncomplete) return { text: output, consumed: next }; + // At completion an unmatched delimiter is ordinary text; later images may still resolve. + output += text.slice(next, next + markerLength); + index = next + markerLength; continue; } - if (code.kind === "open") { - if (!holdIncomplete) break; - return { text: output, consumed: next }; - } - const resolved = relativeImagesOnly ? null : resolveLocalMediaPath(code.content, roots, exists); + const resolved = + relativeImagesOnly || markerLength !== 1 || /[\r\n]/u.test(code.content) + ? null + : resolveLocalMediaPath(code.content, roots, exists); output += resolved === null ? text.slice(next, code.end) : mediaMarkdownImage(resolved); index = code.end; } @@ -279,18 +283,20 @@ function earliestIndex(candidates: readonly number[]): number { return next; } -/** `text`: ordinary text through `end`. `open`: the text ends before the construct is decided. */ -type ParsedInlineCode = - { kind: "code"; content: string; end: number } | { kind: "open" } | { kind: "text" }; +type ParsedInlineCode = { kind: "code"; content: string; end: number } | { kind: "open" }; -function parseInlineCode(text: string, start: number): ParsedInlineCode { - for (let index = start + 1; index < text.length; index += 1) { - const character = text[index]; - // A path never spans lines, so leave the tick as text rather than hold everything after it. - if (character === "\n") return { kind: "text" }; - if (character === "`") { - return { kind: "code", content: text.slice(start + 1, index), end: index + 1 }; +/** Code spans can cross lines, but only an equally long backtick run closes them. */ +function parseInlineCode(text: string, start: number, length: number): ParsedInlineCode { + let index = start + length; + while (index < text.length) { + const tick = text.indexOf("`", index); + if (tick === -1) break; + let end = tick + 1; + while (text[end] === "`") end += 1; + if (end - tick === length) { + return { kind: "code", content: text.slice(start + length, tick), end }; } + index = end; } return { kind: "open" }; } diff --git a/packages/harness-adapter/test/local-media-markdown.test.ts b/packages/harness-adapter/test/local-media-markdown.test.ts index 1673931de..1bc49944d 100644 --- a/packages/harness-adapter/test/local-media-markdown.test.ts +++ b/packages/harness-adapter/test/local-media-markdown.test.ts @@ -53,6 +53,19 @@ describe("rewriteLocalMediaMarkdown", () => { ); }); + it("preserves matching multi-backtick and multiline code spans", () => { + const spans = [ + "``![a](a.png) `a.png` ``", + "`first\n![a](a.png)\nlast`", + "``first\n![a](a.png) `not a closer`\nlast``", + "``a.png``", + "`\na.png\n`", + ]; + for (const span of spans) { + expect(rewrite(`${span} then \`a.png\``)).toBe(`${span} then ![a.png](${at("a.png")})`); + } + }); + it("keeps non-ASCII paths and brackets paths with spaces", () => { expect(rewrite('![e](截图/e.png) ![c]( "title")')).toBe( `![e](${at("截图/e.png")}) ![c](<${at("dir with space/c.png")}> "title")`, @@ -152,6 +165,12 @@ describe("LocalMediaMarkdownStream", () => { "![a]\n![c]()", "a!![b](sub/b.png)!", "``a.png`` and `a.png`` and ```", + "``![a](a.png)``", + "`first\n![a](a.png)\nlast`", + "``![a](a.png) `a.png` `` after ![a](a.png)", + "`first\n![a](a.png)\nlast` after ![a](a.png)", + "``first\n![a](a.png) `not a closer`\nlast`` after `a.png`", + "`a.png`` and ![a](a.png)", "鱼：`a.png`。后文 [图](a.png)", "| `a.png` |\n# `sub/b.png`：", "````\n```\n`a.png`\n```\n````\nthen `a.png`", @@ -204,6 +223,13 @@ describe("LocalMediaMarkdownStream", () => { expect(stream.flush()).toBe("sub/"); }); + it("waits for the closing backtick run's length before rewriting a code path", () => { + const stream = new LocalMediaMarkdownStream(roots, exists); + expect(stream.append("`a.png`")).toBe(""); + expect(stream.append("` and ![a](a.png)")).toBe(""); + expect(stream.flush()).toBe(`\`a.png\`\` and ![a](${at("a.png")})`); + }); + it("remembers negative and positive checks when files appear", () => { let present = false; const check = vi.fn((absolute: string) => present || exists(absolute)); diff --git a/packages/protocol-core/test/codex-local-media.test.ts b/packages/protocol-core/test/codex-local-media.test.ts new file mode 100644 index 000000000..34859a8a9 --- /dev/null +++ b/packages/protocol-core/test/codex-local-media.test.ts @@ -0,0 +1,110 @@ +import type { HostAgentMessageItem, HostTurnSnapshot } from "@codexhost/harness-adapter"; +import { + hostItemIdSchema, + hostTurnIdSchema, + nativeTurnRefSchema, + type JsonObject, +} from "@codexhost/shared-contracts"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { CodexTurnProjector, projectHistoricalTurn } from "../src/index.js"; + +const turnId = hostTurnIdSchema.parse("code-media"); +const spans = [ + "``![example](a.png)``", + "``![example](a.png) `not a closer` ![another](a.png)``", + "``![example](a.png) ```not a closer``` ![another](a.png)``", + "`first\n![example](a.png)\nlast`", + "``first\n![example](a.png)\n`not a closer`\nlast``", +]; + +describe("agent media code spans", () => { + let cwd: string; + let outside: string; + beforeEach(async () => { + cwd = await mkdtemp(path.join(os.tmpdir(), "codex-code-media-")); + await writeFile(path.join(cwd, "a.png"), "synthetic image"); + outside = `![outside](<${path.join(cwd, "a.png").replaceAll("\\", "/").replaceAll("%", "%2525")}>)`; + }); + afterEach(async () => { + await rm(cwd, { recursive: true, force: true }); + }); + + it.each(spans)("preserves %j through character-sized live updates", (span) => { + const raw = `${span}\nafter ![outside]()`; + const expected = `${span}\nafter ${outside}`; + const item: HostAgentMessageItem = { + type: "agentMessage", + itemId: hostItemIdSchema.parse("answer"), + text: "", + }; + const live = new CodexTurnProjector({ threadId: "external", turnId, cwd, startedAtMs: 1 }); + live.project({ type: "turn.started", turnId }); + const wire: JsonObject[] = live.project({ type: "item.started", turnId, item }).messages; + for (const text of raw) { + wire.push( + ...live.project({ + type: "item.updated", + turnId, + itemId: item.itemId, + update: { type: "text.append", text }, + }).messages, + ); + const published = wire + .filter(({ method }) => method === "item/agentMessage/delta") + .map(({ params }) => (params as JsonObject).delta) + .join(""); + expect(expected.startsWith(published)).toBe(true); + } + wire.push( + ...live.project({ + type: "item.completed", + turnId, + snapshot: { item: { ...item, text: raw }, outcome: { status: "succeeded" } }, + }).messages, + ); + expect( + wire + .filter(({ method }) => method === "item/agentMessage/delta") + .map(({ params }) => (params as JsonObject).delta) + .join(""), + ).toBe(expected); + expect(wire.filter(({ method }) => method === "item/completed")).toMatchObject([ + { params: { item: { type: "agentMessage", text: expected } } }, + ]); + expect( + live.project({ type: "turn.completed", turnId, outcome: { status: "succeeded" } }) + .completedTurn, + ).toMatchObject({ items: [{ text: expected, phase: "final_answer" }] }); + }); + + it.each(spans)("preserves %j through historical projection", (span) => { + const snapshot: HostTurnSnapshot = { + nativeTurnRef: nativeTurnRefSchema.parse({ + formatVersion: 1, + harnessId: "pi", + nativeSessionId: "session", + nativeTurnKey: "turn", + }), + input: [], + items: [ + { + item: { + type: "agentMessage", + itemId: hostItemIdSchema.parse("answer"), + text: `${span}\nafter ![outside]()`, + }, + outcome: { status: "succeeded" }, + }, + ], + outcome: { status: "succeeded" }, + }; + expect(projectHistoricalTurn({ turnId, cwd, snapshot }).items).toMatchObject([ + { type: "userMessage" }, + { type: "agentMessage", text: `${span}\nafter ${outside}`, phase: "final_answer" }, + ]); + }); +}); diff --git a/packages/protocol-core/test/codex-ui-projector.test.ts b/packages/protocol-core/test/codex-ui-projector.test.ts index 599bc554c..c33542d06 100644 --- a/packages/protocol-core/test/codex-ui-projector.test.ts +++ b/packages/protocol-core/test/codex-ui-projector.test.ts @@ -186,7 +186,7 @@ describe("Codex UI projector", () => { ]); }); - it("restores Desktop image attachments as native localImage input", () => { + it.each([true, false])("restores Desktop image attachments (request heading: %s)", (request) => { const text = [ "", "# Files mentioned by the user:", @@ -196,8 +196,9 @@ describe("Codex UI projector", () => { "", "## notes.md: /work/notes.md", "", - "## My request:", - "Compare /work/typed.png with the screenshot", + ...(request + ? ["## My request:", "## typed.png: /work/typed.png", "Image attachment: true"] + : []), ].join("\n"); const input = [{ type: "text" as const, text }]; const content = [ diff --git a/packages/renderer-extension/test/renderer-remote-attachments.test.ts b/packages/renderer-extension/test/renderer-remote-attachments.test.ts index 254955967..19ee0bfc3 100644 --- a/packages/renderer-extension/test/renderer-remote-attachments.test.ts +++ b/packages/renderer-extension/test/renderer-remote-attachments.test.ts @@ -122,6 +122,31 @@ describe("remote external attachments through the native Desktop file store", () } }); + it("reuses an image-only native manifest without requiring attachment storage", async () => { + const f = fixture(); + Reflect.deleteProperty(f.manager, "createFileAttachment"); + const params = request([ + { + type: "text", + text: "# Files mentioned by the user:\n\n## original.png: /remote/.codex/attachments/native/original.png\nImage attachment: true\n", + }, + image, + ]); + try { + await expect(f.manager.sendRequest("turn/start", params)).resolves.toEqual({ + turn: { id: "turn" }, + }); + expect(f.createFileAttachment).not.toHaveBeenCalled(); + expect(f.rpc).toHaveBeenCalledExactlyOnceWith( + "turn/start", + { ...params, codexhostSourceHostId: "ssh:remote" }, + undefined, + ); + } finally { + f.dispose(); + } + }); + it("does not mistake an example in the user's request for native attachment context", async () => { const f = fixture(); try { diff --git a/packages/shared-contracts/src/desktop-attachments.ts b/packages/shared-contracts/src/desktop-attachments.ts index 8c64ee567..4ad03f33d 100644 --- a/packages/shared-contracts/src/desktop-attachments.ts +++ b/packages/shared-contracts/src/desktop-attachments.ts @@ -23,14 +23,13 @@ const IMAGE_ATTACHMENT_PATTERN = const CONTROL_CHARACTER_PATTERN = /[\u0000-\u001f\u007f]/u; /** - * Absolute image paths in the attachment context Desktop serialized into a prompt. Only a - * complete context counts: it opens the prompt and ends at the request heading, so text the - * user typed or pasted that merely looks like one is never read as attachments. + * Absolute image paths in the attachment context Desktop serialized into a prompt. The context + * opens the prompt and ends at the request heading when present, or at the end of an image-only + * prompt. Paths in the request body are not attachments. */ export function desktopImageAttachmentPaths(text: string): string[] { const end = text.search(REQUEST_HEADING); - if (end < 0) return []; - const context = text.slice(0, end); + const context = end < 0 ? text : text.slice(0, end); if (!ATTACHED_FILES_HEADING.test(context)) return []; return [...context.matchAll(IMAGE_ATTACHMENT_PATTERN)] .map((match) => match[1] ?? "") diff --git a/packages/shared-contracts/test/desktop-attachments.test.ts b/packages/shared-contracts/test/desktop-attachments.test.ts index eec1186c8..b796e8712 100644 --- a/packages/shared-contracts/test/desktop-attachments.test.ts +++ b/packages/shared-contracts/test/desktop-attachments.test.ts @@ -45,11 +45,23 @@ describe("desktopImageAttachmentPaths", () => { expect(desktopImageAttachmentPaths(typed)).toEqual([]); }); - it("reads only a complete context that opens the prompt", () => { + it.each(["\n", "\r\n"])("reads an image-only context through the end (%j)", (newline) => { + const context = [ + "# Files mentioned by the user:", + "", + "## a.png: /a.png", + "Image attachment: true", + "", + ].join(newline); + expect(desktopImageAttachmentPaths(context)).toEqual(["/a.png"]); + // An inline mention of the request heading does not end the context. + expect(desktopImageAttachmentPaths(`${context}${newline}## My request: inline`)).toEqual([ + "/a.png", + ]); + }); + + it("reads only a context that opens the prompt", () => { const context = "# Files mentioned by the user:\n\n## a.png: /a.png\nImage attachment: true\n"; - // Pasted without the request heading that ends a Desktop context. - expect(desktopImageAttachmentPaths(context)).toEqual([]); - expect(desktopImageAttachmentPaths(`${context}\n## My request: inline`)).toEqual([]); // Preceded by other text, so it is part of what the user typed. expect(desktopImageAttachmentPaths(`Look:\n${context}\n## My request:\nhi`)).toEqual([]); expect(desktopImageAttachmentPaths(`\n\n${context}\n## My request:\nhi`)).toEqual(["/a.png"]); From 5bb78cfa8ab7379559f8884a75a09ef7a97858fc Mon Sep 17 00:00:00 2001 From: BytePioneer-AI <28417416@qq.com> Date: Sun, 1 Nov 2026 23:54:45 +0800 Subject: [PATCH 15/15] fix: release unmatched code spans at paragraph boundaries --- docs/architecture/external-thread-media.md | 2 +- .../src/local-media-markdown.ts | 14 ++-- .../test/local-media-markdown.test.ts | 25 +++++++ .../test/codex-local-media.test.ts | 71 +++++++++++++++++++ 4 files changed, 106 insertions(+), 6 deletions(-) diff --git a/docs/architecture/external-thread-media.md b/docs/architecture/external-thread-media.md index 94edca638..dfd81fccb 100644 --- a/docs/architecture/external-thread-media.md +++ b/docs/architecture/external-thread-media.md @@ -45,7 +45,7 @@ Desktop 的 Markdown 图片 token 转换器和媒体加载组件各归一化一 实时输出保持追加式，原始完成全文必须等于原始初始文本加所有增量。共享投影层为每个 Agent Item 创建独立的 `LocalMediaMarkdownStream`，不把显示文本写回原始账本；显示完成文本同样等于已发布的初始显示文本加增量。pending、terminal 和 final-answer phase replay 读取缓存显示文本，不重新检查文件；历史用同一 stream 消费全文并 flush。 -共享投影层、Grok 与 DSH 均使用 `LocalMediaMarkdownStream`：只暂缓结尾未决的图片、代码跨度或可能增长的围栏标记，不等待整行、不等待后续标点。未闭合的反引号代码跨度可以跨行，暂缓到闭合或 Item 结束；末尾的闭合反引号串也须等待后续字符确认长度。Item 结束时将未匹配的反引号串作为普通文本处理，继续解析其后的图片。同一条消息内缓存每个候选路径的存在性检查，包括未找到的结果，避免文件出现或根目录命中变化导致已发布文本需要撤回；新的消息使用新的缓存。文件变化后历史重载的显示结果可能不同。每次增量仍会重扫累计文本，超长回复与细粒度增量会增加 CPU 开销。 +共享投影层、Grok 与 DSH 均使用 `LocalMediaMarkdownStream`：只暂缓结尾未决的图片、代码跨度或可能增长的围栏标记，不等待整行、不等待后续标点。未闭合的反引号代码跨度可以跨行但不能跨空行（两个换行符之间仅有空白），暂缓到闭合、空行或 Item 结束；末尾的闭合反引号串也须等待后续字符确认长度。遇到空行或 Item 结束时将未匹配的反引号串作为普通文本处理，继续解析其后的图片，不扣住后续段落。同一条消息内缓存每个候选路径的存在性检查，包括未找到的结果，避免文件出现或根目录命中变化导致已发布文本需要撤回；新的消息使用新的缓存。文件变化后历史重载的显示结果可能不同。每次增量仍会重扫累计文本，超长回复与细粒度增量会增加 CPU 开销。 Grok/DSH 的额外目录仍由各自 Adapter 处理，优先于共享层： diff --git a/packages/harness-adapter/src/local-media-markdown.ts b/packages/harness-adapter/src/local-media-markdown.ts index b19f9dca0..bec98d4f2 100644 --- a/packages/harness-adapter/src/local-media-markdown.ts +++ b/packages/harness-adapter/src/local-media-markdown.ts @@ -216,10 +216,10 @@ function projectLocalMedia( continue; } const code = parseInlineCode(text, next, markerLength); - if (code.kind === "open" || (holdIncomplete && code.end === text.length)) { + if (code.kind !== "code" || (holdIncomplete && code.end === text.length)) { // A trailing closing run can still grow into a different-length, non-closing run. - if (holdIncomplete) return { text: output, consumed: next }; - // At completion an unmatched delimiter is ordinary text; later images may still resolve. + if (holdIncomplete && code.kind !== "text") return { text: output, consumed: next }; + // Once the paragraph or Item ends, an unmatched delimiter is ordinary text. output += text.slice(next, next + markerLength); index = next + markerLength; continue; @@ -283,13 +283,17 @@ function earliestIndex(candidates: readonly number[]): number { return next; } -type ParsedInlineCode = { kind: "code"; content: string; end: number } | { kind: "open" }; +type ParsedInlineCode = + { kind: "code"; content: string; end: number } | { kind: "open" } | { kind: "text" }; -/** Code spans can cross lines, but only an equally long backtick run closes them. */ +/** Code spans can cross lines, not blank lines; only an equally long backtick run closes them. */ function parseInlineCode(text: string, start: number, length: number): ParsedInlineCode { let index = start + length; while (index < text.length) { const tick = text.indexOf("`", index); + if (/\n[^\S\n]*\n/u.test(text.slice(index, tick < 0 ? text.length : tick))) { + return { kind: "text" }; + } if (tick === -1) break; let end = tick + 1; while (text[end] === "`") end += 1; diff --git a/packages/harness-adapter/test/local-media-markdown.test.ts b/packages/harness-adapter/test/local-media-markdown.test.ts index 1bc49944d..7c333f426 100644 --- a/packages/harness-adapter/test/local-media-markdown.test.ts +++ b/packages/harness-adapter/test/local-media-markdown.test.ts @@ -171,6 +171,9 @@ describe("LocalMediaMarkdownStream", () => { "`first\n![a](a.png)\nlast` after ![a](a.png)", "``first\n![a](a.png) `not a closer`\nlast`` after `a.png`", "`a.png`` and ![a](a.png)", + "before `unclosed\n\n![a](a.png)", + "before `unclosed\n \t\n![a](a.png)\n\nlast `unclosed", + "before ``unclosed\r\n\t\r\n![a](a.png)\r\n\r\nlast ``unclosed", "鱼：`a.png`。后文 [图](a.png)", "| `a.png` |\n# `sub/b.png`：", "````\n```\n`a.png`\n```\n````\nthen `a.png`", @@ -230,6 +233,28 @@ describe("LocalMediaMarkdownStream", () => { expect(stream.flush()).toBe(`\`a.png\`\` and ![a](${at("a.png")})`); }); + it.each(["\n\n", "\n \t\n", "\r\n \t\r\n"])( + "releases an unmatched backtick at a paragraph boundary (%j)", + (boundary) => { + for (const relativeImagesOnly of [false, true]) { + const stream = new LocalMediaMarkdownStream(roots, exists, { relativeImagesOnly }); + const prefix = `before \`unclosed${boundary}`; + const suffix = `after ![a](a.png)${boundary}last \`unclosed`; + const expected = `${prefix}after ![a](${at("a.png")})${boundary}last \`unclosed`; + expect(stream.append("before `unclosed")).toBe("before "); + expect(stream.append(boundary.slice(0, -1))).toBe(""); + expect(stream.append(boundary.slice(-1))).toBe(`\`unclosed${boundary}`); + expect(stream.text).toBe(prefix); + expect(stream.append(suffix)).toBe(`after ![a](${at("a.png")})${boundary}last `); + expect(stream.flush()).toBe("`unclosed"); + expect(stream.text).toBe(expected); + expect( + rewriteLocalMediaMarkdown(prefix + suffix, roots, { exists, relativeImagesOnly }), + ).toBe(expected); + } + }, + ); + it("remembers negative and positive checks when files appear", () => { let present = false; const check = vi.fn((absolute: string) => present || exists(absolute)); diff --git a/packages/protocol-core/test/codex-local-media.test.ts b/packages/protocol-core/test/codex-local-media.test.ts index 34859a8a9..46e1e5cba 100644 --- a/packages/protocol-core/test/codex-local-media.test.ts +++ b/packages/protocol-core/test/codex-local-media.test.ts @@ -81,6 +81,77 @@ describe("agent media code spans", () => { ).toMatchObject({ items: [{ text: expected, phase: "final_answer" }] }); }); + it("releases an unmatched backtick at a blank line during character-sized updates", () => { + const prefix = "before `unclosed\n \t\n"; + const raw = `${prefix}![outside]()\n\nlast \`unclosed`; + const expected = `${prefix}${outside}\n\nlast \`unclosed`; + const item: HostAgentMessageItem = { + type: "agentMessage", + itemId: hostItemIdSchema.parse("answer"), + text: "", + }; + const live = new CodexTurnProjector({ threadId: "external", turnId, cwd, startedAtMs: 1 }); + live.project({ type: "turn.started", turnId }); + live.project({ type: "item.started", turnId, item }); + let native = ""; + let published = ""; + for (const text of raw) { + native += text; + const { messages } = live.project({ + type: "item.updated", + turnId, + itemId: item.itemId, + update: { type: "text.append", text }, + }); + published += messages + .filter(({ method }) => method === "item/agentMessage/delta") + .map(({ params }) => (params as JsonObject).delta) + .join(""); + expect(expected.startsWith(published)).toBe(true); + if (native === prefix) expect(published).toBe(prefix); + } + const completed = live.project({ + type: "item.completed", + turnId, + snapshot: { item: { ...item, text: raw }, outcome: { status: "succeeded" } }, + }); + published += completed.messages + .filter(({ method }) => method === "item/agentMessage/delta") + .map(({ params }) => (params as JsonObject).delta) + .join(""); + expect(published).toBe(expected); + expect(completed.messages.filter(({ method }) => method === "item/completed")).toMatchObject([ + { params: { item: { type: "agentMessage", text: expected } } }, + ]); + }); + + it("does not pair isolated backticks across paragraphs during historical projection", () => { + const snapshot: HostTurnSnapshot = { + nativeTurnRef: nativeTurnRefSchema.parse({ + formatVersion: 1, + harnessId: "pi", + nativeSessionId: "session", + nativeTurnKey: "turn", + }), + input: [], + items: [ + { + item: { + type: "agentMessage", + itemId: hostItemIdSchema.parse("answer"), + text: "before `unclosed\n\n![outside]()\n\nlast `unclosed", + }, + outcome: { status: "succeeded" }, + }, + ], + outcome: { status: "succeeded" }, + }; + expect(projectHistoricalTurn({ turnId, cwd, snapshot }).items).toMatchObject([ + { type: "userMessage" }, + { type: "agentMessage", text: `before \`unclosed\n\n${outside}\n\nlast \`unclosed` }, + ]); + }); + it.each(spans)("preserves %j through historical projection", (span) => { const snapshot: HostTurnSnapshot = { nativeTurnRef: nativeTurnRefSchema.parse({