fix(store): split sibling fields back out when they leak into content

.github/workflows/ci.yml

@@ -42,6 +42,14 @@ jobs:
       - name: Unit tests (series detection — keeps timelines out of the supersede band)
         run: node series-detection.test.mjs
 
+      - name: Unit tests (tag-leak repair — sibling fields leaked into content)
+        run: node tag-leak-repair.test.mjs
+
+      - name: Integration test (tag-leak repair through store_memory)
+        env:
+          TOKENMEM_DB_PATH: ${{ runner.temp }}/mneme-ci-tag-leak.db
+        run: node tag-leak.integration.test.mjs
+
       - name: Integration test (meta-gate write-gate)
         env:
           TOKENMEM_DB_PATH: ${{ runner.temp }}/mneme-ci-meta-gate.db

mcp-server.mjs

@@ -48,6 +48,7 @@ import {
 } from './index.mjs'
 import { migrateVectorsToBlob } from './index.mjs'   // Migration 013 background runner (below)
 import { parseHostTokens, resolveAuthMode, resolveHost } from './auth.mjs'
+import { repairTagLeak } from './tag-leak-repair.mjs'
 import { recallClaudeMarkdownMemory } from './lib/claude-markdown-memory.mjs'
 
 // ── Load .env.local BEFORE initMemory() ────────────────────────────────
@@ -256,6 +257,19 @@ function createServer(hostId = DEFAULT_HOST) {
     async ({ content, summary, importance = 6, memory_type = 'long_term', memory_level = 'semi_abstract', category = 'general', tags = [], supersedes, event_time, is_anchor, is_pinned }) => {
       const out = {}
 
+      // Close-tag leak: sibling fields parsed into content/summary as raw XML
+      // while the real fields fell back to zod defaults. Split them back out
+      // before anything else sees the args — see tag-leak-repair.mjs.
+      const leak = repairTagLeak({ content, summary, importance, memory_type, memory_level, category, tags, supersedes, event_time, is_anchor, is_pinned })
+      if (leak.repaired) {
+        ({ content, summary, importance, memory_type, memory_level, category, tags, supersedes, event_time, is_anchor, is_pinned } = leak.args)
+      }
+      const leakNote = leak.repaired
+        ? `\n🩹 close-tag leak repaired: ${leak.moved.join(', ') || '(no valid fields)'} moved back out of ${leak.from.join(' + ')} — a field was closed with the wrong tag; each <parameter> must end with </parameter>`
+        : leak.suspect
+          ? `\n⚠️ possible close-tag leak: content/summary contains a closing tag followed by a field tag, but it didn't parse cleanly — recall_by_id this row and check its tail`
+          : ''
+
       // v2.9: not-yet-trusted hosts write into quarantine — a separate table
       // the recall pool never reads. Requested supersedes are recorded but
       // execute only if the reviewer approves.
@@ -273,6 +287,7 @@ function createServer(hostId = DEFAULT_HOST) {
         if (!qid) return { content: [{ type: 'text', text: 'Quarantine storage failed' }] }
         let qtext = `🔒 Quarantined (qid: ${qid}, host: ${hostId}) — pending review by '${PRIMARY_HOST}'. Not recallable until approved.`
         if (is_anchor || is_pinned) qtext += `\n(anchor/pinned flags are dropped for quarantined writes — the reviewer can re-add them after merge)`
+        qtext += leakNote
         if (out.encodingWarning) {
           const e = out.encodingWarning
           qtext += `\n⚠️ ENCODING DAMAGE: ${e.qmarkCount} '?' chars (longest run ${e.maxRun}). CJK was likely lost to a non-UTF-8 code page (cp936) — this is IRREVERSIBLE, not a display glitch. If you just wrote Chinese, it did NOT save; re-store via a UTF-8-safe path (codex exec / CC-side), not Codex Desktop.`
@@ -309,7 +324,7 @@ function createServer(hostId = DEFAULT_HOST) {
       if (is_anchor && !out.quotaRejected?.find(q => q.flag === 'is_anchor')) flags.push('anchor')
       if (is_pinned && !out.quotaRejected?.find(q => q.flag === 'is_pinned')) flags.push('pinned')
       const flagStr = flags.length ? `, flags: [${flags.join(', ')}]` : ''
-      let text = `Stored memory (id: ${id}, importance: ${importance}, type: ${memory_type}, level: ${finalLevel}${flagStr})`
+      let text = `Stored memory (id: ${id}, importance: ${importance}, type: ${memory_type}, level: ${finalLevel}${flagStr})` + leakNote
       // First among the warnings on purpose: the others say a policy adjusted the
       // write, this one says the content that landed is already damaged.
       if (out.encodingWarning) {

tag-leak-repair.mjs

@@ -0,0 +1,175 @@
+// tag-leak-repair.mjs — tool-call close-tag leak detection + repair
+//
+// When a caller closes a long `content` argument with the wrong tag
+// (`</content>` instead of `</parameter>`, or opens a bare `<summary>`),
+// the tool-call parser keeps reading and the sibling fields — summary,
+// importance, category, tags — land inside `content` as raw XML. The
+// fields themselves arrive empty, so the zod defaults (importance 6,
+// category general) silently replace what the caller wrote.
+//
+// An instruction-level rule against this was in place for months and the
+// leak kept happening, so the fix lives at the write path:
+// detect the leak, split the tail back into its fields, report the repair.
+//
+// Detection is structural, not substring. The closing tag must be followed
+// by a field tag AND the whole tail must parse as field tags to the end of
+// the string. A memory that *describes* this bug quotes the same strings
+// mid-prose (often in backticks) — that one must pass through untouched.
+
+const FIELDS = [
+  'summary', 'importance', 'category', 'tags', 'memory_type', 'memory_level',
+  'supersedes', 'event_time', 'is_anchor', 'is_pinned',
+]
+const F = FIELDS.join('|')
+
+const ENUMS = {
+  category: ['general', 'people', 'project', 'decision', 'feedback', 'bug', 'relationship', 'skill', 'preference'],
+  memory_type: ['working', 'short_term', 'long_term', 'permanent'],
+  memory_level: ['concrete_trace', 'semi_abstract', 'meta_knowledge'],
+}
+
+// zod defaults in the store_memory schema. A passed value equal to one of
+// these may be the default standing in for a value that leaked.
+export const DEFAULTS = { importance: 6, category: 'general', memory_type: 'long_term', memory_level: 'semi_abstract' }
+
+// Tail tokens, matched in place with sticky regexes — no slicing, tails can
+// be tens of KB. Only field / `parameter` closers and the tool-call envelope
+// (`invoke` / `function_calls`, optionally namespaced — real leaks usually
+// end with it) count. A tail that ends in any other closer (`</entry>`,
+// `</div>`) is markup, not a leak.
+const WS = /\s*/y
+const CLOSE = new RegExp(`</(?:parameter|${F}|(?:[A-Za-z]+:)?(?:invoke|function_calls))>`, 'y')
+const OPEN = new RegExp(`<(?:parameter name="(${F})"|(${F}))>`, 'y')
+const VALUE_END = new RegExp(`</parameter>|</(?:${F})>|<parameter name="|<(?:${F})>`, 'g')
+
+// Parse field runs from `start` to the end of `s`. Returns { fields } when the
+// whole tail is field openers / field closers / whitespace, else { failAt }:
+// the position of the first token that isn't one.
+function parseTail(s, start) {
+  const out = {}
+  let i = start
+  while (true) {
+    WS.lastIndex = i; WS.exec(s); i = WS.lastIndex
+    if (i >= s.length) break
+    CLOSE.lastIndex = i
+    if (CLOSE.exec(s)) { i = CLOSE.lastIndex; continue }
+    OPEN.lastIndex = i
+    const m = OPEN.exec(s)
+    if (!m) return { failAt: i }
+    const name = m[1] || m[2]
+    VALUE_END.lastIndex = OPEN.lastIndex
+    const end = VALUE_END.exec(s)
+    const stop = end ? end.index : s.length
+    out[name] = s.slice(OPEN.lastIndex, stop).trim()
+    i = stop
+  }
+  return { fields: out }
+}
+
+function coerce(name, raw) {
+  if (name === 'importance') {
+    const n = Number(raw)
+    return Number.isFinite(n) && n >= 1 && n <= 10 ? n : undefined
+  }
+  if (ENUMS[name]) return ENUMS[name].includes(raw) ? raw : undefined
+  if (name === 'tags' || name === 'supersedes') {
+    try {
+      const v = JSON.parse(raw)
+      if (Array.isArray(v)) return v.map(String)
+    } catch {}
+    const parts = raw.split(/[,，]/).map(t => t.trim().replace(/^["']|["']$/g, '')).filter(Boolean)
+    return parts.length ? parts : undefined
+  }
+  if (name === 'is_anchor' || name === 'is_pinned') {
+    return raw === 'true' ? true : raw === 'false' ? false : undefined
+  }
+  return raw || undefined
+}
+
+// Find the first structural leak in `text`: a closer from `closerNames`, then
+// a field opener, then a tail that parses as fields to the end of the string.
+// Returns { head, fields }, { suspect: true } when a leak-shaped match exists
+// but nothing parsed cleanly, or null.
+function splitLeak(text, closerNames) {
+  if (typeof text !== 'string' || !text) return null
+  const re = new RegExp(`</(${closerNames})>\\s*<(?:parameter name="(?:${F})"|(?:${F}))>`, 'g')
+
+  // A leaked closer has no opener in the text — the caller's own opening tag
+  // was consumed by the tool-call parser. A closer that closes an element
+  // opened earlier (`<content>…</content>` in a quoted Atom entry, say) is
+  // markup. Open/close depth per name, advanced as matches move right.
+  const tagRe = new RegExp(`<(/?)(${closerNames})(?=[\\s>])`, 'g')
+  const depth = Object.fromEntries(closerNames.split('|').map(n => [n, 0]))
+  let scanned = 0
+  const advanceTo = (to) => {
+    tagRe.lastIndex = scanned
+    let t
+    while ((t = tagRe.exec(text)) && t.index < to) depth[t[2]] += t[1] ? -1 : 1
+    scanned = to
+  }
+
+  let m
+  let suspect = false
+  while ((m = re.exec(text))) {
+    if (text[m.index - 1] === '`') continue  // quoted in inline code
+    advanceTo(m.index)
+    if (depth[m[1]] > 0) continue
+    const r = parseTail(text, m.index + m[1].length + 3)
+    if (r.fields) return { head: text.slice(0, m.index).trimEnd(), fields: r.fields }
+    suspect = true
+    // Any later start before failAt runs into the same non-field token, so
+    // skip past it — keeps the scan linear on long, repetitive content.
+    re.lastIndex = Math.max(re.lastIndex, r.failAt)
+  }
+  return suspect ? { suspect: true } : null
+}
+
+const isEmpty = v => v === undefined || v === null || v === '' || (Array.isArray(v) && v.length === 0)
+
+/**
+ * Repair a store_memory argument set whose fields leaked into content/summary.
+ * Pure: returns a new args object, never mutates the input.
+ *
+ * Merge rule: a leaked value only replaces a value the caller didn't really
+ * choose — an empty one, or one equal to the zod default (`defaults`). An
+ * explicitly passed non-default value always wins. When content and summary
+ * both leak the same field, the content-side value is kept.
+ *
+ * The text is only truncated when the split pays for itself: at least one
+ * field is recovered and something of the original text is left. Otherwise
+ * the args pass through unchanged and the result is flagged `suspect`.
+ *
+ * @returns {{ repaired: boolean, suspect: boolean, args: object, moved: string[], from: string[] }}
+ */
+export function repairTagLeak(args, defaults = DEFAULTS) {
+  const next = { ...args }
+  const moved = new Set()
+  const from = []
+  let suspect = false
+
+  const replaceable = (name) => isEmpty(next[name]) || (name in defaults && next[name] === defaults[name])
+
+  for (const [field, closerNames] of [['content', 'content|parameter'], ['summary', 'summary|parameter']]) {
+    const hit = splitLeak(next[field], closerNames)
+    if (!hit) continue
+    if (hit.suspect) { suspect = true; continue }
+    // `accounted`: tail fields that are valid and either applied now or
+    // already recovered from the other side — proof the tail is a real leak.
+    const updates = {}
+    let accounted = 0
+    for (const [name, raw] of Object.entries(hit.fields)) {
+      if (name === field) continue
+      const v = coerce(name, raw)
+      if (v === undefined) continue
+      if (moved.has(name)) { accounted++; continue }
+      if (replaceable(name)) { updates[name] = v; accounted++ }
+    }
+    if (!accounted || !hit.head.trim()) { suspect = true; continue }
+    next[field] = hit.head
+    Object.assign(next, updates)
+    for (const k of Object.keys(updates)) moved.add(k)
+    from.push(field)
+  }
+
+  return { repaired: from.length > 0, suspect, args: next, moved: [...moved], from }
+}

tag-leak-repair.test.mjs

@@ -0,0 +1,121 @@
+// Self-check for tag-leak-repair. Run: node tag-leak-repair.test.mjs
+// Leak cases are shaped after real corrupted rows found in a production store.
+import { repairTagLeak } from './tag-leak-repair.mjs'
+
+const base = { summary: undefined, importance: 6, category: 'general', tags: [], memory_type: 'long_term', memory_level: 'semi_abstract' }
+let fail = 0
+const check = (label, cond, detail) => {
+  if (!cond) { fail++; console.log(`✗ ${label}`, detail ?? '') } else console.log(`✓ ${label}`)
+}
+
+// 1. </content> then an unclosed summary param (most common Aug shape)
+{
+  const r = repairTagLeak({ ...base, content: '测量工具介入系统就改变了系统。</content> <parameter name="summary">性能测量前须采样空载基线' })
+  check('content leak: summary moved out', r.repaired && r.args.summary === '性能测量前须采样空载基线', r)
+  check('content leak: content truncated', r.args.content === '测量工具介入系统就改变了系统。', r.args.content)
+}
+
+// 2. bare <summary> / <importance> tags (Jun–Aug shape)
+{
+  const r = repairTagLeak({ ...base, content: '方案是假的。</content> <summary>备份凭据不能只存在被备份的机器上</summary> <importance>8</importance>' })
+  check('bare tags: summary + importance', r.args.summary === '备份凭据不能只存在被备份的机器上' && r.args.importance === 8, r.args)
+}
+
+// 3. full sibling set with category / tags / level
+{
+  const content = '正文\n</content>\n<parameter name="summary">S</parameter>\n<parameter name="importance">9</parameter>\n<parameter name="category">decision</parameter>\n<parameter name="tags">["a","b"]</parameter>\n<parameter name="memory_level">meta_knowledge</parameter>'
+  const r = repairTagLeak({ ...base, content })
+  check('full set: all fields', r.args.content === '正文' && r.args.summary === 'S' && r.args.importance === 9
+    && r.args.category === 'decision' && r.args.tags.join() === 'a,b' && r.args.memory_level === 'meta_knowledge', r.args)
+}
+
+// 4. summary-side leak (9/21 shape): importance stuck inside summary
+{
+  const r = repairTagLeak({ ...base, content: '内容无损', summary: '摘要文本</summary><parameter name="importance">7' })
+  check('summary leak: importance recovered', r.args.summary === '摘要文本' && r.args.importance === 7 && r.args.content === '内容无损', r.args)
+}
+
+// 4b. tail ending in the tool-call envelope closer (most summary-side rows)
+{
+  const r = repairTagLeak({ ...base, content: '正文', summary: '摘要</summary>\n<importance>9</importance>\n<category>project</category>\n<tags>["a","b"]</tags>\n</invoke>' })
+  check('envelope closer at the end', r.repaired && r.args.summary === '摘要' && r.args.importance === 9 && r.args.category === 'project' && r.args.tags.join() === 'a,b', r.args)
+}
+
+// 5. an existing non-empty summary is never overwritten by a leaked one
+{
+  const r = repairTagLeak({ ...base, summary: '已有摘要', content: 'x</content><parameter name="summary">泄漏摘要</parameter><parameter name="category">bug</parameter>' })
+  check('keeps passed summary, takes leaked category', r.args.summary === '已有摘要' && r.args.category === 'bug' && r.args.content === 'x', r.args)
+}
+
+// 6. prose that DESCRIBES the bug must pass untouched
+for (const [label, content] of [
+  ['prose: backticked closer + field', '尾巴上挂着一段 `</content><parameter name="summary">…` 的裸 XML。\n\n## 分类判据\n关键是分清两种'],
+  ['prose: closer followed by Chinese', '把结尾 `</parameter>` 手滑成 </content>, 污染其后 importance'],
+  ['prose: field tag mid-paragraph then prose', '写 <parameter name="summary"> 时要记得关 tag，然后继续写正文。这里还有更多说明文字。'],
+  ['clean content', '普通的一条记忆，没有任何标签。'],
+]) {
+  const r = repairTagLeak({ ...base, content })
+  check(label, !r.repaired && r.args.content === content, r)
+}
+
+// 7. nothing valid recovered → no truncation, flagged suspect
+{
+  const content = 'x</content><parameter name="category">nonsense</parameter><parameter name="importance">42</parameter>'
+  const r = repairTagLeak({ ...base, content })
+  check('invalid values only → untouched + suspect', !r.repaired && r.suspect && r.args.content === content && r.args.importance === 6, r)
+}
+
+// 8. leak-shaped closer whose tail is prose → suspect, not repaired
+{
+  const content = '正文</content><parameter name="summary">摘要</parameter> 然后又接了一段正文'
+  const r = repairTagLeak({ ...base, content })
+  check('unparseable tail → suspect only', !r.repaired && r.suspect && r.args.content === content, r)
+}
+
+// 9. markup that happens to use field-named elements is not a leak
+for (const [label, content] of [
+  ['atom entry with wrapper', 'Atom feed 示例：\n<entry>\n  <content>正文内容</content>\n  <summary>摘要内容</summary>\n</entry>'],
+  ['atom elements at the very end', 'Atom 里正文和摘要这样写：\n<content>正文内容</content>\n<summary>摘要内容</summary>'],
+  ['jsx-ish', '组件结构：<Foo>\n<content>bar</content>\n<tags>x</tags>\n</Foo>'],
+  ['html with invalid enum', '这是页面结构：\n<div>\n  <content>正文</content>\n  <category>news</category>\n</div>'],
+]) {
+  const r = repairTagLeak({ ...base, content })
+  check(`markup: ${label}`, !r.repaired && r.args.content === content && r.args.summary === undefined, r)
+}
+
+// 10. an explicitly passed non-default value is never overridden
+{
+  const r = repairTagLeak({ ...base, importance: 10, category: 'decision', content: '示例正文</content><parameter name="importance">3</parameter><parameter name="category">bug</parameter><parameter name="summary">S</parameter>' })
+  check('explicit importance/category kept', r.args.importance === 10 && r.args.category === 'decision' && r.args.summary === 'S' && r.args.content === '示例正文', r.args)
+}
+
+// 11. leak consumes the whole field → nothing left, don't store a blank row
+{
+  const content = '</content><parameter name="summary">S</parameter>'
+  const r = repairTagLeak({ ...base, content })
+  check('empty head → untouched + suspect', !r.repaired && r.suspect && r.args.content === content, r)
+}
+
+// 12. content and summary both leak the same field → content side wins
+{
+  const r = repairTagLeak({ ...base, content: 'c</content><parameter name="importance">3</parameter>', summary: 's</summary><parameter name="importance">8' })
+  check('content-side value wins on conflict', r.args.importance === 3 && r.args.summary === 's' && r.args.content === 'c', r.args)
+}
+
+// 13. remaining fields: memory_type / supersedes / event_time / is_anchor
+{
+  const r = repairTagLeak({ ...base, content: 'x</content><parameter name="memory_type">permanent</parameter><parameter name="supersedes">["12","34"]</parameter><parameter name="event_time">2026-01-02</parameter><parameter name="is_anchor">true</parameter>' })
+  check('other fields recovered', r.args.memory_type === 'permanent' && r.args.supersedes.join() === '12,34' && r.args.event_time === '2026-01-02' && r.args.is_anchor === true, r.args)
+}
+
+// 14. long repetitive near-leak stays linear
+{
+  const content = 'Z</content>' + '<parameter name="summary">Y</parameter>'.repeat(8000) + '\u0000POISON'
+  const t0 = Date.now()
+  const r = repairTagLeak({ ...base, content })
+  const ms = Date.now() - t0
+  check(`${Math.round(content.length / 1000)}KB adversarial input in ${ms}ms (< 500)`, ms < 500 && !r.repaired, { ms, repaired: r.repaired })
+}
+
+console.log(fail ? `\n${fail} FAILED` : '\nall passed')
+process.exit(fail ? 1 : 0)