fix(v2.11.1): stop the recall feedback loop; recall on what the user typed

.github/workflows/ci.yml

@@ -66,6 +66,16 @@ jobs:
           TOKENMEM_DB_PATH: ${{ runner.temp }}/mneme-ci-ranking-importance.db
         run: node ranking-importance.integration.test.mjs
 
+      - name: Integration test (access bump — only returned rows, not the candidate pool)
+        env:
+          TOKENMEM_DB_PATH: ${{ runner.temp }}/mneme-ci-access-bump.db
+        run: node access-bump-scope.integration.test.mjs
+
+      - name: Integration test (DB path read at open time, not import time)
+        env:
+          TOKENMEM_DB_PATH: ${{ runner.temp }}/mneme-ci-late-db.db
+        run: node db-path-late-env.integration.test.mjs
+
       - name: Integration test (encoding-damage detection, both write paths)
         env:
           TOKENMEM_DB_PATH: ${{ runner.temp }}/mneme-ci-encoding-damage.db

access-bump-scope.integration.test.mjs

@@ -0,0 +1,63 @@
+// recallForClients over-fetches a candidate pool (up to 30 rows when filtering)
+// and trims it to the caller's limit. Access counts must move only for the rows
+// that were returned. When the whole pool was bumped, every hook call added +1 to
+// ~30 rows while showing 3, and since access frequency feeds ranking and decay the
+// rows that were already on top stayed there: on one 10k-row store a single row
+// reached 46% of two weeks of hook recalls.
+//
+// Run: TOKENMEM_DB_PATH=/tmp/x.db node access-bump-scope.integration.test.mjs
+import { initMemory, closeMemory, storeMemory, recallForClients } from './index.mjs'
+import Database from 'better-sqlite3'
+
+const DB_PATH = process.env.TOKENMEM_DB_PATH
+if (!DB_PATH) { console.error('FATAL: set TOKENMEM_DB_PATH'); process.exit(2) }
+
+let pass = 0, fail = 0
+const check = (label, cond, detail = '') => {
+  if (cond) { pass++; console.log(`✓ ${label}`) }
+  else { fail++; console.log(`✗ ${label}${detail ? ' — ' + detail : ''}`) }
+}
+
+initMemory()
+const marker = `zzbump${Math.floor(Math.random() * 1e6)}`
+const ids = []
+for (let i = 0; i < 12; i++) {
+  ids.push(storeMemory({
+    content: `${marker} pool row ${i} ${'x'.repeat(i)}`,
+    importance: 7, memoryLevel: 'semi_abstract', memoryType: 'long_term',
+  }))
+}
+
+const readCounts = () => {
+  const db = new Database(DB_PATH, { readonly: true })
+  const rows = db.prepare(`SELECT rowid, access_count FROM memories WHERE rowid IN (${ids.map(() => '?').join(',')})`).all(...ids)
+  db.close()
+  return new Map(rows.map(r => [String(r.rowid), r.access_count || 0]))
+}
+
+const before = readCounts()
+// min_importance > 0 is what makes recallForClients over-fetch — the shape the hooks send.
+const res = await recallForClients({ query: marker, limit: 2, minImportance: 1, source: 'test' })
+const after = readCounts()
+
+const returned = new Set(res.hits.map(h => String(h.id)))
+const bumped = [...after].filter(([id, n]) => n > (before.get(id) || 0)).map(([id]) => id)
+
+check('the pool held more candidates than were returned', res.hits.length === 2 && bumped.length <= 2,
+  `returned=${res.hits.length} bumped=${bumped.length}`)
+check('every returned row was bumped', [...returned].every(id => bumped.includes(id)),
+  `returned=${[...returned]} bumped=${bumped}`)
+check('no row outside the result was bumped', bumped.every(id => returned.has(id)),
+  `bumped=${bumped} returned=${[...returned]}`)
+
+// preferVec is the fail-open twin of requireVec. This temp DB has no embedding
+// config, i.e. the zero-config install: requireVec must return nothing and
+// preferVec must fall back to the FTS rows in the same call.
+const strict = await recallForClients({ query: marker, limit: 2, minImportance: 1, requireVec: true, source: 'test' })
+const lenient = await recallForClients({ query: marker, limit: 2, minImportance: 1, preferVec: true, source: 'test' })
+check('requireVec without embeddings returns nothing', strict.hits.length === 0, `got ${strict.hits.length}`)
+check('preferVec without embeddings falls back to FTS rows', lenient.hits.length === 2, `got ${lenient.hits.length}`)
+
+closeMemory()
+console.log(`\n${fail === 0 ? 'PASS' : 'FAIL'}: ${pass} passed / ${fail} failed`)
+process.exit(fail === 0 ? 0 : 1)

db-path-late-env.integration.test.mjs

@@ -0,0 +1,47 @@
+// The DB path is read when the store opens, not when the module is imported.
+//
+// A host that imports mneme and then loads its own env file sets
+// TOKENMEM_DB_PATH after ES import hoisting has already run this module's top
+// level. With a module-level const the store silently opened the fallback DB
+// beside index.mjs — and with more than one tenant on a machine, one tenant's
+// writes landed in another's store.
+//
+// This file re-runs itself as a child with TOKENMEM_DB_PATH unset at import
+// time, then sets it in the body — the host's order — and checks where the
+// store actually opened.
+//
+// Run: TOKENMEM_DB_PATH=/tmp/x.db node db-path-late-env.integration.test.mjs
+import { spawnSync } from 'node:child_process'
+import { existsSync, unlinkSync } from 'node:fs'
+import { fileURLToPath } from 'node:url'
+
+if (process.env.MNEME_LATE_DB_CHILD) {
+  const { initMemory, closeMemory, storeMemory } = await import('./index.mjs')
+  process.env.TOKENMEM_DB_PATH = process.env.MNEME_LATE_DB_CHILD   // host sets it after import
+  initMemory()
+  storeMemory({ content: 'late env marker', importance: 5, memoryLevel: 'semi_abstract', memoryType: 'long_term' })
+  closeMemory()
+  process.exit(0)
+}
+
+const target = process.env.TOKENMEM_DB_PATH
+if (!target) { console.error('FATAL: set TOKENMEM_DB_PATH'); process.exit(2) }
+const latePath = target + '.late.db'
+for (const sfx of ['', '-shm', '-wal']) { if (existsSync(latePath + sfx)) unlinkSync(latePath + sfx) }
+
+const env = { ...process.env, MNEME_LATE_DB_CHILD: latePath }
+delete env.TOKENMEM_DB_PATH
+delete env.MNEME_DB_PATH
+const r = spawnSync(process.execPath, [fileURLToPath(import.meta.url)], { env, encoding: 'utf-8', timeout: 30000 })
+
+let pass = 0, fail = 0
+const check = (label, cond, detail = '') => {
+  if (cond) { pass++; console.log(`✓ ${label}`) }
+  else { fail++; console.log(`✗ ${label}${detail ? ' — ' + detail : ''}`) }
+}
+check('child ran cleanly', r.status === 0, (r.stderr || '').slice(0, 300))
+check('store opened the path set after import', existsSync(latePath), latePath)
+
+for (const sfx of ['', '-shm', '-wal']) { try { unlinkSync(latePath + sfx) } catch {} }
+console.log(`\n${fail === 0 ? 'PASS' : 'FAIL'}: ${pass} passed / ${fail} failed`)
+process.exit(fail === 0 ? 0 : 1)

docs/configuring-your-agent.md

@@ -234,9 +234,10 @@ be re-injected within one Claude Code session.
 | `MNEME_DB_PATH` | mneme's own `engram.db` | Alias for `TOKENMEM_DB_PATH`; picks the DB file |
 | `MNEME_INDEX_PATH` | `<mneme>/index.mjs` | Override the engine entry point |
 | `MNEME_MIN_IMPORTANCE` | `6` | Floor for prompt-recall hits |
-| `MNEME_LEVEL` | `meta_knowledge` | Prompt-recall level filter |
+| `MNEME_LEVEL` | `meta_knowledge,semi_abstract` | Prompt-recall level filter |
+| `MNEME_MAX_VEC_DISTANCE` | `0.95` | Prompt-recall drops hits farther than this when vectors are available |
 | `MNEME_LIMIT` | `5` | Prompt-recall candidate cap |
-| `MNEME_MIN_CONSENSUS` | `2` | Prompt-recall skip if fewer hits |
+| `MNEME_MIN_CONSENSUS` | `2` | Prompt-recall skip if fewer hits (only when no vector evidence) |
 | `MNEME_TOOL_MIN_IMPORTANCE` | `6` | Floor for tool-recall hits |
 | `MNEME_TOOL_LEVEL` | `meta_knowledge,semi_abstract` | Tool-recall level filter |
 | `MNEME_TOOL_LIMIT` | `4` | Tool-recall candidate cap |

docs/configuring-your-agent.zh-CN.md

@@ -206,9 +206,10 @@ MCP tool 是 pull 模式——Agent 自己决定何时 `recall_memory`。有些
 | `MNEME_DB_PATH` | mneme 自带 `engram.db` | `TOKENMEM_DB_PATH` 的别名，选 DB 文件 |
 | `MNEME_INDEX_PATH` | `<mneme>/index.mjs` | 覆盖引擎入口 |
 | `MNEME_MIN_IMPORTANCE` | `6` | prompt-recall 命中门槛 |
-| `MNEME_LEVEL` | `meta_knowledge` | prompt-recall level 过滤 |
+| `MNEME_LEVEL` | `meta_knowledge,semi_abstract` | prompt-recall level 过滤 |
+| `MNEME_MAX_VEC_DISTANCE` | `0.95` | 有向量时，prompt-recall 丢弃距离大于此值的命中 |
 | `MNEME_LIMIT` | `5` | prompt-recall 候选上限 |
-| `MNEME_MIN_CONSENSUS` | `2` | prompt-recall 少于此数就不注入 |
+| `MNEME_MIN_CONSENSUS` | `2` | prompt-recall 少于此数就不注入（仅在没有向量证据时生效） |
 | `MNEME_TOOL_MIN_IMPORTANCE` | `6` | tool-recall 命中门槛 |
 | `MNEME_TOOL_LEVEL` | `meta_knowledge,semi_abstract` | tool-recall level 过滤 |
 | `MNEME_TOOL_LIMIT` | `4` | tool-recall 候选上限 |

hooks/prompt-recall-trigger.mjs

@@ -22,6 +22,27 @@ const TRIGGERS = [
   /(?:路径|目录|位置|端口|凭证|密钥|环境变量|配置|脚本|工具|命令|启动|重启|守护进程)/,
 ]
 
+// UserPromptSubmit carries more than what the user typed. In Claude Code,
+// background-agent reports arrive as <task-notification>, messages from other
+// sessions as <cross-session-message>, and app notices get prepended as
+// <system-reminder>. Recalling on the raw prompt means recalling on that
+// wrapper text: on one store, 631 of 688 fast-path queries over two weeks were
+// wrapper text, and those reports are dense with exactly the infrastructure
+// nouns the triggers below look for.
+//
+// Returns '' when nothing user-authored remains. Known trade-off: a user who
+// pastes text that itself starts with one of these tags gets no recall for it.
+const NOT_USER_INPUT = /^<(task-notification|cross-session-message)\b/
+
+export function userPromptText(raw) {
+  if (!raw || typeof raw !== 'string') return ''
+  const text = raw.replace(/<system-reminder>[\s\S]*?<\/system-reminder>/g, '').trim()
+  // An unterminated block (truncated upstream) would otherwise survive whole
+  // and be sent as the query.
+  if (NOT_USER_INPUT.test(text) || text.startsWith('<system-reminder')) return ''
+  return text
+}
+
 export function shouldTriggerPromptRecall(prompt) {
   if (!prompt || prompt.length < 4) return false
   if (prompt.length > 1500) return false

hooks/prompt-recall-trigger.test.mjs

@@ -16,3 +16,22 @@ test('short steering and long pasted text stay outside the gate', () => {
   assert.equal(shouldTriggerPromptRecall('推进'), false)
   assert.equal(shouldTriggerPromptRecall('召回'.repeat(751)), false)
 })
+
+test('only user-authored text reaches the trigger and the query', async () => {
+  const { userPromptText } = await import('./prompt-recall-trigger.mjs')
+  // App notice prepended to a real prompt: the notice goes, the prompt stays.
+  assert.equal(
+    userPromptText('<system-reminder>\nThe user started background task X ("fix daemon restart")\n</system-reminder>\nwhy is recall noisy'),
+    'why is recall noisy',
+  )
+  // Background-agent reports and other sessions' messages are not user input,
+  // even though they are full of the nouns the triggers look for.
+  assert.equal(userPromptText('<task-notification>\n<result>daemon config path port restart</result>\n</task-notification>'), '')
+  assert.equal(userPromptText('<cross-session-message from="x">where is the config</cross-session-message>'), '')
+  assert.equal(shouldTriggerPromptRecall(userPromptText('<task-notification>how to restart the daemon</task-notification>')), false)
+  // Plain prompts pass through untouched.
+  assert.equal(userPromptText('  how to restart the daemon '), 'how to restart the daemon')
+  assert.equal(userPromptText(undefined), '')
+  // A truncated reminder is not user input either.
+  assert.equal(userPromptText('<system-reminder>\nThe user started task X and the text was cut'), '')
+})

hooks/prompt-recall.mjs

@@ -25,7 +25,9 @@
 //   MNEME_DB_PATH          alias for TOKENMEM_DB_PATH; where mneme's engram.db lives
 //   MNEME_INDEX_PATH       override for index.mjs (default: ../index.mjs)
 //   MNEME_MIN_IMPORTANCE   floor for hits (default: 6)
-//   MNEME_LEVEL            recall level filter (default: meta_knowledge)
+//   MNEME_LEVEL            recall level filter (default: meta_knowledge,semi_abstract)
+//   MNEME_MAX_VEC_DISTANCE drop hits farther than this when the server returned
+//                          vector evidence (default: 0.95; ignored without embeddings)
 //   MNEME_LIMIT            max recall candidates (default: 5)
 //   MNEME_MIN_CONSENSUS    hide injection if hits < this (default: 2)
 //   MNEME_STATE_DIR        session-dedup file dir (default: ~/.claude/hooks)
@@ -43,7 +45,7 @@ import { spawnSync } from 'node:child_process'
 import { readFileSync, writeFileSync, existsSync, mkdirSync, renameSync } from 'node:fs'
 import { resolve, dirname } from 'node:path'
 import { fileURLToPath } from 'node:url'
-import { shouldTriggerPromptRecall } from './prompt-recall-trigger.mjs'
+import { shouldTriggerPromptRecall, userPromptText } from './prompt-recall-trigger.mjs'
 
 const __dirname = dirname(fileURLToPath(import.meta.url))
 const HOME = process.env.USERPROFILE || process.env.HOME || __dirname
@@ -98,7 +100,12 @@ const CFG = {
   httpTimeoutMs: intEnv('MNEME_HTTP_TIMEOUT_MS', 1500),
   indexPath: process.env.MNEME_INDEX_PATH || resolve(__dirname, '..', 'index.mjs'),
   minImportance: intEnv('MNEME_MIN_IMPORTANCE', 6),
-  level: process.env.MNEME_LEVEL || 'meta_knowledge',
+  // semi_abstract is in the default on purpose. The triggers are operational
+  // (paths, ports, restarts, config), and the write-time meta gate downgrades
+  // anything carrying a path, port or version to semi_abstract — so under a
+  // meta-only filter the answers this hook exists to find could never match.
+  level: process.env.MNEME_LEVEL || 'meta_knowledge,semi_abstract',
+  maxVecDistance: (() => { const n = Number(process.env.MNEME_MAX_VEC_DISTANCE); return Number.isFinite(n) && n > 0 ? n : 0.95 })(),
   limit: intEnv('MNEME_LIMIT', 5),
   minConsensus: intEnv('MNEME_MIN_CONSENSUS', 2),
   stateDir: process.env.MNEME_STATE_DIR || resolve(HOME, '.claude', 'hooks'),
@@ -166,6 +173,13 @@ async function runRecall(query, sessionId) {
     limit: CFG.limit,
     min_importance: CFG.minImportance,
     level: CFG.level,
+    // Prefer rows with vector evidence BEFORE the server trims to `limit`.
+    // Otherwise the few slots go to character-level FTS and entity matches —
+    // on CJK text that is mostly generic rows sharing one common noun — and the
+    // semantically relevant rows sit just below the cut. Fail-open: with no
+    // embeddings the server returns plain FTS rows in the same round trip.
+    // (Servers older than 2.11.1 ignore the field and behave as before.)
+    prefer_vec: true,
     source: 'mneme-prompt-recall',
     session_id: sessionId,
     // Tell the server how long we will actually wait, so a slow embedding
@@ -185,6 +199,7 @@ async function runRecall(query, sessionId) {
     '--level', CFG.level,
     '--limit', String(CFG.limit),
     '--source', 'mneme-prompt-recall',
+    '--prefer-vec',
     '--session-id', sessionId,
   ]
   const r = spawnSync(process.execPath, args, {
@@ -219,16 +234,24 @@ process.stdin.on('end', async () => {
   try { payload = JSON.parse(input || '{}') } catch { process.exit(0) }
 
   const sessionId = payload.session_id || payload.sessionId || 'unknown'
-  const prompt = (payload.prompt || '').trim()
+  const prompt = userPromptText(payload.prompt || '')
 
   if (!shouldTriggerPromptRecall(prompt)) process.exit(0)
 
   const query = prompt.slice(0, 500)
   const recalled = await runRecall(query, sessionId)
   if (!recalled || !Array.isArray(recalled.hits)) process.exit(0)
 
-  const hits = recalled.hits
-  if (hits.length < CFG.minConsensus) process.exit(0)
+  // With vector evidence present, relevance is measurable — gate on it and
+  // skip the count heuristic. Without it (no embeddings), fall back to
+  // requiring agreement between several FTS hits.
+  let hits = recalled.hits
+  if (hits.some(h => typeof h.vec_distance === 'number')) {
+    hits = hits.filter(h => typeof h.vec_distance === 'number' && h.vec_distance <= CFG.maxVecDistance)
+    if (hits.length === 0) process.exit(0)
+  } else if (hits.length < CFG.minConsensus) {
+    process.exit(0)
+  }
 
   const injected = loadInjected(sessionId)
   const fresh = hits.filter(h => !injected.has(h.id))