From adac8e0df1b31b229b723a5b06485e65f567230d Mon Sep 17 00:00:00 2001 From: Jean-Baptiste THERY Date: Sun, 1 Nov 2026 23:25:16 +0700 Subject: [PATCH 1/8] feat(core): refuse questions about absent topics without a model and sharpen ranking - weight function words such as "what", "de", or "les" at 0.3 in the keyword query, and lift passages whose file name or heading path holds words of the question by up to 20% - ignore words that more than half of the passages hold in those boosts, so a heading word that every file shares no longer buries the passage both channels ranked first - with a semantic model, require half of the question's IDF weight for a passage that only the keyword search returned, and credit fuzzy matches and identifier parts in coverage - with local-hash, answer nothing when none of the first five passages holds 30% of the question's weighted words, or when words found nowhere in the index carry half of its weight - weigh a missing word as if one passage held it, so one unusual verb on a small index does not pass for the question's focus - memoize query-independent work per passage text and file path and read Arrow rows as plain objects, for faster warm searches - raise the ranking policy to version 9 --- packages/ragmir-core/src/query.test.ts | 28 +++ packages/ragmir-core/src/query.ts | 111 ++++++-- packages/ragmir-core/src/ranking.test.ts | 108 +++++++- packages/ragmir-core/src/ranking.ts | 236 +++++++++++++----- .../ragmir-core/src/retrieval-diversity.ts | 6 +- .../ragmir-core/src/source-weights.test.ts | 79 +++++- packages/ragmir-core/src/source-weights.ts | 167 +++++++++---- packages/ragmir-core/src/text-memo.test.ts | 14 ++ packages/ragmir-core/src/text-memo.ts | 29 +++ packages/ragmir-core/src/text.ts | 27 ++ 10 files changed, 661 insertions(+), 144 deletions(-) create mode 100644 packages/ragmir-core/src/text-memo.test.ts create mode 100644 packages/ragmir-core/src/text-memo.ts diff --git a/packages/ragmir-core/src/query.test.ts b/packages/ragmir-core/src/query.test.ts index 9d84479..2f1afe5 100644 --- a/packages/ragmir-core/src/query.test.ts +++ b/packages/ragmir-core/src/query.test.ts @@ -872,6 +872,34 @@ describe("search", { timeout: integrationTimeout(20_000) }, () => { expect(typo[0]?.relativePath).toBe(".ragmir/raw/security.md") expect(typo[0]?.score?.lexicalQueryVariants).toBeGreaterThan(1) expect(wrongWord).toEqual([]) + const typoAmongKnownWords = await search("sceurity controls", { cwd: root, topK: 1 }) + expect(typoAmongKnownWords[0]?.relativePath).toBe(".ragmir/raw/security.md") + expect(await search("pinecone controls", { cwd: root, topK: 1 })).toEqual([]) + }) + + it("should not let a heading word that every passage holds outrank the agreed answer", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-query-common-heading-")) + tempDirs.push(root) + await initProject(root) + const raw = path.join(root, ".ragmir", "raw") + await mkdir(raw, { recursive: true }) + for (const name of ["contract", "runbook", "policy"]) { + await writeFile( + path.join(raw, `${name}.md`), + `# Synthetic ${name}\n\nSynthetic ${name} notes for the demo.\n`, + "utf8", + ) + } + await writeFile( + path.join(raw, "scan.md"), + "Synthetic confidential scan of the signed control evidence.\n", + "utf8", + ) + await ingest({ cwd: root }) + + const results = await search("synthetic confidential evidence", { cwd: root, topK: 2 }) + + expect(results[0]?.relativePath).toBe(".ragmir/raw/scan.md") }) it("blocks search when the active index policy differs", async () => { diff --git a/packages/ragmir-core/src/query.ts b/packages/ragmir-core/src/query.ts index 5de0e52..5b02541 100644 --- a/packages/ragmir-core/src/query.ts +++ b/packages/ragmir-core/src/query.ts @@ -31,9 +31,11 @@ import { operationSignal, throwIfAborted } from "./operation.js" import { sanitizeRetrievalQuery } from "./query-sanitizer.js" import type { QueryTermStatistics, RankedRow } from "./ranking.js" import { + absentShareOf, abstentionFilter, fusionCandidateDepth, fuzzyMatchPrefilter, + isLowInformationWord, isWeakAnswer, queryEvidence, queryTermCoverage, @@ -51,7 +53,13 @@ import { resolveExistingPath, unicodePathVariants, } from "./source-paths.js" -import { definitionBoostOf, localeDocumentKey, sourceWeightOf } from "./source-weights.js" +import { + definitionBoostOf, + fileNameBoostOf, + headingBoostOf, + localeDocumentKey, + sourceWeightOf, +} from "./source-weights.js" import type { IndexReadSnapshot } from "./store.js" import { closeIndexReadSnapshot, loadIndexReadSnapshot } from "./store.js" import { normalizeForMatch, tokenize } from "./text.js" @@ -283,9 +291,9 @@ async function searchWithinGeneration( manifest.vectorIndex, options.vectorSearchMode === "exact", ) - return (await (predicate ? vectorQuery.where(predicate) : vectorQuery) - .limit(limit) - .toArray()) as SearchRow[] + return plainRows( + await (predicate ? vectorQuery.where(predicate) : vectorQuery).limit(limit).toArray(), + ) } const rows = await nearest(retrievalPredicate) const crowding = crowdingDocuments(rows, topK) @@ -314,25 +322,31 @@ async function searchWithinGeneration( } const vectorRows = vectorRowsOutcome.value const lexicalCandidates = lexicalCandidatesOutcome.value - const termStatistics = withFuzzyMatches(await termFrequenciesPromise, queryTokens, [ - ...lexicalCandidates.rows, - ...vectorRows, - ]) - throwIfAborted(signal) const rankingPolicy = rankingPolicyFor( config.embeddingProvider, config.retrievalProfile, maxChunksPerDocument, config.embeddingModel, ) + const termStatistics = withFuzzyMatches( + await termFrequenciesPromise, + queryTokens, + [...lexicalCandidates.rows, ...vectorRows], + rankingPolicy.absentFocusShare, + ) + throwIfAborted(signal) const weightOf = sourceWeightOf(config) - const definitionBoost = definitionBoostOf(queryTokens) + const boostTokens = distinctiveTokens(queryTokens, termStatistics) + const definitionBoost = definitionBoostOf(boostTokens) + const fileNameBoost = fileNameBoostOf(boostTokens) + const headingBoost = headingBoostOf(boostTokens) const rankedRows = rankHybridRows( sanitized.query, vectorRows, lexicalCandidates.rows, rankingPolicy, - (row) => weightOf(row.relativePath) * definitionBoost(row), + (row) => + weightOf(row.relativePath) * definitionBoost(row) * fileNameBoost(row) * headingBoost(row), ) const isExactPathMatch = (row: SearchRow): boolean => lexicalCandidates.exactPathMatches.has(rowKey(row)) @@ -358,7 +372,6 @@ async function searchWithinGeneration( // than noise. An exact path match is never weak. const rows = isWeakAnswer( diversity.rows, - rankingPolicy, (row) => !isExactPathMatch(row.row) && hasWeakEvidence(row.row), ) ? [] @@ -545,21 +558,42 @@ async function queryTermFrequencies( } } +/** + * Query words that can tell passages apart. A word held by more than half of the passages has no + * positive inverse document frequency, so a file-name, heading, or definition match on it would lift + * nearly every passage above the one both channels ranked first. + */ +function distinctiveTokens( + tokens: readonly string[], + statistics: QueryTermStatistics | null, +): readonly string[] { + if (!statistics) { + return tokens + } + return tokens.filter( + (token) => (statistics.documentFrequencies.get(token) ?? 0) * 2 <= statistics.documentCount, + ) +} + /** Mark absent words that a candidate still matches fuzzily, so typos are not treated as noise. */ function withFuzzyMatches( statistics: QueryTermStatistics | null, tokens: readonly string[], candidateRows: readonly SearchRow[], + absentFocusShare: number, ): QueryTermStatistics | null { if (!statistics) { return null } - const measured = [...new Set(tokens)].filter((token) => !statistics.unmeasuredTerms?.has(token)) - // Fuzzy matches only decide whether no query word is known at all; skip the scan otherwise. - if (measured.some((token) => (statistics.documentFrequencies.get(token) ?? 0) > 0)) { + // Fuzzy matches only decide whether the question's focus is absent, so a misspelled word does not + // make an answerable question look unanswerable; skip the scan when the focus is present anyway. + if (absentShareOf(tokens, statistics) < absentFocusShare) { return statistics } - const absent = measured.filter((token) => token.length >= 7) + const measured = [...new Set(tokens)].filter((token) => !statistics.unmeasuredTerms?.has(token)) + const absent = measured.filter( + (token) => token.length >= 7 && (statistics.documentFrequencies.get(token) ?? 0) === 0, + ) if (absent.length === 0 || candidateRows.length === 0) { return statistics } @@ -973,9 +1007,21 @@ async function executeFullTextQuery( pathPredicate: string | null, ): Promise { const searchQuery = table.search(query, "fts", "searchText").select(FTS_SEARCH_COLUMNS) - return (await (pathPredicate ? searchQuery.where(pathPredicate) : searchQuery) - .limit(limit) - .toArray()) as SearchRow[] + return plainRows( + await (pathPredicate ? searchQuery.where(pathPredicate) : searchQuery).limit(limit).toArray(), + ) +} + +/** + * LanceDB returns Arrow row proxies that decode a field on every read, and ranking reads the path, + * text, and score of each candidate many times, so candidates become plain objects once. + */ +function plainRows(rows: unknown[]): SearchRow[] { + return rows.map((row) => + typeof (row as { toJSON?: unknown }).toJSON === "function" + ? (row as { toJSON(): SearchRow }).toJSON() + : (row as SearchRow), + ) } async function executeSourcePathQuery( @@ -1304,6 +1350,31 @@ function citationForRow(row: SearchRow): string { return citationForCoordinates(row) } +// Function words still help phrase-like matches but must not rank a passage on their own. The +// keyword analyzer drops English stop words such as "the", yet keeps question words and French +// function words: "de" and "les" match most passages of a French corpus and "what" a rare few that +// then look distinctive. Weighting them 0.3 raised top-3 hits on three real corpora with and without +// an embedding model (from 45% to 60% on a French and Java monorepo without one), refusing no answer. +const FUNCTION_WORD_WEIGHT = 0.3 + +function keywordQuery(tokens: readonly string[]): FullTextQuery { + const content = tokens.filter((token) => !isLowInformationWord(token)) + const functionWords = tokens.filter(isLowInformationWord) + if (content.length === 0 || functionWords.length === 0) { + return new MatchQuery(tokens.join(" "), "searchText", { operator: Operator.Or }) + } + return new BooleanQuery([ + [Occur.Should, new MatchQuery(content.join(" "), "searchText", { operator: Operator.Or })], + [ + Occur.Should, + new MatchQuery(functionWords.join(" "), "searchText", { + operator: Operator.Or, + boost: FUNCTION_WORD_WEIGHT, + }), + ], + ]) +} + function lexicalQuery( query: string, ): { primary: FullTextQuery; supplemental: FullTextQuery[] } | null { @@ -1312,7 +1383,7 @@ function lexicalQuery( return null } const joined = tokens.join(" ") - const broadQuery = new MatchQuery(joined, "searchText", { operator: Operator.Or }) + const broadQuery = keywordQuery(tokens) const supplemental: FullTextQuery[] = [] const identifierTerms = [...normalizeForMatch(query).matchAll(LEXICAL_IDENTIFIER_PATTERN)] .map((match) => match[0]) diff --git a/packages/ragmir-core/src/ranking.test.ts b/packages/ragmir-core/src/ranking.test.ts index a35c312..18bffc5 100644 --- a/packages/ragmir-core/src/ranking.test.ts +++ b/packages/ragmir-core/src/ranking.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest" import { + absentShareOf, abstentionFilter, candidatePassesAbstention, effectiveVectorDistances, @@ -30,7 +31,7 @@ describe("hybrid ranking", () => { const alpha = row("alpha.md", "Shared policy evidence.", { distance: 0.5, score: 3 }) const beta = row("beta.md", "Shared policy evidence.", { distance: 0.5, score: 3 }) const policy = rankingPolicyFor("local-hash", "balanced", 1) - expect(policy.version).toBe(8) + expect(policy.version).toBe(9) const forward = rankHybridRows("policy evidence", [beta, alpha], [alpha, beta], policy) const reversed = rankHybridRows("policy evidence", [alpha, beta], [beta, alpha], policy) @@ -358,15 +359,108 @@ describe("noise floor abstention", () => { expect(weak(row("far.md", "Daily operations.", { distance: 0.4 }))).toBe(true) }) - it("should refuse a semantic answer only when every leading result is weak", () => { + it("should need half of the question's weight for a passage that only keywords returned", () => { + const evidence = queryEvidence("weather forecast tomorrow") + const weak = weakEvidenceFilter(evidence, uncalibrated, terms, 0.32) + const localHash = rankingPolicyFor("local-hash", "balanced", 1) + + expect(uncalibrated.lexicalOnlyEvidenceCoverage).toBe(0.5) + expect(weak(row("keyword.md", "Weather station notes.", {}))).toBe(true) + expect(weak(row("vector.md", "Weather station notes.", { distance: 0.4 }))).toBe(false) + expect(weak(row("both.md", "Weather forecast notes.", {}))).toBe(false) + expect(localHash.lexicalOnlyEvidenceCoverage).toBe(localHash.weakEvidenceCoverage) + expect( + weakEvidenceFilter( + evidence, + localHash, + terms, + )(row("keyword.md", "Weather station notes.", {})), + ).toBe(false) + }) + + it("should credit fuzzy keyword matches and the parts of joined words", () => { + const typo = queryEvidence("sceurity") + const typoTerms: QueryTermStatistics = { + documentCount: 10, + documentFrequencies: new Map([["sceurity", 0]]), + fuzzyMatchedTerms: new Set(["sceurity"]), + } + const joined = queryEvidence("compliance abstention") + const joinedTerms: QueryTermStatistics = { + documentCount: 10, + documentFrequencies: new Map([ + ["compliance", 1], + ["abstention", 1], + ]), + } + + expect(queryTermCoverage(typo, typoTerms)("Security controls require signed evidence.")).toBe(1) + expect(queryTermCoverage(typo, typoTerms)("The secretary signs.")).toBe(0) + expect( + queryTermCoverage(joined, joinedTerms)("$.compliance.description: règles de l'abstention"), + ).toBe(1) + }) + + it("should share one memoized coverage function per query and term statistics", () => { + const evidence = queryEvidence("weather forecast tomorrow") + const coverage = queryTermCoverage(evidence, terms) + + expect(queryTermCoverage(evidence, terms)).toBe(coverage) + expect(queryTermCoverage(queryEvidence("weather forecast tomorrow"), terms)).not.toBe(coverage) + expect(coverage("Weather forecast notes.")).toBe(coverage("Weather forecast notes.")) + }) + + it("should refuse local-hash answers when the question's focus is absent from the corpus", () => { + const evidence = queryEvidence("pinecone sync index") + const absentFocus: QueryTermStatistics = { + documentCount: 100, + documentFrequencies: new Map([ + ["pinecone", 0], + ["sync", 5], + ["index", 40], + ]), + } + const knownFocus: QueryTermStatistics = { + ...absentFocus, + documentFrequencies: new Map([...absentFocus.documentFrequencies, ["pinecone", 3]]), + } + const passage = row("notes.md", "Index sync notes.", { distance: 0.5 }) + const localHash = rankingPolicyFor("local-hash", "balanced", 1) + + expect(absentShareOf(evidence.tokens, absentFocus)).toBeGreaterThan(0.5) + expect(abstentionFilter(evidence, localHash, absentFocus)(passage)).toBe(false) + expect(abstentionFilter(evidence, localHash, knownFocus)(passage)).toBe(true) + // A semantic model keeps its vector evidence unless every word of the question is absent. + expect(abstentionFilter(evidence, uncalibrated, absentFocus, 0.31)(passage)).toBe(true) + expect( + absentShareOf(evidence.tokens, { ...absentFocus, fuzzyMatchedTerms: new Set(["pinecone"]) }), + ).toBe(0) + }) + + it("should not let one unseen word outweigh the rest of a question on a small index", () => { + const evidence = queryEvidence("What proves offline text-to-speech is required?") + const smallIndex: QueryTermStatistics = { + documentCount: 5, + documentFrequencies: new Map([ + ["proves", 0], + ["offline", 4], + ["text", 2], + ["speech", 2], + ["required", 4], + ]), + } + + expect(absentShareOf(evidence.tokens, smallIndex)).toBeCloseTo(0.373, 3) + }) + + it("should refuse an answer only when every leading result is weak", () => { const flags = (values: boolean[]) => values.map((weak, index) => ({ index, weak })) const isWeak = (entry: { weak: boolean }) => entry.weak - expect(isWeakAnswer(flags([true, true, true, true, true, false]), e5, isWeak)).toBe(true) - expect(isWeakAnswer(flags([true, false, true]), e5, isWeak)).toBe(false) - expect(isWeakAnswer([], e5, isWeak)).toBe(false) - const localHash = rankingPolicyFor("local-hash", "balanced", 1) - expect(isWeakAnswer(flags([true, true]), localHash, isWeak)).toBe(false) + expect(isWeakAnswer(flags([true, true, true, true, true, false]), isWeak)).toBe(true) + expect(isWeakAnswer(flags([true, false, true]), isWeak)).toBe(false) + expect(isWeakAnswer([], isWeak)).toBe(false) + expect(isWeakAnswer(flags([true, true]), isWeak)).toBe(true) }) }) diff --git a/packages/ragmir-core/src/ranking.ts b/packages/ragmir-core/src/ranking.ts index 44fa56a..eb55465 100644 --- a/packages/ragmir-core/src/ranking.ts +++ b/packages/ragmir-core/src/ranking.ts @@ -5,7 +5,8 @@ import { lexicalDocument, lexicalDocumentScore, } from "./lexical-scoring.js" -import { identifierParts, normalizeForMatch, tokenize } from "./text.js" +import { identifierParts, normalizeForMatch, shareStem, stemPrefix, tokenize } from "./text.js" +import { memoizeByText } from "./text-memo.js" import type { EmbeddingProvider, RetrievalProfile } from "./types.js" export interface RankingRow { @@ -29,7 +30,7 @@ export interface RankedRow { } export interface RankingPolicy { - version: 8 + version: 9 embeddingProvider: EmbeddingProvider /** Semantic model whose calibrated evidence profile applies; null for local-hash. */ embeddingModel: string | null @@ -47,6 +48,13 @@ export interface RankingPolicy { minimumLexicalCoverage: number /** Below this coverage, and without a strong vector match, a passage is flagged weakEvidence. */ weakEvidenceCoverage: number + /** Coverage a passage needs when only the keyword channel returned it, without vector support. */ + lexicalOnlyEvidenceCoverage: number + /** + * Share of the question's IDF weight on words found nowhere in the index at which its focus counts + * as absent: only a passage within the vocabulary-free distance can then answer, none without one. + */ + absentFocusShare: number } export interface QueryEvidence { @@ -126,9 +134,24 @@ const LOCAL_HASH_PROFILE: EvidenceProfile = { minimumLexicalCoverage: 0, weakEvidenceCoverage: 0.3, } -const MIN_STEM_PREFIX_LENGTH = 5 -const MAX_STEM_SUFFIX_LENGTH = 4 +// With a semantic model, a passage the vector channel did not return at all has keyword support +// only. Off-topic questions share one everyday word with many documents ("height", "capital"), +// while answers on three real corpora came from the keyword channel alone in 2 of 250 questions. +// With the other 6.5 rules, requiring half of the question's IDF weight there raised refusals of +// off-topic questions on those corpora from 54 to 68 of 90, without refusing an answer. +const LEXICAL_ONLY_EVIDENCE_COVERAGE = 0.5 +// A question whose most distinctive words occur nowhere in the index asks about something the corpus +// lacks ("How does it sync with Pinecone?"). Without a model nothing else can vouch for a passage, +// so local-hash refuses once absent words carry half of the question's IDF weight: on three real +// corpora, 65 of 90 off-topic and 27 of 110 near-domain questions instead of 56 and 17, with none of +// 250 answers refused, and 7 instead of 5 once an unknown word was added to each of those questions. +// Semantic models keep the vector channel's judgment and only refuse when every word is absent: a +// 40% share there refused 17 more of those 250 altered questions for 17 more unanswerable ones. +const SEMANTIC_ABSENT_FOCUS_SHARE = 1 +const LOCAL_HASH_ABSENT_FOCUS_SHARE = 0.5 const IDENTIFIER_PATTERN = /[\p{L}\p{N}]+(?:[-_./][\p{L}\p{N}]+)+/gu +const WORD_JOINER_PATTERN = /[^\p{L}\p{N}]/u +const WORD_JOINERS = /[^\p{L}\p{N}]+/u const LOW_INFORMATION_WORDS = new Set( ( "a about an and are as at be been by can could describe did do does for from had has have how " + @@ -141,6 +164,11 @@ const LOW_INFORMATION_WORDS = new Set( ).split(" "), ) +/** Whether a folded word carries little meaning on its own ("the", "what", "de", "les"). */ +export function isLowInformationWord(token: string): boolean { + return LOW_INFORMATION_WORDS.has(token) +} + export function rankingPolicyFor( embeddingProvider: EmbeddingProvider, retrievalProfile: RetrievalProfile, @@ -153,7 +181,7 @@ export function rankingPolicyFor( DEFAULT_TRANSFORMERS_PROFILE) : LOCAL_HASH_PROFILE return { - version: 8, + version: 9, embeddingProvider, embeddingModel: semantic ? (embeddingModel ?? null) : null, retrievalProfile, @@ -162,6 +190,10 @@ export function rankingPolicyFor( vectorWeight: semantic ? RRF_SEMANTIC_VECTOR_WEIGHT : RRF_LOCAL_HASH_VECTOR_WEIGHT, lexicalWeight: RRF_LEXICAL_WEIGHT, ...profile, + lexicalOnlyEvidenceCoverage: semantic + ? LEXICAL_ONLY_EVIDENCE_COVERAGE + : profile.weakEvidenceCoverage, + absentFocusShare: semantic ? SEMANTIC_ABSENT_FOCUS_SHARE : LOCAL_HASH_ABSENT_FOCUS_SHARE, } } @@ -216,8 +248,8 @@ export function rankHybridRows( } return key } - const compareKeys = (left: Row, right: Row): number => keyOf(left).localeCompare(keyOf(right)) - const rows = mergeRows(vectorRows, textRows, keyOf, compareKeys) + const compareKeys = (left: Row, right: Row): number => compareOrdinal(keyOf(left), keyOf(right)) + const rows = mergeRows(vectorRows, textRows, keyOf) const exactAnchorMatches = new Map() if (evidence.anchors.length > 0) { const requestedAnchors = new Set(evidence.anchors) @@ -329,36 +361,68 @@ export function effectiveVectorDistances( const WEAK_ANSWER_DEPTH = 5 /** - * Whether a semantic answer holds no real evidence: every leading result is weak. Off-topic and - * near-domain questions without an answer typically land here. Local-hash search keeps such - * results, since its weak flag was never calibrated to refuse. + * Whether an answer holds no real evidence: every leading result is weak. Off-topic and + * near-domain questions without an answer typically land here. With local-hash, whose weak flag + * is keyword coverage alone, this refused 56 of 90 off-topic questions on three real corpora + * instead of 12, and none of 250 answerable ones: the least covered correct answer still held 35% + * of its question's IDF-weighted words. */ -export function isWeakAnswer( - rows: readonly Row[], - policy: RankingPolicy, - isWeak: (row: Row) => boolean, -): boolean { - if (policy.embeddingProvider === "local-hash" || rows.length === 0) { +export function isWeakAnswer(rows: readonly Row[], isWeak: (row: Row) => boolean): boolean { + if (rows.length === 0) { return false } return rows.slice(0, WEAK_ANSWER_DEPTH).every(isWeak) } +function idfWeightOf(documentFrequency: number, documentCount: number): number { + const frequency = Math.min(documentFrequency, documentCount) + return Math.log(1 + (documentCount - frequency + 0.5) / (frequency + 0.5)) +} + +/** Inverse document frequency weight of a query word, as the coverage rules use it. */ +function idfWeight(token: string, terms: QueryTermStatistics): number { + return idfWeightOf(terms.documentFrequencies.get(token) ?? 0, terms.documentCount) +} + /** - * Whether no query word occurs in the corpus, exactly or through a fuzzy keyword match. Only - * vector evidence can then support a passage, and nonsense or unrelated questions get none. + * Share of the IDF weight of these query words carried by words found nowhere in the index, neither + * exactly nor through a fuzzy keyword match. A missing word weighs as if one document held it: on a + * small index an unseen word would otherwise outweigh the rest of the question, so one unusual verb + * ("proves") could pass for the question's focus. Words in unsegmented scripts are not measured, + * and a query holding one never counts as absent. */ -function vocabularyFree(evidence: QueryEvidence, terms: QueryTermStatistics | null): boolean { +export function absentShareOf(tokens: readonly string[], terms: QueryTermStatistics): number { + const unique = [...new Set(tokens)] + if (unique.length === 0 || unique.some((token) => terms.unmeasuredTerms?.has(token))) { + return 0 + } + let total = 0 + let absent = 0 + for (const token of unique) { + const documentFrequency = terms.documentFrequencies.get(token) ?? 0 + const weight = idfWeightOf(Math.max(1, documentFrequency), terms.documentCount) + total += weight + if (documentFrequency === 0 && !terms.fuzzyMatchedTerms?.has(token)) { + absent += weight + } + } + return total > 0 ? absent / total : 0 +} + +/** + * Whether the question's focus is absent from the corpus: its words carrying `absentFocusShare` of + * the IDF weight occur nowhere, as with nonsense, unrelated, or near-domain questions about a missing + * feature. Only vector evidence can then support a passage. + */ +function focusAbsent( + evidence: QueryEvidence, + terms: QueryTermStatistics | null, + policy: RankingPolicy, +): boolean { return ( terms !== null && evidence.anchors.length === 0 && - evidence.tokens.length > 0 && - evidence.tokens.every( - (token) => - (terms.documentFrequencies.get(token) ?? 0) === 0 && - !terms.fuzzyMatchedTerms?.has(token) && - !terms.unmeasuredTerms?.has(token), - ) + absentShareOf(evidence.tokens, terms) >= policy.absentFocusShare ) } @@ -378,7 +442,7 @@ export function abstentionFilter( if (evidence.anchors.length > 0) { return (row) => hasLexicalEvidence(row.searchText) } - const noKnownWords = vocabularyFree(evidence, terms) + const noKnownWords = focusAbsent(evidence, terms, policy) if (policy.embeddingProvider === "local-hash") { return noKnownWords ? () => false : (row) => hasLexicalEvidence(row.searchText) } @@ -399,7 +463,8 @@ export function abstentionFilter( /** * Flag a selected passage whose support is thin: it holds a small IDF-weighted share of the query - * words and has no strong vector match. Search still returns it; agents should verify it first. + * words and has no strong vector match, or only the keyword channel returned it and it covers less + * than its policy share. Search still returns it; agents should verify it first. */ export function weakEvidenceFilter( evidence: QueryEvidence, @@ -412,49 +477,87 @@ export function weakEvidenceFilter( } const coverage = queryTermCoverage(evidence, terms) const strong = effectiveVectorDistances(policy, noiseFloor).strongVectorDistance - return (row) => - coverage(row.searchText) < policy.weakEvidenceCoverage && - !(strong !== null && rowDistance(row) <= strong) + return (row) => { + const distance = rowDistance(row) + if (strong !== null && distance <= strong) { + return false + } + const required = Number.isFinite(distance) + ? policy.weakEvidenceCoverage + : policy.lexicalOnlyEvidenceCoverage + return coverage(row.searchText) < required + } } +const coverageByTerms = new WeakMap< + QueryTermStatistics, + WeakMap number> +>() + /** * IDF-weighted share of query words present in a text. Words match exactly or as one stem with a * short ending (`permission`, `permissions`), so a rare word weighs more than a frequent one and a - * word missing from the corpus weighs the most. + * word missing from the corpus weighs the most. The abstention and weak-evidence filters of one + * query share a single memoized function, so each candidate text is segmented once. */ export function queryTermCoverage( evidence: QueryEvidence, terms: QueryTermStatistics, +): (text: string) => number { + let byEvidence = coverageByTerms.get(terms) + if (!byEvidence) { + byEvidence = new WeakMap() + coverageByTerms.set(terms, byEvidence) + } + let coverage = byEvidence.get(evidence) + if (!coverage) { + const measure = measureQueryTermCoverage(evidence, terms) + const known = new Map() + coverage = (text) => { + let value = known.get(text) + if (value === undefined) { + value = measure(text) + known.set(text, value) + } + return value + } + byEvidence.set(evidence, coverage) + } + return coverage +} + +function measureQueryTermCoverage( + evidence: QueryEvidence, + terms: QueryTermStatistics, ): (text: string) => number { const tokens = [...new Set(evidence.tokens)].filter((token) => !terms.unmeasuredTerms?.has(token)) - const weights = tokens.map((token) => { - const documentFrequency = Math.min( - terms.documentFrequencies.get(token) ?? 0, - terms.documentCount, - ) - return Math.log(1 + (terms.documentCount - documentFrequency + 0.5) / (documentFrequency + 0.5)) - }) + const weights = tokens.map((token) => idfWeight(token, terms)) const totalWeight = weights.reduce((sum, weight) => sum + weight, 0) if (totalWeight <= 0) { return () => 0 } + // A misspelled word that the keyword search matched fuzzily covers its fuzzy relatives, as there. + const fuzzy = tokens.map((token) => terms.fuzzyMatchedTerms?.has(token) ?? false) // Every word sharing a stem with a query word contains its stem prefix, so a passage whose folded // text holds none of them covers nothing. Unrelated candidates then skip word segmentation. const stemPrefixes = tokens.map(stemPrefix) return (text) => { - const folded = normalizeForMatch(text) - const present = stemPrefixes.map((prefix) => folded.includes(prefix)) + const folded = foldedText(text) + const present = stemPrefixes.map((prefix, index) => fuzzy[index] || folded.includes(prefix)) if (!present.includes(true)) { return 0 } - const textTokens = new Set([...tokenize(text), ...identifierParts(text)]) + const textTokens = coverageWords(text) let matched = 0 tokens.forEach((token, index) => { if (!present[index]) { return } for (const textToken of textTokens) { - if (shareStem(token, textToken)) { + if ( + shareStem(token, textToken) || + (fuzzy[index] && tokensAreFuzzyRelated(token, textToken)) + ) { matched += weights[index] ?? 0 break } @@ -464,25 +567,25 @@ export function queryTermCoverage( } } -/** The prefix that every word passing shareStem with this query word contains. */ -function stemPrefix(token: string): string { - return token.length < MIN_STEM_PREFIX_LENGTH - ? token - : token.slice(0, Math.max(MIN_STEM_PREFIX_LENGTH, token.length - MAX_STEM_SUFFIX_LENGTH)) -} +const foldedText = memoizeByText(normalizeForMatch) -function shareStem(queryToken: string, textToken: string): boolean { - if (queryToken === textToken) { - return true +/** + * The words of a passage as the keyword index splits them: word segmentation keeps JSONPath and + * dotted identifiers (`compliance.description`) and French elisions (`l'abstention`) whole, while + * the keyword index also matches each part. + */ +const coverageWords = memoizeByText((text: string): ReadonlySet => { + const words = new Set() + for (const word of [...tokenize(text), ...identifierParts(text)]) { + words.add(word) + if (WORD_JOINER_PATTERN.test(word)) { + for (const part of word.split(WORD_JOINERS)) { + if (part.length > 1) words.add(part) + } + } } - const shorter = queryToken.length <= textToken.length ? queryToken : textToken - const longer = shorter === queryToken ? textToken : queryToken - return ( - shorter.length >= MIN_STEM_PREFIX_LENGTH && - longer.length - shorter.length <= MAX_STEM_SUFFIX_LENGTH && - longer.startsWith(shorter) - ) -} + return words +}) export function tokensAreLexicallyRelated(queryToken: string, textToken: string): boolean { if (queryToken === textToken) { @@ -655,17 +758,17 @@ function isSingleEditApart(left: string, right: string): boolean { return true } +/** Merge both candidate lists by row key; the final ranking sort fixes the order. */ function mergeRows( vectorRows: Row[], textRows: Row[], keyOf: (row: Row) => string, - compareKeys: (left: Row, right: Row) => number, ): Row[] { const rows = new Map() - for (const row of [...textRows].sort(compareKeys)) { + for (const row of textRows) { rows.set(keyOf(row), row) } - for (const row of [...vectorRows].sort(compareKeys)) { + for (const row of vectorRows) { const key = keyOf(row) const existing = rows.get(key) if (!existing) { @@ -708,7 +811,12 @@ function bm25Scores( } function compareScoredKeys(left: [string, number], right: [string, number]): number { - return right[1] - left[1] || left[0].localeCompare(right[0]) + return right[1] - left[1] || compareOrdinal(left[0], right[0]) +} + +/** Locale-independent tie-break, so equal scores rank the same on every machine. */ +function compareOrdinal(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0 } function rowDistance(row: RankingRow): number { diff --git a/packages/ragmir-core/src/retrieval-diversity.ts b/packages/ragmir-core/src/retrieval-diversity.ts index 115f1b2..7b10935 100644 --- a/packages/ragmir-core/src/retrieval-diversity.ts +++ b/packages/ragmir-core/src/retrieval-diversity.ts @@ -1,4 +1,8 @@ import type { RankedRow, RankingRow } from "./ranking.js" +import { memoizeByText } from "./text-memo.js" + +/** Texts that differ only in spacing or case count as one passage. */ +const duplicateKeyOf = memoizeByText((text) => text.replace(/\s+/gu, " ").trim().toLowerCase()) interface DiversifiableRow extends RankingRow { text: string @@ -102,7 +106,7 @@ function uniqueRelevantRows( isRelevant: ((row: Row) => boolean) | undefined, isExactPathMatch: ((row: Row) => boolean) | undefined, ): (index: number) => RankedRow | undefined { - const textKeys = rows.map((ranked) => ranked.row.text.replace(/\s+/gu, " ").trim().toLowerCase()) + const textKeys = rows.map((ranked) => duplicateKeyOf(ranked.row.text)) const rowsByText = new Map() textKeys.forEach((textKey, index) => { const group = rowsByText.get(textKey) diff --git a/packages/ragmir-core/src/source-weights.test.ts b/packages/ragmir-core/src/source-weights.test.ts index e87bda6..df1dd87 100644 --- a/packages/ragmir-core/src/source-weights.test.ts +++ b/packages/ragmir-core/src/source-weights.test.ts @@ -1,5 +1,12 @@ import { describe, expect, it } from "vitest" -import { definitionBoostOf, localeDocumentKey, sourceWeightOf } from "./source-weights.js" +import { chunkLexicalText } from "./chunking.js" +import { + definitionBoostOf, + fileNameBoostOf, + headingBoostOf, + localeDocumentKey, + sourceWeightOf, +} from "./source-weights.js" describe("localeDocumentKey", () => { it("collapses translations of one document across common layouts", () => { @@ -54,17 +61,69 @@ describe("definitionBoostOf", () => { const boost = definitionBoostOf(["permissions", "checked"]) expect( - boost({ - relativePath: "src/auth/PermissionService.java", - text: "public class PermissionService {\n public boolean check(Role role) {", - }), + boost( + codeRow( + "src/auth/PermissionService.java", + "public class PermissionService {\n public boolean check(Role role) {", + ), + ), ).toBeCloseTo(1.14) + expect(boost(codeRow("docs/permissions.md", "class PermissionService checks access."))).toBe(1) + expect(boost(codeRow("src/util.ts", "export function formatDate() {}"))).toBe(1) + }) + + it("skips passages whose keyword text holds no stem of a query word", () => { + const boost = definitionBoostOf(["permissions"]) + const row = codeRow("src/auth/PermissionService.java", "public class PermissionService {}") + + expect(boost(row)).toBeCloseTo(1.07) + expect(boost({ ...row, searchText: "unrelated keyword text" })).toBe(1) + }) +}) + +describe("headingBoostOf", () => { + it("boosts passages whose heading path holds words of the question", () => { + const boost = headingBoostOf(["authentication", "tokens", "refresh"]) + + expect(boost({ contextPath: "Security > Authentication > Refresh tokens" })).toBeCloseTo(1.2) + expect(boost({ contextPath: "$.auth.refresh" })).toBeCloseTo(1.1) + expect(boost({ contextPath: "" })).toBe(1) + expect(boost({})).toBe(1) + }) +}) + +describe("fileNameBoostOf", () => { + it("boosts files whose name holds words of the question, two words at most", () => { expect( - boost({ - relativePath: "docs/permissions.md", - text: "class PermissionService checks access.", + fileNameBoostOf(["generated", "openapi", "document"])({ + relativePath: "src/OpenApiConfig.java", }), - ).toBe(1) - expect(boost({ relativePath: "src/util.ts", text: "export function formatDate() {}" })).toBe(1) + ).toBeCloseTo(1.1) + expect( + fileNameBoostOf(["permission", "service"])({ relativePath: "auth/PermissionService.java" }), + ).toBeCloseTo(1.2) + expect( + fileNameBoostOf(["languages", "seeded"])({ relativePath: "sql-data/init_language.sql" }), + ).toBeCloseTo(1.1) + expect( + fileNameBoostOf(["role", "keys", "reference"])({ + relativePath: "security/RoleProfiles.java", + }), + ).toBeCloseTo(1.1) + expect( + fileNameBoostOf(["account", "principal", "profile"])({ + relativePath: "AccountPrincipalProfile.java", + }), + ).toBeCloseTo(1.2) + expect(fileNameBoostOf(["permissions"])({ relativePath: "docs/overview.md" })).toBe(1) + expect(fileNameBoostOf(["id"])({ relativePath: "id.ts" })).toBe(1) }) }) + +function codeRow(relativePath: string, text: string) { + return { + relativePath, + text, + searchText: chunkLexicalText({ relativePath, text, contextPath: "" }), + } +} diff --git a/packages/ragmir-core/src/source-weights.ts b/packages/ragmir-core/src/source-weights.ts index 3d076a3..bd361a5 100644 --- a/packages/ragmir-core/src/source-weights.ts +++ b/packages/ragmir-core/src/source-weights.ts @@ -1,6 +1,7 @@ import path from "node:path" import picomatch from "picomatch" -import { identifierParts, normalizeForMatch } from "./text.js" +import { identifierParts, normalizeForMatch, shareStem, stemPrefix } from "./text.js" +import { memoizeByText } from "./text-memo.js" import type { Config } from "./types.js" /** @@ -26,7 +27,8 @@ interface CompiledWeight { weight: number } -const compiledWeights = new Map() +/** One weight function per distinct weight table, so its per-path memo outlives each query. */ +const weightFunctions = new Map number>() /** * Ranking multiplier for a source path: the product of every matching path prior and configured @@ -40,27 +42,28 @@ export function sourceWeightOf( ...config.sourceWeights, } const key = JSON.stringify(entries) - let compiled = compiledWeights.get(key) - if (!compiled) { - compiled = Object.entries(entries) + let weightOf = weightFunctions.get(key) + if (!weightOf) { + const weights: CompiledWeight[] = Object.entries(entries) .filter(([, weight]) => weight !== 1) .map(([pattern, weight]) => ({ matches: picomatch(pattern, { dot: true, nocase: true, basename: !pattern.includes("/") }), weight, })) - compiledWeights.set(key, compiled) - } - const weights = compiled - if (weights.length === 0) { - return () => 1 - } - return (relativePath) => { - let weight = 1 - for (const entry of weights) { - if (entry.matches(relativePath)) weight *= entry.weight - } - return weight + // Every chunk of a file, in every query, shares its path: match the patterns once per path. + weightOf = + weights.length === 0 + ? () => 1 + : memoizeByText((relativePath) => { + let weight = 1 + for (const entry of weights) { + if (entry.matches(relativePath)) weight *= entry.weight + } + return weight + }) + weightFunctions.set(key, weightOf) } + return weightOf } const LOCALE_CONTAINERS = new Set([ @@ -211,7 +214,103 @@ const DEFINITION_PATTERNS = [ // tests and notes that mention it, not enough to outrank prose that answers the question better. const DEFINITION_BOOST_PER_PART = 0.07 const MAX_DEFINITION_BOOST = 0.2 -const MIN_STEM_LENGTH = 5 + +// A question that names a file ("how is the OpenAPI document titled") usually wants that file, and +// a heading path names what its section is about: a question word there is stronger evidence than +// the same word in the body, as field weighting does without a second index. On six real indexes, +// with and without an embedding model, 10% per matching word, two at most, raised MRR@10 on most +// of them for file names and on four for headings, without changing a refusal. +const WORD_MATCH_BOOST = 0.1 +const MAX_MATCHED_WORDS = 2 +const MIN_MATCHED_WORD_LENGTH = 4 +const NAME_SEPARATORS = /[^\p{L}\p{N}]+/u + +/** Folded words of a file name: whole, by identifier part and separator, and adjacent parts joined. */ +const fileNameWords = memoizeByText((relativePath: string): ReadonlySet => { + const name = path.posix.basename(relativePath, path.posix.extname(relativePath)) + const folded = normalizeForMatch(name) + const parts = identifierParts(name) + const words = new Set([folded, ...parts, ...folded.split(NAME_SEPARATORS)]) + for (let index = 0; index + 1 < parts.length; index += 1) { + words.add(`${parts[index]}${parts[index + 1]}`) + } + words.delete("") + return words +}) + +/** Folded words of a Markdown heading path or JSONPath. */ +const headingWords = memoizeByText( + (contextPath: string): ReadonlySet => + new Set(normalizeForMatch(contextPath).split(NAME_SEPARATORS).filter(Boolean)), +) + +/** A multiplier of 1 to 1.2: 10% per query word sharing a stem with a word of the row's set. */ +function wordMatchBoostOf( + queryTokens: readonly string[], + wordsOf: (row: Row) => ReadonlySet | null, +): (row: Row) => number { + const tokens = [...new Set(queryTokens)].filter( + (token) => token.length >= MIN_MATCHED_WORD_LENGTH, + ) + if (tokens.length === 0) { + return () => 1 + } + return (row) => { + const words = wordsOf(row) + if (!words) { + return 1 + } + let matched = 0 + for (const token of tokens) { + for (const word of words) { + if (shareStem(token, word)) { + matched += 1 + break + } + } + if (matched === MAX_MATCHED_WORDS) { + break + } + } + return 1 + matched * WORD_MATCH_BOOST + } +} + +/** + * Boost passages from files whose name holds words of the question, such as `OpenApiConfig.java` + * for "how is the OpenAPI document titled". + */ +export function fileNameBoostOf( + queryTokens: readonly string[], +): (row: { relativePath: string }) => number { + return wordMatchBoostOf(queryTokens, (row: { relativePath: string }) => + fileNameWords(row.relativePath), + ) +} + +/** Boost passages whose heading path or JSONPath holds words of the question. */ +export function headingBoostOf( + queryTokens: readonly string[], +): (row: { contextPath?: string | null }) => number { + return wordMatchBoostOf(queryTokens, (row: { contextPath?: string | null }) => + row.contextPath ? headingWords(row.contextPath) : null, + ) +} + +/** Folded names, and their identifier parts, that a code passage declares; one scan per text. */ +const definedNameParts = memoizeByText((text: string): ReadonlySet => { + const parts = new Set() + for (const pattern of DEFINITION_PATTERNS) { + pattern.lastIndex = 0 + for (const match of text.matchAll(pattern)) { + const name = match[1] + if (!name) continue + parts.add(normalizeForMatch(name)) + for (const part of identifierParts(name)) parts.add(part) + } + } + return parts +}) /** * Boost code passages that define what the question names: "how are permissions checked" should @@ -220,32 +319,29 @@ const MIN_STEM_LENGTH = 5 */ export function definitionBoostOf( queryTokens: readonly string[], -): (row: { relativePath: string; text: string }) => number { +): (row: { relativePath: string; text: string; searchText: string }) => number { const tokens = [...new Set(queryTokens)].filter((token) => token.length >= 3) if (tokens.length === 0) { return () => 1 } + const prefixes = tokens.map(stemPrefix) return (row) => { if (!CODE_EXTENSIONS.has(path.posix.extname(row.relativePath).toLowerCase())) { return 1 } - const parts = new Set() - for (const pattern of DEFINITION_PATTERNS) { - pattern.lastIndex = 0 - for (const match of row.text.matchAll(pattern)) { - const name = match[1] - if (!name) continue - parts.add(normalizeForMatch(name)) - for (const part of identifierParts(name)) parts.add(part) - } + // A defined name sharing a stem with a query word puts that word's stem prefix in the folded + // keyword text, so a passage without any of them skips the definition patterns. + if (!prefixes.some((prefix) => row.searchText.includes(prefix))) { + return 1 } + const parts = definedNameParts(row.text) if (parts.size === 0) { return 1 } let matched = 0 for (const token of tokens) { for (const part of parts) { - if (sameStem(token, part)) { + if (shareStem(token, part)) { matched += 1 break } @@ -254,16 +350,3 @@ export function definitionBoostOf( return 1 + Math.min(MAX_DEFINITION_BOOST, matched * DEFINITION_BOOST_PER_PART) } } - -function sameStem(left: string, right: string): boolean { - if (left === right) { - return true - } - const shorter = left.length <= right.length ? left : right - const longer = shorter === left ? right : left - return ( - shorter.length >= MIN_STEM_LENGTH && - longer.startsWith(shorter) && - longer.length - shorter.length <= 4 - ) -} diff --git a/packages/ragmir-core/src/text-memo.test.ts b/packages/ragmir-core/src/text-memo.test.ts new file mode 100644 index 0000000..d647436 --- /dev/null +++ b/packages/ragmir-core/src/text-memo.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it, vi } from "vitest" +import { memoizeByText } from "./text-memo.js" + +describe("memoizeByText", () => { + it("computes once per text and forgets the oldest text beyond its capacity", () => { + const compute = vi.fn((text: string) => text.length) + const memo = memoizeByText(compute, 2) + + expect([memo("alpha"), memo("alpha"), memo("beta"), memo("gamma"), memo("alpha")]).toEqual([ + 5, 5, 4, 5, 5, + ]) + expect(compute.mock.calls.map(([text]) => text)).toEqual(["alpha", "beta", "gamma", "alpha"]) + }) +}) diff --git a/packages/ragmir-core/src/text-memo.ts b/packages/ragmir-core/src/text-memo.ts new file mode 100644 index 0000000..db1e0cd --- /dev/null +++ b/packages/ragmir-core/src/text-memo.ts @@ -0,0 +1,29 @@ +/** Passage texts remembered per memoized function; about 4 MB of text at the default chunk size. */ +const MAX_REMEMBERED_TEXTS = 4096 + +/** + * Remember a query-independent result per passage text for the whole process. A warm process ranks + * the same popular passages for many queries, and recomputing their words, defined names, and + * duplicate keys for each one took a large share of a search. The oldest text leaves first. + */ +export function memoizeByText( + compute: (text: string) => T, + capacity = MAX_REMEMBERED_TEXTS, +): (text: string) => T { + const known = new Map() + return (text) => { + const cached = known.get(text) + if (cached !== undefined) { + return cached + } + const value = compute(text) + if (known.size >= capacity) { + const oldest = known.keys().next().value + if (oldest !== undefined) { + known.delete(oldest) + } + } + known.set(text, value) + return value + } +} diff --git a/packages/ragmir-core/src/text.ts b/packages/ragmir-core/src/text.ts index a300bab..dbaf537 100644 --- a/packages/ragmir-core/src/text.ts +++ b/packages/ragmir-core/src/text.ts @@ -5,6 +5,8 @@ export function normalizeForMatch(text: string): string { .replace(/\p{Diacritic}/gu, "") } +const MIN_STEM_LENGTH = 5 +const MAX_STEM_SUFFIX_LENGTH = 4 const wordSegmenters = new Map() const IDENTIFIER_WORD_PATTERN = /[\p{L}\p{N}]+/gu const IDENTIFIER_PART_BOUNDARY = /\p{Ll}\p{Lu}|\p{Lu}\p{Lu}\p{Ll}|\p{L}\p{N}|\p{N}\p{L}/u @@ -27,6 +29,31 @@ export function tokenize(text: string, locale = "und"): string[] { return tokens } +/** + * Whether two folded words are one stem with a short ending, such as `permission` and + * `permissions`: equal, or the shorter word has at least five letters and starts the longer one, + * which adds at most four. + */ +export function shareStem(left: string, right: string): boolean { + if (left === right) { + return true + } + const shorter = left.length <= right.length ? left : right + const longer = shorter === left ? right : left + return ( + shorter.length >= MIN_STEM_LENGTH && + longer.length - shorter.length <= MAX_STEM_SUFFIX_LENGTH && + longer.startsWith(shorter) + ) +} + +/** The prefix that every word sharing a stem with this word contains. */ +export function stemPrefix(word: string): string { + return word.length < MIN_STEM_LENGTH + ? word + : word.slice(0, Math.max(MIN_STEM_LENGTH, word.length - MAX_STEM_SUFFIX_LENGTH)) +} + function wordSegmenter(locale: string): Intl.Segmenter { const cached = wordSegmenters.get(locale) if (cached) { From d0e91d8902623d3eaea63bd116f9378a37321ec6 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste THERY Date: Sun, 1 Nov 2026 23:25:16 +0700 Subject: [PATCH 2/8] feat(core): embed document batches on the GPU during ingestion - add `ingestEmbeddingDevice` ("cpu" by default, or "webgpu"): Transformers ingestion then embeds document batches through ONNX Runtime's bundled WebGPU provider and falls back to the CPU when the runtime offers no WebGPU - keep query embeddings on the CPU, where short inputs run faster, and key cached pipelines by device so CPU and GPU extractors never mix --- packages/ragmir-core/src/config.test.ts | 13 +++++ packages/ragmir-core/src/config.ts | 5 ++ packages/ragmir-core/src/defaults.ts | 1 + packages/ragmir-core/src/embeddings.test.ts | 36 ++++++++++++ packages/ragmir-core/src/embeddings.ts | 58 +++++++++++++++---- .../ragmir-core/src/test-support/config.ts | 1 + packages/ragmir-core/src/types.ts | 7 +++ 7 files changed, 110 insertions(+), 11 deletions(-) diff --git a/packages/ragmir-core/src/config.test.ts b/packages/ragmir-core/src/config.test.ts index fd6babd..86b3a00 100644 --- a/packages/ragmir-core/src/config.test.ts +++ b/packages/ragmir-core/src/config.test.ts @@ -56,6 +56,19 @@ describe("loadConfig", () => { expect(config.legacyWordCommand).toEqual([]) expect(config.legacyWordTimeoutMs).toBe(120_000) expect(config.sourceFingerprintMode).toBe("fast") + expect(config.ingestEmbeddingDevice).toBe("cpu") + }) + + it("should load the ingestion embedding device and reject unknown devices", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-config-device-")) + tempDirs.push(root) + await mkdir(path.join(root, ".ragmir"), { recursive: true }) + const configPath = path.join(root, ".ragmir/config.json") + await writeFile(configPath, JSON.stringify({ ingestEmbeddingDevice: "webgpu" }), "utf8") + + await expect(loadConfig(root)).resolves.toMatchObject({ ingestEmbeddingDevice: "webgpu" }) + await writeFile(configPath, JSON.stringify({ ingestEmbeddingDevice: "cuda" }), "utf8") + await expect(loadConfig(root)).rejects.toThrow(/ingestEmbeddingDevice/) }) it("should load strict source fingerprint mode", async () => { diff --git a/packages/ragmir-core/src/config.ts b/packages/ragmir-core/src/config.ts index a2eb05a..17b8827 100644 --- a/packages/ragmir-core/src/config.ts +++ b/packages/ragmir-core/src/config.ts @@ -52,6 +52,7 @@ const embeddingModelDigestSchema = z .nullable() const retrievalProfileSchema = z.enum(["fast", "balanced", "quality", "custom"]) const sourceFingerprintModeSchema = z.enum(["fast", "strict"]) +const ingestEmbeddingDeviceSchema = z.enum(["cpu", "webgpu"]) const incrementalFailurePolicySchema = z.enum(["preserve-last-good", "remove-stale"]) const configPathSchema = z.string().min(1).max(MAX_CONFIG_PATH_CHARACTERS) const configTextSchema = z.string().min(1).max(MAX_CONFIG_TEXT_CHARACTERS) @@ -124,6 +125,9 @@ const rawConfigSchema = z maxFileBytes: z.number().int().positive().default(DEFAULT_CONFIG.maxFileBytes), ingestConcurrency: z.number().int().positive().default(DEFAULT_CONFIG.ingestConcurrency), embeddingBatchSize: z.number().int().positive().default(DEFAULT_CONFIG.embeddingBatchSize), + ingestEmbeddingDevice: ingestEmbeddingDeviceSchema.default( + DEFAULT_CONFIG.ingestEmbeddingDevice, + ), sourceFingerprintMode: sourceFingerprintModeSchema.default( DEFAULT_CONFIG.sourceFingerprintMode, ), @@ -354,6 +358,7 @@ export async function loadConfig(start = process.cwd()): Promise { maxFileBytes: effective.maxFileBytes, ingestConcurrency: effective.ingestConcurrency, embeddingBatchSize: effective.embeddingBatchSize, + ingestEmbeddingDevice: effective.ingestEmbeddingDevice, sourceFingerprintMode: effective.sourceFingerprintMode, incrementalFailurePolicy: effective.incrementalFailurePolicy, hybridTextScanLimit: effective.hybridTextScanLimit, diff --git a/packages/ragmir-core/src/defaults.ts b/packages/ragmir-core/src/defaults.ts index e56f200..bd80c13 100644 --- a/packages/ragmir-core/src/defaults.ts +++ b/packages/ragmir-core/src/defaults.ts @@ -83,6 +83,7 @@ export const DEFAULT_CONFIG: Omit = { maxFileBytes: 50_000_000, ingestConcurrency: 4, embeddingBatchSize: 32, + ingestEmbeddingDevice: "cpu", sourceFingerprintMode: "fast", incrementalFailurePolicy: "preserve-last-good", hybridTextScanLimit: 5_000, diff --git a/packages/ragmir-core/src/embeddings.test.ts b/packages/ragmir-core/src/embeddings.test.ts index 4f0e759..f72ee01 100644 --- a/packages/ragmir-core/src/embeddings.test.ts +++ b/packages/ragmir-core/src/embeddings.test.ts @@ -131,6 +131,42 @@ describe("embedding model adapters", () => { expect(extractor).toHaveBeenCalledTimes(3) }) + it("embeds ingestion batches on WebGPU when configured and keeps queries on the CPU", async () => { + const extractor = vi.fn(async (texts: string[]) => ({ + tolist: () => texts.map(() => [1, 0]), + })) + transformersMock.pipeline.mockResolvedValue(extractor) + const devices = () => + transformersMock.pipeline.mock.calls.map( + (call) => (call[2] as { device?: string } | undefined)?.device ?? "default", + ) + + await embedTexts(["chunk one", "chunk two"], transformerConfig()) + expect(devices()).toEqual(["default"]) + + const webgpu: Config = { ...transformerConfig(), ingestEmbeddingDevice: "webgpu" } + await embedTexts(["chunk one", "chunk two"], webgpu, "document") + await embedText("short question", webgpu) + + expect(devices()).toEqual(["default", "webgpu", "default"]) + }) + + it("falls back to the CPU when the runtime offers no WebGPU", async () => { + const extractor = vi.fn(async (texts: string[]) => ({ + tolist: () => texts.map(() => [1, 0]), + })) + transformersMock.pipeline.mockImplementation( + async (_task: string, _model: string, options?: { device?: string }) => { + if (options?.device === "webgpu") throw new Error("Unsupported device: webgpu") + return extractor + }, + ) + const config: Config = { ...transformerConfig(), ingestEmbeddingDevice: "webgpu" } + + await expect(embedTexts(["chunk"], config, "document")).resolves.toEqual([[1, 0]]) + expect(transformersMock.pipeline).toHaveBeenCalledTimes(2) + }) + it("isolates query vectors by project root and exact model identity", async () => { const extractor = vi.fn(async (texts: string[]) => ({ tolist: () => texts.map(() => [1, 0]), diff --git a/packages/ragmir-core/src/embeddings.ts b/packages/ragmir-core/src/embeddings.ts index 112780a..db1bccf 100644 --- a/packages/ragmir-core/src/embeddings.ts +++ b/packages/ragmir-core/src/embeddings.ts @@ -7,7 +7,7 @@ import { removeTree, renameWithRetry } from "./fs-retry.js" import { throwIfAborted } from "./operation.js" import { ensurePrivateDirectory } from "./permissions.js" import { tokenize } from "./text.js" -import type { Config, IngestionEmbeddingModelState } from "./types.js" +import type { Config, IngestEmbeddingDevice, IngestionEmbeddingModelState } from "./types.js" import { runWorkload, type WorkloadAdmission } from "./workload.js" const LOCAL_HASH_DIMENSIONS = 384 @@ -177,7 +177,21 @@ export function embeddingModelState( if (config.embeddingProvider === "local-hash") { return "stateless" } - return transformersPipelines.has(transformersCacheKey(config)) ? "warm" : "cold" + return transformersPipelines.has( + transformersCacheKey(config, embeddingDeviceFor(config, "document")), + ) + ? "warm" + : "cold" +} + +/** + * Document batches run on the configured ingestion device. On an Apple M3 Max, WebGPU embedded + * 300-token passages 2.5 times faster with mxbai-embed-xsmall and 3.7 times faster with + * multilingual-e5-base, with vectors within 4e-7 of the CPU's; short queries were faster on the + * CPU, so search keeps it. + */ +function embeddingDeviceFor(config: Config, inputType: EmbeddingInputType): IngestEmbeddingDevice { + return inputType === "document" ? config.ingestEmbeddingDevice : "cpu" } async function embedWithTransformers( @@ -186,7 +200,7 @@ async function embedWithTransformers( inputType: EmbeddingInputType, signal?: AbortSignal, ): Promise { - const lease = await acquireTransformersPipeline(config) + const lease = await acquireTransformersPipeline(config, embeddingDeviceFor(config, inputType)) try { throwIfAborted(signal) // Cache only single-query vectors. Search still reads and ranks the current index snapshot. @@ -266,8 +280,11 @@ export function prepareEmbeddingText( return text } -async function acquireTransformersPipeline(config: Config): Promise { - const key = transformersCacheKey(config) +async function acquireTransformersPipeline( + config: Config, + device: IngestEmbeddingDevice, +): Promise { + const key = transformersCacheKey(config, device) let entry = transformersPipelines.get(key) if (entry) { transformersPipelines.delete(key) @@ -275,7 +292,7 @@ async function acquireTransformersPipeline(config: Config): Promise { + if (device === "cpu") { + return createTransformersExtractor(config) + } + return createTransformersExtractor(config, undefined, device).catch(() => + createTransformersExtractor(config), + ) +} + async function createTransformersExtractor( config: Config, files: TransformersFileEnvironment = { cacheDir: config.embeddingModelPath, allowLocalModels: true, }, + device: IngestEmbeddingDevice = "cpu", ): Promise { return withTransformersEnvironment(async () => { const transformers = await import("@huggingface/transformers").catch((error: unknown) => { @@ -356,6 +388,7 @@ async function createTransformersExtractor( try { const extractor = (await transformers.pipeline("feature-extraction", config.embeddingModel, { revision: config.embeddingModelRevision, + ...(device === "cpu" ? {} : { device }), })) as TransformersExtractor if ("tokenizer" in extractor && extractor.tokenizer == null) { await extractor.dispose?.() @@ -435,10 +468,13 @@ export async function disposeTransformersCache(): Promise { } export async function disposeTransformersModel(config: Config): Promise { - const entry = transformersPipelines.get(transformersCacheKey(config)) - if (entry) { - await retireTransformersPipeline(entry) - } + const devices: IngestEmbeddingDevice[] = ["cpu", "webgpu"] + await Promise.all( + devices.map((device) => { + const entry = transformersPipelines.get(transformersCacheKey(config, device)) + return entry ? retireTransformersPipeline(entry) : undefined + }), + ) } export function retainEmbeddingModel(config: Config): () => Promise { diff --git a/packages/ragmir-core/src/test-support/config.ts b/packages/ragmir-core/src/test-support/config.ts index b87caf4..b1e938d 100644 --- a/packages/ragmir-core/src/test-support/config.ts +++ b/packages/ragmir-core/src/test-support/config.ts @@ -36,6 +36,7 @@ export function testConfig( maxFileBytes: DEFAULT_CONFIG.maxFileBytes, ingestConcurrency: DEFAULT_CONFIG.ingestConcurrency, embeddingBatchSize: DEFAULT_CONFIG.embeddingBatchSize, + ingestEmbeddingDevice: DEFAULT_CONFIG.ingestEmbeddingDevice, sourceFingerprintMode: DEFAULT_CONFIG.sourceFingerprintMode, incrementalFailurePolicy: DEFAULT_CONFIG.incrementalFailurePolicy, hybridTextScanLimit: DEFAULT_CONFIG.hybridTextScanLimit, diff --git a/packages/ragmir-core/src/types.ts b/packages/ragmir-core/src/types.ts index fc26ee5..6849cb9 100644 --- a/packages/ragmir-core/src/types.ts +++ b/packages/ragmir-core/src/types.ts @@ -56,6 +56,11 @@ export interface Config { maxFileBytes: number ingestConcurrency: number embeddingBatchSize: number + /** + * Device for document embeddings during ingestion. `webgpu` falls back to the CPU when the + * runtime offers no WebGPU; search embeds short queries on the CPU, where they are faster. + */ + ingestEmbeddingDevice: IngestEmbeddingDevice sourceFingerprintMode: SourceFingerprintMode incrementalFailurePolicy: IncrementalFailurePolicy hybridTextScanLimit: number @@ -100,6 +105,8 @@ export interface SearchScope { export type RetrievalProfile = "fast" | "balanced" | "quality" | "custom" export type IncrementalFailurePolicy = "preserve-last-good" | "remove-stale" export type SourceFingerprintMode = "fast" | "strict" +/** Where Transformers models embed document batches during ingestion; queries stay on the CPU. */ +export type IngestEmbeddingDevice = "cpu" | "webgpu" export type WorkloadKind = "search" | "embedding" | "ingestion" export interface WorkloadLimit { From 089a92d58cb7e081e9eb2577eae643c64057b17a Mon Sep 17 00:00:00 2001 From: Jean-Baptiste THERY Date: Sun, 1 Nov 2026 23:25:16 +0700 Subject: [PATCH 3/8] fix(core): name an explicit ingest when external extractors block repairs - have `rgr doctor` and `rgr upgrade` recommend `rgr ingest` or `rgr ingest --rebuild` when configured OCR or legacy Word commands keep setup, repair, and upgrade from ingesting on their own, instead of a repair step that cannot ingest --- packages/ragmir-core/src/doctor.test.ts | 20 +++++++++++++ packages/ragmir-core/src/doctor.ts | 8 ++++- packages/ragmir-core/src/setup.ts | 5 ++++ packages/ragmir-core/src/upgrade.ts | 40 +++++++++++++++++-------- 4 files changed, 59 insertions(+), 14 deletions(-) diff --git a/packages/ragmir-core/src/doctor.test.ts b/packages/ragmir-core/src/doctor.test.ts index e76ba5a..b8fa535 100644 --- a/packages/ragmir-core/src/doctor.test.ts +++ b/packages/ragmir-core/src/doctor.test.ts @@ -74,6 +74,26 @@ describe("doctor", { timeout: integrationTimeout(30_000) }, () => { ) }) + it("recommends an explicit ingest when external extractors keep repair from ingesting", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-doctor-extractors-")) + tempDirs.push(root) + await initProject(root) + await writeFile(path.join(root, ".ragmir", "raw", "evidence.md"), "Local evidence.\n", "utf8") + await writeFile( + path.join(root, ".ragmir", "config.json"), + JSON.stringify({ imageOcrCommand: ["./ocr.sh", "{input}"] }), + "utf8", + ) + + const report = await doctor(root, { deep: true }) + + expect(report.securityWarnings.length).toBeGreaterThan(0) + expect(report.nextSteps[0]).toMatch( + /^Run `pnpm exec rgr ingest` to index stale or missing files\./, + ) + expect(report.nextSteps.join("\n")).not.toContain("doctor --fix` to rebuild") + }) + it("reports files added or edited after ingestion instead of trusting the ingest snapshot", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-doctor-freshness-")) tempDirs.push(root) diff --git a/packages/ragmir-core/src/doctor.ts b/packages/ragmir-core/src/doctor.ts index 6a1b67d..212a8cc 100644 --- a/packages/ragmir-core/src/doctor.ts +++ b/packages/ragmir-core/src/doctor.ts @@ -289,7 +289,13 @@ function nextActions(input: NextActionInput): string[] { } if (input.chunksIndexed === 0 || input.missingFromIndex > 0 || input.staleInIndex > 0) { - steps.push(`Run \`${input.run(["doctor", "--fix"])}\` to rebuild stale or missing index data.`) + // Setup, repair, and upgrade never ingest on their own while a security warning stands, such as + // configured external extractors: only an explicit ingest runs those commands. + steps.push( + input.warnings > 0 + ? `Run \`${input.run(["ingest"])}\` to index stale or missing files. \`doctor --fix\` and \`upgrade\` do not ingest while a security warning stands, such as configured external extractors.` + : `Run \`${input.run(["doctor", "--fix"])}\` to rebuild stale or missing index data.`, + ) steps.push(`Run \`${input.run(["audit"])}\` to verify missingFromIndex=0 and staleInIndex=0.`) if (input.chunksIndexed > 0) { steps.push( diff --git a/packages/ragmir-core/src/setup.ts b/packages/ragmir-core/src/setup.ts index 57bc622..dffecce 100644 --- a/packages/ragmir-core/src/setup.ts +++ b/packages/ragmir-core/src/setup.ts @@ -149,6 +149,11 @@ async function setupSemanticEmbeddings(cwd: string): Promise 0 && diff --git a/packages/ragmir-core/src/upgrade.ts b/packages/ragmir-core/src/upgrade.ts index f6fed2b..dd8a76d 100644 --- a/packages/ragmir-core/src/upgrade.ts +++ b/packages/ragmir-core/src/upgrade.ts @@ -62,13 +62,21 @@ export async function inspectUpgrade(cwd = process.cwd()): Promise Date: Sun, 1 Nov 2026 23:25:16 +0700 Subject: [PATCH 4/8] fix(evaluate): time latency on sequential warm searches - keep scoring cases concurrently, but take p50 and p95 latency from an untimed warm-up and up to ten sequential searches; concurrent cases waited behind each other and read several times higher than real searches --- packages/ragmir-core/src/evaluate.ts | 39 +++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/packages/ragmir-core/src/evaluate.ts b/packages/ragmir-core/src/evaluate.ts index 8bfa58a..d4d6af9 100644 --- a/packages/ragmir-core/src/evaluate.ts +++ b/packages/ragmir-core/src/evaluate.ts @@ -246,6 +246,32 @@ export async function evaluateGoldenQueries(options: EvaluationOptions): Promise return evaluateGoldenQueriesWithConfig(options, config) } +/** Sequential searches timed for the latency percentiles, after one untimed warm-up. */ +const LATENCY_SAMPLE_SIZE = 10 + +/** + * Cases run concurrently for throughput, so a case's own timing includes the time it waited behind + * the others. Report latency the way an agent searches instead: an untimed search loads the model + * and opens the index, then a few searches run one at a time. + */ +async function sequentialSearchLatencies( + queries: readonly Query[], + search: (query: Query) => Promise, +): Promise { + const [warmUp] = queries + if (warmUp === undefined) { + return [] + } + await search(warmUp) + const latencies: number[] = [] + for (const query of queries.slice(0, LATENCY_SAMPLE_SIZE)) { + const startedAt = performance.now() + await search(query) + latencies.push(performance.now() - startedAt) + } + return latencies +} + async function evaluateGoldenQueriesWithConfig( options: EvaluationOptions, config: Config, @@ -280,8 +306,17 @@ async function evaluateGoldenQueriesWithConfig( ? fingerprintIndexManifest(initialManifest) : "missing-index" + let latencySample: number[] = [] try { throwIfAborted(signal) + latencySample = await sequentialSearchLatencies(goldenFile.queries, (goldenQuery) => + evaluationClient.search(goldenQuery.query, { + cwd, + topK: boundedTopK(goldenQuery.topK ?? defaultTopK, options.maxTopK), + contextRadius: 0, + ...(signal === undefined ? {} : { signal }), + }), + ) for (let offset = 0; offset < goldenFile.queries.length; offset += evaluationConcurrency) { throwIfAborted(signal) const batch = goldenFile.queries.slice(offset, offset + evaluationConcurrency) @@ -366,7 +401,9 @@ async function evaluateGoldenQueriesWithConfig( : null const hits = cases.filter((result) => result.hit).length - const latencies = cases.map((result) => result.latencyMs).sort((a, b) => a - b) + const latencies = ( + latencySample.length > 0 ? latencySample : cases.map((result) => result.latencyMs) + ).sort((a, b) => a - b) const recall = mean(answerableCases.map((result) => result.recall)) const precision = mean(answerableCases.map((result) => result.precision)) const meanReciprocalRank = mean(answerableCases.map((result) => result.reciprocalRank)) From d6418b99be03eff6c26eaafde5438037ddd6797d Mon Sep 17 00:00:00 2001 From: Jean-Baptiste THERY Date: Sun, 1 Nov 2026 23:25:16 +0700 Subject: [PATCH 5/8] test(bench): refresh the repository fixture and the benchmark gates - move the questions about the retired Chat and TTS packages from the answerable list to the near-domain list of the repository fixture - check observability phases by name, so adding or removing one fails visibly - derive the runtime-reuse topK change from the balanced default, which setup leaves unset --- .../fixtures/repository-questions.json | 108 +++--------------- .../ragmir-core/benchmarks/observability.mjs | 16 ++- .../ragmir-core/benchmarks/runtime-reuse.mjs | 4 +- 3 files changed, 32 insertions(+), 96 deletions(-) diff --git a/packages/ragmir-core/benchmarks/fixtures/repository-questions.json b/packages/ragmir-core/benchmarks/fixtures/repository-questions.json index bc743ad..b2bda29 100644 --- a/packages/ragmir-core/benchmarks/fixtures/repository-questions.json +++ b/packages/ragmir-core/benchmarks/fixtures/repository-questions.json @@ -509,30 +509,6 @@ "llms.txt" ] }, - { - "id": "chat-2", - "query": "What are the download sizes for lite fast and quality Chat profiles?", - "expectedPaths": [ - "README.md", - "llms.txt" - ] - }, - { - "id": "chat-3", - "query": "How does Ragmir Chat validate visible citation markers in a local answer?", - "expectedPaths": [ - "README.md", - "llms.txt" - ] - }, - { - "id": "chat-4", - "query": "Which command verifies a Chat model profile is ready for offline generation?", - "expectedPaths": [ - "README.md", - "llms.txt" - ] - }, { "id": "chat-5", "query": "Does installing Ragmir Core require Qwen Gemma or the Chat package?", @@ -541,46 +517,6 @@ "llms.txt" ] }, - { - "id": "tts-1", - "query": "Which languages have an offline Ragmir TTS model?", - "expectedPaths": [ - "README.md", - "llms.txt" - ] - }, - { - "id": "tts-2", - "query": "When may narration text be sent to the Edge speech service?", - "expectedPaths": [ - "README.md", - "llms.txt" - ] - }, - { - "id": "tts-3", - "query": "How should a TTS model be preloaded with non-sensitive text?", - "expectedPaths": [ - "README.md", - "llms.txt" - ] - }, - { - "id": "tts-4", - "query": "Which output format is produced by offline Transformers speech?", - "expectedPaths": [ - "README.md", - "llms.txt" - ] - }, - { - "id": "tts-5", - "query": "Does Ragmir TTS retrieve evidence or only render caller-provided text?", - "expectedPaths": [ - "README.md", - "llms.txt" - ] - }, { "id": "troubleshooting-1", "query": "What should a user inspect when Ragmir search results are weak?", @@ -611,16 +547,6 @@ "docs/cli-reference.md" ] }, - { - "id": "troubleshooting-4", - "query": "How should a missing optional Chat or TTS peer package be diagnosed?", - "expectedPaths": [ - "docs/troubleshooting.md", - "README.md", - "packages/ragmir-core/README.md", - "docs/cli-reference.md" - ] - }, { "id": "troubleshooting-5", "query": "Which checks distinguish privacy warnings from operational readiness?", @@ -703,18 +629,6 @@ "scripts/semantic-release-smoke.mjs" ] }, - { - "id": "release-2", - "query": "How are Core Chat and TTS package compatibility checked before publishing?", - "expectedPaths": [ - "RELEASING.md", - "README.md", - "package.json", - "AGENTS.md", - "CLAUDE.md", - "scripts/semantic-release-smoke.mjs" - ] - }, { "id": "release-3", "query": "Which release notes sections describe highlights details and verification?", @@ -816,15 +730,11 @@ }, { "id": "scripts-2", - "query": "Which package scripts build Core Chat TTS and the landing?", + "query": "Which package scripts build Core and the landing?", "expectedPaths": [ "package.json", - "README.md", - "AGENTS.md", - "CLAUDE.md", - "RELEASING.md", - "scripts/public-surface-smoke.mjs", - "scripts/semantic-release-smoke.mjs" + "packages/ragmir-core/package.json", + "packages/ragmir-landing/README.md" ] }, { @@ -1167,6 +1077,16 @@ "How do I configure the Ragmir Chrome extension?", "How does Ragmir OCR Arabic right-to-left scans?", "How does Ragmir handle Excel macros in .xlsm files?", - "How does Ragmir federated search query several remote servers?" + "How does Ragmir federated search query several remote servers?", + "What are the download sizes for lite fast and quality Chat profiles?", + "How does Ragmir Chat validate visible citation markers in a local answer?", + "Which command verifies a Chat model profile is ready for offline generation?", + "Which languages have an offline Ragmir TTS model?", + "When may narration text be sent to the Edge speech service?", + "How should a TTS model be preloaded with non-sensitive text?", + "Which output format is produced by offline Transformers speech?", + "Does Ragmir TTS retrieve evidence or only render caller-provided text?", + "How should a missing optional Chat or TTS peer package be diagnosed?", + "How are Core Chat and TTS package compatibility checked before publishing?" ] } diff --git a/packages/ragmir-core/benchmarks/observability.mjs b/packages/ragmir-core/benchmarks/observability.mjs index 9c85d0a..0412ed5 100644 --- a/packages/ragmir-core/benchmarks/observability.mjs +++ b/packages/ragmir-core/benchmarks/observability.mjs @@ -42,12 +42,26 @@ try { ((median(disabledProbeMs) - median(baselineProbeMs)) * 1_000_000) / PROBE_ITERATIONS, ) const metrics = ingestion.metrics + // Named phases, so removing or adding one fails this gate visibly instead of shifting a count. + const phaseNames = [ + "queueMs", + "writeLockWaitMs", + "discoveryMs", + "hashingMs", + "parsingMs", + "chunkingMs", + "embeddingMs", + "storageWriteMs", + "maintenanceMs", + "totalMs", + ] const phaseValues = metrics ? Object.values(metrics.phaseDurations) : [] const serializedDiagnostics = JSON.stringify(events) const gates = { metricsReturned: metrics !== undefined, phaseAttribution: - phaseValues.length === 11 && + phaseValues.length === phaseNames.length && + phaseNames.every((name) => Object.hasOwn(metrics?.phaseDurations ?? {}, name)) && phaseValues.every((value) => Number.isFinite(value) && value >= 0) && (metrics?.phaseDurations.hashingMs ?? 0) > 0 && (metrics?.phaseDurations.parsingMs ?? 0) > 0 && diff --git a/packages/ragmir-core/benchmarks/runtime-reuse.mjs b/packages/ragmir-core/benchmarks/runtime-reuse.mjs index 8d938c8..70e60ac 100644 --- a/packages/ragmir-core/benchmarks/runtime-reuse.mjs +++ b/packages/ragmir-core/benchmarks/runtime-reuse.mjs @@ -247,7 +247,9 @@ async function measureConfigInvalidation(projectRoot) { await first.search("signed comet evidence") const configPath = path.join(projectRoot, ".ragmir", "config.json") const config = JSON.parse(await readFile(configPath, "utf8")) - await writeFile(configPath, `${JSON.stringify({ ...config, topK: config.topK + 1 }, null, 2)}\n`) + // Setup leaves topK to the retrieval profile, so derive a changed value from the balanced default. + const topK = (config.topK ?? 8) + 1 + await writeFile(configPath, `${JSON.stringify({ ...config, topK }, null, 2)}\n`) const events = [] const onEvent = (event) => events.push(event) subscribe("ragmir:index-read", onEvent) From a4294f69f4d47e20fc37ca6591a6d4f6ed9f485f Mon Sep 17 00:00:00 2001 From: Jean-Baptiste THERY Date: Sun, 1 Nov 2026 23:25:16 +0700 Subject: [PATCH 6/8] docs: publish the 6.5 measurements, absent-topic refusals, and GPU ingestion - update both READMEs, configuration, migration, CLI, troubleshooting, llms.txt, and the benchmark guide with the measured 6.4 to 6.5 results on six indexes and the stack comparison - document the absent-topic rule, the distinctive-word boosts, and `ingestEmbeddingDevice` --- README.md | 124 ++++++++++++++++------ docs/cli-reference.md | 8 +- docs/configuration.md | 38 +++++-- docs/migration.md | 34 ++++++ docs/troubleshooting.md | 7 +- llms.txt | 5 +- packages/ragmir-core/README.md | 17 +-- packages/ragmir-core/benchmarks/README.md | 6 +- 8 files changed, 184 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index 65fb0b7..b292c03 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@

- 93% of real developer questions find the right file in the top 10 · 39 ms median search on a - 29,778-chunk monorepo · benchmarks + 94% of real developer questions find the right file in the top 10 · 36 ms median search on a + 30,171-chunk monorepo · benchmarks

@@ -56,18 +56,19 @@ npx rgr search "how are permissions checked?" --compact - **Code and documents in one index.** Hybrid BM25 and vector search understands paths and camelCase identifiers and boosts the code that defines what you ask about, so "how are permissions checked" reaches `PermissionService.java`, not only its tests and notes. -- **Fast on real repositories.** Median search takes 39 ms on a 29,778-chunk Java and TypeScript - monorepo with Office files, and the right file reaches the top 10 for 93% of 101 real developer - questions. -- **Ahead of the usual RAG stacks.** On this repository, the right file reaches the top 3 for 76% - of developer questions, against 56% for a LangChain.js pipeline and 70% for MiniSearch BM25 with - the same embedding model, and Ragmir refuses 27 of 30 off-topic questions where they refuse at - most 3. +- **Fast on real repositories.** Median search takes 36 ms on a 30,171-chunk Java and TypeScript + monorepo with Office files, and the right file reaches the top 10 for 94% of 101 real developer + questions. GPU ingestion rebuilds this repository's index twice as fast on Apple silicon. +- **Ahead of the usual RAG stacks.** On this repository, the right file reaches the top 3 for 83% + of developer questions, against 63% for a LangChain.js pipeline and 79% for MiniSearch BM25 with + the same embedding model, and Ragmir refuses 24 of 30 off-topic questions, 28 without a model, + where they refuse at most 2. - **Evidence you can verify.** Citations point to source lines, PDF pages, slides, sheet cells, or EPUB sections. Results say when their file changed after indexing (`sourceStatus`) and when their support is thin (`weakEvidence`). Questions your sources cannot answer return nothing - instead of noise: 58 of 90 off-topic and 24 of 90 near-domain questions on real bases, up from - 41 and 1. + instead of noise: 68 of 90 off-topic and 26 of 110 near-domain questions on three real bases, up + from 54 and 23 in 6.4, and 65 and 27 without any model, up from 12 and 1, with no correct answer + refused. - **Always current.** `rgr doctor` compares the files on disk with the index on every run, and `rgr hooks install` refreshes it in the background after commits, merges, and checkouts. - **Private by default.** No account, telemetry, or hosted storage. The default `local-hash` @@ -188,46 +189,65 @@ use the same retrieval API without one. ### Real projects -Ragmir 6.4 against 6.3.1 on the same machine, configuration, and indexes. Each answerable question +Ragmir 6.5 against 6.4 on the same machine, configuration, and indexes. Each answerable question lists the files that answer it; "top 3" and "top 10" count the questions with one of them there. Off-topic questions (sports, cooking, weather, nonsense) and near-domain questions about features the project does not have should return nothing. Latency is the median of warm sequential searches in one process. -| Corpus | Embeddings | Answer in top 3 | Answer in top 10 | Off-topic refused | Near-domain refused | Correct answers flagged weak | Median search | -| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| Java/TypeScript monorepo with Office files and scans: 2,297 files, 29,778 chunks, 101 developer questions | multilingual-e5-base | 74% → **74%** | 93% → **93%** | 0 → **16** of 30 | 1 → **11** of 30 | 11 → **8** of 94 | 39 → 39 ms | -| French and English product documentation: 299 files, 21,011 chunks, 60 questions | mxbai-embed-xsmall | 100% → **100%** | 100% → **100%** | 16 → **17** of 30 | 0 → **6** of 30 | 0 → **0** of 60 | 28 → 28 ms | -| This repository: 229 files, 2,490 chunks, 99 developer questions | mxbai-embed-xsmall | 75% → **75%** | 97% → **97%** | 25 → **25** of 30 | 0 → **7** of 30 | 32 → **6** of 96 | 22 → 22 ms | +| Corpus | Embeddings | Answer first | Answer in top 3 | Answer in top 10 | Off-topic refused | Near-domain refused | Correct answers flagged weak | Median search | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Java/TypeScript monorepo with Office files and scans: 2,325 files, 30,171 chunks, 101 developer questions | multilingual-e5-base | 44% → **54%** | 73% → **77%** | 93% → **94%** | 16 → **22** of 30 | 11 → **11** of 30 | 8 of 94 → **10** of 95 | 39 → **36 ms** | +| French and English product documentation: 299 files, 21,011 chunks, 60 questions | mxbai-embed-xsmall | 97% → **100%** | 100% → **100%** | 100% → **100%** | 17 → **23** of 30 | 6 → **9** of 30 | 0 → **0** of 60 | 29 → **26 ms** | +| This repository: 237 files, 2,541 chunks, 89 developer questions | mxbai-embed-xsmall | 51% → **54%** | 81% → **83%** | 100% → **100%** | 21 → **23** of 30 | 6 → **6** of 40 | 6 → **4** of 89 | 23 → **17 ms** | +| The same three corpora with `local-hash`, the monorepo without OCR text | none | 28 to 83% → **35 to 88%** | 46 to 90% → **63 to 90%** | 79 to 99% → **84 to 98%** | 12 → **65** of 90 | 1 → **27** of 110 | 15 of 222 → **9** of 226 | 20 to 31 → **16 to 30 ms** | -Measured on November 1, 2026 on an Apple M3 Max with Node.js 26 and LanceDB 0.30, with the +Measured on November 1, 2026 on an Apple M3 Max with Node.js 24 and LanceDB 0.30, with the `quality` profile these projects use. Near-domain questions pair the project's own vocabulary with an integration, service, or format it does not have, and each was checked absent from every -indexed file. 6.4 kept every answer in place and caught them by measuring each index's noise -floor: a passage only counts as a strong vector match when it is nearer than 24 off-topic probe -questions, and a search whose first five results are all weak answers nothing. The two private -corpora are described rather than published. Run the [benchmarks](./packages/ragmir-core/benchmarks/README.md) -and `rgr evaluate` on your own corpus before relying on any number. +indexed file. 6.5 refuses no correct answer and ranks the right file first more often on all six +indexes: the keyword query weights function words such as "de" or "what" at 0.3, and passages +whose file name or heading path holds words of the question gain up to 20%. The one loss is +`local-hash` on this repository: three fewer questions find their file in the top 5, and one fewer +in the top 10. It refuses more because a passage that only the keyword search returned now needs +half of the question's weighted words: one shared everyday word ("height", "capital") no longer +makes an off-topic question look answered. Without a model, `local-hash` now also answers nothing +when none of its first five passages holds 30% of the question's weighted words, or when words +found nowhere in the index carry half of the question's weight, as in a question about an +integration the project lacks. A long-running process remembers the query-independent work done +per passage, so warm searches are faster. This repository's question set moved ten questions about +the retired Chat and TTS packages to the near-domain list. The two private corpora are described +rather than published. Run the [benchmarks](./packages/ragmir-core/benchmarks/README.md) and +`rgr evaluate` on your own corpus before relying on any number. + +### Ingestion on the GPU + +`ingestEmbeddingDevice: "webgpu"` embeds document batches through the WebGPU runtime that ONNX +Runtime bundles; search keeps embedding short queries on the CPU, where they are faster. On the same +Apple M3 Max, a full rebuild of this repository (2,519 chunks, `mxbai-embed-xsmall-v1`) took 28 s +instead of 59 s, and `multilingual-e5-base` embedded 300-token passages 3.7 times faster. Both +indexes returned identical results for all 159 benchmark questions, and GPU vectors stayed within +4e-7 of the CPU's. ### Against other retrieval stacks -`pnpm bench:competitors` indexes the 210 tracked documentation and source files of this -repository and asks its 99 developer questions plus 30 off-topic and 30 near-domain questions. Every +`pnpm bench:competitors` indexes the 218 tracked documentation and source files of this +repository and asks its 89 developer questions plus 30 off-topic and 40 near-domain questions. Every vector side uses the same model and vectors (`mxbai-embed-xsmall-v1`); the other stacks read the same files split by LangChain's `RecursiveCharacterTextSplitter` (1,000 characters, 200 overlap). | System | Answer in top 3 | Answer in top 10 | MRR@10 | Off-topic refused | Near-domain refused | Median search | | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| **Ragmir**, hybrid with `mxbai-embed-xsmall-v1` | **76%** | **97%** | **0.647** | **27** of 30 | **5** of 30 | 22 ms | -| **Ragmir**, `local-hash` without a model | **79%** | 96% | 0.611 | 9 of 30 | 0 of 30 | 20 ms | -| LangChain.js splitter with exact cosine search, as `MemoryVectorStore` | 56% | 89% | 0.509 | 0 of 30 | 0 of 30 | 5 ms | -| Orama 3 hybrid, default similarity 0.8 | 18% | 35% | 0.144 | 2 of 30 | 0 of 30 | 5 ms | -| Orama 3 hybrid, similarity 0.1 | 38% | 64% | 0.317 | 0 of 30 | 0 of 30 | 6 ms | -| MiniSearch 7 BM25 | 70% | 92% | 0.566 | 3 of 30 | 0 of 30 | 1 ms | +| **Ragmir**, hybrid with `mxbai-embed-xsmall-v1` | **83%** | **100%** | **0.705** | **24** of 30 | **4** of 40 | 18 ms | +| **Ragmir**, `local-hash` without a model | **88%** | **99%** | **0.701** | **28** of 30 | **13** of 40 | 17 ms | +| LangChain.js splitter with exact cosine search, as `MemoryVectorStore` | 63% | 87% | 0.530 | 0 of 30 | 0 of 40 | 5 ms | +| Orama 3 hybrid, default similarity 0.8 | 19% | 38% | 0.154 | 2 of 30 | 0 of 40 | 6 ms | +| Orama 3 hybrid, similarity 0.1 | 42% | 71% | 0.347 | 0 of 30 | 0 of 40 | 7 ms | +| MiniSearch 7 BM25 | 79% | 96% | 0.617 | 2 of 30 | 0 of 40 | 1 ms | In-memory libraries answer faster, but they return results for almost every off-topic question, -and none ranks the answering file as high: it reaches the top 3 for 76 to 79% of questions with Ragmir, -against 56% with the LangChain.js pipeline and 70% with MiniSearch. Measured on the same machine on +and none ranks the answering file as high: it reaches the top 3 for 83 to 88% of questions with Ragmir, +against 63% with the LangChain.js pipeline and 79% with MiniSearch. Measured on the same machine on November 1, 2026; [the benchmark guide](./packages/ragmir-core/benchmarks/README.md) explains every setting. @@ -292,6 +312,44 @@ pnpm exec rgr ingest Embedded PDF text is extracted first; OCR runs only on blank pages, in bounded batches with a local cache. Normal retrieval keeps remote model loading disabled. +### Choosing an embedding model + +Start without one: `local-hash` needs no download and put the answering file in the top 3 for 90% +of this repository's developer questions. Add a model when questions use words your sources do not +(synonyms, paraphrases, another language) or when refusals should follow meaning, not keywords. + +| Your sources | Model | Why | +| --- | --- | --- | +| English code and documentation | `mixedbread-ai/mxbai-embed-xsmall-v1` | Smallest and fastest; pinned revision and calibrated abstention bundled. | +| French, English, or mixed | `intfloat/multilingual-e5-small`, the `rgr setup --semantic` default | Multilingual; pinned revision and calibrated abstention bundled. | +| Large mixed-language monorepo | `intfloat/multilingual-e5-base` | On a 30,171-chunk French and Java monorepo it ranked answers above e5-small, while e5-large scored lower. Pin its revision and digest yourself. | + +Newer open models score higher on public benchmarks, and Ragmir is evaluating them before it +supports them: IBM's Granite Embedding Multilingual R2 (97M and 311M, Apache-2.0, ONNX signed by +IBM), Google's EmbeddingGemma 2 (Apache-2.0), and Qwen3-Embedding-0.6B (Apache-2.0). Do not +configure them yet: they need a different pooling or task prompts than the mean pooling Ragmir +applies today, so their vectors would be degraded. Jina embeddings use a non-commercial license. + +A model is code you run on your sources, so treat it as a dependency. Ragmir loads only ONNX +weights through Transformers.js (no pickled objects, no remote code), pins a 40-character revision, +records the artifact digest, and keeps downloads disabled after setup. Before switching, check +that the ONNX files come from the original lab or the Transformers.js maintainers, that the license +allows your use, and that `rgr evaluate` on your own questions beats your current model. A new +model needs `rgr ingest --rebuild`. + +### Claude Code and Codex as the model + +Your agent's model already reads every cited passage and decides what it means, so Ragmir leaves +reasoning to it and returns evidence, `weakEvidence` flags, and refusals. It cannot borrow that +model for its own retrieval: + +- **Embeddings.** Claude and Codex models do not produce embeddings. Anthropic points to Voyage AI, + a cloud API, which would send your whole corpus off the machine at ingest. +- **Reranking or judging through the agent.** MCP sampling, which lets a server ask the client's + model, is deprecated since the 2026-07-28 MCP specification, and neither Claude Code nor Codex + supports it. A judge that reads untrusted documents can also be steered by instructions planted + in them, and small local judges did not improve Ragmir's refusals on real corpora. + ## Supported documents Text and code, Markdown, JSON/JSONL, YAML, CSV, HTML, PDF, DOCX, XLSX, PPTX, OpenDocument, EPUB, diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 485a491..f7dfd2e 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -119,7 +119,9 @@ instruction instead of reading an untrusted layout. `ready` describes upgrade an continuity. Repeated `advisory` lines report separate security follow-ups; they do not turn a compatible operational index into `repair-required`. -`upgrade` refreshes managed agent helpers and performs any required ingest or rebuild. Schema, +`upgrade` refreshes managed agent helpers and performs any required ingest or rebuild. While a +security advisory stands, such as configured external extractors, it does not ingest on its own and +its reason names the explicit `rgr ingest` or `rgr ingest --rebuild` to run. Schema, embedding, chunking, extraction, and index-policy changes use the staged-generation flow: Ragmir never deletes the active index first, and only a replacement that passes row-count, checksum, and duplicate-ID validation activates. Failed or interrupted rebuilds never activate a partial table @@ -295,7 +297,9 @@ rgr destroy-index --yes cases. Wrapped files can declare graded `relevanceJudgments`, `answerable: false` hard negatives, categories, locales, exact citations, and independent thresholds for Recall@1/3/5/10, Precision@5, MRR@10, nDCG@10, citation accuracy, and false-positive rate. One run pins a single - index generation and evaluates cases with bounded concurrency while preserving report order. + index generation and evaluates cases with bounded concurrency while preserving report order. Its + latency percentiles come from an untimed warm-up and up to ten sequential searches, as an agent + runs them. - A passing suite with at least 100 cases, graded relevance, exact citations, hard negatives, and every threshold stores a fingerprint in the active manifest. `rgr doctor --deep` reports retrieval quality as verified only while that report still matches the golden file, corpus, model revision, diff --git a/docs/configuration.md b/docs/configuration.md index 64f6692..4a64ee5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -42,6 +42,7 @@ does not create a local copy. With both files, CLI changes go to `.ragmir/config | `maxFileBytes` | `50000000` | Lower the per-file parser budget; 50 MB is the hard ceiling. | | `ingestConcurrency` | `4` | Bound concurrent parsers; values above `8` are rejected. | | `embeddingBatchSize` | `32` | Bound one model call; values above `128` are rejected. | +| `ingestEmbeddingDevice` | `cpu` | Set `webgpu` to embed document batches on the GPU during ingestion. Search keeps embedding short queries on the CPU, where they are faster, and a runtime without WebGPU falls back to the CPU. On an Apple M3 Max, it rebuilt this repository's index in 28 s instead of 59 s with `mxbai-embed-xsmall-v1` and embedded passages 3.7 times faster with `multilingual-e5-base`, with vectors within 4e-7 of the CPU's. | | `sourceFingerprintMode` | `fast` | Use `strict` to hash every source on every inventory instead of reusing unchanged private fingerprints. | | `incrementalFailurePolicy` | `preserve-last-good` | Use `remove-stale` only when failed changed files must disappear immediately. | | `hybridTextScanLimit` | `5000` | Set the batch size for the complete lexical scan used when FTS is unavailable; values above 10,000 are rejected. The fallback scans every matching chunk across as many batches as the corpus requires. | @@ -124,6 +125,18 @@ adds more work and corpus-dependent ordering, and the current golden benchmark d advantage over the simple cap default. It can be evaluated later as an explicit opt-in strategy without changing this predictable default. +The keyword query weights function words such as "what", "which", "de", or "les" at 0.3: the +analyzer keeps question words and French function words, and a rare one would otherwise rank +passages on its own. Passages from a file whose name holds words of the question gain 10% per word, +up to two, so "how is the OpenAPI document titled" lifts `OpenApiConfig.java`, and passages whose +Markdown heading path or JSONPath holds them gain the same, as a title field would; a code passage +that defines what the question names gains up to 20% more. These boosts ignore words that more than +half of the passages hold: on a small corpus whose headings all start with the same word, they would +otherwise lift every passage above the one both channels ranked first. On three real corpora with +and without an embedding model, these rules put the right file first more often on all six indexes, +raised top-3 hits on three and kept the three others, from 46% to 63% on a French and Java monorepo +without a model, and refused no correct answer. + Hybrid ranking uses deterministic reciprocal-rank fusion with `k = 60`. Transformers vectors and full-text results have equal weights. The `local-hash` vector is a hashed lexical signal without inverse document frequency, so it receives a weight of 0.01: BM25 evidence leads, and hashed @@ -142,7 +155,12 @@ distinctive word counts more than a common one: questions then return nothing instead of the nearest noise. Words in scripts written without spaces (Thai, Chinese, Japanese, Korean) are not measured this way. - `local-hash` otherwise requires lexical evidence and gives query identifiers precedence over - coincidental section numbers. + coincidental section numbers. It also answers nothing when the words found nowhere in the index + carry half of the question's IDF weight, as in "How does it sync with Pinecone?" on a corpus + that never mentions Pinecone. A missing word weighs as if one passage held it, so on a small + corpus one unusual verb does not pass for the question's focus. Semantic models keep their vector + evidence until every word is absent: the same rule at 40% refused 17 more answerable questions + when each held one unknown word. - Calibrated Transformers models need a normalized L2 distance within the model's limit, or passages holding at least 30% of the question's IDF-weighted words. `multilingual-e5-small` packs every text into a narrow distance band and uses a limit of 0.4; @@ -157,12 +175,18 @@ match, and a question with no known word needs one. An index built before this r the floor once per process on its first search; the next `rgr ingest` stores it. Each result whose passage holds few of the question's distinctive words and has no strong vector -match carries `weakEvidence: true`. When the first five results of a Transformers search are all -weak, the answer holds no real evidence and search returns nothing. `local-hash` search keeps such -results. On a Java and TypeScript monorepo, a product documentation base, and this repository -(260 answerable questions), these rules kept every answer in place while refusing 58 of 90 -off-topic and 24 of 90 near-domain unanswerable questions, against 41 and 1 before, and flagged -14 correct answers as weak instead of 43. Evaluate your own corpus with `rgr evaluate`. +match carries `weakEvidence: true`. With a Transformers model, a passage that only the keyword +search returned, without vector support, needs half of the question's IDF-weighted words: an +off-topic question often shares one everyday word with many documents. Coverage counts a misspelled +word through the fuzzy match that found it, and the parts of JSONPath, dotted identifiers, and +elided words (`l'abstention`). When the first five results of a search are all weak, the answer +holds no real evidence and search returns nothing; with `local-hash`, weak means under 30% of the +question's IDF-weighted words. On a Java and TypeScript monorepo, a product documentation base, +and this repository (250 answerable questions), these rules kept every answer and ranking in place +while refusing 68 of 90 off-topic and 26 of 110 near-domain unanswerable questions with +Transformers models, against 54 and 23 with the 6.4 rules. `local-hash` refused 65 and 27 instead +of 12 and 1. +Evaluate your own corpus with `rgr evaluate`. Changing an embedding provider, model, revision, digest, or chunking field requires `rgr ingest --rebuild`. Revision and artifact digest participate in the index, vector-index, and diff --git a/docs/migration.md b/docs/migration.md index b8e5199..214125e 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -3,6 +3,40 @@ This is a breaking surface change. Ragmir now concentrates on local indexing, search, exact citations, OCR, and integration with developer agents. +## Keyword-only evidence and GPU ingestion (6.5) + +Existing indexes stay searchable after the update; nothing needs rebuilding. + +- **Ranking policy version 9.** With a Transformers model, a passage that only the keyword search + returned, without vector support, is flagged `weakEvidence` below half of the question's + IDF-weighted words instead of 30%. One shared everyday word no longer makes an off-topic question + look answered: three real bases refused 68 of 90 off-topic questions instead of 54, with the + same rankings and no correct answer refused. Rerun `rgr evaluate`: stored quality reports no + longer verify the active policy. +- **`local-hash` refuses unsupported answers.** Without a model, search now also answers nothing + when none of its first five passages holds 30% of the question's IDF-weighted words, or when + words found nowhere in the index carry half of the question's IDF weight. The same bases refused + 65 of 90 off-topic and 27 of 110 near-domain questions instead of 12 and 1, and none of 250 + answerable ones. +- **Coverage follows the keyword index.** A misspelled word counts through the fuzzy match that + found it, and JSONPath, dotted identifiers, and elided words (`l'abstention`) count by part. +- **Keyword query, file names, and headings.** The keyword query weights function words such as + "what", "de", or "les" at 0.3, and passages gain up to 20% when their file name or heading path + holds words of the question, unless more than half of the passages hold those words. + On a French and Java monorepo the right file reached the top 3 for 77% of questions instead of + 73% with a model, and 63% instead of 46% without one. +- **Faster warm searches.** A long-running process remembers the query-independent work done per + passage text and file path. Median warm searches on three real bases took 16 to 36 ms instead + of 20 to 39 ms. +- **GPU ingestion.** Set `ingestEmbeddingDevice: "webgpu"` to embed document batches on the GPU. + Search keeps embedding queries on the CPU, and a runtime without WebGPU falls back to it. +- **Explicit ingest with external extractors.** `rgr upgrade` and `rgr doctor --fix` never run + configured OCR or legacy Word commands on their own. When such an advisory stands, their next + step now names `rgr ingest` (or `rgr ingest --rebuild`) instead of a repair that cannot ingest. +- **Evaluation latency.** `rgr evaluate` still scores cases concurrently, but takes `p50LatencyMs` + and `p95LatencyMs` from an untimed warm-up and up to ten sequential searches. Earlier reports + timed concurrent cases, which waited behind each other and read several times higher. + ## Index noise floor and weak answers (6.4) Existing indexes stay searchable after the update; nothing needs deleting. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 1ba2419..f33ca14 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -95,8 +95,8 @@ need explicitly configured local extractors. Results marked `weakEvidence` hold few of the question's distinctive words and have no strong semantic match: rephrase with the project's own terms or narrow the search with a `scope`. A -semantic search returns nothing when its first five results are all weak; a rephrased question -with the project's vocabulary usually finds the evidence if it exists. When one +search returns nothing when its first five results are all weak; a rephrased question with the +project's vocabulary usually finds the evidence if it exists. When one export or log dominates results, `rgr audit` reports index concentration; exclude it, lower it with `sourceWeights`, or cap it with `maxChunksPerDataFile`. @@ -122,6 +122,9 @@ activates. If the process is interrupted, rerun the command to resume; do not de rebuild, then restart or cut over once the upgrade reports `status=current` and `ready=true`. Use `rgr doctor --fix` for current-config setup repairs. Retired configuration fields require `rgr upgrade` first. +With configured external extractors (OCR or legacy Word commands), `rgr upgrade` and +`rgr doctor --fix` end with `action=none`: they never run those commands on their own. Run the +`rgr ingest` (or `rgr ingest --rebuild`) that their reason names. `advisory=...` can accompany `status=current`: retrieval is compatible, but a separate local security control still needs review. Run `rgr security-audit` for the exact follow-up instead of deleting a healthy index. diff --git a/llms.txt b/llms.txt index 99e8c38..e4a221c 100644 --- a/llms.txt +++ b/llms.txt @@ -15,6 +15,9 @@ the query. It owns reasoning, generation, and actions, with the model chosen by first, including source paths and camelCase identifier parts, and uses hashed vectors for ties. - Semantic retrieval requires the optional `@huggingface/transformers` peer and a local model. Use `rgr setup --semantic` to configure and preload one. Default installs exclude that runtime. + `ingestEmbeddingDevice: "webgpu"` embeds ingestion batches on the GPU, with a CPU fallback. + Supported models: `mxbai-embed-xsmall-v1` (English), `multilingual-e5-small` (default) and + `multilingual-e5-base`. Granite R2, EmbeddingGemma 2, and Qwen3-Embedding are not supported yet. - Text, Markdown, structured files, PDF, Office, EPUB, and configured local extractors are supported. `rgr audit --unsupported` reports gaps. Do not claim universal binary support. - Paths are stored and cited with forward slashes and in Unicode NFC on every platform, so @@ -31,7 +34,7 @@ the query. It owns reasoning, generation, and actions, with the model chosen by session as short stubs. Instructions name the base; unrelated workspaces get a scope notice, or a refusal with `mcpAllowedRoots`. Register Ragmir per project, not for every project. - Results carry `sourceStatus` when the cited file changed after indexing and `weakEvidence` when - support is thin. An empty semantic answer means the base holds no evidence for the question. + support is thin. An empty answer means the base holds no evidence for the question. `rgr doctor` checks freshness on every run; `rgr hooks install` refreshes the index after Git operations. - Agents search, expand selected evidence, and refine queries. Ragmir does not run the agent loop. diff --git a/packages/ragmir-core/README.md b/packages/ragmir-core/README.md index 81fe5e0..100c106 100644 --- a/packages/ragmir-core/README.md +++ b/packages/ragmir-core/README.md @@ -11,8 +11,8 @@

- 93% of real developer questions find the right file in the top 10 · 39 ms median search on a - 29,778-chunk monorepo · benchmarks + 94% of real developer questions find the right file in the top 10 · 36 ms median search on a + 30,171-chunk monorepo · benchmarks

@@ -31,11 +31,11 @@ model you choose. Ragmir does not bundle a chat model or host your documents. - **Code and documents in one index:** hybrid BM25 and vector search that understands paths and camelCase identifiers and boosts the code that defines what you ask about. -- **Fast on real repositories:** 39 ms median search on a 29,778-chunk monorepo, with the right - source in the top 10 for 93% of real developer questions +- **Fast on real repositories:** 36 ms median search on a 30,171-chunk monorepo, with the right + source in the top 10 for 94% of real developer questions ([benchmarks](https://github.com/jcode-works/jcode-ragmir#benchmarks)). -- **Ahead of the usual RAG stacks:** the right file reaches the top 3 for 76% of this repository's - developer questions, against 56% for a LangChain.js pipeline and 70% for MiniSearch BM25. +- **Ahead of the usual RAG stacks:** the right file reaches the top 3 for 83% of this repository's + developer questions, against 63% for a LangChain.js pipeline and 79% for MiniSearch BM25. - **Evidence you can verify:** citations point to source lines or native page, slide, sheet/cell, and EPUB coordinates. Results flag changed files (`sourceStatus`) and thin support (`weakEvidence`); questions your sources cannot answer return nothing instead of noise. @@ -113,7 +113,10 @@ scripts. See the [API reference](https://github.com/jcode-works/jcode-ragmir/blo Index Markdown, code, JSON, CSV, HTML, PDF, DOCX, XLSX, PPTX, OpenDocument, EPUB, and RTF; scanned PDFs, photos, and legacy `.doc` files use local extractors (`rgr ocr setup --images --legacy-word`). The default `local-hash` provider needs no model download. For semantic -embeddings, install `@huggingface/transformers` and run `npx rgr setup --semantic`. +embeddings, install `@huggingface/transformers` and run `npx rgr setup --semantic`. Use +`mxbai-embed-xsmall-v1` for English, `multilingual-e5-small` (the default) or `multilingual-e5-base` +for French and mixed languages; newer models such as Granite R2 or EmbeddingGemma 2 are not +supported yet. See [choosing an embedding model](https://github.com/jcode-works/jcode-ragmir#choosing-an-embedding-model). For guided setup, give your agent this prompt: diff --git a/packages/ragmir-core/benchmarks/README.md b/packages/ragmir-core/benchmarks/README.md index 7622a68..17a8ae3 100644 --- a/packages/ragmir-core/benchmarks/README.md +++ b/packages/ragmir-core/benchmarks/README.md @@ -37,9 +37,9 @@ and explicit reranker, compression, hashing, and content-dedup experiments. The gate covers exact references, long-query preservation, and spreadsheet header retrieval. Other outcomes are reported, not presented as representative developer-task success rates. - `bench:competitors` indexes the tracked documentation and source of this repository at the - current commit and asks the questions in `fixtures/repository-questions.json`: 99 developer - questions with every file that answers them, 30 off-topic questions, and 30 near-domain - questions about features Ragmir does not have. The other stacks read the same files split by + current commit and asks the questions in `fixtures/repository-questions.json`: 89 developer + questions with every file that answers them, 30 off-topic questions, and 40 near-domain + questions about features Ragmir does not have, including the retired Chat and TTS packages. The other stacks read the same files split by LangChain's `RecursiveCharacterTextSplitter` (1,000 characters, 200 overlap), and every vector side uses Ragmir's embedding function, so models, prefixes, and vectors are identical. Orama runs with its default 0.8 hybrid similarity and with 0.1, which lets its vector side rank. A From f58d2c94a709d03004910fe9923af7cfa024ecd8 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste THERY Date: Sun, 1 Nov 2026 23:25:16 +0700 Subject: [PATCH 7/8] feat(landing): place the GitHub and npm buttons under the terminal demo - move the hero's View on GitHub and View on npm buttons to the right column, under the terminal animation and aligned with its right edge, on desktop and mobile - show the measured 94% top-10 recall and 36 ms median search in the hero proof --- packages/ragmir-landing/messages/en.json | 6 +-- packages/ragmir-landing/messages/fr.json | 6 +-- .../src/components/sections/hero.astro | 42 +++++++++---------- .../src/public-contract.test.ts | 6 +-- 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/packages/ragmir-landing/messages/en.json b/packages/ragmir-landing/messages/en.json index 566baa6..8bc78e5 100644 --- a/packages/ragmir-landing/messages/en.json +++ b/packages/ragmir-landing/messages/en.json @@ -28,10 +28,10 @@ "hero_github_cta": "View on GitHub", "hero_npm_cta": "View on npm", "hero_npm_downloads_label": "Ragmir downloads/month on npm", - "hero_proof_recall_value": "93%", + "hero_proof_recall_value": "94%", "hero_proof_recall_label": "of real developer questions find the right file in the top 10", - "hero_proof_speed_value": "39 ms", - "hero_proof_speed_label": "median search on a 29,778-chunk monorepo", + "hero_proof_speed_value": "36 ms", + "hero_proof_speed_label": "median search on a 30,171-chunk monorepo", "hero_proof_link": "See the benchmarks", "hero_metric_private_value": "Fewer tokens", "hero_metric_private_label": "Cited passages, never the same one twice.", diff --git a/packages/ragmir-landing/messages/fr.json b/packages/ragmir-landing/messages/fr.json index 9ef3bc6..4d100d6 100644 --- a/packages/ragmir-landing/messages/fr.json +++ b/packages/ragmir-landing/messages/fr.json @@ -28,10 +28,10 @@ "hero_github_cta": "Voir sur GitHub", "hero_npm_cta": "Voir sur npm", "hero_npm_downloads_label": "téléchargements/mois de Ragmir sur npm", - "hero_proof_recall_value": "93 %", + "hero_proof_recall_value": "94 %", "hero_proof_recall_label": "des vraies questions de développeurs trouvent le bon fichier dans le top 10", - "hero_proof_speed_value": "39 ms", - "hero_proof_speed_label": "de recherche médiane sur un monorepo de 29 778 passages", + "hero_proof_speed_value": "36 ms", + "hero_proof_speed_label": "de recherche médiane sur un monorepo de 30 171 passages", "hero_proof_link": "Voir les benchmarks", "hero_metric_private_value": "Moins de tokens", "hero_metric_private_label": "Des passages cités, jamais deux fois le même.", diff --git a/packages/ragmir-landing/src/components/sections/hero.astro b/packages/ragmir-landing/src/components/sections/hero.astro index 6ba497b..088ed50 100644 --- a/packages/ragmir-landing/src/components/sections/hero.astro +++ b/packages/ragmir-landing/src/components/sections/hero.astro @@ -66,26 +66,6 @@ const heroTitleLines = [ {t("hero_secondary_cta")} -

- - - {t("hero_github_cta")} - - - - {t("hero_npm_cta")} - -
{monthlyNpmDownloads !== null && (
-
+
+
+ + + {t("hero_github_cta")} + + + + {t("hero_npm_cta")} + +