diff --git a/frontend/src/pages/ProvidersPage.tsx b/frontend/src/pages/ProvidersPage.tsx
index 95e80f1..ed92186 100644
--- a/frontend/src/pages/ProvidersPage.tsx
+++ b/frontend/src/pages/ProvidersPage.tsx
@@ -5,6 +5,7 @@ import {
setLLMPreferences,
getLLMPreferences,
getProviderSettings,
+ type ProviderRoutingSettings,
setProviderSettings,
} from '../api';
import type {
@@ -31,31 +32,43 @@ import {
Zap,
} from 'lucide-react';
import { useNavigate } from 'react-router-dom';
+import { CACHE_KEYS } from '../cacheKeys'
+import { useOptimisticMutation } from '../useOptimisticMutation'
export function ProvidersPage() {
const navigate = useNavigate();
- const { data: providers, refetch: refetchProviders, loading } = useApi(getProviders, { cacheKey: 'providers-list' });
- const { data: prefs, refetch: refetchPrefs } = useApi(getLLMPreferences, { cacheKey: 'llm-preferences' });
- const { data: routingSettings, refetch: refetchSettings } = useApi(getProviderSettings, { cacheKey: 'provider-settings' });
+ const { data: providers, refetch: refetchProviders, loading } = useApi(getProviders, { cacheKey: CACHE_KEYS.providersList });
+ const { data: prefs, refetch: refetchPrefs } = useApi(getLLMPreferences, { cacheKey: CACHE_KEYS.llmPreferences });
+ const { data: routingSettings, refetch: refetchSettings } = useApi(getProviderSettings, { cacheKey: CACHE_KEYS.providerSettings });
+
+ // Both of these write provider settings, so they share one cache entry and one
+ // optimistic path: the list reorders and the checkbox moves on click instead
+ // of waiting for the round trip and the refetch behind it.
+ const updateRoutingSettings = useOptimisticMutation<
+ Partial,
+ unknown,
+ ProviderRoutingSettings
+ >({
+ mutationFn: setProviderSettings,
+ cacheKey: CACHE_KEYS.providerSettings,
+ optimistic: (current, patch) => (current ? { ...current, ...patch } : current),
+ });
- const handleMoveFallback = async (index: number, direction: number) => {
+ const handleMoveFallback = (index: number, direction: number) => {
if (!routingSettings) return;
const newChain = [...routingSettings.fallback_chain];
const targetIndex = index + direction;
if (targetIndex < 0 || targetIndex >= newChain.length) return;
-
+
// Swap
const temp = newChain[index];
newChain[index] = newChain[targetIndex];
newChain[targetIndex] = temp;
-
- await setProviderSettings({
+
+ updateRoutingSettings.mutate({
provider: routingSettings.provider,
- fallback_chain: newChain
+ fallback_chain: newChain,
});
-
- invalidateCache('provider-settings');
- refetchSettings();
};
const [selectedId, setSelectedId] = useState('gemini');
@@ -429,10 +442,9 @@ export function ProvidersPage() {
{
- await setProviderSettings({ cloud_privacy_consent: e.target.checked });
- invalidateCache('provider-settings');
- refetchSettings();
+ disabled={updateRoutingSettings.isPending}
+ onChange={e => {
+ updateRoutingSettings.mutate({ cloud_privacy_consent: e.target.checked });
}}
className="rounded border-amber-500/40"
/>
diff --git a/frontend/src/pages/SearchPage.tsx b/frontend/src/pages/SearchPage.tsx
index aea6eca..5e7982f 100644
--- a/frontend/src/pages/SearchPage.tsx
+++ b/frontend/src/pages/SearchPage.tsx
@@ -9,6 +9,7 @@ import { MessageBubble } from '../components/chat/MessageBubble';
import { FilterBar } from '../components/chat/FilterBar';
import { ModelPicker } from '../components/providers/ModelPicker';
import { useDreamscapeStore } from '../store/dreamscapeStore';
+import { CACHE_KEYS } from '../cacheKeys'
export function SearchPage() {
const selectedChunks = useDreamscapeStore(state => state.selectedChunks);
@@ -22,10 +23,10 @@ export function SearchPage() {
const [selectedFolderTag, setSelectedFolderTag] = useState('');
const [selectedMode, setSelectedMode] = useState('');
- const { data: historyData, refetch: refetchHistory } = useApi(getQueryHistory, { cacheKey: 'query-history' });
+ const { data: historyData, refetch: refetchHistory } = useApi(getQueryHistory, { cacheKey: CACHE_KEYS.queryHistory });
const { messages, executeSearch, resetChat: resetChatStream, stopStream } = useChatStream(() => {
- invalidateCache('query-history');
+ invalidateCache(CACHE_KEYS.queryHistory);
refetchHistory();
});
@@ -35,7 +36,7 @@ export function SearchPage() {
// already pushes those events. A fixed 15s poll re-fetched the whole tree
// forever on a corpus that had not moved.
const { data: fileTree, refetch: refetchFileTree } = useApi(getFileTree, {
- cacheKey: 'files-tree',
+ cacheKey: CACHE_KEYS.fileTree,
});
useEffect(() => {
@@ -45,7 +46,7 @@ export function SearchPage() {
// frame - those arrive continuously during a scan.
const status = evt?.status ?? '';
if (status !== last && (status === 'completed' || status === 'idle')) {
- invalidateCache('files-tree');
+ invalidateCache(CACHE_KEYS.fileTree);
refetchFileTree();
}
last = status;
@@ -109,7 +110,7 @@ export function SearchPage() {
const doClear = async () => {
try {
const res = await clearQueryHistory();
- invalidateCache('query-history');
+ invalidateCache(CACHE_KEYS.queryHistory);
refetchHistory();
resetChatStream();
// The backend reports whether the persistent semantic cache went with
diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsx
index 016959e..cb7c976 100644
--- a/frontend/src/pages/SettingsPage.tsx
+++ b/frontend/src/pages/SettingsPage.tsx
@@ -15,6 +15,7 @@ import {
getProviderLaunchStatus,
launchProvider,
getOcrStatus,
+ type OcrStatus,
getOcrTiers,
selectOcrTier,
getVlmModels,
@@ -37,6 +38,8 @@ import {
type OcrInstallState,
type OcrQueueItem
} from '../api'
+import { CACHE_KEYS } from '../cacheKeys'
+import { useOptimisticMutation } from '../useOptimisticMutation'
// STATIC_FALLBACK_MODELS removed in favor of dynamic backend discovery and persistent model heaps.
@@ -68,8 +71,8 @@ function StartLocalProviderButton({ providerId, displayName, offlineHint, onStar
try {
const res = await launchProvider(providerId)
if (res.ok) {
- invalidateCache('local-models')
- invalidateCache('providers-list')
+ invalidateCache(CACHE_KEYS.localModels)
+ invalidateCache(CACHE_KEYS.providersList)
onStarted()
} else {
setError(res.message)
@@ -231,8 +234,8 @@ function LLMPreferencesSection({
saving: boolean
}>) {
const navigate = useNavigate()
- const { data: llmPrefs } = useApi(getLLMPreferences, { cacheKey: 'llm-prefs' })
- const { data: providers } = useApi(getProviders, { cacheKey: 'providers-list' })
+ const { data: llmPrefs } = useApi(getLLMPreferences, { cacheKey: CACHE_KEYS.llmPreferences })
+ const { data: providers } = useApi(getProviders, { cacheKey: CACHE_KEYS.providersList })
const [provider, setProvider] = useState('auto')
const [selectedModels, setSelectedModels] = useState>({})
@@ -395,9 +398,9 @@ function VlmPicker({
onChanged,
setNote,
}: Readonly<{ onChanged: () => void; setNote: (s: string) => void }>) {
- const { data, loading, refetch } = useApi(getVlmModels, { cacheKey: 'ocr-vlm-models' })
+ const { data, loading, refetch } = useApi(getVlmModels, { cacheKey: CACHE_KEYS.ocrVlmModels })
const { data: current, refetch: refetchSelection } = useApi(getVlmSelection, {
- cacheKey: 'ocr-vlm-selection',
+ cacheKey: CACHE_KEYS.ocrVlmSelection,
})
const [saving, setSaving] = useState('')
const [isChecking, setIsChecking] = useState(false)
@@ -412,8 +415,8 @@ function VlmPicker({
return
}
setNote(`OCR will use ${model}.`)
- invalidateCache('ocr-tiers')
- invalidateCache('ocr-status-settings')
+ invalidateCache(CACHE_KEYS.ocrTiers)
+ invalidateCache(CACHE_KEYS.ocrStatus)
refetchSelection()
onChanged()
} catch (e) {
@@ -425,7 +428,7 @@ function VlmPicker({
const handleCheckAgain = async () => {
setIsChecking(true)
- invalidateCache('ocr-vlm-models')
+ invalidateCache(CACHE_KEYS.ocrVlmModels)
try {
await refetch()
} finally {
@@ -440,8 +443,8 @@ function VlmPicker({
const res = await launchProvider(providerId)
if (res.ok) {
setNote(`${displayName} started successfully.`)
- invalidateCache('ocr-vlm-models')
- invalidateCache('local-models')
+ invalidateCache(CACHE_KEYS.ocrVlmModels)
+ invalidateCache(CACHE_KEYS.localModels)
await refetch()
} else {
setNote(res.message || `Could not start ${displayName}.`)
@@ -589,10 +592,10 @@ function VlmPicker({
function OcrSection() {
const { data: ocr, refetch } = useApi(getOcrStatus, {
- cacheKey: 'ocr-status-settings',
+ cacheKey: CACHE_KEYS.ocrStatus,
refetchInterval: 10_000,
})
- const { data: tierData, refetch: refetchTiers } = useApi(getOcrTiers, { cacheKey: 'ocr-tiers' })
+ const { data: tierData, refetch: refetchTiers } = useApi(getOcrTiers, { cacheKey: CACHE_KEYS.ocrTiers })
const tiers = tierData?.tiers ?? []
const [selectedTier, setSelectedTier] = useState('cpu')
const selectedTierInfo = tiers.find((t) => t.id === selectedTier)
@@ -621,8 +624,8 @@ function OcrSection() {
const state = await getOcrInstallState()
setInstall(state)
if (state.status !== 'running') {
- invalidateCache('ocr-tiers')
- invalidateCache('ocr-status-settings')
+ invalidateCache(CACHE_KEYS.ocrTiers)
+ invalidateCache(CACHE_KEYS.ocrStatus)
refetch()
refetchTiers()
setBusy(false)
@@ -667,8 +670,8 @@ function OcrSection() {
return
}
setNote(`OCR switched to ${TIER_COPY[tier]?.short ?? tier}.`)
- invalidateCache('ocr-tiers')
- invalidateCache('ocr-status-settings')
+ invalidateCache(CACHE_KEYS.ocrTiers)
+ invalidateCache(CACHE_KEYS.ocrStatus)
refetch()
refetchTiers()
} catch (e) {
@@ -685,8 +688,8 @@ function OcrSection() {
await uninstallOcrTier(target)
setInstall(null)
setNote(`Uninstalled ${TIER_COPY[target]?.short ?? target} tier.`)
- invalidateCache('ocr-tiers')
- invalidateCache('ocr-status-settings')
+ invalidateCache(CACHE_KEYS.ocrTiers)
+ invalidateCache(CACHE_KEYS.ocrStatus)
refetch()
refetchTiers()
} catch (e) {
@@ -696,16 +699,25 @@ function OcrSection() {
}
}
- const handleToggle = async (enabled: boolean) => {
- try {
- const res = await setOcrEnabled(enabled)
+ // The checkbox reads `checked={!!ocr?.enabled}` straight off server data, so
+ // before this it visibly snapped back to its old position and stayed there
+ // until the refetch landed. The optimistic write moves it on click and the
+ // rollback puts it back only if the server actually refuses.
+ const toggleOcr = useOptimisticMutation>, OcrStatus>({
+ mutationFn: setOcrEnabled,
+ cacheKey: CACHE_KEYS.ocrStatus,
+ invalidates: [CACHE_KEYS.ocrTiers],
+ optimistic: (current, enabled) => (current ? { ...current, enabled } : current),
+ onSuccess: (res) => {
+ // A 200 that says "no" is not an error, so it does not roll back. The
+ // onSettled invalidation refetches and the real value wins; this only
+ // explains why.
if (!res.ok) setNote(`Cannot enable: ${res.error_code}`)
- invalidateCache('ocr-tiers')
- invalidateCache('ocr-status-settings')
- refetch()
- refetchTiers()
- } catch (e) { setNote(e instanceof Error ? e.message : 'Could not change OCR state.') }
- }
+ },
+ onError: (e) => setNote(e instanceof Error ? e.message : 'Could not change OCR state.'),
+ })
+
+ const handleToggle = (enabled: boolean) => toggleOcr.mutate(enabled)
const handleResume = async () => {
try {
@@ -1133,10 +1145,10 @@ function ResetSection({ onRestartOnboarding, onFullReset }: Readonly<{ onRestart
// ── Main Page Component ──────────────────────────────────────────────
export function SettingsPage() {
- const { data: localModels, refetch: refetchLocalModels } = useApi(getLocalModels, { cacheKey: 'local-models' })
- const { data: sysInfo } = useApi(getSystemInfo, { cacheKey: 'system-info' })
- const { refetch: refetchPrefs } = useApi(getLLMPreferences, { cacheKey: 'llm-prefs' })
- const { data: driveInfo } = useApi(getDriveInfo, { cacheKey: 'drive-info' })
+ const { data: localModels, refetch: refetchLocalModels } = useApi(getLocalModels, { cacheKey: CACHE_KEYS.localModels })
+ const { data: sysInfo } = useApi(getSystemInfo, { cacheKey: CACHE_KEYS.systemInfo })
+ const { refetch: refetchPrefs } = useApi(getLLMPreferences, { cacheKey: CACHE_KEYS.llmPreferences })
+ const { data: driveInfo } = useApi(getDriveInfo, { cacheKey: CACHE_KEYS.driveInfo })
const [message, setMessage] = useState<{ type: 'ok' | 'err'; text: string } | null>(null)
const [savingPrefs, setSavingPrefs] = useState(false)
@@ -1154,8 +1166,8 @@ export function SettingsPage() {
await setProviderDefaultModel(providerId, val).catch(() => {})
}
}
- invalidateCache('llm-prefs')
- invalidateCache('providers-list')
+ invalidateCache(CACHE_KEYS.llmPreferences)
+ invalidateCache(CACHE_KEYS.providersList)
refetchPrefs()
setMessage({ type: 'ok', text: 'LLM preferences saved.' })
} catch (e) {
diff --git a/frontend/src/pages/SetupPage.tsx b/frontend/src/pages/SetupPage.tsx
index 509046d..b4d17b3 100644
--- a/frontend/src/pages/SetupPage.tsx
+++ b/frontend/src/pages/SetupPage.tsx
@@ -3,6 +3,7 @@ import { Brain, Shield, ArrowRight, CheckCircle2, ChevronRight, HardDrive, Alert
import { useNavigate } from 'react-router-dom'
import { useApi } from '../useApi'
import { getLocalModels, getDriveInfo, enableSplitBrain, getProviders, setProviderKey } from '../api'
+import { CACHE_KEYS } from '../cacheKeys'
const PROVIDERS = [
{ id: 'gemini', name: 'Google Gemini', icon: '✨' },
@@ -12,7 +13,7 @@ const PROVIDERS = [
]
function ApiKeyInput({ provider }: { provider: typeof PROVIDERS[0] }) {
- const { data: providers, refetch } = useApi(getProviders, { cacheKey: 'providers-list' })
+ const { data: providers, refetch } = useApi(getProviders, { cacheKey: CACHE_KEYS.providersList })
const pData = providers?.find(p => p.spec.id === provider.id)
const [key, setKey] = useState('')
const [saving, setSaving] = useState(false)
@@ -84,9 +85,9 @@ function ApiKeyInput({ provider }: { provider: typeof PROVIDERS[0] }) {
export function SetupPage() {
const navigate = useNavigate()
- const { data: providers, refetch: refetchProviders, loading: providersLoading } = useApi(getProviders, { cacheKey: 'providers-list' })
- const { data: localModels, loading: localModelsLoading } = useApi(getLocalModels, { cacheKey: 'local-models' })
- const { data: driveInfo, loading: driveLoading } = useApi(getDriveInfo, { cacheKey: 'drive-info' })
+ const { data: providers, refetch: refetchProviders, loading: providersLoading } = useApi(getProviders, { cacheKey: CACHE_KEYS.providersList })
+ const { data: localModels, loading: localModelsLoading } = useApi(getLocalModels, { cacheKey: CACHE_KEYS.localModels })
+ const { data: driveInfo, loading: driveLoading } = useApi(getDriveInfo, { cacheKey: CACHE_KEYS.driveInfo })
const isLoading = providersLoading || localModelsLoading || driveLoading
@@ -181,7 +182,14 @@ export function SetupPage() {
)}
- {!isLoading && !isDriveConfigSafe && driveInfo && !requiresRestart && (
+ {/* Gated on data, not on fetching state. `isLoading` is
+ `isLoading || isFetching`, and the focus/visibilitychange
+ listeners above refetch on every alt-tab - so `!isLoading`
+ made this storage-compatibility warning blink out exactly
+ when the user returned to the page. `driveInfo` presence is
+ the real guard, and isDriveConfigSafe already defaults to
+ true while it is absent. */}
+ {!isDriveConfigSafe && driveInfo && !requiresRestart && (
@@ -230,7 +238,9 @@ export function SetupPage() {
)}
{/* Drive OK badge — shown when split_brain is active on portable drive */}
- {!isLoading && isDriveConfigSafe && driveInfo?.is_portable_fs && (
+ {/* Same reasoning as the warning above: `driveInfo?.is_portable_fs`
+ already requires the data to have arrived. */}
+ {isDriveConfigSafe && driveInfo?.is_portable_fs && (
diff --git a/frontend/src/providers/ProviderRecipes.tsx b/frontend/src/providers/ProviderRecipes.tsx
index c26d7ed..195cd72 100644
--- a/frontend/src/providers/ProviderRecipes.tsx
+++ b/frontend/src/providers/ProviderRecipes.tsx
@@ -2,6 +2,7 @@ import { Rocket, Zap, Shield, Star, X, Loader2 } from 'lucide-react';
import { useState } from 'react';
import { setProviderSettings, setLLMPreferences, getLLMPreferences } from '../api';
import { invalidateCache } from '../useApi';
+import { CACHE_KEYS } from '../cacheKeys'
export function ProviderRecipes({
onRecipeApplied,
@@ -32,8 +33,8 @@ export function ProviderRecipes({
[`${defaultModel.provider}_model`]: defaultModel.model,
});
- invalidateCache('provider-settings');
- invalidateCache('llm-preferences');
+ invalidateCache(CACHE_KEYS.providerSettings);
+ invalidateCache(CACHE_KEYS.llmPreferences);
onRecipeApplied();
} catch (e: any) {
setApplyError(e.message || 'Failed to apply recipe');
diff --git a/frontend/src/useApi.ts b/frontend/src/useApi.ts
index e08453f..ab32fe4 100644
--- a/frontend/src/useApi.ts
+++ b/frontend/src/useApi.ts
@@ -1,5 +1,6 @@
import { useQuery } from '@tanstack/react-query'
import { queryClient } from './queryClient'
+import { CORPUS_DERIVED_KEYS } from './cacheKeys'
import { useId } from 'react'
interface UseApiOptions {
@@ -48,12 +49,32 @@ export function useApi(
}
}
-/** Invalidate cache entries matching a prefix query key */
-export function invalidateCache(prefix?: string) {
- if (!prefix) {
- queryClient.clear()
- return
- }
+/**
+ * Invalidate cache entries matching a prefix query key.
+ *
+ * The prefix is required. It used to be optional, and calling with no argument
+ * ran `queryClient.clear()` — which *removes* every query rather than marking it
+ * stale (`getAll().forEach(q => this.remove(q))` in query-core 5.94.5). Every
+ * mounted observer therefore dropped to `data: undefined` and every page fell
+ * back to its cold-start spinner. Four call sites did that, including the one
+ * that fires when an index run finishes, so completing a scan blanked the whole
+ * app. Invalidation refetches while keeping the previous data on screen, which
+ * is what all of them wanted.
+ */
+export function invalidateCache(prefix: string) {
// React Query fuzzy-matches keys, so passing ['prefix'] invalidates any key starting with ['prefix']
queryClient.invalidateQueries({ queryKey: [prefix] })
}
+
+/**
+ * Refresh everything derived from the indexed corpus.
+ *
+ * The replacement for the bare `invalidateCache()` calls: indexing, clearing and
+ * a manual refresh all change the same set of views, and naming that set beats
+ * evicting unrelated caches (providers, API keys, OCR tiers) to reach it.
+ */
+export function invalidateCorpusCaches() {
+ for (const key of CORPUS_DERIVED_KEYS) {
+ invalidateCache(key)
+ }
+}
diff --git a/frontend/src/useOptimisticMutation.ts b/frontend/src/useOptimisticMutation.ts
new file mode 100644
index 0000000..bdefe0c
--- /dev/null
+++ b/frontend/src/useOptimisticMutation.ts
@@ -0,0 +1,63 @@
+import { useMutation, useQueryClient } from '@tanstack/react-query'
+
+interface Options {
+ /** The write itself. */
+ mutationFn: (vars: TVars) => Promise
+ /**
+ * The cache entry this write changes. Updated immediately, restored if the
+ * server rejects. Matches `useApi`'s query key, which is `[cacheKey]`.
+ */
+ cacheKey: string
+ /** Next cached value, given the current one. Return it unchanged to opt out. */
+ optimistic: (current: TCache | undefined, vars: TVars) => TCache | undefined
+ /** Other keys the write invalidates once the server has answered. */
+ invalidates?: readonly string[]
+ onError?: (error: unknown, vars: TVars) => void
+ onSuccess?: (data: TData, vars: TVars) => void
+}
+
+/**
+ * A mutation that shows its result before the server confirms it.
+ *
+ * Every write in this app used to be a bare `await` followed by a refetch, so a
+ * control bound to server state (a checkbox reading `checked={!!ocr?.enabled}`,
+ * a list rendered from `prefs.fallback_chain`) visibly snapped back to the old
+ * value until the round-trip finished — and destructive actions with no pending
+ * state could be fired twice.
+ *
+ * The ceremony that makes this safe is easy to get wrong once, let alone four
+ * times, hence one helper: cancel in-flight refetches so a stale response cannot
+ * land on top of the optimistic value, snapshot for rollback, restore on error,
+ * and re-sync from the server either way.
+ */
+export function useOptimisticMutation(
+ opts: Options,
+) {
+ const client = useQueryClient()
+
+ return useMutation({
+ mutationFn: opts.mutationFn,
+ onMutate: async (vars) => {
+ // Without this, a refetch already in flight can resolve after the
+ // optimistic write and overwrite it with the pre-mutation server state.
+ await client.cancelQueries({ queryKey: [opts.cacheKey] })
+ const previous = client.getQueryData([opts.cacheKey])
+ client.setQueryData([opts.cacheKey], (current) =>
+ opts.optimistic(current, vars),
+ )
+ return { previous }
+ },
+ onError: (error, vars, context) => {
+ // Restore exactly what was there, including "nothing".
+ client.setQueryData([opts.cacheKey], context?.previous)
+ opts.onError?.(error, vars)
+ },
+ onSuccess: opts.onSuccess,
+ onSettled: () => {
+ client.invalidateQueries({ queryKey: [opts.cacheKey] })
+ for (const key of opts.invalidates ?? []) {
+ client.invalidateQueries({ queryKey: [key] })
+ }
+ },
+ })
+}
diff --git a/tests/test_coverage_booster_gen3.py b/tests/test_coverage_booster_gen3.py
index cdae94e..00bbf0d 100644
--- a/tests/test_coverage_booster_gen3.py
+++ b/tests/test_coverage_booster_gen3.py
@@ -80,19 +80,33 @@ async def test_db_manager_coverage_booster(mock_db):
(10, file_id, 0, 20, compressed_text),
)
+ # A second chunk so the graph below spans two of them. Needed now that
+ # kg_nodes.chunk_id is populated: the column carries
+ # FOREIGN KEY (chunk_id) REFERENCES chunks(id), which a NULL column left
+ # unenforced. Production always has the referent, so the fixture should too.
+ await db.execute_write(
+ "INSERT INTO chunks (id, file_id, start_offset, end_offset, text_preview) VALUES (?, ?, ?, ?, ?)",
+ (11, file_id, 20, 40, compressed_text),
+ )
+
# 5. Test BFS and knowledge graph queries
assert await db.bfs_from_chunks([]) == []
assert await db.get_relational_paths([]) == []
- # Insert kg_nodes and kg_edges
- # properties contains chunk_id
+ # Insert kg_nodes and kg_edges.
+ # chunk_id goes in the column, not only in the properties JSON. This fixture
+ # used to omit the column and rely on json_extract(properties,'$.chunk_id'),
+ # which is a row shape production never writes: insert_kg_nodes_bulk fills
+ # both from the same variable (app/indexing/service.py), and the _migrate
+ # back-fill populates the column for pre-existing rows. The traversal reads
+ # the column now, so writing only the JSON tested a path nothing produced.
await db.execute_write(
- "INSERT INTO kg_nodes (id, label, type, properties) VALUES (?, ?, ?, ?)",
- ("node1", "Class", "type_foo", '{"chunk_id": 10}'),
+ "INSERT INTO kg_nodes (id, label, type, properties, chunk_id) VALUES (?, ?, ?, ?, ?)",
+ ("node1", "Class", "type_foo", '{"chunk_id": 10}', 10),
)
await db.execute_write(
- "INSERT INTO kg_nodes (id, label, type, properties) VALUES (?, ?, ?, ?)",
- ("node2", "Method", "type_bar", '{"chunk_id": 11}'),
+ "INSERT INTO kg_nodes (id, label, type, properties, chunk_id) VALUES (?, ?, ?, ?, ?)",
+ ("node2", "Method", "type_bar", '{"chunk_id": 11}', 11),
)
await db.execute_write(
"INSERT INTO kg_edges (source, target, relation) VALUES (?, ?, ?)",
diff --git a/tests/test_indexing_executor_lifecycle.py b/tests/test_indexing_executor_lifecycle.py
index 2ea22fe..062a906 100644
--- a/tests/test_indexing_executor_lifecycle.py
+++ b/tests/test_indexing_executor_lifecycle.py
@@ -394,3 +394,79 @@ async def test_root_path_is_added_to_an_existing_index(self, tmp_path):
# derives a fallback root for exactly these rows.
assert [tuple(r) for r in rows] == [("/srv/proj/a.py", "")]
assert "idx_files_root_path" in idx
+
+ @pytest.mark.asyncio
+ async def test_the_redundant_text_lookup_index_is_gone_and_stays_gone(self, tmp_path):
+ """`idx_chunks_text_lookup` duplicated the whole compressed corpus.
+
+ `chunks.id` is the rowid, so an index on (id, text_preview) is a second
+ full copy of the text column. Measured on 21,584 chunks: +90.2% on disk
+ and +57.8% insert time, against no read benefit on any query shape -
+ including the FTS-rebuild projection, the only one the planner chose it
+ for. This is the same defect as `idx_chunks_covering`, which was already
+ dropped for the same reason.
+
+ What this test pins is the DROP in `_migrate`. It is the load-bearing
+ half: `executescript(schema.sql)` runs *before* `_migrate`, so the drop
+ wins regardless of what schema.sql declares - verified by restoring the
+ CREATE line and watching this still pass. Removing the CREATE is
+ therefore not what makes the behaviour correct; it is what stops every
+ startup from building the index and immediately dropping it again.
+
+ The reverse control does bite: disable the DROP and the legacy half
+ fails, because `executescript` never reconciles away a declaration that
+ schema.sql has stopped making.
+ """
+ import aiosqlite
+
+ from app.storage.db import DatabaseManager
+
+ # Half one: an existing index that already carries it must lose it.
+ legacy = tmp_path / "legacy.db"
+ async with aiosqlite.connect(str(legacy)) as conn:
+ await conn.execute(
+ "CREATE TABLE chunks (id INTEGER PRIMARY KEY AUTOINCREMENT,"
+ " file_id INTEGER NOT NULL, start_offset INTEGER NOT NULL,"
+ " end_offset INTEGER NOT NULL, text_preview TEXT NOT NULL)"
+ )
+ await conn.execute("CREATE INDEX idx_chunks_text_lookup ON chunks(id, text_preview)")
+ await conn.commit()
+ async with conn.execute(
+ "SELECT name FROM sqlite_master WHERE type='index' AND name=?",
+ ("idx_chunks_text_lookup",),
+ ) as cur:
+ assert await cur.fetchone(), "fixture did not create the index"
+
+ mgr = DatabaseManager(str(legacy))
+ try:
+ await mgr.init_db(schema_path="app/storage/schema.sql")
+ migrated = {
+ r[0]
+ for r in await mgr.execute_query(
+ "SELECT name FROM sqlite_master WHERE type='index'"
+ )
+ }
+ finally:
+ await mgr.close()
+
+ assert "idx_chunks_text_lookup" not in migrated, (
+ "an existing database kept the redundant index through init_db"
+ )
+ # The FK index is the one that earns its keep; it must survive.
+ assert "idx_chunks_file_id" in migrated
+
+ # Half two: a fresh database must never grow it in the first place.
+ fresh_mgr = DatabaseManager(str(tmp_path / "fresh.db"))
+ try:
+ await fresh_mgr.init_db(schema_path="app/storage/schema.sql")
+ fresh = {
+ r[0]
+ for r in await fresh_mgr.execute_query(
+ "SELECT name FROM sqlite_master WHERE type='index'"
+ )
+ }
+ finally:
+ await fresh_mgr.close()
+
+ assert "idx_chunks_text_lookup" not in fresh, "schema.sql still declares it"
+ assert "idx_chunks_file_id" in fresh
diff --git a/tests/test_retrieval_hardening.py b/tests/test_retrieval_hardening.py
index 563f6f9..f425526 100644
--- a/tests/test_retrieval_hardening.py
+++ b/tests/test_retrieval_hardening.py
@@ -897,3 +897,84 @@ async def test_agentic_loop_is_off_by_default(monkeypatch):
"q", plan, MagicMock(), MagicMock(), MagicMock(), MagicMock(), 10, None, None, None
)
assert got is None and trace is None
+
+
+@pytest.mark.asyncio
+async def test_graph_seed_lookup_uses_an_index_rather_than_scanning(real_db):
+ """The seed step must be an index search, not a full scan of kg_nodes.
+
+ `_bfs_cte` used to select seeds with
+ `json_extract(properties, '$.chunk_id') IN (...)`, which is not sargable, so
+ every GRAPH_SEARCH began by scanning kg_nodes and JSON-parsing each row -
+ while `kg_nodes.chunk_id` sat right there, written on every insert and
+ indexed by nothing. Measured on 64,752 nodes with 20 seeds: 22.54 ms
+ scanning against 0.06 ms through the index.
+
+ Asserted against the query plan because that is the property; a behavioural
+ test passes either way, which is exactly why this survived so long.
+ """
+ chunk_ids = await _seed_cycle(real_db, size=3)
+
+ query = DatabaseManager._bfs_cte("?") + " SELECT COUNT(*) FROM bfs_nodes" # noqa: S608
+ plan = [
+ row[3]
+ for row in await real_db.execute_query("EXPLAIN QUERY PLAN " + query, (chunk_ids[0], 3, 3))
+ ]
+
+ seed_steps = [step for step in plan if "kg_nodes" in step]
+ assert seed_steps, plan
+ assert any("idx_kg_nodes_chunk_id" in step for step in seed_steps), plan
+ assert not any(step.startswith("SCAN kg_nodes") for step in seed_steps), plan
+
+
+@pytest.mark.asyncio
+async def test_nodes_whose_chunk_id_lives_only_in_json_are_backfilled(tmp_path: Path):
+ """A NULL chunk_id column would silently drop a node from every graph seed.
+
+ The traversal reads `kg_nodes.chunk_id` rather than parsing
+ `properties`. The existing rebuild migration back-fills that column, but
+ only when the column is absent altogether - a database that already had it
+ keeps whatever was written. Verified divergent before this back-fill
+ existed: with one JSON-only row, the old predicate returned it and the
+ column predicate did not.
+
+ Orphans stay NULL on purpose: chunk_id is a FOREIGN KEY onto chunks(id) with
+ foreign_keys ON, so adopting a deleted chunk would fail the migration.
+ """
+ db_file = tmp_path / "backfill.db"
+ mgr = DatabaseManager(str(db_file))
+ await mgr.init_db(schema_path="app/storage/schema.sql")
+ conn = mgr._get_conn()
+ await conn.execute("INSERT INTO files (path,size,modified_at,type) VALUES ('f',1,'now','.py')")
+ await conn.execute(
+ "INSERT INTO chunks (id,file_id,start_offset,end_offset,text_preview) VALUES (7,1,0,10,?)",
+ (zlib.compress(b"body"),),
+ )
+ # Provenance in the JSON only, as a pre-existing row could carry it.
+ await conn.execute(
+ "INSERT INTO kg_nodes (id,type,label,properties,chunk_id) VALUES (?,?,?,?,?)",
+ ("orphaned_col", "func", "f", json.dumps({"chunk_id": 7}), None),
+ )
+ # Same shape, but the chunk it names is gone.
+ await conn.execute(
+ "INSERT INTO kg_nodes (id,type,label,properties,chunk_id) VALUES (?,?,?,?,?)",
+ ("dangling", "func", "g", json.dumps({"chunk_id": 999}), None),
+ )
+ await conn.commit()
+ await mgr.close()
+
+ # Re-open: _migrate runs again and should adopt the recoverable row.
+ mgr = DatabaseManager(str(db_file))
+ try:
+ await mgr.init_db(schema_path="app/storage/schema.sql")
+ rows = dict(
+ (r[0], r[1])
+ for r in await mgr.execute_query("SELECT id, chunk_id FROM kg_nodes ORDER BY id")
+ )
+ seeds = await mgr.bfs_from_chunks([7], max_depth=1, limit=5)
+ finally:
+ await mgr.close()
+
+ assert rows["orphaned_col"] == 7, "a recoverable node was left unreachable"
+ assert rows["dangling"] is None, "a node pointing at a deleted chunk must stay NULL"
+ assert 7 in seeds, "the back-filled node is still not reachable as a graph seed"
diff --git a/tests/test_stream_and_history_hardening.py b/tests/test_stream_and_history_hardening.py
index 100e0a5..d283985 100644
--- a/tests/test_stream_and_history_hardening.py
+++ b/tests/test_stream_and_history_hardening.py
@@ -2,6 +2,7 @@
import asyncio
import json
+import os
import pytest
from httpx import AsyncClient
@@ -93,3 +94,152 @@ async def test_injected_token_script_carries_a_non_empty_nonce(client: AsyncClie
csp = response.headers["content-security-policy"]
assert "'nonce-'" not in csp
+
+
+@pytest.mark.asyncio
+async def test_the_ndjson_stream_opts_out_of_gzip(client: AsyncClient, monkeypatch):
+ """Cheap guard on the header that keeps GZipMiddleware off this route.
+
+ The expensive property - that bytes actually leave early - is asserted by
+ the test below. This one just pins the mechanism, because it is a single
+ kwarg in app/api/search.py that a later edit could drop without any content
+ assertion in this file noticing.
+ """
+
+ async def _one_record(*args, **kwargs):
+ yield {"type": "content", "text": "hello"}
+
+ monkeypatch.setattr("app.search.retrieval.stream_rag", _one_record)
+
+ response = await client.post("/api/query/stream", json={"question": "hi"})
+ assert response.status_code == 200
+ assert response.headers.get("content-encoding") == "identity"
+
+
+@pytest.mark.asyncio
+async def test_first_token_is_sent_before_the_generator_finishes(client, monkeypatch):
+ """The NDJSON stream must actually stream, not arrive as one block.
+
+ GZipMiddleware (app/main.py) compresses any streaming response whose content
+ type it does not exempt, and it exempts only text/event-stream. Its
+ GZipResponder writes each chunk into a GzipFile without flushing, so deflate
+ holds every token until the generator closes. Content is unaffected - which
+ is why every other test in this file passed throughout - so only arrival
+ *order* can catch it.
+
+ Driven against the ASGI app directly rather than through the `client`
+ fixture, because httpx's ASGITransport cannot express this: it awaits the
+ app to completion and accumulates the body before constructing the Response
+ (httpx/_transports/asgi.py:170,185). Nothing routed through that transport
+ can observe streaming at all.
+
+ Asserted without wall-clock thresholds. The fake provider blocks on
+ `release` after its first record, and only this test sets `release` - and
+ only once a *decodable* record has actually been sent. Buffered, no record
+ is ever sent, `release` is never set, the provider never returns, and
+ wait_for trips.
+
+ "Decodable" rather than "non-empty" for a reason the negative control
+ caught: gzip emits its 10-byte header on the first write, so a non-empty
+ check passes even when the payload is fully buffered.
+
+ Negative control: drop headers={"Content-Encoding": "identity"} from the
+ StreamingResponse in app/api/search.py and this fails with the buffered
+ message below.
+ """
+ from app.main import app
+
+ release = asyncio.Event()
+ first_body = asyncio.Event()
+
+ async def _blocks_until_its_first_record_is_sent(*args, **kwargs):
+ yield {"type": "content", "text": "first"}
+ await release.wait()
+ yield {"type": "content", "text": "second"}
+
+ monkeypatch.setattr("app.search.retrieval.stream_rag", _blocks_until_its_first_record_is_sent)
+
+ payload = json.dumps({"question": "does this stream?"}).encode()
+ token = os.environ.get("X_LOCAL_ACCESS_TOKEN", "test_token")
+ scope = {
+ "type": "http",
+ "asgi": {"version": "3.0", "spec_version": "2.1"},
+ "http_version": "1.1",
+ "method": "POST",
+ "scheme": "http",
+ "path": "/api/query/stream",
+ "raw_path": b"/api/query/stream",
+ "query_string": b"",
+ "root_path": "",
+ "headers": [
+ (b"host", b"test"),
+ (b"content-type", b"application/json"),
+ (b"content-length", str(len(payload)).encode()),
+ (b"x-local-access-token", token.encode()),
+ # What every browser sends and cannot be told not to send.
+ (b"accept-encoding", b"gzip, deflate"),
+ ],
+ "client": ("127.0.0.1", 1234),
+ "server": ("testserver", 80),
+ "state": {},
+ }
+
+ sent: list[dict] = []
+ body_delivered = False
+ still_connected = asyncio.Event()
+
+ async def receive():
+ # The body once, then block, which is what a live connection does.
+ # Returning http.request twice trips BaseHTTPMiddleware's wrapped_receive,
+ # and returning http.disconnect makes the endpoint abandon generation.
+ nonlocal body_delivered
+ if not body_delivered:
+ body_delivered = True
+ return {"type": "http.request", "body": payload, "more_body": False}
+ await still_connected.wait()
+ return {"type": "http.disconnect"}
+
+ streamed = bytearray()
+
+ async def send(message):
+ sent.append(message)
+ if message["type"] != "http.response.body":
+ return
+ streamed.extend(message.get("body", b""))
+ # A non-empty body message is not enough: gzip emits its 10-byte header
+ # on the first write, so the naive check passes even when fully buffered.
+ # The bar is a record the client could actually decode.
+ head, sep, _ = bytes(streamed).partition(b"\n")
+ if not sep:
+ return
+ try:
+ json.loads(head)
+ except (UnicodeDecodeError, ValueError):
+ return
+ first_body.set()
+
+ task = asyncio.ensure_future(app(scope, receive, send))
+ try:
+ try:
+ await asyncio.wait_for(first_body.wait(), timeout=10)
+ except TimeoutError: # pragma: no cover - only on regression
+ pytest.fail(
+ "No response body was sent while the provider was still "
+ "producing: the NDJSON stream is buffered end-to-end."
+ )
+ finally:
+ release.set()
+ await asyncio.wait_for(task, timeout=10)
+ finally:
+ release.set()
+ still_connected.set()
+ task.cancel()
+
+ start = next(m for m in sent if m["type"] == "http.response.start")
+ headers = {k.decode().lower(): v.decode() for k, v in start["headers"]}
+ assert headers.get("content-encoding") == "identity"
+
+ body = b"".join(m.get("body", b"") for m in sent if m["type"] == "http.response.body")
+ records = [json.loads(line) for line in body.decode().splitlines() if line.strip()]
+ texts = [r.get("text") for r in records if r.get("type") == "content"]
+ assert texts == ["first", "second"], records