Updates

app/api/search.py

@@ -257,7 +257,22 @@ async def stream_results():
                 with contextlib.suppress(Exception):
                     await agen.aclose()
 
-    return StreamingResponse(stream_results(), media_type="application/x-ndjson")
+    # Content-Encoding opts this stream out of GZipMiddleware (app/main.py), and
+    # it has to: Starlette exempts only text/event-stream by content type
+    # (starlette.middleware.gzip.DEFAULT_EXCLUDED_CONTENT_TYPES), and its
+    # GZipResponder writes each chunk into a GzipFile without flushing, so
+    # deflate holds every token until the generator closes. Measured on this
+    # stack: 30 records yielded 50ms apart arrived with mean lag 0.795s / max
+    # 1.593s, i.e. all at once at the end - which silently defeats the keepalive
+    # above and the 50ms flush throttle in useChatStream.ts. Browsers cannot opt
+    # out; Accept-Encoding is a forbidden header name. IdentityResponder passes
+    # the body through untouched once content-encoding is already set.
+    # This is per-endpoint: a new NDJSON stream elsewhere needs the same header.
+    return StreamingResponse(
+        stream_results(),
+        media_type="application/x-ndjson",
+        headers={"Content-Encoding": "identity"},
+    )
 
 
 @router.get("/history")

app/storage/db.py

@@ -520,6 +520,45 @@ async def _migrate(self, conn: "aiosqlite.Connection") -> None:
         except Exception as exc:
             logger.warning("kg_nodes schema migration failed: %s", exc)
 
+        # Back-fill any node whose provenance lives only in the properties JSON.
+        # The rebuild above back-fills the column, but only when the column was
+        # missing entirely; a database that already had it keeps whatever was
+        # written, and NULL there is now the difference between a node being
+        # reachable as a graph seed and being invisible. Verified divergent: with
+        # one such row, the old json_extract predicate returned it and the column
+        # predicate did not. Orphans are left NULL deliberately - chunk_id is a
+        # FOREIGN KEY onto chunks(id) and foreign_keys is ON, so pointing it at a
+        # deleted chunk would fail the whole migration.
+        try:
+            await conn.execute("""
+                UPDATE kg_nodes
+                   SET chunk_id = CAST(json_extract(properties, '$.chunk_id') AS INTEGER)
+                 WHERE chunk_id IS NULL
+                   AND json_extract(properties, '$.chunk_id') IS NOT NULL
+                   AND EXISTS (
+                       SELECT 1 FROM chunks c
+                        WHERE c.id = CAST(json_extract(kg_nodes.properties, '$.chunk_id') AS INTEGER)
+                   )
+            """)
+            await conn.commit()
+        except Exception as exc:
+            logger.warning("kg_nodes chunk_id back-fill failed: %s", exc)
+
+        # The seed step of every graph traversal looks nodes up by chunk_id, and
+        # kg_nodes carried no index at all beyond its id PRIMARY KEY. Measured on
+        # 64,752 nodes: the shipped json_extract predicate scanned the table and
+        # parsed JSON per row at 22.54 ms; the column with this index is 0.06 ms.
+        # Created here rather than beside the CREATE TABLE above because the
+        # migration block just above may rename and rebuild kg_nodes, which would
+        # take the index with it.
+        try:
+            await conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_kg_nodes_chunk_id ON kg_nodes(chunk_id)"
+            )
+            await conn.commit()
+        except Exception as exc:
+            logger.warning("Failed to create idx_kg_nodes_chunk_id: %s", exc)
+
         # GraphRAG edges
         try:
             await conn.execute("""
@@ -603,6 +642,20 @@ async def _migrate(self, conn: "aiosqlite.Connection") -> None:
         except Exception as exc:
             logger.warning("Failed to drop idx_chunks_covering: %s", exc)
 
+        # The same defect, reintroduced under a second name: `id` is the rowid,
+        # so indexing (id, text_preview) kept a second full copy of the
+        # compressed corpus. Measured +90.2% on disk and +57.8% insert time for
+        # no read benefit (see the note in schema.sql). Dropped here as well as
+        # removed from schema.sql, because executescript(schema.sql) above does
+        # not reconcile away what it no longer declares - an existing database
+        # would keep the index forever. Space returns via the incremental_vacuum
+        # already running at startup.
+        try:
+            await conn.execute("DROP INDEX IF EXISTS idx_chunks_text_lookup")
+            await conn.commit()
+        except Exception as exc:
+            logger.warning("Failed to drop idx_chunks_text_lookup: %s", exc)
+
         # Phase 9.2: Rebuild chunk_fts with detail=column to save ~40% space
         try:
             cur = await conn.execute(
@@ -1155,7 +1208,7 @@ def _bfs_cte(placeholders: str) -> str:
         bfs_nodes(id, depth) AS (
             SELECT id, 0
             FROM kg_nodes
-            WHERE json_extract(properties, '$.chunk_id') IN ({placeholders})
+            WHERE chunk_id IN ({placeholders})
 
             UNION
 
@@ -1183,10 +1236,10 @@ async def bfs_from_chunks(
         placeholders = ",".join("?" for _ in chunk_ids)
         # Only bound placeholders are interpolated; every value is parameterized.
         projection = """
-        SELECT DISTINCT CAST(json_extract(n.properties, '$.chunk_id') AS INTEGER) as chunk_id
+        SELECT DISTINCT n.chunk_id as chunk_id
         FROM bfs_nodes b
         JOIN kg_nodes n ON b.id = n.id
-        WHERE json_extract(n.properties, '$.chunk_id') IS NOT NULL
+        WHERE n.chunk_id IS NOT NULL
         LIMIT ?
         """
         query = self._bfs_cte(placeholders) + projection  # nosec B608
@@ -1218,7 +1271,7 @@ async def get_relational_paths(
         paths(id, path_str, depth, visited) AS (
             SELECT id, label || ' ' || id, 0, ',' || id || ','
             FROM kg_nodes
-            WHERE json_extract(properties, '$.chunk_id') IN ({placeholders})
+            WHERE chunk_id IN ({placeholders})
 
             UNION ALL
 

app/storage/schema.sql

@@ -40,7 +40,16 @@ CREATE TABLE IF NOT EXISTS chunks (
 
 -- FK index for efficient joins / cascading deletes
 CREATE INDEX IF NOT EXISTS idx_chunks_file_id ON chunks(file_id);
-CREATE INDEX IF NOT EXISTS idx_chunks_text_lookup ON chunks(id, text_preview);
+
+-- NOTE: idx_chunks_text_lookup ON chunks(id, text_preview) was dropped for the
+-- same reason idx_chunks_covering was (see below), and is dropped from existing
+-- databases in db.py _migrate. `id` is the rowid, so indexing (id, text_preview)
+-- stored a second full copy of the compressed corpus. Measured on 21,584 chunks
+-- of 512-char zlib bodies: +90.2% on disk (4792 -> 9112 KiB) and +57.8% insert
+-- time, for no read benefit at all. The planner did pick it, but only for scans
+-- that read the same bytes either way - the FTS-rebuild projection
+-- `SELECT id, zlib_decompress(text_preview) FROM chunks` measured 0.0815s with
+-- it against 0.0803s without. Do not reintroduce it under a third name.
 
 -- Chunk embeddings table for storing vector math as binary BLOBs
 -- This strictly isolates heavy binary data from normal metadata queries

frontend/src/__tests__/cacheKeys.test.ts

@@ -0,0 +1,64 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest'
+
+import { CACHE_KEYS, CORPUS_DERIVED_KEYS, launchStatusKey } from '../cacheKeys'
+
+vi.mock('../queryClient', () => ({
+  queryClient: {
+    invalidateQueries: vi.fn(),
+    clear: vi.fn(),
+  },
+}))
+
+import { queryClient } from '../queryClient'
+import { invalidateCache, invalidateCorpusCaches } from '../useApi'
+
+describe('cache keys', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('maps every endpoint to a distinct key', () => {
+    // The defect this file exists to prevent: three endpoints were each cached
+    // under two spellings ('file-tree'/'files-tree',
+    // 'llm-preferences'/'llm-prefs', 'ocr-status'/'ocr-status-settings'), so the
+    // same payload was fetched twice and invalidating one never reached the
+    // other. Two names for one endpoint is the bug; two *values* colliding here
+    // would be a different one, so assert the values are unique.
+    const values = Object.values(CACHE_KEYS)
+    expect(new Set(values).size).toBe(values.length)
+  })
+
+  it('builds a distinct launch-status key per provider', () => {
+    expect(launchStatusKey('ollama')).not.toBe(launchStatusKey('lm_studio'))
+  })
+
+  it('invalidates rather than evicting', () => {
+    // queryClient.clear() *removes* every query (query-core 5.94.5:
+    // getAll().forEach(q => this.remove(q))), so each mounted observer drops to
+    // data: undefined and every page falls back to its cold-start spinner.
+    // invalidateQueries refetches while keeping the previous data on screen.
+    invalidateCache(CACHE_KEYS.fileTree)
+
+    expect(queryClient.clear).not.toHaveBeenCalled()
+    expect(queryClient.invalidateQueries).toHaveBeenCalledWith({
+      queryKey: [CACHE_KEYS.fileTree],
+    })
+  })
+
+  it('refreshes corpus-derived views without touching configuration state', () => {
+    invalidateCorpusCaches()
+
+    const invalidated = vi
+      .mocked(queryClient.invalidateQueries)
+      .mock.calls.map((call) => (call[0] as { queryKey: string[] }).queryKey[0])
+
+    expect(new Set(invalidated)).toEqual(new Set(CORPUS_DERIVED_KEYS))
+    // An index run cannot change which providers are configured, so blowing
+    // those away to refresh the file tree is what the old bare
+    // invalidateCache() did wrong.
+    expect(invalidated).not.toContain(CACHE_KEYS.providersList)
+    expect(invalidated).not.toContain(CACHE_KEYS.providerSettings)
+    expect(invalidated).not.toContain(CACHE_KEYS.llmPreferences)
+    expect(queryClient.clear).not.toHaveBeenCalled()
+  })
+})

frontend/src/__tests__/components/ExplorerPage.test.tsx

@@ -1,5 +1,5 @@
 import { describe, it, expect, vi, beforeEach } from 'vitest';
-import { screen, fireEvent } from '@testing-library/react';
+import { screen, fireEvent, waitFor } from '@testing-library/react';
 import { ExplorerPage } from '../../pages/ExplorerPage';
 import { renderWithProviders } from '../test-utils';
 import { removeFolderIndex } from '../../api';
@@ -11,6 +11,9 @@ import { removeFolderIndex } from '../../api';
 // broken tree it produced was never caught here.
 const ROOT = 'C:/projects/test';
 
+// Mutable so a test can report a background refetch without re-mocking.
+const state = { loading: false };
+
 vi.mock('../../useApi', () => ({
   useApi: vi.fn((_, opts) => {
     if (opts?.cacheKey === 'file-tree') {
@@ -25,7 +28,7 @@ vi.mock('../../useApi', () => ({
           total_files: 2,
           total_size: 3072,
         },
-        loading: false,
+        loading: state.loading,
         error: null,
         refetch: vi.fn(),
       };
@@ -50,6 +53,7 @@ vi.mock('../../api', () => ({
 describe('ExplorerPage Component', () => {
   beforeEach(() => {
     vi.clearAllMocks();
+    state.loading = false;
     vi.spyOn(window, 'confirm').mockReturnValue(true);
     vi.spyOn(window, 'alert').mockImplementation(() => {});
   });
@@ -92,6 +96,43 @@ describe('ExplorerPage Component', () => {
     const del = screen.getAllByTitle('Delete this folder index')[0];
     fireEvent.click(del);
 
-    expect(removeFolderIndex).toHaveBeenCalledWith([ROOT]);
+    // Awaited now that the delete goes through useMutation: `mutate` schedules
+    // the mutationFn rather than entering it synchronously the way the old bare
+    // `await removeFolderIndex(...)` in the click handler did.
+    await waitFor(() => expect(removeFolderIndex).toHaveBeenCalledWith([ROOT]));
+  });
+
+  it('does not fire a second removal while the first is in flight', async () => {
+    // The delete button had no pending state at all, so a double-click sent two
+    // requests for the same folder.
+    let resolveDelete: (v: unknown) => void = () => {};
+    vi.mocked(removeFolderIndex).mockImplementationOnce(
+      () => new Promise((resolve) => { resolveDelete = resolve; }) as ReturnType<typeof removeFolderIndex>,
+    );
+
+    renderWithProviders(<ExplorerPage />);
+
+    const del = screen.getAllByTitle('Delete this folder index')[0];
+    fireEvent.click(del);
+    await waitFor(() => expect(removeFolderIndex).toHaveBeenCalledTimes(1));
+
+    fireEvent.click(del);
+    expect(removeFolderIndex).toHaveBeenCalledTimes(1);
+
+    resolveDelete({ message: 'ok', chunks_removed: 0 });
+  });
+  it('keeps the tree on screen while a background refetch is in flight', () => {
+    // useApi reports `isLoading || isFetching`, so this goes true on every
+    // background refetch - and with refetchOnWindowFocus enabled, alt-tabbing
+    // back replaced the entire explorer with a full-panel spinner. The guard is
+    // `loading && !tree`: spin only when there is nothing to show yet.
+    state.loading = true;
+
+    renderWithProviders(<ExplorerPage />);
+
+    expect(
+      screen.getAllByTitle('Delete this folder index').length,
+      'the rendered tree was replaced by a spinner during a refetch',
+    ).toBeGreaterThan(0);
   });
 });

frontend/src/__tests__/components/InsightsPage.test.tsx

@@ -76,12 +76,17 @@ vi.mock('../../api', () => ({
 }));
 
 describe('InsightsPage Component', () => {
-  it('renders Insights page title and cards', () => {
+  it('renders Insights page title and cards', async () => {
     renderWithProviders(<InsightsPage />);
 
     expect(screen.getByText('Insights')).toBeDefined();
     expect(screen.getByText('Total Files')).toBeDefined();
     expect(screen.getByText('Indexed Files Size')).toBeDefined();
     expect(screen.getByText('Database Size')).toBeDefined();
+
+    // KnowledgePortrait's getPortrait() promise (mocked to resolve with no
+    // themes) settles after the assertions above return, so its setState
+    // lands outside act() unless we wait for the resulting empty-state text.
+    expect(await screen.findByText('No Portrait Available')).toBeDefined();
   });
 });

frontend/src/__tests__/components/SearchPage.test.tsx

@@ -10,7 +10,7 @@ vi.mock('../../useApi', () => ({
     if (opts?.cacheKey === 'query-history') {
       return { data: { history: [] }, loading: false, error: null, refetch: vi.fn() };
     }
-    if (opts?.cacheKey === 'files-tree') {
+    if (opts?.cacheKey === 'file-tree') {
       return { data: { folders: {}, total_files: 0, total_size: 0 }, loading: false, error: null, refetch: vi.fn() };
     }
     if (opts?.cacheKey === 'app-config') {