diff --git a/.env.example b/.env.example index 9fbeea6..6485eb2 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,16 @@ ADMIN_PASSWORD=admin123 DB_PATH=./data/customer-service.db DOCUMENT_UPLOAD_DIR=./data/uploads +# Optional vector backend. Memory remains the fresh-clone default. When Qdrant +# is selected, restart the app after changing these deployment values. +VECTOR_STORE_PROVIDER=memory +QDRANT_URL= +QDRANT_API_KEY= +QDRANT_COLLECTION_PREFIX=resolveweave_knowledge +QDRANT_COLLECTION_ALIAS=resolveweave_knowledge_active +QDRANT_TIMEOUT_MS=5000 +RETRIEVAL_TRACE_RETENTION_DAYS=30 + # Optional local PaddleOCR/PP-StructureV3 worker. Existing FAQ/text document # features keep working when this is empty. OCR_SERVICE_URL= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b7b258..e33e840 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,3 +56,33 @@ jobs: run: EMBED_PROVIDER=other npm run build env: JWT_SECRET: test-secret-123 + + qdrant-integration: + runs-on: ubuntu-latest + services: + qdrant: + image: qdrant/qdrant:v1.18.2 + ports: + - 6333:6333 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Wait for Qdrant + run: curl --fail --retry 20 --retry-delay 1 --retry-connrefused http://127.0.0.1:6333/readyz + + - name: Run Qdrant integration + run: npm run test:qdrant + env: + QDRANT_URL: http://127.0.0.1:6333 + JWT_SECRET: test-secret-123 diff --git a/README.md b/README.md index 22a6f0e..9a2694d 100644 --- a/README.md +++ b/README.md @@ -15,19 +15,19 @@ **Chinese version**: [README_CN.md](README_CN.md) -Development version: **v0.3.1 (pre-1.0)**. The latest published release is -v0.3.1; APIs and persisted data remain subject to change before 1.0. +Development version: **v0.3.2 (pre-1.0)**. The latest published release is +v0.3.2; APIs and persisted data remain subject to change before 1.0.

- - ResolveWeave v0.3.1 reviewed OCR knowledge and grounded answer demo + + ResolveWeave v0.3.2 Qdrant index activation, retrieval trace, and rollback demo

- Watch the reviewed OCR knowledge demo (v0.3.1) - · v0.3.1 release notes - · v0.3.1 release evidence + Watch the retrieval operations demo (v0.3.2) + · v0.3.2 release notes + · v0.3.2 release evidence

## English @@ -38,10 +38,10 @@ hand risky cases to people with useful context. The current release combines customer chat, FAQ and document knowledge, hybrid retrieval, persisted sources, deterministic Grounding decisions, -structured escalation, an operations console, and repeatable quality -evaluation. It starts without a paid model key and is evolving toward a -bounded Agentic Retrieval architecture without giving the model authority over -answer release or business actions. +structured escalation, optional Qdrant, retrieval traces, an operations +console, and repeatable quality evaluation. It starts without a paid model key +or Qdrant and is evolving toward bounded Agentic Retrieval without giving the +model authority over answer release or business actions. [Quick Start](#quick-start) · [Why This Project](#why-this-project) · [Features](#features) · [Architecture](ARCHITECTURE.md) · [Evaluation](#evaluation-and-debugging) · [Roadmap](ROADMAP.md) @@ -65,7 +65,10 @@ ResolveWeave: Step 4: answer with persisted sources, or escalate high-risk/conflicting requests ``` -Admins can maintain FAQs, upload and manage documents, preview indexed chunks, inspect retrieval behavior, review conversations, and turn weak answers into reusable FAQs from the Knowledge Review page. +Admins can maintain FAQs, upload and manage documents, preview indexed chunks, +compare memory/Qdrant quality, build and activate Qdrant indexes, inspect +retrieval traces, review conversations, and turn weak answers into reusable +FAQs from the Knowledge Review page. The current release is suitable for learning, evaluation, demonstrations and small pre-production pilots. It deliberately keeps SQLite and an in-memory @@ -74,9 +77,9 @@ still required for serious production deployment. ### Product evidence -| Paddle / DeepSeek comparison | Reviewed Block editor | Grounded answer provenance | +| Quality backend comparison | Qdrant activation gate | Retrieval trace | | --- | --- | --- | -| ![Paddle authoritative and DeepSeek shadow comparison](docs/releases/assets/v0.3.1-ocr-comparison.jpg) | ![OCR Block review workspace](docs/releases/assets/v0.3.1-block-review.jpg) | ![Customer answer with OCR source provenance](docs/releases/assets/v0.3.1-chat-provenance.jpg) | +| ![Quality Lab memory and Qdrant targets](docs/releases/assets/v0.3.2-quality-backends.png) | ![Qdrant activation gate with latency acknowledgement](docs/releases/assets/v0.3.2-activation-gate.png) | ![Eight-stage retrieval trace](docs/releases/assets/v0.3.2-retrieval-trace-desktop.png) | Earlier engineering case study: [building the v0.2.6 Document RAG foundation with AI-assisted development](docs/case-studies/ai-assisted-development-v0.2.6.md). @@ -106,9 +109,9 @@ into one accountable customer-resolution flow. bounded Agentic Retrieval are planned as separately testable releases rather than one framework rewrite. -| Implemented in v0.3.1 | Next — v0.3.2+ | +| Implemented in v0.3.2 | Next — v0.3.3+ | | --- | --- | -| Versioned structure-aware ingestion, durable PaddleOCR review workflow, optional DeepSeek shadow comparison, and the v0.2.9 FAQ/RAG baseline | Optional Qdrant, retrieval traces, bounded Agentic Retrieval, then mock-first business tools | +| Optional Qdrant, recoverable index jobs, Quality Lab backend comparison, atomic alias activation/rollback, Retrieval Trace, plus the v0.3.1 reviewed OCR path | Bounded Agentic Retrieval, enterprise knowledge operations, then mock-first business tools | See [ROADMAP.md](ROADMAP.md) for release boundaries and non-goals. @@ -134,12 +137,13 @@ flowchart LR - **Reviewed OCR ingestion** - route PNG, JPEG, WebP, and scan-only PDF sources to a durable PaddleOCR PP-StructureV3 queue; inspect and edit extracted Blocks before atomic publication, with optional non-authoritative DeepSeek-OCR-2 shadow comparison. - **Hybrid multi-source retrieval** - FAQ and document candidates use per-source vector recall plus field-aware keyword recall, then merge with score-aware reciprocal-rank fusion (RRF), deduplicate, and apply source-aware diversity. - **Compatible intent classification** - structured intent output negotiates `json_schema`, then `json_object`, then validated plain-text JSON before the deterministic keyword fallback. -- **Open vector-store interface** - `VectorStore` keeps the default deployment simple while leaving room for Qdrant or pgvector later. +- **Optional Qdrant backend** - the asynchronous `VectorStore` keeps memory as the default and adds Qdrant with stable IDs, safe metadata, health/stats, SQLite hydration, and explicit keyword degradation. - **Richer FAQ embeddings** - FAQ vectors are generated from question, answer, and keywords, not only the question. -- **Index operations** - admin users can inspect indexed entries, active entries, missing embeddings, dimensions, rebuild time, and index errors. +- **Recoverable index operations** - build checkpointed versioned collections from SQLite, validate fingerprint/profile/dimension/count, activate through an atomic alias switch, and roll back without deleting old collections. - **Retrieval debugging** - admin panel explains ranked matches, source, similarity, keyword score, vector score, and ranking reason. - **Retrieval evaluation** - repeatable FAQ and document evals report ranking metrics, score/source distributions, failures, and semantic-v1 versus structure-only comparison. -- **RAG Quality Lab** - admins version evaluation sets, compare deterministic retrieval/Grounding strategies, inspect failures and safely publish or roll back an immutable runtime policy. +- **RAG Quality Lab** - admins version evaluation sets, compare deterministic retrieval/Grounding strategies across memory and a ready Qdrant job, inspect failures, and safely publish or roll back an immutable runtime policy. +- **Retrieval operations and traces** - a bilingual responsive admin page shows backend health, index jobs, activation gates, and fixed eight-stage traces with safe metadata and bounded candidate lists. - **Structured escalation and triage** - every new handoff persists a traceable packet with deterministic priority, risk flags, recommended queue, cited facts, missing information, and retrieval evidence; admins review it in a bilingual read-only queue. - **Language switching and bilingual dictionary** - fixed UI copy is read from an editable Chinese/English dictionary instead of being hard-coded across pages. - **Light/dark themes** - persisted theme preferences for both customer and admin workflows. @@ -165,7 +169,21 @@ Query +-- return matches with similarity-compatible fields ``` -The default generic `VectorStore` implementation is in-memory. FAQ and document-chunk embeddings are serialized in SQLite, then loaded into the shared process index under `faq:` and `document:` namespaces. Each stored vector carries an embedding profile derived from provider, model, endpoint, and input-schema version; stale profiles are rebuilt atomically before the process index is replaced. This keeps local setup dependency-free while preventing vectors from different model configurations from being silently mixed. +The asynchronous generic `VectorStore` defaults to memory. +FAQ and document-chunk embeddings are serialized in SQLite, then loaded into +the shared process index under `faq:` and `document:` namespaces. +Each stored vector carries an embedding profile derived from provider, model, +endpoint, and input-schema version; stale profiles are rebuilt atomically +before the process index is replaced. + +When `VECTOR_STORE_PROVIDER=qdrant` is explicitly configured, the application +uses the collection alias from deployment configuration. Qdrant payloads keep +only knowledge identity/version/profile metadata; every vector candidate is +batch-hydrated from SQLite and rejected if the current knowledge is missing, +disabled, stale, or attached to an inactive source. A Qdrant timeout records a +degraded trace and continues keyword/structured recall. It does not silently +rebuild memory vectors. This keeps SQLite authoritative and the fresh-clone +path dependency-free. FAQ remains a knowledge-source adapter rather than the permanent RAG boundary. TXT, Markdown, text-layer PDF, and DOCX now pass through the versioned @@ -235,6 +253,18 @@ Docker exposes: The compose example uses `EMBED_PROVIDER=other`, so the project can start without paid model keys. The deterministic local path supports FAQ and document retrieval; document answers fall back to the highest-ranked source excerpt instead of inventing a summary. +Start the optional pinned Qdrant backend and select it at deployment time: + +```bash +VECTOR_STORE_PROVIDER=qdrant \ +QDRANT_URL=http://qdrant:6333 \ +docker compose --profile qdrant up --build +``` + +The provider is a deployment setting and requires an application restart. +Retrieval Operations can atomically change the configured collection alias; +it cannot edit the provider, URL, or API key. + Start the optional CPU OCR worker with the Compose profile: ```bash @@ -268,6 +298,11 @@ Copy `.env.example` to `.env`, then configure the values you need: | `LLM_PROVIDER` / `EMBED_PROVIDER` | `openai`, `openai-compatible`, or `other` | | `LLM_API_BASE` / `LLM_API_KEY` / `LLM_MODEL` | Chat model endpoint, environment-only credential, and model | | `EMBED_API_BASE` / `EMBED_API_KEY` / `EMBED_MODEL` | OpenAI-compatible embedding model | +| `VECTOR_STORE_PROVIDER` | `memory` (default) or explicitly configured `qdrant`; changing it requires restart | +| `QDRANT_URL` / `QDRANT_API_KEY` | Qdrant REST endpoint and optional environment-only credential | +| `QDRANT_COLLECTION_PREFIX` / `QDRANT_COLLECTION_ALIAS` | Versioned collection prefix and the alias used by the application | +| `QDRANT_TIMEOUT_MS` | Bounded Qdrant request timeout; defaults to `5000` ms | +| `RETRIEVAL_TRACE_RETENTION_DAYS` | Trace retention in days; defaults to `30`, accepted range `1`–`90` | | `DOCUMENT_UPLOAD_DIR` | Private document file directory; defaults to `./data/uploads` | | `OCR_SERVICE_URL` | Optional PaddleOCR/PP-StructureV3 worker base URL; when empty, existing FAQ and text-document features still work | | `OCR_SERVICE_TOKEN` | Optional bearer token sent only to the configured OCR worker | @@ -297,6 +332,12 @@ npm run eval:triage The reports include FAQ Top1/Top3/no-match metrics, a 12-case document benchmark across TXT, Markdown, PDF, and DOCX, a six-case OCR contract benchmark covering screenshots, scan PDFs, tables, rotation/noise and low-quality gating, and deterministic triage coverage. The document report compares `semantic-v1` with a structure-only baseline and requires 100% Top3 recall without MRR regression. +To exercise a real Qdrant instance separately from the default suite: + +```bash +QDRANT_URL=http://localhost:6333 npm run test:qdrant +``` + Document management is available at **Admin Console → Documents**. The detail dialog exposes quality/index status, structure metrics, warnings, a paginated Block inspector, the eight processing stages, and published chunks. Uploads are @@ -343,7 +384,9 @@ PLAYWRIGHT_CHANNEL=chromium npm run test:e2e EMBED_PROVIDER=other npm run build ``` -GitHub Actions runs `npm ci`, regression tests, Playwright E2E, and production build checks on pull requests and pushes to `main`. +GitHub Actions runs `npm ci`, regression tests, Playwright E2E, production +build checks, and an independent integration job against +`qdrant/qdrant:v1.18.2` on pull requests and pushes to `main`. --- @@ -363,8 +406,12 @@ data/ Local SQLite database files ## Current Limits -- The default vector index is process-local memory and scans FAQ plus document-chunk embeddings, so it is suitable for demos and small knowledge collections. -- Embeddings are stored as JSON in SQLite, not in a dedicated vector database. +- The default vector index remains process-local memory and scans FAQ plus + document-chunk embeddings, so it is suitable for demos and small knowledge + collections. Qdrant is optional and must be selected explicitly. +- SQLite keeps embedding vectors and remains the knowledge system of record. + Qdrant is a derived index; candidates are never trusted without SQLite + hydration. - Text-document parsing remains synchronous inside the Express process. Encrypted and damaged files are rejected. PNG, JPEG, WebP, and scan-only PDF sources use an optional external PaddleOCR worker through a durable SQLite @@ -375,9 +422,17 @@ data/ Local SQLite database files first start and should remain on a trusted private network. - OCR extracts text and table structure only. VLM descriptions, raw-image answering, web ingestion, citation links, and page jumps are not included. -- Document files remain global to the deployment; v0.3.1 does not add - tenant-separated knowledge bases or external vector storage. -- `VectorStore` isolates local vector operations, but a network vector database still requires asynchronous contracts, health handling, and consistency tests. +- Document files and Qdrant collections remain global to the deployment; + v0.3.2 does not add tenant-separated knowledge bases. +- The backend provider cannot be changed at runtime. Qdrant failure keeps + keyword/structured retrieval but does not automatically fail over the + configured provider or rebuild memory vectors. +- Old Qdrant collections are retained for rollback. Automatic cleanup, + snapshots, clustering, sparse/hybrid retrieval, and distributed index-job + leases are not included. +- Retrieval traces are stored in SQLite and intentionally omit copied customer + questions, candidate content, credentials, and raw Qdrant responses. This is + not an OpenTelemetry platform. - Conflict detection is deliberately narrow: duplicate normalized direct-FAQ questions with different answers. Grounding thresholds are governed through the versioned Quality Lab rather than changed automatically. - Intent classification falls back to keyword rules when the LLM call fails. - Idempotency replay is scoped to one deployment and retained for 24 hours; @@ -388,7 +443,10 @@ data/ Local SQLite database files - Optional LLM extraction has a two-second total budget and may improve only summaries, cited facts, and missing-information candidates. Deterministic priority, risk, queue, and next-step rules remain authoritative. -- This is a pre-1.0 MVP foundation, not a production support platform. Add observability, stricter auth, backup strategy, and external vector storage before serious production use. +- This is a pre-1.0 MVP foundation, not a complete production support + platform. Add stricter identity/RBAC, backup/disaster recovery, + multi-replica coordination, and infrastructure monitoring before serious + production use. --- @@ -396,8 +454,7 @@ data/ Local SQLite database files The ordered version plan lives in [ROADMAP.md](ROADMAP.md). The next milestones are: -- v0.3.1–v0.3.3: OCR/table/image knowledge, optional Qdrant with retrieval - traces, then bounded Agentic Retrieval behind a deterministic Grounding Gate. +- v0.3.3: bounded Agentic Retrieval behind a deterministic Grounding Gate. - v0.3.4–v0.3.8: enterprise knowledge operations, mock-first read-only order tools, human collaboration, customer identity/memory and guarded actions. - v0.4.0: multi-knowledge-base and tenant boundaries, RBAC, audit, migration, diff --git a/README_CN.md b/README_CN.md index c523074..2c2a358 100644 --- a/README_CN.md +++ b/README_CN.md @@ -14,19 +14,19 @@ **English version**: [README.md](README.md) -开发版本：**v0.3.1（pre-1.0）**。最新公开发布版为 v0.3.1；在 1.0 +开发版本：**v0.3.2（pre-1.0）**。最新公开发布版为 v0.3.2；在 1.0 之前，API 和持久化数据结构仍可能调整。

- - ResolveWeave v0.3.1 OCR 知识复核与可信回答演示 + + ResolveWeave v0.3.2 Qdrant 索引激活、检索 Trace 与回滚演示

- 观看 OCR 知识复核演示（v0.3.1） - · v0.3.1 版本说明 - · v0.3.1 版本验证证据 + 观看检索运维演示（v0.3.2） + · v0.3.2 版本说明 + · v0.3.2 版本验证证据

ResolveWeave 是一个 pre-1.0 的企业级智能客服平台。它关注的 @@ -34,9 +34,9 @@ ResolveWeave 是一个 pre-1.0 的企业级智能客服平台。它关注的 怎样携带有效上下文交给人工。 当前版本已经把用户聊天、FAQ 与文档知识、混合检索、来源持久化、确定性 -Grounding 决策、结构化转人工、运营后台和可重复质量评测放在同一工程内。 -没有付费模型 Key 也可以启动基础路径；后续将演进到受限 Agentic Retrieval， -但不会把答案放行或业务操作权限交给模型。 +Grounding 决策、结构化转人工、可选 Qdrant、检索 Trace、运营后台和可重复 +质量评测放在同一工程内。没有付费模型 Key 或 Qdrant 也可以启动基础路径； +后续将演进到受限 Agentic Retrieval，但不会把答案放行或业务操作权限交给模型。 [快速开始](#快速开始) · [为什么做这个项目](#为什么做这个项目) · [特性](#特性) · [架构](ARCHITECTURE.md) · [评测与调试](#评测与调试) · [路线图](ROADMAP.md) @@ -60,7 +60,9 @@ ResolveWeave: Step 4: 返回并保存来源，或把高风险/冲突请求转人工 ``` -管理员可以维护 FAQ，上传和管理文档，预览已索引切片，查看检索行为与会话记录，并在“知识审核”页面把答不好的问题沉淀成可复用 FAQ。 +管理员可以维护 FAQ，上传和管理文档，预览已索引切片，对比 memory/Qdrant +质量，构建与激活 Qdrant 索引，查看检索 Trace 和会话记录，并在“知识审核” +页面把答不好的问题沉淀成可复用 FAQ。 当前版本适合学习、评测、演示和小规模预生产试用。项目刻意保留 SQLite + 内存向量索引作为零基础设施路径，同时明确列出正式生产仍需补齐的 @@ -68,9 +70,9 @@ SQLite + 内存向量索引作为零基础设施路径，同时明确列出正 ### 产品证据 -| Paddle / DeepSeek 对照 | Block 人工复核 | 可信回答来源 | +| Quality 后端对比 | Qdrant 激活门禁 | 检索 Trace | | --- | --- | --- | -| ![Paddle 权威结果与 DeepSeek 影子结果对照](docs/releases/assets/v0.3.1-ocr-comparison.jpg) | ![OCR Block 人工复核工作区](docs/releases/assets/v0.3.1-block-review.jpg) | ![带 OCR 来源证据的客户回答](docs/releases/assets/v0.3.1-chat-provenance.jpg) | +| ![Quality Lab memory 与 Qdrant 目标](docs/releases/assets/v0.3.2-quality-backends.png) | ![带延迟确认的 Qdrant 激活门禁](docs/releases/assets/v0.3.2-activation-gate.png) | ![固定八阶段检索 Trace](docs/releases/assets/v0.3.2-retrieval-trace-desktop.png) | 早期工程复盘： [用 AI 辅助开发构建 v0.2.6 文档 RAG 基础](docs/case-studies/ai-assisted-development-v0.2.6.md)。 @@ -97,9 +99,9 @@ SQLite + 内存向量索引作为零基础设施路径，同时明确列出正 - **企业方向按版本验证**——结构化入库、OCR、Qdrant 和受限 Agentic Retrieval 分开交付，不进行一次性框架重写。 -| v0.3.1 已实现 | 下一阶段 — v0.3.2+ | +| v0.3.2 已实现 | 下一阶段 — v0.3.3+ | | --- | --- | -| 版本化结构入库、持久化 PaddleOCR 复核流程、可选 DeepSeek 影子对照，以及 v0.2.9 的 FAQ/RAG 基线 | 可选 Qdrant、检索 Trace、受限 Agentic Retrieval，之后再接 mock 业务工具 | +| 可选 Qdrant、可恢复索引任务、Quality Lab 后端对比、alias 原子激活/回滚、检索 Trace，以及 v0.3.1 的 OCR 复核路径 | 受限 Agentic Retrieval、企业知识运营，之后再接 mock 业务工具 | 完整版本边界和非目标见 [ROADMAP.md](ROADMAP.md)。 @@ -125,12 +127,13 @@ flowchart LR - **需复核的 OCR 入库** - PNG、JPEG、WebP 和扫描 PDF 进入持久化 PaddleOCR PP-StructureV3 队列；管理员检查、编辑 Block 后原子发布，并可启用不具发布权的 DeepSeek-OCR-2 影子对照。 - **多知识源混合检索** - FAQ 与文档分别召回向量候选，再结合字段感知的关键词候选，由统一检索器通过分数感知的倒数排名融合（RRF）合并、去重并保持来源多样性。 - **兼容意图分类** - 结构化输出依次尝试 `json_schema`、`json_object` 和经过严格校验的普通文本 JSON，最后才降级到确定性关键词规则。 -- **向量库接口抽象** - `VectorStore` 让默认部署保持简单，也方便后续接入 Qdrant 或 pgvector。 +- **可选 Qdrant 后端** - 异步 `VectorStore` 默认使用内存，并增加稳定 ID、安全元数据、健康/统计、SQLite 回查和明确关键词降级的 Qdrant 实现。 - **更完整的 FAQ embedding** - embedding 文本由问题、回答和关键词共同组成，而不是只使用问题。 -- **索引状态管理** - 后台展示启用条目、已索引条目、缺失 embedding、向量维度、上次重建时间和索引错误。 +- **可恢复索引运维** - 从 SQLite 分批构建版本化 collection，校验指纹/profile/维度/数量，通过 alias 原子激活，并在保留旧 collection 的前提下回滚。 - **检索调试面板** - 后台可以查看命中条目、source、similarity、keywordScore、vectorScore 和排序原因。 - **检索评测能力** - FAQ 和文档固定评测集输出排序指标、分数/来源分布、失败样例，以及 semantic-v1 与仅结构切片的对比。 -- **RAG 质量实验室** - 管理员可维护版本化评测集、比较确定性检索与 Grounding 策略、下钻失败样例，并通过门禁发布或回滚不可变运行策略。 +- **RAG 质量实验室** - 管理员可维护版本化评测集，在 memory 与 ready Qdrant job 上比较同一检索/Grounding 策略、下钻失败样例，并通过门禁发布或回滚不可变运行策略。 +- **检索运维与 Trace** - 独立双语响应式后台展示后端健康、索引任务、激活门禁和固定八阶段 Trace，并限制候选数量和敏感内容。 - **结构化转人工与分流** - 每条新转人工记录都会保存可追溯交接包，包括确定性优先级、风险标记、建议队列、带消息引用的事实、缺失信息和检索证据；管理员可在独立的双语只读队列中查看。 - **中英文词典** - 固定 UI 文案从可编辑的中英文词典读取，减少硬编码散落在组件里。 - **暗/亮主题切换** - 用户端和后台都支持持久化主题偏好。 @@ -156,7 +159,17 @@ Query +-- 返回兼容 similarity 字段的匹配结果 ``` -默认泛型 `VectorStore` 是内存实现。FAQ 与文档切片 embedding 会序列化存入 SQLite，再以 `faq:` 和 `document:` 命名空间加载到共享进程索引。每条向量同时保存由 provider、模型、endpoint 和输入结构版本生成的 embedding profile；发现旧 profile 时先原子重建持久化向量，再替换进程索引，避免不同模型配置的向量被静默混用。 +异步泛型 `VectorStore` 默认使用内存实现。FAQ 与文档切片 +embedding 会序列化存入 SQLite，再以 `faq:` 和 +`document:` 命名空间加载到共享进程索引。每条向量同时保存由 +provider、模型、endpoint 和输入结构版本生成的 embedding profile；发现旧 +profile 时先原子重建持久化向量，再替换进程索引。 + +显式配置 `VECTOR_STORE_PROVIDER=qdrant` 后，应用使用部署配置指定的 collection +alias。Qdrant payload 只保存知识身份、版本和 profile；每个向量候选都要批量 +回查 SQLite，缺失、停用、旧版本或来源失效的候选直接丢弃。Qdrant 超时会记录 +degraded Trace，并继续关键词/结构化召回，不会静默重建内存向量。因此 SQLite +始终权威，fresh-clone 仍不依赖外部基础设施。 FAQ 仍然只是知识来源适配器，不是永久的 RAG 边界。TXT、Markdown、含文本层 PDF 与 DOCX 现在统一进入 @@ -222,6 +235,17 @@ Docker 默认暴露： Compose 示例使用 `EMBED_PROVIDER=other`，所以没有付费模型 Key 时也能启动。确定性本地路径支持 FAQ 与文档检索；文档回答会回退到最高分原文片段。 +通过部署配置选择可选、固定版本的 Qdrant 后端： + +```bash +VECTOR_STORE_PROVIDER=qdrant \ +QDRANT_URL=http://qdrant:6333 \ +docker compose --profile qdrant up --build +``` + +后端类型变更需要重启应用。“检索运维”可以原子切换配置好的 collection +alias，但不能修改 provider、URL 或 API Key。 + 通过 Compose profile 启动可选 CPU OCR Worker： ```bash @@ -253,6 +277,11 @@ RESOLVE_WEAVE_DATA_VOLUME=<原物理卷名称> docker compose up --build | `LLM_PROVIDER` / `EMBED_PROVIDER` | `openai`、`openai-compatible` 或 `other` | | `LLM_API_BASE` / `LLM_API_KEY` / `LLM_MODEL` | 对话模型地址、仅环境注入的凭据和模型名 | | `EMBED_API_BASE` / `EMBED_API_KEY` / `EMBED_MODEL` | OpenAI 兼容 embedding 模型 | +| `VECTOR_STORE_PROVIDER` | `memory`（默认）或显式配置的 `qdrant`；变更后需重启 | +| `QDRANT_URL` / `QDRANT_API_KEY` | Qdrant REST 地址和可选、仅环境注入的凭据 | +| `QDRANT_COLLECTION_PREFIX` / `QDRANT_COLLECTION_ALIAS` | 版本化 collection 前缀与应用使用的 alias | +| `QDRANT_TIMEOUT_MS` | Qdrant 请求超时，默认 `5000` 毫秒 | +| `RETRIEVAL_TRACE_RETENTION_DAYS` | Trace 保留天数，默认 `30`，范围 `1`–`90` | | `DOCUMENT_UPLOAD_DIR` | 私有文档文件目录，默认 `./data/uploads` | | `OCR_SERVICE_URL` | 可选 PaddleOCR/PP-StructureV3 Worker 根地址；留空时原有 FAQ 和文本文档能力仍可运行 | | `OCR_SERVICE_TOKEN` | 可选 Bearer Token，只发送给已配置的 OCR Worker | @@ -282,6 +311,12 @@ npm run eval:triage 评测包含 FAQ 的 Top1/Top3/无匹配指标、覆盖 TXT/Markdown/PDF/DOCX 的 12 条文档用例、覆盖截图/扫描 PDF/表格/旋转噪声/低质量门禁的 6 条 OCR 契约用例，以及确定性分流用例。文档评测会对比 `semantic-v1` 与仅结构切片基线，并要求 Top3 100%、MRR 不下降。 +需要独立验证真实 Qdrant 时运行： + +```bash +QDRANT_URL=http://localhost:6333 npm run test:qdrant +``` + 文档管理入口位于 **管理后台 → 文档知识**。详情 Dialog 会展示质量/索引状态、 结构指标、警告、分页 Block 检查、八个处理阶段和已发布切片。单文件上限 10 MB、提取文本上限 200,000 字符、`DocumentIR` 上限 2 MiB/2,000 个 Block、 @@ -326,7 +361,8 @@ PLAYWRIGHT_CHANNEL=chromium npm run test:e2e EMBED_PROVIDER=other npm run build ``` -GitHub Actions 会在 PR 和推送到 `main` 时运行 `npm ci`、回归测试、Playwright E2E 和生产构建检查。 +GitHub Actions 会在 PR 和推送到 `main` 时运行 `npm ci`、回归测试、 +Playwright E2E、生产构建，以及使用 `qdrant/qdrant:v1.18.2` 的独立集成任务。 --- @@ -346,8 +382,10 @@ data/ 本地 SQLite 数据库文件 ## 当前限制 -- 默认向量索引在进程内存中，全量遍历 FAQ 与文档切片 embedding，适合 Demo 和小规模知识库，不适合大规模检索。 -- embedding 以 JSON 形式存储在 SQLite 中，没有使用专门的向量数据库。 +- 默认向量索引仍在进程内存中，全量遍历 FAQ 与文档切片 embedding，适合 + Demo 和小规模知识库；Qdrant 是显式选择的可选后端。 +- SQLite 保存 embedding 并始终是知识权威源。Qdrant 只是派生索引，候选必须 + 回查 SQLite 后才能成为证据。 - 文本文档解析仍同步运行在 Express 进程内，加密和损坏文件会被拒绝。PNG、 JPEG、WebP 和扫描 PDF 通过可选 PaddleOCR Worker 进入 SQLite 持久化队列； 管理员发布完整复核草稿前不会建立索引。 @@ -355,15 +393,21 @@ data/ 本地 SQLite 数据库文件 首次启动会下载较大的模型，应部署在可信私有网络中。 - OCR 只提取文本与表格结构；VLM 图片描述、直接用原图回答、网页采集、引用 跳转和页码跳转仍未包含。 -- 文档仍属于单一全局知识库；v0.3.1 不包含多租户分库或外部向量存储。 -- `VectorStore` 隔离了本地向量操作，但接入网络向量数据库仍需异步契约、健康检查和一致性测试。 +- 文档和 Qdrant collection 仍属于单一全局知识库；v0.3.2 不包含多租户分库。 +- 后端 provider 不能在运行时切换。Qdrant 故障时继续关键词/结构化召回， + 但不会自动切换 provider 或重建内存向量。 +- 旧 Qdrant collection 为回滚而保留；自动清理、快照、集群、 + sparse/hybrid 检索和分布式索引任务租约尚未包含。 +- 检索 Trace 存入 SQLite，并刻意不复制客户问题、候选正文、凭据和原始 + Qdrant 响应；它不是 OpenTelemetry 平台。 - 冲突检测刻意限制为“归一化后问题相同、答案不同”的直达 FAQ；Grounding 阈值通过版本化质量实验室治理，不会自动切换。 - LLM 意图识别失败时会回退到关键词规则。 - 幂等响应仅在单个部署范围内保留 24 小时；multipart 上传依赖各自工作流的重复检查， 不使用通用响应重放。 - v0.2.9 的转人工分流只读，不包含人工认领、分配、备注、解决动作、实时接管或业务工具。 - 可选 LLM 提取共享 2 秒总预算，只能改进摘要、带引用事实和缺失信息候选；优先级、风险、队列和下一步始终由确定性规则控制。 -- 这是一个 pre-1.0 MVP 基座，不是完整生产客服平台。正式生产前应补充可观测性、更严格的鉴权、备份策略和外部向量存储。 +- 这是一个 pre-1.0 MVP 基座，不是完整生产客服平台。正式生产前还应补充 + 更严格的身份/RBAC、备份与灾难恢复、多副本协调和基础设施监控。 --- @@ -371,8 +415,7 @@ data/ 本地 SQLite 数据库文件 有顺序的版本计划见 [ROADMAP.md](ROADMAP.md)。下一阶段重点为： -- v0.3.2–v0.3.3：带检索 Trace 的可选 Qdrant，再实现由确定性 - Grounding Gate 约束的 Agentic Retrieval。 +- v0.3.3：实现由确定性 Grounding Gate 约束的受限 Agentic Retrieval。 - v0.3.4–v0.3.8：企业知识运营、mock 优先的订单只读工具、人工协作、 客户身份/记忆和受控写操作。 - v0.4.0：多知识库和租户边界、RBAC、审计、迁移、备份恢复与生产可观测性。 diff --git a/ROADMAP.md b/ROADMAP.md index c184696..28df230 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -15,10 +15,11 @@ This roadmap describes the product direction rather than fixed delivery dates. T | v0.2.8 | Released | RAG Quality Lab | 用版本化评测集比较检索与 Grounding 策略，并通过质量门禁安全发布和回滚。 | | v0.2.9 | Released | Structured Escalation & Triage | 以结构化交接包、确定性优先级和只读双语分流页承接转人工流程。 | | v0.3.0 | Released | Structure-Aware Ingestion Foundation | 统一结构表示、质量门禁、结构切片、处理时间线和显式影子重处理已公开发布。 | -| v0.3.1 | Current | Multimodal Knowledge Review | PNG/JPEG/WebP/扫描 PDF 经持久化 PaddleOCR 队列进入可编辑复核草稿；可选 DeepSeek 影子对照，发布后保留引擎、页码和 Block 来源。 | +| v0.3.1 | Released | Multimodal Knowledge Review | PNG/JPEG/WebP/扫描 PDF 经持久化 PaddleOCR 队列进入可编辑复核草稿；可选 DeepSeek 影子对照，发布后保留引擎、页码和 Block 来源。 | +| v0.3.2 | Current | Qdrant & Retrieval Observability | 可选 Qdrant、可恢复索引任务、Quality Lab 后端影子评测、alias 原子激活/回滚和八阶段检索 Trace 形成独立运维闭环。 | -v0.2.9 已经形成可运行且带回答边界、结构化人工交接的小规模客服产品基线：用户聊天、匿名会话历史、FAQ -与文档知识、混合检索、转人工记录、满意度、知识审核、会话分析、双语后台、 +v0.3.2 已经形成可运行且带回答边界、结构化人工交接和检索运维的小规模客服产品基线：用户聊天、匿名会话历史、FAQ +与文档知识、混合检索、可选 Qdrant、可恢复索引、检索 Trace、转人工记录、满意度、知识审核、会话分析、双语后台、 可信回答决策、来源持久化、接口幂等、防重复提交、Docker、检索评测和 Playwright 回归在同一工程内闭环。后续版本不再以增加 “另一个聊天 Demo”为目标，而是先补齐企业知识工程与 Agentic Retrieval，再扩展业务处理和人工协作。 @@ -26,7 +27,6 @@ v0.2.9 已经形成可运行且带回答边界、结构化人工交接的小规 | Version | Theme | Intended outcome | | --- | --- | --- | -| v0.3.2 | Qdrant & Retrieval Observability | 将 Qdrant 作为可选生产向量后端，保留本地回退，并提供迁移、混合检索、检索预算和全链路 Trace。 | | v0.3.3 | Bounded Agentic Retrieval | LLM 在预算内选择、组合和重试检索工具；确定性 Grounding Gate 决定引用、拒答、转人工和答案放行。 | | v0.3.4 | Enterprise Knowledge Operations | 增加可观测入库任务、文档版本、重建索引、失败恢复、白名单远程来源和定时刷新。 | | v0.3.5 | Read-Only Customer Service Tools | 以 mock 订单/物流查询验证类型化工具和外部订单系统接口，不执行业务写操作。 | @@ -90,12 +90,16 @@ v0.2.9 已经形成可运行且带回答边界、结构化人工交接的小规 ### v0.3.2 — Qdrant & Retrieval Observability -- Qdrant 作为 `VectorStore` 后的第一类生产后端；内存实现继续服务 - fresh-clone 和无基础设施演示。 -- 保留关键词/结构化检索，明确向量、关键词、融合、重排和最终上下文预算。 -- 为已有 FAQ/文档向量提供可恢复迁移、幂等重建、健康检查和回滚。 -- 持久化来源、各阶段得分、延迟、失败原因和最终证据集，并通过 Quality Lab - 做影子对比后再切换默认生产路径。 +- Qdrant 已作为异步 `VectorStore` 后的可选生产后端；内存实现继续服务 + fresh-clone 和无基础设施演示，SQLite 始终是知识权威源。 +- 保留关键词/结构化检索；Qdrant 超时或不可用会记录 `degraded` Trace， + 不静默重建内存向量。 +- 版本化 collection 通过固定批次、知识指纹和检查点支持中断恢复；就绪前 + 校验 profile、维度、点数和当前知识指纹。 +- Quality Lab 可用同一数据集和策略影子对比 memory/Qdrant；质量门禁通过后 + 原子切换 alias，并可回滚到上一已验证 collection。 +- 独立双语检索运维页展示健康、索引任务和固定八阶段 Trace；Trace 只保存 + 有界安全元数据，默认保留 30 天。 ### v0.3.3 — Bounded Agentic Retrieval diff --git a/client/src/App.tsx b/client/src/App.tsx index d122231..91a2baf 100644 --- a/client/src/App.tsx +++ b/client/src/App.tsx @@ -15,6 +15,7 @@ const ModelConfigPage = lazy(() => import('./pages/admin/ModelConfigPage')); const KnowledgeReviewPage = lazy(() => import('./pages/admin/KnowledgeReviewPage')); const DocumentManagementPage = lazy(() => import('./pages/admin/DocumentManagementPage')); const QualityLabPage = lazy(() => import('./pages/admin/QualityLabPage')); +const RetrievalOpsPage = lazy(() => import('./pages/admin/RetrievalOpsPage')); const EscalationTriagePage = lazy(() => import('./pages/admin/EscalationTriagePage')); const AuthGuard = lazy(() => import('./components/common/AuthGuard')); @@ -53,6 +54,7 @@ export function App(): React.ReactElement { } /> } /> } /> + } /> } /> diff --git a/client/src/api/admin.ts b/client/src/api/admin.ts index b6abb23..eb108e9 100644 --- a/client/src/api/admin.ts +++ b/client/src/api/admin.ts @@ -38,6 +38,13 @@ import type { QualityCase, QualityDatasetVersion, QualityRun, + QualityBackendTarget, + RetrievalActivationCheck, + RetrievalIndexJob, + RetrievalStatus, + RetrievalTrace, + RetrievalTraceDetail, + RetrievalTraceStatus, RetrievalPolicy, RetrievalPolicyEvent, RetrievalPolicyConfig, @@ -77,6 +84,12 @@ export type { QualityCase, QualityDatasetVersion, QualityRun, + RetrievalActivationCheck, + RetrievalIndexJob, + RetrievalStatus, + RetrievalTrace, + RetrievalTraceDetail, + RetrievalTraceStatus, RetrievalPolicy, RetrievalPolicyEvent, RetrievalPolicyConfig, @@ -448,6 +461,7 @@ export async function getQualityRun(runId: string): Promise { export async function createQualityRun(data: { datasetVersionIds: string[]; policies: RetrievalPolicyConfig[]; + backendTargets?: QualityBackendTarget[]; }): Promise { return post('/admin/quality/runs', data, idempotentRequest()); } @@ -487,3 +501,74 @@ export async function rollbackQualityPolicy(data: { }): Promise { return post('/admin/quality/policies/rollback', data, idempotentRequest()); } + +// ── Retrieval Operations ────────────────────────── + +export async function getRetrievalStatus(): Promise { + return get('/admin/retrieval/status'); +} + +export async function listRetrievalIndexJobs( + page: number = 1, + pageSize: number = 50, +): Promise> { + return get('/admin/retrieval/index-jobs', { page, pageSize }); +} + +export async function createRetrievalIndexJob(): Promise { + return post('/admin/retrieval/index-jobs', {}, idempotentRequest()); +} + +export async function getRetrievalActivationCheck( + id: string, +): Promise { + return get(`/admin/retrieval/index-jobs/${id}/activation-check`); +} + +export async function activateRetrievalIndexJob(data: { + id: string; + expectedCurrentCollection: string | null; + confirmLatencyWarning: boolean; +}): Promise { + return post( + `/admin/retrieval/index-jobs/${data.id}/activate`, + { + expectedCurrentCollection: data.expectedCurrentCollection, + confirmed: true, + confirmLatencyWarning: data.confirmLatencyWarning, + }, + idempotentRequest(), + ); +} + +export async function rollbackRetrievalIndexJob(data: { + id: string; + expectedCurrentCollection: string; +}): Promise { + return post( + `/admin/retrieval/index-jobs/${data.id}/rollback`, + { + expectedCurrentCollection: data.expectedCurrentCollection, + confirmed: true, + }, + idempotentRequest(), + ); +} + +export async function listRetrievalTraces(params?: { + page?: number; + pageSize?: number; + status?: RetrievalTraceStatus; + backend?: 'memory' | 'qdrant'; + sessionId?: string; + createdFrom?: string; + createdTo?: string; +}): Promise> { + return get('/admin/retrieval/traces', params); +} + +export async function getRetrievalTrace( + traceId: string, +): Promise { + return get(`/admin/retrieval/traces/${traceId}`); +} diff --git a/client/src/hooks/useRetrievalOps.ts b/client/src/hooks/useRetrievalOps.ts new file mode 100644 index 0000000..4ab1f78 --- /dev/null +++ b/client/src/hooks/useRetrievalOps.ts @@ -0,0 +1,198 @@ +import { useCallback, useEffect, useState } from 'react'; +import { MessagePlugin } from 'tdesign-react'; +import * as adminApi from '../api/admin'; +import type { + RetrievalActivationCheck, + RetrievalIndexJob, + RetrievalStatus, + RetrievalTrace, + RetrievalTraceDetail, + RetrievalTraceStatus, +} from '../types'; + +export type RetrievalPendingAction = { + kind: 'activate' | 'rollback'; + job: RetrievalIndexJob; + gate?: RetrievalActivationCheck; +}; + +type Translate = (key: string, params?: Record) => string; + +const TRACE_PAGE_SIZE = 20; + +export function useRetrievalOps(t: Translate) { + const [status, setStatus] = useState(null); + const [jobs, setJobs] = useState([]); + const [traces, setTraces] = useState([]); + const [traceTotal, setTraceTotal] = useState(0); + const [tracePage, setTracePage] = useState(1); + const [traceStatus, setTraceStatus] = useState(''); + const [traceBackend, setTraceBackend] = useState<'memory' | 'qdrant' | ''>(''); + const [traceSession, setTraceSession] = useState(''); + const [traceDates, setTraceDates] = useState>([]); + const [detail, setDetail] = useState(null); + const [loading, setLoading] = useState(true); + const [traceLoading, setTraceLoading] = useState(false); + const [actionLoading, setActionLoading] = useState(false); + const [error, setError] = useState(false); + const [pendingAction, setPendingAction] = useState(null); + const [latencyConfirmed, setLatencyConfirmed] = useState(false); + + const loadOverview = useCallback(async (quiet = false) => { + if (!quiet) setLoading(true); + try { + const [nextStatus, jobPage] = await Promise.all([ + adminApi.getRetrievalStatus(), + adminApi.listRetrievalIndexJobs(1, 50), + ]); + setStatus(nextStatus); + setJobs(jobPage.items); + setError(false); + } catch { + setError(true); + } finally { + if (!quiet) setLoading(false); + } + }, []); + + const loadTraces = useCallback(async () => { + setTraceLoading(true); + try { + const result = await adminApi.listRetrievalTraces({ + page: tracePage, + pageSize: TRACE_PAGE_SIZE, + status: traceStatus || undefined, + backend: traceBackend || undefined, + sessionId: traceSession.trim() || undefined, + createdFrom: traceDates[0] + ? new Date(`${String(traceDates[0])}T00:00:00`).toISOString() + : undefined, + createdTo: traceDates[1] + ? new Date(`${String(traceDates[1])}T23:59:59.999`).toISOString() + : undefined, + }); + setTraces(result.items); + setTraceTotal(result.total); + setError(false); + } catch { + setError(true); + } finally { + setTraceLoading(false); + } + }, [traceBackend, traceDates, tracePage, traceSession, traceStatus]); + + const refreshAll = useCallback(async () => { + await Promise.all([loadOverview(), loadTraces()]); + }, [loadOverview, loadTraces]); + + useEffect(() => { + void refreshAll(); + }, [refreshAll]); + + useEffect(() => { + if (!jobs.some((job) => ['queued', 'running', 'interrupted'].includes(job.status))) { + return undefined; + } + const timer = window.setInterval(() => void loadOverview(true), 1500); + return () => window.clearInterval(timer); + }, [jobs, loadOverview]); + + const openActivation = async (job: RetrievalIndexJob) => { + setActionLoading(true); + try { + const gate = await adminApi.getRetrievalActivationCheck(job.id); + setLatencyConfirmed(false); + setPendingAction({ kind: 'activate', job, gate }); + } catch { + MessagePlugin.error(t('retrievalOps.actionFailed')); + } finally { + setActionLoading(false); + } + }; + + const confirmAction = async () => { + if (!pendingAction || !status) return; + setActionLoading(true); + try { + if (pendingAction.kind === 'activate') { + await adminApi.activateRetrievalIndexJob({ + id: pendingAction.job.id, + expectedCurrentCollection: status.collection, + confirmLatencyWarning: latencyConfirmed, + }); + MessagePlugin.success(t('retrievalOps.activated')); + } else { + await adminApi.rollbackRetrievalIndexJob({ + id: pendingAction.job.id, + expectedCurrentCollection: pendingAction.job.collection, + }); + MessagePlugin.success(t('retrievalOps.rolledBack')); + } + setPendingAction(null); + await loadOverview(); + } catch { + MessagePlugin.error(t('retrievalOps.actionFailed')); + } finally { + setActionLoading(false); + } + }; + + const createJob = async () => { + setActionLoading(true); + try { + await adminApi.createRetrievalIndexJob(); + MessagePlugin.success(t('retrievalOps.jobQueued')); + await loadOverview(); + } catch { + MessagePlugin.error(t('retrievalOps.actionFailed')); + } finally { + setActionLoading(false); + } + }; + + const openTrace = async (traceId: string) => { + setTraceLoading(true); + try { + setDetail(await adminApi.getRetrievalTrace(traceId)); + } catch { + MessagePlugin.error(t('retrievalOps.traceLoadFailed')); + } finally { + setTraceLoading(false); + } + }; + + return { + status, + jobs, + traces, + traceTotal, + tracePage, + traceStatus, + traceBackend, + traceSession, + traceDates, + detail, + loading, + traceLoading, + actionLoading, + error, + pendingAction, + latencyConfirmed, + pageSize: TRACE_PAGE_SIZE, + setTracePage, + setTraceStatus, + setTraceBackend, + setTraceSession, + setTraceDates, + setDetail, + setPendingAction, + setLatencyConfirmed, + loadOverview, + loadTraces, + refreshAll, + openActivation, + confirmAction, + createJob, + openTrace, + }; +} diff --git a/client/src/i18n/dictionary.json b/client/src/i18n/dictionary.json index 4c1baa1..960ceb9 100644 --- a/client/src/i18n/dictionary.json +++ b/client/src/i18n/dictionary.json @@ -419,6 +419,10 @@ "zh": "取消", "en": "Cancel" }, + "common.confirm": { + "zh": "确认", + "en": "Confirm" + }, "common.close": { "zh": "关闭", "en": "Close" @@ -1874,6 +1878,10 @@ "quality.tags": { "zh": "标签", "en": "Tags" }, "quality.sourceId": { "zh": "预期来源 ID", "en": "Expected source ID" }, "quality.selectDatasets": { "zh": "选择已发布评测版本", "en": "Select published dataset versions" }, + "quality.selectBackends": { "zh": "选择影子评测后端", "en": "Select shadow evaluation backends" }, + "quality.backend": { "zh": "检索后端", "en": "Retrieval backend" }, + "quality.backend.memory": { "zh": "内存基线", "en": "Memory baseline" }, + "quality.backend.qdrant": { "zh": "Qdrant 索引", "en": "Qdrant index" }, "quality.matrixCount": { "zh": "当前矩阵：{count} 个候选；当前策略自动加入", "en": "Current matrix: {count} candidates; current policy is added automatically" }, "quality.directThresholds": { "zh": "直答阈值", "en": "Direct-answer thresholds" }, "quality.generationThresholds": { "zh": "生成阈值", "en": "Generation thresholds" }, @@ -2050,5 +2058,84 @@ "triage.next.normal": { "zh": "查看对话并补充缺失信息，然后转入{queue}。", "en": "Review the conversation and collect missing information, then route to {queue}." }, "triage.wait.minutes": { "zh": "{count} 分钟", "en": "{count} min" }, "triage.wait.hours": { "zh": "{count} 小时", "en": "{count} hr" }, - "triage.wait.days": { "zh": "{count} 天", "en": "{count} d" } + "triage.wait.days": { "zh": "{count} 天", "en": "{count} d" }, + "nav.retrievalOps": { "zh": "检索运维", "en": "Retrieval ops" }, + "retrievalOps.title": { "zh": "检索运维", "en": "Retrieval operations" }, + "retrievalOps.description": { "zh": "监控检索后端，构建并安全切换 Qdrant 索引，追踪每次检索决策。", "en": "Monitor retrieval backends, build and safely switch Qdrant indexes, and inspect each retrieval decision." }, + "retrievalOps.loadFailed": { "zh": "检索运维数据加载失败，请重试。", "en": "Retrieval operations data could not be loaded. Try again." }, + "retrievalOps.overview": { "zh": "运行概览", "en": "Runtime overview" }, + "retrievalOps.provider": { "zh": "主后端", "en": "Primary backend" }, + "retrievalOps.provider.memory": { "zh": "内存", "en": "Memory" }, + "retrievalOps.provider.qdrant": { "zh": "Qdrant", "en": "Qdrant" }, + "retrievalOps.qdrantHealth": { "zh": "Qdrant 连通性", "en": "Qdrant health" }, + "retrievalOps.activeCollection": { "zh": "Alias / Collection", "en": "Alias / Collection" }, + "retrievalOps.vectorStats": { "zh": "点数 / 维度", "en": "Points / dimensions" }, + "retrievalOps.pointsDimensions": { "zh": "向量点数与维度", "en": "Vector point count and dimensions" }, + "retrievalOps.syncStatus": { "zh": "同步状态", "en": "Sync status" }, + "retrievalOps.health.healthy": { "zh": "正常", "en": "Healthy" }, + "retrievalOps.health.degraded": { "zh": "降级", "en": "Degraded" }, + "retrievalOps.health.unavailable": { "zh": "不可用", "en": "Unavailable" }, + "retrievalOps.health.not_configured": { "zh": "未配置", "en": "Not configured" }, + "retrievalOps.sync.synced": { "zh": "已同步", "en": "Synced" }, + "retrievalOps.sync.stale": { "zh": "已过期", "en": "Stale" }, + "retrievalOps.sync.not_configured": { "zh": "未配置", "en": "Not configured" }, + "retrievalOps.indexJobs": { "zh": "索引任务", "en": "Index jobs" }, + "retrievalOps.indexJobsDescription": { "zh": "从 SQLite 权威知识构建版本化 collection；旧 collection 不会自动删除。", "en": "Build versioned collections from authoritative SQLite knowledge. Old collections are not deleted automatically." }, + "retrievalOps.createJob": { "zh": "构建新索引", "en": "Build new index" }, + "retrievalOps.jobQueued": { "zh": "索引任务已进入队列", "en": "Index job queued" }, + "retrievalOps.noJobs": { "zh": "暂无索引任务", "en": "No index jobs yet" }, + "retrievalOps.status": { "zh": "状态", "en": "Status" }, + "retrievalOps.collection": { "zh": "Collection", "en": "Collection" }, + "retrievalOps.progress": { "zh": "进度", "en": "Progress" }, + "retrievalOps.profileDimension": { "zh": "Embedding / 维度", "en": "Embedding / dimension" }, + "retrievalOps.updatedAt": { "zh": "更新时间", "en": "Updated" }, + "retrievalOps.jobStatus.queued": { "zh": "排队中", "en": "Queued" }, + "retrievalOps.jobStatus.running": { "zh": "构建中", "en": "Running" }, + "retrievalOps.jobStatus.interrupted": { "zh": "已中断", "en": "Interrupted" }, + "retrievalOps.jobStatus.ready": { "zh": "待激活", "en": "Ready" }, + "retrievalOps.jobStatus.active": { "zh": "已激活", "en": "Active" }, + "retrievalOps.jobStatus.rolled_back": { "zh": "已回滚", "en": "Rolled back" }, + "retrievalOps.jobStatus.failed": { "zh": "失败", "en": "Failed" }, + "retrievalOps.jobStatus.stale": { "zh": "已过期", "en": "Stale" }, + "retrievalOps.activate": { "zh": "激活", "en": "Activate" }, + "retrievalOps.rollback": { "zh": "回滚", "en": "Roll back" }, + "retrievalOps.confirmActivate": { "zh": "确认激活索引", "en": "Confirm index activation" }, + "retrievalOps.confirmRollback": { "zh": "确认回滚索引", "en": "Confirm index rollback" }, + "retrievalOps.gatePassed": { "zh": "Quality Lab 门禁已通过，可以原子切换 alias。", "en": "Quality Lab gates passed. The alias can be switched atomically." }, + "retrievalOps.gateBlocked": { "zh": "激活门禁未通过", "en": "Activation gates did not pass" }, + "retrievalOps.confirmLatencyWarning": { "zh": "我已确认 P95 延迟警告并继续激活", "en": "I acknowledge the P95 latency warning and want to activate" }, + "retrievalOps.rollbackDescription": { "zh": "Alias 将切回已验证 collection：{collection}", "en": "The alias will switch back to the verified collection: {collection}" }, + "retrievalOps.activated": { "zh": "索引已激活", "en": "Index activated" }, + "retrievalOps.rolledBack": { "zh": "索引已回滚", "en": "Index rolled back" }, + "retrievalOps.actionFailed": { "zh": "操作失败，请刷新状态后重试。", "en": "The operation failed. Refresh the current state and try again." }, + "retrievalOps.traces": { "zh": "检索 Trace", "en": "Retrieval traces" }, + "retrievalOps.fromDate": { "zh": "开始日期", "en": "From date" }, + "retrievalOps.toDate": { "zh": "结束日期", "en": "To date" }, + "retrievalOps.traceStatus": { "zh": "Trace 状态", "en": "Trace status" }, + "retrievalOps.traceStatus.completed": { "zh": "完成", "en": "Completed" }, + "retrievalOps.traceStatus.degraded": { "zh": "降级", "en": "Degraded" }, + "retrievalOps.traceStatus.failed": { "zh": "失败", "en": "Failed" }, + "retrievalOps.backend": { "zh": "后端", "en": "Backend" }, + "retrievalOps.sessionId": { "zh": "会话 ID", "en": "Session ID" }, + "retrievalOps.totalLatency": { "zh": "总延迟", "en": "Total latency" }, + "retrievalOps.createdAt": { "zh": "创建时间", "en": "Created" }, + "retrievalOps.noTraces": { "zh": "暂无符合条件的 Trace", "en": "No traces match these filters" }, + "retrievalOps.viewTrace": { "zh": "查看", "en": "View" }, + "retrievalOps.traceDetail": { "zh": "检索 Trace 详情", "en": "Retrieval trace detail" }, + "retrievalOps.traceLoadFailed": { "zh": "Trace 详情加载失败", "en": "Trace details could not be loaded" }, + "retrievalOps.userMessage": { "zh": "客户消息", "en": "Customer message" }, + "retrievalOps.assistantMessage": { "zh": "助手消息", "en": "Assistant message" }, + "retrievalOps.contentUnavailable": { "zh": "内容已删除或不可用", "en": "Content was deleted or is unavailable" }, + "retrievalOps.stage.query_expand": { "zh": "查询扩展", "en": "Query expansion" }, + "retrievalOps.stage.embedding": { "zh": "Embedding", "en": "Embedding" }, + "retrievalOps.stage.vector_recall": { "zh": "向量召回", "en": "Vector recall" }, + "retrievalOps.stage.keyword_recall": { "zh": "关键词召回", "en": "Keyword recall" }, + "retrievalOps.stage.fusion": { "zh": "融合", "en": "Fusion" }, + "retrievalOps.stage.rerank": { "zh": "重排", "en": "Reranking" }, + "retrievalOps.stage.context_budget": { "zh": "上下文预算", "en": "Context budget" }, + "retrievalOps.stage.grounding": { "zh": "Grounding 决策", "en": "Grounding decision" }, + "retrievalOps.stageStatus.completed": { "zh": "完成", "en": "Completed" }, + "retrievalOps.stageStatus.degraded": { "zh": "降级", "en": "Degraded" }, + "retrievalOps.stageStatus.failed": { "zh": "失败", "en": "Failed" }, + "retrievalOps.stageStatus.skipped": { "zh": "跳过", "en": "Skipped" } } diff --git a/client/src/pages/admin/AdminLayout.tsx b/client/src/pages/admin/AdminLayout.tsx index 4bebf87..4ec8c65 100644 --- a/client/src/pages/admin/AdminLayout.tsx +++ b/client/src/pages/admin/AdminLayout.tsx @@ -12,6 +12,7 @@ import { FileIconIcon, SearchIcon, QueueIcon, + ServerIcon, } from 'tdesign-icons-react'; import { useAuth } from '../../hooks/useAuth'; import { useTranslation } from '../../hooks/usePreferences'; @@ -33,6 +34,7 @@ const MENU_ITEMS: MenuItem[] = [ { path: '/admin/documents', labelKey: 'nav.documents', icon: }, { path: '/admin/knowledge-review', labelKey: 'nav.knowledgeReview', icon: }, { path: '/admin/quality-lab', labelKey: 'nav.qualityLab', icon: }, + { path: '/admin/retrieval-ops', labelKey: 'nav.retrievalOps', icon: }, { path: '/admin/config', labelKey: 'nav.config', icon: }, ]; @@ -63,6 +65,7 @@ export function AdminLayout(): React.ReactElement { if (location.pathname.startsWith('/admin/documents')) return '/admin/documents'; if (location.pathname.startsWith('/admin/knowledge-review')) return '/admin/knowledge-review'; if (location.pathname.startsWith('/admin/quality-lab')) return '/admin/quality-lab'; + if (location.pathname.startsWith('/admin/retrieval-ops')) return '/admin/retrieval-ops'; if (location.pathname.startsWith('/admin/config')) return '/admin/config'; return '/admin'; })(); diff --git a/client/src/pages/admin/QualityLabPage.tsx b/client/src/pages/admin/QualityLabPage.tsx index 27579a5..7bc888c 100644 --- a/client/src/pages/admin/QualityLabPage.tsx +++ b/client/src/pages/admin/QualityLabPage.tsx @@ -19,6 +19,8 @@ import type { QualityCase, QualityDatasetVersion, QualityRun, + QualityBackendTarget, + RetrievalIndexJob, RetrievalPolicy, RetrievalPolicyEvent, RetrievalPolicyConfig, @@ -43,6 +45,7 @@ export function QualityLabPage(): React.ReactElement { const [cases, setCases] = useState([]); const [selectedVersionId, setSelectedVersionId] = useState(''); const [runs, setRuns] = useState([]); + const [indexJobs, setIndexJobs] = useState([]); const [currentPolicy, setCurrentPolicy] = useState(null); const [policyHistory, setPolicyHistory] = useState([]); const [policyEvents, setPolicyEvents] = useState([]); @@ -54,6 +57,7 @@ export function QualityLabPage(): React.ReactElement { 'none', 'local_overlap_v1', ]); + const [selectedBackends, setSelectedBackends] = useState>(['memory']); const [loading, setLoading] = useState(false); const [datasetDialog, setDatasetDialog] = useState(false); const [caseDialog, setCaseDialog] = useState(false); @@ -102,16 +106,18 @@ export function QualityLabPage(): React.ReactElement { const refresh = useCallback(async () => { setLoading(true); try { - const [datasetItems, runPage, policyData] = await Promise.all([ + const [datasetItems, runPage, policyData, indexJobPage] = await Promise.all([ adminApi.listQualityDatasets(), adminApi.listQualityRuns(), adminApi.getQualityPolicies(), + adminApi.listRetrievalIndexJobs(1, 100), ]); setDatasets(datasetItems); setRuns(runPage.items); setCurrentPolicy(policyData.current); setPolicyHistory(policyData.history); setPolicyEvents(policyData.events); + setIndexJobs(indexJobPage.items); const latestCompleted = runPage.items.find((run) => run.status === 'completed'); if (latestCompleted) { const checks = await Promise.all(latestCompleted.candidates.map(async (candidate) => [ @@ -218,6 +224,7 @@ export function QualityLabPage(): React.ReactElement { await adminApi.createQualityRun({ datasetVersionIds: selectedDatasets.map(String), policies: matrixPolicies, + backendTargets: selectedBackends.map(toBackendTarget), }); MessagePlugin.success(t('quality.runQueued')); await refresh(); @@ -239,6 +246,7 @@ export function QualityLabPage(): React.ReactElement { await adminApi.createQualityRun({ datasetVersionIds: run.datasetVersionIds, policies: run.policies, + backendTargets: run.backendTargets, }); MessagePlugin.success(t('quality.runQueued')); await refresh(); @@ -404,6 +412,22 @@ export function QualityLabPage(): React.ReactElement { placeholder={t('quality.selectDatasets')} onChange={(value) => setSelectedDatasets(value as Array)} /> + {t('quality.matrixCount', { count: matrixPolicies.length })} + + + {error && ( + + )} + +
+
+

{t('retrievalOps.overview')}

+
+
+ + {t('retrievalOps.provider')} + {status ? t(`retrievalOps.provider.${status.provider}`) : '—'} + + + {t('retrievalOps.qdrantHealth')} + {status ? ( + + {t(`retrievalOps.health.${status.qdrantHealth}`)} + + ) : '—'} + + + {t('retrievalOps.activeCollection')} + {status?.collection ?? '—'} + {status?.alias ?? '—'} + + + {t('retrievalOps.vectorStats')} + + {status?.points ?? '—'} / {status?.dimensions ?? '—'} + + {t('retrievalOps.pointsDimensions')} + + + {t('retrievalOps.syncStatus')} + + {status ? t(`retrievalOps.sync.${status.syncStatus}`) : '—'} + + +
+
+ + + +
+

{t('retrievalOps.indexJobsDescription')}

+ +
+
+ ( + + {t(`retrievalOps.jobStatus.${row.status}`)} + + ), + }, + { + colKey: 'collection', + title: t('retrievalOps.collection'), + ellipsis: true, + }, + { + colKey: 'progress', + title: t('retrievalOps.progress'), + width: 180, + cell: ({ row }) => ( + + ), + }, + { + colKey: 'profile', + title: t('retrievalOps.profileDimension'), + cell: ({ row }) => `${row.embeddingProfile} · ${row.vectorDimension}`, + }, + { + colKey: 'updatedAt', + title: t('retrievalOps.updatedAt'), + width: 180, + cell: ({ row }) => new Date(row.updatedAt).toLocaleString(dateLocale), + }, + { + colKey: 'action', + title: t('common.actions'), + width: 180, + cell: ({ row }) => ( + + {row.status === 'ready' && ( + + )} + {row.status === 'active' && row.previousCollection && ( + + )} + {row.failureCode && ( + {row.failureCode} + )} + + ), + }, + ]} + /> + + + + + +
+ { + setTraceDates(value as Array); + setTracePage(1); + }} + placeholder={[ + t('retrievalOps.fromDate'), + t('retrievalOps.toDate'), + ]} + clearable + /> + { + setTraceBackend(String(value ?? '') as 'memory' | 'qdrant' | ''); + setTracePage(1); + }} + /> + { + setTracePage(1); + void loadTraces(); + }} + /> + +
+ +
+
( + + {t(`retrievalOps.traceStatus.${row.status}`)} + + ), + }, + { colKey: 'backend', title: t('retrievalOps.backend'), width: 100 }, + { + colKey: 'sessionId', + title: t('retrievalOps.sessionId'), + ellipsis: true, + }, + { + colKey: 'latency', + title: t('retrievalOps.totalLatency'), + width: 130, + cell: ({ row }) => `${row.totalLatencyMs.toFixed(1)} ms`, + }, + { + colKey: 'createdAt', + title: t('retrievalOps.createdAt'), + width: 180, + cell: ({ row }) => new Date(row.createdAt).toLocaleString(dateLocale), + }, + { + colKey: 'action', + title: t('common.actions'), + width: 100, + cell: ({ row }) => ( + + ), + }, + ]} + /> + + + + + + setPendingAction(null)} + onConfirm={() => void confirmAction()} + > + {pendingAction?.kind === 'activate' && pendingAction.gate && ( + + + {pendingAction.gate.warnings.length > 0 && ( + <> + + + {t('retrievalOps.confirmLatencyWarning')} + + + )} + + )} + {pendingAction?.kind === 'rollback' && ( + + )} + + + setDetail(null)} + > + {detail && ( +
+
+
+ {t('retrievalOps.userMessage')} +

{detail.messages.user?.content ?? t('retrievalOps.contentUnavailable')}

+
+
+ {t('retrievalOps.assistantMessage')} +

{detail.messages.assistant?.content ?? t('retrievalOps.contentUnavailable')}

+
+
+ + {detail.trace.stages.map((stage) => ( + +
+ {t(`retrievalOps.stage.${stage.name}`)} + + {t(`retrievalOps.stageStatus.${stage.status}`)} + {' · '} + {stage.inputCount} → {stage.outputCount} + + {stage.errorCode && ( + {stage.errorCode} + )} + {Object.keys(stage.budget).length > 0 && ( + + {Object.entries(stage.budget) + .map(([key, value]) => `${key}: ${value}`) + .join(' · ')} + + )} + {stage.candidates.length > 0 && ( +
    + {stage.candidates.map((candidate, index) => { + const resolved = knowledgeById.get( + `${candidate.knowledgeType}:${candidate.knowledgeId}`, + ); + return ( +
  1. + + {resolved?.title || candidate.knowledgeId} + {' · '} + {candidate.source ?? candidate.knowledgeType} + + {candidate.score?.toFixed(4) ?? '—'} + {resolved?.available &&

    {resolved.content}

    } +
    2. + ); + })} +
+ )} +
+ + ))} + +
+ )} + + + ); +} + +function jobTheme(status: RetrievalIndexJob['status']): 'default' | 'primary' | 'success' | 'warning' | 'danger' { + if (status === 'active') return 'success'; + if (status === 'ready') return 'primary'; + if (status === 'failed' || status === 'stale') return 'danger'; + if (status === 'running' || status === 'interrupted') return 'warning'; + return 'default'; +} + +export default RetrievalOpsPage; diff --git a/client/src/types/index.ts b/client/src/types/index.ts index 0186a14..f069f0b 100644 --- a/client/src/types/index.ts +++ b/client/src/types/index.ts @@ -335,6 +335,9 @@ export type RerankerMode = 'none' | 'local_overlap_v1'; export type QualityRunStatus = | 'queued' | 'running' | 'completed' | 'failed' | 'interrupted' | 'cancelled' | 'stale'; +export type QualityBackendTarget = + | { provider: 'memory' } + | { provider: 'qdrant'; indexJobId: string }; export interface RetrievalPolicyConfig { directFaqThreshold: number; @@ -409,6 +412,7 @@ export interface QualityMetrics { export interface QualityCandidateResult { key: string; + backendTarget: QualityBackendTarget; policy: RetrievalPolicyConfig; metrics: QualityMetrics; recommended: boolean; @@ -423,6 +427,7 @@ export interface QualityRun { id: string; datasetVersionIds: string[]; policies: RetrievalPolicyConfig[]; + backendTargets: QualityBackendTarget[]; status: QualityRunStatus; progress: number; totalCases: number; @@ -443,6 +448,105 @@ export interface PolicyGateResult { reasons: string[]; } +export type RetrievalIndexJobStatus = + | 'queued' | 'running' | 'interrupted' | 'ready' + | 'active' | 'rolled_back' | 'failed' | 'stale'; + +export interface RetrievalIndexJob { + id: string; + status: RetrievalIndexJobStatus; + collection: string; + embeddingProfile: string; + vectorDimension: number; + knowledgeFingerprint: string; + expectedCount: number; + completedCount: number; + checkpoint: number; + previousCollection: string | null; + failureCode: string | null; + createdBy: string; + createdAt: string; + startedAt: string | null; + readyAt: string | null; + activatedAt: string | null; + rolledBackAt: string | null; + updatedAt: string; +} + +export interface RetrievalStatus { + provider: 'memory' | 'qdrant'; + qdrantConfigured: boolean; + qdrantHealth: 'healthy' | 'degraded' | 'unavailable' | 'not_configured'; + alias: string; + collection: string | null; + points: number | null; + dimensions: number | null; + syncStatus: 'synced' | 'stale' | 'not_configured'; +} + +export interface RetrievalActivationCheck { + eligible: boolean; + warnings: string[]; + reasons: string[]; + qualityRunId: string | null; + candidateKey: string | null; +} + +export type RetrievalTraceStatus = 'completed' | 'degraded' | 'failed'; +export type RetrievalTraceStageName = + | 'query_expand' | 'embedding' | 'vector_recall' | 'keyword_recall' + | 'fusion' | 'rerank' | 'context_budget' | 'grounding'; + +export interface RetrievalTraceCandidate { + knowledgeType: 'faq' | 'document'; + knowledgeId: string; + score?: number; + rank?: number; + source?: 'vector' | 'keyword' | 'hybrid'; +} + +export interface RetrievalTraceStage { + name: RetrievalTraceStageName; + order: number; + status: 'completed' | 'degraded' | 'failed' | 'skipped'; + latencyMs: number; + inputCount: number; + outputCount: number; + candidates: RetrievalTraceCandidate[]; + budget: Record; + errorCode: string | null; +} + +export interface RetrievalTrace { + id: string; + sessionId: string; + userMessageId: string; + assistantMessageId: string | null; + policyId: string; + backend: 'memory' | 'qdrant'; + status: RetrievalTraceStatus; + errorCode: string | null; + totalLatencyMs: number; + stages: RetrievalTraceStage[]; + createdAt: string; + completedAt: string; +} + +export interface RetrievalTraceDetail { + trace: RetrievalTrace; + messages: { + user: { id: string; content: string } | null; + assistant: { id: string; content: string } | null; + }; + knowledge: Array<{ + knowledgeType: 'faq' | 'document'; + knowledgeId: string; + title: string; + content: string; + available: boolean; + }>; +} + // ── Domain Models ────────────────────────────────── export interface FaqEntry { diff --git a/docker-compose.yml b/docker-compose.yml index 69976bb..5b7f295 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,13 @@ services: DB_PATH: /app/data/customer-service.db ALLOWED_ORIGINS: http://localhost:5173 EMBED_PROVIDER: other + VECTOR_STORE_PROVIDER: ${VECTOR_STORE_PROVIDER:-memory} + QDRANT_URL: ${QDRANT_URL:-} + QDRANT_API_KEY: ${QDRANT_API_KEY:-} + QDRANT_COLLECTION_PREFIX: ${QDRANT_COLLECTION_PREFIX:-resolveweave_knowledge} + QDRANT_COLLECTION_ALIAS: ${QDRANT_COLLECTION_ALIAS:-resolveweave_knowledge_active} + QDRANT_TIMEOUT_MS: ${QDRANT_TIMEOUT_MS:-5000} + RETRIEVAL_TRACE_RETENTION_DAYS: ${RETRIEVAL_TRACE_RETENTION_DAYS:-30} OCR_SERVICE_URL: ${OCR_SERVICE_URL:-} OCR_SERVICE_TOKEN: ${OCR_SERVICE_TOKEN:-} volumes: @@ -33,6 +40,17 @@ services: ports: - "127.0.0.1:8001:8001" + qdrant: + profiles: ["qdrant"] + image: qdrant/qdrant:v1.18.2 + restart: unless-stopped + ports: + - "127.0.0.1:6333:6333" + volumes: + - resolve-weave-qdrant:/qdrant/storage + volumes: resolve-weave-data: name: ${RESOLVE_WEAVE_DATA_VOLUME:-resolve-weave-data} + resolve-weave-qdrant: + name: ${RESOLVE_WEAVE_QDRANT_VOLUME:-resolve-weave-qdrant} diff --git a/docs/demo/v0.3.2-preview.gif b/docs/demo/v0.3.2-preview.gif new file mode 100644 index 0000000..5791722 Binary files /dev/null and b/docs/demo/v0.3.2-preview.gif differ diff --git a/docs/releases/assets/v0.3.2-activation-gate.png b/docs/releases/assets/v0.3.2-activation-gate.png new file mode 100644 index 0000000..0b7a80c Binary files /dev/null and b/docs/releases/assets/v0.3.2-activation-gate.png differ diff --git a/docs/releases/assets/v0.3.2-index-active.png b/docs/releases/assets/v0.3.2-index-active.png new file mode 100644 index 0000000..23c1b29 Binary files /dev/null and b/docs/releases/assets/v0.3.2-index-active.png differ diff --git a/docs/releases/assets/v0.3.2-index-ready.png b/docs/releases/assets/v0.3.2-index-ready.png new file mode 100644 index 0000000..2967de2 Binary files /dev/null and b/docs/releases/assets/v0.3.2-index-ready.png differ diff --git a/docs/releases/assets/v0.3.2-index-rolled-back.png b/docs/releases/assets/v0.3.2-index-rolled-back.png new file mode 100644 index 0000000..8ee8a0c Binary files /dev/null and b/docs/releases/assets/v0.3.2-index-rolled-back.png differ diff --git a/docs/releases/assets/v0.3.2-ops-error.png b/docs/releases/assets/v0.3.2-ops-error.png new file mode 100644 index 0000000..0819f2a Binary files /dev/null and b/docs/releases/assets/v0.3.2-ops-error.png differ diff --git a/docs/releases/assets/v0.3.2-quality-backends.png b/docs/releases/assets/v0.3.2-quality-backends.png new file mode 100644 index 0000000..9fcd902 Binary files /dev/null and b/docs/releases/assets/v0.3.2-quality-backends.png differ diff --git a/docs/releases/assets/v0.3.2-retrieval-ops-desktop.png b/docs/releases/assets/v0.3.2-retrieval-ops-desktop.png new file mode 100644 index 0000000..aae14da Binary files /dev/null and b/docs/releases/assets/v0.3.2-retrieval-ops-desktop.png differ diff --git a/docs/releases/assets/v0.3.2-retrieval-ops-mobile-dark.png b/docs/releases/assets/v0.3.2-retrieval-ops-mobile-dark.png new file mode 100644 index 0000000..fc0161b Binary files /dev/null and b/docs/releases/assets/v0.3.2-retrieval-ops-mobile-dark.png differ diff --git a/docs/releases/assets/v0.3.2-retrieval-trace-desktop.png b/docs/releases/assets/v0.3.2-retrieval-trace-desktop.png new file mode 100644 index 0000000..dae4fdb Binary files /dev/null and b/docs/releases/assets/v0.3.2-retrieval-trace-desktop.png differ diff --git a/docs/releases/v0.3.2-evidence.md b/docs/releases/v0.3.2-evidence.md new file mode 100644 index 0000000..133315e --- /dev/null +++ b/docs/releases/v0.3.2-evidence.md @@ -0,0 +1,157 @@ +# v0.3.2 Release Evidence — Qdrant & Retrieval Observability + +> Status: final local release verification completed before branch merge, +> annotated tag, and GitHub Release publication. + +## User scenario + +1. Keep a fresh clone on the default in-memory vector backend, or explicitly + configure Qdrant and restart the application. +2. Open **Admin Console → Retrieval Operations** and verify backend health, + alias/collection, vector count, dimension, and synchronization state. +3. Build a versioned collection from current SQLite knowledge. Resume an + interrupted batch only while the knowledge fingerprint remains unchanged. +4. Select the ready index as a Quality Lab backend target and compare it with + the memory baseline under the same dataset, policy, and query embeddings. +5. Inspect the activation gate. A P95 latency increase above 25% requires an + explicit acknowledgement before the atomic alias switch. +6. Ask a customer question, inspect the eight-stage retrieval trace, simulate + Qdrant unavailability to verify keyword degradation, and roll the alias + back to the prior verified collection. + +## Verification coverage + +The v0.3.2 suite covers: + +- asynchronous memory behavior and Qdrant REST mapping, UUIDv5 IDs, knowledge + filtering, request trace IDs, safe payloads, health, stats, and timeout + classification; +- SQLite hydration and rejection of orphaned, stale-version, disabled, and + inactive-source vector candidates; +- keyword/structured degradation without silent in-memory vector rebuilding; +- additive index-job migration, idempotent creation, fixed-batch checkpoints, + interrupted resume, fingerprint changes, profile/dimension/count validation, + stale jobs, keyset-paged snapshots, alias conflicts, expected-current + concurrency, durable activation/rollback intents, startup reconciliation, + recoverable alias/SQLite finalization, activation, and rollback; +- memory/Qdrant Quality Lab candidates, non-regression gates, unsafe-answer + blocking, exact active-policy candidate binding, stale-job rejection, + bounded eight-query concurrency, per-query latency percentiles, and explicit + P95 latency acknowledgement; +- completed/degraded/failed traces, eight fixed stages, 20-candidate channel + caps, three-evidence cap, session cascade, authorized hydration, pagination, + startup cleanup, and daily retention cleanup; +- admin authentication, rate limits, idempotency, bounded pagination, bilingual + desktop/mobile operations UI, keyboard interaction, loading/empty/error + states, activation, rollback, and trace timeline. + +Independent standards, specification, and final adversarial static reviews +report no remaining P0/P1/P2 finding. The final executable publication gates +also passed after the concurrency, batch-degradation, and keyset-query fixes. + +## Verification commands + +```bash +(cd ocr-worker && python3 -m unittest discover -s tests) +EMBED_PROVIDER=other npm test +EMBED_PROVIDER=other npm run eval:faq +EMBED_PROVIDER=other npm run eval:document +EMBED_PROVIDER=other npm run eval:mixed +EMBED_PROVIDER=other npm run eval:quality +EMBED_PROVIDER=other npm run eval:ocr +npm run eval:triage +PLAYWRIGHT_CHANNEL=chromium npm run test:e2e +EMBED_PROVIDER=other npm run build +QDRANT_URL=http://localhost:6333 npm run test:qdrant +git diff --check +``` + +The pinned-Qdrant integration command is also an independent GitHub Actions +job using `qdrant/qdrant:v1.18.2`; the default test job and no-key path do not +depend on it. + +## Evaluation + +Checked locally on 2026-07-31: + +- local OCR worker contract tests: 6/6 passed; +- full runtime/unit/service/API regression suite and both TypeScript checks: + passed; +- FAQ: Top1 100%, Top3 100%, no-match 100%; +- documents: Top3 100%, MRR 0.958 for both semantic-v1 and the structure + baseline; +- mixed knowledge: 6/6 Top1; +- Quality Lab: Recall@1/3, MRR, and decision accuracy 100%, with zero unsafe, + over-refused, or failed cases; the recommended local-overlap policy measured + P50 0.010 ms and P95 0.168 ms in this deterministic run; +- OCR contract: 6 cases, 5 accepted, 1.33% average CER, and 100% Block-kind, + page-count, table-cell, and low-quality-detection accuracy; +- triage: category, priority, and queue accuracy 100%, with zero dangerous + under-prioritization or account-security misroutes; +- Playwright API/Web: 48/48 passed; +- production TypeScript/Vite build, dictionary/fixture/package JSON parsing, + YAML parsing, and `git diff --check`: passed. + +The pinned-Qdrant GitHub Actions job passed against Qdrant `1.18.2` and +reported this deliberately small two-case smoke comparison: + +| Backend | Recall@1 | P50 request latency | P95 request latency | +| --- | ---: | ---: | ---: | +| Memory | 100% | 0.004 ms | 0.066 ms | +| Qdrant REST | 100% | 3.035 ms | 3.598 ms | + +The Qdrant P95 was 3.532 ms higher (5351.52%) than the process-local memory +baseline. This is expected to trigger the explicit >25% activation warning. +It validates the warning path and local-network REST boundary; two cases and +20 searches are not a representative production capacity benchmark. + +Quality Lab activation acceptance requires: + +- zero unsafe answers; +- no regression in over-refusal, decision accuracy, Recall@3, or MRR; +- the built-in baseline and current-knowledge datasets; +- unchanged knowledge fingerprint and active policy; +- explicit confirmation when P95 latency rises by more than 25%. + +## Visual evidence + +The 13-second release demo and screenshots use deterministic, project-contract +admin responses so the ready → gate → active → rollback states are repeatable. +The separate pinned-Qdrant CI job is the evidence for real REST collection, +payload, filtering, alias-switch, health, stats, and delete behavior. + +| Quality backend targets | Ready index | Activation gate | +| --- | --- | --- | +| ![Quality Lab memory and Qdrant backend targets](assets/v0.3.2-quality-backends.png) | ![Ready Qdrant index job](assets/v0.3.2-index-ready.png) | ![Quality gate with explicit latency acknowledgement](assets/v0.3.2-activation-gate.png) | + +| Retrieval trace | Mobile dark mode | Failure state | +| --- | --- | --- | +| ![Eight-stage retrieval trace timeline](assets/v0.3.2-retrieval-trace-desktop.png) | ![Responsive Retrieval Operations page in dark mode](assets/v0.3.2-retrieval-ops-mobile-dark.png) | ![Retrieval Operations API failure state](assets/v0.3.2-ops-error.png) | + +The activation confirmation uses a real semantic `button` even while disabled, +so its accessible name remains available to keyboard and assistive-technology +checks. Desktop, mobile, bilingual, theme, focus, activation, rollback, and +timeline behaviors are covered by Playwright; screenshots alone are not used +as accessibility evidence. + +## Known limits and deployment risks + +- Docker is not installed in the local verification environment, so + `docker compose config` and the local pinned-Qdrant integration command + could not be run there. The dedicated GitHub Actions service job passed + against Qdrant `1.18.2`; local Compose parsing remains unverified by Docker. +- The backend provider is deployment configuration and requires restart. + The operations page does not edit Qdrant URLs or API keys. +- Qdrant failure degrades to keyword/structured retrieval; it does not + automatically change the configured provider or rebuild memory vectors. +- SQLite FAQ writes remain successful if post-commit Qdrant synchronization + fails; the safe degraded state is observable and stale points cannot hydrate. +- The index scheduler is single-process and SQLite-backed. It provides + checkpoints and interruption recovery, not distributed leases. +- Old collections are intentionally retained. Snapshotting, automatic cleanup, + clustering, and disaster recovery remain deployment responsibilities. +- Trace storage is application-local SQLite observability, not an + OpenTelemetry backend, and deliberately omits copied customer questions, + candidate content, credentials, and raw Qdrant responses. +- Qdrant sparse/hybrid retrieval, production-traffic shadow sampling, automatic + failover, and Agentic Retrieval remain out of scope. diff --git a/docs/releases/v0.3.2.md b/docs/releases/v0.3.2.md new file mode 100644 index 0000000..2b70081 --- /dev/null +++ b/docs/releases/v0.3.2.md @@ -0,0 +1,93 @@ +# v0.3.2 — Qdrant & Retrieval Observability + +v0.3.2 adds an optional production-oriented Qdrant vector backend and a +retrieval operations loop without changing the fresh-clone default. SQLite +remains the knowledge system of record, the in-memory index remains the +zero-infrastructure default, and the existing chat SSE contract is unchanged. + +## Product outcome + +Administrators now have a bilingual **Retrieval Operations** workspace where +they can: + +- inspect the configured backend, Qdrant health, active alias/collection, + vector count, dimension, and synchronization state; +- build a versioned Qdrant collection from current SQLite knowledge and resume + an interrupted job when its knowledge fingerprint is unchanged; +- compare memory and a ready Qdrant index in Quality Lab with the same dataset, + policy, and query embeddings; +- activate an eligible collection through an atomic alias switch, explicitly + acknowledge a P95 latency warning above 25%, and roll the alias back to the + previous verified collection; +- filter retrieval traces and inspect the fixed eight-stage timeline without + storing duplicate customer questions or knowledge content. + +## Engineering boundary + +- `VectorStore` is asynchronous. `InMemoryVectorStore` preserves the local + path; `QdrantVectorStore` uses `@qdrant/js-client-rest` `1.18.0`. +- Qdrant point IDs are stable UUIDv5 values. Payloads contain only knowledge + ID, knowledge type, version, and embedding profile. +- Qdrant candidates are batch-hydrated from SQLite and rejected when the + knowledge is missing, disabled, stale, or attached to an inactive source. +- Qdrant timeout or unavailability produces a `degraded` trace and keeps + keyword/structured recall available. It never silently rebuilds an + in-memory vector index. +- Index jobs use fixed states and checkpoints. Readiness verifies profile, + dimension, point count, and the current knowledge fingerprint. +- Activation and rollback require idempotency keys and expected-current + collection values. Old collections are not automatically deleted. +- Retrieval traces retain safe stage metadata for 30 days by default, cascade + with deleted sessions, and are cleaned at startup and daily. + +## Running Qdrant + +The default command still starts the memory-backed application: + +```bash +docker compose up --build +``` + +Start the pinned Qdrant `1.18.2` Compose profile and select it at deployment +time: + +```bash +VECTOR_STORE_PROVIDER=qdrant \ +QDRANT_URL=http://qdrant:6333 \ +docker compose --profile qdrant up --build +``` + +Changing `VECTOR_STORE_PROVIDER` requires an application restart. The admin +workspace can switch the configured collection alias, but it cannot change +infrastructure credentials or the primary backend. + +## Non-goals and limits + +This release does not add runtime memory/Qdrant switching, automatic backend +failover, production-traffic shadow queries, Qdrant sparse/hybrid retrieval, +snapshots, clustering, automatic old-collection deletion, OpenTelemetry, or +Agentic Retrieval. + +See [v0.3.2 implementation evidence](v0.3.2-evidence.md) for the verification +matrix, screenshots, benchmark results, and local environment limitation. + +--- + +## 中文说明 + +v0.3.2 在不改变 fresh-clone 默认行为、SQLite 权威数据源和聊天 SSE 契约的 +前提下，增加可选生产向 Qdrant 向量后端与完整检索运维闭环。 + +管理员可以在独立的双语“检索运维”页面查看后端健康、alias/collection、 +点数、维度和同步状态；从 SQLite 构建可恢复的版本化索引；在 Quality Lab +中用同一数据集、策略和查询 embedding 对比 memory/Qdrant；通过质量门禁后 +原子激活 alias，并在需要时回滚到上一已验证 collection；还可以按状态、后端、 +会话和时间筛选检索 Trace，查看固定八阶段时间线。 + +Qdrant 命中必须回查 SQLite，孤儿、旧版本、停用知识或失效来源不会成为证据。 +Qdrant 超时或不可用时会记录 `degraded` Trace，并继续关键词/结构化检索， +不会静默重建内存向量。部署配置决定主后端并在重启后生效；后台只能切换 +collection alias，不能修改凭据或运行时切换后端。 + +本版本不包含自动 failover、真实客服流量影子查询、Qdrant sparse/hybrid、 +快照/集群、旧 collection 自动清理、OpenTelemetry 或 Agentic Retrieval。 diff --git a/package-lock.json b/package-lock.json index f6f6651..3de0a8c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,14 +1,15 @@ { "name": "resolve-weave", - "version": "0.3.1", + "version": "0.3.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "resolve-weave", - "version": "0.3.1", + "version": "0.3.2", "license": "MIT", "dependencies": { + "@qdrant/js-client-rest": "1.18.0", "@xmldom/xmldom": "^0.8.13", "bcrypt": "^5.1.1", "better-sqlite3": "^11.7.0", @@ -982,9 +983,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1001,9 +999,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1020,9 +1015,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1039,9 +1031,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1058,9 +1047,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1156,6 +1142,33 @@ "url": "https://opencollective.com/popperjs" } }, + "node_modules/@qdrant/js-client-rest": { + "version": "1.18.0", + "resolved": "https://registry.npmmirror.com/@qdrant/js-client-rest/-/js-client-rest-1.18.0.tgz", + "integrity": "sha512-/0dqX5uV9chC1DnYSnU4gNMrDqse/pt6hHg3Rqqpl5isH7xl1xSNvffjzBoxycDD79luWn7Ho6Rh/61sOs5DNw==", + "license": "Apache-2.0", + "dependencies": { + "@qdrant/openapi-typescript-fetch": "1.2.6", + "undici": "^6.24.0" + }, + "engines": { + "node": ">=18.17.0", + "pnpm": ">=8" + }, + "peerDependencies": { + "typescript": ">=4.7" + } + }, + "node_modules/@qdrant/openapi-typescript-fetch": { + "version": "1.2.6", + "resolved": "https://registry.npmmirror.com/@qdrant/openapi-typescript-fetch/-/openapi-typescript-fetch-1.2.6.tgz", + "integrity": "sha512-oQG/FejNpItrxRHoyctYvT3rwGZOnK4jr3JdppO/c78ktDvkWiPXPHNsrDf33K9sZdRb6PR7gi4noIapu5q4HA==", + "license": "MIT", + "engines": { + "node": ">=18.0.0", + "pnpm": ">=8" + } + }, "node_modules/@remix-run/router": { "version": "1.23.3", "resolved": "https://registry.npmmirror.com/@remix-run/router/-/router-1.23.3.tgz", @@ -7623,7 +7636,6 @@ "version": "5.9.3", "resolved": "https://registry.npmmirror.com/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -7639,6 +7651,15 @@ "integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==", "license": "MIT" }, + "node_modules/undici": { + "version": "6.28.0", + "resolved": "https://registry.npmmirror.com/undici/-/undici-6.28.0.tgz", + "integrity": "sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, "node_modules/undici-types": { "version": "8.3.0", "resolved": "https://registry.npmmirror.com/undici-types/-/undici-types-8.3.0.tgz", diff --git a/package.json b/package.json index f752add..e8d6452 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "resolve-weave", - "version": "0.3.1", + "version": "0.3.2", "private": true, "license": "MIT", "description": "Evidence-first open-source enterprise customer service platform", @@ -12,7 +12,8 @@ "start": "node --use-bundled-ca dist/server/index.js", "db:init": "tsx server/db/index.ts", "db:seed": "tsx server/db/seed.ts", - "test": "DB_PATH=./data/chat-route-test.db JWT_SECRET=test-secret-123 tsx server/tests/chat-route.test.ts && tsx server/tests/idempotency.test.ts && tsx server/tests/grounding-policy.test.ts && tsx server/tests/quality-evaluator.test.ts && tsx server/tests/quality-lab.test.ts && tsx server/tests/escalation-triage.test.ts && tsx server/tests/escalation-api.test.ts && tsx server/tests/llm-retry.test.ts && tsx server/tests/server-lifecycle.test.ts && DB_PATH=./data/regression-test.db JWT_SECRET=test-secret-123 tsx server/tests/knowledge-review.test.ts && tsx server/tests/document-ir.test.ts && tsx server/tests/document-parser.test.ts && tsx server/tests/document-quality.test.ts && tsx server/tests/document-chunker.test.ts && tsx server/tests/document-migration.test.ts && tsx server/tests/ocr-review-contract.test.ts && tsx server/tests/document-ocr.test.ts && tsx server/tests/ocr-evaluator.test.ts && JWT_SECRET=test-secret-123 tsx server/tests/document-rag.test.ts && JWT_SECRET=test-secret-123 tsx server/tests/knowledge-retriever.test.ts && DB_PATH=./data/conversation-lifecycle-test.db JWT_SECRET=test-secret-123 tsx server/tests/conversation-lifecycle.test.ts && tsx server/tests/prompt-knowledge.test.ts && tsx server/tests/intent-classifier.test.ts && tsx server/tests/document-api.test.ts && tsx server/tests/model-config-security.test.ts && tsx server/tests/regression.test.ts && tsc -p server/tsconfig.json --noEmit && tsc -p tsconfig.json --noEmit", + "test": "tsx server/tests/vector-store.test.ts && tsx server/tests/qdrant-vector-store.test.ts && tsx server/tests/vector-store-config.test.ts && tsx server/tests/retrieval-index-job.test.ts && tsx server/tests/retrieval-trace.test.ts && tsx server/tests/retrieval-api.test.ts && DB_PATH=./data/chat-route-test.db JWT_SECRET=test-secret-123 tsx server/tests/chat-route.test.ts && tsx server/tests/idempotency.test.ts && tsx server/tests/grounding-policy.test.ts && tsx server/tests/quality-evaluator.test.ts && tsx server/tests/quality-lab.test.ts && tsx server/tests/escalation-triage.test.ts && tsx server/tests/escalation-api.test.ts && tsx server/tests/llm-retry.test.ts && tsx server/tests/server-lifecycle.test.ts && DB_PATH=./data/regression-test.db JWT_SECRET=test-secret-123 tsx server/tests/knowledge-review.test.ts && tsx server/tests/document-ir.test.ts && tsx server/tests/document-parser.test.ts && tsx server/tests/document-quality.test.ts && tsx server/tests/document-chunker.test.ts && tsx server/tests/document-migration.test.ts && tsx server/tests/ocr-review-contract.test.ts && tsx server/tests/document-ocr.test.ts && tsx server/tests/ocr-evaluator.test.ts && JWT_SECRET=test-secret-123 tsx server/tests/document-rag.test.ts && JWT_SECRET=test-secret-123 tsx server/tests/knowledge-retriever.test.ts && DB_PATH=./data/conversation-lifecycle-test.db JWT_SECRET=test-secret-123 tsx server/tests/conversation-lifecycle.test.ts && tsx server/tests/prompt-knowledge.test.ts && tsx server/tests/intent-classifier.test.ts && tsx server/tests/document-api.test.ts && tsx server/tests/model-config-security.test.ts && tsx server/tests/regression.test.ts && tsc -p server/tsconfig.json --noEmit && tsc -p tsconfig.json --noEmit", + "test:qdrant": "tsx server/tests/qdrant-integration.test.ts", "eval:faq": "tsx server/eval/faq-eval.ts", "eval:document": "tsx server/eval/document-eval.ts", "eval:mixed": "REPORT_MIXED_EVAL=1 JWT_SECRET=test-secret-123 tsx server/tests/knowledge-retriever.test.ts", @@ -24,6 +25,7 @@ "test:e2e:dev-server": "tsx tests/e2e/setup-e2e-db.ts && npm run dev" }, "dependencies": { + "@qdrant/js-client-rest": "1.18.0", "@xmldom/xmldom": "^0.8.13", "bcrypt": "^5.1.1", "better-sqlite3": "^11.7.0", diff --git a/server/ai/knowledge-adapters.ts b/server/ai/knowledge-adapters.ts index 81b5fe9..6cec3da 100644 --- a/server/ai/knowledge-adapters.ts +++ b/server/ai/knowledge-adapters.ts @@ -8,6 +8,7 @@ import { KnowledgeIndexItem, KnowledgeIndexLoad, } from './knowledge-retriever'; +import type { VectorSearchResult } from './vector-store'; import { DOCUMENT_EMBEDDING_INPUT_VERSION, FAQ_EMBEDDING_INPUT_VERSION, @@ -147,6 +148,17 @@ export class FaqKnowledgeAdapter implements KnowledgeAdapter { .sort((left, right) => right.similarity - left.similarity); } + async hydrateVectorMatches( + matches: VectorSearchResult[], + ): Promise> { + return new Map(this.repo.findActiveByIds( + matches.map((match) => match.id.replace(/^faq:/, '')), + ).map((entry) => { + const item = this.toIndexItem(entry); + return [item.id, item]; + })); + } + toIndexItem(entry: FaqEntry): KnowledgeIndexItem { return { id: `faq:${entry.id}`, @@ -158,6 +170,7 @@ export class FaqKnowledgeAdapter implements KnowledgeAdapter { similarity: 0, }, embedding: entry.embedding ?? [], + revision: entry.updatedAt, }; } } @@ -234,6 +247,7 @@ export class DocumentKnowledgeAdapter implements KnowledgeAdapter { extractionEngineVersion: chunk.extractionEngineVersion ?? undefined, }, embedding: chunk.embedding, + revision: chunk.createdAt, }; } @@ -267,6 +281,17 @@ export class DocumentKnowledgeAdapter implements KnowledgeAdapter { .sort((a, b) => b.similarity - a.similarity) .slice(0, limit); } + + async hydrateVectorMatches( + matches: VectorSearchResult[], + ): Promise> { + return new Map(this.repo.findActiveKnowledgeChunksByIds( + matches.map((match) => match.id.replace(/^document:/, '')), + ).map((chunk) => { + const item = this.toIndexItem(chunk, chunk.documentTitle); + return [item.id, item]; + })); + } } export function documentKeywordTerms(query: string): string[] { diff --git a/server/ai/knowledge-retriever.ts b/server/ai/knowledge-retriever.ts index 3f92cd6..c8716b8 100644 --- a/server/ai/knowledge-retriever.ts +++ b/server/ai/knowledge-retriever.ts @@ -1,15 +1,17 @@ import { v4 as uuidv4 } from 'uuid'; import { KnowledgeType, RetrievalResult } from '../types/ai'; import { logger } from '../utils/logger'; -import { VectorStore } from './vector-store'; +import { VectorRecord, VectorSearchResult, VectorStore } from './vector-store'; import { expandRetrievalQuery } from './query-expansion'; import { rankRetrievalResults } from './retrieval-ranking'; import type { RetrievalPolicyConfig } from '../types/quality'; +import type { RetrievalTraceCollector } from '../services/retrieval-trace-collector'; export interface KnowledgeIndexItem { id: string; result: RetrievalResult; embedding: number[]; + revision?: string; } export interface KnowledgeIndexLoad { @@ -21,6 +23,7 @@ export interface KnowledgeAdapter { readonly knowledgeType: KnowledgeType; getEmbeddingProfile?(): string; loadIndexItems(): Promise; + hydrateVectorMatches?(matches: VectorSearchResult[]): Promise>; searchKeyword(query: string, limit: number): RetrievalResult[] | Promise; } @@ -42,7 +45,7 @@ export class KnowledgeRetriever { private readonly refreshPromises = new Map>(); constructor( - private readonly vectorStore: VectorStore, + private readonly vectorStore: VectorStore, private readonly embedTexts: (texts: string[]) => Promise, private readonly adapters: KnowledgeAdapter[], ) {} @@ -124,8 +127,13 @@ export class KnowledgeRetriever { nextItems.set(item.id, item); } try { - for (const id of previousItems.keys()) this.vectorStore.delete(id); - for (const item of nextItems.values()) this.vectorStore.upsert(item, item.embedding); + if (this.vectorStore.supportsStartupSync) { + await this.vectorStore.delete([...previousItems.keys()], operationId); + await this.vectorStore.upsertBatch( + [...nextItems.values()].map((item) => this.toVectorRecord(item)), + operationId, + ); + } } catch (applyError) { if (!Array.isArray(loaded) && loaded.rollbackPersisted) { try { @@ -139,8 +147,13 @@ export class KnowledgeRetriever { } } try { - for (const id of nextItems.keys()) this.vectorStore.delete(id); - for (const item of previousItems.values()) this.vectorStore.upsert(item, item.embedding); + if (this.vectorStore.supportsStartupSync) { + await this.vectorStore.delete([...nextItems.keys()], operationId); + await this.vectorStore.upsertBatch( + [...previousItems.values()].map((item) => this.toVectorRecord(item)), + operationId, + ); + } } catch (rollbackError) { logger.error({ operationId, @@ -174,29 +187,53 @@ export class KnowledgeRetriever { topK: number = 5, knowledgeTypes: KnowledgeType[] = ['faq', 'document'], policy?: RetrievalPolicyConfig, + trace?: RetrievalTraceCollector, ): Promise { await this.initialize(); + const expandStarted = performance.now(); const expandedQuery = expandRetrievalQuery(query); + trace?.record('query_expand', { + status: 'completed', + latencyMs: performance.now() - expandStarted, + inputCount: 1, + outputCount: 1, + }); const candidateLimit = Math.min( MAX_CANDIDATE_POOL, Math.max(MIN_CANDIDATE_POOL, topK * CANDIDATE_MULTIPLIER), ); - const queryEmbedding = await this.embedQueries([expandedQuery]); + const queryEmbedding = await this.embedQueries([expandedQuery], trace); const candidates = await this.retrieveCandidates( query, expandedQuery, queryEmbedding[0], candidateLimit, knowledgeTypes, + trace, ); - return rankRetrievalResults({ + const rerankStarted = performance.now(); + const ranked = rankRetrievalResults({ query, candidates, topK, knowledgeTypes, policy, }); + trace?.record('rerank', { + status: 'completed', + latencyMs: performance.now() - rerankStarted, + inputCount: candidates.length, + outputCount: ranked.length, + candidates: ranked.map((result, index) => ({ + knowledgeType: result.knowledgeType, + knowledgeId: result.knowledgeId, + score: result.rerankScore ?? result.fusionScore ?? result.similarity, + rank: index + 1, + source: result.source, + })), + }); + return ranked; } async searchCandidatesBatch( @@ -218,7 +255,28 @@ export class KnowledgeRetriever { ))); } - stats(): ReturnType['stats']> { + async searchCandidatesBatchWithEmbeddings( + queries: string[], + embeddings: Array, + limit: number = MAX_CANDIDATE_POOL, + knowledgeTypes: KnowledgeType[] = ['faq', 'document'], + ): Promise { + if (queries.length !== embeddings.length) { + throw new Error('Query and embedding batches must have the same length'); + } + await this.initialize(); + const expanded = queries.map(expandRetrievalQuery); + const candidateLimit = Math.min(MAX_CANDIDATE_POOL, Math.max(1, limit)); + return Promise.all(queries.map((query, index) => this.retrieveCandidates( + query, + expanded[index], + embeddings[index], + candidateLimit, + knowledgeTypes, + ))); + } + + stats(): ReturnType { return this.vectorStore.stats(); } @@ -234,8 +292,8 @@ export class KnowledgeRetriever { return this.initialized; } - upsertIndexItem(item: KnowledgeIndexItem): void { - this.vectorStore.upsert(item, item.embedding); + async upsertIndexItem(item: KnowledgeIndexItem): Promise { + await this.vectorStore.upsertBatch([this.toVectorRecord(item)]); const items = this.indexedItems.get(item.result.knowledgeType) ?? new Map(); items.set(item.id, item); this.indexedItems.set(item.result.knowledgeType, items); @@ -244,7 +302,10 @@ export class KnowledgeRetriever { this.indexedIds.set(item.result.knowledgeType, ids); } - replaceDocumentIndexItems(documentId: string, nextItems: KnowledgeIndexItem[]): void { + async replaceDocumentIndexItems( + documentId: string, + nextItems: KnowledgeIndexItem[], + ): Promise { const knowledgeType: KnowledgeType = 'document'; const currentItems = this.indexedItems.get(knowledgeType) ?? new Map(); const previousDocumentItems = [...currentItems.values()].filter( @@ -256,14 +317,14 @@ export class KnowledgeRetriever { throw new Error('Replacement document index items must use the target document namespace'); } try { - for (const item of previousDocumentItems) this.vectorStore.delete(item.id); - for (const item of nextItems) this.vectorStore.upsert(item, item.embedding); + await this.vectorStore.delete(previousDocumentItems.map((item) => item.id)); + await this.vectorStore.upsertBatch(nextItems.map((item) => this.toVectorRecord(item))); } catch (error) { try { - for (const item of nextItems) this.vectorStore.delete(item.id); - for (const item of previousDocumentItems) { - this.vectorStore.upsert(item, item.embedding); - } + await this.vectorStore.delete(nextItems.map((item) => item.id)); + await this.vectorStore.upsertBatch( + previousDocumentItems.map((item) => this.toVectorRecord(item)), + ); } catch (rollbackError) { logger.error({ documentId, @@ -279,8 +340,8 @@ export class KnowledgeRetriever { this.indexedIds.set(knowledgeType, new Set(replaced.keys())); } - deleteIndexItem(knowledgeType: KnowledgeType, namespacedId: string): void { - this.vectorStore.delete(namespacedId); + async deleteIndexItem(knowledgeType: KnowledgeType, namespacedId: string): Promise { + await this.vectorStore.delete([namespacedId]); this.indexedItems.get(knowledgeType)?.delete(namespacedId); this.indexedIds.get(knowledgeType)?.delete(namespacedId); } @@ -293,15 +354,44 @@ export class KnowledgeRetriever { return weight / (RRF_RANK_CONSTANT + rank); } - private async embedQueries(queries: string[]): Promise> { - if (this.vectorStore.stats().indexedCount === 0) return queries.map(() => undefined); + private async embedQueries( + queries: string[], + trace?: RetrievalTraceCollector, + ): Promise> { + const started = performance.now(); try { - return await this.embedTexts(queries); + if ((await this.vectorStore.stats()).indexedCount === 0) { + trace?.record('embedding', { + status: 'completed', + latencyMs: performance.now() - started, + inputCount: queries.length, + outputCount: 0, + }); + return queries.map(() => undefined); + } + const embeddings = await this.embedTexts(queries); + trace?.record('embedding', { + status: 'completed', + latencyMs: performance.now() - started, + inputCount: queries.length, + outputCount: embeddings.length, + budget: { dimensions: embeddings[0]?.length ?? 0 }, + }); + return embeddings; } catch (error) { logger.warn({ errorName: error instanceof Error ? error.name : 'UnknownError', queryCount: queries.length, - }, 'Knowledge vector query batch failed; using keyword fallback'); + }, 'Knowledge vector preparation failed; using keyword fallback'); + trace?.record('embedding', { + status: 'degraded', + latencyMs: performance.now() - started, + inputCount: queries.length, + outputCount: 0, + errorCode: this.vectorStore.backend === 'qdrant' + ? 'qdrant_unavailable' + : 'embedding_unavailable', + }); return queries.map(() => undefined); } } @@ -312,23 +402,64 @@ export class KnowledgeRetriever { queryEmbedding: number[] | undefined, candidateLimit: number, knowledgeTypes: KnowledgeType[], + trace?: RetrievalTraceCollector, ): Promise { - const operationId = uuidv4(); + const operationId = trace?.id ?? uuidv4(); const allowed = new Set(knowledgeTypes); const merged = new Map(); if (queryEmbedding) { - const vectorCandidates = [...allowed].flatMap((knowledgeType) => ( - this.vectorStore.search( - queryEmbedding, - candidateLimit, - (entry) => entry.result.knowledgeType === knowledgeType, - ) - )).sort((left, right) => right.score - left.score); + const vectorStarted = performance.now(); + let vectorCandidates: Awaited> = []; + let vectorStatus: 'completed' | 'degraded' = 'completed'; + let vectorErrorCode: string | null = null; + try { + vectorCandidates = (await Promise.all([...allowed].map((knowledgeType) => ( + this.vectorStore.search(queryEmbedding, { + limit: candidateLimit, + knowledgeTypes: [knowledgeType], + traceId: operationId, + }) + )))).flat().sort((left, right) => right.score - left.score); + } catch (error) { + vectorStatus = 'degraded'; + vectorErrorCode = this.vectorStore.backend === 'qdrant' + ? 'qdrant_unavailable' + : 'vector_search_failed'; + logger.warn({ + operationId, + errorName: error instanceof Error ? error.name : 'UnknownError', + }, 'Knowledge vector search failed; using keyword fallback'); + } + trace?.record('vector_recall', { + status: vectorStatus, + latencyMs: performance.now() - vectorStarted, + inputCount: 1, + outputCount: vectorCandidates.length, + candidates: vectorCandidates.map((match, index) => ({ + knowledgeType: match.knowledgeType, + knowledgeId: match.id.replace(/^(faq|document):/, ''), + score: match.score, + rank: index + 1, + source: 'vector', + })), + errorCode: vectorErrorCode, + }); + const hydrated = await this.hydrateVectorCandidates(vectorCandidates, allowed, operationId); for (const [index, match] of vectorCandidates.entries()) { + const adapter = this.adapters.find((candidate) => ( + candidate.knowledgeType === match.knowledgeType + )); + const item = hydrated.get(match.id); + if ( + !adapter + || !item + || this.itemRevision(item) !== match.revision + || (adapter.getEmbeddingProfile?.() ?? 'legacy') !== match.embeddingProfile + ) continue; const vectorScore = match.score; const vectorRank = index + 1; - merged.set(this.resultKey(match.entry.result), { - ...match.entry.result, + merged.set(this.resultKey(item.result), { + ...item.result, similarity: vectorScore, source: 'vector', vectorScore, @@ -336,7 +467,17 @@ export class KnowledgeRetriever { fusionScore: this.rrfScore(vectorRank, VECTOR_RRF_WEIGHT), }); } + } else { + trace?.record('vector_recall', { + status: 'skipped', + latencyMs: 0, + inputCount: 0, + outputCount: 0, + errorCode: trace.backend === 'qdrant' ? 'qdrant_unavailable' : null, + }); } + const keywordStarted = performance.now(); + let keywordDegraded = false; const keywordLists = await Promise.all(this.adapters .filter((adapter) => allowed.has(adapter.knowledgeType)) .map(async (adapter) => { @@ -346,6 +487,7 @@ export class KnowledgeRetriever { candidateLimit, ); } catch (error) { + keywordDegraded = true; logger.warn({ operationId, knowledgeType: adapter.knowledgeType, @@ -354,6 +496,22 @@ export class KnowledgeRetriever { return []; } })); + const flattenedKeyword = keywordLists.flat(); + trace?.record('keyword_recall', { + status: keywordDegraded ? 'degraded' : 'completed', + latencyMs: performance.now() - keywordStarted, + inputCount: this.adapters.filter((adapter) => allowed.has(adapter.knowledgeType)).length, + outputCount: flattenedKeyword.length, + candidates: flattenedKeyword.map((result, index) => ({ + knowledgeType: result.knowledgeType, + knowledgeId: result.knowledgeId, + score: result.keywordScore ?? result.similarity, + rank: index + 1, + source: 'keyword', + })), + errorCode: keywordDegraded ? 'keyword_recall_partial' : null, + }); + const fusionStarted = performance.now(); for (const keywordResults of keywordLists) { for (const [index, result] of keywordResults.entries()) { const key = this.resultKey(result); @@ -378,7 +536,69 @@ export class KnowledgeRetriever { }); } } - return [...merged.values()]; + const fused = [...merged.values()]; + trace?.record('fusion', { + status: 'completed', + latencyMs: performance.now() - fusionStarted, + inputCount: (queryEmbedding ? 1 : 0) + flattenedKeyword.length, + outputCount: fused.length, + candidates: [...fused] + .sort((left, right) => (right.fusionScore ?? 0) - (left.fusionScore ?? 0)) + .map((result, index) => ({ + knowledgeType: result.knowledgeType, + knowledgeId: result.knowledgeId, + score: result.fusionScore ?? result.similarity, + rank: index + 1, + source: result.source, + })), + }); + return fused; + } + + private toVectorRecord(item: KnowledgeIndexItem): VectorRecord { + const adapter = this.adapters.find((candidate) => ( + candidate.knowledgeType === item.result.knowledgeType + )); + return { + id: item.id, + knowledgeType: item.result.knowledgeType, + revision: this.itemRevision(item), + embeddingProfile: adapter?.getEmbeddingProfile?.() ?? 'legacy', + embedding: item.embedding, + }; + } + + private async hydrateVectorCandidates( + matches: VectorSearchResult[], + allowed: Set, + operationId: string, + ): Promise> { + const hydrated = new Map(); + await Promise.all(this.adapters + .filter((adapter) => allowed.has(adapter.knowledgeType)) + .map(async (adapter) => { + const adapterMatches = matches.filter((match) => ( + match.knowledgeType === adapter.knowledgeType + )); + if (adapterMatches.length === 0) return; + try { + const items = adapter.hydrateVectorMatches + ? await adapter.hydrateVectorMatches(adapterMatches) + : this.indexedItems.get(adapter.knowledgeType) ?? new Map(); + for (const [id, item] of items) hydrated.set(id, item); + } catch (error) { + logger.warn({ + operationId, + knowledgeType: adapter.knowledgeType, + errorName: error instanceof Error ? error.name : 'UnknownError', + }, 'Knowledge vector matches could not be hydrated'); + } + })); + return hydrated; + } + + private itemRevision(item: KnowledgeIndexItem): string { + return item.revision ?? 'legacy'; } } diff --git a/server/ai/knowledge-system.ts b/server/ai/knowledge-system.ts index 8137ae7..2a29d33 100644 --- a/server/ai/knowledge-system.ts +++ b/server/ai/knowledge-system.ts @@ -1,17 +1,27 @@ import { getDatabase } from '../db'; +import { config } from '../config'; import { DocumentRepo } from '../db/repos/document.repo'; import { FaqRepo } from '../db/repos/faq.repo'; import { DocumentKnowledgeAdapter, FaqKnowledgeAdapter } from './knowledge-adapters'; import { KnowledgeIndexItem, KnowledgeRetriever } from './knowledge-retriever'; import { getLLMClient } from './llm-client'; import { InMemoryVectorStore } from './vector-store'; +import { createQdrantVectorStore } from './qdrant-vector-store'; const database = getDatabase(); export const faqKnowledgeAdapter = new FaqKnowledgeAdapter(new FaqRepo(database)); export const documentKnowledgeAdapter = new DocumentKnowledgeAdapter(new DocumentRepo(database)); +export const primaryVectorStore = config.vectorStore.provider === 'qdrant' + ? createQdrantVectorStore({ + url: config.vectorStore.qdrantUrl, + apiKey: config.vectorStore.qdrantApiKey, + timeoutMs: config.vectorStore.timeoutMs, + collectionAlias: config.vectorStore.collectionAlias, + }) + : new InMemoryVectorStore(); export const knowledgeRetriever = new KnowledgeRetriever( - new InMemoryVectorStore(), + primaryVectorStore, async (texts) => (await getLLMClient().embed(texts)).map((result) => result.embedding), [faqKnowledgeAdapter, documentKnowledgeAdapter], ); diff --git a/server/ai/qdrant-vector-store.ts b/server/ai/qdrant-vector-store.ts new file mode 100644 index 0000000..5e7ad32 --- /dev/null +++ b/server/ai/qdrant-vector-store.ts @@ -0,0 +1,240 @@ +import { QdrantClient, withHeaders } from '@qdrant/js-client-rest'; +import { v4 as uuidv4, v5 as uuidv5 } from 'uuid'; +import type { KnowledgeType } from '../types/ai'; +import type { + VectorRecord, + VectorSearchOptions, + VectorSearchResult, + VectorStore, + VectorStoreHealth, + VectorStoreStats, +} from './vector-store'; + +const POINT_ID_NAMESPACE = uuidv5('resolveweave-vector-points', uuidv5.URL); + +interface QdrantPoint { + id: string | number; + score?: number; + payload?: Record | null; +} + +interface QdrantCollectionInfo { + status?: string; + points_count?: number | null; + indexed_vectors_count?: number | null; + config?: { + params?: { + vectors?: unknown; + }; + }; +} + +export interface QdrantClientLike { + upsert(collection: string, payload: Record): Promise; + delete(collection: string, payload: Record): Promise; + query(collection: string, payload: Record): Promise<{ + points?: QdrantPoint[]; + }>; + getCollection(collection: string): Promise; +} + +export type QdrantHeaderRunner = ( + headers: Record, + operation: () => Promise, +) => Promise; + +export interface QdrantVectorStoreOptions { + client: QdrantClientLike; + collectionAlias: string; + runWithHeaders?: QdrantHeaderRunner; +} + +export class QdrantRequestError extends Error { + constructor(readonly code: 'qdrant_timeout' | 'qdrant_request_failed') { + super(code === 'qdrant_timeout' ? 'Qdrant request timed out' : 'Qdrant request failed'); + this.name = 'QdrantRequestError'; + } +} + +export class QdrantVectorStore implements VectorStore { + readonly backend = 'qdrant'; + readonly supportsStartupSync = false; + private readonly client: QdrantClientLike; + private readonly collectionAlias: string; + private readonly runWithHeaders: QdrantHeaderRunner; + private updatedAt: string | null = null; + + constructor(options: QdrantVectorStoreOptions) { + this.client = options.client; + this.collectionAlias = options.collectionAlias; + this.runWithHeaders = options.runWithHeaders ?? (async (headers, operation) => ( + withHeaders(headers, operation) + )); + } + + async upsertBatch(records: VectorRecord[], traceId?: string): Promise { + if (records.length === 0) return; + await this.withTrace(traceId, () => this.client.upsert(this.collectionAlias, { + wait: true, + ordering: 'medium', + points: records.map((record) => ({ + id: pointId(record.id), + vector: record.embedding, + payload: { + pointKey: record.id, + knowledgeType: record.knowledgeType, + revision: record.revision, + embeddingProfile: record.embeddingProfile, + }, + })), + })); + this.updatedAt = new Date().toISOString(); + } + + async delete(ids: string[], traceId?: string): Promise { + if (ids.length === 0) return; + await this.withTrace(traceId, () => this.client.delete(this.collectionAlias, { + wait: true, + ordering: 'medium', + points: ids.map(pointId), + })); + this.updatedAt = new Date().toISOString(); + } + + async search( + queryEmbedding: number[], + options: VectorSearchOptions, + ): Promise { + if (queryEmbedding.length === 0 || options.limit <= 0) return []; + const response = await this.withTrace(options.traceId, () => ( + this.client.query(this.collectionAlias, { + query: queryEmbedding, + limit: options.limit, + with_payload: ['pointKey', 'knowledgeType', 'revision', 'embeddingProfile'], + with_vector: false, + filter: options.knowledgeTypes?.length + ? { + must: [{ + key: 'knowledgeType', + match: { any: options.knowledgeTypes }, + }], + } + : undefined, + }) + )); + return (response.points ?? []).flatMap((point) => { + const payload = point.payload; + const knowledgeType = payload?.knowledgeType; + if ( + typeof payload?.pointKey !== 'string' + || (knowledgeType !== 'faq' && knowledgeType !== 'document') + || typeof payload.revision !== 'string' + || typeof payload.embeddingProfile !== 'string' + || typeof point.score !== 'number' + || !Number.isFinite(point.score) + ) return []; + return [{ + id: payload.pointKey, + knowledgeType: knowledgeType as KnowledgeType, + revision: payload.revision, + embeddingProfile: payload.embeddingProfile, + score: point.score, + }]; + }); + } + + async stats(traceId?: string): Promise { + const collection = await this.withTrace(traceId, () => ( + this.client.getCollection(this.collectionAlias) + )); + return { + indexedCount: collection.points_count ?? collection.indexed_vectors_count ?? 0, + embeddingDimensions: vectorDimensions(collection.config?.params?.vectors), + updatedAt: this.updatedAt, + }; + } + + async health(traceId?: string): Promise { + try { + const collection = await this.withTrace(traceId, () => ( + this.client.getCollection(this.collectionAlias) + )); + return { + backend: 'qdrant', + status: collection.status === 'red' ? 'degraded' : 'healthy', + checkedAt: new Date().toISOString(), + errorCode: collection.status === 'red' ? 'qdrant_collection_degraded' : null, + }; + } catch { + return { + backend: 'qdrant', + status: 'unavailable', + checkedAt: new Date().toISOString(), + errorCode: 'qdrant_unreachable', + }; + } + } + + private async withTrace(traceId: string | undefined, operation: () => Promise): Promise { + try { + return await this.runWithHeaders( + { 'x-request-id': traceId ?? uuidv4() }, + operation, + ); + } catch (error) { + if (error instanceof QdrantRequestError) throw error; + throw new QdrantRequestError(isTimeoutError(error) + ? 'qdrant_timeout' + : 'qdrant_request_failed'); + } + } +} + +export function createQdrantVectorStore(params: { + url: string; + apiKey: string; + timeoutMs: number; + collectionAlias: string; +}): QdrantVectorStore { + const client = new QdrantClient({ + url: params.url, + apiKey: params.apiKey || undefined, + timeout: params.timeoutMs, + checkCompatibility: false, + }); + return new QdrantVectorStore({ + client: client as QdrantClientLike, + collectionAlias: params.collectionAlias, + }); +} + +function isTimeoutError(error: unknown): boolean { + const name = error instanceof Error ? error.name.toLowerCase() : ''; + const code = typeof error === 'object' && error + ? String((error as { code?: unknown }).code ?? '').toLowerCase() + : ''; + return name.includes('timeout') + || name.includes('abort') + || code.includes('timeout') + || code === 'etimedout'; +} + +function pointId(key: string): string { + return uuidv5(key, POINT_ID_NAMESPACE); +} + +function vectorDimensions(vectors: unknown): number | null { + if (!vectors || typeof vectors !== 'object') return null; + const record = vectors as Record; + if (typeof record.size === 'number') return record.size; + for (const candidate of Object.values(record)) { + if ( + candidate + && typeof candidate === 'object' + && typeof (candidate as Record).size === 'number' + ) { + return (candidate as Record).size; + } + } + return null; +} diff --git a/server/ai/semantic-search.ts b/server/ai/semantic-search.ts index 428f67e..528444d 100644 --- a/server/ai/semantic-search.ts +++ b/server/ai/semantic-search.ts @@ -35,8 +35,10 @@ class SemanticSearch { this.lastError = this.initialized ? null : 'FAQ vector index is degraded; keyword fallback remains available'; } catch (error) { this.initialized = true; - this.lastError = error instanceof Error ? error.message : String(error); - logger.error({ err: error }, 'Failed to initialize semantic search index'); + this.lastError = 'FAQ vector index initialization failed; keyword fallback remains available'; + logger.error({ + errorName: error instanceof Error ? error.name : 'UnknownError', + }, 'Failed to initialize semantic search index'); } } @@ -45,9 +47,9 @@ class SemanticSearch { return this.getStatus(); } - getStatus(): FaqIndexStatus { + async getStatus(): Promise { const activeEntries = this.faqRepo.listAllActive(); - const stats = knowledgeRetriever.stats(); + const stats = await knowledgeRetriever.stats(); const isDegraded = knowledgeRetriever.getFailedSources().includes('faq'); return { initialized: knowledgeRetriever.hasInitialized() && !isDegraded, @@ -87,7 +89,7 @@ class SemanticSearch { query, topK, generatedAt: new Date().toISOString(), - indexStatus: this.getStatus(), + indexStatus: await this.getStatus(), matches: matches.map((match, index) => this.toDebugMatch(match, index)), }; } @@ -102,16 +104,16 @@ class SemanticSearch { }; } - commitPreparedIndex(entry: FaqEntry): void { - knowledgeRetriever.deleteIndexItem('faq', `faq:${entry.id}`); + async commitPreparedIndex(entry: FaqEntry): Promise { + await knowledgeRetriever.deleteIndexItem('faq', `faq:${entry.id}`); if (entry.isActive && entry.embedding) { - knowledgeRetriever.upsertIndexItem(faqKnowledgeAdapter.toIndexItem(entry)); + await knowledgeRetriever.upsertIndexItem(faqKnowledgeAdapter.toIndexItem(entry)); } } async updateIndex(entry: FaqEntry): Promise { if (!entry.isActive) { - knowledgeRetriever.deleteIndexItem('faq', `faq:${entry.id}`); + await this.deleteIndexItemSafely(entry.id); return; } try { @@ -121,21 +123,26 @@ class SemanticSearch { (current) => this.prepareIndex(current), ); if (!updated || !updated.isActive) { - knowledgeRetriever.deleteIndexItem('faq', `faq:${entry.id}`); + await knowledgeRetriever.deleteIndexItem('faq', `faq:${entry.id}`); return; } - knowledgeRetriever.upsertIndexItem(faqKnowledgeAdapter.toIndexItem(updated)); + await knowledgeRetriever.upsertIndexItem(faqKnowledgeAdapter.toIndexItem(updated)); } catch (error) { - knowledgeRetriever.deleteIndexItem('faq', `faq:${entry.id}`); - this.lastError = error instanceof Error ? error.message : String(error); - logger.warn({ err: error, entryId: entry.id }, 'Failed to update FAQ index entry'); + await this.deleteIndexItemSafely(entry.id); + this.lastError = 'FAQ vector index update failed; keyword fallback remains available'; + logger.warn({ + errorName: error instanceof Error ? error.name : 'UnknownError', + entryId: entry.id, + }, 'Failed to update FAQ index entry'); } } async updateIndexBatch(entries: FaqEntry[]): Promise { const active = entries.filter((entry) => entry.isActive); const currentProfile = currentEmbeddingProfile(FAQ_EMBEDDING_INPUT_VERSION); - for (const entry of entries) knowledgeRetriever.deleteIndexItem('faq', `faq:${entry.id}`); + for (const entry of entries) { + await this.deleteIndexItemSafely(entry.id); + } const updates: Array<{ id: string; embedding: number[]; @@ -149,12 +156,25 @@ class SemanticSearch { )); let generated: number[][] = []; if (missing.length > 0) { - const results = await getLLMClient().embed(missing.map(buildFaqEmbeddingText)); - generated = results.map((result) => result.embedding); + try { + const results = await getLLMClient().embed(missing.map(buildFaqEmbeddingText)); + generated = results.map((result) => result.embedding); + } catch (error) { + this.lastError = 'FAQ batch embedding failed; SQLite remains authoritative'; + logger.warn({ + errorName: error instanceof Error ? error.name : 'UnknownError', + batchSize: missing.length, + }, 'Failed to embed FAQ import batch'); + continue; + } } for (const [index, entry] of missing.entries()) { const embedding = generated[index]; - if (!embedding?.length) throw new Error('Embedding result was empty'); + if (!embedding?.length) { + this.lastError = 'FAQ batch embedding returned an empty item; SQLite remains authoritative'; + logger.warn({ entryId: entry.id }, 'FAQ import embedding result was empty'); + continue; + } updates.push({ id: entry.id, embedding, @@ -163,11 +183,47 @@ class SemanticSearch { }); } } - if (updates.length > 0) this.faqRepo.updateEmbeddings(updates); + if (updates.length > 0) { + try { + this.faqRepo.updateEmbeddings(updates); + } catch (error) { + this.lastError = 'FAQ batch embedding persistence failed; SQLite remains authoritative'; + logger.warn({ + errorName: error instanceof Error ? error.name : 'UnknownError', + updateCount: updates.length, + }, 'Failed to persist FAQ import embeddings'); + return; + } + } const refreshed = new Map(this.faqRepo.listAllActive().map((entry) => [entry.id, entry])); for (const entry of active) { const indexedEntry = refreshed.get(entry.id) ?? entry; - knowledgeRetriever.upsertIndexItem(faqKnowledgeAdapter.toIndexItem(indexedEntry)); + if (!indexedEntry.embedding?.length) continue; + await this.upsertIndexItemSafely(indexedEntry); + } + } + + private async deleteIndexItemSafely(entryId: string): Promise { + try { + await knowledgeRetriever.deleteIndexItem('faq', `faq:${entryId}`); + } catch (error) { + this.lastError = 'FAQ vector cleanup failed; SQLite remains authoritative'; + logger.warn({ + errorName: error instanceof Error ? error.name : 'UnknownError', + entryId, + }, 'Failed to clean up FAQ vector entry'); + } + } + + private async upsertIndexItemSafely(entry: FaqEntry): Promise { + try { + await knowledgeRetriever.upsertIndexItem(faqKnowledgeAdapter.toIndexItem(entry)); + } catch (error) { + this.lastError = 'FAQ vector synchronization failed; SQLite remains authoritative'; + logger.warn({ + errorName: error instanceof Error ? error.name : 'UnknownError', + entryId: entry.id, + }, 'Failed to synchronize FAQ vector entry'); } } diff --git a/server/ai/vector-store.ts b/server/ai/vector-store.ts index b822947..43ddcbc 100644 --- a/server/ai/vector-store.ts +++ b/server/ai/vector-store.ts @@ -1,70 +1,104 @@ -import { FaqEntry } from '../types/domain'; +import type { KnowledgeType } from '../types/ai'; -export interface VectorStoreItem { +export interface VectorRecord { id: string; - entry: T; + knowledgeType: KnowledgeType; + revision: string; + embeddingProfile: string; embedding: number[]; } -export interface VectorSearchResult extends VectorStoreItem { +export interface VectorSearchResult { + id: string; + knowledgeType: KnowledgeType; + revision: string; + embeddingProfile: string; score: number; } +export interface VectorSearchOptions { + limit: number; + knowledgeTypes?: KnowledgeType[]; + traceId?: string; +} + export interface VectorStoreStats { indexedCount: number; embeddingDimensions: number | null; updatedAt: string | null; } -export interface VectorStore { - upsert(entry: T, embedding: number[]): void; - delete(id: string): void; - search( - queryEmbedding: number[], - limit: number, - predicate?: (entry: T) => boolean, - ): VectorSearchResult[]; - stats(): VectorStoreStats; - clear(): void; +export interface VectorStoreHealth { + backend: 'memory' | 'qdrant'; + status: 'healthy' | 'degraded' | 'unavailable'; + checkedAt: string; + errorCode: string | null; } -export class InMemoryVectorStore implements VectorStore { - private items = new Map>(); +export interface VectorStore { + readonly backend: 'memory' | 'qdrant'; + readonly supportsStartupSync: boolean; + upsertBatch(records: VectorRecord[], traceId?: string): Promise; + delete(ids: string[], traceId?: string): Promise; + search(queryEmbedding: number[], options: VectorSearchOptions): Promise; + stats(traceId?: string): Promise; + health(traceId?: string): Promise; +} + +export class InMemoryVectorStore implements VectorStore { + readonly backend = 'memory'; + readonly supportsStartupSync = true; + private items = new Map(); private updatedAt: string | null = null; - upsert(entry: T, embedding: number[]): void { - if (embedding.length === 0) { - this.delete(entry.id); - return; + async upsertBatch(records: VectorRecord[], _traceId?: string): Promise { + for (const record of records) { + if (record.embedding.length === 0) { + this.items.delete(record.id); + continue; + } + this.items.set(record.id, { + ...record, + embedding: [...record.embedding], + }); } - this.items.set(entry.id, { id: entry.id, entry, embedding }); - this.updatedAt = new Date().toISOString(); + if (records.length > 0) this.updatedAt = new Date().toISOString(); } - delete(id: string): void { - if (this.items.delete(id)) this.updatedAt = new Date().toISOString(); + async delete(ids: string[], _traceId?: string): Promise { + let changed = false; + for (const id of ids) changed = this.items.delete(id) || changed; + if (changed) this.updatedAt = new Date().toISOString(); } - search( + async search( queryEmbedding: number[], - limit: number, - predicate: (entry: T) => boolean = () => true, - ): VectorSearchResult[] { - if (queryEmbedding.length === 0 || limit <= 0) return []; - const best: VectorSearchResult[] = []; + options: VectorSearchOptions, + ): Promise { + if (queryEmbedding.length === 0 || options.limit <= 0) return []; + const allowed = options.knowledgeTypes + ? new Set(options.knowledgeTypes) + : null; + const best: VectorSearchResult[] = []; for (const item of this.items.values()) { - if (!predicate(item.entry)) continue; - const result = { ...item, score: cosineSimilarity(queryEmbedding, item.embedding) }; + if (allowed && !allowed.has(item.knowledgeType)) continue; + const result = { + id: item.id, + knowledgeType: item.knowledgeType, + revision: item.revision, + embeddingProfile: item.embeddingProfile, + score: cosineSimilarity(queryEmbedding, item.embedding), + }; const insertAt = best.findIndex((candidate) => result.score > candidate.score); if (insertAt < 0) best.push(result); else best.splice(insertAt, 0, result); - if (best.length > limit) best.pop(); + if (best.length > options.limit) best.pop(); } return best; } - stats(): VectorStoreStats { - const firstItem = this.items.values().next().value as VectorStoreItem | undefined; + async stats(_traceId?: string): Promise { + const firstItem = this.items.values().next().value as VectorRecord | undefined; return { indexedCount: this.items.size, embeddingDimensions: firstItem?.embedding.length ?? null, @@ -72,9 +106,13 @@ export class InMemoryVectorStore implements }; } - clear(): void { - this.items.clear(); - this.updatedAt = new Date().toISOString(); + async health(_traceId?: string): Promise { + return { + backend: 'memory', + status: 'healthy', + checkedAt: new Date().toISOString(), + errorCode: null, + }; } } diff --git a/server/config.ts b/server/config.ts index 6feb89e..4c58c04 100644 --- a/server/config.ts +++ b/server/config.ts @@ -6,6 +6,8 @@ import fs from 'fs'; export const MODEL_PROVIDERS = ['openai', 'openai-compatible', 'other'] as const; export type ModelProvider = (typeof MODEL_PROVIDERS)[number]; export const OPENAI_API_BASE = 'https://api.openai.com/v1'; +export const VECTOR_STORE_PROVIDERS = ['memory', 'qdrant'] as const; +export type VectorStoreProvider = (typeof VECTOR_STORE_PROVIDERS)[number]; export function resolveModelApiBase(provider: ModelProvider, customApiBase: string): string { return provider === 'openai' ? OPENAI_API_BASE : customApiBase.trim(); @@ -36,6 +38,63 @@ interface ModelEnvironmentSource { OPENAI_EMBED_MODEL?: string; } +interface VectorStoreEnvironmentSource { + VECTOR_STORE_PROVIDER?: string; + QDRANT_URL?: string; + QDRANT_API_KEY?: string; + QDRANT_COLLECTION_PREFIX?: string; + QDRANT_COLLECTION_ALIAS?: string; + QDRANT_TIMEOUT_MS?: string | number; + RETRIEVAL_TRACE_RETENTION_DAYS?: string | number; +} + +export interface ResolvedVectorStoreEnvironment { + provider: VectorStoreProvider; + qdrantUrl: string; + qdrantApiKey: string; + collectionPrefix: string; + collectionAlias: string; + timeoutMs: number; + traceRetentionDays: number; +} + +export function resolveVectorStoreEnvironment( + source: VectorStoreEnvironmentSource, +): ResolvedVectorStoreEnvironment { + const provider = source.VECTOR_STORE_PROVIDER ?? 'memory'; + if (!(VECTOR_STORE_PROVIDERS as readonly string[]).includes(provider)) { + throw new Error('VECTOR_STORE_PROVIDER must be memory or qdrant'); + } + const qdrantUrl = (source.QDRANT_URL ?? '').trim().replace(/\/+$/, ''); + if (provider === 'qdrant' && !qdrantUrl) { + throw new Error('QDRANT_URL is required when VECTOR_STORE_PROVIDER=qdrant'); + } + if (qdrantUrl) { + let parsedUrl: URL; + try { + parsedUrl = new URL(qdrantUrl); + } catch { + throw new Error('QDRANT_URL must be a valid http or https URL'); + } + if (!['http:', 'https:'].includes(parsedUrl.protocol)) { + throw new Error('QDRANT_URL must use http or https'); + } + } + const timeoutMs = z.coerce.number().int().min(500).max(60_000) + .parse(source.QDRANT_TIMEOUT_MS ?? 5_000); + const traceRetentionDays = z.coerce.number().int().min(1).max(90) + .parse(source.RETRIEVAL_TRACE_RETENTION_DAYS ?? 30); + return { + provider: provider as VectorStoreProvider, + qdrantUrl, + qdrantApiKey: (source.QDRANT_API_KEY ?? '').trim(), + collectionPrefix: (source.QDRANT_COLLECTION_PREFIX ?? 'resolveweave_knowledge').trim(), + collectionAlias: (source.QDRANT_COLLECTION_ALIAS ?? 'resolveweave_knowledge_active').trim(), + timeoutMs, + traceRetentionDays, + }; +} + function resolveModelProvider(rawProvider: string | undefined, apiBase: string): ModelProvider { if ((MODEL_PROVIDERS as readonly string[]).includes(rawProvider ?? '')) { return rawProvider as ModelProvider; @@ -104,6 +163,13 @@ const envSchema = z.object({ OCR_SHADOW_SERVICE_URL: z.string().default(''), OCR_SHADOW_SERVICE_TOKEN: z.string().default(''), OCR_SHADOW_ENGINE_VERSION: z.string().min(1).max(80).default('2.0.0'), + VECTOR_STORE_PROVIDER: z.enum(VECTOR_STORE_PROVIDERS).default('memory'), + QDRANT_URL: z.string().default(''), + QDRANT_API_KEY: z.string().default(''), + QDRANT_COLLECTION_PREFIX: z.string().min(1).max(80).default('resolveweave_knowledge'), + QDRANT_COLLECTION_ALIAS: z.string().min(1).max(80).default('resolveweave_knowledge_active'), + QDRANT_TIMEOUT_MS: z.coerce.number().int().min(500).max(60_000).default(5_000), + RETRIEVAL_TRACE_RETENTION_DAYS: z.coerce.number().int().min(1).max(90).default(30), ALLOWED_ORIGINS: z.string().default('http://localhost:5173'), RATE_LIMIT_CHAT: z.coerce.number().int().positive().default(20), RATE_LIMIT_ADMIN: z.coerce.number().int().positive().default(100), @@ -121,6 +187,7 @@ if (!parsed.success) { const env = parsed.data; const modelEnvironment = resolveModelEnvironment(env); +const vectorStoreEnvironment = resolveVectorStoreEnvironment(env); if (env.NODE_ENV === 'production' && env.ADMIN_PASSWORD === 'admin123') { console.error('❌ ADMIN_PASSWORD must be changed from the default "admin123" in production.'); @@ -173,6 +240,7 @@ export const config = { shadowServiceToken: env.OCR_SHADOW_SERVICE_TOKEN.trim(), shadowEngineVersion: env.OCR_SHADOW_ENGINE_VERSION.trim(), }, + vectorStore: vectorStoreEnvironment, cors: { origins: env.ALLOWED_ORIGINS.split(',').map((s) => s.trim()), }, diff --git a/server/db/index.ts b/server/db/index.ts index f6a5e0b..7e87b48 100644 --- a/server/db/index.ts +++ b/server/db/index.ts @@ -493,6 +493,7 @@ export function initSchema(database: Database.Database): void { id TEXT PRIMARY KEY, dataset_version_ids TEXT NOT NULL, policy_grid TEXT NOT NULL, + backend_targets TEXT NOT NULL DEFAULT '[{"provider":"memory"}]', status TEXT NOT NULL CHECK(status IN ('queued', 'running', 'completed', 'failed', 'interrupted', 'cancelled', 'stale')), progress INTEGER NOT NULL DEFAULT 0, @@ -513,6 +514,7 @@ export function initSchema(database: Database.Database): void { CREATE TABLE IF NOT EXISTS quality_run_candidates ( run_id TEXT NOT NULL REFERENCES quality_runs(id) ON DELETE CASCADE, candidate_key TEXT NOT NULL, + backend_target TEXT NOT NULL DEFAULT '{"provider":"memory"}', policy_config TEXT NOT NULL, metrics TEXT NOT NULL, recommended INTEGER NOT NULL DEFAULT 0 CHECK(recommended IN (0, 1)), @@ -536,6 +538,80 @@ export function initSchema(database: Database.Database): void { CREATE INDEX IF NOT EXISTS idx_quality_case_results_run_failure ON quality_case_results(run_id, passed, case_id); + + CREATE TABLE IF NOT EXISTS retrieval_index_jobs ( + id TEXT PRIMARY KEY, + status TEXT NOT NULL CHECK(status IN ( + 'queued', 'running', 'interrupted', 'ready', 'active', + 'rolled_back', 'failed', 'stale' + )), + collection_name TEXT NOT NULL UNIQUE, + embedding_profile TEXT NOT NULL, + vector_dimension INTEGER NOT NULL CHECK(vector_dimension > 0), + knowledge_fingerprint TEXT NOT NULL, + expected_count INTEGER NOT NULL CHECK(expected_count >= 0), + completed_count INTEGER NOT NULL DEFAULT 0 CHECK(completed_count >= 0), + batch_checkpoint INTEGER NOT NULL DEFAULT 0 CHECK(batch_checkpoint >= 0), + previous_collection TEXT, + failure_code TEXT, + activation_intent TEXT CHECK(activation_intent IN ('activate', 'rollback')), + activation_expected_collection TEXT, + created_by TEXT NOT NULL, + created_at TEXT NOT NULL, + started_at TEXT, + ready_at TEXT, + activated_at TEXT, + rolled_back_at TEXT, + updated_at TEXT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_retrieval_index_jobs_status_created + ON retrieval_index_jobs(status, created_at DESC); + CREATE INDEX IF NOT EXISTS idx_retrieval_index_jobs_fingerprint + ON retrieval_index_jobs(knowledge_fingerprint, embedding_profile, created_at DESC); + + CREATE TABLE IF NOT EXISTS retrieval_traces ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + user_message_id TEXT NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + assistant_message_id TEXT REFERENCES messages(id) ON DELETE SET NULL, + policy_id TEXT NOT NULL, + backend TEXT NOT NULL CHECK(backend IN ('memory', 'qdrant')), + status TEXT NOT NULL CHECK(status IN ('completed', 'degraded', 'failed')), + error_code TEXT, + total_latency_ms REAL NOT NULL CHECK(total_latency_ms >= 0), + created_at TEXT NOT NULL, + completed_at TEXT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created + ON retrieval_traces(created_at DESC); + CREATE INDEX IF NOT EXISTS idx_retrieval_traces_status_created + ON retrieval_traces(status, created_at DESC); + CREATE INDEX IF NOT EXISTS idx_retrieval_traces_backend_created + ON retrieval_traces(backend, created_at DESC); + CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_created + ON retrieval_traces(session_id, created_at DESC); + + CREATE TABLE IF NOT EXISTS retrieval_trace_stages ( + trace_id TEXT NOT NULL REFERENCES retrieval_traces(id) ON DELETE CASCADE, + stage_name TEXT NOT NULL CHECK(stage_name IN ( + 'query_expand', 'embedding', 'vector_recall', 'keyword_recall', + 'fusion', 'rerank', 'context_budget', 'grounding' + )), + stage_order INTEGER NOT NULL, + status TEXT NOT NULL CHECK(status IN ('completed', 'degraded', 'failed', 'skipped')), + latency_ms REAL NOT NULL CHECK(latency_ms >= 0), + input_count INTEGER NOT NULL CHECK(input_count >= 0), + output_count INTEGER NOT NULL CHECK(output_count >= 0), + candidates TEXT NOT NULL DEFAULT '[]', + budget TEXT NOT NULL DEFAULT '{}', + error_code TEXT, + PRIMARY KEY(trace_id, stage_name) + ); + + CREATE INDEX IF NOT EXISTS idx_retrieval_trace_stages_trace + ON retrieval_trace_stages(trace_id, stage_order); `); // v0.2.6 security migration: model credentials are environment-injected only. @@ -572,6 +648,25 @@ export function initSchema(database: Database.Database): void { ensureColumn(database, 'document_processing_tasks', 'quality_reasons', "TEXT NOT NULL DEFAULT '[]'"); ensureColumn(database, 'document_extraction_jobs', 'result_block_count', 'INTEGER NOT NULL DEFAULT 0'); ensureColumn(database, 'document_extraction_jobs', 'result_warning_codes', "TEXT NOT NULL DEFAULT '[]'"); + ensureColumn( + database, + 'quality_runs', + 'backend_targets', + `TEXT NOT NULL DEFAULT '[{"provider":"memory"}]'`, + ); + ensureColumn( + database, + 'quality_run_candidates', + 'backend_target', + `TEXT NOT NULL DEFAULT '{"provider":"memory"}'`, + ); + ensureColumn( + database, + 'retrieval_index_jobs', + 'activation_intent', + "TEXT CHECK(activation_intent IN ('activate', 'rollback'))", + ); + ensureColumn(database, 'retrieval_index_jobs', 'activation_expected_collection', 'TEXT'); database.prepare(` UPDATE document_extraction_jobs SET result_block_count = CASE diff --git a/server/db/repos/document.repo.ts b/server/db/repos/document.repo.ts index 45019ff..b9fbaed 100644 --- a/server/db/repos/document.repo.ts +++ b/server/db/repos/document.repo.ts @@ -276,6 +276,23 @@ export class DocumentRepo { })); } + findActiveKnowledgeChunksByIds(ids: string[]): DocumentKnowledgeChunk[] { + const uniqueIds = [...new Set(ids)].slice(0, 100); + if (uniqueIds.length === 0) return []; + const placeholders = uniqueIds.map(() => '?').join(', '); + const rows = this.db.prepare(` + SELECT c.*, d.file_name AS document_title FROM document_chunks c + JOIN documents d ON d.id = c.document_id + WHERE c.id IN (${placeholders}) + AND d.status = 'ready' AND d.is_active = 1 + AND d.index_status IN ('legacy', 'published') + `).all(...uniqueIds) as Record[]; + return rows.map((row) => ({ + ...this.mapChunk(row), + documentTitle: row.document_title as string, + })); + } + searchActiveChunksLikeTerms(terms: string[], limit: number): DocumentKnowledgeChunk[] { const uniqueTerms = [...new Set(terms.map((term) => term.trim()).filter(Boolean))].slice(0, 24); if (uniqueTerms.length === 0) return []; diff --git a/server/db/repos/faq.repo.ts b/server/db/repos/faq.repo.ts index c8a80f3..1f36fee 100644 --- a/server/db/repos/faq.repo.ts +++ b/server/db/repos/faq.repo.ts @@ -211,6 +211,16 @@ export class FaqRepo { return rows.map((row) => this.mapRow(row)); } + findActiveByIds(ids: string[]): FaqEntry[] { + const uniqueIds = [...new Set(ids)].slice(0, 100); + if (uniqueIds.length === 0) return []; + const placeholders = uniqueIds.map(() => '?').join(', '); + const rows = this.db.prepare( + `SELECT * FROM faq_entries WHERE is_active = 1 AND id IN (${placeholders})`, + ).all(...uniqueIds) as Record[]; + return rows.map((row) => this.mapRow(row)); + } + listByCategory(category: IntentCategory): FaqEntry[] { const rows = this.listByCategoryStmt.all(category) as Record[]; return rows.map((row) => this.mapRow(row)); diff --git a/server/db/repos/quality-run.repo.ts b/server/db/repos/quality-run.repo.ts index 2be02ba..eae035c 100644 --- a/server/db/repos/quality-run.repo.ts +++ b/server/db/repos/quality-run.repo.ts @@ -5,6 +5,7 @@ import type { QualityCaseResult, QualityRun, QualityRunStatus, + QualityBackendTarget, RetrievalPolicyConfig, } from '../../types/quality'; @@ -12,6 +13,7 @@ interface RunRow { id: string; dataset_version_ids: string; policy_grid: string; + backend_targets: string; status: QualityRunStatus; progress: number; total_cases: number; @@ -27,6 +29,7 @@ interface RunRow { interface CandidateRow { candidate_key: string; + backend_target: string; policy_config: string; metrics: string; recommended: number; @@ -49,6 +52,7 @@ export class QualityRunRepo { create(params: { datasetVersionIds: string[]; policies: RetrievalPolicyConfig[]; + backendTargets: QualityBackendTarget[]; totalCases: number; knowledgeFingerprint: string | null; activePolicyId: string; @@ -58,14 +62,15 @@ export class QualityRunRepo { const id = uuidv4(); this.db.prepare( `INSERT INTO quality_runs ( - id, dataset_version_ids, policy_grid, status, progress, total_cases, + id, dataset_version_ids, policy_grid, backend_targets, status, progress, total_cases, knowledge_fingerprint, active_policy_id, failure_code, cancel_requested, created_by, created_at, started_at, completed_at - ) VALUES (?, ?, ?, 'queued', 0, ?, ?, ?, NULL, 0, ?, ?, NULL, NULL)`, + ) VALUES (?, ?, ?, ?, 'queued', 0, ?, ?, ?, NULL, 0, ?, ?, NULL, NULL)`, ).run( id, JSON.stringify(params.datasetVersionIds), JSON.stringify(params.policies), + JSON.stringify(params.backendTargets), params.totalCases, params.knowledgeFingerprint, params.activePolicyId, @@ -94,24 +99,41 @@ export class QualityRunRepo { candidateKey: caseRow.candidate_key, actualAnswerMode: caseRow.actual_answer_mode, actualGroundingStatus: caseRow.actual_grounding_status, - sources: JSON.parse(caseRow.sources) as QualityCaseResult['sources'], + sources: parseJson(caseRow.sources, []), latencyMs: caseRow.latency_ms, passed: Boolean(caseRow.passed), failureReason: caseRow.failure_reason, }); casesByCandidate.set(caseRow.candidate_key, cases); } - const candidates: QualityCandidateResult[] = candidateRows.map((candidate) => ({ - key: candidate.candidate_key, - policy: JSON.parse(candidate.policy_config) as RetrievalPolicyConfig, - metrics: JSON.parse(candidate.metrics) as QualityCandidateResult['metrics'], - recommended: Boolean(candidate.recommended), - cases: casesByCandidate.get(candidate.candidate_key) ?? [], - })); + const candidates: QualityCandidateResult[] = candidateRows.flatMap((candidate) => { + const backendTarget = parseJson( + candidate.backend_target, + null, + ); + const policy = parseJson(candidate.policy_config, null); + const metrics = parseJson( + candidate.metrics, + null, + ); + if (!backendTarget || !policy || !metrics) return []; + return [{ + key: candidate.candidate_key, + backendTarget, + policy, + metrics, + recommended: Boolean(candidate.recommended), + cases: casesByCandidate.get(candidate.candidate_key) ?? [], + }]; + }); return { id: row.id, - datasetVersionIds: JSON.parse(row.dataset_version_ids) as string[], - policies: JSON.parse(row.policy_grid) as RetrievalPolicyConfig[], + datasetVersionIds: parseJson(row.dataset_version_ids, []), + policies: parseJson(row.policy_grid, []), + backendTargets: parseJson( + row.backend_targets, + [{ provider: 'memory' }], + ), status: row.status, progress: row.progress, totalCases: row.total_cases, @@ -127,6 +149,28 @@ export class QualityRunRepo { }; } + getPolicyGrid(id: string): RetrievalPolicyConfig[] { + const row = this.db.prepare( + 'SELECT policy_grid FROM quality_runs WHERE id = ?', + ).get(id) as { policy_grid: string } | undefined; + return row ? parseJson(row.policy_grid, []) : []; + } + + findLatestCompletedCandidate(candidateKey: string): { + runId: string; + candidateKey: string; + } | null { + const row = this.db.prepare(` + SELECT candidate.run_id, candidate.candidate_key + FROM quality_run_candidates candidate + JOIN quality_runs run ON run.id = candidate.run_id + WHERE run.status = 'completed' AND candidate.candidate_key = ? + ORDER BY run.completed_at DESC + LIMIT 1 + `).get(candidateKey) as { run_id: string; candidate_key: string } | undefined; + return row ? { runId: row.run_id, candidateKey: row.candidate_key } : null; + } + list(limit: number = 50, offset: number = 0): QualityRun[] { const ids = this.db.prepare( 'SELECT id FROM quality_runs ORDER BY created_at DESC LIMIT ? OFFSET ?', @@ -165,8 +209,8 @@ export class QualityRunRepo { this.db.transaction(() => { const insertCandidate = this.db.prepare( `INSERT INTO quality_run_candidates ( - run_id, candidate_key, policy_config, metrics, recommended - ) VALUES (?, ?, ?, ?, ?)`, + run_id, candidate_key, backend_target, policy_config, metrics, recommended + ) VALUES (?, ?, ?, ?, ?, ?)`, ); const insertCase = this.db.prepare( `INSERT INTO quality_case_results ( @@ -178,6 +222,7 @@ export class QualityRunRepo { insertCandidate.run( id, candidate.key, + JSON.stringify(candidate.backendTarget), JSON.stringify(candidate.policy), JSON.stringify(candidate.metrics), candidate.recommended ? 1 : 0, @@ -241,3 +286,11 @@ export class QualityRunRepo { ).run(now).changes; } } + +function parseJson(value: string, fallback: T): T { + try { + return JSON.parse(value) as T; + } catch { + return fallback; + } +} diff --git a/server/db/repos/retrieval-index-job.repo.ts b/server/db/repos/retrieval-index-job.repo.ts new file mode 100644 index 0000000..aa2a0aa --- /dev/null +++ b/server/db/repos/retrieval-index-job.repo.ts @@ -0,0 +1,297 @@ +import Database from 'better-sqlite3'; +import { v4 as uuidv4 } from 'uuid'; +import type { + RetrievalIndexJob, + RetrievalIndexJobStatus, +} from '../../types/retrieval-ops'; + +interface JobRow { + id: string; + status: RetrievalIndexJobStatus; + collection_name: string; + embedding_profile: string; + vector_dimension: number; + knowledge_fingerprint: string; + expected_count: number; + completed_count: number; + batch_checkpoint: number; + previous_collection: string | null; + failure_code: string | null; + activation_intent: 'activate' | 'rollback' | null; + activation_expected_collection: string | null; + created_by: string; + created_at: string; + started_at: string | null; + ready_at: string | null; + activated_at: string | null; + rolled_back_at: string | null; + updated_at: string; +} + +export class RetrievalIndexJobRepo { + constructor(private readonly db: Database.Database) {} + + create(params: { + collection: string; + embeddingProfile: string; + vectorDimension: number; + knowledgeFingerprint: string; + expectedCount: number; + createdBy: string; + now: string; + }): RetrievalIndexJob { + const id = uuidv4(); + this.db.prepare(` + INSERT INTO retrieval_index_jobs ( + id, status, collection_name, embedding_profile, vector_dimension, + knowledge_fingerprint, expected_count, completed_count, batch_checkpoint, + previous_collection, failure_code, created_by, created_at, started_at, + ready_at, activated_at, rolled_back_at, updated_at + ) VALUES (?, 'queued', ?, ?, ?, ?, ?, 0, 0, NULL, NULL, ?, ?, NULL, NULL, NULL, NULL, ?) + `).run( + id, + params.collection, + params.embeddingProfile, + params.vectorDimension, + params.knowledgeFingerprint, + params.expectedCount, + params.createdBy, + params.now, + params.now, + ); + return this.get(id) as RetrievalIndexJob; + } + + get(id: string): RetrievalIndexJob | null { + const row = this.db.prepare( + 'SELECT * FROM retrieval_index_jobs WHERE id = ?', + ).get(id) as JobRow | undefined; + return row ? this.map(row) : null; + } + + list(limit: number, offset: number): RetrievalIndexJob[] { + const rows = this.db.prepare( + 'SELECT * FROM retrieval_index_jobs ORDER BY created_at DESC LIMIT ? OFFSET ?', + ).all(limit, offset) as JobRow[]; + return rows.map((row) => this.map(row)); + } + + count(): number { + return (this.db.prepare( + 'SELECT COUNT(*) AS total FROM retrieval_index_jobs', + ).get() as { total: number }).total; + } + + findReusable(fingerprint: string, embeddingProfile: string): RetrievalIndexJob | null { + const row = this.db.prepare(` + SELECT * FROM retrieval_index_jobs + WHERE knowledge_fingerprint = ? AND embedding_profile = ? + AND status IN ('queued', 'running', 'interrupted', 'ready', 'active') + ORDER BY created_at DESC LIMIT 1 + `).get(fingerprint, embeddingProfile) as JobRow | undefined; + return row ? this.map(row) : null; + } + + nextPending(): RetrievalIndexJob | null { + const row = this.db.prepare(` + SELECT * FROM retrieval_index_jobs + WHERE status IN ('queued', 'interrupted') + ORDER BY CASE status WHEN 'interrupted' THEN 0 ELSE 1 END, created_at + LIMIT 1 + `).get() as JobRow | undefined; + return row ? this.map(row) : null; + } + + markRunning(id: string, now: string): boolean { + return this.db.prepare(` + UPDATE retrieval_index_jobs + SET status = 'running', started_at = COALESCE(started_at, ?), updated_at = ? + WHERE id = ? AND status IN ('queued', 'interrupted') + `).run(now, now, id).changes === 1; + } + + saveCheckpoint(id: string, checkpoint: number, completedCount: number, now: string): void { + this.db.prepare(` + UPDATE retrieval_index_jobs + SET batch_checkpoint = ?, completed_count = ?, updated_at = ? + WHERE id = ? AND status = 'running' + `).run(checkpoint, completedCount, now, id); + } + + markReady(id: string, now: string): void { + this.db.prepare(` + UPDATE retrieval_index_jobs + SET status = 'ready', completed_count = expected_count, + ready_at = ?, updated_at = ? + WHERE id = ? AND status = 'running' + `).run(now, now, id); + } + + markFailed(id: string, failureCode: string, now: string): void { + this.db.prepare(` + UPDATE retrieval_index_jobs + SET status = 'failed', failure_code = ?, updated_at = ? + WHERE id = ? AND status IN ('queued', 'running', 'interrupted') + `).run(failureCode, now, id); + } + + markStale(id: string, now: string): void { + this.db.prepare(` + UPDATE retrieval_index_jobs SET status = 'stale', updated_at = ? + WHERE id = ? AND status IN ('queued', 'running', 'interrupted', 'ready') + `).run(now, id); + } + + interruptRunning(now: string): number { + return this.db.prepare(` + UPDATE retrieval_index_jobs SET status = 'interrupted', updated_at = ? + WHERE status = 'running' + `).run(now).changes; + } + + prepareActivation(id: string, previousCollection: string | null, now: string): void { + this.db.transaction(() => { + this.requireNoOtherIntent(id); + const result = this.db.prepare(` + UPDATE retrieval_index_jobs + SET previous_collection = ?, activation_intent = 'activate', + activation_expected_collection = ?, updated_at = ? + WHERE id = ? AND status = 'ready' + `).run(previousCollection, previousCollection, now, id); + if (result.changes !== 1) { + throw new Error('Retrieval index job is no longer ready'); + } + })(); + } + + completeActivation(id: string, now: string): void { + this.db.transaction(() => { + this.db.prepare(` + UPDATE retrieval_index_jobs + SET status = 'rolled_back', rolled_back_at = ?, updated_at = ? + WHERE status = 'active' AND id <> ? + `).run(now, now, id); + const result = this.db.prepare(` + UPDATE retrieval_index_jobs + SET status = 'active', activated_at = ?, + rolled_back_at = NULL, activation_intent = NULL, + activation_expected_collection = NULL, updated_at = ? + WHERE id = ? AND status IN ('ready', 'stale') + AND activation_intent = 'activate' + `).run(now, now, id); + if (result.changes !== 1) { + throw new Error('Retrieval index activation state changed'); + } + })(); + } + + prepareRollback(id: string, now: string): void { + this.db.transaction(() => { + this.requireNoOtherIntent(id); + const result = this.db.prepare(` + UPDATE retrieval_index_jobs + SET activation_intent = 'rollback', + activation_expected_collection = collection_name, updated_at = ? + WHERE id = ? AND status = 'active' AND previous_collection IS NOT NULL + `).run(now, id); + if (result.changes !== 1) { + throw new Error('Retrieval index job is no longer rollback-ready'); + } + })(); + } + + completeRollback(sourceId: string, targetId: string, now: string): void { + this.db.transaction(() => { + const source = this.db.prepare(` + UPDATE retrieval_index_jobs + SET status = 'rolled_back', rolled_back_at = ?, + activation_intent = NULL, activation_expected_collection = NULL, + updated_at = ? + WHERE id = ? AND status = 'active' AND activation_intent = 'rollback' + `).run(now, now, sourceId); + if (source.changes !== 1) { + throw new Error('Retrieval rollback source state changed'); + } + const result = this.db.prepare(` + UPDATE retrieval_index_jobs + SET status = 'active', activated_at = ?, rolled_back_at = NULL, updated_at = ? + WHERE id = ? AND status = 'rolled_back' + `).run(now, now, targetId); + if (result.changes !== 1) { + throw new Error('Retrieval rollback target state changed'); + } + })(); + } + + findPendingIntent(): { + job: RetrievalIndexJob; + intent: 'activate' | 'rollback'; + expectedCollection: string | null; + } | null { + const row = this.db.prepare(` + SELECT * FROM retrieval_index_jobs + WHERE activation_intent IS NOT NULL + ORDER BY updated_at + LIMIT 1 + `).get() as JobRow | undefined; + return row && row.activation_intent ? { + job: this.map(row), + intent: row.activation_intent, + expectedCollection: row.activation_expected_collection, + } : null; + } + + hasPendingIntent(id: string): boolean { + return Boolean((this.db.prepare(` + SELECT activation_intent FROM retrieval_index_jobs WHERE id = ? + `).get(id) as { activation_intent: string | null } | undefined)?.activation_intent); + } + + rollBack(id: string, now: string): void { + this.db.prepare(` + UPDATE retrieval_index_jobs + SET status = 'rolled_back', rolled_back_at = ?, updated_at = ? + WHERE id = ? AND status = 'active' + `).run(now, now, id); + } + + findByCollection(collection: string): RetrievalIndexJob | null { + const row = this.db.prepare(` + SELECT * FROM retrieval_index_jobs WHERE collection_name = ? + ORDER BY created_at DESC LIMIT 1 + `).get(collection) as JobRow | undefined; + return row ? this.map(row) : null; + } + + private requireNoOtherIntent(id: string): void { + const pending = this.db.prepare(` + SELECT id FROM retrieval_index_jobs + WHERE activation_intent IS NOT NULL AND id <> ? + LIMIT 1 + `).get(id); + if (pending) throw new Error('Another retrieval alias operation is pending'); + } + + private map(row: JobRow): RetrievalIndexJob { + return { + id: row.id, + status: row.status, + collection: row.collection_name, + embeddingProfile: row.embedding_profile, + vectorDimension: row.vector_dimension, + knowledgeFingerprint: row.knowledge_fingerprint, + expectedCount: row.expected_count, + completedCount: row.completed_count, + checkpoint: row.batch_checkpoint, + previousCollection: row.previous_collection, + failureCode: row.failure_code, + createdBy: row.created_by, + createdAt: row.created_at, + startedAt: row.started_at, + readyAt: row.ready_at, + activatedAt: row.activated_at, + rolledBackAt: row.rolled_back_at, + updatedAt: row.updated_at, + }; + } +} diff --git a/server/db/repos/retrieval-index-knowledge.repo.ts b/server/db/repos/retrieval-index-knowledge.repo.ts new file mode 100644 index 0000000..278d8e4 --- /dev/null +++ b/server/db/repos/retrieval-index-knowledge.repo.ts @@ -0,0 +1,165 @@ +import Database from 'better-sqlite3'; +import type { VectorRecord } from '../../ai/vector-store'; +import { ValidationError } from '../../utils/errors'; + +interface KnowledgeVectorRow { + knowledge_type: 'faq' | 'document'; + id: string; + revision: string; + embedding_profile: string | null; + embedding: string; +} + +export interface RetrievalIndexKnowledgeMetadata { + count: number; + vectorDimension: number; + embeddingProfiles: string[]; +} + +export interface RetrievalIndexKnowledgeCursor { + knowledgeType: 'faq' | 'document'; + id: string; +} + +export class RetrievalIndexKnowledgeRepo { + constructor(private readonly db: Database.Database) {} + + inspect(batchSize: number): RetrievalIndexKnowledgeMetadata { + let count = 0; + let cursor: RetrievalIndexKnowledgeCursor | null = null; + const dimensions = new Set(); + const profiles = new Set(); + + while (true) { + const page = this.readPage(cursor, batchSize); + const { records } = page; + if (records.length === 0) break; + count += records.length; + for (const record of records) { + dimensions.add(record.embedding.length); + profiles.add(record.embeddingProfile); + } + cursor = page.nextCursor; + } + + if (count === 0) { + throw new ValidationError('No active embedded knowledge to index'); + } + if (dimensions.size !== 1) { + throw new ValidationError('Active knowledge embeddings must use one vector dimension'); + } + + return { + count, + vectorDimension: [...dimensions][0], + embeddingProfiles: [...profiles].sort(), + }; + } + + readPage( + cursor: RetrievalIndexKnowledgeCursor | null, + limit: number, + ): { records: VectorRecord[]; nextCursor: RetrievalIndexKnowledgeCursor | null } { + const rows: KnowledgeVectorRow[] = []; + if (!cursor || cursor.knowledgeType === 'faq') { + rows.push(...this.readFaqRows( + cursor?.knowledgeType === 'faq' ? cursor.id : '', + limit, + )); + } + if (rows.length < limit) { + rows.push(...this.readDocumentRows( + cursor?.knowledgeType === 'document' ? cursor.id : '', + limit - rows.length, + )); + } + + const last = rows.at(-1); + return { + records: rows.map((row) => this.map(row)), + nextCursor: last + ? { knowledgeType: last.knowledge_type, id: last.id } + : cursor, + }; + } + + cursorAt(checkpoint: number): RetrievalIndexKnowledgeCursor | null { + if (checkpoint <= 0) return null; + const faqCount = (this.db.prepare(` + SELECT COUNT(*) AS total + FROM faq_entries + WHERE is_active = 1 AND embedding IS NOT NULL + `).get() as { total: number }).total; + if (checkpoint <= faqCount) { + const row = this.db.prepare(` + SELECT id FROM faq_entries + WHERE is_active = 1 AND embedding IS NOT NULL + ORDER BY id LIMIT 1 OFFSET ? + `).get(checkpoint - 1) as { id: string } | undefined; + if (!row) throw new ValidationError('Knowledge checkpoint is out of range'); + return { knowledgeType: 'faq', id: row.id }; + } + const row = this.db.prepare(` + SELECT chunk.id + FROM document_chunks chunk + JOIN documents document ON document.id = chunk.document_id + WHERE document.is_active = 1 + AND document.status = 'ready' + AND document.index_status IN ('legacy', 'published') + ORDER BY chunk.id + LIMIT 1 OFFSET ? + `).get(checkpoint - faqCount - 1) as { id: string } | undefined; + if (!row) throw new ValidationError('Knowledge checkpoint is out of range'); + return { knowledgeType: 'document', id: row.id }; + } + + private readFaqRows(afterId: string, limit: number): KnowledgeVectorRow[] { + return this.db.prepare(` + SELECT 'faq' AS knowledge_type, id, updated_at AS revision, + embedding_profile, embedding + FROM faq_entries + WHERE is_active = 1 AND embedding IS NOT NULL AND id > ? + ORDER BY id + LIMIT ? + `).all(afterId, limit) as KnowledgeVectorRow[]; + } + + private readDocumentRows(afterId: string, limit: number): KnowledgeVectorRow[] { + return this.db.prepare(` + SELECT 'document' AS knowledge_type, chunk.id, chunk.created_at AS revision, + chunk.embedding_profile, chunk.embedding + FROM document_chunks chunk + JOIN documents document ON document.id = chunk.document_id + WHERE document.is_active = 1 + AND document.status = 'ready' + AND document.index_status IN ('legacy', 'published') + AND chunk.id > ? + ORDER BY chunk.id + LIMIT ? + `).all(afterId, limit) as KnowledgeVectorRow[]; + } + + private map(row: KnowledgeVectorRow): VectorRecord { + let embedding: unknown; + try { + embedding = JSON.parse(row.embedding); + } catch { + throw new ValidationError('Knowledge embedding is malformed'); + } + if ( + !Array.isArray(embedding) + || embedding.length === 0 + || embedding.some((value) => typeof value !== 'number' || !Number.isFinite(value)) + || !row.embedding_profile + ) { + throw new ValidationError('All active knowledge must have a valid embedding profile'); + } + return { + id: `${row.knowledge_type}:${row.id}`, + knowledgeType: row.knowledge_type, + revision: row.revision, + embeddingProfile: row.embedding_profile, + embedding, + }; + } +} diff --git a/server/db/repos/retrieval-trace-detail.repo.ts b/server/db/repos/retrieval-trace-detail.repo.ts new file mode 100644 index 0000000..6887995 --- /dev/null +++ b/server/db/repos/retrieval-trace-detail.repo.ts @@ -0,0 +1,69 @@ +import Database from 'better-sqlite3'; + +export interface RetrievalTraceMessageDetail { + id: string; + content: string; +} + +export interface RetrievalTraceKnowledgeDetail { + knowledgeType: 'faq' | 'document'; + knowledgeId: string; + title: string; + content: string; + available: true; +} + +export class RetrievalTraceDetailRepo { + constructor(private readonly db: Database.Database) {} + + findMessages(ids: string[]): RetrievalTraceMessageDetail[] { + const unique = this.uniqueBounded(ids); + if (unique.length === 0) return []; + return this.db.prepare(` + SELECT id, content FROM messages + WHERE id IN (${this.placeholders(unique)}) + `).all(...unique) as RetrievalTraceMessageDetail[]; + } + + findFaqs(ids: string[]): RetrievalTraceKnowledgeDetail[] { + const unique = this.uniqueBounded(ids); + if (unique.length === 0) return []; + const rows = this.db.prepare(` + SELECT id, question, answer FROM faq_entries + WHERE id IN (${this.placeholders(unique)}) + `).all(...unique) as Array<{ id: string; question: string; answer: string }>; + return rows.map((row) => ({ + knowledgeType: 'faq', + knowledgeId: row.id, + title: row.question, + content: row.answer, + available: true, + })); + } + + findDocumentChunks(ids: string[]): RetrievalTraceKnowledgeDetail[] { + const unique = this.uniqueBounded(ids); + if (unique.length === 0) return []; + const rows = this.db.prepare(` + SELECT chunk.id, document.file_name, chunk.content + FROM document_chunks chunk + JOIN documents document ON document.id = chunk.document_id + WHERE chunk.id IN (${this.placeholders(unique)}) + `).all(...unique) as Array<{ id: string; file_name: string; content: string }>; + return rows.map((row) => ({ + knowledgeType: 'document', + knowledgeId: row.id, + title: row.file_name, + content: row.content, + available: true, + })); + } + + private uniqueBounded(ids: string[]): string[] { + return [...new Set(ids.filter(Boolean))].slice(0, 160); + } + + private placeholders(values: string[]): string { + return values.map(() => '?').join(', '); + } +} diff --git a/server/db/repos/retrieval-trace.repo.ts b/server/db/repos/retrieval-trace.repo.ts new file mode 100644 index 0000000..1a81eef --- /dev/null +++ b/server/db/repos/retrieval-trace.repo.ts @@ -0,0 +1,180 @@ +import Database from 'better-sqlite3'; +import { + RetrievalTrace, + RetrievalTraceCandidate, + RetrievalTraceStage, + RetrievalTraceStageName, + RetrievalTraceStageStatus, + RetrievalTraceStatus, +} from '../../types/retrieval-ops'; + +interface TraceRow { + id: string; + session_id: string; + user_message_id: string; + assistant_message_id: string | null; + policy_id: string; + backend: 'memory' | 'qdrant'; + status: RetrievalTraceStatus; + error_code: string | null; + total_latency_ms: number; + created_at: string; + completed_at: string; +} + +interface StageRow { + stage_name: RetrievalTraceStageName; + stage_order: number; + status: RetrievalTraceStageStatus; + latency_ms: number; + input_count: number; + output_count: number; + candidates: string; + budget: string; + error_code: string | null; +} + +export class RetrievalTraceRepo { + constructor(private readonly db: Database.Database) {} + + create(trace: RetrievalTrace): void { + this.db.transaction(() => { + this.db.prepare(` + INSERT INTO retrieval_traces ( + id, session_id, user_message_id, assistant_message_id, policy_id, + backend, status, error_code, total_latency_ms, created_at, completed_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + trace.id, + trace.sessionId, + trace.userMessageId, + trace.assistantMessageId, + trace.policyId, + trace.backend, + trace.status, + trace.errorCode, + trace.totalLatencyMs, + trace.createdAt, + trace.completedAt, + ); + const insertStage = this.db.prepare(` + INSERT INTO retrieval_trace_stages ( + trace_id, stage_name, stage_order, status, latency_ms, input_count, + output_count, candidates, budget, error_code + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + for (const stage of trace.stages) { + insertStage.run( + trace.id, + stage.name, + stage.order, + stage.status, + stage.latencyMs, + stage.inputCount, + stage.outputCount, + JSON.stringify(stage.candidates), + JSON.stringify(stage.budget), + stage.errorCode, + ); + } + })(); + } + + get(id: string): RetrievalTrace | null { + const row = this.db.prepare( + 'SELECT * FROM retrieval_traces WHERE id = ?', + ).get(id) as TraceRow | undefined; + if (!row) return null; + const stages = this.db.prepare(` + SELECT * FROM retrieval_trace_stages + WHERE trace_id = ? ORDER BY stage_order + `).all(id) as StageRow[]; + return this.map(row, stages); + } + + list(filters: { + status?: RetrievalTraceStatus; + backend?: 'memory' | 'qdrant'; + sessionId?: string; + createdFrom?: string; + createdTo?: string; + limit: number; + offset: number; + }): { items: RetrievalTrace[]; total: number } { + const clauses: string[] = []; + const values: string[] = []; + if (filters.status) { + clauses.push('status = ?'); + values.push(filters.status); + } + if (filters.backend) { + clauses.push('backend = ?'); + values.push(filters.backend); + } + if (filters.sessionId) { + clauses.push('session_id = ?'); + values.push(filters.sessionId); + } + if (filters.createdFrom) { + clauses.push('created_at >= ?'); + values.push(filters.createdFrom); + } + if (filters.createdTo) { + clauses.push('created_at <= ?'); + values.push(filters.createdTo); + } + const where = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : ''; + const rows = this.db.prepare(` + SELECT * FROM retrieval_traces ${where} + ORDER BY created_at DESC LIMIT ? OFFSET ? + `).all(...values, filters.limit, filters.offset) as TraceRow[]; + const total = (this.db.prepare( + `SELECT COUNT(*) AS total FROM retrieval_traces ${where}`, + ).get(...values) as { total: number }).total; + return { + items: rows.map((row) => this.map(row, [])), + total, + }; + } + + deleteBefore(cutoff: string): number { + return this.db.prepare( + 'DELETE FROM retrieval_traces WHERE created_at < ?', + ).run(cutoff).changes; + } + + private map(row: TraceRow, stages: StageRow[]): RetrievalTrace { + return { + id: row.id, + sessionId: row.session_id, + userMessageId: row.user_message_id, + assistantMessageId: row.assistant_message_id, + policyId: row.policy_id, + backend: row.backend, + status: row.status, + errorCode: row.error_code, + totalLatencyMs: row.total_latency_ms, + stages: stages.map((stage): RetrievalTraceStage => ({ + name: stage.stage_name, + order: stage.stage_order, + status: stage.status, + latencyMs: stage.latency_ms, + inputCount: stage.input_count, + outputCount: stage.output_count, + candidates: parseJson(stage.candidates, []).slice(0, 20), + budget: parseJson>(stage.budget, {}), + errorCode: stage.error_code, + })), + createdAt: row.created_at, + completedAt: row.completed_at, + }; + } +} + +function parseJson(value: string, fallback: T): T { + try { + return JSON.parse(value) as T; + } catch { + return fallback; + } +} diff --git a/server/eval/document-eval.ts b/server/eval/document-eval.ts index 1b54ae8..acca2bd 100644 --- a/server/eval/document-eval.ts +++ b/server/eval/document-eval.ts @@ -146,7 +146,7 @@ async function evaluate(cases: EvalCase[], index: IndexedChunk[]): Promise(), + new InMemoryVectorStore(), embedTexts, [new EvalDocumentAdapter(index)], ); diff --git a/server/eval/quality-evaluator.ts b/server/eval/quality-evaluator.ts index 38bafc7..6ba8dff 100644 --- a/server/eval/quality-evaluator.ts +++ b/server/eval/quality-evaluator.ts @@ -7,6 +7,7 @@ import type { QualityCandidateResult, QualityCase, QualityCaseResult, + QualityBackendTarget, RetrievalPolicyConfig, } from '../types/quality'; @@ -21,9 +22,11 @@ export function evaluateQualityCandidates(params: { policies: RetrievalPolicyConfig[]; embeddingCallCount: number; estimatedTokenCount: number; + backendTarget?: QualityBackendTarget; }): QualityCandidateResult[] { + const backendTarget = params.backendTarget ?? { provider: 'memory' as const }; const results: QualityCandidateResult[] = params.policies.map((policy) => { - const candidateKey = policyKey(policy); + const candidateKey = qualityCandidateKey(backendTarget, policy); const caseResults = params.cases.map(({ testCase, candidates, latencyMs }) => { const started = performance.now(); const result = evaluateCase(testCase, candidates, latencyMs, policy, candidateKey); @@ -64,6 +67,7 @@ export function evaluateQualityCandidates(params: { const denominator = Math.max(answerable.length, 1); return { key: candidateKey, + backendTarget, policy, recommended: false, cases: caseResults, @@ -94,6 +98,16 @@ export function policyKey(policy: RetrievalPolicyConfig): string { return createHash('sha256').update(JSON.stringify(policy)).digest('hex').slice(0, 16); } +export function qualityCandidateKey( + target: QualityBackendTarget, + policy: RetrievalPolicyConfig, +): string { + const backend = target.provider === 'memory' + ? 'memory' + : `qdrant:${target.indexJobId}`; + return `${backend}:${policyKey(policy)}`; +} + function evaluateCase( testCase: QualityCase, candidates: RetrievalResult[], diff --git a/server/index.ts b/server/index.ts index e85212c..9512738 100644 --- a/server/index.ts +++ b/server/index.ts @@ -18,6 +18,7 @@ let adminKnowledgeReviewRoutes: express.Router; let adminDocumentRoutes: express.Router; let adminQualityRoutes: express.Router; let adminEscalationRoutes: express.Router; +let adminRetrievalRoutes: express.Router; let ready = false; function createApp(): express.Application { @@ -113,6 +114,13 @@ function createApp(): express.Application { return adminEscalationRoutes(_req, _res, next); }); + app.use('/api/admin/retrieval', (_req, _res, next) => { + if (!adminRetrievalRoutes) { + adminRetrievalRoutes = require('./routes/admin/retrieval').default; + } + return adminRetrievalRoutes(_req, _res, next); + }); + // ---- Health check ---- app.get('/api/health', (_req, res) => { res.json({ code: 0, data: { status: 'ok', uptime: process.uptime() }, message: 'ok' }); @@ -147,6 +155,12 @@ async function start(): Promise { getQualityLabService().bootstrap(); const { getQualityRunService } = await import('./services/quality-run.service'); getQualityRunService().start(); + const { getRetrievalIndexJobService } = await import( + './services/retrieval-index-job.service' + ); + getRetrievalIndexJobService().start(); + const { getRetrievalTraceService } = await import('./services/retrieval-trace.service'); + getRetrievalTraceService().start(); logger.info('RAG quality lab initialized'); // Hydrate runtime config from environment-owned model settings. @@ -214,6 +228,8 @@ async function closeDatabaseAndExit(code: number): Promise { try { const { documentOcrScheduler } = await import('./services/document-runtime'); await documentOcrScheduler.stop(); + const { getRetrievalTraceService } = await import('./services/retrieval-trace.service'); + getRetrievalTraceService().stop(); const { closeDatabase } = await import('./db'); closeDatabase(); } catch (error) { diff --git a/server/routes/admin/faq.ts b/server/routes/admin/faq.ts index 9e207dc..ddb111c 100644 --- a/server/routes/admin/faq.ts +++ b/server/routes/admin/faq.ts @@ -81,7 +81,7 @@ router.get('/', async (req: Request, res: Response, next: NextFunction) => { */ router.get('/index/status', async (_req: Request, res: Response, next: NextFunction) => { try { - const status = faqService.getIndexStatus(); + const status = await faqService.getIndexStatus(); res.json({ code: 0, data: status, message: 'ok' }); } catch (err) { next(err); diff --git a/server/routes/admin/quality.ts b/server/routes/admin/quality.ts index 8ab99e2..8d5cd72 100644 --- a/server/routes/admin/quality.ts +++ b/server/routes/admin/quality.ts @@ -38,6 +38,10 @@ const policySchema = z.object({ (value) => value.generationEvidenceThreshold <= value.directFaqThreshold, { message: 'generationEvidenceThreshold cannot exceed directFaqThreshold' }, ); +const backendTargetSchema = z.discriminatedUnion('provider', [ + z.object({ provider: z.literal('memory') }).strict(), + z.object({ provider: z.literal('qdrant'), indexJobId: uuid }).strict(), +]); const paginationSchema = z.object({ page: z.coerce.number().int().positive().default(1), pageSize: z.coerce.number().int().positive().max(100).default(20), @@ -83,6 +87,7 @@ router.post('/runs', (req, res, next) => handle(res, next, () => { const data = parse(z.object({ datasetVersionIds: z.array(resourceId).min(1).max(20), policies: z.array(policySchema).max(64), + backendTargets: z.array(backendTargetSchema).min(1).max(10).optional(), }).strict(), req.body); return qualityRuns.createRun({ ...data, createdBy: actor(req) }); }, 202)); diff --git a/server/routes/admin/retrieval.ts b/server/routes/admin/retrieval.ts new file mode 100644 index 0000000..d9325c2 --- /dev/null +++ b/server/routes/admin/retrieval.ts @@ -0,0 +1,112 @@ +import { NextFunction, Request, Response, Router } from 'express'; +import { z } from 'zod'; +import { adminOnlyMiddleware } from '../../middleware/adminOnly'; +import { authMiddleware } from '../../middleware/auth'; +import { idempotencyMiddleware } from '../../middleware/idempotency'; +import { getRetrievalIndexJobService } from '../../services/retrieval-index-job.service'; +import { ValidationError } from '../../utils/errors'; +import { getRetrievalTraceService } from '../../services/retrieval-trace.service'; + +const router = Router(); +router.use(authMiddleware); +router.use(adminOnlyMiddleware); +router.use(idempotencyMiddleware); + +const service = getRetrievalIndexJobService(); +const traces = getRetrievalTraceService(); +const uuid = z.string().uuid(); +const pagination = z.object({ + page: z.coerce.number().int().positive().default(1), + pageSize: z.coerce.number().int().positive().max(100).default(20), +}); + +router.get('/status', (_req, res, next) => handle(res, next, () => service.status())); +router.get('/index-jobs', (req, res, next) => handle(res, next, () => { + const data = parse(pagination, req.query); + return service.listJobs(data.page, data.pageSize); +})); +router.post('/index-jobs', (req, res, next) => handle(res, next, () => { + requireIdempotencyKey(req); + parse(z.object({}).strict(), req.body ?? {}); + return service.createJob({ createdBy: actor(req) }); +}, 202)); +router.get('/index-jobs/:id/activation-check', (req, res, next) => handle( + res, + next, + () => service.activationCheck(parse(uuid, req.params.id)), +)); +router.post('/index-jobs/:id/activate', (req, res, next) => handle(res, next, () => { + requireIdempotencyKey(req); + const data = parse(z.object({ + expectedCurrentCollection: z.string().min(1).max(200).nullable(), + confirmed: z.literal(true), + confirmLatencyWarning: z.boolean().default(false), + }).strict(), req.body); + return service.activate({ + id: parse(uuid, req.params.id), + expectedCurrentCollection: data.expectedCurrentCollection, + confirmLatencyWarning: data.confirmLatencyWarning ?? false, + }); +})); +router.post('/index-jobs/:id/rollback', (req, res, next) => handle(res, next, () => { + requireIdempotencyKey(req); + const data = parse(z.object({ + expectedCurrentCollection: z.string().min(1).max(200), + confirmed: z.literal(true), + }).strict(), req.body); + return service.rollback({ id: parse(uuid, req.params.id), ...data }); +})); +router.get('/index-jobs/:id', (req, res, next) => handle( + res, + next, + () => service.getJob(parse(uuid, req.params.id)), +)); +router.get('/traces', (req, res, next) => handle(res, next, () => { + const data = parse(pagination.extend({ + status: z.enum(['completed', 'degraded', 'failed']).optional(), + backend: z.enum(['memory', 'qdrant']).optional(), + sessionId: z.string().uuid().optional(), + createdFrom: z.string().datetime().optional(), + createdTo: z.string().datetime().optional(), + }), req.query); + return traces.listTraces({ + ...data, + page: data.page ?? 1, + pageSize: data.pageSize ?? 20, + }); +})); +router.get('/traces/:traceId', (req, res, next) => handle( + res, + next, + () => traces.getTraceDetail(parse(uuid, req.params.traceId)), +)); + +function parse(schema: z.ZodType, value: unknown): T { + const result = schema.safeParse(value); + if (!result.success) { + throw new ValidationError(result.error.errors.map((item) => item.message).join('; ')); + } + return result.data; +} + +function actor(req: Request): string { + return req.user?.username ?? 'unknown'; +} + +function requireIdempotencyKey(req: Request): void { + if (!req.get('Idempotency-Key')) throw new ValidationError('Idempotency-Key is required'); +} + +function handle( + res: Response, + next: NextFunction, + work: () => unknown | Promise, + status: number = 200, +): void { + Promise.resolve() + .then(work) + .then((data) => res.status(status).json({ code: 0, data, message: 'ok' })) + .catch(next); +} + +export default router; diff --git a/server/routes/chat.ts b/server/routes/chat.ts index 6283dfe..4d0a8ec 100644 --- a/server/routes/chat.ts +++ b/server/routes/chat.ts @@ -19,6 +19,9 @@ import { } from '../services/grounding-policy'; import { idempotencyMiddleware } from '../middleware/idempotency'; import { getQualityLabService } from '../services/quality-lab.service'; +import { RetrievalTraceCollector } from '../services/retrieval-trace-collector'; +import { getRetrievalTraceService } from '../services/retrieval-trace.service'; +import { config } from '../config'; const router = Router(); router.use(idempotencyMiddleware); @@ -110,6 +113,30 @@ function captureKnowledgeGapSafely( * Core SSE streaming endpoint for chat messages. */ router.post('/', async (req: Request, res: Response, next: NextFunction) => { + let trace: RetrievalTraceCollector | null = null; + let traceLink: { + sessionId: string; + userMessageId: string; + policyId: string; + } | null = null; + let tracePersisted = false; + const finalizeTrace = (assistantMessageId: string | null, errorCode?: string): void => { + if (!trace || !traceLink || tracePersisted) return; + if (errorCode) trace.fail(errorCode); + try { + getRetrievalTraceService().persist(trace.complete({ + ...traceLink, + assistantMessageId, + })); + tracePersisted = true; + } catch (error) { + logger.error({ + traceId: trace.id, + sessionId: traceLink.sessionId, + errorName: error instanceof Error ? error.name : 'UnknownError', + }, 'Retrieval trace persistence failed'); + } + }; try { const parsed = chatSchema.safeParse(req.body); if (!parsed.success) { @@ -119,6 +146,7 @@ router.post('/', async (req: Request, res: Response, next: NextFunction) => { const { message, sessionId: inputSessionId, userIdent: inputUserIdent } = parsed.data; const userIdent = inputUserIdent || req.ip || 'anonymous'; const retrievalPolicy = getQualityLabService().getCurrentPolicy(); + trace = new RetrievalTraceCollector({ backend: config.vectorStore.provider }); // Step 1: Get or create session const session = conversationService.resolveSessionForMessage(inputSessionId, userIdent); @@ -130,6 +158,11 @@ router.post('/', async (req: Request, res: Response, next: NextFunction) => { role: MessageRole.USER, content: message, }); + traceLink = { + sessionId, + userMessageId: userMessage.id, + policyId: retrievalPolicy.id, + }; // Step 3: Build LLM message history from DB messages const previousMessages = conversationService.getMessages(sessionId); @@ -145,7 +178,9 @@ router.post('/', async (req: Request, res: Response, next: NextFunction) => { message, llmHistory, retrievalPolicy.config, + trace, ); + const groundingStarted = performance.now(); const grounding = evaluateGrounding({ message, intent: intentResult.intent.intent, @@ -154,6 +189,37 @@ router.post('/', async (req: Request, res: Response, next: NextFunction) => { explicitEscalation: intentResult.escalationType === 'explicit', policy: retrievalPolicy.config, }); + trace.record('context_budget', { + status: 'completed', + latencyMs: 0, + inputCount: intentResult.retrievalResults.length, + outputCount: grounding.citations.length, + candidates: grounding.citations.map((result, index) => ({ + knowledgeType: result.knowledgeType, + knowledgeId: result.knowledgeId, + score: result.similarity, + rank: index + 1, + source: result.source, + })), + budget: { maxEvidence: 3, selectedEvidence: grounding.citations.length }, + }); + trace.record('grounding', { + status: 'completed', + latencyMs: performance.now() - groundingStarted, + inputCount: intentResult.retrievalResults.length, + outputCount: grounding.citations.length, + candidates: grounding.citations.map((result, index) => ({ + knowledgeType: result.knowledgeType, + knowledgeId: result.knowledgeId, + score: result.similarity, + rank: index + 1, + source: result.source, + })), + budget: { + directFaqThreshold: retrievalPolicy.config.directFaqThreshold, + generationEvidenceThreshold: retrievalPolicy.config.generationEvidenceThreshold, + }, + }); // Set up SSE headers res.setHeader('Content-Type', 'text/event-stream'); @@ -234,6 +300,7 @@ router.post('/', async (req: Request, res: Response, next: NextFunction) => { const assistantMessage = escalationReason ? await conversationService.saveMessageAndEscalate(messageParams, escalationReason) : conversationService.saveMessage(messageParams); + finalizeTrace(assistantMessage.id); if (grounding.groundingStatus !== 'high_risk') { captureKnowledgeGapSafely({ @@ -310,6 +377,7 @@ router.post('/', async (req: Request, res: Response, next: NextFunction) => { } catch (streamErr) { logger.error({ err: streamErr, sessionId }, 'LLM stream failed'); sseSend({ type: 'error', content: 'AI响应生成失败，请稍后重试' }); + finalizeTrace(null, 'generation_failed'); res.end(); return; } @@ -344,6 +412,7 @@ router.post('/', async (req: Request, res: Response, next: NextFunction) => { } logger.error({ sessionId }, 'LLM stream completed without answer content'); sseSend({ type: 'error', content: 'AI响应生成失败，请稍后重试' }); + finalizeTrace(null, 'empty_generation'); res.end(); return; } @@ -366,6 +435,7 @@ router.post('/', async (req: Request, res: Response, next: NextFunction) => { ? await conversationService.saveMessageAndEscalate(messageParams, escalationReason) : conversationService.saveMessage(messageParams); assistantMessageId = assistantMessage.id; + finalizeTrace(assistantMessage.id); captureKnowledgeGapSafely({ userMessage, @@ -397,6 +467,7 @@ router.post('/', async (req: Request, res: Response, next: NextFunction) => { logger.info({ sessionId, messageId: assistantMessageId, intent: intentResult.intent.intent }, 'Chat interaction completed'); res.end(); } catch (err) { + finalizeTrace(null, 'chat_request_failed'); next(err); } }); diff --git a/server/services/document-runtime.ts b/server/services/document-runtime.ts index 8bc2d7b..44dcb46 100644 --- a/server/services/document-runtime.ts +++ b/server/services/document-runtime.ts @@ -28,9 +28,9 @@ export const documentService = new DocumentService(getDatabase(), { : undefined, ocrMode: config.ocr.backgroundEnabled ? 'queued' : 'inline', embedTexts: async (texts) => (await getLLMClient().embed(texts)).map((result) => result.embedding), - publishChunks: (chunks, document) => { + publishChunks: async (chunks, document) => { if (!document) throw new Error('Document metadata is required for index publication'); - knowledgeRetriever.replaceDocumentIndexItems( + await knowledgeRetriever.replaceDocumentIndexItems( document.id, chunks.map((chunk) => documentKnowledgeAdapter.toIndexItem(chunk, document.fileName)), ); @@ -38,7 +38,7 @@ export const documentService = new DocumentService(getDatabase(), { synchronizeIndex: async () => knowledgeRetriever.refreshSource('document'), removeDocumentFromIndex: async (_documentId, chunks) => { for (const chunk of chunks) { - knowledgeRetriever.deleteIndexItem('document', `document:${chunk.id}`); + await knowledgeRetriever.deleteIndexItem('document', `document:${chunk.id}`); } }, }); diff --git a/server/services/faq.service.ts b/server/services/faq.service.ts index d816b33..e75569f 100644 --- a/server/services/faq.service.ts +++ b/server/services/faq.service.ts @@ -121,16 +121,17 @@ export class FaqService { return updated; } - deleteFaq(id: string): void { + async deleteFaq(id: string): Promise { + const entry = this.faqRepo.findById(id); + if (!entry) { + throw new NotFoundError('FAQ条目不存在'); + } const deleted = this.faqRepo.delete(id); if (!deleted) { throw new NotFoundError('FAQ条目不存在'); } // Remove from index by marking inactive - const entry = this.faqRepo.findById(id); - if (entry) { - semanticSearch.updateIndex({ ...entry, isActive: 0 }); - } + await semanticSearch.updateIndex({ ...entry, isActive: 0 }); logger.info({ faqId: id }, 'FAQ entry deleted'); } @@ -166,7 +167,7 @@ export class FaqService { return this.faqRepo.listAllActive(); } - getIndexStatus(): FaqIndexStatus { + getIndexStatus(): Promise { return semanticSearch.getStatus(); } diff --git a/server/services/intent.service.ts b/server/services/intent.service.ts index 8810f9e..eb8176b 100644 --- a/server/services/intent.service.ts +++ b/server/services/intent.service.ts @@ -4,6 +4,7 @@ import { IntentResult, FaqMatch, LLMMessage, RetrievalResult } from '../types/ai import { IntentCategory } from '../types/domain'; import { logger } from '../utils/logger'; import type { RetrievalPolicyConfig } from '../types/quality'; +import type { RetrievalTraceCollector } from './retrieval-trace-collector'; const HIGH_CONFIDENCE_THRESHOLD = 0.7; const LOW_CONFIDENCE_THRESHOLD = 0.4; @@ -22,6 +23,7 @@ export class IntentService { message: string, history: LLMMessage[] = [], policy?: RetrievalPolicyConfig, + trace?: RetrievalTraceCollector, ): Promise { // Step 1: Classify intent const intent = await classify(message, history); @@ -35,13 +37,13 @@ export class IntentService { if (intent.confidence >= HIGH_CONFIDENCE_THRESHOLD) { // High confidence: search FAQ by category + semantics - retrievalResults = await knowledgeRetriever.search(message, 5, undefined, policy); + retrievalResults = await knowledgeRetriever.search(message, 5, undefined, policy, trace); } else if (intent.confidence >= LOW_CONFIDENCE_THRESHOLD) { // Medium confidence: semantic search only - retrievalResults = await knowledgeRetriever.search(message, 3, undefined, policy); + retrievalResults = await knowledgeRetriever.search(message, 3, undefined, policy, trace); } else { // Low confidence: flag for escalation - retrievalResults = await knowledgeRetriever.search(message, 3, undefined, policy); + retrievalResults = await knowledgeRetriever.search(message, 3, undefined, policy, trace); if (retrievalResults.length === 0 || retrievalResults[0].similarity < 0.5) { needsEscalation = true; escalationReason = '意图置信度低且无匹配FAQ，建议转人工'; diff --git a/server/services/knowledge-fingerprint.ts b/server/services/knowledge-fingerprint.ts new file mode 100644 index 0000000..9369b8a --- /dev/null +++ b/server/services/knowledge-fingerprint.ts @@ -0,0 +1,20 @@ +import { createHash } from 'node:crypto'; +import Database from 'better-sqlite3'; + +export function knowledgeFingerprint(db: Database.Database): string { + const rows = db.prepare( + `SELECT 'faq' AS type, id, updated_at AS revision, + COALESCE(embedding_profile, '') AS profile + FROM faq_entries WHERE is_active = 1 + UNION ALL + SELECT 'document' AS type, chunk.id, chunk.created_at AS revision, + COALESCE(chunk.embedding_profile, '') AS profile + FROM document_chunks chunk + JOIN documents document ON document.id = chunk.document_id + WHERE document.is_active = 1 + AND document.status = 'ready' + AND document.index_status IN ('legacy', 'published') + ORDER BY type, id`, + ).all(); + return createHash('sha256').update(JSON.stringify(rows)).digest('hex'); +} diff --git a/server/services/knowledge-review.service.ts b/server/services/knowledge-review.service.ts index b5a4f67..453bd76 100644 --- a/server/services/knowledge-review.service.ts +++ b/server/services/knowledge-review.service.ts @@ -21,7 +21,7 @@ import { ConflictError, NotFoundError, ServiceUnavailableError, ValidationError const KNOWLEDGE_GAP_THRESHOLD = 0.55; type IndexPreparation = (faq: FaqEntry) => Promise; -type IndexCommit = (faq: FaqEntry) => void; +type IndexCommit = (faq: FaqEntry) => void | Promise; export class KnowledgeReviewService { private readonly reviewRepo: KnowledgeReviewRepo; @@ -221,7 +221,7 @@ export class KnowledgeReviewService { isActive: 1, }; try { - this.commitFaqIndex(preparedFaq); + await this.commitFaqIndex(preparedFaq); } catch { throw new ServiceUnavailableError('FAQ索引同步失败，请稍后重试'); } @@ -246,7 +246,7 @@ export class KnowledgeReviewService { return { review, faq }; })(); } catch (error) { - this.commitFaqIndex({ ...preparedFaq, isActive: 0 }); + await this.commitFaqIndex({ ...preparedFaq, isActive: 0 }); throw error; } } diff --git a/server/services/quality-run.service.ts b/server/services/quality-run.service.ts index 615fefd..2f3dc29 100644 --- a/server/services/quality-run.service.ts +++ b/server/services/quality-run.service.ts @@ -1,10 +1,11 @@ -import { createHash } from 'node:crypto'; import Database from 'better-sqlite3'; +import { config } from '../config'; import { getDatabase } from '../db'; import { QualityRunRepo } from '../db/repos/quality-run.repo'; import { evaluateQualityCandidates, policyKey, + qualityCandidateKey, type RetrievedQualityCase, } from '../eval/quality-evaluator'; import { @@ -15,28 +16,50 @@ import type { RetrievalResult } from '../types/ai'; import type { PolicyGateResult, QualityCase, + QualityBackendTarget, QualityRun, RetrievalPolicy, RetrievalPolicyConfig, } from '../types/quality'; +import { InMemoryVectorStore } from '../ai/vector-store'; +import { QdrantVectorStore, createQdrantVectorStore } from '../ai/qdrant-vector-store'; +import { KnowledgeRetriever } from '../ai/knowledge-retriever'; +import { DocumentKnowledgeAdapter, FaqKnowledgeAdapter } from '../ai/knowledge-adapters'; +import { getLLMClient } from '../ai/llm-client'; +import { expandRetrievalQuery } from '../ai/query-expansion'; +import { knowledgeFingerprint } from './knowledge-fingerprint'; +import { getRetrievalIndexJobService } from './retrieval-index-job.service'; +import { FaqRepo } from '../db/repos/faq.repo'; +import { DocumentRepo } from '../db/repos/document.repo'; import { ConflictError, NotFoundError, ValidationError } from '../utils/errors'; import { logger } from '../utils/logger'; import { QualityLabService, getQualityLabService } from './quality-lab.service'; +const QUALITY_SEARCH_CONCURRENCY = 8; + interface QualityRunServiceOptions { qualityLab?: QualityLabService; searchCurrent?: (query: string) => Promise; searchCurrentBatch?: (queries: string[]) => Promise; + searchBackendBatch?: ( + target: QualityBackendTarget, + queries: string[], + embeddings: number[][], + ) => Promise; now?: () => Date; autoDrain?: boolean; + getIndexJob?: ReturnType['getJob']; } export class QualityRunService { private readonly repo: QualityRunRepo; private readonly qualityLab: QualityLabService; private readonly searchCurrentBatch: (queries: string[]) => Promise; + private readonly searchBackendBatch: QualityRunServiceOptions['searchBackendBatch']; private readonly now: () => Date; private readonly autoDrain: boolean; + private readonly usesInjectedSearch: boolean; + private readonly getIndexJob: ReturnType['getJob']; private draining = false; constructor( @@ -54,8 +77,14 @@ export class QualityRunService { const { knowledgeRetriever } = await import('../ai/knowledge-system'); return knowledgeRetriever.searchCandidatesBatch(queries, 100); }); + this.searchBackendBatch = options.searchBackendBatch; + this.usesInjectedSearch = Boolean( + options.searchBackendBatch || options.searchCurrentBatch || options.searchCurrent, + ); this.now = options.now ?? (() => new Date()); this.autoDrain = options.autoDrain ?? true; + this.getIndexJob = options.getIndexJob + ?? ((id) => getRetrievalIndexJobService().getJob(id)); } start(): void { @@ -65,6 +94,7 @@ export class QualityRunService { createRun(params: { datasetVersionIds: string[]; policies: RetrievalPolicyConfig[]; + backendTargets?: QualityBackendTarget[]; createdBy: string; }): QualityRun { const versionIds = [...new Set(params.datasetVersionIds)]; @@ -77,16 +107,19 @@ export class QualityRunService { } return version; }); - const totalCases = versions.reduce((sum, version) => sum + version.caseCount, 0); - if (totalCases === 0 || totalCases > 500) { + const caseCount = versions.reduce((sum, version) => sum + version.caseCount, 0); + if (caseCount === 0 || caseCount > 500) { throw new ValidationError('A run must contain between 1 and 500 cases'); } + const backendTargets = this.validateBackendTargets(params.backendTargets ?? [{ provider: 'memory' }]); + const totalCases = caseCount * backendTargets.length; const currentPolicy = this.qualityLab.getCurrentPolicy(); const policies = this.validatePolicies([currentPolicy.config, ...params.policies]); const includesCurrentKnowledge = versions.some((version) => version.targetKind === 'current'); const run = this.repo.create({ datasetVersionIds: versionIds, policies, + backendTargets, totalCases, knowledgeFingerprint: includesCurrentKnowledge ? this.knowledgeFingerprint() : null, activePolicyId: currentPolicy.id, @@ -157,29 +190,7 @@ export class QualityRunService { const reasons: string[] = []; const warnings: string[] = []; if (run.status !== 'completed') reasons.push('run_not_completed'); - if (!run.datasetVersionIds.includes(QUALITY_BASELINE_VERSION_ID)) { - reasons.push('builtin_baseline_required'); - } - const currentVersions = run.datasetVersionIds - .map((id) => this.qualityLab.getVersion(id)) - .filter((version) => version?.targetKind === 'current'); - if (currentVersions.length === 0) reasons.push('current_knowledge_dataset_required'); - const hasCompleteCurrentVersion = currentVersions.some((version) => { - const cases = this.qualityLab.listCases(version!.id); - const answerable = cases.filter( - (testCase) => testCase.expectedGroundingStatus === 'sufficient', - ).length; - const insufficient = cases.filter( - (testCase) => testCase.expectedGroundingStatus === 'insufficient', - ).length; - const highRisk = cases.filter( - (testCase) => ['high_risk', 'escalated'].includes(testCase.expectedGroundingStatus), - ).length; - return cases.length >= 12 && answerable >= 6 && insufficient >= 4 && highRisk >= 2; - }); - if (currentVersions.length > 0 && !hasCompleteCurrentVersion) { - reasons.push('current_knowledge_coverage_insufficient'); - } + this.appendDatasetGateReasons(run, reasons); if (!run.knowledgeFingerprint || run.knowledgeFingerprint !== this.knowledgeFingerprint()) { reasons.push('knowledge_fingerprint_changed'); } @@ -187,9 +198,12 @@ export class QualityRunService { if (run.activePolicyId !== currentPolicy.id) reasons.push('current_policy_changed'); const candidate = run.candidates.find((item) => item.key === candidateKey); if (!candidate) reasons.push('candidate_not_found'); - const baseline = run.candidates.find( - (item) => item.key === policyKey(currentPolicy.config), - ); + if (candidate?.backendTarget.provider !== 'memory') { + reasons.push('policy_candidate_must_use_memory_backend'); + } + const baseline = run.candidates.find((item) => ( + item.key === qualityCandidateKey({ provider: 'memory' }, currentPolicy.config) + )); if (!baseline) reasons.push('current_policy_result_missing'); if (candidate && baseline) { if (candidate.metrics.unsafeAnswerCount !== 0) reasons.push('unsafe_answers_present'); @@ -213,6 +227,34 @@ export class QualityRunService { return { eligible: reasons.length === 0, warnings, reasons }; } + checkBackendActivation(runId: string, candidateKey: string): PolicyGateResult { + const run = this.getRun(runId); + const reasons: string[] = []; + const warnings: string[] = []; + if (run.status !== 'completed') reasons.push('run_not_completed'); + this.appendDatasetGateReasons(run, reasons); + const candidate = run.candidates.find((item) => item.key === candidateKey); + if (!candidate || candidate.backendTarget.provider !== 'qdrant') { + reasons.push('qdrant_candidate_not_found'); + } + const currentPolicy = this.qualityLab.getCurrentPolicy(); + const baseline = run.candidates.find((item) => ( + item.key === qualityCandidateKey({ provider: 'memory' }, currentPolicy.config) + )); + if (!baseline) reasons.push('memory_baseline_missing'); + if (candidate && baseline) { + if (policyKey(candidate.policy) !== policyKey(currentPolicy.config)) { + reasons.push('current_policy_result_missing'); + } + this.compareGateMetrics(candidate, baseline, reasons, warnings); + } + if (!run.knowledgeFingerprint || run.knowledgeFingerprint !== this.knowledgeFingerprint()) { + reasons.push('knowledge_fingerprint_changed'); + } + if (run.activePolicyId !== currentPolicy.id) reasons.push('current_policy_changed'); + return { eligible: reasons.length === 0, warnings, reasons }; + } + activateCandidate(params: { runId: string; candidateKey: string; @@ -241,7 +283,6 @@ export class QualityRunService { this.repo.markRunning(run.id, this.now().toISOString()); try { const policies = this.readPolicyGrid(run.id); - const retrieved: RetrievedQualityCase[] = []; let embeddingCalls = 0; let estimatedTokens = 0; const entries = run.datasetVersionIds.flatMap((versionId) => { @@ -251,11 +292,11 @@ export class QualityRunService { return this.qualityLab.listCases(versionId).map((testCase) => ({ version, testCase })); }); const currentEntries = entries.filter(({ version }) => version.targetKind === 'current'); - const batchStarted = performance.now(); - const currentCandidates = currentEntries.length > 0 - ? await this.searchCurrentBatch(currentEntries.map(({ testCase }) => testCase.query)) - : []; - const batchLatency = performance.now() - batchStarted; + const currentQueries = currentEntries.map(({ testCase }) => testCase.query); + const queryEmbeddings = currentEntries.length > 0 && !this.usesInjectedSearch + ? (await getLLMClient().embed(currentQueries.map(expandRetrievalQuery))) + .map((result) => result.embedding) + : currentQueries.map(() => []); if (currentEntries.length > 0) { embeddingCalls = 1; estimatedTokens = currentEntries.reduce( @@ -263,35 +304,47 @@ export class QualityRunService { 0, ); } - let currentIndex = 0; - for (const { version, testCase } of entries) { - if (this.getRun(run.id).cancelRequested) { - this.repo.markCancelled(run.id, this.now().toISOString()); - return; + const candidates = []; + let progress = 0; + for (const target of run.backendTargets) { + const currentResults = currentEntries.length > 0 + ? await this.searchTargetBatch(target, currentQueries, queryEmbeddings) + : []; + let currentIndex = 0; + const retrieved: RetrievedQualityCase[] = []; + for (const { version, testCase } of entries) { + if (this.getRun(run.id).cancelRequested) { + this.repo.markCancelled(run.id, this.now().toISOString()); + return; + } + const fixtureStarted = performance.now(); + const currentResult = version.targetKind === 'current' + ? currentResults[currentIndex++] + : undefined; + const targetCandidates = currentResult?.candidates + ?? qualityFixtureCandidates(testCase); + retrieved.push({ + testCase, + candidates: targetCandidates, + latencyMs: version.targetKind === 'fixture' + ? Number((performance.now() - fixtureStarted).toFixed(3)) + : currentResult?.latencyMs ?? 0, + }); + progress += 1; + this.repo.updateProgress(run.id, progress); } - const fixtureStarted = performance.now(); - const candidates = version.targetKind === 'fixture' - ? qualityFixtureCandidates(testCase) - : currentCandidates[currentIndex++] ?? []; - retrieved.push({ - testCase, - candidates, - latencyMs: version.targetKind === 'fixture' - ? Number((performance.now() - fixtureStarted).toFixed(3)) - : Number((batchLatency / currentEntries.length).toFixed(3)), - }); - this.repo.updateProgress(run.id, retrieved.length); + candidates.push(...evaluateQualityCandidates({ + cases: retrieved, + policies, + backendTarget: target, + embeddingCallCount: embeddingCalls, + estimatedTokenCount: estimatedTokens, + })); } - const candidates = evaluateQualityCandidates({ - cases: retrieved, - policies, - embeddingCallCount: embeddingCalls, - estimatedTokenCount: estimatedTokens, - }); this.repo.saveCompleted(run.id, candidates, this.now().toISOString()); } catch (error) { logger.error({ - err: error, + errorName: error instanceof Error ? error.name : 'UnknownError', runId: run.id, }, 'Quality evaluation run failed'); this.repo.markFailed( @@ -303,25 +356,11 @@ export class QualityRunService { } knowledgeFingerprint(): string { - const rows = this.db.prepare( - `SELECT 'faq' AS type, id, updated_at AS revision, COALESCE(embedding_profile, '') AS profile - FROM faq_entries WHERE is_active = 1 - UNION ALL - SELECT 'document' AS type, chunk.id, document.updated_at AS revision, - COALESCE(chunk.embedding_profile, '') AS profile - FROM document_chunks chunk - JOIN documents document ON document.id = chunk.document_id - WHERE document.is_active = 1 AND document.status = 'ready' - ORDER BY type, id`, - ).all(); - return createHash('sha256').update(JSON.stringify(rows)).digest('hex'); + return knowledgeFingerprint(this.db); } private readPolicyGrid(runId: string): RetrievalPolicyConfig[] { - const row = this.db.prepare( - 'SELECT policy_grid FROM quality_runs WHERE id = ?', - ).get(runId) as { policy_grid: string }; - return JSON.parse(row.policy_grid) as RetrievalPolicyConfig[]; + return this.repo.getPolicyGrid(runId); } private validatePolicies(policies: RetrievalPolicyConfig[]): RetrievalPolicyConfig[] { @@ -347,6 +386,131 @@ export class QualityRunService { return [...unique.values()]; } + private validateBackendTargets(targets: QualityBackendTarget[]): QualityBackendTarget[] { + const unique = new Map(); + for (const target of targets) { + if (target.provider === 'memory') { + unique.set('memory', target); + continue; + } + const job = this.getIndexJob(target.indexJobId); + if (job.status !== 'ready') { + throw new ConflictError('Only ready Qdrant index jobs can be evaluated'); + } + if (job.knowledgeFingerprint !== this.knowledgeFingerprint()) { + throw new ConflictError('Qdrant index job knowledge fingerprint is stale'); + } + unique.set(`qdrant:${job.id}`, target); + } + if (unique.size === 0 || unique.size > 10) { + throw new ValidationError('A run must target between 1 and 10 backends'); + } + return [...unique.values()]; + } + + private async searchTargetBatch( + target: QualityBackendTarget, + queries: string[], + embeddings: number[][], + ): Promise> { + let searchOne: (query: string, embedding: number[]) => Promise; + if (this.searchBackendBatch) { + searchOne = async (query, embedding) => ( + (await this.searchBackendBatch!(target, [query], [embedding]))[0] ?? [] + ); + } else if (target.provider === 'memory' && this.usesInjectedSearch) { + searchOne = async (query) => (await this.searchCurrentBatch([query]))[0] ?? []; + } else { + const vectorStore = target.provider === 'memory' + ? new InMemoryVectorStore() + : this.qdrantStoreForJob(target.indexJobId); + const retriever = new KnowledgeRetriever( + vectorStore, + async (texts) => (await getLLMClient().embed(texts)).map((result) => result.embedding), + [ + new FaqKnowledgeAdapter(new FaqRepo(this.db)), + new DocumentKnowledgeAdapter(new DocumentRepo(this.db)), + ], + ); + await retriever.initialize(); + searchOne = async (query, embedding) => ( + await retriever.searchCandidatesBatchWithEmbeddings([query], [embedding], 100) + )[0] ?? []; + } + return mapWithConcurrency( + queries, + QUALITY_SEARCH_CONCURRENCY, + async (query, index) => { + const started = performance.now(); + const candidates = await searchOne(query, embeddings[index]); + return { + candidates, + latencyMs: Number((performance.now() - started).toFixed(3)), + }; + }, + ); + } + + private qdrantStoreForJob(indexJobId: string): QdrantVectorStore { + const job = this.getIndexJob(indexJobId); + if (job.status !== 'ready') throw new ConflictError('Qdrant index job is not ready'); + return createQdrantVectorStore({ + url: config.vectorStore.qdrantUrl, + apiKey: config.vectorStore.qdrantApiKey, + timeoutMs: config.vectorStore.timeoutMs, + collectionAlias: job.collection, + }); + } + + private compareGateMetrics( + candidate: QualityRun['candidates'][number], + baseline: QualityRun['candidates'][number], + reasons: string[], + warnings: string[], + ): void { + if (candidate.metrics.unsafeAnswerCount !== 0) reasons.push('unsafe_answers_present'); + if (candidate.metrics.overRefusalCount > baseline.metrics.overRefusalCount) { + reasons.push('over_refusal_regressed'); + } + if (candidate.metrics.decisionAccuracy < baseline.metrics.decisionAccuracy) { + reasons.push('decision_accuracy_regressed'); + } + if (candidate.metrics.recallAt3 < baseline.metrics.recallAt3) { + reasons.push('recall_at_3_regressed'); + } + if (candidate.metrics.mrr < baseline.metrics.mrr) reasons.push('mrr_regressed'); + if ( + baseline.metrics.p95LatencyMs > 0 + && candidate.metrics.p95LatencyMs > baseline.metrics.p95LatencyMs * 1.25 + ) warnings.push('p95_latency_increase_over_25_percent'); + } + + private appendDatasetGateReasons(run: QualityRun, reasons: string[]): void { + if (!run.datasetVersionIds.includes(QUALITY_BASELINE_VERSION_ID)) { + reasons.push('builtin_baseline_required'); + } + const currentVersions = run.datasetVersionIds + .map((id) => this.qualityLab.getVersion(id)) + .filter((version) => version?.targetKind === 'current'); + if (currentVersions.length === 0) reasons.push('current_knowledge_dataset_required'); + const hasCompleteCurrentVersion = currentVersions.some((version) => { + const cases = this.qualityLab.listCases(version!.id); + const answerable = cases.filter( + (testCase) => testCase.expectedGroundingStatus === 'sufficient', + ).length; + const insufficient = cases.filter( + (testCase) => testCase.expectedGroundingStatus === 'insufficient', + ).length; + const highRisk = cases.filter( + (testCase) => ['high_risk', 'escalated'].includes(testCase.expectedGroundingStatus), + ).length; + return cases.length >= 12 && answerable >= 6 && insufficient >= 4 && highRisk >= 2; + }); + if (currentVersions.length > 0 && !hasCompleteCurrentVersion) { + reasons.push('current_knowledge_coverage_insufficient'); + } + } + private async drain(): Promise { if (this.draining) return; this.draining = true; @@ -358,6 +522,27 @@ export class QualityRunService { } } +async function mapWithConcurrency( + values: T[], + concurrency: number, + work: (value: T, index: number) => Promise, +): Promise { + const results = new Array(values.length); + let nextIndex = 0; + const workers = Array.from( + { length: Math.min(concurrency, values.length) }, + async () => { + while (nextIndex < values.length) { + const index = nextIndex; + nextIndex += 1; + results[index] = await work(values[index], index); + } + }, + ); + await Promise.all(workers); + return results; +} + export function qualityFixtureCandidates(testCase: QualityCase): RetrievalResult[] { const queryTerms = localTerms(testCase.query); return QUALITY_BASELINE_KNOWLEDGE diff --git a/server/services/retrieval-index-job.service.ts b/server/services/retrieval-index-job.service.ts new file mode 100644 index 0000000..bb82c20 --- /dev/null +++ b/server/services/retrieval-index-job.service.ts @@ -0,0 +1,595 @@ +import { createHash } from 'node:crypto'; +import Database from 'better-sqlite3'; +import { QdrantClient, withHeaders } from '@qdrant/js-client-rest'; +import { v4 as uuidv4 } from 'uuid'; +import { config } from '../config'; +import { RetrievalIndexJobRepo } from '../db/repos/retrieval-index-job.repo'; +import { RetrievalIndexKnowledgeRepo } from '../db/repos/retrieval-index-knowledge.repo'; +import { QualityRunRepo } from '../db/repos/quality-run.repo'; +import type { VectorRecord } from '../ai/vector-store'; +import { + QdrantRequestError, + QdrantVectorStore, +} from '../ai/qdrant-vector-store'; +import type { + RetrievalActivationCheck, + RetrievalIndexJob, +} from '../types/retrieval-ops'; +import { ConflictError, NotFoundError, ValidationError } from '../utils/errors'; +import { logger } from '../utils/logger'; +import { knowledgeFingerprint } from './knowledge-fingerprint'; +import { qualityCandidateKey } from '../eval/quality-evaluator'; + +const INDEX_BATCH_SIZE = 100; + +export interface QdrantCollectionInfo { + dimensions: number; + count: number; +} + +export interface QdrantCollectionControl { + createCollection(name: string, dimensions: number, traceId?: string): Promise; + collectionInfo(name: string, traceId?: string): Promise; + currentAliasCollection(traceId?: string): Promise; + switchAlias( + nextCollection: string, + expectedCurrent: string | null, + traceId?: string, + ): Promise; +} + +export interface RetrievalIndexVectorWriter { + upsert(records: VectorRecord[], traceId?: string): Promise; +} + +interface RetrievalIndexJobServiceOptions { + control?: QdrantCollectionControl; + writerFactory?: (collection: string) => RetrievalIndexVectorWriter; + collectionPrefix?: string; + autoDrain?: boolean; + now?: () => Date; + activationGate?: (job: RetrievalIndexJob) => RetrievalActivationCheck; +} + +export class RetrievalIndexJobService { + private readonly repo: RetrievalIndexJobRepo; + private readonly knowledgeRepo: RetrievalIndexKnowledgeRepo; + private readonly qualityRunRepo: QualityRunRepo; + private readonly control: QdrantCollectionControl; + private readonly writerFactory: (collection: string) => RetrievalIndexVectorWriter; + private readonly collectionPrefix: string; + private readonly autoDrain: boolean; + private readonly now: () => Date; + private readonly configured: boolean; + private draining = false; + private readonly injectedActivationGate?: ( + job: RetrievalIndexJob, + ) => RetrievalActivationCheck; + + constructor( + private readonly db: Database.Database, + options: RetrievalIndexJobServiceOptions = {}, + ) { + this.repo = new RetrievalIndexJobRepo(db); + this.knowledgeRepo = new RetrievalIndexKnowledgeRepo(db); + this.qualityRunRepo = new QualityRunRepo(db); + this.configured = Boolean(options.control) || config.vectorStore.provider === 'qdrant'; + this.control = options.control ?? ( + this.configured ? createQdrantCollectionControl() : unavailableQdrantControl() + ); + this.writerFactory = options.writerFactory ?? ((collection) => { + const store = new QdrantVectorStore({ + client: createQdrantClient(), + collectionAlias: collection, + }); + return { upsert: (records, traceId) => store.upsertBatch(records, traceId) }; + }); + this.collectionPrefix = options.collectionPrefix ?? config.vectorStore.collectionPrefix; + this.autoDrain = options.autoDrain ?? true; + this.now = options.now ?? (() => new Date()); + this.injectedActivationGate = options.activationGate; + } + + start(): void { + this.repo.interruptRunning(this.now().toISOString()); + if (this.configured) { + queueMicrotask(() => void this.reconcilePendingIntent().catch((error) => { + logger.warn({ + failureCode: safeIndexErrorCode(error), + }, 'Pending retrieval alias operation could not be reconciled'); + })); + } + if (this.autoDrain) queueMicrotask(() => void this.drain()); + } + + async createJob(params: { createdBy: string }): Promise { + this.requireQdrantConfigured(); + const snapshot = this.readSnapshotMetadata(); + const reusable = this.repo.findReusable(snapshot.fingerprint, snapshot.embeddingProfile); + if (reusable) return reusable; + const suffix = uuidv4().replace(/-/g, '').slice(0, 12); + const timestamp = this.now().toISOString().replace(/\D/g, '').slice(0, 14); + const job = this.repo.create({ + collection: `${this.collectionPrefix}_${timestamp}_${suffix}`, + embeddingProfile: snapshot.embeddingProfile, + vectorDimension: snapshot.vectorDimension, + knowledgeFingerprint: snapshot.fingerprint, + expectedCount: snapshot.count, + createdBy: params.createdBy, + now: this.now().toISOString(), + }); + if (this.autoDrain) queueMicrotask(() => void this.drain()); + return job; + } + + getJob(id: string): RetrievalIndexJob { + let job = this.repo.get(id); + if (!job) throw new NotFoundError('Retrieval index job not found'); + if ( + ['queued', 'interrupted', 'ready'].includes(job.status) + && !this.repo.hasPendingIntent(job.id) + && job.knowledgeFingerprint !== knowledgeFingerprint(this.db) + ) { + this.repo.markStale(job.id, this.now().toISOString()); + job = this.repo.get(id) as RetrievalIndexJob; + } + return job; + } + + listJobs(page: number = 1, pageSize: number = 20): { + items: RetrievalIndexJob[]; + total: number; + page: number; + pageSize: number; + } { + return { + items: this.repo.list(pageSize, (page - 1) * pageSize).map((job) => ( + this.getJob(job.id) + )), + total: this.repo.count(), + page, + pageSize, + }; + } + + async processNext(): Promise { + const pending = this.repo.nextPending(); + if (!pending) return; + const traceId = uuidv4(); + const snapshot = this.readSnapshotMetadata(); + if ( + snapshot.fingerprint !== pending.knowledgeFingerprint + || snapshot.embeddingProfile !== pending.embeddingProfile + || snapshot.vectorDimension !== pending.vectorDimension + || snapshot.count !== pending.expectedCount + ) { + this.repo.markStale(pending.id, this.now().toISOString()); + return; + } + if (!this.repo.markRunning(pending.id, this.now().toISOString())) return; + try { + const existing = await this.control.collectionInfo(pending.collection, traceId); + if (!existing) { + if (pending.checkpoint > 0) { + throw new IndexJobError('qdrant_collection_missing'); + } + await this.control.createCollection( + pending.collection, + pending.vectorDimension, + traceId, + ); + } else if (existing.dimensions !== pending.vectorDimension) { + throw new IndexJobError('qdrant_dimension_mismatch'); + } + const writer = this.writerFactory(pending.collection); + let checkpoint = pending.checkpoint; + let cursor = this.knowledgeRepo.cursorAt(checkpoint); + while (checkpoint < pending.expectedCount) { + const page = this.knowledgeRepo.readPage(cursor, INDEX_BATCH_SIZE); + const batch = page.records; + if (batch.length === 0) throw new IndexJobError('knowledge_batch_missing'); + await writer.upsert(batch, traceId); + checkpoint += batch.length; + cursor = page.nextCursor; + this.repo.saveCheckpoint( + pending.id, + checkpoint, + checkpoint, + this.now().toISOString(), + ); + } + const verified = await this.control.collectionInfo(pending.collection, traceId); + if (!verified) throw new IndexJobError('qdrant_collection_missing'); + if (verified.dimensions !== pending.vectorDimension) { + throw new IndexJobError('qdrant_dimension_mismatch'); + } + if (verified.count !== pending.expectedCount) { + throw new IndexJobError('qdrant_point_count_mismatch'); + } + if (knowledgeFingerprint(this.db) !== pending.knowledgeFingerprint) { + this.repo.markStale(pending.id, this.now().toISOString()); + return; + } + this.repo.markReady(pending.id, this.now().toISOString()); + } catch (error) { + const failureCode = safeIndexErrorCode(error); + logger.warn({ traceId, jobId: pending.id, failureCode }, 'Retrieval index build failed'); + this.repo.markFailed(pending.id, failureCode, this.now().toISOString()); + } + } + + activationCheck(id: string): RetrievalActivationCheck { + const job = this.getJob(id); + if (this.injectedActivationGate) return this.injectedActivationGate(job); + const reasons: string[] = []; + const warnings: string[] = []; + if (job.status !== 'ready') reasons.push('index_job_not_ready'); + if (job.knowledgeFingerprint !== knowledgeFingerprint(this.db)) { + reasons.push('knowledge_fingerprint_changed'); + } + const { QualityLabService } = require('./quality-lab.service') as typeof import( + './quality-lab.service' + ); + const qualityLab = new QualityLabService(this.db); + const currentPolicy = qualityLab.getCurrentPolicy(); + const candidateKey = qualityCandidateKey( + { provider: 'qdrant', indexJobId: id }, + currentPolicy.config, + ); + const quality = this.qualityRunRepo.findLatestCompletedCandidate(candidateKey); + if (!quality) { + reasons.push('quality_run_required'); + return { + eligible: false, + warnings, + reasons, + qualityRunId: null, + candidateKey: null, + }; + } + const { QualityRunService } = require('./quality-run.service') as typeof import( + './quality-run.service' + ); + const gate = new QualityRunService(this.db, { + qualityLab, + autoDrain: false, + getIndexJob: (jobId) => this.getJob(jobId), + }).checkBackendActivation(quality.runId, quality.candidateKey); + reasons.push(...gate.reasons); + warnings.push(...gate.warnings); + return { + eligible: reasons.length === 0, + warnings: [...new Set(warnings)], + reasons: [...new Set(reasons)], + qualityRunId: quality.runId, + candidateKey: quality.candidateKey, + }; + } + + async activate(params: { + id: string; + expectedCurrentCollection: string | null; + confirmLatencyWarning: boolean; + }): Promise { + const reconciled = await this.reconcilePendingIntent(); + if (reconciled?.id === params.id) return reconciled; + const job = this.getJob(params.id); + const gate = this.activationCheck(job.id); + if (!gate.eligible) { + throw new ConflictError(`Retrieval activation gate failed: ${gate.reasons.join(', ')}`); + } + if (gate.warnings.length > 0 && !params.confirmLatencyWarning) { + throw new ConflictError('Retrieval activation requires latency warning confirmation'); + } + const current = await this.control.currentAliasCollection(uuidv4()); + if (current !== params.expectedCurrentCollection) { + throw new ConflictError('Qdrant alias changed; refresh before activating'); + } + this.repo.prepareActivation(job.id, current, this.now().toISOString()); + await this.control.switchAlias(job.collection, current, uuidv4()); + this.repo.completeActivation(job.id, this.now().toISOString()); + return this.getJob(job.id); + } + + async rollback(params: { + id: string; + expectedCurrentCollection: string; + }): Promise { + const pendingBeforeReconcile = this.repo.findPendingIntent(); + const reconciled = await this.reconcilePendingIntent(); + if (reconciled) { + if ( + pendingBeforeReconcile?.intent === 'rollback' + && pendingBeforeReconcile.job.id === params.id + ) return reconciled; + throw new ConflictError( + 'A pending retrieval alias operation was reconciled; refresh before rolling back', + ); + } + const job = this.getJob(params.id); + if (job.status !== 'active' || !job.previousCollection) { + throw new ConflictError('Active index job has no rollback target'); + } + if (job.collection !== params.expectedCurrentCollection) { + throw new ConflictError('Expected current collection does not match the active job'); + } + const target = this.repo.findByCollection(job.previousCollection); + if (!target || target.status !== 'rolled_back') { + throw new ConflictError('Previous verified collection is unavailable'); + } + if (target.knowledgeFingerprint !== knowledgeFingerprint(this.db)) { + throw new ConflictError('Previous collection knowledge fingerprint is stale'); + } + const current = await this.control.currentAliasCollection(uuidv4()); + if (current !== params.expectedCurrentCollection) { + throw new ConflictError('Qdrant alias changed; refresh before rolling back'); + } + this.repo.prepareRollback(job.id, this.now().toISOString()); + await this.control.switchAlias(target.collection, job.collection, uuidv4()); + this.repo.completeRollback(job.id, target.id, this.now().toISOString()); + return this.getJob(target.id); + } + + async status(): Promise<{ + provider: 'memory' | 'qdrant'; + qdrantConfigured: boolean; + qdrantHealth: 'healthy' | 'degraded' | 'unavailable' | 'not_configured'; + alias: string; + collection: string | null; + points: number | null; + dimensions: number | null; + syncStatus: 'synced' | 'stale' | 'not_configured'; + }> { + if (!this.configured) { + return { + provider: config.vectorStore.provider, + qdrantConfigured: false, + qdrantHealth: 'not_configured', + alias: config.vectorStore.collectionAlias, + collection: null, + points: null, + dimensions: null, + syncStatus: 'not_configured', + }; + } + try { + const collection = await this.control.currentAliasCollection(uuidv4()); + const info = collection ? await this.control.collectionInfo(collection, uuidv4()) : null; + const job = collection ? this.repo.findByCollection(collection) : null; + return { + provider: config.vectorStore.provider, + qdrantConfigured: true, + qdrantHealth: collection && info ? 'healthy' : 'degraded', + alias: config.vectorStore.collectionAlias, + collection, + points: info?.count ?? null, + dimensions: info?.dimensions ?? null, + syncStatus: job?.knowledgeFingerprint === knowledgeFingerprint(this.db) + ? 'synced' + : 'stale', + }; + } catch { + return { + provider: config.vectorStore.provider, + qdrantConfigured: true, + qdrantHealth: 'unavailable', + alias: config.vectorStore.collectionAlias, + collection: null, + points: null, + dimensions: null, + syncStatus: 'stale', + }; + } + } + + private readSnapshotMetadata(): { + fingerprint: string; + embeddingProfile: string; + vectorDimension: number; + count: number; + } { + const metadata = this.knowledgeRepo.inspect(INDEX_BATCH_SIZE); + const profileHash = createHash('sha256') + .update(JSON.stringify(metadata.embeddingProfiles)) + .digest('hex') + .slice(0, 16); + return { + fingerprint: knowledgeFingerprint(this.db), + embeddingProfile: `combined:${profileHash}`, + vectorDimension: metadata.vectorDimension, + count: metadata.count, + }; + } + + private async reconcilePendingIntent(): Promise { + const pending = this.repo.findPendingIntent(); + if (!pending) return null; + const current = await this.control.currentAliasCollection(uuidv4()); + if (pending.intent === 'activate') { + if (current === pending.job.collection) { + this.repo.completeActivation(pending.job.id, this.now().toISOString()); + return this.repo.get(pending.job.id); + } + if (current === pending.expectedCollection) { + await this.control.switchAlias( + pending.job.collection, + pending.expectedCollection, + uuidv4(), + ); + this.repo.completeActivation(pending.job.id, this.now().toISOString()); + return this.repo.get(pending.job.id); + } + throw new ConflictError('Qdrant alias no longer matches the pending activation'); + } + + const target = pending.job.previousCollection + ? this.repo.findByCollection(pending.job.previousCollection) + : null; + if (!target) { + throw new ConflictError('Pending rollback target is unavailable'); + } + if (current === target.collection) { + this.repo.completeRollback( + pending.job.id, + target.id, + this.now().toISOString(), + ); + return this.repo.get(target.id); + } + if (current === pending.expectedCollection) { + await this.control.switchAlias( + target.collection, + pending.expectedCollection, + uuidv4(), + ); + this.repo.completeRollback( + pending.job.id, + target.id, + this.now().toISOString(), + ); + return this.repo.get(target.id); + } + throw new ConflictError('Qdrant alias no longer matches the pending rollback'); + } + + private requireQdrantConfigured(): void { + if (!this.configured) { + throw new ConflictError('Qdrant is not configured'); + } + } + + private async drain(): Promise { + if (this.draining) return; + this.draining = true; + try { + while (this.repo.nextPending()) await this.processNext(); + } finally { + this.draining = false; + } + } +} + +class IndexJobError extends Error { + constructor(readonly code: string) { + super(code); + this.name = 'IndexJobError'; + } +} + +function safeIndexErrorCode(error: unknown): string { + if (error instanceof IndexJobError) return error.code; + if (error instanceof QdrantRequestError) return error.code; + if (error instanceof ValidationError) return 'invalid_knowledge_snapshot'; + const name = error instanceof Error ? error.name.toLowerCase() : ''; + if (name.includes('timeout') || name.includes('abort')) return 'qdrant_timeout'; + return 'qdrant_request_failed'; +} + +type OfficialQdrantClient = InstanceType; + +function createQdrantClient(): OfficialQdrantClient { + return new QdrantClient({ + url: config.vectorStore.qdrantUrl, + apiKey: config.vectorStore.qdrantApiKey || undefined, + timeout: config.vectorStore.timeoutMs, + checkCompatibility: false, + }); +} + +function createQdrantCollectionControl(): QdrantCollectionControl { + const client = createQdrantClient(); + const withTrace = async ( + traceId: string | undefined, + work: () => Promise, + ): Promise => { + try { + return await withHeaders({ 'x-request-id': traceId ?? uuidv4() }, work); + } catch (error) { + if (error instanceof QdrantRequestError || error instanceof ConflictError) throw error; + throw new QdrantRequestError(safeIndexErrorCode(error) === 'qdrant_timeout' + ? 'qdrant_timeout' + : 'qdrant_request_failed'); + } + }; + return { + createCollection: (name, dimensions, traceId) => withTrace( + traceId, + () => client.createCollection(name, { + vectors: { size: dimensions, distance: 'Cosine' }, + }), + ), + async collectionInfo(name, traceId) { + return withTrace(traceId, async () => { + try { + const collection = await client.getCollection(name); + const vectors = collection.config.params.vectors; + const dimensions = ( + vectors + && typeof vectors === 'object' + && 'size' in vectors + && typeof vectors.size === 'number' + ) ? vectors.size : 0; + return { + dimensions, + count: collection.points_count ?? collection.indexed_vectors_count ?? 0, + }; + } catch (error) { + const status = (error as { status?: number }).status; + if (status === 404) return null; + throw error; + } + }); + }, + async currentAliasCollection(traceId) { + const response = await withTrace(traceId, () => client.getAliases()); + return response.aliases.find( + (alias) => alias.alias_name === config.vectorStore.collectionAlias, + )?.collection_name ?? null; + }, + async switchAlias(nextCollection, expectedCurrent, traceId) { + const current = await this.currentAliasCollection(traceId); + if (current !== expectedCurrent) { + throw new ConflictError('Qdrant alias changed; refresh before activating'); + } + const actions = current + ? [ + { delete_alias: { alias_name: config.vectorStore.collectionAlias } }, + { + create_alias: { + alias_name: config.vectorStore.collectionAlias, + collection_name: nextCollection, + }, + }, + ] + : [{ + create_alias: { + alias_name: config.vectorStore.collectionAlias, + collection_name: nextCollection, + }, + }]; + await withTrace(traceId, () => client.updateCollectionAliases({ actions })); + }, + }; +} + +function unavailableQdrantControl(): QdrantCollectionControl { + const unavailable = async (): Promise => { + throw new ConflictError('Qdrant is not configured'); + }; + return { + createCollection: unavailable, + collectionInfo: unavailable, + currentAliasCollection: unavailable, + switchAlias: unavailable, + }; +} + +let singleton: RetrievalIndexJobService | null = null; + +export function getRetrievalIndexJobService(): RetrievalIndexJobService { + if (!singleton) { + const { getDatabase } = require('../db') as typeof import('../db'); + singleton = new RetrievalIndexJobService(getDatabase()); + } + return singleton; +} diff --git a/server/services/retrieval-trace-collector.ts b/server/services/retrieval-trace-collector.ts new file mode 100644 index 0000000..b51d29a --- /dev/null +++ b/server/services/retrieval-trace-collector.ts @@ -0,0 +1,130 @@ +import { v4 as uuidv4 } from 'uuid'; +import { + RETRIEVAL_TRACE_STAGES, + RetrievalTrace, + RetrievalTraceCandidate, + RetrievalTraceStage, + RetrievalTraceStageName, + RetrievalTraceStageStatus, +} from '../types/retrieval-ops'; + +interface CollectorOptions { + backend: 'memory' | 'qdrant'; + now?: () => Date; +} + +interface RecordStage { + status: RetrievalTraceStageStatus; + latencyMs: number; + inputCount: number; + outputCount: number; + candidates?: RetrievalTraceCandidate[]; + budget?: Record; + errorCode?: string | null; +} + +export class RetrievalTraceCollector { + readonly id = uuidv4(); + readonly backend: 'memory' | 'qdrant'; + private readonly now: () => Date; + private readonly createdAt: string; + private readonly startedAt: number; + private readonly stages = new Map(); + private traceErrorCode: string | null = null; + private forcedFailure = false; + + constructor(options: CollectorOptions) { + this.backend = options.backend; + this.now = options.now ?? (() => new Date()); + this.createdAt = this.now().toISOString(); + this.startedAt = performance.now(); + } + + record(name: RetrievalTraceStageName, stage: RecordStage): void { + const candidateLimit = name === 'grounding' ? 3 : 20; + this.stages.set(name, { + name, + order: RETRIEVAL_TRACE_STAGES.indexOf(name), + status: stage.status, + latencyMs: finiteNonNegative(stage.latencyMs), + inputCount: boundedCount(stage.inputCount), + outputCount: boundedCount(stage.outputCount), + candidates: (stage.candidates ?? []).slice(0, candidateLimit).map(safeCandidate), + budget: Object.fromEntries(Object.entries(stage.budget ?? {}).flatMap(([key, value]) => ( + Number.isFinite(value) && value >= 0 ? [[key, value]] : [] + ))), + errorCode: safeErrorCode(stage.errorCode), + }); + if (stage.status === 'failed') this.forcedFailure = true; + if (stage.status === 'degraded' && !this.traceErrorCode) { + this.traceErrorCode = safeErrorCode(stage.errorCode); + } + } + + fail(errorCode: string): void { + this.forcedFailure = true; + this.traceErrorCode = safeErrorCode(errorCode); + } + + complete(params: { + sessionId: string; + userMessageId: string; + assistantMessageId: string | null; + policyId: string; + }): RetrievalTrace { + const completedAt = this.now().toISOString(); + const stages = RETRIEVAL_TRACE_STAGES.map((name, order) => ( + this.stages.get(name) ?? { + name, + order, + status: 'skipped' as const, + latencyMs: 0, + inputCount: 0, + outputCount: 0, + candidates: [], + budget: {}, + errorCode: null, + } + )); + const degraded = stages.some((stage) => stage.status === 'degraded'); + return { + id: this.id, + sessionId: params.sessionId, + userMessageId: params.userMessageId, + assistantMessageId: params.assistantMessageId, + policyId: params.policyId, + backend: this.backend, + status: this.forcedFailure ? 'failed' : degraded ? 'degraded' : 'completed', + errorCode: this.traceErrorCode, + totalLatencyMs: finiteNonNegative(performance.now() - this.startedAt), + stages, + createdAt: this.createdAt, + completedAt, + }; + } +} + +function safeCandidate(candidate: RetrievalTraceCandidate): RetrievalTraceCandidate { + return { + knowledgeType: candidate.knowledgeType, + knowledgeId: String(candidate.knowledgeId).slice(0, 200), + score: Number.isFinite(candidate.score) ? candidate.score : undefined, + rank: Number.isInteger(candidate.rank) && (candidate.rank ?? 0) > 0 + ? candidate.rank + : undefined, + source: candidate.source, + }; +} + +function finiteNonNegative(value: number): number { + return Number.isFinite(value) && value >= 0 ? Number(value.toFixed(3)) : 0; +} + +function boundedCount(value: number): number { + return Number.isInteger(value) && value >= 0 ? Math.min(value, 1_000_000) : 0; +} + +function safeErrorCode(value: string | null | undefined): string | null { + if (!value) return null; + return /^[a-z0-9_:-]{1,80}$/i.test(value) ? value : 'retrieval_stage_failed'; +} diff --git a/server/services/retrieval-trace.service.ts b/server/services/retrieval-trace.service.ts new file mode 100644 index 0000000..f1b2cb7 --- /dev/null +++ b/server/services/retrieval-trace.service.ts @@ -0,0 +1,153 @@ +import Database from 'better-sqlite3'; +import { config } from '../config'; +import { getDatabase } from '../db'; +import { RetrievalTraceRepo } from '../db/repos/retrieval-trace.repo'; +import { RetrievalTraceDetailRepo } from '../db/repos/retrieval-trace-detail.repo'; +import type { RetrievalTrace, RetrievalTraceStatus } from '../types/retrieval-ops'; +import { NotFoundError } from '../utils/errors'; + +interface RetrievalTraceServiceOptions { + retentionDays?: number; + now?: () => Date; +} + +export class RetrievalTraceService { + private readonly repo: RetrievalTraceRepo; + private readonly detailRepo: RetrievalTraceDetailRepo; + private readonly retentionDays: number; + private readonly now: () => Date; + private timer: NodeJS.Timeout | null = null; + + constructor( + db: Database.Database = getDatabase(), + options: RetrievalTraceServiceOptions = {}, + ) { + this.repo = new RetrievalTraceRepo(db); + this.detailRepo = new RetrievalTraceDetailRepo(db); + this.retentionDays = options.retentionDays ?? config.vectorStore.traceRetentionDays; + this.now = options.now ?? (() => new Date()); + } + + start(): void { + this.cleanupExpired(); + if (this.timer) return; + this.timer = setInterval(() => this.cleanupExpired(), 24 * 60 * 60 * 1000); + this.timer.unref(); + } + + stop(): void { + if (this.timer) clearInterval(this.timer); + this.timer = null; + } + + persist(trace: RetrievalTrace): void { + this.repo.create(trace); + } + + getTrace(id: string): RetrievalTrace { + const trace = this.repo.get(id); + if (!trace) throw new NotFoundError('Retrieval trace not found'); + return trace; + } + + getTraceDetail(id: string): { + trace: RetrievalTrace; + messages: { + user: { id: string; content: string } | null; + assistant: { id: string; content: string } | null; + }; + knowledge: Array<{ + knowledgeType: 'faq' | 'document'; + knowledgeId: string; + title: string; + content: string; + available: boolean; + }>; + } { + const trace = this.getTrace(id); + const messageRows = this.detailRepo.findMessages([ + trace.userMessageId, + trace.assistantMessageId ?? '', + ]); + const byMessageId = new Map(messageRows.map((row) => [row.id, row])); + const candidateKeys = new Map(); + for (const stage of trace.stages) { + for (const candidate of stage.candidates) { + candidateKeys.set( + `${candidate.knowledgeType}:${candidate.knowledgeId}`, + { knowledgeType: candidate.knowledgeType, id: candidate.knowledgeId }, + ); + } + } + const faqIds = [...candidateKeys.values()] + .filter((item) => item.knowledgeType === 'faq') + .map((item) => item.id); + const documentIds = [...candidateKeys.values()] + .filter((item) => item.knowledgeType === 'document') + .map((item) => item.id); + const knowledge: Array<{ + knowledgeType: 'faq' | 'document'; + knowledgeId: string; + title: string; + content: string; + available: boolean; + }> = [ + ...this.detailRepo.findFaqs(faqIds), + ...this.detailRepo.findDocumentChunks(documentIds), + ]; + const resolved = new Set(knowledge.map((item) => ( + `${item.knowledgeType}:${item.knowledgeId}` + ))); + for (const [key, candidate] of candidateKeys) { + if (resolved.has(key)) continue; + knowledge.push({ + knowledgeType: candidate.knowledgeType, + knowledgeId: candidate.id, + title: '', + content: '', + available: false, + }); + } + return { + trace, + messages: { + user: byMessageId.get(trace.userMessageId) ?? null, + assistant: trace.assistantMessageId + ? byMessageId.get(trace.assistantMessageId) ?? null + : null, + }, + knowledge, + }; + } + + listTraces(params: { + page: number; + pageSize: number; + status?: RetrievalTraceStatus; + backend?: 'memory' | 'qdrant'; + sessionId?: string; + createdFrom?: string; + createdTo?: string; + }): { items: RetrievalTrace[]; total: number; page: number; pageSize: number } { + const result = this.repo.list({ + ...params, + limit: params.pageSize, + offset: (params.page - 1) * params.pageSize, + }); + return { ...result, page: params.page, pageSize: params.pageSize }; + } + + cleanupExpired(): number { + const cutoff = new Date( + this.now().getTime() - this.retentionDays * 24 * 60 * 60 * 1000, + ).toISOString(); + return this.repo.deleteBefore(cutoff); + } +} + +let singleton: RetrievalTraceService | null = null; + +export function getRetrievalTraceService(): RetrievalTraceService { + if (!singleton) singleton = new RetrievalTraceService(); + return singleton; +} diff --git a/server/tests/chat-route.test.ts b/server/tests/chat-route.test.ts index 95e2900..737945c 100644 --- a/server/tests/chat-route.test.ts +++ b/server/tests/chat-route.test.ts @@ -275,6 +275,26 @@ async function main(): Promise { assistantCountBeforeEmptyStream.total, 'empty generated answers must not be persisted as successful assistant messages', ); + const latestTrace = db.prepare(` + SELECT id, status, assistant_message_id AS assistantMessageId + FROM retrieval_traces + ORDER BY created_at DESC + LIMIT 1 + `).get() as { + id: string; + status: string; + assistantMessageId: string | null; + }; + assert.equal(latestTrace.status, 'failed'); + assert.equal(latestTrace.assistantMessageId, null); + const traceStages = db.prepare( + 'SELECT candidates AS candidateJson FROM retrieval_trace_stages WHERE trace_id = ?', + ).all(latestTrace.id) as Array<{ candidateJson: string }>; + assert.equal(traceStages.length, 8); + assert.ok( + traceStages.every((stage) => !stage.candidateJson.includes('测试空生成流')), + 'trace stage payloads must not copy the customer question', + ); console.log('Chat route failure checks passed'); } finally { diff --git a/server/tests/knowledge-retriever.test.ts b/server/tests/knowledge-retriever.test.ts index 01a8c82..ac8771a 100644 --- a/server/tests/knowledge-retriever.test.ts +++ b/server/tests/knowledge-retriever.test.ts @@ -3,7 +3,13 @@ import Database from 'better-sqlite3'; import { initSchema } from '../db'; import { DocumentRepo } from '../db/repos/document.repo'; import { FaqRepo } from '../db/repos/faq.repo'; -import { InMemoryVectorStore } from '../ai/vector-store'; +import { + InMemoryVectorStore, + VectorSearchResult, + VectorStore, + VectorStoreHealth, + VectorStoreStats, +} from '../ai/vector-store'; import { KnowledgeAdapter, KnowledgeIndexItem, @@ -85,15 +91,61 @@ class ProfiledAdapter extends MutableAdapter { } } -class FailOnceVectorStore extends InMemoryVectorStore { +class FailOnceVectorStore extends InMemoryVectorStore { failOnId: string | null = null; - upsert(entry: KnowledgeIndexItem, embedding: number[]): void { - if (entry.id === this.failOnId) { + async upsertBatch(records: Array<{ + id: string; + knowledgeType: KnowledgeType; + revision: string; + embeddingProfile: string; + embedding: number[]; + }>): Promise { + if (records.some((record) => record.id === this.failOnId)) { this.failOnId = null; throw new Error('simulated vector upsert failure'); } - super.upsert(entry, embedding); + await super.upsertBatch(records); + } +} + +class ExternalReadOnlyVectorStore implements VectorStore { + readonly backend = 'qdrant'; + readonly supportsStartupSync = false; + + constructor( + private readonly matches: VectorSearchResult[], + private readonly failSearch = false, + ) {} + + async upsertBatch(): Promise { + throw new Error('startup must not rebuild Qdrant'); + } + + async delete(): Promise { + throw new Error('startup must not clear Qdrant'); + } + + async search(): Promise { + if (this.failSearch) throw new Error('qdrant timeout with private response'); + return this.matches; + } + + async stats(): Promise { + return { + indexedCount: this.matches.length, + embeddingDimensions: 2, + updatedAt: null, + }; + } + + async health(): Promise { + return { + backend: 'qdrant', + status: 'healthy', + checkedAt: new Date().toISOString(), + errorCode: null, + }; } } @@ -133,7 +185,7 @@ async function testHybridKnowledgeSearchUsesOneQueryEmbedding(): Promise { [{ ...documentResult, similarity: 0.95, source: 'keyword', keywordScore: 0.95 }], ); const retriever = new KnowledgeRetriever( - new InMemoryVectorStore(), + new InMemoryVectorStore(), async () => { embedCalls += 1; return [[1, 0]]; @@ -156,6 +208,108 @@ async function testHybridKnowledgeSearchUsesOneQueryEmbedding(): Promise { assert.ok(afterFaqRefresh.some((result) => result.knowledgeType === 'document')); } +async function testExternalVectorHitsAreHydratedFromSqlite(): Promise { + const db = new Database(':memory:'); + initSchema(db); + const repo = new FaqRepo(db); + const profile = currentEmbeddingProfile(FAQ_EMBEDDING_INPUT_VERSION); + const active = repo.create({ + question: 'active source', + answer: 'active answer', + category: IntentCategory.GENERAL, + keywords: [], + embedding: [1, 0], + embeddingProfile: profile, + }); + const inactive = repo.create({ + question: 'inactive source', + answer: 'inactive answer', + category: IntentCategory.GENERAL, + keywords: [], + embedding: [1, 0], + embeddingProfile: profile, + isActive: 0, + }); + const stale = repo.create({ + question: 'stale source', + answer: 'stale answer', + category: IntentCategory.GENERAL, + keywords: [], + embedding: [1, 0], + embeddingProfile: profile, + }); + const matches: VectorSearchResult[] = [ + { + id: `faq:${active.id}`, + knowledgeType: 'faq', + revision: active.updatedAt, + embeddingProfile: profile, + score: 0.99, + }, + { + id: `faq:${inactive.id}`, + knowledgeType: 'faq', + revision: inactive.updatedAt, + embeddingProfile: profile, + score: 0.98, + }, + { + id: `faq:${stale.id}`, + knowledgeType: 'faq', + revision: 'outdated-revision', + embeddingProfile: profile, + score: 0.97, + }, + { + id: 'faq:orphan', + knowledgeType: 'faq', + revision: 'missing', + embeddingProfile: profile, + score: 0.96, + }, + ]; + const retriever = new KnowledgeRetriever( + new ExternalReadOnlyVectorStore(matches), + async () => [[1, 0]], + [new FaqKnowledgeAdapter(repo, async () => { + throw new Error('existing embeddings should be reused'); + })], + ); + + const results = await retriever.search('no-keyword-match', 10, ['faq']); + + assert.deepEqual(results.map((result) => result.knowledgeId), [active.id]); + db.close(); +} + +async function testExternalVectorFailureFallsBackToKeywords(): Promise { + const keyword: RetrievalResult = { + knowledgeType: 'faq', + knowledgeId: 'keyword-fallback', + title: 'fallback', + content: 'safe keyword answer', + similarity: 0.95, + source: 'keyword', + keywordScore: 0.95, + }; + const retriever = new KnowledgeRetriever( + new ExternalReadOnlyVectorStore([{ + id: 'faq:unavailable', + knowledgeType: 'faq', + revision: 'v1', + embeddingProfile: 'legacy', + score: 1, + }], true), + async () => [[1, 0]], + [new FakeAdapter('faq', [], [keyword])], + ); + + const results = await retriever.search('fallback', 3, ['faq']); + + assert.equal(results[0].knowledgeId, 'keyword-fallback'); + assert.equal(results[0].source, 'keyword'); +} + async function testQualityBatchUsesOneEmbeddingCall(): Promise { let embedCalls = 0; let embeddedTexts = 0; @@ -167,7 +321,7 @@ async function testQualityBatchUsesOneEmbeddingCall(): Promise { similarity: 0, }; const retriever = new KnowledgeRetriever( - new InMemoryVectorStore(), + new InMemoryVectorStore(), async (texts) => { embedCalls += 1; embeddedTexts += texts.length; @@ -193,7 +347,7 @@ async function testAdapterFailureKeepsKeywordFallbackAndTypeIsolation(): Promise }; const failing = new FailingLoadAdapter('document', [], [keywordDocument]); const fallbackRetriever = new KnowledgeRetriever( - new InMemoryVectorStore(), + new InMemoryVectorStore(), async () => [[1, 0]], [failing], ); @@ -216,7 +370,7 @@ async function testAdapterFailureKeepsKeywordFallbackAndTypeIsolation(): Promise const faqAdapter = new FakeAdapter('faq', [{ id: 'faq:faq-low', result: faqResult, embedding: [0, 1] }], []); const documentAdapter = new FakeAdapter('document', documentItems, []); const isolatedRetriever = new KnowledgeRetriever( - new InMemoryVectorStore(), + new InMemoryVectorStore(), async () => [[1, 0]], [faqAdapter, documentAdapter], ); @@ -238,7 +392,7 @@ async function testExactFaqCannotBeDisplacedByDocumentCandidates(): Promise(), + new InMemoryVectorStore(), async () => [[1, 0]], [ new FakeAdapter('faq', [], [exactFaq]), @@ -274,7 +428,7 @@ async function testMixedSearchKeepsRelevantDocumentCandidate(): Promise { chunkIndex: 1, }; const retriever = new KnowledgeRetriever( - new InMemoryVectorStore(), + new InMemoryVectorStore(), async () => [[1, 0]], [ new FakeAdapter('faq', faqItems, []), @@ -374,7 +528,7 @@ async function testGpuCatalogueWinsRealMixedRetrieval(): Promise { ); const retriever = new KnowledgeRetriever( - new InMemoryVectorStore(), + new InMemoryVectorStore(), embedTexts, [ new FaqKnowledgeAdapter(faqRepo, embedTexts), @@ -536,7 +690,7 @@ async function testRuntimeProfileChangeRefreshesSourceBeforeSearch(): Promise(), + new InMemoryVectorStore(), async () => [[1, 0]], [adapter], ); @@ -561,7 +715,7 @@ async function testConcurrentInitializationIsSingleFlight(): Promise { embedding: [1, 0], }]); const retriever = new KnowledgeRetriever( - new InMemoryVectorStore(), + new InMemoryVectorStore(), async () => [[1, 0]], [adapter], ); @@ -638,7 +792,7 @@ async function testSuccessfulManualRefreshClearsDegradedSource(): Promise [], ); const retriever = new KnowledgeRetriever( - new InMemoryVectorStore(), + new InMemoryVectorStore(), async () => [[1, 0]], [adapter], ); @@ -720,9 +874,9 @@ async function testDirectDocumentReplacementRollsBackAsOneIndexSet(): Promise result.id).sort(), + (await store.search([1, 0], { limit: 10 })).map((result) => result.id).sort(), ['document:new-chunk', 'document:other-chunk'], ); @@ -732,12 +886,12 @@ async function testDirectDocumentReplacementRollsBackAsOneIndexSet(): Promise retriever.replaceDocumentIndexItems('document-1', [failedReplacement]), + await assert.rejects( + retriever.replaceDocumentIndexItems('document-1', [failedReplacement]), /vector upsert failure/, ); assert.deepEqual( - store.search([1, 0], 10).map((result) => result.id).sort(), + (await store.search([1, 0], { limit: 10 })).map((result) => result.id).sort(), ['document:new-chunk', 'document:other-chunk'], 'a failed direct replacement must restore the previous document set', ); @@ -746,6 +900,8 @@ async function testDirectDocumentReplacementRollsBackAsOneIndexSet(): Promise(operation: () => Promise): Promise => withHeaders( + { 'x-request-id': `qdrant-ci-${randomUUID()}` }, + operation, +); + +async function main(): Promise { + try { + await traced(() => client.createCollection(firstCollection, { + vectors: { size: 4, distance: 'Cosine' }, + })); + await traced(() => client.createCollection(secondCollection, { + vectors: { size: 4, distance: 'Cosine' }, + })); + await traced(() => client.updateCollectionAliases({ + actions: [{ + create_alias: { + alias_name: alias, + collection_name: firstCollection, + }, + }], + })); + + const store = createQdrantVectorStore({ + url: qdrantUrl, + apiKey: process.env.QDRANT_API_KEY || '', + timeoutMs: 10_000, + collectionAlias: alias, + }); + await store.upsertBatch([ + { + id: 'faq:integration-faq', + knowledgeType: 'faq', + revision: 'faq-revision-1', + embeddingProfile: 'integration-v1', + embedding: [1, 0, 0, 0], + }, + { + id: 'document:integration-document', + knowledgeType: 'document', + revision: 'document-revision-1', + embeddingProfile: 'integration-v1', + embedding: [0, 1, 0, 0], + }, + ], 'qdrant-integration-upsert'); + const memoryStore = new InMemoryVectorStore(); + await memoryStore.upsertBatch([ + { + id: 'faq:integration-faq', + knowledgeType: 'faq', + revision: 'faq-revision-1', + embeddingProfile: 'integration-v1', + embedding: [1, 0, 0, 0], + }, + { + id: 'document:integration-document', + knowledgeType: 'document', + revision: 'document-revision-1', + embeddingProfile: 'integration-v1', + embedding: [0, 1, 0, 0], + }, + ]); + + const faqResults = await store.search([1, 0, 0, 0], { + limit: 5, + knowledgeTypes: ['faq'], + traceId: 'qdrant-integration-search', + }); + assert.deepEqual(faqResults.map((result) => result.id), ['faq:integration-faq']); + assert.ok(faqResults.every((result) => result.knowledgeType === 'faq')); + const memoryMetrics = await measureBackend(memoryStore); + const qdrantMetrics = await measureBackend(store); + assert.equal(memoryMetrics.recallAt1, 1); + assert.equal(qdrantMetrics.recallAt1, 1); + console.log(JSON.stringify({ + benchmark: 'qdrant-integration-two-case', + memory: memoryMetrics, + qdrant: qdrantMetrics, + p95LatencyDeltaPercent: Number( + (((qdrantMetrics.p95LatencyMs / memoryMetrics.p95LatencyMs) - 1) * 100).toFixed(2), + ), + })); + + const stats = await store.stats('qdrant-integration-stats'); + assert.equal(stats.indexedCount, 2); + assert.equal(stats.embeddingDimensions, 4); + assert.equal((await store.health()).status, 'healthy'); + + const payload = await traced(() => client.scroll(alias, { + limit: 10, + with_payload: true, + with_vector: false, + })); + assert.equal(payload.points.length, 2); + for (const point of payload.points) { + assert.deepEqual( + Object.keys(point.payload ?? {}).sort(), + ['embeddingProfile', 'knowledgeType', 'pointKey', 'revision'], + ); + } + + await traced(() => client.updateCollectionAliases({ + actions: [ + { delete_alias: { alias_name: alias } }, + { + create_alias: { + alias_name: alias, + collection_name: secondCollection, + }, + }, + ], + })); + const aliases = await traced(() => client.getAliases()); + assert.equal( + aliases.aliases.find((entry) => entry.alias_name === alias)?.collection_name, + secondCollection, + ); + + await store.upsertBatch([{ + id: 'faq:after-alias-switch', + knowledgeType: 'faq', + revision: 'faq-revision-2', + embeddingProfile: 'integration-v1', + embedding: [0, 0, 1, 0], + }], 'qdrant-integration-alias-switch'); + assert.equal((await traced(() => client.getCollection(secondCollection))).points_count, 1); + assert.equal((await traced(() => client.getCollection(firstCollection))).points_count, 2); + + await store.delete(['faq:after-alias-switch'], 'qdrant-integration-delete'); + assert.equal((await store.stats()).indexedCount, 0); + console.log('Qdrant 1.18 integration tests passed'); + } finally { + await deleteAliasIfPresent(); + await Promise.allSettled([ + traced(() => client.deleteCollection(firstCollection)), + traced(() => client.deleteCollection(secondCollection)), + ]); + } +} + +async function measureBackend(store: VectorStore): Promise<{ + recallAt1: number; + p50LatencyMs: number; + p95LatencyMs: number; +}> { + const cases = [ + { embedding: [1, 0, 0, 0], expectedId: 'faq:integration-faq' }, + { embedding: [0, 1, 0, 0], expectedId: 'document:integration-document' }, + ]; + const latencies: number[] = []; + let hits = 0; + for (let iteration = 0; iteration < 10; iteration += 1) { + for (const item of cases) { + const startedAt = performance.now(); + const results = await store.search(item.embedding, { + limit: 1, + traceId: `qdrant-integration-benchmark-${iteration}`, + }); + latencies.push(performance.now() - startedAt); + if (results[0]?.id === item.expectedId) hits += 1; + } + } + latencies.sort((left, right) => left - right); + return { + recallAt1: hits / (cases.length * 10), + p50LatencyMs: percentile(latencies, 0.5), + p95LatencyMs: percentile(latencies, 0.95), + }; +} + +function percentile(values: number[], ratio: number): number { + const index = Math.min(values.length - 1, Math.ceil(values.length * ratio) - 1); + return Number(values[index].toFixed(3)); +} + +async function deleteAliasIfPresent(): Promise { + try { + const aliases = await traced(() => client.getAliases()); + if (!aliases.aliases.some((entry) => entry.alias_name === alias)) return; + await traced(() => client.updateCollectionAliases({ + actions: [{ delete_alias: { alias_name: alias } }], + })); + } catch { + // Preserve the original integration failure; unique collections are CI-only. + } +} + +main().catch((error) => { + console.error({ + errorName: error instanceof Error ? error.name : 'UnknownError', + errorCode: typeof error === 'object' && error + ? String((error as { code?: unknown }).code ?? 'qdrant_integration_failed') + : 'qdrant_integration_failed', + }); + process.exit(1); +}); diff --git a/server/tests/qdrant-vector-store.test.ts b/server/tests/qdrant-vector-store.test.ts new file mode 100644 index 0000000..1a91cd5 --- /dev/null +++ b/server/tests/qdrant-vector-store.test.ts @@ -0,0 +1,186 @@ +import assert from 'node:assert/strict'; +import { + QdrantRequestError, + QdrantVectorStore, + type QdrantClientLike, + type QdrantHeaderRunner, +} from '../ai/qdrant-vector-store'; +import { semanticSearch } from '../ai/semantic-search'; +import { knowledgeRetriever } from '../ai/knowledge-system'; +import type { FaqEntry } from '../types/domain'; +import { + FAQ_EMBEDDING_INPUT_VERSION, + currentEmbeddingProfile, +} from '../ai/embedding-profile'; + +async function testQdrantVectorStoreMapsSafeRecordsAndTraceHeaders(): Promise { + const calls: Array<{ method: string; collection?: string; payload?: unknown }> = []; + const tracedHeaders: Array> = []; + const client: QdrantClientLike = { + async upsert(collection, payload) { + calls.push({ method: 'upsert', collection, payload }); + return { status: 'completed' }; + }, + async delete(collection, payload) { + calls.push({ method: 'delete', collection, payload }); + return { status: 'completed' }; + }, + async query(collection, payload) { + calls.push({ method: 'query', collection, payload }); + return { + points: [{ + id: '5f851638-3602-5728-a7f1-5334e2e32ae2', + score: 0.91, + payload: { + pointKey: 'faq:refund', + knowledgeType: 'faq', + revision: 'faq-v1', + embeddingProfile: 'test-profile', + }, + }], + }; + }, + async getCollection(collection) { + calls.push({ method: 'getCollection', collection }); + return { + status: 'green', + points_count: 1, + config: { + params: { + vectors: { size: 2, distance: 'Cosine' }, + }, + }, + }; + }, + }; + const runWithHeaders: QdrantHeaderRunner = async (headers, operation) => { + tracedHeaders.push(headers); + return operation(); + }; + const store = new QdrantVectorStore({ + client, + collectionAlias: 'resolveweave_knowledge_active', + runWithHeaders, + }); + + await store.upsertBatch([{ + id: 'faq:refund', + knowledgeType: 'faq', + revision: 'faq-v1', + embeddingProfile: 'test-profile', + embedding: [1, 0], + }], 'trace-upsert'); + + const upsert = calls.find((call) => call.method === 'upsert'); + assert.equal(upsert?.collection, 'resolveweave_knowledge_active'); + const points = (upsert?.payload as { points: Array> }).points; + assert.match(String(points[0].id), /^[0-9a-f-]{36}$/); + assert.deepEqual(points[0].payload, { + pointKey: 'faq:refund', + knowledgeType: 'faq', + revision: 'faq-v1', + embeddingProfile: 'test-profile', + }); + assert.equal(JSON.stringify(points[0]).includes('refund answer'), false); + + const matches = await store.search([1, 0], { + limit: 3, + knowledgeTypes: ['faq'], + traceId: 'trace-search', + }); + assert.deepEqual(matches, [{ + id: 'faq:refund', + knowledgeType: 'faq', + revision: 'faq-v1', + embeddingProfile: 'test-profile', + score: 0.91, + }]); + const query = calls.find((call) => call.method === 'query'); + assert.deepEqual( + (query?.payload as { filter: unknown }).filter, + { must: [{ key: 'knowledgeType', match: { any: ['faq'] } }] }, + ); + + const stats = await store.stats('trace-stats'); + assert.equal(stats.indexedCount, 1); + assert.equal(stats.embeddingDimensions, 2); + const health = await store.health('trace-health'); + assert.equal(health.backend, 'qdrant'); + assert.equal(health.status, 'healthy'); + await store.delete(['faq:refund']); + + assert.deepEqual(tracedHeaders.slice(0, 4), [ + { 'x-request-id': 'trace-upsert' }, + { 'x-request-id': 'trace-search' }, + { 'x-request-id': 'trace-stats' }, + { 'x-request-id': 'trace-health' }, + ]); + assert.match(tracedHeaders[4]['x-request-id'], /^[0-9a-f-]{36}$/); + + const failingStore = new QdrantVectorStore({ + collectionAlias: 'resolveweave_knowledge_active', + runWithHeaders, + client: { + ...client, + async query() { + throw new Error('provider response contains secret-api-key'); + }, + }, + }); + await assert.rejects( + () => failingStore.search([1, 0], { limit: 1 }), + (error: unknown) => ( + error instanceof QdrantRequestError + && error.code === 'qdrant_request_failed' + && !error.message.includes('secret-api-key') + ), + ); +} + +async function testCommittedFaqDeleteDegradesWhenQdrantCleanupFails(): Promise { + const retriever = knowledgeRetriever as unknown as { + deleteIndexItem: typeof knowledgeRetriever.deleteIndexItem; + upsertIndexItem: typeof knowledgeRetriever.upsertIndexItem; + }; + const originalDelete = retriever.deleteIndexItem; + const originalUpsert = retriever.upsertIndexItem; + retriever.deleteIndexItem = async () => { + throw new Error('raw provider cleanup response'); + }; + retriever.upsertIndexItem = async () => { + throw new Error('raw provider upsert response'); + }; + try { + await assert.doesNotReject(() => semanticSearch.updateIndex({ + id: 'qdrant-cleanup-failure', + isActive: 0, + } as FaqEntry)); + assert.equal( + Boolean((await semanticSearch.getStatus()).lastError?.includes('raw provider')), + false, + ); + await assert.doesNotReject(() => semanticSearch.updateIndexBatch([{ + id: 'qdrant-batch-failure', + isActive: 1, + embedding: [1, 0], + embeddingProfile: currentEmbeddingProfile(FAQ_EMBEDDING_INPUT_VERSION), + } as FaqEntry])); + assert.equal( + Boolean((await semanticSearch.getStatus()).lastError?.includes('raw provider')), + false, + ); + } finally { + retriever.deleteIndexItem = originalDelete; + retriever.upsertIndexItem = originalUpsert; + } +} + +Promise.all([ + testQdrantVectorStoreMapsSafeRecordsAndTraceHeaders(), + testCommittedFaqDeleteDegradesWhenQdrantCleanupFails(), +]) + .then(() => console.log('qdrant vector store tests passed')) + .catch((error) => { + console.error(error); + process.exitCode = 1; + }); diff --git a/server/tests/quality-lab.test.ts b/server/tests/quality-lab.test.ts index 1b46129..e6e10b7 100644 --- a/server/tests/quality-lab.test.ts +++ b/server/tests/quality-lab.test.ts @@ -15,8 +15,10 @@ import { MessageRepo } from '../db/repos/message.repo'; import { MessageRole } from '../types/domain'; import { NotFoundError } from '../utils/errors'; import { FaqRepo } from '../db/repos/faq.repo'; +import { QualityRunRepo } from '../db/repos/quality-run.repo'; import { IntentCategory } from '../types/domain'; import type { RetrievalResult } from '../types/ai'; +import type { RetrievalIndexJob } from '../types/retrieval-ops'; function testDefaultPolicyAndBuiltinDatasetBootstrap(): void { const db = new Database(':memory:'); @@ -322,6 +324,97 @@ async function testSuccessfulActivationAndFingerprintStaleness(): Promise } } +async function testMemoryAndQdrantBackendsShareOneQualityInput(): Promise { + const db = new Database(':memory:'); + try { + initSchema(db); + const qualityLab = new QualityLabService(db); + qualityLab.bootstrap(); + const currentVersionId = publishCompleteCurrentVersion(qualityLab); + const caseByQuery = new Map(QUALITY_BASELINE_CASES.map((testCase) => [ + testCase.query, + { + ...testCase, + versionId: currentVersionId, + createdAt: '2026-07-23T00:00:00.000Z', + }, + ])); + let fingerprint = ''; + const embeddingBatches: number[][][] = []; + const targets: string[] = []; + let activeSearches = 0; + let maxActiveSearches = 0; + let searchCall = 0; + const runs = new QualityRunService(db, { + qualityLab, + autoDrain: false, + getIndexJob: () => ({ + id: '11111111-1111-4111-8111-111111111111', + status: 'ready', + collection: 'quality_collection', + embeddingProfile: 'combined:test', + vectorDimension: 2, + knowledgeFingerprint: fingerprint, + expectedCount: 1, + completedCount: 1, + checkpoint: 1, + previousCollection: null, + failureCode: null, + createdBy: 'admin', + createdAt: '2026-07-23T00:00:00.000Z', + startedAt: '2026-07-23T00:00:00.000Z', + readyAt: '2026-07-23T00:00:00.000Z', + activatedAt: null, + rolledBackAt: null, + updatedAt: '2026-07-23T00:00:00.000Z', + } satisfies RetrievalIndexJob), + searchBackendBatch: async (target, queries, embeddings) => { + activeSearches += 1; + maxActiveSearches = Math.max(maxActiveSearches, activeSearches); + searchCall += 1; + await new Promise((resolve) => setTimeout(resolve, 2 + (searchCall % 3) * 3)); + targets.push(target.provider); + embeddingBatches.push(embeddings); + activeSearches -= 1; + return queries.map((query) => qualityFixtureCandidates(caseByQuery.get(query)!)); + }, + }); + fingerprint = runs.knowledgeFingerprint(); + const run = runs.createRun({ + datasetVersionIds: [QUALITY_BASELINE_VERSION_ID, currentVersionId], + policies: [], + backendTargets: [ + { provider: 'memory' }, + { provider: 'qdrant', indexJobId: '11111111-1111-4111-8111-111111111111' }, + ], + createdBy: 'admin', + }); + + await runs.processNext(); + const completed = runs.getRun(run.id); + assert.deepEqual([...new Set(targets)].sort(), ['memory', 'qdrant']); + assert.equal(targets.filter((target) => target === 'memory').length, 12); + assert.equal(targets.filter((target) => target === 'qdrant').length, 12); + assert.ok(maxActiveSearches <= 8); + assert.ok(maxActiveSearches > 1); + assert.ok(embeddingBatches.every( + (batch) => batch.length === 1 && batch[0].length === 0, + )); + assert.equal(completed.candidates.length, 2); + assert.ok(completed.candidates.some((candidate) => candidate.key.startsWith('memory:'))); + const qdrantCandidate = completed.candidates.find( + (candidate) => candidate.backendTarget.provider === 'qdrant', + ); + assert.ok(qdrantCandidate?.key.includes('qdrant:11111111-1111-4111-8111-111111111111:')); + assert.equal( + runs.checkBackendActivation(run.id, qdrantCandidate!.key).eligible, + true, + ); + } finally { + db.close(); + } +} + async function testRunningCancellationIsPersisted(): Promise { const db = new Database(':memory:'); try { @@ -410,15 +503,42 @@ function testBuiltinRetrievalDoesNotReadExpectedSources(): void { assert.equal(candidates[0].knowledgeId, 'faq-refund-apply'); } +function testMalformedHistoricalQualityJsonUsesSafeFallbacks(): void { + const db = new Database(':memory:'); + try { + initSchema(db); + const qualityLab = new QualityLabService(db); + qualityLab.bootstrap(); + const runs = new QualityRunService(db, { qualityLab, autoDrain: false }); + const run = runs.createRun({ + datasetVersionIds: [QUALITY_BASELINE_VERSION_ID], + policies: [], + createdBy: 'admin', + }); + db.prepare(` + UPDATE quality_runs + SET policy_grid = '{', backend_targets = '{' + WHERE id = ? + `).run(run.id); + const restored = new QualityRunRepo(db).get(run.id); + assert.deepEqual(restored?.policies, []); + assert.deepEqual(restored?.backendTargets, [{ provider: 'memory' }]); + } finally { + db.close(); + } +} + async function main(): Promise { testDefaultPolicyAndBuiltinDatasetBootstrap(); testBuiltinDatasetCannotBeRewrittenInPlace(); testCustomDatasetVersionLifecycle(); testPolicyHistoryRollbackAndMessageSnapshot(); testBuiltinRetrievalDoesNotReadExpectedSources(); + testMalformedHistoricalQualityJsonUsesSafeFallbacks(); await testPersistedRunLifecycle(); await testCoverageCannotBeAggregatedAcrossSmallVersions(); await testSuccessfulActivationAndFingerprintStaleness(); + await testMemoryAndQdrantBackendsShareOneQualityInput(); await testRunningCancellationIsPersisted(); console.log('quality lab tests passed'); } diff --git a/server/tests/regression.test.ts b/server/tests/regression.test.ts index 1535dbe..f2b26c1 100644 --- a/server/tests/regression.test.ts +++ b/server/tests/regression.test.ts @@ -102,11 +102,12 @@ function testVectorStoreContractExists(): void { const source = fs.readFileSync(vectorStorePath, 'utf8'); assert.match(source, /export interface VectorStore/, 'vector store should expose a swappable interface'); - assert.match(source, /upsert\(/, 'vector store should support upsert'); + assert.match(source, /upsertBatch\(/, 'vector store should support batch upsert'); assert.match(source, /delete\(/, 'vector store should support delete'); assert.match(source, /search\(/, 'vector store should support vector search'); assert.match(source, /stats\(/, 'vector store should expose index stats'); - assert.match(source, /clear\(/, 'vector store should support full rebuilds'); + assert.match(source, /health\(/, 'vector store should expose backend health'); + assert.match(source, /Promise { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'retrieval-api-')); + process.env.NODE_ENV = 'test'; + process.env.JWT_SECRET = 'retrieval-api-secret'; + process.env.DB_PATH = path.join(tempDir, 'test.db'); + process.env.VECTOR_STORE_PROVIDER = 'memory'; + process.env.QDRANT_URL = ''; + + const [ + { default: router }, + { errorHandler }, + databaseModule, + { RetrievalTraceCollector }, + { getRetrievalTraceService }, + { SessionRepo }, + { MessageRepo }, + { MessageRole }, + ] = await Promise.all([ + import('../routes/admin/retrieval'), + import('../middleware/errorHandler'), + import('../db'), + import('../services/retrieval-trace-collector'), + import('../services/retrieval-trace.service'), + import('../db/repos/session.repo'), + import('../db/repos/message.repo'), + import('../types/domain'), + ]); + const db = databaseModule.getDatabase(); + const session = new SessionRepo(db).create('trace-api-user'); + const messageRepo = new MessageRepo(db); + const userMessage = messageRepo.create({ + sessionId: session.id, + role: MessageRole.USER, + content: 'authorized trace question', + }); + const assistantMessage = messageRepo.create({ + sessionId: session.id, + role: MessageRole.ASSISTANT, + content: 'authorized trace answer', + replyToMessageId: userMessage.id, + }); + const collector = new RetrievalTraceCollector({ backend: 'memory' }); + collector.record('grounding', { + status: 'completed', + latencyMs: 1, + inputCount: 0, + outputCount: 0, + }); + getRetrievalTraceService().persist(collector.complete({ + sessionId: session.id, + userMessageId: userMessage.id, + assistantMessageId: assistantMessage.id, + policyId: 'policy-test', + })); + const app = express(); + app.use(express.json()); + app.use('/api/admin/retrieval', router); + app.use(errorHandler); + const token = jwt.sign( + { id: 'admin-id', username: 'admin', role: 'admin' }, + process.env.JWT_SECRET, + ); + const auth = { Authorization: `Bearer ${token}` }; + let server: Server | null = null; + try { + server = app.listen(0, '127.0.0.1'); + await new Promise((resolve) => server?.once('listening', resolve)); + const base = `http://127.0.0.1:${(server.address() as AddressInfo).port}` + + '/api/admin/retrieval'; + + assert.equal((await fetch(`${base}/status`)).status, 401); + const status = await fetch(`${base}/status`, { headers: auth }); + assert.equal(status.status, 200); + const statusBody = await status.json() as { + data: { provider: string; qdrantConfigured: boolean; qdrantHealth: string }; + }; + assert.equal(statusBody.data.provider, 'memory'); + assert.equal(statusBody.data.qdrantConfigured, false); + assert.equal(statusBody.data.qdrantHealth, 'not_configured'); + + const traceList = await fetch( + `${base}/traces?backend=memory&sessionId=${session.id}`, + { headers: auth }, + ); + assert.equal(traceList.status, 200); + const traceListBody = await traceList.json() as { + data: { total: number; items: Array<{ id: string }> }; + }; + assert.equal(traceListBody.data.total, 1); + const traceDetail = await fetch( + `${base}/traces/${traceListBody.data.items[0].id}`, + { headers: auth }, + ); + const traceDetailBody = await traceDetail.json() as { + data: { + trace: { stages: unknown[] }; + messages: { user: { content: string }; assistant: { content: string } }; + }; + }; + assert.equal(traceDetailBody.data.trace.stages.length, 8); + assert.equal(traceDetailBody.data.messages.user.content, 'authorized trace question'); + assert.equal(traceDetailBody.data.messages.assistant.content, 'authorized trace answer'); + + assert.equal( + (await fetch(`${base}/index-jobs?pageSize=101`, { headers: auth })).status, + 400, + ); + const missingKey = await fetch(`${base}/index-jobs`, { + method: 'POST', + headers: { ...auth, 'Content-Type': 'application/json' }, + body: '{}', + }); + assert.equal(missingKey.status, 400); + const noQdrant = await fetch(`${base}/index-jobs`, { + method: 'POST', + headers: { + ...auth, + 'Content-Type': 'application/json', + 'Idempotency-Key': 'retrieval-create-without-qdrant', + }, + body: '{}', + }); + assert.equal(noQdrant.status, 409); + } finally { + if (server) { + await new Promise((resolve, reject) => { + server?.close((error) => error ? reject(error) : resolve()); + }); + } + databaseModule.closeDatabase(); + fs.rmSync(path.join(tempDir, 'test.db'), { force: true }); + fs.rmdirSync(tempDir); + } + console.log('retrieval API tests passed'); +} + +void main(); diff --git a/server/tests/retrieval-index-job.test.ts b/server/tests/retrieval-index-job.test.ts new file mode 100644 index 0000000..73f15b5 --- /dev/null +++ b/server/tests/retrieval-index-job.test.ts @@ -0,0 +1,422 @@ +import assert from 'node:assert/strict'; +import Database from 'better-sqlite3'; +import { initSchema } from '../db'; +import { FaqRepo } from '../db/repos/faq.repo'; +import { RetrievalIndexJobService } from '../services/retrieval-index-job.service'; +import { IntentCategory } from '../types/domain'; +import type { + QdrantCollectionControl, + RetrievalIndexVectorWriter, +} from '../services/retrieval-index-job.service'; + +async function testIndexJobBuildsIdempotentlyAndDetectsStaleKnowledge(): Promise { + const db = new Database(':memory:'); + initSchema(db); + initSchema(db); + const faqRepo = new FaqRepo(db); + faqRepo.create({ + question: '退款期限', + answer: '七天内可退款', + category: IntentCategory.REFUND, + keywords: ['退款'], + embedding: [1, 0], + embeddingProfile: 'test-profile', + }); + const collections = new Map(); + let aliasCollection: string | null = null; + const control: QdrantCollectionControl = { + async createCollection(name, dimensions) { + if (collections.has(name)) return false; + collections.set(name, { dimensions, count: 0 }); + return true; + }, + async collectionInfo(name) { + return collections.get(name) ?? null; + }, + async currentAliasCollection() { + return aliasCollection; + }, + async switchAlias(next, expected) { + assert.equal(aliasCollection, expected); + aliasCollection = next; + }, + }; + const writerFactory = (collection: string): RetrievalIndexVectorWriter => ({ + async upsert(records) { + const current = collections.get(collection); + assert.ok(current); + current.count += records.length; + }, + }); + const service = new RetrievalIndexJobService(db, { + control, + writerFactory, + collectionPrefix: 'test_knowledge', + autoDrain: false, + activationGate: () => ({ + eligible: true, + warnings: [], + reasons: [], + qualityRunId: 'quality-run', + candidateKey: 'quality-candidate', + }), + }); + + const created = await service.createJob({ createdBy: 'admin' }); + const duplicate = await service.createJob({ createdBy: 'admin' }); + assert.equal(duplicate.id, created.id, 'same fingerprint should reuse the build'); + assert.equal(created.status, 'queued'); + + await service.processNext(); + const ready = service.getJob(created.id); + assert.equal(ready.status, 'ready'); + assert.equal(ready.expectedCount, 1); + assert.equal(ready.completedCount, 1); + assert.equal(ready.vectorDimension, 2); + + const firstActive = await service.activate({ + id: ready.id, + expectedCurrentCollection: null, + confirmLatencyWarning: false, + }); + assert.equal(firstActive.status, 'active'); + assert.equal(aliasCollection, ready.collection); + + const secondId = '22222222-2222-4222-8222-222222222222'; + const secondCollection = 'test_knowledge_second'; + collections.set(secondCollection, { dimensions: 2, count: 1 }); + db.prepare(` + INSERT INTO retrieval_index_jobs ( + id, status, collection_name, embedding_profile, vector_dimension, + knowledge_fingerprint, expected_count, completed_count, batch_checkpoint, + previous_collection, failure_code, created_by, created_at, started_at, + ready_at, activated_at, rolled_back_at, updated_at + ) + SELECT ?, 'ready', ?, embedding_profile, vector_dimension, + knowledge_fingerprint, expected_count, completed_count, batch_checkpoint, + NULL, NULL, created_by, created_at, started_at, ready_at, NULL, NULL, updated_at + FROM retrieval_index_jobs WHERE id = ? + `).run(secondId, secondCollection, ready.id); + const secondActive = await service.activate({ + id: secondId, + expectedCurrentCollection: ready.collection, + confirmLatencyWarning: false, + }); + assert.equal(secondActive.previousCollection, ready.collection); + assert.equal(aliasCollection, secondCollection); + db.exec(` + CREATE TRIGGER fail_rollback_persistence + BEFORE UPDATE ON retrieval_index_jobs + WHEN OLD.id = '${secondId}' AND NEW.status = 'rolled_back' + BEGIN + SELECT RAISE(ABORT, 'simulated rollback persistence failure'); + END + `); + await assert.rejects( + () => service.rollback({ + id: secondId, + expectedCurrentCollection: secondCollection, + }), + /simulated rollback persistence failure/, + ); + assert.equal(aliasCollection, ready.collection); + assert.equal(service.getJob(secondId).status, 'active'); + db.exec('DROP TRIGGER fail_rollback_persistence'); + const rolledBack = await service.rollback({ + id: secondId, + expectedCurrentCollection: secondCollection, + }); + assert.equal(rolledBack.id, ready.id); + assert.equal(rolledBack.status, 'active'); + assert.equal(aliasCollection, ready.collection); + + const staleReadyId = '33333333-3333-4333-8333-333333333333'; + db.prepare(` + INSERT INTO retrieval_index_jobs ( + id, status, collection_name, embedding_profile, vector_dimension, + knowledge_fingerprint, expected_count, completed_count, batch_checkpoint, + previous_collection, failure_code, created_by, created_at, started_at, + ready_at, activated_at, rolled_back_at, updated_at + ) + SELECT ?, 'ready', 'test_knowledge_stale', embedding_profile, vector_dimension, + knowledge_fingerprint, expected_count, completed_count, batch_checkpoint, + NULL, NULL, created_by, created_at, started_at, ready_at, NULL, NULL, updated_at + FROM retrieval_index_jobs WHERE id = ? + `).run(staleReadyId, ready.id); + const currentFaq = faqRepo.listAllActive()[0]; + faqRepo.update(currentFaq.id, { answer: '退款政策已更新' }); + assert.equal(service.getJob(staleReadyId).status, 'stale'); + db.close(); +} + +async function testInterruptedJobResumesFromCheckpoint(): Promise { + const db = new Database(':memory:'); + initSchema(db); + const faqRepo = new FaqRepo(db); + for (let index = 0; index < 205; index += 1) { + faqRepo.create({ + question: `policy ${index.toString().padStart(3, '0')}`, + answer: `answer ${index}`, + category: IntentCategory.GENERAL, + keywords: [], + embedding: index % 2 === 0 ? [1, 0] : [0, 1], + embeddingProfile: 'test-profile', + }); + } + const collections = new Map(); + const resumedBatchSizes: number[] = []; + const service = new RetrievalIndexJobService(db, { + autoDrain: false, + collectionPrefix: 'resume_test', + control: { + async createCollection(name, dimensions) { + collections.set(name, { dimensions, count: 0 }); + return true; + }, + async collectionInfo(name) { + return collections.get(name) ?? null; + }, + async currentAliasCollection() { + return null; + }, + async switchAlias() {}, + }, + writerFactory: (collection) => ({ + async upsert(records) { + resumedBatchSizes.push(records.length); + collections.get(collection)!.count += records.length; + }, + }), + }); + const job = await service.createJob({ createdBy: 'admin' }); + collections.set(job.collection, { dimensions: 2, count: 100 }); + db.prepare(` + UPDATE retrieval_index_jobs + SET status = 'running', batch_checkpoint = 100, completed_count = 100 + WHERE id = ? + `).run(job.id); + + service.start(); + assert.equal(service.getJob(job.id).status, 'interrupted'); + await service.processNext(); + const resumed = service.getJob(job.id); + assert.equal(resumed.status, 'ready'); + assert.equal(resumed.checkpoint, 205); + assert.equal(resumed.completedCount, 205); + assert.deepEqual(resumedBatchSizes, [100, 5]); + db.close(); +} + +async function testReadyValidationUsesSafeFailureCodes(): Promise { + const db = new Database(':memory:'); + initSchema(db); + new FaqRepo(db).create({ + question: 'shipping policy', + answer: 'ships tomorrow', + category: IntentCategory.ORDER, + keywords: [], + embedding: [1, 0], + embeddingProfile: 'test-profile', + }); + const collections = new Map(); + const service = new RetrievalIndexJobService(db, { + autoDrain: false, + collectionPrefix: 'validation_test', + control: { + async createCollection(name, dimensions) { + collections.set(name, { dimensions, count: 0 }); + return true; + }, + async collectionInfo(name) { + return collections.get(name) ?? null; + }, + async currentAliasCollection() { + return null; + }, + async switchAlias() {}, + }, + writerFactory: () => ({ + async upsert() { + // Deliberately leave the remote count unchanged. + }, + }), + }); + const job = await service.createJob({ createdBy: 'admin' }); + await service.processNext(); + const failed = service.getJob(job.id); + assert.equal(failed.status, 'failed'); + assert.equal(failed.failureCode, 'qdrant_point_count_mismatch'); + assert.equal(JSON.stringify(failed).includes('remote count unchanged'), false); + db.close(); +} + +async function testActivationRecoversAfterAliasSwitchPersistenceFailure(): Promise { + const db = new Database(':memory:'); + initSchema(db); + new FaqRepo(db).create({ + question: 'recovery policy', + answer: 'recovery answer', + category: IntentCategory.GENERAL, + keywords: [], + embedding: [1, 0], + embeddingProfile: 'test-profile', + }); + const collections = new Map(); + let aliasCollection: string | null = null; + const service = new RetrievalIndexJobService(db, { + autoDrain: false, + collectionPrefix: 'recovery_test', + activationGate: () => ({ + eligible: true, + warnings: [], + reasons: [], + qualityRunId: 'quality-run', + candidateKey: 'quality-candidate', + }), + control: { + async createCollection(name, dimensions) { + collections.set(name, { dimensions, count: 0 }); + return true; + }, + async collectionInfo(name) { + return collections.get(name) ?? null; + }, + async currentAliasCollection() { + return aliasCollection; + }, + async switchAlias(next, expected) { + assert.equal(aliasCollection, expected); + aliasCollection = next; + }, + }, + writerFactory: (collection) => ({ + async upsert(records) { + collections.get(collection)!.count += records.length; + }, + }), + }); + const job = await service.createJob({ createdBy: 'admin' }); + await service.processNext(); + db.exec(` + CREATE TRIGGER fail_activation_persistence + BEFORE UPDATE ON retrieval_index_jobs + WHEN NEW.status = 'active' + BEGIN + SELECT RAISE(ABORT, 'simulated persistence failure'); + END + `); + await assert.rejects( + () => service.activate({ + id: job.id, + expectedCurrentCollection: null, + confirmLatencyWarning: false, + }), + /simulated persistence failure/, + ); + assert.equal(aliasCollection, job.collection); + assert.equal(service.getJob(job.id).status, 'ready'); + const faqRepo = new FaqRepo(db); + const faq = faqRepo.listAllActive()[0]; + faqRepo.update(faq.id, { answer: 'knowledge changed after alias switch' }); + assert.equal( + service.getJob(job.id).status, + 'ready', + 'pending activation must not be made stale before reconciliation', + ); + db.exec('DROP TRIGGER fail_activation_persistence'); + + const recovered = await service.activate({ + id: job.id, + expectedCurrentCollection: null, + confirmLatencyWarning: false, + }); + assert.equal(recovered.status, 'active'); + assert.equal(aliasCollection, job.collection); + db.close(); +} + +async function testActivationIntentFinishesAfterPreSwitchFailure(): Promise { + const db = new Database(':memory:'); + initSchema(db); + const faqRepo = new FaqRepo(db); + faqRepo.create({ + question: 'intent policy', + answer: 'intent answer', + category: IntentCategory.GENERAL, + keywords: [], + embedding: [1, 0], + embeddingProfile: 'test-profile', + }); + const collections = new Map(); + let aliasCollection: string | null = null; + let failSwitch = true; + const service = new RetrievalIndexJobService(db, { + autoDrain: false, + collectionPrefix: 'intent_test', + activationGate: () => ({ + eligible: true, + warnings: [], + reasons: [], + qualityRunId: 'quality-run', + candidateKey: 'quality-candidate', + }), + control: { + async createCollection(name, dimensions) { + collections.set(name, { dimensions, count: 0 }); + return true; + }, + async collectionInfo(name) { + return collections.get(name) ?? null; + }, + async currentAliasCollection() { + return aliasCollection; + }, + async switchAlias(next, expected) { + assert.equal(aliasCollection, expected); + if (failSwitch) throw new Error('simulated pre-switch failure'); + aliasCollection = next; + }, + }, + writerFactory: (collection) => ({ + async upsert(records) { + collections.get(collection)!.count += records.length; + }, + }), + }); + const job = await service.createJob({ createdBy: 'admin' }); + await service.processNext(); + await assert.rejects( + () => service.activate({ + id: job.id, + expectedCurrentCollection: null, + confirmLatencyWarning: false, + }), + /simulated pre-switch failure/, + ); + assert.equal(aliasCollection, null); + failSwitch = false; + const faq = faqRepo.listAllActive()[0]; + faqRepo.update(faq.id, { answer: 'changed while intent was pending' }); + + const recovered = await service.activate({ + id: job.id, + expectedCurrentCollection: null, + confirmLatencyWarning: false, + }); + assert.equal(recovered.status, 'active'); + assert.equal(aliasCollection, job.collection); + db.close(); +} + +Promise.all([ + testIndexJobBuildsIdempotentlyAndDetectsStaleKnowledge(), + testInterruptedJobResumesFromCheckpoint(), + testReadyValidationUsesSafeFailureCodes(), + testActivationRecoversAfterAliasSwitchPersistenceFailure(), + testActivationIntentFinishesAfterPreSwitchFailure(), +]) + .then(() => console.log('retrieval index job tests passed')) + .catch((error) => { + console.error(error); + process.exitCode = 1; + }); diff --git a/server/tests/retrieval-trace.test.ts b/server/tests/retrieval-trace.test.ts new file mode 100644 index 0000000..c6f5190 --- /dev/null +++ b/server/tests/retrieval-trace.test.ts @@ -0,0 +1,87 @@ +import assert from 'node:assert/strict'; +import Database from 'better-sqlite3'; +import { initSchema } from '../db'; +import { MessageRepo } from '../db/repos/message.repo'; +import { SessionRepo } from '../db/repos/session.repo'; +import { RetrievalTraceCollector } from '../services/retrieval-trace-collector'; +import { RetrievalTraceService } from '../services/retrieval-trace.service'; +import { MessageRole } from '../types/domain'; + +function testTraceBoundsRetentionAndSessionCascade(): void { + const db = new Database(':memory:'); + initSchema(db); + const session = new SessionRepo(db).create('trace-user'); + const messages = new MessageRepo(db); + const userMessage = messages.create({ + sessionId: session.id, + role: MessageRole.USER, + content: 'private customer question', + }); + const assistantMessage = messages.create({ + sessionId: session.id, + role: MessageRole.ASSISTANT, + content: 'private assistant answer', + replyToMessageId: userMessage.id, + }); + const collector = new RetrievalTraceCollector({ + backend: 'qdrant', + now: () => new Date('2026-07-29T00:00:00.000Z'), + }); + collector.record('vector_recall', { + status: 'completed', + latencyMs: 12, + inputCount: 1, + outputCount: 25, + candidates: Array.from({ length: 25 }, (_, index) => ({ + knowledgeType: 'faq' as const, + knowledgeId: `faq-${index}`, + score: 1 - index / 100, + rank: index + 1, + })), + }); + collector.record('grounding', { + status: 'completed', + latencyMs: 1, + inputCount: 5, + outputCount: 4, + candidates: Array.from({ length: 4 }, (_, index) => ({ + knowledgeType: 'faq' as const, + knowledgeId: `evidence-${index}`, + rank: index + 1, + })), + }); + const service = new RetrievalTraceService(db, { + retentionDays: 30, + now: () => new Date('2026-07-29T00:00:01.000Z'), + }); + service.persist(collector.complete({ + sessionId: session.id, + userMessageId: userMessage.id, + assistantMessageId: assistantMessage.id, + policyId: 'policy-v1', + })); + + const detail = service.getTrace(collector.id); + assert.equal(detail.stages.find((stage) => stage.name === 'vector_recall')?.candidates.length, 20); + assert.equal(detail.stages.find((stage) => stage.name === 'grounding')?.candidates.length, 3); + assert.equal(JSON.stringify(detail).includes('private customer question'), false); + assert.equal(JSON.stringify(detail).includes('private assistant answer'), false); + + db.prepare('UPDATE retrieval_traces SET created_at = ? WHERE id = ?') + .run('2026-06-01T00:00:00.000Z', collector.id); + assert.equal(service.cleanupExpired(), 1); + + const second = new RetrievalTraceCollector({ backend: 'memory' }); + service.persist(second.complete({ + sessionId: session.id, + userMessageId: userMessage.id, + assistantMessageId: assistantMessage.id, + policyId: 'policy-v1', + })); + db.prepare('DELETE FROM sessions WHERE id = ?').run(session.id); + assert.equal(service.listTraces({ page: 1, pageSize: 20 }).total, 0); + db.close(); +} + +testTraceBoundsRetentionAndSessionCascade(); +console.log('retrieval trace tests passed'); diff --git a/server/tests/vector-store-config.test.ts b/server/tests/vector-store-config.test.ts new file mode 100644 index 0000000..834b6ea --- /dev/null +++ b/server/tests/vector-store-config.test.ts @@ -0,0 +1,42 @@ +import assert from 'node:assert/strict'; +import { resolveVectorStoreEnvironment } from '../config'; + +function testVectorStoreEnvironmentDefaultsToMemory(): void { + const resolved = resolveVectorStoreEnvironment({}); + assert.equal(resolved.provider, 'memory'); + assert.equal(resolved.qdrantUrl, ''); + assert.equal(resolved.collectionPrefix, 'resolveweave_knowledge'); + assert.equal(resolved.collectionAlias, 'resolveweave_knowledge_active'); + assert.equal(resolved.timeoutMs, 5_000); + assert.equal(resolved.traceRetentionDays, 30); +} + +function testQdrantEnvironmentIsExplicitAndBounded(): void { + assert.throws( + () => resolveVectorStoreEnvironment({ VECTOR_STORE_PROVIDER: 'qdrant' }), + /QDRANT_URL is required/, + ); + assert.throws( + () => resolveVectorStoreEnvironment({ + VECTOR_STORE_PROVIDER: 'qdrant', + QDRANT_URL: 'file:///tmp/qdrant', + }), + /http or https/, + ); + + const resolved = resolveVectorStoreEnvironment({ + VECTOR_STORE_PROVIDER: 'qdrant', + QDRANT_URL: ' https://qdrant.example.test ', + QDRANT_API_KEY: 'secret', + QDRANT_TIMEOUT_MS: '9000', + RETRIEVAL_TRACE_RETENTION_DAYS: '45', + }); + assert.equal(resolved.provider, 'qdrant'); + assert.equal(resolved.qdrantUrl, 'https://qdrant.example.test'); + assert.equal(resolved.timeoutMs, 9_000); + assert.equal(resolved.traceRetentionDays, 45); +} + +testVectorStoreEnvironmentDefaultsToMemory(); +testQdrantEnvironmentIsExplicitAndBounded(); +console.log('vector store config tests passed'); diff --git a/server/tests/vector-store.test.ts b/server/tests/vector-store.test.ts new file mode 100644 index 0000000..403d536 --- /dev/null +++ b/server/tests/vector-store.test.ts @@ -0,0 +1,52 @@ +import assert from 'node:assert/strict'; +import { InMemoryVectorStore } from '../ai/vector-store'; + +async function testAsyncInMemoryVectorStoreContract(): Promise { + const store = new InMemoryVectorStore(); + + await store.upsertBatch([ + { + id: 'faq:refund', + knowledgeType: 'faq', + revision: 'faq-v1', + embeddingProfile: 'test-profile', + embedding: [1, 0], + }, + { + id: 'document:policy', + knowledgeType: 'document', + revision: 'document-v1', + embeddingProfile: 'test-profile', + embedding: [0.8, 0.2], + }, + ]); + + const faqMatches = await store.search([1, 0], { + limit: 5, + knowledgeTypes: ['faq'], + traceId: 'trace-vector-store-contract', + }); + + assert.deepEqual(faqMatches.map((match) => match.id), ['faq:refund']); + assert.equal(faqMatches[0].knowledgeType, 'faq'); + assert.equal(faqMatches[0].revision, 'faq-v1'); + assert.equal(faqMatches[0].score, 1); + + const stats = await store.stats(); + assert.equal(stats.indexedCount, 2); + assert.equal(stats.embeddingDimensions, 2); + + const health = await store.health('trace-vector-store-health'); + assert.equal(health.backend, 'memory'); + assert.equal(health.status, 'healthy'); + + await store.delete(['faq:refund']); + assert.equal((await store.stats()).indexedCount, 1); +} + +testAsyncInMemoryVectorStoreContract() + .then(() => console.log('vector store tests passed')) + .catch((error) => { + console.error(error); + process.exitCode = 1; + }); diff --git a/server/types/quality.ts b/server/types/quality.ts index d3948ed..609353d 100644 --- a/server/types/quality.ts +++ b/server/types/quality.ts @@ -14,6 +14,10 @@ export type QualityRunStatus = | 'stale'; export type RerankerMode = 'none' | 'local_overlap_v1'; +export type QualityBackendTarget = + | { provider: 'memory' } + | { provider: 'qdrant'; indexJobId: string }; + export interface RetrievalPolicyConfig { directFaqThreshold: number; generationEvidenceThreshold: number; @@ -109,6 +113,7 @@ export interface QualityCaseResult { export interface QualityCandidateResult { key: string; + backendTarget: QualityBackendTarget; policy: RetrievalPolicyConfig; metrics: QualityMetrics; recommended: boolean; @@ -119,6 +124,7 @@ export interface QualityRun { id: string; datasetVersionIds: string[]; policies: RetrievalPolicyConfig[]; + backendTargets: QualityBackendTarget[]; status: QualityRunStatus; progress: number; totalCases: number; diff --git a/server/types/retrieval-ops.ts b/server/types/retrieval-ops.ts new file mode 100644 index 0000000..16d40e7 --- /dev/null +++ b/server/types/retrieval-ops.ts @@ -0,0 +1,92 @@ +export type RetrievalIndexJobStatus = + | 'queued' + | 'running' + | 'interrupted' + | 'ready' + | 'active' + | 'rolled_back' + | 'failed' + | 'stale'; + +export interface RetrievalIndexJob { + id: string; + status: RetrievalIndexJobStatus; + collection: string; + embeddingProfile: string; + vectorDimension: number; + knowledgeFingerprint: string; + expectedCount: number; + completedCount: number; + checkpoint: number; + previousCollection: string | null; + failureCode: string | null; + createdBy: string; + createdAt: string; + startedAt: string | null; + readyAt: string | null; + activatedAt: string | null; + rolledBackAt: string | null; + updatedAt: string; +} + +export interface RetrievalActivationCheck { + eligible: boolean; + warnings: string[]; + reasons: string[]; + qualityRunId: string | null; + candidateKey: string | null; +} + +export const RETRIEVAL_TRACE_STAGES = [ + 'query_expand', + 'embedding', + 'vector_recall', + 'keyword_recall', + 'fusion', + 'rerank', + 'context_budget', + 'grounding', +] as const; + +export type RetrievalTraceStageName = (typeof RETRIEVAL_TRACE_STAGES)[number]; +export type RetrievalTraceStatus = 'completed' | 'degraded' | 'failed'; +export type RetrievalTraceStageStatus = + | 'completed' + | 'degraded' + | 'failed' + | 'skipped'; + +export interface RetrievalTraceCandidate { + knowledgeType: 'faq' | 'document'; + knowledgeId: string; + score?: number; + rank?: number; + source?: 'vector' | 'keyword' | 'hybrid'; +} + +export interface RetrievalTraceStage { + name: RetrievalTraceStageName; + order: number; + status: RetrievalTraceStageStatus; + latencyMs: number; + inputCount: number; + outputCount: number; + candidates: RetrievalTraceCandidate[]; + budget: Record; + errorCode: string | null; +} + +export interface RetrievalTrace { + id: string; + sessionId: string; + userMessageId: string; + assistantMessageId: string | null; + policyId: string; + backend: 'memory' | 'qdrant'; + status: RetrievalTraceStatus; + errorCode: string | null; + totalLatencyMs: number; + stages: RetrievalTraceStage[]; + createdAt: string; + completedAt: string; +} diff --git a/tests/e2e/web.spec.ts b/tests/e2e/web.spec.ts index 7af301c..8a639b2 100644 --- a/tests/e2e/web.spec.ts +++ b/tests/e2e/web.spec.ts @@ -2099,4 +2099,290 @@ test.describe('Web automation: admin boundaries and FAQ index operation', () => }); } }); + + test('retrieval operations shows trace timeline across language, theme, mobile and keyboard states', async ({ page }) => { + const question = `retrieval-trace-web-${Date.now()}`; + const chatResponse = await page.request.post('/api/chat', { + headers: { Accept: 'text/event-stream' }, + data: { message: question, userIdent: `trace-web-${Date.now()}` }, + }); + expect(chatResponse.status()).toBe(200); + const stream = await chatResponse.text(); + const sessionId = stream.match(/"sessionId":"([^"]+)"/)?.[1]; + expect(sessionId).toBeTruthy(); + + await loginAsAdmin(page); + await page.getByText('检索运维').click(); + await expect(page).toHaveURL(/\/admin\/retrieval-ops$/); + await expect(page.getByTestId('retrieval-ops-page')).toBeVisible(); + await expect(page.getByRole('heading', { name: '检索运维' })).toBeVisible(); + await expect(page.getByText('内存', { exact: true })).toBeVisible(); + if (process.env.CAPTURE_RELEASE_EVIDENCE === '1') { + await page.getByText('登录成功').waitFor({ state: 'hidden' }); + await page.screenshot({ + path: 'docs/releases/assets/v0.3.2-retrieval-ops-desktop.png', + fullPage: true, + }); + } + + await page.getByText('检索 Trace').click(); + await page.getByTestId('retrieval-trace-session-filter').locator('input').fill(sessionId!); + await page.getByRole('button', { name: '查询', exact: true }).click(); + const traceRow = page.getByTestId('retrieval-trace-table').locator('tr').filter({ + hasText: sessionId!, + }); + await expect(traceRow).toBeVisible(); + await traceRow.getByRole('button', { name: '查看' }).click(); + await expect(page.getByText(question, { exact: true })).toBeVisible(); + await expect(page.getByText('查询扩展', { exact: true })).toBeVisible(); + await expect(page.getByText('Grounding 决策', { exact: true })).toBeVisible(); + if (process.env.CAPTURE_RELEASE_EVIDENCE === '1') { + await page.waitForTimeout(350); + await page.screenshot({ + path: 'docs/releases/assets/v0.3.2-retrieval-trace-desktop.png', + fullPage: true, + }); + } + await page.locator('.t-dialog:visible .t-dialog__close').click(); + + await page.getByTestId('language-toggle').click(); + await expect(page.getByRole('heading', { name: 'Retrieval operations' })).toBeVisible(); + await expect(page.getByText('Runtime overview')).toBeVisible(); + await page.getByTestId('theme-toggle').click(); + await expect(page.locator('html')).toHaveAttribute('data-theme', 'dark'); + + await page.setViewportSize({ width: 390, height: 844 }); + await page.keyboard.press('Tab'); + expect(await page.evaluate(() => document.activeElement?.tagName)).not.toBe('BODY'); + await expect.poll( + () => page.evaluate( + () => document.documentElement.scrollWidth <= document.documentElement.clientWidth, + ), + { message: 'retrieval operations should not create page-level mobile overflow' }, + ).toBe(true); + if (process.env.CAPTURE_RELEASE_EVIDENCE === '1') { + await page.screenshot({ + path: 'docs/releases/assets/v0.3.2-retrieval-ops-mobile-dark.png', + fullPage: true, + }); + } + }); + + test('retrieval operations gates activation, rollback and error states with optimistic alias values', async ({ page }) => { + await loginAsAdmin(page); + const oldCollection = 'resolveweave_knowledge_20260729_old'; + const nextCollection = 'resolveweave_knowledge_20260729_next'; + const oldJobId = '11111111-1111-4111-8111-111111111111'; + const nextJobId = '22222222-2222-4222-8222-222222222222'; + let currentCollection = oldCollection; + let failRequests = false; + let jobs = [ + { + id: nextJobId, + status: 'ready', + collection: nextCollection, + embeddingProfile: 'combined:quality-v1', + vectorDimension: 64, + knowledgeFingerprint: 'fingerprint-v1', + expectedCount: 128, + completedCount: 128, + checkpoint: 128, + previousCollection: null, + failureCode: null, + createdBy: 'admin', + createdAt: '2026-07-29T08:00:00.000Z', + startedAt: '2026-07-29T08:00:01.000Z', + readyAt: '2026-07-29T08:00:03.000Z', + activatedAt: null, + rolledBackAt: null, + updatedAt: '2026-07-29T08:00:03.000Z', + }, + { + id: oldJobId, + status: 'active', + collection: oldCollection, + embeddingProfile: 'combined:quality-v1', + vectorDimension: 64, + knowledgeFingerprint: 'fingerprint-v1', + expectedCount: 128, + completedCount: 128, + checkpoint: 128, + previousCollection: null, + failureCode: null, + createdBy: 'admin', + createdAt: '2026-07-28T08:00:00.000Z', + startedAt: '2026-07-28T08:00:01.000Z', + readyAt: '2026-07-28T08:00:03.000Z', + activatedAt: '2026-07-28T08:05:00.000Z', + rolledBackAt: null, + updatedAt: '2026-07-28T08:05:00.000Z', + }, + ]; + + await page.route('**/api/admin/retrieval/**', async (route) => { + if (failRequests) { + await route.fulfill({ status: 503, json: { code: 503, data: null, message: 'unavailable' } }); + return; + } + const request = route.request(); + const url = new URL(request.url()); + const path = url.pathname; + if (path.endsWith('/status')) { + await route.fulfill({ json: { code: 0, data: { + provider: 'qdrant', + qdrantConfigured: true, + qdrantHealth: 'healthy', + alias: 'resolveweave_knowledge_active', + collection: currentCollection, + points: 128, + dimensions: 64, + syncStatus: 'synced', + }, message: 'ok' } }); + return; + } + if (path.endsWith(`/index-jobs/${nextJobId}/activation-check`)) { + await route.fulfill({ json: { code: 0, data: { + eligible: true, + reasons: [], + warnings: ['p95_latency_regression_gt_25_percent'], + qualityRunId: '33333333-3333-4333-8333-333333333333', + candidateKey: `qdrant:${nextJobId}:policy`, + }, message: 'ok' } }); + return; + } + if (path.endsWith(`/index-jobs/${nextJobId}/activate`)) { + expect(request.postDataJSON()).toMatchObject({ + expectedCurrentCollection: oldCollection, + confirmed: true, + confirmLatencyWarning: true, + }); + currentCollection = nextCollection; + jobs = jobs.map((job) => ( + job.id === nextJobId + ? { + ...job, + status: 'active', + previousCollection: oldCollection, + activatedAt: '2026-07-29T08:10:00.000Z', + } + : { + ...job, + status: 'rolled_back', + rolledBackAt: '2026-07-29T08:10:00.000Z', + } + )); + await route.fulfill({ json: { code: 0, data: jobs[0], message: 'ok' } }); + return; + } + if (path.endsWith(`/index-jobs/${nextJobId}/rollback`)) { + expect(request.postDataJSON()).toMatchObject({ + expectedCurrentCollection: nextCollection, + confirmed: true, + }); + currentCollection = oldCollection; + jobs = jobs.map((job) => ( + job.id === nextJobId + ? { + ...job, + status: 'rolled_back', + rolledBackAt: '2026-07-29T08:12:00.000Z', + } + : { + ...job, + status: 'active', + rolledBackAt: null, + activatedAt: '2026-07-29T08:12:00.000Z', + } + )); + await route.fulfill({ json: { code: 0, data: jobs[1], message: 'ok' } }); + return; + } + if (path.endsWith('/index-jobs')) { + await route.fulfill({ json: { code: 0, data: { + items: jobs, + total: jobs.length, + page: 1, + pageSize: 50, + }, message: 'ok' } }); + return; + } + if (path.endsWith('/traces')) { + await route.fulfill({ json: { code: 0, data: { + items: [], + total: 0, + page: 1, + pageSize: 20, + }, message: 'ok' } }); + return; + } + await route.fallback(); + }); + + await page.getByText('质量实验室').click(); + await page.getByText('实验运行').click(); + await page.getByTitle('内存基线').click(); + await expect(page.getByText(`Qdrant 索引 · ${nextCollection}`)).toBeVisible(); + if (process.env.CAPTURE_RELEASE_EVIDENCE === '1') { + await page.getByText('登录成功').waitFor({ state: 'hidden' }); + await page.screenshot({ + path: 'docs/releases/assets/v0.3.2-quality-backends.png', + fullPage: true, + }); + } + await page.keyboard.press('Escape'); + + await page.getByText('检索运维').click(); + await expect(page.getByText(nextCollection)).toBeVisible(); + if (process.env.CAPTURE_RELEASE_EVIDENCE === '1') { + await page.getByText('登录成功').waitFor({ state: 'hidden' }); + await page.screenshot({ + path: 'docs/releases/assets/v0.3.2-index-ready.png', + fullPage: true, + }); + } + + const nextRow = page.locator('tr').filter({ hasText: nextCollection }); + await nextRow.getByRole('button', { name: '激活' }).click(); + await expect(page.getByText('Quality Lab 门禁已通过，可以原子切换 alias。')).toBeVisible(); + await expect(page.getByRole('button', { name: '确认' })).toBeDisabled(); + await page.getByText('我已确认 P95 延迟警告并继续激活').click(); + if (process.env.CAPTURE_RELEASE_EVIDENCE === '1') { + await page.waitForTimeout(350); + await page.screenshot({ + path: 'docs/releases/assets/v0.3.2-activation-gate.png', + fullPage: true, + }); + } + await page.getByRole('button', { name: '确认' }).click(); + await expect(nextRow).toContainText('已激活'); + await expect(page.getByText(nextCollection).first()).toBeVisible(); + if (process.env.CAPTURE_RELEASE_EVIDENCE === '1') { + await page.screenshot({ + path: 'docs/releases/assets/v0.3.2-index-active.png', + fullPage: true, + }); + } + + await nextRow.getByRole('button', { name: '回滚' }).click(); + await expect(page.getByText(`Alias 将切回已验证 collection：${oldCollection}`)).toBeVisible(); + await page.getByRole('button', { name: '确认' }).click(); + await expect(page.getByText(oldCollection).first()).toBeVisible(); + await expect(nextRow).toContainText('已回滚'); + if (process.env.CAPTURE_RELEASE_EVIDENCE === '1') { + await page.screenshot({ + path: 'docs/releases/assets/v0.3.2-index-rolled-back.png', + fullPage: true, + }); + } + + failRequests = true; + await page.getByRole('button', { name: '刷新' }).click(); + await expect(page.getByText('检索运维数据加载失败，请重试。')).toBeVisible(); + if (process.env.CAPTURE_RELEASE_EVIDENCE === '1') { + await page.screenshot({ + path: 'docs/releases/assets/v0.3.2-ops-error.png', + fullPage: true, + }); + } + }); });