From b7d8d698b4cd6d702fcf12700bc4523951056e35 Mon Sep 17 00:00:00 2001 From: aircode610 Date: Thu, 2 Jul 2026 17:14:58 +0200 Subject: [PATCH 1/4] fix(extraction): guard empty provider choices in PDF extraction path (#8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The issue-#8 fix (PR #13) taught the OCR and form-fill response parsers to reject an empty provider `choices` list, but the primary PDF extraction entry point still blind-indexed `response.choices[0]`. A content-filtered or empty vision completion for a blank/unreadable scanned PDF can return `choices == []`; `choices[0]` then raised a raw IndexError (surfacing as a 500) instead of the typed ExtractionError — the exact crash pattern issue #8 set out to eliminate. Fixed the same pattern at all three call sites in extraction.py (extract_from_letter_file, generate_reply_text, generate_checklist) and added a regression test for the empty-choices scanned-PDF case. Verified: the new test reproduces `IndexError: list index out of range` at extraction.py:284 without the fix, and passes as a typed ExtractionError with it. Full backend suite: 22 passed. --- backend/app/services/extraction.py | 22 ++++++++++++-- backend/tests/test_scanned_pdf_graceful.py | 35 ++++++++++++++++++---- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/backend/app/services/extraction.py b/backend/app/services/extraction.py index 1c04fc6..7d13fff 100644 --- a/backend/app/services/extraction.py +++ b/backend/app/services/extraction.py @@ -281,7 +281,15 @@ async def extract_from_letter_file( temperature=0.1, ) - tool_calls = response.choices[0].message.tool_calls or [] + # Defensive: never blind-index the provider response. A content-filtered or + # empty completion for a blank/unreadable scan can come back with an empty + # `choices` list (or a null message) — indexing `choices[0]` then raised a + # raw IndexError that surfaced as a 500. This is the same defect pattern the + # OCR / form-fill parsers already guard against (issue #8); handle it here at + # the primary extraction entry point too, mapping it to the typed error. + choices = response.choices or [] + message = choices[0].message if choices else None + tool_calls = (message.tool_calls if message else None) or [] if not tool_calls: # The model couldn't find structured content to extract. For a scanned, # image-only PDF with no readable text this is the expected outcome — @@ -417,7 +425,11 @@ async def generate_reply_text( messages=[{"role": "user", "content": prompt}], temperature=0.3, ) - return (response.choices[0].message.content or "").strip() + # Same defensive guard as the extraction path: an empty `choices` list or a + # null message must not raise a raw IndexError — return empty text instead. + choices = response.choices or [] + message = choices[0].message if choices else None + return ((message.content if message else None) or "").strip() def _response_prompt(extracted: ExtractedLetter) -> str: @@ -482,7 +494,11 @@ async def generate_checklist(extracted: ExtractedLetter, lang: str) -> list[str] messages=[{"role": "user", "content": _checklist_prompt(extracted, lang)}], temperature=0.2, ) - raw = (response.choices[0].message.content or "").strip() + # Same defensive guard as the extraction path: tolerate an empty `choices` + # list / null message instead of blind-indexing (which raised IndexError). + choices = response.choices or [] + message = choices[0].message if choices else None + raw = ((message.content if message else None) or "").strip() # Best-effort parse — model may return code-fenced JSON. if raw.startswith("```"): raw = raw.strip("`") diff --git a/backend/tests/test_scanned_pdf_graceful.py b/backend/tests/test_scanned_pdf_graceful.py index 4a07122..064a637 100644 --- a/backend/tests/test_scanned_pdf_graceful.py +++ b/backend/tests/test_scanned_pdf_graceful.py @@ -90,21 +90,26 @@ def __init__(self, tool_calls): class _FakeResponse: - def __init__(self, tool_calls): - self.choices = [_FakeChoice(tool_calls)] + def __init__(self, tool_calls, *, empty_choices=False): + # `empty_choices=True` models a content-filtered / empty provider + # completion for a blank scan, where `choices` comes back as []. + self.choices = [] if empty_choices else [_FakeChoice(tool_calls)] class _FakeCompletions: - def __init__(self, tool_calls): + def __init__(self, tool_calls, *, empty_choices=False): self._tool_calls = tool_calls + self._empty_choices = empty_choices async def create(self, **kwargs): - return _FakeResponse(self._tool_calls) + return _FakeResponse(self._tool_calls, empty_choices=self._empty_choices) class _FakeClient: - def __init__(self, tool_calls): - self.chat = types.SimpleNamespace(completions=_FakeCompletions(tool_calls)) + def __init__(self, tool_calls, *, empty_choices=False): + self.chat = types.SimpleNamespace( + completions=_FakeCompletions(tool_calls, empty_choices=empty_choices) + ) async def test_extract_from_letter_file_no_tool_call(monkeypatch): @@ -124,6 +129,24 @@ async def test_extract_from_letter_file_empty_pages(monkeypatch): await extraction.extract_from_letter_file("/tmp/scan.pdf", "application/pdf") +async def test_extract_from_letter_file_empty_choices(monkeypatch): + # A content-filtered / empty provider completion for a blank scan can return + # an empty `choices` list. Blind-indexing `choices[0]` used to raise a raw + # IndexError (→ 500); it must surface as a typed ExtractionError instead. + monkeypatch.setattr( + extraction, "split_to_image_bytes", lambda p, m: [(b"\x89PNG", "image/png")] + ) + monkeypatch.setattr(extraction.store, "search", lambda *a, **k: []) + monkeypatch.setattr( + extraction, + "_get_client", + lambda: _FakeClient(tool_calls=[], empty_choices=True), + ) + + with pytest.raises(ExtractionError): + await extraction.extract_from_letter_file("/tmp/scan.pdf", "application/pdf") + + # -------------------------------------------------------------------------- # 4. orchestrator._ocr_letter_file — handles PDFs + blank scans gracefully # -------------------------------------------------------------------------- From 5109d5341a71843992d372180b4fa253141df787 Mon Sep 17 00:00:00 2001 From: aircode610 Date: Thu, 2 Jul 2026 17:27:24 +0200 Subject: [PATCH 2/4] ci: add Lint Python workflow so the tests job runs on this branch This branch was missing .github/workflows/lint.yml (present on every other feature branch), so PR #20 had 0 check runs and the tests job never executed. Add the canonical workflow. Verified the backend test suite passes under the exact CI setup (Python 3.12 + backend/requirements.txt + pytest-asyncio): 22 passed. --- .github/workflows/lint.yml | 45 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..823c361 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,45 @@ +name: Lint Python + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install ruff + run: pip install ruff + + - name: Ruff check + run: ruff check backend/ ai/ + + - name: Ruff format check + run: ruff format --check backend/ ai/ + + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install -r backend/requirements.txt pytest pytest-asyncio + + - name: Run tests + working-directory: backend + env: + DATABASE_URL: "sqlite:///test.db" + JWT_SECRET: "test-secret-test-secret-test-secret-32" + run: python -m pytest tests/ -v From faa6953bf1b3e5ff26e30c46461ca11ba66ab1c4 Mon Sep 17 00:00:00 2001 From: aircode610 Date: Thu, 2 Jul 2026 17:39:47 +0200 Subject: [PATCH 3/4] fix(ci): resolve ruff lint errors and apply ruff formatting The newly added Lint Python workflow's ruff job was failing: - Removed unused imports (datetime, json, Any, RagHit, generate_reply_text) - Removed f-strings without placeholders in ai/rag/ingest.py - Removed dead locals (classification_data, action_titles, reply_actions) - Moved public.py logger below imports (fixes E402) - Ran 'ruff format' across backend/ and ai/ so 'ruff format --check' passes Backend test suite (22 tests) still passes. --- ai/form_fill.py | 8 +- ai/rag/generator.py | 14 +- ai/rag/ingest.py | 90 ++++++---- ai/rag/retrieval.py | 38 ++-- ai/rag/schemas.py | 4 +- ai/react_agent/agent.py | 48 +++-- ai/react_agent/ocr.py | 4 +- ai/schemas.py | 47 +++-- backend/app/auth/dependencies.py | 11 +- backend/app/auth/router.py | 2 - backend/app/auth/utils.py | 4 +- backend/app/config.py | 6 +- backend/app/database.py | 1 + backend/app/errors.py | 49 ++--- backend/app/main.py | 1 + backend/app/models.py | 38 ++-- backend/app/pipeline/orchestrator.py | 141 +++++++++++---- backend/app/rag/store.py | 1 + backend/app/routers/actions.py | 11 +- backend/app/routers/deadlines.py | 11 +- backend/app/routers/letters.py | 37 +++- backend/app/routers/public.py | 115 ++++++------ backend/app/routers/rag.py | 2 +- backend/app/schemas.py | 20 ++- backend/app/services/__init__.py | 6 +- backend/app/services/ai_bridge.py | 257 ++++++++++++++------------- backend/app/services/extraction.py | 21 ++- backend/app/services/pdf_pages.py | 4 +- backend/app/services/persistence.py | 4 +- 29 files changed, 622 insertions(+), 373 deletions(-) diff --git a/ai/form_fill.py b/ai/form_fill.py index 6c3b868..50de5f3 100644 --- a/ai/form_fill.py +++ b/ai/form_fill.py @@ -11,9 +11,7 @@ import httpx DASHSCOPE_API_KEY = os.environ.get("DASHSCOPE_API_KEY", "") -DASHSCOPE_INTL_URL = ( - "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" -) +DASHSCOPE_INTL_URL = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" # Standard placeholder patterns for common German form fields PLACEHOLDER_MAP = { @@ -70,7 +68,9 @@ async def generate_filled_form( b64 = base64.b64encode(image_bytes).decode() ext = image_path.rsplit(".", 1)[-1].lower() - mime = {"jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png"}.get(ext, "image/jpeg") + mime = {"jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png"}.get( + ext, "image/jpeg" + ) data_url = f"data:{mime};base64,{b64}" field_instructions = _build_field_instructions(placeholders) diff --git a/ai/rag/generator.py b/ai/rag/generator.py index 11e64ff..fe78f81 100644 --- a/ai/rag/generator.py +++ b/ai/rag/generator.py @@ -14,8 +14,14 @@ QWEN_AGENT_MODEL = os.environ.get("QWEN_AGENT_MODEL", "qwen3.7-plus") LANGUAGE_NAMES = { - "en": "English", "de": "German", "tr": "Turkish", "ar": "Arabic", - "es": "Spanish", "fr": "French", "zh": "Chinese", "fa": "Persian", + "en": "English", + "de": "German", + "tr": "Turkish", + "ar": "Arabic", + "es": "Spanish", + "fr": "French", + "zh": "Chinese", + "fa": "Persian", } _model = ChatOpenAI( @@ -34,7 +40,9 @@ async def generate_response( language: str = "en", ) -> GenerationOutput: """Retrieve legal context from ChromaDB, inject into prompt, return structured output.""" - legal_context = retrieve_as_context(agent_result.letter_type, agent_result.consequence) + legal_context = retrieve_as_context( + agent_result.letter_type, agent_result.consequence + ) prompt = GENERATION_PROMPT.format( ocr_text=ocr_text[:3000], diff --git a/ai/rag/ingest.py b/ai/rag/ingest.py index fb26cff..c489210 100644 --- a/ai/rag/ingest.py +++ b/ai/rag/ingest.py @@ -26,31 +26,32 @@ # ── Paths ───────────────────────────────────────────────────────────────────── -ROOT = Path(__file__).resolve().parent.parent # ai/ -LAWS_DIR = ROOT / "data" / "laws" # ai/data/laws/ -CHROMA_DIR = ROOT / "data" / "chroma" # ai/data/chroma/ +ROOT = Path(__file__).resolve().parent.parent # ai/ +LAWS_DIR = ROOT / "data" / "laws" # ai/data/laws/ +CHROMA_DIR = ROOT / "data" / "chroma" # ai/data/chroma/ COLLECTION_NAME = "german_laws" # ── Law file → abbreviation map ─────────────────────────────────────────────── LAWS = { - "aufenthg.md": "AufenthG", - "aufenthv.md": "AufenthV", - "beschv.md": "BeschV", - "vwvfg.md": "VwVfG", - "bafoeg.md": "BAföG", - "asylg.md": "AsylG", - "asylblg.md": "AsylbLG", - "wogg.md": "WoGG", - "bmg.md": "BMG", - "intv.md": "IntV", - "owig.md": "OWiG", - "estg.md": "EStG", - "sgb5.md": "SGB V", + "aufenthg.md": "AufenthG", + "aufenthv.md": "AufenthV", + "beschv.md": "BeschV", + "vwvfg.md": "VwVfG", + "bafoeg.md": "BAföG", + "asylg.md": "AsylG", + "asylblg.md": "AsylbLG", + "wogg.md": "WoGG", + "bmg.md": "BMG", + "intv.md": "IntV", + "owig.md": "OWiG", + "estg.md": "EStG", + "sgb5.md": "SGB V", } # ── Qwen client ─────────────────────────────────────────────────────────────── + def get_qwen_client() -> OpenAI: api_key = os.getenv("DASHSCOPE_API_KEY") if not api_key: @@ -64,8 +65,8 @@ def get_qwen_client() -> OpenAI: # Qwen text-embedding-v3 limits -MAX_BATCH_SIZE = 10 # max texts per API call -MAX_CHARS = 6000 # conservative char limit (~8192 tokens safety margin) +MAX_BATCH_SIZE = 10 # max texts per API call +MAX_CHARS = 6000 # conservative char limit (~8192 tokens safety margin) def truncate(text: str) -> str: @@ -85,7 +86,7 @@ def embed_texts(client: OpenAI, texts: list[str]) -> list[list[float]]: texts = [truncate(t) for t in texts] for i in range(0, len(texts), MAX_BATCH_SIZE): - batch = texts[i:i + MAX_BATCH_SIZE] + batch = texts[i : i + MAX_BATCH_SIZE] response = client.embeddings.create( model="text-embedding-v3", input=batch, @@ -101,6 +102,7 @@ def embed_texts(client: OpenAI, texts: list[str]) -> list[list[float]]: # ── Chunking ────────────────────────────────────────────────────────────────── + def parse_paragraphs(text: str, law_abbrev: str) -> list[dict]: """ Split a law's markdown into one chunk per § paragraph. @@ -110,7 +112,7 @@ def parse_paragraphs(text: str, law_abbrev: str) -> list[dict]: id, text, paragraph, title, law """ # Match § headers at any heading level: ### § 1, #### § 4a, etc. - pattern = r'^#{1,4} (§ \d+[a-z]?\b.*?)$' + pattern = r"^#{1,4} (§ \d+[a-z]?\b.*?)$" matches = list(re.finditer(pattern, text, re.MULTILINE)) chunks = [] @@ -121,7 +123,7 @@ def parse_paragraphs(text: str, law_abbrev: str) -> list[dict]: end = matches[i + 1].start() if i + 1 < len(matches) else len(text) header = match.group(1).strip() - para_num_match = re.match(r'(§ \d+[a-z]?)', header) + para_num_match = re.match(r"(§ \d+[a-z]?)", header) para_num = para_num_match.group(1) if para_num_match else header body = text[start:end].strip() @@ -141,19 +143,22 @@ def parse_paragraphs(text: str, law_abbrev: str) -> list[dict]: counter += 1 seen_ids.add(unique_id) - chunks.append({ - "id": unique_id, - "text": body, - "paragraph": para_num, - "title": header, - "law": law_abbrev, - }) + chunks.append( + { + "id": unique_id, + "text": body, + "paragraph": para_num, + "title": header, + "law": law_abbrev, + } + ) return chunks # ── Main ────────────────────────────────────────────────────────────────────── + def ingest_all(): print("── Klar RAG Ingestion ──────────────────────────────────────") @@ -197,32 +202,39 @@ def ingest_all(): print(f" ⚠ No paragraphs parsed in {filename}") continue - print(f" Embedding {law_abbrev}: {len(chunks)} paragraphs ...", end=" ", flush=True) + print( + f" Embedding {law_abbrev}: {len(chunks)} paragraphs ...", + end=" ", + flush=True, + ) texts_to_embed = [c["text"] for c in chunks] embeddings = embed_texts(client, texts_to_embed) batch_size = 100 for i in range(0, len(chunks), batch_size): - batch_chunks = chunks[i:i + batch_size] - batch_embeddings = embeddings[i:i + batch_size] + batch_chunks = chunks[i : i + batch_size] + batch_embeddings = embeddings[i : i + batch_size] collection.add( ids=[c["id"] for c in batch_chunks], documents=[c["text"] for c in batch_chunks], - metadatas=[{ - "law": c["law"], - "paragraph": c["paragraph"], - "title": c["title"], - } for c in batch_chunks], + metadatas=[ + { + "law": c["law"], + "paragraph": c["paragraph"], + "title": c["title"], + } + for c in batch_chunks + ], embeddings=batch_embeddings, ) - print(f"✅") + print("✅") total_chunks += len(chunks) print() - print(f"── Done ────────────────────────────────────────────────────") + print("── Done ────────────────────────────────────────────────────") print(f" Total chunks ingested : {total_chunks}") print(f" ChromaDB saved to : {CHROMA_DIR.resolve()}") print() @@ -230,4 +242,4 @@ def ingest_all(): if __name__ == "__main__": - ingest_all() \ No newline at end of file + ingest_all() diff --git a/ai/rag/retrieval.py b/ai/rag/retrieval.py index aee8302..81c275c 100644 --- a/ai/rag/retrieval.py +++ b/ai/rag/retrieval.py @@ -25,20 +25,21 @@ # ── Paths ───────────────────────────────────────────────────────────────────── -ROOT = Path(__file__).resolve().parent.parent # ai/ -CHROMA_DIR = ROOT / "data" / "chroma" # ai/data/chroma/ +ROOT = Path(__file__).resolve().parent.parent # ai/ +CHROMA_DIR = ROOT / "data" / "chroma" # ai/data/chroma/ COLLECTION_NAME = "german_laws" # ── Schema ──────────────────────────────────────────────────────────────────── + @dataclass class LegalChunk: - section: str # e.g. "§ 81" - law: str # e.g. "AufenthG" - title: str # e.g. "§ 81 Beantragung des Aufenthaltstitels" - text: str # full paragraph text - citation: str # e.g. "§ 81 AufenthG" - score: float # cosine similarity, higher = more relevant + section: str # e.g. "§ 81" + law: str # e.g. "AufenthG" + title: str # e.g. "§ 81 Beantragung des Aufenthaltstitels" + text: str # full paragraph text + citation: str # e.g. "§ 81 AufenthG" + score: float # cosine similarity, higher = more relevant # ── Singleton clients ───────────────────────────────────────────────────────── @@ -87,6 +88,7 @@ def _embed_query(query: str) -> list[float]: # ── Core retrieval ──────────────────────────────────────────────────────────── + def retrieve_legal_context( letter_type: str, consequence: str, @@ -124,14 +126,16 @@ def retrieve_legal_context( results["distances"][0], ): section = meta.get("paragraph", meta.get("section", "Unknown")) - chunks.append(LegalChunk( - section=section, - law=meta["law"], - title=meta["title"], - text=doc, - citation=f"{section} {meta['law']}", - score=round(1 - distance, 4), - )) + chunks.append( + LegalChunk( + section=section, + law=meta["law"], + title=meta["title"], + text=doc, + citation=f"{section} {meta['law']}", + score=round(1 - distance, 4), + ) + ) return chunks @@ -156,4 +160,4 @@ def retrieve_as_context( text_preview = c.text[:1000] + "..." if len(c.text) > 1000 else c.text parts.append(f"[{c.citation}] {c.title}\n{text_preview}") - return "\n\n---\n\n".join(parts) \ No newline at end of file + return "\n\n---\n\n".join(parts) diff --git a/ai/rag/schemas.py b/ai/rag/schemas.py index edc7ba4..bcae2c2 100644 --- a/ai/rag/schemas.py +++ b/ai/rag/schemas.py @@ -8,7 +8,7 @@ @dataclass class RAGEvent: - type: str # "explanation" | "response_draft" | "checklist" | "citations" | "error" + type: str # "explanation" | "response_draft" | "checklist" | "citations" | "error" data: dict confidence: str = "high" # "high" if RAG matched well, "low" if no strong matches # data shapes per type: @@ -16,4 +16,4 @@ class RAGEvent: # response_draft: {"chunk": str} — streamed token by token # checklist: {"items": list[str]} — emitted once, complete # citations: {"items": list[dict]} — emitted once, [{section, text}, ...] - # error: {"message": str} \ No newline at end of file + # error: {"message": str} diff --git a/ai/react_agent/agent.py b/ai/react_agent/agent.py index 155c349..40a671a 100644 --- a/ai/react_agent/agent.py +++ b/ai/react_agent/agent.py @@ -43,9 +43,15 @@ async def run_react_agent(ocr_text: str) -> AsyncGenerator[AgentEvent, None]: """Run the ReAct agent with structured output via LangGraph response_format.""" try: - result = await _agent.ainvoke({ - "messages": [HumanMessage(content=f"Analyze this German official letter:\n\n{ocr_text}")], - }) + result = await _agent.ainvoke( + { + "messages": [ + HumanMessage( + content=f"Analyze this German official letter:\n\n{ocr_text}" + ) + ], + } + ) analysis: AgentAnalysis = result["structured_response"] @@ -57,10 +63,22 @@ async def run_react_agent(ocr_text: str) -> AsyncGenerator[AgentEvent, None]: except ValueError: pass - yield AgentEvent("classification", {"type": analysis.classification.type, "agency": analysis.classification.agency}) - yield AgentEvent("risk_score", {"score": analysis.risk_score.score, "label": analysis.risk_score.label}) + yield AgentEvent( + "classification", + { + "type": analysis.classification.type, + "agency": analysis.classification.agency, + }, + ) + yield AgentEvent( + "risk_score", + {"score": analysis.risk_score.score, "label": analysis.risk_score.label}, + ) if analysis.deadline.date: - yield AgentEvent("deadline", {"date": analysis.deadline.date, "days_remaining": days_remaining}) + yield AgentEvent( + "deadline", + {"date": analysis.deadline.date, "days_remaining": days_remaining}, + ) yield AgentEvent("consequence", {"text": analysis.consequence.text}) except Exception as e: @@ -70,9 +88,19 @@ async def run_react_agent(ocr_text: str) -> AsyncGenerator[AgentEvent, None]: def get_last_agent_result(events: list[AgentEvent], ocr_text: str) -> AgentResult: """Reconstruct an AgentResult from collected events.""" data = {e.type: e.data for e in events} - c, d, r, q = data.get("classification", {}), data.get("deadline", {}), data.get("risk_score", {}), data.get("consequence", {}) + c, d, r, q = ( + data.get("classification", {}), + data.get("deadline", {}), + data.get("risk_score", {}), + data.get("consequence", {}), + ) return AgentResult( - ocr_text=ocr_text, letter_type=c.get("type", "Unknown"), agency=c.get("agency", "Unknown"), - deadline_date=d.get("date"), days_remaining=d.get("days_remaining"), - consequence=q.get("text", ""), risk_score=r.get("score", 3), risk_label=r.get("label", "Medium"), + ocr_text=ocr_text, + letter_type=c.get("type", "Unknown"), + agency=c.get("agency", "Unknown"), + deadline_date=d.get("date"), + days_remaining=d.get("days_remaining"), + consequence=q.get("text", ""), + risk_score=r.get("score", 3), + risk_label=r.get("label", "Medium"), ) diff --git a/ai/react_agent/ocr.py b/ai/react_agent/ocr.py index 8f047f7..e1359f1 100644 --- a/ai/react_agent/ocr.py +++ b/ai/react_agent/ocr.py @@ -42,7 +42,9 @@ def _parse_ocr_response(result: object) -> str: "image without readable content." ) - message = (choices[0] or {}).get("message") if isinstance(choices[0], dict) else None + message = ( + (choices[0] or {}).get("message") if isinstance(choices[0], dict) else None + ) content = (message or {}).get("content") if isinstance(message, dict) else None if not content or not str(content).strip(): diff --git a/ai/schemas.py b/ai/schemas.py index 0151249..17803a2 100644 --- a/ai/schemas.py +++ b/ai/schemas.py @@ -4,19 +4,32 @@ # --- Structured output models (used by LLM response_format) --- + class Classification(BaseModel): - type: str = Field(description="Letter type, e.g. 'Residence Permit - Document Request', 'Health Insurance - Tax ID Request'") - agency: str = Field(description="Sender agency name, e.g. 'Techniker Krankenkasse', 'Ausländerbehörde München'") + type: str = Field( + description="Letter type, e.g. 'Residence Permit - Document Request', 'Health Insurance - Tax ID Request'" + ) + agency: str = Field( + description="Sender agency name, e.g. 'Techniker Krankenkasse', 'Ausländerbehörde München'" + ) class Deadline(BaseModel): - date: str | None = Field(description="Deadline date in YYYY-MM-DD format, or null if no deadline") - days_remaining: int | None = Field(description="Days until deadline from today, or null") - source: str = Field(description="'letter' if read directly, 'calculated' if computed from letter date, 'searched' if from web, 'none' if no deadline applies") + date: str | None = Field( + description="Deadline date in YYYY-MM-DD format, or null if no deadline" + ) + days_remaining: int | None = Field( + description="Days until deadline from today, or null" + ) + source: str = Field( + description="'letter' if read directly, 'calculated' if computed from letter date, 'searched' if from web, 'none' if no deadline applies" + ) class Consequence(BaseModel): - text: str = Field(description="Detailed consequence description of what happens if deadline is missed or action not taken") + text: str = Field( + description="Detailed consequence description of what happens if deadline is missed or action not taken" + ) severity: str = Field(description="One-line severity summary") @@ -28,6 +41,7 @@ class RiskScore(BaseModel): class AgentAnalysis(BaseModel): """Structured output from the ReAct agent letter analysis.""" + classification: Classification deadline: Deadline consequence: Consequence @@ -35,23 +49,34 @@ class AgentAnalysis(BaseModel): class Citation(BaseModel): - section: str = Field(description="Legal paragraph reference, e.g. '§ 81 Abs. 4 AufenthG'") + section: str = Field( + description="Legal paragraph reference, e.g. '§ 81 Abs. 4 AufenthG'" + ) text: str = Field(description="Brief explanation of why this citation is relevant") class GenerationOutput(BaseModel): """Structured output from the response generation LLM.""" - explanation: str = Field(description="Clear plain-language explanation of the letter") + + explanation: str = Field( + description="Clear plain-language explanation of the letter" + ) response_draft: str = Field(description="Formal response letter in Behördendeutsch") - checklist: list[str] = Field(description="List of documents the user needs to prepare, with German terms in parentheses") - citations: list[Citation] = Field(default_factory=list, description="Legal § references that are relevant. Empty list if none found.") + checklist: list[str] = Field( + description="List of documents the user needs to prepare, with German terms in parentheses" + ) + citations: list[Citation] = Field( + default_factory=list, + description="Legal § references that are relevant. Empty list if none found.", + ) # --- Internal data transfer objects --- + @dataclass class AgentEvent: - type: str # "classification", "risk_score", "deadline", "consequence", "error" + type: str # "classification", "risk_score", "deadline", "consequence", "error" data: dict diff --git a/backend/app/auth/dependencies.py b/backend/app/auth/dependencies.py index 7659868..0c10f16 100644 --- a/backend/app/auth/dependencies.py +++ b/backend/app/auth/dependencies.py @@ -1,7 +1,6 @@ """FastAPI dependency that resolves the current user from the session cookie.""" import logging -from datetime import datetime from fastapi import Depends, Request, status from sqlmodel import Session as DBSession, select @@ -37,7 +36,8 @@ def _resolve_user(token: str | None, db: DBSession) -> User: logger.warning( "AUTH_SESSION_NOT_FOUND: no Session row for token=%s... " "(DB=%s; possible causes: db wiped, multiple workers, env mismatch)", - token_prefix, settings.database_url, + token_prefix, + settings.database_url, ) raise KlarHTTPException( status_code=status.HTTP_401_UNAUTHORIZED, @@ -47,7 +47,9 @@ def _resolve_user(token: str | None, db: DBSession) -> User: if session_row.expires_at < utcnow(): logger.info( "AUTH_SESSION_EXPIRED: token=%s... expired_at=%s now=%s", - token_prefix, session_row.expires_at, utcnow(), + token_prefix, + session_row.expires_at, + utcnow(), ) raise KlarHTTPException( status_code=status.HTTP_401_UNAUTHORIZED, @@ -59,7 +61,8 @@ def _resolve_user(token: str | None, db: DBSession) -> User: # Session row exists but the user it points to is gone — corrupt FK. logger.error( "AUTH: orphan Session row token=%s... user_id=%s has no User", - token_prefix, session_row.user_id, + token_prefix, + session_row.user_id, ) raise KlarHTTPException( status_code=status.HTTP_401_UNAUTHORIZED, diff --git a/backend/app/auth/router.py b/backend/app/auth/router.py index 151767b..67af069 100644 --- a/backend/app/auth/router.py +++ b/backend/app/auth/router.py @@ -1,7 +1,5 @@ """Authentication routes: signup, login, logout, me, forgot/reset password.""" -from datetime import datetime - from fastapi import APIRouter, Depends, Request, Response, status from pydantic import BaseModel, EmailStr, Field, field_validator from sqlmodel import Session as DBSession, select diff --git a/backend/app/auth/utils.py b/backend/app/auth/utils.py index 9d9a5f5..7198836 100644 --- a/backend/app/auth/utils.py +++ b/backend/app/auth/utils.py @@ -12,7 +12,9 @@ def hash_password(plain: str) -> str: """bcrypt with cost factor 12 — ~250ms on a modern laptop.""" - return bcrypt.hashpw(plain.encode("utf-8"), bcrypt.gensalt(rounds=12)).decode("utf-8") + return bcrypt.hashpw(plain.encode("utf-8"), bcrypt.gensalt(rounds=12)).decode( + "utf-8" + ) def verify_password(plain: str, password_hash: str) -> bool: diff --git a/backend/app/config.py b/backend/app/config.py index 15037ca..5a60bc6 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -46,7 +46,11 @@ def effective_llm_api_key(self) -> str: @property def effective_llm_base_url(self) -> str: - return self.qwen_api_base or self.llm_base_url or "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" + return ( + self.qwen_api_base + or self.llm_base_url + or "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" + ) @property def effective_llm_model(self) -> str: diff --git a/backend/app/database.py b/backend/app/database.py index f666408..836a132 100644 --- a/backend/app/database.py +++ b/backend/app/database.py @@ -21,6 +21,7 @@ def init_db() -> None: path = settings.database_url.replace("sqlite:///", "", 1) Path(path).parent.mkdir(parents=True, exist_ok=True) from app import models # noqa: F401 — populate SQLModel metadata + SQLModel.metadata.create_all(engine) diff --git a/backend/app/errors.py b/backend/app/errors.py index 20478a8..1240501 100644 --- a/backend/app/errors.py +++ b/backend/app/errors.py @@ -56,35 +56,43 @@ class ErrorCode(str, Enum): """ # --- generic --- - HTTP_ERROR = "HTTP_ERROR" # untyped fallback (legacy HTTPException) - INTERNAL_ERROR = "INTERNAL_ERROR" # unhandled exception - VALIDATION_ERROR = "VALIDATION_ERROR" # request body / query / path + HTTP_ERROR = "HTTP_ERROR" # untyped fallback (legacy HTTPException) + INTERNAL_ERROR = "INTERNAL_ERROR" # unhandled exception + VALIDATION_ERROR = "VALIDATION_ERROR" # request body / query / path # --- auth --- - AUTH_NOT_AUTHENTICATED = "AUTH_NOT_AUTHENTICATED" # no cookie at all - AUTH_SESSION_NOT_FOUND = "AUTH_SESSION_NOT_FOUND" # cookie present, but no Session row in DB - AUTH_SESSION_EXPIRED = "AUTH_SESSION_EXPIRED" # Session row exists but past expires_at + AUTH_NOT_AUTHENTICATED = "AUTH_NOT_AUTHENTICATED" # no cookie at all + AUTH_SESSION_NOT_FOUND = ( + "AUTH_SESSION_NOT_FOUND" # cookie present, but no Session row in DB + ) + AUTH_SESSION_EXPIRED = ( + "AUTH_SESSION_EXPIRED" # Session row exists but past expires_at + ) AUTH_INVALID_CREDENTIALS = "AUTH_INVALID_CREDENTIALS" # wrong email / password - AUTH_EMAIL_TAKEN = "AUTH_EMAIL_TAKEN" # signup with existing email - AUTH_INVALID_RESET_TOKEN = "AUTH_INVALID_RESET_TOKEN" # token unknown / already used + AUTH_EMAIL_TAKEN = "AUTH_EMAIL_TAKEN" # signup with existing email + AUTH_INVALID_RESET_TOKEN = ( + "AUTH_INVALID_RESET_TOKEN" # token unknown / already used + ) AUTH_RESET_TOKEN_EXPIRED = "AUTH_RESET_TOKEN_EXPIRED" # token past 15-min TTL # --- letters --- LETTER_NOT_FOUND = "LETTER_NOT_FOUND" - LETTER_FILE_MISSING = "LETTER_FILE_MISSING" # row exists but file gone + LETTER_FILE_MISSING = "LETTER_FILE_MISSING" # row exists but file gone LETTER_EMPTY_UPLOAD = "LETTER_EMPTY_UPLOAD" LETTER_TOO_LARGE = "LETTER_TOO_LARGE" LETTER_UNSUPPORTED_TYPE = "LETTER_UNSUPPORTED_TYPE" - LETTER_CORRUPT_FILE = "LETTER_CORRUPT_FILE" # magic-bytes mismatch - LETTER_MIME_MISMATCH = "LETTER_MIME_MISMATCH" # declared ≠ detected + LETTER_CORRUPT_FILE = "LETTER_CORRUPT_FILE" # magic-bytes mismatch + LETTER_MIME_MISMATCH = "LETTER_MIME_MISMATCH" # declared ≠ detected # --- actions --- ACTION_NOT_FOUND = "ACTION_NOT_FOUND" # --- pipeline / AI --- - EXTRACTION_FAILED = "EXTRACTION_FAILED" # SSE-only: model returned no tool call, parse error, etc. - LLM_PROVIDER_ERROR = "LLM_PROVIDER_ERROR" # network / 5xx from Qwen - PDF_RENDER_FAILED = "PDF_RENDER_FAILED" # pdf2image / poppler missing + EXTRACTION_FAILED = ( + "EXTRACTION_FAILED" # SSE-only: model returned no tool call, parse error, etc. + ) + LLM_PROVIDER_ERROR = "LLM_PROVIDER_ERROR" # network / 5xx from Qwen + PDF_RENDER_FAILED = "PDF_RENDER_FAILED" # pdf2image / poppler missing # User-facing default messages per code. Keep short, no jargon, no secrets. @@ -93,7 +101,6 @@ class ErrorCode(str, Enum): ErrorCode.HTTP_ERROR: "Something went wrong with that request.", ErrorCode.INTERNAL_ERROR: "Something went wrong on our end. Please try again.", ErrorCode.VALIDATION_ERROR: "Some fields in your request are invalid.", - ErrorCode.AUTH_NOT_AUTHENTICATED: "Please sign in to continue.", ErrorCode.AUTH_SESSION_NOT_FOUND: "Your session is no longer recognized. Please sign in again.", ErrorCode.AUTH_SESSION_EXPIRED: "Your session has expired. Please sign in again.", @@ -101,7 +108,6 @@ class ErrorCode(str, Enum): ErrorCode.AUTH_EMAIL_TAKEN: "An account with that email already exists.", ErrorCode.AUTH_INVALID_RESET_TOKEN: "This reset link is invalid or has already been used.", ErrorCode.AUTH_RESET_TOKEN_EXPIRED: "This reset link has expired. Please request a new one.", - ErrorCode.LETTER_NOT_FOUND: "That letter doesn't exist or you don't have access to it.", ErrorCode.LETTER_FILE_MISSING: "We can't find the uploaded file for this letter.", ErrorCode.LETTER_EMPTY_UPLOAD: "The uploaded file is empty.", @@ -109,9 +115,7 @@ class ErrorCode(str, Enum): ErrorCode.LETTER_UNSUPPORTED_TYPE: "We can only read JPEG, PNG, HEIC, WebP, or PDF letters.", ErrorCode.LETTER_CORRUPT_FILE: "The file looks corrupted or isn't the type it claims to be.", ErrorCode.LETTER_MIME_MISMATCH: "The file's content doesn't match its declared type.", - ErrorCode.ACTION_NOT_FOUND: "That action doesn't exist or you don't have access to it.", - ErrorCode.EXTRACTION_FAILED: "We couldn't read this letter. Try a clearer photo or PDF.", ErrorCode.LLM_PROVIDER_ERROR: "Our AI provider is having trouble. Please try again in a moment.", ErrorCode.PDF_RENDER_FAILED: "We couldn't open that PDF. Try uploading it as an image instead.", @@ -188,8 +192,9 @@ async def generic_http_exception_handler( # If detail is already a Klar envelope dict (from KlarHTTPException # routing through the default handler), pass it through. if isinstance(exc.detail, dict) and "code" in exc.detail: - return JSONResponse(status_code=exc.status_code, content=exc.detail, - headers=exc.headers) + return JSONResponse( + status_code=exc.status_code, content=exc.detail, headers=exc.headers + ) # Auth-shaped status codes get more specific codes by default. code = ErrorCode.HTTP_ERROR @@ -239,7 +244,9 @@ async def unhandled_exception_handler(req: Request, exc: Exception) -> JSONRespo """ logger.exception( "Unhandled exception on %s %s: %s", - req.method, req.url.path, exc, + req.method, + req.url.path, + exc, ) return JSONResponse( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/backend/app/main.py b/backend/app/main.py index 04f428b..ef34d42 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -85,6 +85,7 @@ def _bridge_env_to_ai_team() -> None: value still wins. """ import os + if settings.effective_llm_api_key: os.environ.setdefault("DASHSCOPE_API_KEY", settings.effective_llm_api_key) if settings.effective_llm_base_url: diff --git a/backend/app/models.py b/backend/app/models.py index 69b4890..19404b4 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -55,20 +55,22 @@ class DocumentCategory(str, Enum): does not fit any defined bucket. """ - HEALTH_INSURANCE = "health_insurance" # AOK, TK, BARMER, private KV - OTHER_INSURANCE = "other_insurance" # Haftpflicht, Hausrat, KFZ, Leben - BANKING = "banking" # bank accounts, credit cards, SCHUFA - TAX = "tax" # Finanzamt - IMMIGRATION = "immigration" # Ausländerbehörde, residence/visa - EDUCATION = "education" # universities, BAföG, Studentenwerk - HOUSING = "housing" # landlord, property management - UTILITIES = "utilities" # Strom, Gas, Wasser, Internet, Mobilfunk - EMPLOYMENT = "employment" # Arbeitgeber, HR, Lohn - GOVERNMENT_BENEFITS = "government_benefits" # ALG I/II, Kindergeld, Elterngeld, Wohngeld - PENSION = "pension" # Deutsche Rentenversicherung - BROADCAST_FEE = "broadcast_fee" # Beitragsservice / Rundfunk - CIVIC = "civic" # Bürgeramt, Personalausweis, Pass - LEGAL_DEBT = "legal_debt" # Mahnbescheid, Inkasso, Bußgeld, Anwalt + HEALTH_INSURANCE = "health_insurance" # AOK, TK, BARMER, private KV + OTHER_INSURANCE = "other_insurance" # Haftpflicht, Hausrat, KFZ, Leben + BANKING = "banking" # bank accounts, credit cards, SCHUFA + TAX = "tax" # Finanzamt + IMMIGRATION = "immigration" # Ausländerbehörde, residence/visa + EDUCATION = "education" # universities, BAföG, Studentenwerk + HOUSING = "housing" # landlord, property management + UTILITIES = "utilities" # Strom, Gas, Wasser, Internet, Mobilfunk + EMPLOYMENT = "employment" # Arbeitgeber, HR, Lohn + GOVERNMENT_BENEFITS = ( + "government_benefits" # ALG I/II, Kindergeld, Elterngeld, Wohngeld + ) + PENSION = "pension" # Deutsche Rentenversicherung + BROADCAST_FEE = "broadcast_fee" # Beitragsservice / Rundfunk + CIVIC = "civic" # Bürgeramt, Personalausweis, Pass + LEGAL_DEBT = "legal_debt" # Mahnbescheid, Inkasso, Bußgeld, Anwalt OTHER = "other" @@ -114,15 +116,15 @@ class Letter(SQLModel, table=True): original_file: str = "" # Spec-flat structured fields (denormalized from ActionItem for /api/letters) - letter_type: str = "" # alias of document_type for spec compat - risk_score: int = 0 # denormalized highest action risk + letter_type: str = "" # alias of document_type for spec compat + risk_score: int = 0 # denormalized highest action risk deadline_date: Optional[date] = None # denormalized most-urgent action deadline # Rich Klar extras institution: str = "" document_type: str = "" category: DocumentCategory = DocumentCategory.OTHER - summary: str = "" # language matches Letter.language + summary: str = "" # language matches Letter.language language: str = "en" # OCR + long-form generation outputs @@ -132,7 +134,7 @@ class Letter(SQLModel, table=True): # "low confidence, get a human" prompt. Computed from extraction outputs. confidence: Optional[float] = None explanation: str = "" - response_draft: str = "" # ALWAYS German (formal reply to German institution) + response_draft: str = "" # ALWAYS German (formal reply to German institution) checklist: list[str] = Field(default_factory=list, sa_column=Column(JSON)) citations: list[dict] = Field(default_factory=list, sa_column=Column(JSON)) consequence: str = "" diff --git a/backend/app/pipeline/orchestrator.py b/backend/app/pipeline/orchestrator.py index 6048a0a..8896e95 100644 --- a/backend/app/pipeline/orchestrator.py +++ b/backend/app/pipeline/orchestrator.py @@ -40,7 +40,7 @@ import os import re import tempfile -from datetime import date, datetime +from datetime import date from typing import AsyncIterator from uuid import UUID @@ -166,10 +166,31 @@ async def _ocr_letter_file(path: str) -> str: # text for common German date patterns and pick the most-likely deadline. _GERMAN_MONTHS = { - "januar": 1, "jan": 1, "februar": 2, "feb": 2, "märz": 3, "mar": 3, "mrz": 3, - "april": 4, "apr": 4, "mai": 5, "juni": 6, "jun": 6, "juli": 7, "jul": 7, - "august": 8, "aug": 8, "september": 9, "sep": 9, "sept": 9, - "oktober": 10, "okt": 10, "november": 11, "nov": 11, "dezember": 12, "dez": 12, + "januar": 1, + "jan": 1, + "februar": 2, + "feb": 2, + "märz": 3, + "mar": 3, + "mrz": 3, + "april": 4, + "apr": 4, + "mai": 5, + "juni": 6, + "jun": 6, + "juli": 7, + "jul": 7, + "august": 8, + "aug": 8, + "september": 9, + "sep": 9, + "sept": 9, + "oktober": 10, + "okt": 10, + "november": 11, + "nov": 11, + "dezember": 12, + "dez": 12, } # Match: "28. Oktober 2021", "28 Oktober 2021", "den 28. Oktober 2021" @@ -269,16 +290,20 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str except Exception as exc: logger.exception( "AI team's modules failed to import — check env (DASHSCOPE_API_KEY, " - "TAVILY_API_KEY): %s", exc, + "TAVILY_API_KEY): %s", + exc, ) - yield sse_event("error", sse_error_payload( - ErrorCode.LLM_PROVIDER_ERROR, - message=( - "AI pipeline failed to initialize. Most likely cause: missing " - "DASHSCOPE_API_KEY or TAVILY_API_KEY env var on the backend " - "process. See server logs." + yield sse_event( + "error", + sse_error_payload( + ErrorCode.LLM_PROVIDER_ERROR, + message=( + "AI pipeline failed to initialize. Most likely cause: missing " + "DASHSCOPE_API_KEY or TAVILY_API_KEY env var on the backend " + "process. See server logs." + ), ), - )) + ) return out_lang = normalize_lang(lang) @@ -316,16 +341,18 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str # STAGE 2 — ReAct agent (LangGraph + Tavily, ~5-15s) # ============================================================ agent_events_collected = [] - classification_data: dict | None = None risk_label = "Medium" async for ev in run_react_agent(ocr_text): agent_events_collected.append(ev) if ev.type == "classification": - classification_data = ev.data - category = ai_bridge.map_classification_to_category(ev.data.get("type", "")) - letter.document_type = ev.data.get("type", "") or letter.document_type + category = ai_bridge.map_classification_to_category( + ev.data.get("type", "") + ) + letter.document_type = ( + ev.data.get("type", "") or letter.document_type + ) letter.letter_type = letter.document_type letter.category = category letter.institution = ev.data.get("agency", "") or letter.institution @@ -364,17 +391,37 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str yield sse_event("consequence", {"text": consequence_text}) elif ev.type == "error": - logger.warning("ReAct agent emitted error: %s", ev.data.get("message")) + logger.warning( + "ReAct agent emitted error: %s", ev.data.get("message") + ) # Don't propagate immediately — try to continue with what we have. await asyncio.sleep(0.05) # Reconstruct AgentAnalysis-like dict from collected events - from ai.schemas import AgentAnalysis, Classification, Deadline, Consequence, RiskScore as TheirRiskScore - cls_data = next((e.data for e in agent_events_collected if e.type == "classification"), {}) - dl_data = next((e.data for e in agent_events_collected if e.type == "deadline"), {}) - rs_data = next((e.data for e in agent_events_collected if e.type == "risk_score"), {"score": 3, "label": "Medium", "reason": ""}) - cq_data = next((e.data for e in agent_events_collected if e.type == "consequence"), {"text": "", "severity": ""}) + from ai.schemas import ( + AgentAnalysis, + Classification, + Deadline, + Consequence, + RiskScore as TheirRiskScore, + ) + + cls_data = next( + (e.data for e in agent_events_collected if e.type == "classification"), + {}, + ) + dl_data = next( + (e.data for e in agent_events_collected if e.type == "deadline"), {} + ) + rs_data = next( + (e.data for e in agent_events_collected if e.type == "risk_score"), + {"score": 3, "label": "Medium", "reason": ""}, + ) + cq_data = next( + (e.data for e in agent_events_collected if e.type == "consequence"), + {"text": "", "severity": ""}, + ) # Fallback: if the agent didn't extract a deadline, scan the OCR # text with our German-date regex. Common for letters where the @@ -396,19 +443,35 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str # Also emit a deadline SSE event so the frontend sees it live yield sse_event( "deadline", - {"date": agent_date_iso, "days_remaining": days_remaining, - "note": "Found via OCR text scan (agent missed it)"}, + { + "date": agent_date_iso, + "days_remaining": days_remaining, + "note": "Found via OCR text scan (agent missed it)", + }, ) analysis = AgentAnalysis( - classification=Classification(type=cls_data.get("type", "Unknown"), agency=cls_data.get("agency", "Unknown")), + classification=Classification( + type=cls_data.get("type", "Unknown"), + agency=cls_data.get("agency", "Unknown"), + ), deadline=Deadline( date=dl_data.get("date"), days_remaining=dl_data.get("days_remaining"), - source="letter" if (dl_data.get("date") and not fallback_date) else fallback_source if fallback_date else "none", + source="letter" + if (dl_data.get("date") and not fallback_date) + else fallback_source + if fallback_date + else "none", + ), + consequence=Consequence( + text=cq_data.get("text", ""), severity=cq_data.get("severity", "") + ), + risk_score=TheirRiskScore( + score=rs_data.get("score", 3), + label=rs_data.get("label", "Medium"), + reason=rs_data.get("reason", ""), ), - consequence=Consequence(text=cq_data.get("text", ""), severity=cq_data.get("severity", "")), - risk_score=TheirRiskScore(score=rs_data.get("score", 3), label=rs_data.get("label", "Medium"), reason=rs_data.get("reason", "")), ) unpacked = ai_bridge.unpack_agent_analysis(analysis) @@ -453,7 +516,11 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str if not dl_data.get("date"): yield sse_event( "deadline", - {"date": None, "days_remaining": None, "note": "No explicit deadline"}, + { + "date": None, + "days_remaining": None, + "note": "No explicit deadline", + }, ) # ============================================================ @@ -461,6 +528,7 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str # ============================================================ try: from ai.rag.retrieval import retrieve_legal_context + # AI team's new signature (commit 61fd2b5): (letter_type, consequence, top_k) legal_chunks = retrieve_legal_context( letter_type=letter.document_type or "", @@ -468,14 +536,18 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str top_k=5, ) except Exception as e: - logger.warning("Legal retrieval failed: %s — continuing without citations", e) + logger.warning( + "Legal retrieval failed: %s — continuing without citations", e + ) legal_chunks = [] # ============================================================ # STAGE 4 — Grounded generation (~5-10s) # ============================================================ agent_result = ai_bridge.synthesize_agent_result(letter, action=action) - agent_result.risk_label = risk_label # use their qualitative label for grounding context + agent_result.risk_label = ( + risk_label # use their qualitative label for grounding context + ) generation = await ai_bridge.generate_grounded_response( ocr_text=ocr_text, @@ -483,7 +555,9 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str language=out_lang, legal_chunks=legal_chunks, ) - explanation, response_draft, checklist, citations = ai_bridge.unpack_generation_output(generation) + explanation, response_draft, checklist, citations = ( + ai_bridge.unpack_generation_output(generation) + ) # Stream explanation chunks for piece in _chunk_text_for_streaming(explanation, chunk_size=50): @@ -519,6 +593,7 @@ async def process_letter_stream(letter_id: UUID, lang: str) -> AsyncIterator[str # Project to the same PublicLetter shape GET /letters/{id} returns. from app.routers.public import _public_letter + public = _public_letter(db, letter) yield sse_event( diff --git a/backend/app/rag/store.py b/backend/app/rag/store.py index cec1126..195b435 100644 --- a/backend/app/rag/store.py +++ b/backend/app/rag/store.py @@ -36,6 +36,7 @@ def init_chroma() -> None: coll = get_collection() if coll.count() == 0: from app.rag.seed import seed_corpus + seed_corpus(coll) diff --git a/backend/app/routers/actions.py b/backend/app/routers/actions.py index 4bdcb58..c152515 100644 --- a/backend/app/routers/actions.py +++ b/backend/app/routers/actions.py @@ -52,8 +52,15 @@ def list_actions( 422, ErrorCode.VALIDATION_ERROR, message=f"Unknown status: {status!r}.", - details={"errors": [{"field": "status", "message": "must be one of " - + ", ".join(s.value for s in ActionStatus)}]}, + details={ + "errors": [ + { + "field": "status", + "message": "must be one of " + + ", ".join(s.value for s in ActionStatus), + } + ] + }, ) stmt = ( diff --git a/backend/app/routers/deadlines.py b/backend/app/routers/deadlines.py index c4dc27b..914f0dd 100644 --- a/backend/app/routers/deadlines.py +++ b/backend/app/routers/deadlines.py @@ -59,8 +59,15 @@ def list_deadlines( 422, ErrorCode.VALIDATION_ERROR, message=f"Unknown status: {status!r}.", - details={"errors": [{"field": "status", "message": "must be one of " - + ", ".join(s.value for s in ActionStatus)}]}, + details={ + "errors": [ + { + "field": "status", + "message": "must be one of " + + ", ".join(s.value for s in ActionStatus), + } + ] + }, ) stmt = ( select(ActionItem, Letter) diff --git a/backend/app/routers/letters.py b/backend/app/routers/letters.py index 4812bbb..1108884 100644 --- a/backend/app/routers/letters.py +++ b/backend/app/routers/letters.py @@ -17,7 +17,12 @@ User, utcnow, ) -from app.schemas import ErrorResponse, LetterListItem, LetterResponse, LetterUploadResponse +from app.schemas import ( + ErrorResponse, + LetterListItem, + LetterResponse, + LetterUploadResponse, +) from app.services.extraction import ( ExtractionError, extract_from_letter_file, @@ -141,7 +146,8 @@ async def upload_letter( raise KlarHTTPException(415, ErrorCode.LETTER_CORRUPT_FILE) # Allow image/jpeg ↔ image/jpg variants; otherwise demand strict match. if actual_mime != file.content_type and not ( - actual_mime.startswith("image/") and file.content_type.startswith("image/") + actual_mime.startswith("image/") + and file.content_type.startswith("image/") and actual_mime.split("/")[-1] == file.content_type.split("/")[-1] ): raise KlarHTTPException( @@ -267,7 +273,10 @@ async def extract_letter( ), responses={ 401: {"model": ErrorResponse, "description": "Not authenticated."}, - 422: {"model": ErrorResponse, "description": "Unknown status or category value."}, + 422: { + "model": ErrorResponse, + "description": "Unknown status or category value.", + }, }, ) def list_letters( @@ -287,8 +296,15 @@ def list_letters( 422, ErrorCode.VALIDATION_ERROR, message=f"Unknown status: {status!r}.", - details={"errors": [{"field": "status", "message": "must be one of " - + ", ".join(s.value for s in LetterStatus)}]}, + details={ + "errors": [ + { + "field": "status", + "message": "must be one of " + + ", ".join(s.value for s in LetterStatus), + } + ] + }, ) parsed_category: DocumentCategory | None = None if category: @@ -299,8 +315,15 @@ def list_letters( 422, ErrorCode.VALIDATION_ERROR, message=f"Unknown category: {category!r}.", - details={"errors": [{"field": "category", "message": "must be one of " - + ", ".join(c.value for c in DocumentCategory)}]}, + details={ + "errors": [ + { + "field": "category", + "message": "must be one of " + + ", ".join(c.value for c in DocumentCategory), + } + ] + }, ) stmt = select(Letter).where(Letter.user_id == user.id) diff --git a/backend/app/routers/public.py b/backend/app/routers/public.py index 4b03459..08ee4f8 100644 --- a/backend/app/routers/public.py +++ b/backend/app/routers/public.py @@ -22,7 +22,6 @@ """ import logging -from datetime import datetime from uuid import UUID from fastapi import APIRouter, Depends, File, Query, UploadFile @@ -30,8 +29,6 @@ from sqlmodel import Session, select from app.auth.dependencies import get_current_user - -logger = logging.getLogger("klar.public") from app.database import get_session from app.errors import ErrorCode, KlarHTTPException from app.models import ( @@ -54,7 +51,6 @@ PublicLetter, ChatRequest, ChatResponse, - RagHit, RagQuery, RagResponse, ReplyDraft, @@ -65,13 +61,14 @@ from app.services.extraction import ( ExtractionError, extract_from_letter_file, - generate_reply_text, normalize_lang, ) from app.services.pdf_pages import PdfRenderError from app.services.persistence import persist_extraction from app.services.storage import detect_magic_mime, save_letter_file +logger = logging.getLogger("klar.public") + router = APIRouter(tags=["public"]) ACCEPTED_MIMES = { @@ -87,9 +84,7 @@ # ---------- shape projection: Letter + ActionItems → PublicLetter ---------- -def _public_action( - action: ActionItem, latest_risk: RiskScore | None -) -> PublicAction: +def _public_action(action: ActionItem, latest_risk: RiskScore | None) -> PublicAction: risk_breakdown = None if latest_risk is not None: risk_breakdown = RiskBreakdown( @@ -120,9 +115,7 @@ def _public_action( ) -def _load_risk_by_action( - db: Session, action_ids: list[UUID] -) -> dict[UUID, RiskScore]: +def _load_risk_by_action(db: Session, action_ids: list[UUID]) -> dict[UUID, RiskScore]: """Batch-load the most recent RiskScore per action — O(1) queries.""" if not action_ids: return {} @@ -140,9 +133,7 @@ def _load_risk_by_action( def _public_letter(db: Session, letter: Letter) -> PublicLetter: actions = list( - db.scalars( - select(ActionItem).where(ActionItem.letter_id == letter.id) - ).all() + db.scalars(select(ActionItem).where(ActionItem.letter_id == letter.id)).all() ) risk_by_action = _load_risk_by_action(db, [a.id for a in actions]) # Citations are stored as a list[dict] on the Letter row, but the public @@ -169,10 +160,7 @@ def _public_letter(db: Session, letter: Letter) -> PublicLetter: summary_en=letter.summary, # field renamed for frontend contract ocr_text=letter.ocr_text or None, confidence=letter.confidence, - actions=[ - _public_action(a, risk_by_action.get(a.id)) - for a in actions - ], + actions=[_public_action(a, risk_by_action.get(a.id)) for a in actions], extraction_warnings=letter.extraction_warnings or [], explanation=letter.explanation or "", consequence=letter.consequence or "", @@ -283,7 +271,8 @@ async def post_letter( # implementation details to the client. logger.exception( "Qwen extraction failed for letter %s: %s", - letter.id, exc, + letter.id, + exc, ) letter.status = LetterStatus.ERROR db.add(letter) @@ -362,8 +351,15 @@ def list_actions_public( 422, ErrorCode.VALIDATION_ERROR, message=f"Unknown status: {status!r}.", - details={"errors": [{"field": "status", "message": "must be one of " - + ", ".join(s.value for s in ActionStatus)}]}, + details={ + "errors": [ + { + "field": "status", + "message": "must be one of " + + ", ".join(s.value for s in ActionStatus), + } + ] + }, ) stmt = ( @@ -456,7 +452,10 @@ def update_action_public( ), responses={ 401: {"model": ErrorResponse, "description": "Not authenticated."}, - 404: {"model": ErrorResponse, "description": "`LETTER_NOT_FOUND` or `ACTION_NOT_FOUND`."}, + 404: { + "model": ErrorResponse, + "description": "`LETTER_NOT_FOUND` or `ACTION_NOT_FOUND`.", + }, 502: {"model": ErrorResponse, "description": "`LLM_PROVIDER_ERROR`."}, }, ) @@ -473,8 +472,7 @@ async def generate_reply( payload = payload or ReplyRequest() - # If action_id is set, scope to that one action; otherwise include all - # actions on the letter that need a reply. + # If action_id is set, validate it belongs to this letter (404 otherwise). if payload.action_id: try: action_uuid = UUID(payload.action_id) @@ -483,16 +481,6 @@ async def generate_reply( action = db.get(ActionItem, action_uuid) if action is None or action.letter_id != letter.id: raise KlarHTTPException(404, ErrorCode.ACTION_NOT_FOUND) - action_titles = [action.title] - else: - actions = list( - db.scalars( - select(ActionItem).where(ActionItem.letter_id == letter.id) - ).all() - ) - # Prefer actions explicitly flagged reply_needed; fall back to all titles. - reply_actions = [a for a in actions if a.reply_needed] or actions - action_titles = [a.title for a in reply_actions] # 1) Retrieve real legal context from the AI team's law corpus try: @@ -532,12 +520,15 @@ async def generate_reply( except Exception as exc: logger.exception( "Reply generation failed for letter %s: %s", - letter_id, exc, + letter_id, + exc, ) raise KlarHTTPException(502, ErrorCode.LLM_PROVIDER_ERROR) # 4) Unpack + persist all 4 long-form fields - explanation, body_text, checklist, citations = ai_bridge.unpack_generation_output(generation) + explanation, body_text, checklist, citations = ai_bridge.unpack_generation_output( + generation + ) letter.explanation = explanation letter.response_draft = body_text letter.checklist = checklist @@ -604,7 +595,6 @@ async def chat_about_letter( db: Session = Depends(get_session), user: User = Depends(get_current_user), ): - import json import os from uuid import UUID as _UUID @@ -653,19 +643,23 @@ async def chat_about_letter( extra_body={"enable_thinking": False}, ) - response = await model.ainvoke([ - SystemMessage(content=system), - HumanMessage(content=payload.query), - ]) + response = await model.ainvoke( + [ + SystemMessage(content=system), + HumanMessage(content=payload.query), + ] + ) # Coerce raw dicts → CitationItem (same as _public_letter) clean_citations = [] for c in citations: if isinstance(c, dict) and c.get("section"): - clean_citations.append(CitationItem( - section=str(c.get("section", "")), - text=str(c.get("text", "")), - )) + clean_citations.append( + CitationItem( + section=str(c.get("section", "")), + text=str(c.get("text", "")), + ) + ) return ChatResponse(answer=response.content, citations=clean_citations) @@ -704,7 +698,7 @@ async def form_fill( placeholders = [] # From checklist (documents to prepare) - for item in (letter.checklist or []): + for item in letter.checklist or []: placeholders.append(str(item)) # From actions (steps the user needs to take) @@ -712,12 +706,25 @@ async def form_fill( db.scalars(select(ActionItem).where(ActionItem.letter_id == letter.id)).all() ) for action in actions: - for step in (action.steps or []): - if any(kw in step.lower() for kw in [ - "iban", "steuer", "name", "adresse", "address", "nummer", - "number", "unterschrift", "signature", "datum", "date", - "versichertennummer", "aktenzeichen", - ]): + for step in action.steps or []: + if any( + kw in step.lower() + for kw in [ + "iban", + "steuer", + "name", + "adresse", + "address", + "nummer", + "number", + "unterschrift", + "signature", + "datum", + "date", + "versichertennummer", + "aktenzeichen", + ] + ): placeholders.append(step) # Always include standard form fields — these match the PLACEHOLDER_MAP @@ -745,7 +752,9 @@ async def form_fill( placeholders=placeholders[:8], # Cap at 8 to keep prompt manageable ) except Exception as exc: - logger.exception("Form-fill generation failed for letter %s: %s", letter_id, exc) + logger.exception( + "Form-fill generation failed for letter %s: %s", letter_id, exc + ) raise KlarHTTPException(502, ErrorCode.LLM_PROVIDER_ERROR) return Response( diff --git a/backend/app/routers/rag.py b/backend/app/routers/rag.py index 419907a..b0a346b 100644 --- a/backend/app/routers/rag.py +++ b/backend/app/routers/rag.py @@ -5,7 +5,7 @@ from app.auth.dependencies import get_current_user from app.models import User from app.rag import store -from app.schemas import ErrorResponse, RagHit, RagQuery, RagResponse +from app.schemas import ErrorResponse, RagQuery, RagResponse from app.services import ai_bridge router = APIRouter(prefix="/api/rag", tags=["rag"]) diff --git a/backend/app/schemas.py b/backend/app/schemas.py index 04a5558..c52b183 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -14,7 +14,7 @@ """ from datetime import date, datetime -from typing import Any, Literal, Optional +from typing import Literal, Optional from uuid import UUID from pydantic import BaseModel, EmailStr, Field @@ -185,7 +185,9 @@ class PublicAction(BaseModel): description="Full RiskScore breakdown — powers the 'why this risk' view.", ) deadline_confidence: Optional[float] = Field( - default=None, ge=0.0, le=1.0, + default=None, + ge=0.0, + le=1.0, description="0..1 confidence in the deadline value (null when unknown).", ) deadline_source: Optional[DeadlineSource] = Field( @@ -200,7 +202,8 @@ class PublicAction(BaseModel): ) reply_needed: bool = False amount_due_eur: Optional[float] = Field( - default=None, ge=0.0, + default=None, + ge=0.0, description=( "Outstanding amount the user must pay for this action, in EUR. " "Extracted from the OCR text by a regex pattern matcher." @@ -227,7 +230,9 @@ class PublicLetter(BaseModel): description="Verbatim German OCR text from the source. Never localized.", ) confidence: Optional[float] = Field( - default=None, ge=0.0, le=1.0, + default=None, + ge=0.0, + le=1.0, description="0..1 overall extraction confidence. <0.85 triggers a 'get a human' UI prompt.", ) actions: list[PublicAction] = Field(default_factory=list) @@ -283,7 +288,8 @@ class PublicActionListItem(BaseModel): status: ActionStatus reply_needed: bool amount_due_eur: Optional[float] = Field( - default=None, ge=0.0, + default=None, + ge=0.0, description=( "Outstanding EUR amount for this action, mirrored from the same " "field on PublicAction. Included on the list shape so the " @@ -414,7 +420,9 @@ class ErrorResponse(BaseModel): description="Stable machine-readable identifier — see docs/06-api-contract.md." ) message: str = Field(description="Localized, user-facing copy.") - detail: str = Field(description="Alias of `message` — for clients that expect FastAPI's default error shape.") + detail: str = Field( + description="Alias of `message` — for clients that expect FastAPI's default error shape." + ) details: Optional[ErrorDetails] = None diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py index 3818b10..0faa8ea 100644 --- a/backend/app/services/__init__.py +++ b/backend/app/services/__init__.py @@ -9,7 +9,11 @@ stream_explanation, stream_response_draft, ) -from app.services.pdf_pages import iter_data_urls, pdf_to_image_bytes, split_to_image_bytes +from app.services.pdf_pages import ( + iter_data_urls, + pdf_to_image_bytes, + split_to_image_bytes, +) from app.services.persistence import persist_extraction from app.services.risk import compute_risk from app.services.storage import detect_magic_mime, is_pdf, save_letter_file, user_dir diff --git a/backend/app/services/ai_bridge.py b/backend/app/services/ai_bridge.py index 0776499..e1ecd88 100644 --- a/backend/app/services/ai_bridge.py +++ b/backend/app/services/ai_bridge.py @@ -50,102 +50,89 @@ _CATEGORY_PATTERNS: list[tuple[str, DocumentCategory]] = [ # Most specific first - ("residence permit", DocumentCategory.IMMIGRATION), - ("aufenthaltstitel", DocumentCategory.IMMIGRATION), - ("ausländerbehörde", DocumentCategory.IMMIGRATION), - ("visa", DocumentCategory.IMMIGRATION), - ("aufenthalts", DocumentCategory.IMMIGRATION), - ("immigration", DocumentCategory.IMMIGRATION), - - ("health insurance", DocumentCategory.HEALTH_INSURANCE), - ("krankenkasse", DocumentCategory.HEALTH_INSURANCE), - ("krankenversicherung", DocumentCategory.HEALTH_INSURANCE), - ("aok", DocumentCategory.HEALTH_INSURANCE), - ("techniker krankenkasse", DocumentCategory.HEALTH_INSURANCE), - ("barmer", DocumentCategory.HEALTH_INSURANCE), - ("dak-gesundheit", DocumentCategory.HEALTH_INSURANCE), - - ("car insurance", DocumentCategory.OTHER_INSURANCE), - ("haftpflicht", DocumentCategory.OTHER_INSURANCE), - ("hausrat", DocumentCategory.OTHER_INSURANCE), - ("kfz-versicherung", DocumentCategory.OTHER_INSURANCE), - ("liability insurance", DocumentCategory.OTHER_INSURANCE), - - ("tax", DocumentCategory.TAX), - ("finanzamt", DocumentCategory.TAX), - ("steuer", DocumentCategory.TAX), - - ("university", DocumentCategory.EDUCATION), - ("universität", DocumentCategory.EDUCATION), - ("hochschule", DocumentCategory.EDUCATION), - ("immatrikulation", DocumentCategory.EDUCATION), - ("studentenwerk", DocumentCategory.EDUCATION), - ("bafög", DocumentCategory.EDUCATION), - ("rückmeldung", DocumentCategory.EDUCATION), - ("enrollment", DocumentCategory.EDUCATION), - - ("rent", DocumentCategory.HOUSING), - ("vermieter", DocumentCategory.HOUSING), - ("hausverwaltung", DocumentCategory.HOUSING), - ("mieterhöhung", DocumentCategory.HOUSING), - ("nebenkosten", DocumentCategory.HOUSING), - ("landlord", DocumentCategory.HOUSING), - - ("electricity", DocumentCategory.UTILITIES), - ("gas bill", DocumentCategory.UTILITIES), - ("internet", DocumentCategory.UTILITIES), - ("telekom", DocumentCategory.UTILITIES), - ("vodafone", DocumentCategory.UTILITIES), - ("stadtwerke", DocumentCategory.UTILITIES), - ("vattenfall", DocumentCategory.UTILITIES), - ("strom", DocumentCategory.UTILITIES), - - ("employer", DocumentCategory.EMPLOYMENT), - ("arbeitgeber", DocumentCategory.EMPLOYMENT), - ("lohn", DocumentCategory.EMPLOYMENT), - ("gehalt", DocumentCategory.EMPLOYMENT), - ("payroll", DocumentCategory.EMPLOYMENT), - ("arbeitsvertrag", DocumentCategory.EMPLOYMENT), - - ("unemployment", DocumentCategory.GOVERNMENT_BENEFITS), - ("kindergeld", DocumentCategory.GOVERNMENT_BENEFITS), - ("elterngeld", DocumentCategory.GOVERNMENT_BENEFITS), - ("wohngeld", DocumentCategory.GOVERNMENT_BENEFITS), - ("arbeitslosengeld", DocumentCategory.GOVERNMENT_BENEFITS), - ("bürgergeld", DocumentCategory.GOVERNMENT_BENEFITS), - ("jobcenter", DocumentCategory.GOVERNMENT_BENEFITS), - ("familienkasse", DocumentCategory.GOVERNMENT_BENEFITS), - - ("pension", DocumentCategory.PENSION), - ("rentenversicherung", DocumentCategory.PENSION), - ("rente", DocumentCategory.PENSION), - - ("rundfunk", DocumentCategory.BROADCAST_FEE), - ("gez", DocumentCategory.BROADCAST_FEE), - ("beitragsservice", DocumentCategory.BROADCAST_FEE), - ("broadcasting fee", DocumentCategory.BROADCAST_FEE), - - ("bürgeramt", DocumentCategory.CIVIC), - ("einwohnermelde", DocumentCategory.CIVIC), - ("standesamt", DocumentCategory.CIVIC), - ("personalausweis", DocumentCategory.CIVIC), - ("reisepass", DocumentCategory.CIVIC), - ("meldebescheinigung", DocumentCategory.CIVIC), - - ("court", DocumentCategory.LEGAL_DEBT), - ("gericht", DocumentCategory.LEGAL_DEBT), - ("mahnbescheid", DocumentCategory.LEGAL_DEBT), - ("vollstreckung", DocumentCategory.LEGAL_DEBT), - ("inkasso", DocumentCategory.LEGAL_DEBT), - ("bußgeld", DocumentCategory.LEGAL_DEBT), - ("anwalt", DocumentCategory.LEGAL_DEBT), - ("debt collection", DocumentCategory.LEGAL_DEBT), - ("fine notice", DocumentCategory.LEGAL_DEBT), - - ("bank", DocumentCategory.BANKING), - ("sparkasse", DocumentCategory.BANKING), - ("schufa", DocumentCategory.BANKING), - ("kreditkarte", DocumentCategory.BANKING), + ("residence permit", DocumentCategory.IMMIGRATION), + ("aufenthaltstitel", DocumentCategory.IMMIGRATION), + ("ausländerbehörde", DocumentCategory.IMMIGRATION), + ("visa", DocumentCategory.IMMIGRATION), + ("aufenthalts", DocumentCategory.IMMIGRATION), + ("immigration", DocumentCategory.IMMIGRATION), + ("health insurance", DocumentCategory.HEALTH_INSURANCE), + ("krankenkasse", DocumentCategory.HEALTH_INSURANCE), + ("krankenversicherung", DocumentCategory.HEALTH_INSURANCE), + ("aok", DocumentCategory.HEALTH_INSURANCE), + ("techniker krankenkasse", DocumentCategory.HEALTH_INSURANCE), + ("barmer", DocumentCategory.HEALTH_INSURANCE), + ("dak-gesundheit", DocumentCategory.HEALTH_INSURANCE), + ("car insurance", DocumentCategory.OTHER_INSURANCE), + ("haftpflicht", DocumentCategory.OTHER_INSURANCE), + ("hausrat", DocumentCategory.OTHER_INSURANCE), + ("kfz-versicherung", DocumentCategory.OTHER_INSURANCE), + ("liability insurance", DocumentCategory.OTHER_INSURANCE), + ("tax", DocumentCategory.TAX), + ("finanzamt", DocumentCategory.TAX), + ("steuer", DocumentCategory.TAX), + ("university", DocumentCategory.EDUCATION), + ("universität", DocumentCategory.EDUCATION), + ("hochschule", DocumentCategory.EDUCATION), + ("immatrikulation", DocumentCategory.EDUCATION), + ("studentenwerk", DocumentCategory.EDUCATION), + ("bafög", DocumentCategory.EDUCATION), + ("rückmeldung", DocumentCategory.EDUCATION), + ("enrollment", DocumentCategory.EDUCATION), + ("rent", DocumentCategory.HOUSING), + ("vermieter", DocumentCategory.HOUSING), + ("hausverwaltung", DocumentCategory.HOUSING), + ("mieterhöhung", DocumentCategory.HOUSING), + ("nebenkosten", DocumentCategory.HOUSING), + ("landlord", DocumentCategory.HOUSING), + ("electricity", DocumentCategory.UTILITIES), + ("gas bill", DocumentCategory.UTILITIES), + ("internet", DocumentCategory.UTILITIES), + ("telekom", DocumentCategory.UTILITIES), + ("vodafone", DocumentCategory.UTILITIES), + ("stadtwerke", DocumentCategory.UTILITIES), + ("vattenfall", DocumentCategory.UTILITIES), + ("strom", DocumentCategory.UTILITIES), + ("employer", DocumentCategory.EMPLOYMENT), + ("arbeitgeber", DocumentCategory.EMPLOYMENT), + ("lohn", DocumentCategory.EMPLOYMENT), + ("gehalt", DocumentCategory.EMPLOYMENT), + ("payroll", DocumentCategory.EMPLOYMENT), + ("arbeitsvertrag", DocumentCategory.EMPLOYMENT), + ("unemployment", DocumentCategory.GOVERNMENT_BENEFITS), + ("kindergeld", DocumentCategory.GOVERNMENT_BENEFITS), + ("elterngeld", DocumentCategory.GOVERNMENT_BENEFITS), + ("wohngeld", DocumentCategory.GOVERNMENT_BENEFITS), + ("arbeitslosengeld", DocumentCategory.GOVERNMENT_BENEFITS), + ("bürgergeld", DocumentCategory.GOVERNMENT_BENEFITS), + ("jobcenter", DocumentCategory.GOVERNMENT_BENEFITS), + ("familienkasse", DocumentCategory.GOVERNMENT_BENEFITS), + ("pension", DocumentCategory.PENSION), + ("rentenversicherung", DocumentCategory.PENSION), + ("rente", DocumentCategory.PENSION), + ("rundfunk", DocumentCategory.BROADCAST_FEE), + ("gez", DocumentCategory.BROADCAST_FEE), + ("beitragsservice", DocumentCategory.BROADCAST_FEE), + ("broadcasting fee", DocumentCategory.BROADCAST_FEE), + ("bürgeramt", DocumentCategory.CIVIC), + ("einwohnermelde", DocumentCategory.CIVIC), + ("standesamt", DocumentCategory.CIVIC), + ("personalausweis", DocumentCategory.CIVIC), + ("reisepass", DocumentCategory.CIVIC), + ("meldebescheinigung", DocumentCategory.CIVIC), + ("court", DocumentCategory.LEGAL_DEBT), + ("gericht", DocumentCategory.LEGAL_DEBT), + ("mahnbescheid", DocumentCategory.LEGAL_DEBT), + ("vollstreckung", DocumentCategory.LEGAL_DEBT), + ("inkasso", DocumentCategory.LEGAL_DEBT), + ("bußgeld", DocumentCategory.LEGAL_DEBT), + ("anwalt", DocumentCategory.LEGAL_DEBT), + ("debt collection", DocumentCategory.LEGAL_DEBT), + ("fine notice", DocumentCategory.LEGAL_DEBT), + ("bank", DocumentCategory.BANKING), + ("sparkasse", DocumentCategory.BANKING), + ("schufa", DocumentCategory.BANKING), + ("kreditkarte", DocumentCategory.BANKING), ] @@ -160,7 +147,9 @@ def map_classification_to_category(free_text_type: str | None) -> DocumentCatego for pattern, cat in _CATEGORY_PATTERNS: if pattern in needle: return cat - logger.debug("map_classification_to_category: no match for %r → OTHER", free_text_type) + logger.debug( + "map_classification_to_category: no match for %r → OTHER", free_text_type + ) return DocumentCategory.OTHER @@ -169,10 +158,10 @@ def map_classification_to_category(free_text_type: str | None) -> DocumentCatego # ============================================================ _LABEL_TO_SEVERITY: dict[str, Severity] = { - "critical": Severity.CRITICAL, - "high": Severity.HIGH, - "medium": Severity.MEDIUM, - "low": Severity.LOW, + "critical": Severity.CRITICAL, + "high": Severity.HIGH, + "medium": Severity.MEDIUM, + "low": Severity.LOW, "informational": Severity.LOW, } @@ -194,10 +183,10 @@ def map_their_severity_label(label: str | None) -> Severity: # ============================================================ _SOURCE_MAPPING: dict[str, DeadlineSource] = { - "letter": DeadlineSource.EXPLICIT, + "letter": DeadlineSource.EXPLICIT, "calculated": DeadlineSource.INFERRED, - "searched": DeadlineSource.INFERRED, # Tavily web search - "none": DeadlineSource.UNKNOWN, + "searched": DeadlineSource.INFERRED, # Tavily web search + "none": DeadlineSource.UNKNOWN, } @@ -218,6 +207,7 @@ def deadline_was_web_searched(source: str | None) -> bool: # Our risk_score (0-100) → their RiskScore.label (for generator input) # ============================================================ + def risk_label_from_score(score: int | None) -> str: """Map our 0-100 score → their label string (used when synthesizing an `AgentResult` to feed their `generate_response`).""" @@ -237,6 +227,7 @@ def risk_label_from_score(score: int | None) -> str: # Letter + (optional) ActionItem → AgentResult (their dataclass) # ============================================================ + def synthesize_agent_result( letter: Letter, action: ActionItem | None = None, @@ -274,6 +265,7 @@ def synthesize_agent_result( # Their LegalChunk → our RagHit (for /rag/search response) # ============================================================ + def legal_chunk_to_rag_hit(chunk: "LegalChunk") -> RagHit: """Map their `LegalChunk` from `ai.rag.retrieval` → our `RagHit` wire shape. @@ -285,9 +277,9 @@ def legal_chunk_to_rag_hit(chunk: "LegalChunk") -> RagHit: text=chunk.text, score=1.0, metadata={ - "section": chunk.section, - "law": chunk.law, - "title": chunk.title, + "section": chunk.section, + "law": chunk.law, + "title": chunk.title, "citation": chunk.citation, }, ) @@ -297,6 +289,7 @@ def legal_chunk_to_rag_hit(chunk: "LegalChunk") -> RagHit: # Their Citation → JSON dict (stored on Letter.citations column) # ============================================================ + def citation_to_dict(c: "Citation") -> dict[str, Any]: """Persist-shape for the `Letter.citations` JSON column. @@ -305,8 +298,8 @@ def citation_to_dict(c: "Citation") -> dict[str, Any]: """ return { "section": c.section, - "text": c.text, - "score": 1.0, # their structured citation doesn't carry a score + "text": c.text, + "score": 1.0, # their structured citation doesn't carry a score } @@ -318,6 +311,7 @@ def citations_to_dicts(cits: list["Citation"]) -> list[dict[str, Any]]: # Their GenerationOutput → (explanation, response_draft, checklist[], citations[dict]) # ============================================================ + def unpack_generation_output( out: "GenerationOutput", ) -> tuple[str, str, list[str], list[dict[str, Any]]]: @@ -336,6 +330,7 @@ def unpack_generation_output( # Their AgentAnalysis → (category, document_type, severity, deadline_date, ...) # ============================================================ + async def generate_grounded_response( ocr_text: str, agent_result: "AgentResult", @@ -369,8 +364,7 @@ async def generate_grounded_response( # Build the legal-context section from retrieved chunks if legal_chunks: legal_lines = [ - f"### {c.citation} — {c.title}\n{c.text}\n" - for c in legal_chunks + f"### {c.citation} — {c.title}\n{c.text}\n" for c in legal_chunks ] legal_context = "\n".join(legal_lines) else: @@ -403,13 +397,18 @@ async def generate_grounded_response( ), temperature=0, max_tokens=4096, - extra_body={"enable_thinking": False, "response_format": {"type": "json_object"}}, + extra_body={ + "enable_thinking": False, + "response_format": {"type": "json_object"}, + }, ) response = await raw_model.ainvoke([HumanMessage(content=prompt)]) raw_text = response.content if hasattr(response, "content") else str(response) if isinstance(raw_text, list): # Some langchain versions return content as a list of parts - raw_text = "".join(p.get("text", "") if isinstance(p, dict) else str(p) for p in raw_text) + raw_text = "".join( + p.get("text", "") if isinstance(p, dict) else str(p) for p in raw_text + ) payload = json.loads(raw_text) @@ -421,10 +420,12 @@ async def generate_grounded_response( # Bare "§ 81 AufenthG" — wrap as Citation with empty explanation cleaned_citations.append(Citation(section=c, text="")) elif isinstance(c, dict): - cleaned_citations.append(Citation( - section=str(c.get("section") or c.get("§") or "§"), - text=str(c.get("text") or c.get("explanation") or ""), - )) + cleaned_citations.append( + Citation( + section=str(c.get("section") or c.get("§") or "§"), + text=str(c.get("text") or c.get("explanation") or ""), + ) + ) else: logger.debug("Skipping unparseable citation: %r", c) @@ -454,17 +455,19 @@ def unpack_agent_analysis( try: parsed_deadline = _date.fromisoformat(deadline_iso) except ValueError: - logger.debug("Their agent returned non-ISO deadline %r — dropping", deadline_iso) + logger.debug( + "Their agent returned non-ISO deadline %r — dropping", deadline_iso + ) return { - "category": map_classification_to_category(analysis.classification.type), - "document_type": analysis.classification.type or "", - "institution": analysis.classification.agency or "", - "deadline": parsed_deadline, - "deadline_source": map_their_deadline_source(analysis.deadline.source), + "category": map_classification_to_category(analysis.classification.type), + "document_type": analysis.classification.type or "", + "institution": analysis.classification.agency or "", + "deadline": parsed_deadline, + "deadline_source": map_their_deadline_source(analysis.deadline.source), "deadline_was_searched": deadline_was_web_searched(analysis.deadline.source), - "consequence": analysis.consequence.text or "", - "severity": map_their_severity_label(analysis.risk_score.label), - "risk_label": analysis.risk_score.label or "Medium", - "risk_reason": analysis.risk_score.reason or "", + "consequence": analysis.consequence.text or "", + "severity": map_their_severity_label(analysis.risk_score.label), + "risk_label": analysis.risk_score.label or "Medium", + "risk_reason": analysis.risk_score.reason or "", } diff --git a/backend/app/services/extraction.py b/backend/app/services/extraction.py index 7d13fff..faec74a 100644 --- a/backend/app/services/extraction.py +++ b/backend/app/services/extraction.py @@ -37,6 +37,7 @@ class ExtractionError(Exception): `EXTRACTION_FAILED` error instead of letting a raw exception become a 500. """ + # ISO 639-1 codes — matches the frontend's docs/06-frontend-integration-contract.md. # Qwen3.7-Plus handles all of these out of the box. Quality bar: # en/de: production-grade, the wedge languages @@ -324,7 +325,9 @@ async def extract_from_letter_file( json_parsed = json.loads(stripped) except json.JSONDecodeError: # Sometimes the model wraps the JSON in code fences - fence_match = re.search(r"```(?:json)?\s*(.*?)\s*```", stripped, re.DOTALL) + fence_match = re.search( + r"```(?:json)?\s*(.*?)\s*```", stripped, re.DOTALL + ) if fence_match: try: json_parsed = json.loads(fence_match.group(1).strip()) @@ -416,8 +419,12 @@ async def generate_reply_text( Used by POST /letters/{id}/reply (frontend contract §4.7). """ - actions_text = "\n".join(f"- {t}" for t in action_titles) or "- (keine spezifische Aktion)" - prompt = _response_prompt_from_letter(institution, document_type, actions_text, applicant) + actions_text = ( + "\n".join(f"- {t}" for t in action_titles) or "- (keine spezifische Aktion)" + ) + prompt = _response_prompt_from_letter( + institution, document_type, actions_text, applicant + ) client = _get_client() response = await client.chat.completions.create( @@ -476,7 +483,9 @@ async def _stream_text(prompt: str) -> AsyncIterator[str]: yield delta.content -async def stream_explanation(extracted: ExtractedLetter, lang: str) -> AsyncIterator[str]: +async def stream_explanation( + extracted: ExtractedLetter, lang: str +) -> AsyncIterator[str]: async for piece in _stream_text(_explanation_prompt(extracted, lang)): yield piece @@ -517,7 +526,9 @@ async def generate_checklist(extracted: ExtractedLetter, lang: str) -> list[str] # ---------- backwards-compat alias for the original entrypoint ---------- -async def extract_from_image(image_bytes: bytes, mime: str = "image/jpeg") -> ExtractedLetter: +async def extract_from_image( + image_bytes: bytes, mime: str = "image/jpeg" +) -> ExtractedLetter: """Legacy entrypoint: write bytes to a temp file then call the new API.""" import tempfile import os diff --git a/backend/app/services/pdf_pages.py b/backend/app/services/pdf_pages.py index 8f4ae61..fd0b810 100644 --- a/backend/app/services/pdf_pages.py +++ b/backend/app/services/pdf_pages.py @@ -20,7 +20,9 @@ class PdfRenderError(Exception): """ -def pdf_to_image_bytes(path: str, *, dpi: int = 200, max_pages: int = 12) -> list[bytes]: +def pdf_to_image_bytes( + path: str, *, dpi: int = 200, max_pages: int = 12 +) -> list[bytes]: """Render up to `max_pages` pages of `path` to PNG bytes. Imported lazily so callers that never touch a PDF don't pay the diff --git a/backend/app/services/persistence.py b/backend/app/services/persistence.py index eb28f81..049833b 100644 --- a/backend/app/services/persistence.py +++ b/backend/app/services/persistence.py @@ -34,7 +34,8 @@ def persist_extraction( # Overall confidence = min of available signals. Frontend uses <0.85 to # surface a "get a human" prompt. signals = [ - s for s in (extracted.language_confidence, extracted.category_confidence) + s + for s in (extracted.language_confidence, extracted.category_confidence) if s and s > 0 ] letter.confidence = min(signals) if signals else None @@ -84,6 +85,7 @@ def persist_extraction( # so totals across letters don't double-count. if letter_amount is not None and saved: from app.models import Severity as _Sev + sev_rank = {_Sev.CRITICAL: 4, _Sev.HIGH: 3, _Sev.MEDIUM: 2, _Sev.LOW: 1} amount_attached_to = max(saved, key=lambda x: sev_rank.get(x.severity, 0)) amount_attached_to.amount_due_eur = letter_amount From 534c4b5d64311a7d7c278ff76bea6be228faab64 Mon Sep 17 00:00:00 2001 From: aircode610 Date: Thu, 2 Jul 2026 17:44:33 +0200 Subject: [PATCH 4/4] ci: auto-label and auto-assign issues via triage workflow Add a GitHub Actions workflow (actions/github-script) that runs on issue opened/edited/reopened and, based on issue content: - Adds a type label (security > bug > documentation > enhancement > question) - Adds component labels (frontend, backend, ai-agent, ai-rag, infra) - Adds a priority label (keyword-driven, with sensible defaults) - Auto-assigns the component owner, never overriding a manual assignee Only adds labels that exist in the repo and are not already set. --- .github/workflows/issue-triage.yml | 188 +++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 .github/workflows/issue-triage.yml diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml new file mode 100644 index 0000000..e75ab2c --- /dev/null +++ b/.github/workflows/issue-triage.yml @@ -0,0 +1,188 @@ +name: Issue Triage + +# Auto-label and auto-assign new issues based on their content. +# Runs whenever an issue is opened, edited, or reopened. + +on: + issues: + types: [opened, edited, reopened] + +permissions: + issues: write + contents: read + +jobs: + triage: + runs-on: ubuntu-latest + steps: + - name: Label and assign based on content + uses: actions/github-script@v7 + with: + script: | + const issue = context.payload.issue; + if (!issue || issue.pull_request) { + core.info('Not an issue (or is a PR); skipping.'); + return; + } + + const text = `${issue.title || ''}\n${issue.body || ''}`.toLowerCase(); + const match = (patterns) => patterns.some((p) => p.test(text)); + + // ---- Rule definitions ------------------------------------------- + // Each label maps to a list of regexes. Word boundaries keep + // matches precise (e.g. "ci" won't match "specific"). + + const TYPE_RULES = { + 'security': [ + /\bsecurity\b/, /\bvulnerabilit/, /\bcve[- ]?\d/, /\bexploit/, + /\binjection\b/, /\bxss\b/, /\bcsrf\b/, /\bauth(entication)? bypass\b/, + /\brce\b/, /\bmalicious\b/, + ], + 'bug': [ + /\bbug\b/, /\berror\b/, /\bcrash/, /\bexception\b/, /\btraceback\b/, + /\bstack ?trace\b/, /\b5\d{2}\b/, /\b4\d{2}\b/, /\bfail(s|ed|ing)?\b/, + /\bbroken\b/, /\bnot working\b/, /\bdoesn'?t work\b/, /\bregression\b/, + /\bunexpected\b/, + ], + 'documentation': [ + /\bdocs?\b/, /\bdocumentation\b/, /\breadme\b/, /\btypo\b/, + /\bclarif/, /\bcomment(s)? (are|is) (missing|wrong)\b/, + ], + 'enhancement': [ + /\bfeature\b/, /\benhancement\b/, /\bimprove/, /\bwould be (nice|great)\b/, + /\bplease add\b/, /\bsupport for\b/, /\bproposal\b/, /\brequest\b/, + /\bnew (feature|endpoint|page|tool)\b/, + ], + 'question': [ + /\bquestion\b/, /\bhow (do|can|to)\b/, /\bwhat is\b/, /\bwhy does\b/, + /\bis it possible\b/, + ], + }; + + const COMPONENT_RULES = { + 'component: frontend': [ + /\bfront[- ]?end\b/, /\bnext\.?js\b/, /\breact\b(?!\s*[- ]?agent)/, /\bui\b/, /\bux\b/, + /\bpwa\b/, /\bservice worker\b/, /\bcss\b/, /\btailwind\b/, /\bvercel\b/, + /\beventsource\b/, /\bcomponent(s)?\b/, /\bpage(s)?\b/, /\bbutton\b/, + ], + 'component: backend': [ + /\bback[- ]?end\b/, /\bfastapi\b/, /\bapi\b/, /\bendpoint\b/, /\/api\//, + /\bsqlite\b/, /\bjwt\b/, /\bauth\b/, /\blogin\b/, /\bsignup\b/, + /\bupload\b/, /\bsse\b/, /\bstreaming\b/, /\bdatabase\b/, /\bletters?\b/, + /\bdeadline(s)?\b/, + ], + 'component: ai-agent': [ + /\bocr\b/, /\bqwen-vl\b/, /\breact[- ]?agent\b/, /\breact_agent\b/, + /\blanggraph\b/, /\bclassification\b/, /\btavily\b/, /\bextract_text\b/, + /\bagent\b/, /\brisk[- ]?score\b/, + ], + 'component: ai-rag': [ + /\brag\b/, /\bchroma\s?db\b/, /\bchroma\b/, /\bembedding(s)?\b/, + /\bretrieval\b/, /\blegal\b/, /\bcitation(s)?\b/, /\bcorpus\b/, + /\bingestion\b/, /\bvector\b/, + ], + 'component: infra': [ + /\bci\/?cd\b/, /\bci\b/, /\bcd\b/, /\bdeploy/, /\bdocker/, /\bworkflow\b/, + /\bgithub action/, /\brailway\b/, /\brender\b/, /\blint\b/, + /\brequirements(\.txt)?\b/, /\benv(ironment)? var/, + ], + }; + + const PRIORITY_RULES = { + 'priority: critical': [ + /\bcritical\b/, /\bsystem down\b/, /\bdata loss\b/, /\boutage\b/, + /\bproduction (is )?down\b/, /\bp0\b/, + ], + 'priority: high': [ + /\bhigh priority\b/, /\bblocker\b/, /\bblocks\b/, /\burgent\b/, + /\basap\b/, /\bp1\b/, + ], + 'priority: low': [ + /\blow priority\b/, /\bnice to have\b/, /\bminor\b/, /\bcosmetic\b/, + /\bwhenever\b/, /\bp3\b/, + ], + }; + + // ---- Compute labels --------------------------------------------- + const toAdd = new Set(); + + // Type: pick the strongest single type (security > bug > others). + const typeOrder = ['security', 'bug', 'documentation', 'enhancement', 'question']; + for (const label of typeOrder) { + if (match(TYPE_RULES[label])) { toAdd.add(label); break; } + } + + // Components: an issue can touch multiple areas. + for (const [label, patterns] of Object.entries(COMPONENT_RULES)) { + if (match(patterns)) toAdd.add(label); + } + + // Priority: pick highest matched; else derive a sensible default. + let priority = null; + for (const label of ['priority: critical', 'priority: high', 'priority: medium', 'priority: low']) { + if (PRIORITY_RULES[label] && match(PRIORITY_RULES[label])) { priority = label; break; } + } + if (!priority) { + // Defaults: security/bugs matter more than enhancements/questions. + if (toAdd.has('security')) priority = 'priority: high'; + else if (toAdd.has('bug')) priority = 'priority: medium'; + else priority = 'priority: low'; + } + toAdd.add(priority); + + // Only add labels that actually exist in the repo and aren't set. + const existingRepoLabels = new Set( + (await github.paginate(github.rest.issues.listLabelsForRepo, { + owner: context.repo.owner, repo: context.repo.repo, per_page: 100, + })).map((l) => l.name) + ); + const current = new Set((issue.labels || []).map((l) => (l.name || l))); + const labels = [...toAdd].filter((l) => existingRepoLabels.has(l) && !current.has(l)); + + if (labels.length) { + await github.rest.issues.addLabels({ + owner: context.repo.owner, repo: context.repo.repo, + issue_number: issue.number, labels, + }); + core.info(`Added labels: ${labels.join(', ')}`); + } else { + core.info('No new labels to add.'); + } + + // ---- Auto-assign owner ------------------------------------------ + // Assign the component owner, but never override a manual assignee. + const OWNERS = { + 'component: frontend': 'saintnuno', + 'component: backend': 'saintnuno', + 'component: ai-agent': 'aircode610', + 'component: ai-rag': 'Alir3zag', + 'component: infra': 'aircode610', + 'security': 'aircode610', + }; + // Preference order when several components match. + const ownerOrder = [ + 'security', 'component: ai-agent', 'component: ai-rag', + 'component: backend', 'component: frontend', 'component: infra', + ]; + + const alreadyAssigned = (issue.assignees || []).length > 0; + if (!alreadyAssigned) { + const finalLabels = new Set([...current, ...labels]); + let assignee = null; + for (const key of ownerOrder) { + if (finalLabels.has(key)) { assignee = OWNERS[key]; break; } + } + if (!assignee) assignee = 'aircode610'; // triage fallback (repo owner) + + try { + await github.rest.issues.addAssignees({ + owner: context.repo.owner, repo: context.repo.repo, + issue_number: issue.number, assignees: [assignee], + }); + core.info(`Assigned to: ${assignee}`); + } catch (e) { + core.warning(`Could not assign ${assignee}: ${e.message}`); + } + } else { + core.info('Issue already has an assignee; leaving as-is.'); + }