diff --git a/.env.example b/.env.example index 18f5d31..7a05702 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,23 @@ +# Copy to .env at the repo root. Both services load it automatically. +# .env is gitignored — never commit real keys. + +# Gemini API key for the backend. Get one at https://aistudio.google.com/apikey +# Not needed if you run the backend with MOCK_LLM=true. GEMINI_API_KEY= + +# Run the backend without Gemini: every LLM call is answered from canned, +# rule-based responses in backend/mock_llm.py. No network, no key, no quota. +# Set to true to try the project without an API key. +MOCK_LLM=false + +# Where the backend subscribes to the vision service's scene stream. VISION_WS_URL=ws://localhost:8000/ws + +# Seconds to wait before reconnecting when that websocket drops. VISION_RECONNECT_DELAY=2.0 -LLM_HTTP_HOST=127.0.0.1 -LLM_HTTP_PORT=8001 \ No newline at end of file + +# Interface the vision service binds to. Defaults to localhost because none of +# its endpoints are authenticated and GET /frame serves live camera stills. +# Only change this on a network you trust. See PRIVACY.md. +VISION_HOST=127.0.0.1 +VISION_PORT=8000 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..6c701f5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: CI + +on: + push: + branches: [master, main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + + # Only the backend is installed. vision_service pulls in InsightFace, + # ultralytics, and OpenCV — hundreds of MB of wheels plus model downloads — + # and its endpoints need a physical camera, so it is not exercised here. + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r backend/requirements.txt -r backend/requirements-dev.txt + + - name: Lint + run: ruff check . + + # No GEMINI_API_KEY is set: the suite runs against backend/mock_llm.py and + # must stay free of network calls. + - name: Test + run: pytest backend/tests -q diff --git a/.gitignore b/.gitignore index 985c091..66cc5cd 100644 --- a/.gitignore +++ b/.gitignore @@ -21,22 +21,28 @@ env/ .env local_settings.py -# IDE +# IDE / agent tooling .vscode/ .idea/ +.claude/ -# Testing/Coverage +# Testing / linting caches .pytest_cache/ +.ruff_cache/ .coverage htmlcov/ -.tox/ +.tox/ # OS .DS_Store +**/.DS_Store # Face embeddings — biometric data, never commit vision_service/storage/data/embeddings.json +# Offline chatbot's local store — caregiver-entered personal data + Chroma index +**/memory_data/ + # Downloaded model weights — large files, re-downloaded on first run *.pt *.onnx diff --git a/DISCLAIMER.md b/DISCLAIMER.md new file mode 100644 index 0000000..90affd3 --- /dev/null +++ b/DISCLAIMER.md @@ -0,0 +1,64 @@ +# Disclaimer + +**This is a research and demonstration project. It is not a medical device, and it must not be +used to make care decisions.** + +Please read this before running the project, showing it to anyone, or reusing any part of it. + +## Not a medical device + +This software has not been evaluated, cleared, or approved by the FDA, the EMA, the MHRA, or any +other regulatory body. It is not certified under any medical-device framework. It was built as a +hackathon prototype to explore whether computer vision and language models could be combined into +a memory-support interface — nothing more. + +## Not clinically validated + +No part of this system has been tested in a clinical setting, reviewed by clinicians, or measured +against any standard of care. The prompts, the retrieval logic, and the behavioral categories were +written by developers, not by medical professionals. + +In particular, the `POST /api/caregiver/analyze` endpoint returns a field named +`clinical_rationale`. **That name describes the shape of the data, not its authority.** It is +language-model output. It is not a clinical assessment, and no clinician reviewed it. + +## The model can be wrong + +The assistant is built on a large language model. Like all such models, it can: + +- state things that are confidently wrong, +- invent details that were never in the patient profile, +- misread a scene, or misidentify a person or object, +- produce guidance that is inappropriate for a specific person's condition. + +Face recognition additionally produces false matches and false rejections. A card naming the wrong +person is an expected failure mode of this system, not an edge case. + +## Do not use this for + +- Diagnosis, screening, triage, or staging of dementia or any other condition. +- Treatment, medication, or dosage decisions of any kind. +- Unsupervised care, monitoring, or companionship for a person with dementia. +- Any situation where a wrong or missing answer could affect someone's safety. +- Storing or processing real patient records, protected health information (PHI), or any data + subject to HIPAA, GDPR, or comparable regimes. + +## Always defer to professionals + +Nothing this software outputs is a substitute for a qualified healthcare professional. If you are +caring for someone with dementia, decisions about their care belong with their doctor and care +team. + +**In an emergency, contact your local emergency services. Do not consult this software.** + +## Sample data + +`patient_profile.json` describes a fictional patient ("Arthur") invented for demonstration. It is +not a real person's record, and the repository contains no real patient data. + +## Liability + +This project is provided under the MIT License, which includes no warranty of any kind. See +[LICENSE](LICENSE). The authors accept no liability for any use of this software. + +See also [PRIVACY.md](PRIVACY.md) for how the system handles biometric data. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c4f0128 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Sharif + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/PRIVACY.md b/PRIVACY.md new file mode 100644 index 0000000..75d09a6 --- /dev/null +++ b/PRIVACY.md @@ -0,0 +1,87 @@ +# Privacy and biometric data + +This project processes **face biometrics** and a **live camera feed**. That carries real legal and +ethical weight, so this document states plainly what is captured, where it goes, and how to delete +it. + +Read this alongside [DISCLAIMER.md](DISCLAIMER.md). + +## What the system captures + +| Data | Where it comes from | Where it is stored | +|---|---|---| +| **Face embeddings** — a 512-float vector per registered face | `POST /register` and `POST /register/capture` in `vision_service` | `vision_service/storage/data/embeddings.json`, on your machine | +| **Name, relationship, note** for each registered person | The registration form | Same file | +| **Live camera frames** | Your webcam, while `vision_service` runs | In memory only. Served on request via `GET /frame`; never written to disk | +| **Patient profile** — name, condition, family, preferences | `patient_profile.json`, edited by hand | That file, on your machine | +| **Scene descriptions and caretaker questions** | The running pipeline | Sent to the Gemini API (see below); held in memory, never written to disk | + +Face embeddings are **biometric identifiers**. Under the GDPR they are special-category data +(Art. 9), and jurisdictions such as Illinois (BIPA) and Texas (CUBI) impose specific consent and +retention duties on anyone who collects them. Treat the embeddings file accordingly. + +## What leaves your machine + +**`vision_service` is fully offline.** Face recognition and object detection run locally. No image, +frame, or embedding is ever transmitted anywhere. Model weights are downloaded once on first run. + +**`backend` calls the Google Gemini API.** What is sent: the text description of the current scene +(names, relationships, object labels), retrieved lines from `patient_profile.json`, and caretaker +questions. **No images are ever sent** — only text. Your prompts are subject to +[Google's Gemini API terms](https://ai.google.dev/gemini-api/terms). If you do not want any data +leaving your machine, run with `MOCK_LLM=true`, which makes no network calls at all. + +**`offline-chatbot/` is fully offline.** It uses a local Ollama model and a local Chroma store. + +## Consent + +**Register only people who have knowingly agreed to it.** Enrolling someone's face without their +informed consent is unlawful in many jurisdictions, regardless of intent. + +Where the person cannot meaningfully consent — which includes many people living with advanced +dementia — consent must come from whoever holds legal authority for their care decisions, in line +with local law. + +## Deleting biometric data + +Remove one person: + +```bash +curl -X DELETE http://localhost:8000/people/NAME +``` + +Remove everyone, permanently: + +```bash +rm vision_service/storage/data/embeddings.json +``` + +The file is recreated empty on the next registration. There is no backup and no recovery — that is +deliberate. + +## Retention + +The system applies **no automatic retention limit**. Embeddings persist until you delete them. If +you deploy this anywhere real, set and enforce a retention policy; the code will not do it for you. + +## Committing data by accident + +`.gitignore` excludes `vision_service/storage/data/embeddings.json`, `**/memory_data/`, and `.env`. +Verify before pushing: + +```bash +git ls-files | grep -E 'embeddings\.json|memory_data|\.env$' # must print nothing +``` + +## Known limitations + +This is a prototype, and its security posture reflects that: + +- **No authentication on any endpoint.** Anyone who can reach the port can register a face, delete + a person, or pull a live camera still from `GET /frame`. +- **CORS is fully open** (`allow_origins=["*"]`) on both services. +- **No encryption at rest.** `embeddings.json` is plain, readable JSON. +- **No audit log.** Nothing records who registered or deleted whom. + +Both services therefore default to binding `127.0.0.1`. Do not expose either to a network, and do +not deploy this as-is. diff --git a/README.md b/README.md index 32ef2bc..ca8f0f9 100644 --- a/README.md +++ b/README.md @@ -1,137 +1,145 @@ -# Memento Mori — A Dementia Companion +# Dementia Memory Assistant -An ambient, camera-based assistant that helps people with dementia recognize -familiar faces and objects, hear a gentle spoken reminder of who's in the -room, and lets caregivers ask for real-time, practical guidance — all without -the patient having to operate anything. +A prototype memory-support system for people living with dementia. A camera +recognises familiar faces and everyday objects; a language model turns what it +sees into a short, calm memory card — *"Sarah (Daughter). She visits every +weekend."* — while a caregiver can ask questions about the same scene and get +practical, grounded suggestions. + +> [!WARNING] +> **This is a research and demonstration project. It is not a medical device and +> must not be used to make care decisions.** It processes face biometrics and is +> not clinically validated. Read [DISCLAIMER.md](DISCLAIMER.md) and +> [PRIVACY.md](PRIVACY.md) before running it. --- -## Why this exists +## How it fits together + +``` + webcam + │ + ▼ +┌───────────────────┐ WS /ws, 1 Hz ┌───────────────────┐ +│ vision_service │ ────────────────► │ backend │ +│ port 8000 │ │ port 8001 │ +│ │ │ │ +│ InsightFace + │ │ dedup → retrieve │ +│ YOLOv8n, offline │ │ → Gemini → card │ +└───────────────────┘ └───────────────────┘ + ▲ ▲ ▲ + │ GET /frame, POST /register │ GET /latest│ POST /ask + │ │ │ + │ ┌────────┴────────────┴───┐ + └───────────────────────│ frontend │ + │ static HTML │ + └─────────────────────────┘ +``` + +- **`vision_service/`** (port 8000) — face recognition and object detection. + Runs entirely locally; no image ever leaves the machine. +- **`backend/`** (port 8001) — subscribes to the vision stream, retrieves + relevant facts from `patient_profile.json`, and calls Gemini to produce memory + cards and caregiver answers. Only text is sent to the API. +- **`frontend/`** — a static page, no build step. Open it directly. +- **`offline-chatbot/`** — a separate, fully offline Streamlit prototype + (Ollama + Chroma). Not connected to the pipeline above. -People with moderate-to-advanced dementia frequently lose the ability to -recognize even their closest family members, and forget routine things like -whether they've taken their medication. That moment of not-recognizing is -frightening for the patient and exhausting for the caregiver, who often has -to repeat the same reassurance dozens of times a day. +Full detail in [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md). Most "AI for dementia" demos stop at a chatbot. We wanted something that works passively, in the background, through a camera that's already pointed at the room — so the patient never has to type, tap, or ask. -The result is a three-stage pipeline: +## Quickstart -``` -Webcam / Photo - │ - ▼ -Vision Service → { person, objects } (face + object recognition) - │ - ├──► LLM / RAG → memory card + voice (turns detections into comfort) - │ - └──► Frontend → shows card, plays audio, caregiver workspace -``` +Requires **Python 3.10+** (the code uses `X | None` type syntax throughout). -Each stage is a separate, independently runnable service, which let us split -work cleanly and swap pieces (e.g. mock vision → real webcam) without -touching the rest of the system. +You can try the whole thing **without an API key and without a camera** — start +here if you just want to see it work. ---- +### 1. Backend, in mock mode -## What it actually does - -- **Recognizes people** the patient knows, and silently tracks who is - currently in frame. -- **Recognizes care-relevant objects** — medicine bottles, glasses, a cane, - keys — distinguishing them from visually similar everyday objects. -- **Generates a memory card** in real time: a short title, a couple of - reassuring sentences, and a spoken line — written to sound like a warm, - unseen companion, never like a system reporting a detection. -- **Speaks that line aloud** to the patient via the browser's text-to-speech, - so no reading is required. -- **Gives caregivers a separate workspace**: a live text Q&A ("he's anxious - and asking where his daughter is — what do I do?") that returns a - structured, non-clinical action plan, and flags anything that looks like a - genuine safety risk. -- **Grounds every response** in a small, editable patient profile (family - members, preferences, personal notes) instead of generic advice, via a - lightweight retrieval step. +```bash +cd backend +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +MOCK_LLM=true uvicorn main:app --port 8001 +``` ---- +`MOCK_LLM=true` swaps Gemini for canned, rule-based responses. No key, no +network calls, no rate limit. Card titles are labelled `(mock)` so the mode is +never ambiguous. -## Architecture & why each part exists +### 2. A fake camera, in a second terminal -| Component | What it does | Why we built it this way | -|---|---|---| -| **Vision Service** (`vision_service/`) | Offline face recognition (InsightFace/ArcFace) + object detection (YOLOv8), served over a FastAPI HTTP + WebSocket API | Runs **entirely on-device**. This matters for two reasons: latency (we need ~1 update/second, a round trip to a cloud vision API would be too slow) and privacy (raw camera frames of the patient never leave the machine — only a small JSON summary like `person: Sarah, daughter` goes downstream). | -| ↳ Face matching | Two-pass cosine similarity: first against each person's *averaged* embedding (fast), falling back to comparing every stored photo individually only if there's no confident match | Keeps recognition fast as the registered-people list grows, while still catching awkward angles that pull a face away from its own average. | -| ↳ Object detection | Base YOLOv8-nano (general COCO objects) + an optional custom-trained model layered on top, with the custom model's detections taking priority on overlap | A generic model can tell you "bottle" but not "medicine bottle vs. water bottle" — the distinction that actually matters for dementia care. Stacking models let us keep broad household context *and* get fine-grained, demo-specific detail without retraining from scratch. | -| **LLM / RAG Service** (`rag-service/`, `llm-service/`) | Consumes the vision stream, deduplicates scene changes, retrieves relevant patient-memory context, and calls Gemini with a forced JSON schema (Pydantic) to produce the memory card and voice line | Raw detections mean nothing to someone with dementia — "Sarah, confidence 0.96" isn't comforting. Structured outputs mean the frontend never parses free text; it just plugs values straight into the UI. Retrieval over a small patient profile means the same "Sarah is here" event becomes personal ("she visits every weekend and loves you") instead of generic. | -| ↳ Scene-change dedup | Only calls the LLM when the detected person/objects actually change | Stops the card from flickering with reworded versions of the same message every second, and keeps API cost and latency down. | -| ↳ Caregiver analysis endpoint | A second, separate LLM call/schema focused on behavioral triggers, rationale, and one concrete non-pharmacological action — flagged separately if it detects a crisis | Patients and caregivers need fundamentally different tones and content. One audience needs to be soothed; the other needs to be informed and given something actionable. Mixing the two would serve neither well. | -| **Frontend** (`frontend/`, `rag-service/static/`) | Displays the live memory card, speaks it via Web Speech API, shows a live camera preview, and gives caregivers a question box | Kept deliberately simple (plain HTML/JS, no build step) so it's trivial to run anywhere and easy to swap for a native app later. | +```bash +cd backend && source .venv/bin/activate +python mock_vision.py # serves the real ws://localhost:8000/ws contract +``` ---- +### 3. Open the page -## Project structure +Open `frontend/index.html` in a browser. A memory card appears within a couple +of seconds, and the caregiver question box works. -``` -├── frontend/ lightweight standalone UI (talks to vision + llm-service) -├── llm-service/ minimal HTTP server variant of the LLM layer -├── rag-service/ fuller FastAPI app: RAG + caregiver analysis + bundled frontend -└── vision_service/ face + object recognition, fully offline - ├── models/ InsightFace + YOLOv8 wrappers - ├── storage/ JSON-backed face embedding store - ├── api/ /recognize, /register, /people, /ws - └── train/ tools + config for fine-tuning a custom object model -``` +On Windows, use `.venv\Scripts\activate` in place of `source .venv/bin/activate`. --- -## Running it +## Running the real thing -Three terminals — vision service first, then whichever LLM path you're using. +### Vision service -**1. Vision Service** (required for everything else) ```bash cd vision_service python3 -m venv .venv && source .venv/bin/activate pip install -r requirements.txt python main.py -# → http://localhost:8000 -# → registration UI: http://localhost:8000/static/register.html ``` -**2a. Full RAG app (recommended — includes caregiver analysis + bundled frontend)** -```bash -cd rag-service -python -m venv .venv && source .venv/bin/activate -pip install -r requirements.txt -cp .env.example .env # add your GEMINI_API_KEY -uvicorn app.main:app --host 0.0.0.0 --port 8080 -# → http://localhost:8080 +Serves on . The first run downloads InsightFace and YOLO +weights (a few hundred MB); after that it is fully offline. A webcam is required. + +**Register a face** at — enter a +name and relationship, capture 3–5 photos, submit. Recognition only matches +people enrolled on this machine, so this step is not optional. + +> Registering someone stores a **face embedding — biometric data** — on your +> disk. Only enrol people who have knowingly agreed. See +> [PRIVACY.md](PRIVACY.md) for what is stored and how to delete it. + +Check the roster with `curl http://localhost:8000/people`. + +### Backend with Gemini + +Copy `.env.example` to `.env` at the repository root and set your key from +: + +``` +GEMINI_API_KEY=your_key_here ``` -**2b. Minimal LLM service + standalone frontend** +Then: + ```bash -cd llm-service -export GEMINI_API_KEY=... -export VISION_WS_URL=ws://localhost:8000/ws -python main.py -# then open frontend/index.html in a browser +cd backend && source .venv/bin/activate +uvicorn main:app --port 8001 ``` -No webcam handy? Run `mock_vision.py` in either service folder to simulate a -person walking in and out of frame — useful for demos and development. - -See `vision_service/README.md` and `rag-service/README.md` for full endpoint -references and configuration options. +Both services read that root `.env` automatically. It is gitignored — keep real +keys out of source. The free Gemini tier allows roughly 5 requests per minute; +`MOCK_LLM=true` avoids it entirely. --- -## Key API shapes +## API + +### vision_service — port 8000 + +`GET /recognize` and `WS /ws` return the same payload. This is the stable +contract: -**Vision Service → everyone else** ```json { "person": { @@ -142,16 +150,97 @@ references and configuration options. "confidence": 0.96, "face_detected": true }, - "objects": [{ "label": "Medicine Bottle", "confidence": 0.91 }], + "objects": [ + { "label": "cup", "confidence": 0.91 } + ], "timestamp": 1751190195 } ``` -**LLM output → frontend** -```json -{ - "card_title": "Sarah, your daughter", - "card_body": "Sarah is here with you. She visits often and cares about you very much.", - "voice_guidance": "Hi Arthur, Sarah is here with you. You are safe, and she is happy to see you." -} +`/ws` pushes once per second. Also available: `GET /health`, `GET /people`, +`POST /register`, `POST /register/capture`, `DELETE /people/{name}`, +`GET /frame` (JPEG still). + +### backend — port 8001 + +| Endpoint | Purpose | +|---|---| +| `GET /health` | Liveness | +| `GET /latest` | Current patient memory card | +| `GET /latest?type=caretaker` | Latest caregiver answer (separate slot, so neither clobbers the other) | +| `POST /ask` | `{"question": "..."}` → advice grounded in the live scene and the patient profile | +| `POST /api/caregiver/analyze` | `{"message": "..."}` → structured behavioural analysis | + +```bash +curl -s -X POST http://localhost:8001/ask \ + -H 'Content-Type: application/json' \ + -d '{"question":"How do I keep him calm at dinner?"}' +``` + +Subscribing to the vision stream directly, if you want to build something else +on top: + +```python +import asyncio, json, websockets + +async def listen(): + async with websockets.connect("ws://localhost:8000/ws") as ws: + async for msg in ws: + scene = json.loads(msg) # ~1 per second +``` + +--- + +## The patient profile + +`patient_profile.json` at the repository root is the single source of patient +facts — name, condition, family, communication preferences. The backend +retrieves matching lines from it and passes only those to the model. Edit this +file, not code, to change who the patient is. + +**The file shipped here is sample data.** "Arthur" is fictional, invented for +demonstration. The repository contains no real patient information, and none +should be added to it — see [DISCLAIMER.md](DISCLAIMER.md). + +--- + +## Security + +This is a prototype and its security posture reflects that. **Neither service +authenticates anything**, and CORS is fully open on both. Anyone who can reach +port 8000 can pull a live camera still from `GET /frame`, enrol a face, or delete +a registered person. + +Both services therefore bind `127.0.0.1` by default. Set `VISION_HOST=0.0.0.0` +only on a network you trust, and do not deploy this as-is. Details and the full +limitation list are in [PRIVACY.md](PRIVACY.md). + +--- + +## Development + +```bash +pip install -r backend/requirements.txt -r backend/requirements-dev.txt +pytest backend/tests +ruff check . ``` + +The test suite runs fully offline against `backend/mock_llm.py` — no API key, no +camera, no network. CI runs it on Python 3.10, 3.11, and 3.12. + +`vision_service` is deliberately excluded from CI: its InsightFace/YOLO/OpenCV +stack is heavy and its endpoints need a physical camera. + +## Documentation + +- [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) — services, contracts, and design decisions +- [docs/DEMO.md](docs/DEMO.md) — running a walkthrough +- [DISCLAIMER.md](DISCLAIMER.md) — what this is not +- [PRIVACY.md](PRIVACY.md) — biometric data handling and deletion +- [vision_service/README.md](vision_service/README.md) — vision service internals +- [backend/README.md](backend/README.md) — backend internals +- [offline-chatbot/README.md](offline-chatbot/README.md) — the separate offline prototype + +## License + +MIT — see [LICENSE](LICENSE). Provided with no warranty of any kind. diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 0000000..cf550aa --- /dev/null +++ b/backend/README.md @@ -0,0 +1,66 @@ +# backend + +Single FastAPI service (port 8001). Consumes the `vision_service` websocket, turns scenes into memory cards via Gemini, answers caregiver questions, and serves both to `frontend/`. + +See [../docs/ARCHITECTURE.md](../docs/ARCHITECTURE.md) for the design rationale behind scene dedup, the two output slots, and the retrieval approach. + +```text +Vision Service (ws://localhost:8000/ws) + ↓ +backend (this service, port 8001) + ↓ +frontend/ (polls GET /latest, calls POST /ask) +``` + +## Files + +- `main.py` — FastAPI app, CORS, `/health`, `/latest`, `/ask`, `/api/caregiver/analyze`. Starts the perception background task on startup. +- `perception.py` — connects to `VISION_WS_URL` as a client, dedups scene changes, calls Gemini, writes the patient card to the store. +- `patient_memory.py` — reads the single shared `../patient_profile.json`, retrieves relevant memory chunks for a query (keyword overlap, no embeddings — good enough for a demo). +- `models.py` — `MemoryCard` (shared shape for both the patient card and caretaker advice), `BehavioralAnalysis`, request bodies. +- `gemini_client.py` — the one `genai.Client()` instance, shared by every call site. Set `MOCK_LLM=true` to swap it for `mock_llm.py`'s canned responses instead (no API key or quota needed). +- `mock_llm.py` — rule-based stand-in for the Gemini client, used only when `MOCK_LLM=true`. +- `store.py` — in-memory state: latest scene, latest patient card, latest caretaker advice (two separate slots so a caretaker question can't clobber the patient-facing card). +- `mock_vision.py` — local test double for `vision_service`, serves at `/ws` only. +- `tests/` — offline pytest suite; runs against `mock_llm.py` with no API key, camera, or network. + +Every Gemini call uses the async client surface (`client.aio.models.generate_content`). The synchronous one blocks the whole event loop for the duration of the round trip, which stalls the vision websocket and every concurrent request. + +## Run + +```bash +cd backend +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +# GEMINI_API_KEY is read from the repo-root .env +uvicorn main:app --host 127.0.0.1 --port 8001 +``` + +To run without a Gemini key or quota, use the canned responses instead: + +```bash +MOCK_LLM=true uvicorn main:app --host 127.0.0.1 --port 8001 +``` + +To test without a real camera: + +```bash +python mock_vision.py # serves ws://localhost:8000/ws +``` + +## Test + +```bash +pip install -r requirements-dev.txt +pytest tests +``` + +## Endpoints + +- `GET /health` → `{"ok": true}` +- `GET /latest` → the current patient-facing `MemoryCard` (or `{"status": "empty"}` before the first scene). Add `?type=caretaker` to get the latest caregiver advice instead. +- `POST /ask` `{"question": "..."}` → a `MemoryCard`-shaped caregiver answer, grounded in the current scene + patient memory, cached by scene + question in a bounded LRU. `400` on an empty question, `503` if Gemini is unreachable or returns nothing parseable. +- `POST /api/caregiver/analyze` `{"message": "..."}` → a `BehavioralAnalysis` for a caregiver's free-text situation report. Same `400`/`503` behaviour. + +`BehavioralAnalysis.clinical_rationale` is language-model output, not a clinical assessment — see [../DISCLAIMER.md](../DISCLAIMER.md). + diff --git a/backend/gemini_client.py b/backend/gemini_client.py new file mode 100644 index 0000000..c6fdd65 --- /dev/null +++ b/backend/gemini_client.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import logging +import os + +from dotenv import load_dotenv + +load_dotenv() + +logger = logging.getLogger(__name__) + +GEMINI_MODEL = "gemini-2.5-flash" + +MOCK_LLM = os.environ.get("MOCK_LLM", "").strip().lower() in ("1", "true", "yes") + +if MOCK_LLM: + logging.basicConfig(level=logging.INFO) + logger.info("MOCK_LLM is set — using canned responses instead of calling Gemini.") + from mock_llm import MockClient + + client = MockClient() +else: + from google import genai + + if not os.environ.get("GEMINI_API_KEY"): + raise SystemExit("Set GEMINI_API_KEY before running, or set MOCK_LLM=true to run without Gemini.") + + client = genai.Client() diff --git a/backend/main.py b/backend/main.py new file mode 100644 index 0000000..844a7c6 --- /dev/null +++ b/backend/main.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import asyncio +import json +import logging +from collections import OrderedDict +from contextlib import asynccontextmanager, suppress +from threading import Lock + +from dotenv import load_dotenv +from fastapi import FastAPI, HTTPException, Query +from fastapi.middleware.cors import CORSMiddleware +from google.genai import types + +import store +from gemini_client import GEMINI_MODEL, client +from models import AskRequest, BehavioralAnalysis, CaregiverInput, MemoryCard +from patient_memory import PatientMemoryStore +from perception import PerceptionStateTracker + +load_dotenv() + +logger = logging.getLogger(__name__) + +patient_memory = PatientMemoryStore() + +# Bounded so a long-running process can't grow the cache without limit — +# the key is (scene signature x question), which is effectively unbounded. +_ASK_CACHE_MAX = 256 +_ask_cache_lock = Lock() +_ask_cache: OrderedDict[str, dict] = OrderedDict() + + +def _cache_get(key: str) -> dict | None: + with _ask_cache_lock: + if key not in _ask_cache: + return None + _ask_cache.move_to_end(key) + return _ask_cache[key] + + +def _cache_put(key: str, value: dict) -> None: + with _ask_cache_lock: + _ask_cache[key] = value + _ask_cache.move_to_end(key) + while len(_ask_cache) > _ASK_CACHE_MAX: + _ask_cache.popitem(last=False) + + +def _normalize_question(question: str) -> str: + return " ".join(question.lower().strip().split()) + + +def _scene_signature(scene: dict) -> str: + if not isinstance(scene, dict): + return "{}" + + person = scene.get("person", {}) if isinstance(scene.get("person", {}), dict) else {} + objects = scene.get("objects", []) if isinstance(scene.get("objects", []), list) else [] + normalized_objects = [] + for item in objects: + if not isinstance(item, dict): + continue + normalized_objects.append({ + "label": item.get("label", ""), + "confidence": round(float(item.get("confidence", 0.0)), 3) if item.get("confidence") is not None else None, + }) + payload = { + "recognized": bool(person.get("recognized")), + "face_detected": bool(person.get("face_detected")), + "name": person.get("name", ""), + "relationship": person.get("relationship", ""), + "note": person.get("note", ""), + "objects": normalized_objects, + } + return json.dumps(payload, sort_keys=True) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + tracker = PerceptionStateTracker(memory=patient_memory) + task = asyncio.create_task(tracker.listen_and_process()) + yield + task.cancel() + # Await the cancellation so shutdown doesn't race the perception loop. + with suppress(asyncio.CancelledError): + await task + + +app = FastAPI(title="Dementia Memory Assistant", lifespan=lifespan) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/health") +async def health(): + return {"ok": True} + + +@app.get("/latest") +async def latest(kind: str = Query("patient", alias="type")): + payload = store.get_latest_caretaker_advice() if kind == "caretaker" else store.get_latest_patient_card() + if not payload: + return {"status": "empty"} + return payload + + +@app.post("/ask", response_model=MemoryCard) +async def ask(request: AskRequest): + question = request.question.strip() + if not question: + raise HTTPException(status_code=400, detail="question must not be empty") + + scene = store.get_latest_scene() + question_key = _normalize_question(question) + cache_key = f"{_scene_signature(scene)}::{question_key}" + + cached = _cache_get(cache_key) + if cached is not None: + return cached + + person = scene.get("person", {}) if isinstance(scene, dict) else {} + objects = scene.get("objects", []) if isinstance(scene, dict) else [] + retrieval_query = f"{question}\nPerson: {person}\nObjects: {objects}" + retrieved_context = patient_memory.retrieve_text(retrieval_query, top_k=5) + + prompt = ( + "You are a calm dementia-care assistant. Give practical, non-judgmental guidance for the caretaker. " + "Do not mention that you are an AI or reference hidden chain of thought.\n" + f"Retrieved patient memory context:\n{retrieved_context}\n" + f"Current scene: {json.dumps(scene)}\n" + f"Caretaker question: {question}" + ) + + try: + response = await client.aio.models.generate_content( + model=GEMINI_MODEL, + contents=prompt, + config=types.GenerateContentConfig( + system_instruction=( + "You answer caretakers with short, concrete suggestions. " + "Focus on safety, reassurance, and immediate next steps. " + "Keep the response short and speak directly to the caretaker." + ), + response_mime_type="application/json", + response_schema=MemoryCard, + temperature=0.2, + ), + ) + except Exception as exc: + logger.exception("Gemini call failed for /ask") + raise HTTPException(status_code=503, detail=f"Assistant service unavailable: {exc}") from exc + + # .parsed is None when the model is safety-blocked or returns unparseable JSON. + if response.parsed is None: + logger.warning("Gemini returned no parseable content for /ask (question=%r)", question) + raise HTTPException( + status_code=503, + detail="The assistant could not produce an answer for that question. Please rephrase and try again.", + ) + + result = response.parsed.model_dump() + store.set_latest_caretaker_advice(result) + _cache_put(cache_key, result) + return result + + +@app.post("/api/caregiver/analyze", response_model=BehavioralAnalysis) +async def analyze_caregiver_input(payload: CaregiverInput): + message = payload.message.strip() + if not message: + raise HTTPException(status_code=400, detail="message must not be empty") + + try: + response = await client.aio.models.generate_content( + model=GEMINI_MODEL, + contents=f"Context: {patient_memory.profile_text()}\nInput: {message}", + config=types.GenerateContentConfig( + system_instruction=( + "You are a clinical expert system in dementia caregiving. Analyze the user's input. " + "Prioritize de-escalation, behavioral redirection, and validating caregiver stress. " + "Never suggest medical prescriptions or changing drug dosages." + ), + response_mime_type="application/json", + response_schema=BehavioralAnalysis, + temperature=0.1, + ), + ) + except Exception as exc: + logger.exception("Gemini call failed for /api/caregiver/analyze") + raise HTTPException(status_code=503, detail=f"Assistant service unavailable: {exc}") from exc + + if response.parsed is None: + logger.warning("Gemini returned no parseable content for /api/caregiver/analyze") + raise HTTPException( + status_code=503, + detail="The assistant could not analyze that report. Please rephrase and try again.", + ) + + return response.parsed diff --git a/backend/mock_llm.py b/backend/mock_llm.py new file mode 100644 index 0000000..3ec9977 --- /dev/null +++ b/backend/mock_llm.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import re +from typing import Any + +from models import BehavioralAnalysis, MemoryCard + + +class MockClient: + """Drop-in stand-in for google.genai.Client. Used when MOCK_LLM=true so the + backend can run without a Gemini API key or quota. Matches the subset of the + real client's interface that this project calls: both the sync + client.models.generate_content(...) and the async + client.aio.models.generate_content(...), each returning an object with a + .parsed attribute.""" + + def __init__(self): + self.models = _MockModels() + self.aio = _MockAio() + + +class _MockModels: + def generate_content(self, *, model: str, contents: str, config: Any = None) -> _MockResponse: + schema = getattr(config, "response_schema", None) + if schema is MemoryCard: + parsed = _mock_memory_card(contents) + elif schema is BehavioralAnalysis: + parsed = _mock_behavioral_analysis(contents) + else: + parsed = None + return _MockResponse(parsed) + + +class _MockAio: + """Mirrors client.aio — the async surface of the real genai client.""" + + def __init__(self): + self.models = _MockAioModels() + + +class _MockAioModels: + def __init__(self): + self._sync = _MockModels() + + async def generate_content(self, *, model: str, contents: str, config: Any = None) -> _MockResponse: + return self._sync.generate_content(model=model, contents=contents, config=config) + + +class _MockResponse: + def __init__(self, parsed: Any | None): + self.parsed = parsed + + +def _mock_memory_card(contents: str) -> MemoryCard: + if "Caretaker question:" in contents: + match = re.search(r"Caretaker question:\s*(.+)", contents) + question = match.group(1).strip() if match else "your question" + return MemoryCard( + card_title="Caretaker Tip (mock)", + card_body=f'Regarding "{question}": stay calm, use short reassuring sentences, and check the patient profile for specifics.', + voice_guidance="Everything is okay. Take a slow breath, and offer calm, simple reassurance.", + ) + + match = re.search(r"Person detected:\s*([^(.\n]+)(?:\(([^)]*)\))?", contents) + name = match.group(1).strip() if match else "" + if match and name.lower() not in ("unrecognized face or no familiar person present", ""): + relationship = (match.group(2) or "").strip() + title = f"{name} ({relationship})" if relationship else name + return MemoryCard( + card_title=title, + card_body=f"{name} is here with you. It is safe and familiar.", + voice_guidance=f"Hello, it's {name}. You are safe, and they are happy to see you.", + ) + + return MemoryCard( + card_title="No One Recognized (mock)", + card_body="No familiar face is in view right now.", + voice_guidance="You're safe. Take your time looking around.", + ) + + +_TRIGGER_KEYWORDS = { + "Sundowning": ["evening", "dusk", "afternoon", "sunset", "4pm", "5pm", "before dinner"], + "Wandering": ["wander", "walk out", "left the house", "door", "outside"], + "Aggression": ["hit", "yell", "scream", "aggress", "angry", "throw"], + "Confusion": ["confus", "lost", "forget", "disorient"], +} + +_CRISIS_KEYWORDS = ["fell", "fall", "bleeding", "injur", "hurt", "chest pain", "emergency", "unconscious", "can't breathe"] + + +def _mock_behavioral_analysis(contents: str) -> BehavioralAnalysis: + text = contents.lower() + + category = "General Behavioral Change" + for label, keywords in _TRIGGER_KEYWORDS.items(): + if any(k in text for k in keywords): + category = label + break + + triggers = [k for keywords in _TRIGGER_KEYWORDS.values() for k in keywords if k in text][:3] or ["Not enough detail provided"] + is_crisis = any(k in text for k in _CRISIS_KEYWORDS) + + return BehavioralAnalysis( + category=category, + observed_triggers=triggers, + clinical_rationale=( + "Mock response (MOCK_LLM=true, no live Gemini call). In a real run, this field explains the " + "likely behavioral cause based on the caregiver's description." + ), + actionable_intervention=( + "Approach calmly, use short reassuring sentences, remove immediate triggers if possible, " + "and redirect to a familiar, calming activity." + ), + is_crisis=is_crisis, + ) diff --git a/rag-service/mock_vision.py b/backend/mock_vision.py similarity index 93% rename from rag-service/mock_vision.py rename to backend/mock_vision.py index f1865df..cd000ae 100644 --- a/rag-service/mock_vision.py +++ b/backend/mock_vision.py @@ -1,10 +1,15 @@ import asyncio import json import time + from websockets.asyncio.server import serve async def generate_frames(websocket): + if websocket.request.path != "/ws": + await websocket.close(code=1008, reason="Expected path /ws") + return + print("Connection established with Perception Engine.") scenarios = [ {"person": {"recognized": False, "name": "Unknown", "relationship": "", "note": ""}, "objects": []}, diff --git a/backend/models.py b/backend/models.py new file mode 100644 index 0000000..30c2846 --- /dev/null +++ b/backend/models.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class MemoryCard(BaseModel): + card_title: str = Field(description="Large, simple title text for the frontend display (e.g., 'Sarah (Daughter)').") + card_body: str = Field(description="Max 2 short sentences reinforcing context (e.g., 'She visits every weekend. She loves you.').") + voice_guidance: str = Field(description="The exact text to convert to audio. Must be short, exceptionally gentle, comforting, and spoken directly to the patient or caretaker.") + + +class AskRequest(BaseModel): + question: str + + +class CaregiverInput(BaseModel): + message: str + + +class BehavioralAnalysis(BaseModel): + category: str = Field(description="Categorization of behavior: e.g., Sundowning, Wandering, Aggression, Confusion") + observed_triggers: list[str] = Field(description="Possible environmental, physical, or temporal triggers extracted from text.") + clinical_rationale: str = Field(description="Brief neuro-clinical context explaining why the patient is exhibiting this specific behavior.") + actionable_intervention: str = Field(description="Direct, actionable, non-pharmacological step for the caregiver to de-escalate.") + is_crisis: bool = Field(description="Set to True ONLY if immediate physical danger or medical emergency is indicated.") diff --git a/rag-service/app/patient_memory.py b/backend/patient_memory.py similarity index 96% rename from rag-service/app/patient_memory.py rename to backend/patient_memory.py index e251e77..875f9cb 100644 --- a/rag-service/app/patient_memory.py +++ b/backend/patient_memory.py @@ -12,8 +12,8 @@ class PatientMemoryStore: query/vision event, and passes only that context into the LLM. """ - def __init__(self, path: str = "data/patient_profile.json"): - self.path = Path(path) + def __init__(self, path: str | None = None): + self.path = Path(path) if path else Path(__file__).resolve().parent.parent / "patient_profile.json" self.profile = self._load_profile() self.chunks = self._build_chunks() diff --git a/backend/perception.py b/backend/perception.py new file mode 100644 index 0000000..6dcf6c0 --- /dev/null +++ b/backend/perception.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import asyncio +import json +import logging +import os + +import websockets +from google.genai import types + +import store +from gemini_client import GEMINI_MODEL, client +from models import MemoryCard +from patient_memory import PatientMemoryStore + +logger = logging.getLogger(__name__) + +VISION_WS_URL = os.environ.get("VISION_WS_URL", "ws://localhost:8000/ws") +VISION_RECONNECT_DELAY = float(os.environ.get("VISION_RECONNECT_DELAY", "2.0")) + +# Shown when Gemini is unreachable or returns nothing parseable, so the patient +# still sees something calm instead of a stale or blank card. +_FALLBACK_CARD = MemoryCard( + card_title="You are safe", + card_body="Take your time. Someone will be with you soon.", + voice_guidance="You're safe. Take a slow breath, and take all the time you need.", +) + + +class PerceptionStateTracker: + def __init__(self, memory: PatientMemoryStore | None = None): + self.memory = memory or PatientMemoryStore() + self.current_scene_signature: str | None = None + self.last_output: MemoryCard | None = None + + def _should_trigger_llm(self, new_person: dict, new_objects: list) -> bool: + """Determines if the scene has changed enough to warrant a new cue.""" + new_name = (new_person.get("name") or "Unknown") if new_person.get("recognized") else "Unknown" + new_obj_labels = sorted(obj.get("label", "") for obj in new_objects) + new_scene_signature = json.dumps( + { + "recognized": bool(new_person.get("recognized")), + "face_detected": bool(new_person.get("face_detected")), + "name": new_name, + "relationship": new_person.get("relationship", ""), + "note": new_person.get("note", ""), + "objects": new_obj_labels, + }, + sort_keys=True, + ) + + if new_scene_signature != self.current_scene_signature: + self.current_scene_signature = new_scene_signature + return True + return False + + async def generate_orientation_cue(self, person: dict, objects: list) -> MemoryCard: + """Generates the structured memory card data using Gemini.""" + person = self.memory.enrich_person(person) + if person.get("recognized"): + vision_context = f"Person detected: {person['name']} ({person.get('relationship', '')}). Note: {person.get('note', '')}.\n" + else: + vision_context = "Person detected: Unrecognized face or no familiar person present.\n" + + if objects: + vision_context += "Objects currently visible in frame: " + ", ".join( + o.get("label", "") for o in objects + ) + + retrieved_context = self.memory.retrieve_text(vision_context, top_k=5) + + try: + response = await client.aio.models.generate_content( + model=GEMINI_MODEL, + contents=f"Patient Memory Context:\n{retrieved_context}\nLive Camera Feed Data:\n{vision_context}", + config=types.GenerateContentConfig( + system_instruction=( + "You are a compassionate, real-time memory assistant for a person with dementia. " + "Your job is to read environmental data from their camera and output a clear, visual " + "memory card specification and spoken audio script to orient them gently. " + "Rule 1: Keep UI text extremely simple and large. " + "Rule 2: Voice guidance must be spoken to the patient, calm, conversational, and slow. " + "Never say 'Based on the camera feed' or look clinical. Act like a loving, unseen companion." + ), + response_mime_type="application/json", + response_schema=MemoryCard, + temperature=0.3, + ), + ) + except Exception: + logger.exception("Gemini call failed while generating an orientation cue") + return self.last_output or _FALLBACK_CARD + + # .parsed is None when the model is safety-blocked or returns unparseable JSON. + if response.parsed is None: + logger.warning("Gemini returned no parseable content for the orientation cue.") + return self.last_output or _FALLBACK_CARD + + result = response.parsed + self.last_output = result + return result + + async def listen_and_process(self) -> None: + logger.info("Connecting to live vision websocket stream at %s ...", VISION_WS_URL) + + while True: + try: + async with websockets.connect(VISION_WS_URL) as ws: + logger.info("Connected to vision service.") + async for msg in ws: + try: + data = json.loads(msg) + store.set_latest_scene(data) + person = data.get("person") or { + "recognized": False, "name": "Unknown", "relationship": "", "note": "", + } + objects = data.get("objects") or [] + + # Dedup the 1Hz stream to protect LLM calls + if self._should_trigger_llm(person, objects): + logger.info("Scene shift detected (timestamp=%s).", data.get("timestamp")) + output = await self.generate_orientation_cue(person, objects) + store.set_latest_patient_card(output.model_dump()) + elif self.last_output is not None: + # Scene is stable, keep the previous LLM card. + store.set_latest_patient_card(self.last_output.model_dump()) + + except asyncio.CancelledError: + raise + except Exception: + logger.exception("Error handling a vision frame — skipping it.") + except asyncio.CancelledError: + logger.info("Perception loop cancelled.") + raise + except Exception as e: + logger.warning( + "Vision connection lost (%s). Reconnecting in %.1fs...", e, VISION_RECONNECT_DELAY + ) + await asyncio.sleep(VISION_RECONNECT_DELAY) diff --git a/backend/requirements-dev.txt b/backend/requirements-dev.txt new file mode 100644 index 0000000..dec9404 --- /dev/null +++ b/backend/requirements-dev.txt @@ -0,0 +1,6 @@ +# Test and lint tooling for the backend suite. +# The suite is fully offline (MOCK_LLM=true), so nothing here pulls in Gemini, +# a camera, or the vision_service ML stack. +pytest>=8.0.0 +httpx>=0.27.0 # required by fastapi.testclient +ruff>=0.6.0 diff --git a/rag-service/requirements.txt b/backend/requirements.txt similarity index 73% rename from rag-service/requirements.txt rename to backend/requirements.txt index 836fccf..4b0259d 100644 --- a/rag-service/requirements.txt +++ b/backend/requirements.txt @@ -1,7 +1,6 @@ +fastapi>=0.115.0 +uvicorn[standard]>=0.30.0 google-genai>=1.0.0 pydantic>=2.0.0 +python-dotenv>=1.0.0 websockets>=12.0 -fastapi>=0.115.0 -uvicorn[standard]>=0.30.0 -httpx>=0.27.0 -python-dotenv>=1.0.1 diff --git a/backend/store.py b/backend/store.py new file mode 100644 index 0000000..44468fd --- /dev/null +++ b/backend/store.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from threading import Lock +from typing import Any + +_lock = Lock() +_latest_scene: dict[str, Any] = {} +_latest_patient_card: dict[str, Any] = {} +_latest_caretaker_advice: dict[str, Any] = {} + + +def set_latest_scene(scene: dict[str, Any]) -> None: + with _lock: + _latest_scene.clear() + _latest_scene.update(scene) + + +def get_latest_scene() -> dict[str, Any]: + with _lock: + return dict(_latest_scene) + + +def set_latest_patient_card(card: dict[str, Any]) -> None: + with _lock: + _latest_patient_card.clear() + _latest_patient_card.update(card) + + +def get_latest_patient_card() -> dict[str, Any]: + with _lock: + return dict(_latest_patient_card) + + +def set_latest_caretaker_advice(advice: dict[str, Any]) -> None: + with _lock: + _latest_caretaker_advice.clear() + _latest_caretaker_advice.update(advice) + + +def get_latest_caretaker_advice() -> dict[str, Any]: + with _lock: + return dict(_latest_caretaker_advice) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 0000000..c46afdd --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,64 @@ +"""Shared pytest setup for the backend suite. + +Two things must happen before any backend module is imported: + +1. ``MOCK_LLM`` must be set. ``gemini_client`` raises ``SystemExit`` at import + time when there is no ``GEMINI_API_KEY``, and CI deliberately has none — the + whole suite runs offline against ``mock_llm.py``. +2. ``backend/`` must be on ``sys.path``. The backend uses flat imports + (``import store``) rather than a package, so it has to be importable by name. +""" + +import os +import sys +from pathlib import Path + +BACKEND_DIR = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(BACKEND_DIR)) + +os.environ["MOCK_LLM"] = "true" +# If a test ever runs the app's lifespan, keep the perception task from +# reconnecting in a tight loop against a vision service that isn't there. +os.environ.setdefault("VISION_RECONNECT_DELAY", "3600") + +import pytest # noqa: E402 + +import store # noqa: E402 + + +@pytest.fixture(autouse=True) +def clean_store(): + """``store`` is module-level global state, so reset it between tests.""" + store.set_latest_scene({}) + store.set_latest_patient_card({}) + store.set_latest_caretaker_advice({}) + yield + store.set_latest_scene({}) + store.set_latest_patient_card({}) + store.set_latest_caretaker_advice({}) + + +@pytest.fixture +def profile_path(tmp_path): + """A small, fixed patient profile so retrieval tests don't depend on the + shipped sample data.""" + import json + + path = tmp_path / "patient_profile.json" + path.write_text( + json.dumps( + { + "patient_name": "Arthur", + "condition_stage": "moderate dementia", + "baseline": "Needs calm, simple orientation cues.", + "family": [ + {"name": "Sarah", "relationship": "Daughter", "note": "Visits on weekends"}, + {"name": "Tom", "relationship": "Son", "note": "Calls every evening"}, + ], + "preferences": ["Prefers short sentences", "Likes gardening talk"], + "memories": ["Worked as a carpenter for forty years"], + } + ), + encoding="utf-8", + ) + return path diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py new file mode 100644 index 0000000..13ea86f --- /dev/null +++ b/backend/tests/test_api.py @@ -0,0 +1,293 @@ +"""Endpoint contract tests. + +Everything here runs against ``mock_llm`` (``MOCK_LLM=true`` is set in +``conftest.py``), so the suite needs no API key and makes no network calls. + +``TestClient`` is deliberately *not* used as a context manager: entering it would +run the app's lifespan, which starts the perception task and tries to dial the +vision service websocket. The routes under test don't need it. +""" + +import pytest +from fastapi.testclient import TestClient + +import main +import store + +client = TestClient(main.app) + +SCENE = { + "person": {"recognized": True, "name": "Sarah", "relationship": "Daughter", "note": "", "face_detected": True}, + "objects": [{"label": "cup", "confidence": 0.9}], + "timestamp": 1751190195, +} + + +@pytest.fixture(autouse=True) +def clear_ask_cache(): + main._ask_cache.clear() + yield + main._ask_cache.clear() + + +class CountingModels: + """Wraps the mock client so a test can prove a response came from cache.""" + + def __init__(self): + self.calls = 0 + self._inner = main.client.aio.models + + async def generate_content(self, **kwargs): + self.calls += 1 + return await self._inner.generate_content(**kwargs) + + +def install_counter(monkeypatch): + counter = CountingModels() + monkeypatch.setattr(main.client, "aio", type("Aio", (), {"models": counter})()) + return counter + + +def install_failure(monkeypatch, exc=RuntimeError("gemini is down")): + class BoomModels: + async def generate_content(self, **kwargs): + raise exc + + monkeypatch.setattr(main.client, "aio", type("Aio", (), {"models": BoomModels()})()) + + +def install_unparseable(monkeypatch): + class NoneModels: + async def generate_content(self, **kwargs): + return type("Response", (), {"parsed": None})() + + monkeypatch.setattr(main.client, "aio", type("Aio", (), {"models": NoneModels()})()) + + +def test_health(): + assert client.get("/health").json() == {"ok": True} + + +# ── GET /latest ────────────────────────────────────────────────────────────── + +def test_latest_is_empty_before_any_scene(): + assert client.get("/latest").json() == {"status": "empty"} + + +def test_latest_returns_the_patient_card_flat(): + """The frontend reads card_title/card_body/voice_guidance off the top level.""" + store.set_latest_patient_card({"card_title": "Sarah", "card_body": "b", "voice_guidance": "v"}) + + body = client.get("/latest").json() + + assert body["card_title"] == "Sarah" + assert body["voice_guidance"] == "v" + + +def test_latest_caretaker_slot_is_separate(): + store.set_latest_patient_card({"card_title": "patient"}) + store.set_latest_caretaker_advice({"card_title": "caretaker"}) + + assert client.get("/latest").json()["card_title"] == "patient" + assert client.get("/latest", params={"type": "caretaker"}).json()["card_title"] == "caretaker" + + +def test_latest_caretaker_slot_can_be_empty_independently(): + store.set_latest_patient_card({"card_title": "patient"}) + + assert client.get("/latest", params={"type": "caretaker"}).json() == {"status": "empty"} + + +def test_latest_unknown_type_falls_back_to_the_patient_card(): + store.set_latest_patient_card({"card_title": "patient"}) + + assert client.get("/latest", params={"type": "nonsense"}).json()["card_title"] == "patient" + + +# ── POST /ask ──────────────────────────────────────────────────────────────── + +def test_ask_returns_a_memory_card(): + body = client.post("/ask", json={"question": "Is Sarah visiting today?"}).json() + + assert set(body) == {"card_title", "card_body", "voice_guidance"} + assert all(isinstance(v, str) and v for v in body.values()) + + +def test_ask_rejects_an_empty_question(): + assert client.post("/ask", json={"question": ""}).status_code == 400 + + +def test_ask_rejects_a_whitespace_question(): + assert client.post("/ask", json={"question": " "}).status_code == 400 + + +def test_ask_rejects_a_malformed_body(): + assert client.post("/ask", json={}).status_code == 422 + + +def test_ask_does_not_clobber_the_patient_card(): + """Regression: a caretaker question used to overwrite the patient-facing + card, so asking anything blanked the screen the patient was looking at.""" + store.set_latest_patient_card({"card_title": "Sarah (Daughter)"}) + + client.post("/ask", json={"question": "What should I do?"}) + + assert store.get_latest_patient_card()["card_title"] == "Sarah (Daughter)" + assert store.get_latest_caretaker_advice() + + +def test_ask_caches_repeated_questions(monkeypatch): + counter = install_counter(monkeypatch) + store.set_latest_scene(SCENE) + + first = client.post("/ask", json={"question": "What should I do?"}).json() + second = client.post("/ask", json={"question": " WHAT should I DO? "}).json() + + assert counter.calls == 1 + assert first == second + + +def test_ask_cache_is_keyed_by_scene(monkeypatch): + counter = install_counter(monkeypatch) + store.set_latest_scene(SCENE) + client.post("/ask", json={"question": "Who is this?"}) + + store.set_latest_scene({**SCENE, "person": {**SCENE["person"], "name": "Tom"}}) + client.post("/ask", json={"question": "Who is this?"}) + + assert counter.calls == 2 + + +def test_ask_cache_ignores_the_timestamp(monkeypatch): + """The scene ticks at 1 Hz; keying on the timestamp would defeat the cache.""" + counter = install_counter(monkeypatch) + store.set_latest_scene(SCENE) + client.post("/ask", json={"question": "Who is this?"}) + + store.set_latest_scene({**SCENE, "timestamp": SCENE["timestamp"] + 30}) + client.post("/ask", json={"question": "Who is this?"}) + + assert counter.calls == 1 + + +def test_ask_returns_503_when_the_llm_fails(monkeypatch): + install_failure(monkeypatch) + + res = client.post("/ask", json={"question": "What should I do?"}) + + assert res.status_code == 503 + assert "unavailable" in res.json()["detail"].lower() + + +def test_ask_returns_503_when_the_llm_output_is_unparseable(monkeypatch): + install_unparseable(monkeypatch) + + assert client.post("/ask", json={"question": "What should I do?"}).status_code == 503 + + +def test_ask_does_not_cache_failures(monkeypatch): + install_failure(monkeypatch) + client.post("/ask", json={"question": "What should I do?"}) + + monkeypatch.undo() + counter = install_counter(monkeypatch) + + assert client.post("/ask", json={"question": "What should I do?"}).status_code == 200 + assert counter.calls == 1 + + +def test_ask_works_with_no_scene_yet(): + """The backend may be asked a question before the camera has produced + anything.""" + assert client.post("/ask", json={"question": "How do I keep him calm?"}).status_code == 200 + + +# ── POST /api/caregiver/analyze ────────────────────────────────────────────── + +def test_analyze_returns_the_full_schema(): + res = client.post( + "/api/caregiver/analyze", + json={"message": "Arthur got agitated and started pacing before dinner."}, + ) + + assert res.status_code == 200 + body = res.json() + assert set(body) == { + "category", + "observed_triggers", + "clinical_rationale", + "actionable_intervention", + "is_crisis", + } + assert isinstance(body["observed_triggers"], list) + assert isinstance(body["is_crisis"], bool) + + +def test_analyze_rejects_an_empty_message(): + assert client.post("/api/caregiver/analyze", json={"message": " "}).status_code == 400 + + +def test_analyze_rejects_a_malformed_body(): + assert client.post("/api/caregiver/analyze", json={}).status_code == 422 + + +def test_analyze_flags_a_crisis(): + res = client.post("/api/caregiver/analyze", json={"message": "He fell and is bleeding."}) + + assert res.json()["is_crisis"] is True + + +def test_analyze_does_not_flag_a_routine_report(): + res = client.post("/api/caregiver/analyze", json={"message": "He seemed a little confused."}) + + assert res.json()["is_crisis"] is False + + +def test_analyze_returns_503_when_the_llm_fails(monkeypatch): + install_failure(monkeypatch) + + assert client.post("/api/caregiver/analyze", json={"message": "He is pacing."}).status_code == 503 + + +def test_analyze_returns_503_when_the_llm_output_is_unparseable(monkeypatch): + install_unparseable(monkeypatch) + + assert client.post("/api/caregiver/analyze", json={"message": "He is pacing."}).status_code == 503 + + +# ── Internals ──────────────────────────────────────────────────────────────── + +def test_scene_signature_survives_junk_input(): + """The signature feeds the cache key and runs on whatever the vision service + sent, so it must never raise.""" + for junk in [None, [], "scene", {}, {"person": "nope", "objects": "nope"}, {"objects": [None, "x", {}]}]: + assert isinstance(main._scene_signature(junk), str) + + +def test_scene_signature_is_stable_across_equal_scenes(): + assert main._scene_signature(dict(SCENE)) == main._scene_signature(dict(SCENE)) + + +def test_ask_cache_evicts_the_oldest_entry(monkeypatch): + """Unbounded, this cache grew with every (scene x question) pair for the life + of the process.""" + monkeypatch.setattr(main, "_ASK_CACHE_MAX", 2) + + main._cache_put("a", {"n": 1}) + main._cache_put("b", {"n": 2}) + main._cache_put("c", {"n": 3}) + + assert main._cache_get("a") is None + assert main._cache_get("c") == {"n": 3} + + +def test_ask_cache_eviction_is_least_recently_used(monkeypatch): + monkeypatch.setattr(main, "_ASK_CACHE_MAX", 2) + + main._cache_put("a", {"n": 1}) + main._cache_put("b", {"n": 2}) + main._cache_get("a") + main._cache_put("c", {"n": 3}) + + assert main._cache_get("a") == {"n": 1} + assert main._cache_get("b") is None diff --git a/backend/tests/test_patient_memory.py b/backend/tests/test_patient_memory.py new file mode 100644 index 0000000..796ac96 --- /dev/null +++ b/backend/tests/test_patient_memory.py @@ -0,0 +1,115 @@ +"""Retrieval and person-enrichment behaviour of the keyword-overlap store.""" + +from pathlib import Path + +from patient_memory import PatientMemoryStore + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + + +def test_baseline_chunk_is_always_returned(profile_path): + """The LLM needs stable patient context even when nothing matches the query.""" + store = PatientMemoryStore(str(profile_path)) + + chunks = store.retrieve("zzzz qqqq no overlap whatsoever") + + assert [c["id"] for c in chunks] == ["baseline"] + assert "Arthur" in chunks[0]["text"] + + +def test_family_chunk_retrieved_for_matching_query(profile_path): + store = PatientMemoryStore(str(profile_path)) + + ids = [c["id"] for c in store.retrieve("Sarah is visiting today")] + + assert "family:sarah" in ids + assert "baseline" in ids + + +def test_preference_chunk_retrieved_for_matching_query(profile_path): + store = PatientMemoryStore(str(profile_path)) + + ids = [c["id"] for c in store.retrieve("he likes talking about gardening")] + + assert any(i.startswith("preference:") for i in ids) + + +def test_retrieve_respects_top_k(profile_path): + store = PatientMemoryStore(str(profile_path)) + + assert len(store.retrieve("Sarah Tom gardening carpenter dementia", top_k=2)) == 2 + + +def test_missing_profile_falls_back(tmp_path): + """A missing profile must not crash the pipeline — it degrades to generic + reassurance.""" + store = PatientMemoryStore(str(tmp_path / "does_not_exist.json")) + + chunks = store.retrieve("anything at all") + + assert chunks == [ + { + "id": "fallback", + "text": "No stored patient memory was found. Use gentle general reassurance.", + } + ] + + +def test_retrieve_text_is_prefixed_by_chunk_id(profile_path): + store = PatientMemoryStore(str(profile_path)) + + text = store.retrieve_text("Sarah") + + assert "[baseline]" in text + assert "[family:sarah]" in text + + +def test_enrich_person_merges_known_family_member(profile_path): + store = PatientMemoryStore(str(profile_path)) + + enriched = store.enrich_person({"recognized": True, "name": "Sarah"}) + + assert enriched["relationship"] == "Daughter" + assert enriched["note"] == "Visits on weekends" + + +def test_enrich_person_is_case_insensitive(profile_path): + store = PatientMemoryStore(str(profile_path)) + + assert store.enrich_person({"recognized": True, "name": "sARAh"})["relationship"] == "Daughter" + + +def test_enrich_person_keeps_vision_service_values(profile_path): + """vision_service already stores a relationship per registered face; the + profile must not overwrite what the camera pipeline supplied.""" + store = PatientMemoryStore(str(profile_path)) + + enriched = store.enrich_person( + {"recognized": True, "name": "Sarah", "relationship": "Nurse", "note": "On shift today"} + ) + + assert enriched["relationship"] == "Nurse" + assert enriched["note"] == "On shift today" + + +def test_enrich_person_passes_unrecognized_through_untouched(profile_path): + store = PatientMemoryStore(str(profile_path)) + person = {"recognized": False, "name": None, "face_detected": True} + + assert store.enrich_person(person) == person + + +def test_enrich_person_passes_unknown_name_through_untouched(profile_path): + store = PatientMemoryStore(str(profile_path)) + person = {"recognized": True, "name": "Nobody"} + + assert store.enrich_person(person) == person + + +def test_shipped_sample_profile_loads(): + """The sample profile in the repo root must stay valid — it is what a fresh + clone runs against.""" + store = PatientMemoryStore(str(REPO_ROOT / "patient_profile.json")) + + assert store.chunks, "sample patient_profile.json produced no chunks" + assert any(c["id"] == "baseline" for c in store.chunks) diff --git a/backend/tests/test_perception.py b/backend/tests/test_perception.py new file mode 100644 index 0000000..caafd9f --- /dev/null +++ b/backend/tests/test_perception.py @@ -0,0 +1,154 @@ +"""Scene deduplication and orientation-cue generation. + +The perception loop receives the vision service's 1 Hz stream, so the dedup check +is what stands between a demo and hundreds of LLM calls per minute. These tests +run entirely against ``mock_llm`` — no network, no API key. + +``asyncio.run`` is used directly rather than pytest-asyncio to keep the CI +dependency list to pytest + httpx. +""" + +import asyncio + +from models import MemoryCard +from patient_memory import PatientMemoryStore +from perception import PerceptionStateTracker + +SARAH = {"recognized": True, "name": "Sarah", "relationship": "Daughter", "note": "", "face_detected": True} +CUP = [{"label": "cup", "confidence": 0.9}] + + +def make_tracker(profile_path): + return PerceptionStateTracker(memory=PatientMemoryStore(str(profile_path))) + + +def test_first_scene_always_triggers(profile_path): + assert make_tracker(profile_path)._should_trigger_llm(SARAH, CUP) is True + + +def test_identical_scene_is_deduplicated(profile_path): + tracker = make_tracker(profile_path) + + assert tracker._should_trigger_llm(SARAH, CUP) is True + assert tracker._should_trigger_llm(SARAH, CUP) is False + assert tracker._should_trigger_llm(dict(SARAH), list(CUP)) is False + + +def test_object_order_does_not_count_as_a_change(profile_path): + """YOLO's output order is not stable; reordering the same objects must not + burn an LLM call.""" + tracker = make_tracker(profile_path) + objects = [{"label": "cup"}, {"label": "phone"}] + + assert tracker._should_trigger_llm(SARAH, objects) is True + assert tracker._should_trigger_llm(SARAH, list(reversed(objects))) is False + + +def test_confidence_drift_does_not_count_as_a_change(profile_path): + """Only labels matter — per-frame confidence jitter must not retrigger.""" + tracker = make_tracker(profile_path) + + assert tracker._should_trigger_llm(SARAH, [{"label": "cup", "confidence": 0.91}]) is True + assert tracker._should_trigger_llm(SARAH, [{"label": "cup", "confidence": 0.62}]) is False + + +def test_new_person_triggers(profile_path): + tracker = make_tracker(profile_path) + tracker._should_trigger_llm(SARAH, CUP) + + tom = {**SARAH, "name": "Tom", "relationship": "Son"} + assert tracker._should_trigger_llm(tom, CUP) is True + + +def test_new_object_triggers(profile_path): + tracker = make_tracker(profile_path) + tracker._should_trigger_llm(SARAH, CUP) + + assert tracker._should_trigger_llm(SARAH, CUP + [{"label": "phone", "confidence": 0.7}]) is True + + +def test_person_leaving_triggers(profile_path): + tracker = make_tracker(profile_path) + tracker._should_trigger_llm(SARAH, CUP) + + assert tracker._should_trigger_llm({"recognized": False, "face_detected": False}, CUP) is True + + +def test_recognized_person_without_a_name_does_not_raise(profile_path): + """vision_service can report ``recognized`` with a null name; indexing that + key directly used to raise KeyError and kill the perception loop.""" + tracker = make_tracker(profile_path) + + assert tracker._should_trigger_llm({"recognized": True}, CUP) is True + assert tracker._should_trigger_llm({"recognized": True, "name": None}, CUP) is False + + +def test_object_without_a_label_does_not_raise(profile_path): + make_tracker(profile_path)._should_trigger_llm(SARAH, [{"confidence": 0.5}]) + + +def test_generate_orientation_cue_returns_a_memory_card(profile_path): + tracker = make_tracker(profile_path) + + card = asyncio.run(tracker.generate_orientation_cue(SARAH, CUP)) + + assert isinstance(card, MemoryCard) + assert "Sarah" in card.card_title + assert card.voice_guidance + assert tracker.last_output is card + + +def test_generate_orientation_cue_handles_nobody_present(profile_path): + tracker = make_tracker(profile_path) + + card = asyncio.run(tracker.generate_orientation_cue({"recognized": False}, [])) + + assert isinstance(card, MemoryCard) + assert card.card_body + + +def test_generate_orientation_cue_falls_back_when_the_llm_fails(profile_path, monkeypatch): + """A Gemini outage must leave the patient with calm text, never a traceback + or a blank card.""" + import perception + + class BoomModels: + async def generate_content(self, **kwargs): + raise RuntimeError("gemini is down") + + monkeypatch.setattr(perception.client, "aio", type("Aio", (), {"models": BoomModels()})()) + + tracker = make_tracker(profile_path) + card = asyncio.run(tracker.generate_orientation_cue(SARAH, CUP)) + + assert card is perception._FALLBACK_CARD + + +def test_generate_orientation_cue_keeps_the_previous_card_on_failure(profile_path, monkeypatch): + import perception + + tracker = make_tracker(profile_path) + good = asyncio.run(tracker.generate_orientation_cue(SARAH, CUP)) + + class BoomModels: + async def generate_content(self, **kwargs): + raise RuntimeError("gemini is down") + + monkeypatch.setattr(perception.client, "aio", type("Aio", (), {"models": BoomModels()})()) + + assert asyncio.run(tracker.generate_orientation_cue(SARAH, CUP)) is good + + +def test_generate_orientation_cue_handles_unparseable_output(profile_path, monkeypatch): + """``response.parsed`` is None on a safety block or malformed JSON.""" + import perception + + class NoneModels: + async def generate_content(self, **kwargs): + return type("Response", (), {"parsed": None})() + + monkeypatch.setattr(perception.client, "aio", type("Aio", (), {"models": NoneModels()})()) + + card = asyncio.run(make_tracker(profile_path).generate_orientation_cue(SARAH, CUP)) + + assert card is perception._FALLBACK_CARD diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..cff8dea --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,236 @@ +# Architecture + +Three independent processes. Nothing shares memory or a database; they talk over +HTTP and one websocket. + +``` + webcam + │ + ▼ +┌───────────────────┐ WS /ws, 1 Hz ┌───────────────────┐ +│ vision_service │ ────────────────► │ backend │ +│ port 8000 │ │ port 8001 │ +│ │ │ │ +│ InsightFace + │ │ dedup → retrieve │ +│ YOLOv8n, offline │ │ → Gemini → card │ +└───────────────────┘ └───────────────────┘ + ▲ ▲ ▲ ▲ + │ │ GET /frame │ GET /latest│ POST /ask + │ │ (JPEG still) │ (poll 2 s) │ + │ └──────────────┬─────────────┴────────────┘ + │ POST /register │ + │ ┌──────────────┐ ┌──────────────────────┐ + └─────────────────│ frontend │ │ patient_profile.json │ + │ static HTML │ │ (retrieval source) │ + └──────────────┘ └──────────────────────┘ +``` + +`offline-chatbot/` is a fourth, entirely separate prototype. It shares no code, +no ports, and no data with the pipeline above. + +--- + +## vision_service (port 8000) + +Face recognition and object detection. Fully offline — no image or embedding +ever leaves the machine. + +| Endpoint | Purpose | +|---|---| +| `GET /health` | Liveness | +| `POST /recognize` | One-shot analysis of the current frame | +| `WS /ws` | Same payload, broadcast once per second | +| `GET /frame` | Current camera frame as JPEG | +| `GET /people` | Registered roster | +| `POST /register` | Enrol a face from uploaded images | +| `POST /register/capture` | Enrol from the live camera | +| `DELETE /people/{name}` | Remove a person and their embeddings | +| `GET /static/register.html` | Registration UI | + +**`/recognize` and `/ws` return the same shape.** Treat it as the stable +contract: + +```json +{ + "person": { + "recognized": true, + "name": "Sarah", + "relationship": "Daughter", + "note": "Visits on weekends", + "confidence": 0.96, + "face_detected": true + }, + "objects": [ + { "label": "cup", "confidence": 0.91 } + ], + "timestamp": 1751190195 +} +``` + +When no face is found, `recognized` and `face_detected` are `false` and `name` +is `null`. An unfamiliar face gives `face_detected: true` with +`recognized: false`. + +Implementation notes: + +- Face matching uses InsightFace `buffalo_sc` embeddings with cosine similarity + above a `0.45` threshold (`config.py`). +- Object detection is YOLOv8n at a `0.50` confidence floor, filtered to a + 27-class care-relevant subset (`object_detector.py`). +- `/ws` broadcasts at `ws_interval = 1.0` s, only while a client is connected. +- Storage is a flat JSON file, `storage/data/embeddings.json`, rewritten on + every change. There is no database. +- `config.frame_skip` is declared but never read; the real cadence is + `ws_interval` plus a separate display tick. +- The `/ws` docstring mentions a 30 s keepalive ping that is not implemented. + Harmless unless a proxy with an idle timeout sits in front of it. + +--- + +## backend (port 8001) + +A single FastAPI process — one event loop, no threads, no worker pool. It turns +scenes into patient-facing memory cards and answers caregiver questions. + +| Endpoint | Purpose | +|---|---| +| `GET /health` | Liveness | +| `GET /latest` | Latest patient memory card | +| `GET /latest?type=caretaker` | Latest caregiver answer | +| `POST /ask` | `{"question": "..."}` → a memory card for the caregiver | +| `POST /api/caregiver/analyze` | `{"message": "..."}` → structured behavioural analysis | + +### The perception loop + +`perception.py` runs as an `asyncio.Task` started by FastAPI's `lifespan`. It +dials `VISION_WS_URL`, and reconnects with a fixed backoff whenever the socket +drops. + +The vision stream arrives at 1 Hz, which would mean 3,600 LLM calls an hour. +`_should_trigger_llm` collapses it: each frame is reduced to a **scene +signature** over `recognized`, `face_detected`, `name`, `relationship`, `note`, +and the sorted set of object labels. Confidence values and timestamps are +excluded deliberately — they jitter every frame. Gemini is called only when that +signature changes; otherwise the previous card is re-published unchanged. + +If Gemini fails or returns nothing parseable, the loop keeps the last good card, +or falls back to fixed reassurance text. It never surfaces an error to the +patient. + +### Retrieval + +`patient_memory.py` (`PatientMemoryStore`) reads the repo-root +`patient_profile.json` and flattens it into id-tagged chunks: one `baseline` +chunk, one `family:` chunk per relative, and one `preference:` chunk +per preference. + +Retrieval is **keyword overlap, not vector search** — the query is tokenised, +scored against each chunk by set intersection, and the top *k* are returned. The +`baseline` chunk is always included so the model never loses core patient +context. A paraphrased question that shares no tokens with a chunk will miss it; +that is a known limitation of the demo-grade approach. + +`enrich_person` merges profile data into a recognized person, without +overwriting anything the vision service already supplied. + +### Two output slots + +`store.py` keeps `latest_patient_card` and `latest_caretaker_advice` separately. +This matters: a caregiver's question must never overwrite the card the patient is +looking at. `GET /latest` reads the first, `?type=caretaker` the second, and +`POST /ask` writes only the second. + +Responses are flat JSON — `{card_title, card_body, voice_guidance}` — or +`{"status": "empty"}` before anything has been generated. + +### The /ask cache + +`/ask` is cached on `(scene signature, normalised question)`, so re-asking the +same thing about an unchanged scene costs nothing. The cache is a bounded LRU +(256 entries) because that key space is effectively unbounded over a long run. +Failures are not cached. + +### Gemini + +One shared `genai.Client` in `gemini_client.py`, constructed at import. Every +call site uses the **async** surface (`client.aio.models.generate_content`) — +the synchronous one blocks the entire event loop for the duration of the round +trip, which stalls the websocket and every other request. + +Structured output is enforced with Pydantic response schemas (`models.py`), so +`response.parsed` is a validated `MemoryCard` or `BehavioralAnalysis`. It is +`None` on a safety block, a quota error, or malformed JSON; every call site +checks for that. + +Setting `MOCK_LLM=true` swaps the client for `mock_llm.py`, which implements the +same sync and async surfaces with canned, rule-based responses. + +**Only text is sent to Gemini** — scene descriptions, retrieved profile lines, +and caregiver questions. Images never leave the machine. + +--- + +## frontend + +Static HTML and JavaScript, no build step. `index.html` opens directly from +disk or from any static server. + +- Subscribes to `ws://localhost:8000/ws` for the live scene. +- Polls `http://localhost:8001/latest` every 2 s for the patient card. +- Posts to `http://localhost:8001/ask` for caregiver questions. +- Pulls `http://localhost:8000/frame` for the optional camera preview. + +The patient card and the caregiver answer render into separate elements, mirroring +the backend's two slots. + +`register.html` exists in two copies — `frontend/register.html` and +`vision_service/static/register.html`, served at +`http://localhost:8000/static/register.html`. The second is the one the home page +links to, because it is same-origin with the API it calls. + +--- + +## offline-chatbot + +A self-contained Streamlit prototype, unconnected to everything above. Ollama +`llama3.2` for generation, `all-MiniLM-L6-v2` for embeddings, Chroma for the +vector store. Two apps share a data directory: a patient chat UI on 8501 and a +caregiver admin UI on 8502. + +It solves a different shape of the same problem — offline document Q&A rather +than live scene orientation. Its Chroma store is a plausible replacement for +`patient_memory.py`'s keyword retrieval, but nothing wires the two together +today. + +See [offline-chatbot/README.md](../offline-chatbot/README.md). + +--- + +## Configuration + +All of it lives in a repo-root `.env`, loaded with `python-dotenv`. See +[.env.example](../.env.example). + +| Variable | Default | Used by | +|---|---|---| +| `GEMINI_API_KEY` | — | backend (required unless `MOCK_LLM=true`) | +| `MOCK_LLM` | `false` | backend | +| `VISION_WS_URL` | `ws://localhost:8000/ws` | backend | +| `VISION_RECONNECT_DELAY` | `2.0` | backend | +| `VISION_HOST` | `127.0.0.1` | vision_service | +| `VISION_PORT` | `8000` | vision_service | + +The backend's own host and port are uvicorn CLI flags, not environment +variables. + +--- + +## Known limitations + +- **No authentication anywhere.** Both services trust every caller. They bind + localhost by default for that reason. +- **CORS is fully open** on both services. +- **No durable state in the backend.** Cards, answers, and analyses live in + process memory and vanish on restart. Only `embeddings.json` persists. +- **Retrieval is keyword overlap**, so paraphrases can miss. +- **No retention policy** on stored biometrics. See [PRIVACY.md](../PRIVACY.md). diff --git a/docs/DEMO.md b/docs/DEMO.md new file mode 100644 index 0000000..4cd8545 --- /dev/null +++ b/docs/DEMO.md @@ -0,0 +1,104 @@ +# Demo guide + +How to run all three pieces locally and record a walkthrough. + +Full setup instructions are in the [README](../README.md); this file covers the +demo flow itself. + +--- + +## Before you record + +**Register a face.** Face recognition only matches people enrolled on the +machine doing the recording. Without this step the demo will show "no one +recognized" for its entire duration. + +1. Start the vision service and open . +2. Enter a name and relationship — "Self" is fine for a solo demo. +3. Capture 3–5 photos from slightly different angles, then submit. +4. Confirm: `curl http://localhost:8000/people` should list the name. + +This stores a face embedding on your machine. See [PRIVACY.md](../PRIVACY.md). + +**Decide on Gemini or mock mode.** The free Gemini tier allows roughly 5 +requests per minute, which a live demo can exhaust. Running with +`MOCK_LLM=true` gives canned responses, no network calls, and no rate limit — +the architecture and every response shape are identical. Mock cards are labelled +`(mock)` so it is obvious on camera. + +**Match the profile to your script.** `patient_profile.json` ships with a +fictional patient named Arthur. If you plan to ask "what does Arthur like?", +either keep it as-is or edit the file first — the answers come from it. + +--- + +## Flow (3–5 minutes) + +1. **Sketch the architecture.** Camera → vision service (8000) → backend + (8001) → browser page. Mention that face recognition runs locally and only + text is sent to Gemini. + +2. **Show the live scene.** Open `frontend/index.html`, point the camera at the + registered face plus a detectable object — a cup, a bottle, a phone. The + memory card fills in with the name, relationship, and note; the object chips + update below it. + +3. **Hold still, then move.** Worth calling out: the card does not regenerate + every second. The backend hashes the scene and only calls the LLM when it + actually changes. Step out of frame and back to trigger a fresh card. + +4. **Ask a caregiver question.** Type something like "How do I keep him calm at + dinner?" into the question box and send. The answer appears in its own panel — + the patient's card above is untouched, which is deliberate. + +5. **Show the analysis endpoint.** + + ```bash + curl -s -X POST http://localhost:8001/api/caregiver/analyze \ + -H "Content-Type: application/json" \ + -d '{"message":"Arthur got agitated and started pacing before dinner."}' + ``` + + Returns a structured `BehavioralAnalysis`: category, observed triggers, + rationale, a non-pharmacological intervention, and a crisis flag. Note that + the rationale is model-generated text, not a clinical assessment — see + [DISCLAIMER.md](../DISCLAIMER.md). + +6. **Close on the data source.** Open `patient_profile.json` and explain that + the backend retrieves matching lines from it and passes only those to the + model. Every card and answer in the demo traces back to this file. + +--- + +## URLs to keep open + +| URL | Shows | +|---|---| +| | Vision service is up | +| | Registered faces | +| | Registration UI | +| | Backend is up | +| | Current memory card, as JSON | +| `frontend/index.html` | The demo page | + +--- + +## Troubleshooting + +**"No one recognized" throughout.** The face on camera was never registered on +this machine. See "Before you record" above. + +**Gemini errors mentioning quota or 429.** The free tier is about 5 requests per +minute. Wait ~15 seconds, or restart the backend with `MOCK_LLM=true`. + +**No card appears.** Check that both services are running and the browser console +is clean. The page polls `http://localhost:8001/latest` every 2 seconds — hitting +that URL directly tells you whether the backend or the page is at fault. + +**No webcam.** Run `python backend/mock_vision.py` instead of the real vision +service. It serves the same websocket contract with synthetic scenes, so the +backend and frontend behave normally. + +**Camera is black or permission was denied.** Browsers only grant camera access +on `localhost` or HTTPS, and only one process can hold the camera. Close other +apps using it, then reload. diff --git a/frontend/index.html b/frontend/index.html index 9db08ac..4e42180 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -365,7 +365,7 @@

Dementia Assistant

Connecting to vision service…

- Register person + Register person
@@ -395,6 +395,13 @@

No scene yet

+
+ +

Waiting for the assistant…

+

The card appears once the assistant has seen a scene.

+

+
+ @@ -429,6 +436,13 @@

Caretaker workspace

+ +
+ Demonstration prototype — not a medical device. Responses are generated by a + language model and can be wrong. Do not use this for care decisions, and never as a substitute + for a healthcare professional. See + DISCLAIMER.md and PRIVACY.md. +
\ No newline at end of file diff --git a/frontend/register.html b/frontend/register.html index 90d079b..81ab804 100644 --- a/frontend/register.html +++ b/frontend/register.html @@ -264,6 +264,11 @@

Camera capture

Photos captured: 0 / 5

+

+ Consent required. This stores a face embedding — biometric data — on + this machine. Only register someone who has knowingly agreed. Delete anytime from the + list below. See PRIVACY.md. +

@@ -412,22 +417,50 @@

Registered people

const data = await res.json(); const people = data.people || []; if (people.length === 0) { - container.innerHTML = 'No one registered yet.'; + container.replaceChildren(emptyNote('No one registered yet.', '#888')); return; } - container.innerHTML = people.map(p => ` -
-
- ${p.name} -
${p.relationship}${p.note ? ' · ' + p.note : ''} · ${p.num_embeddings} photo(s)
-
- -
`).join(''); + // Built as DOM nodes rather than an innerHTML string: a name is caregiver-entered + // text and must never be parsed as markup or spliced into an inline handler. + container.replaceChildren(...people.map(personCard)); } catch { - container.innerHTML = 'Could not load people list.'; + container.replaceChildren(emptyNote('Could not load people list.', '#b91c1c')); } } + function personCard(p) { + const card = document.createElement('div'); + card.className = 'person-card'; + + const info = document.createElement('div'); + info.className = 'person-info'; + + const name = document.createElement('strong'); + name.textContent = p.name; + + const rel = document.createElement('div'); + rel.className = 'rel'; + const note = p.note ? ` · ${p.note}` : ''; + rel.textContent = `${p.relationship}${note} · ${p.num_embeddings} photo(s)`; + + info.append(name, rel); + + const del = document.createElement('button'); + del.className = 'del-btn'; + del.textContent = 'Delete'; + del.addEventListener('click', () => deletePerson(p.name)); + + card.append(info, del); + return card; + } + + function emptyNote(text, color) { + const em = document.createElement('em'); + em.style.color = color; + em.textContent = text; + return em; + } + async function deletePerson(name) { if (!confirm(`Delete ${name}?`)) return; const res = await fetch(`${API}/people/${encodeURIComponent(name)}`, { method: 'DELETE' }); diff --git a/llm-service/assistant.py b/llm-service/assistant.py deleted file mode 100644 index e603e2b..0000000 --- a/llm-service/assistant.py +++ /dev/null @@ -1,9 +0,0 @@ -from __future__ import annotations - -from pydantic import BaseModel, Field - - -class CaretakerAdvice(BaseModel): - card_title: str = Field(description="Short title for the advice card.") - card_body: str = Field(description="Short advice for the caretaker.") - voice_guidance: str = Field(description="Short sentence to speak aloud to the caretaker.") \ No newline at end of file diff --git a/llm-service/caregiver.py b/llm-service/caregiver.py deleted file mode 100644 index 9c2a40a..0000000 --- a/llm-service/caregiver.py +++ /dev/null @@ -1,72 +0,0 @@ -import os -import json -from typing import List -from pydantic import BaseModel, Field -from dotenv import load_dotenv -from google import genai -from google.genai import types - -load_dotenv() - -# 1. Define the local data payload structure -class BehavioralAnalysis(BaseModel): - category: str = Field(description="Categorization of behavior: e.g., Sundowning, Wandering, Aggression, Confusion") - observed_triggers: List[str] = Field(description="Possible environmental, physical, or temporal triggers extracted from text.") - clinical_rationale: str = Field(description="Brief neuro-clinical context explaining why the patient is exhibiting this specific behavior.") - actionable_intervention: str = Field(description="Direct, actionable, non-pharmacological step for the caregiver to de-escalate.") - is_crisis: bool = Field(description="Set to True ONLY if immediate physical danger or medical emergency is indicated.") - -# 2. Local Test Runner -def run_local_gemini_test(): - # The modern SDK automatically looks for GEMINI_API_KEY - if not os.environ.get("GEMINI_API_KEY"): - print("Error: Please set your GEMINI_API_KEY environment variable.") - return - - # Initialize the modern unified client - client = genai.Client() - - # Mock context normally sourced from your app's state tracking - patient_profile = ( - "Patient: Robert (80yo). Diagnosed with Moderate Stage Alzheimer's. " - "Tends to exhibit high anxiety late afternoon (sundowning). Has a background in carpentry." - ) - - # Raw unstructured text representing a live caregiver application update - sample_caregiver_input = ( - "It's 5 PM and Robert is getting really frantic. He's trying to tear up the baseboards " - "in the hallway with a flathead screwdriver saying he 'needs to finish the trim before dark'. " - "He swiped at my hand when I tried to take the screwdriver away. I don't know how to calm him down." - ) - - print("--- Sending Prompt to Gemini Locally ---") - - try: - # Generate content with structured schemas forced on the engine - response = client.models.generate_content( - model="gemini-2.5-flash", - contents=f"Context: {patient_profile}\nInput: {sample_caregiver_input}", - config=types.GenerateContentConfig( - system_instruction=( - "You are a clinical expert system in dementia caregiving. Analyze the user's input. " - "Prioritize de-escalation, behavioral redirection, and validating caregiver stress. " - "Never suggest medical prescriptions or changing drug dosages." - ), - # Enforce JSON formatting targeting our Pydantic class - response_mime_type="application/json", - response_schema=BehavioralAnalysis, - temperature=0.1, - ), - ) - - # Extract the native, fully validated Python object directly from the response - result: BehavioralAnalysis = response.parsed - - print("\n--- Structured Gemini Output Received ---") - print(json.dumps(result.model_dump(), indent=2)) - - except Exception as e: - print(f"An error occurred: {e}") - -if __name__ == "__main__": - run_local_gemini_test() \ No newline at end of file diff --git a/llm-service/http_server.py b/llm-service/http_server.py deleted file mode 100644 index da9d4e2..0000000 --- a/llm-service/http_server.py +++ /dev/null @@ -1,162 +0,0 @@ -from __future__ import annotations - -import json -from urllib.parse import parse_qs -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from threading import Lock - -from google import genai -from google.genai import types - -from assistant import CaretakerAdvice -from store import get_latest_output, get_latest_scene, set_latest_output - - -_client = genai.Client() -_cache_lock = Lock() -_ask_cache: dict[str, dict] = {} - - -def _normalize_question(question: str) -> str: - return " ".join(question.lower().strip().split()) - - -def _scene_signature(scene: dict) -> str: - if not isinstance(scene, dict): - return "{}" - - person = scene.get("person", {}) if isinstance(scene.get("person", {}), dict) else {} - objects = scene.get("objects", []) if isinstance(scene.get("objects", []), list) else [] - normalized_objects = [] - for item in objects: - if not isinstance(item, dict): - continue - normalized_objects.append({ - "label": item.get("label", ""), - "confidence": round(float(item.get("confidence", 0.0)), 3) if item.get("confidence") is not None else None, - }) - payload = { - "recognized": bool(person.get("recognized")), - "face_detected": bool(person.get("face_detected")), - "name": person.get("name", ""), - "relationship": person.get("relationship", ""), - "note": person.get("note", ""), - "objects": normalized_objects, - } - return json.dumps(payload, sort_keys=True) - - -def _build_caretaker_prompt(question: str, scene: dict) -> str: - person = scene.get("person", {}) if isinstance(scene, dict) else {} - objects = scene.get("objects", []) if isinstance(scene, dict) else [] - scene_summary = { - "person": person, - "objects": objects, - "timestamp": scene.get("timestamp") if isinstance(scene, dict) else None, - } - return ( - "You are a calm dementia-care assistant. Give practical, non-judgmental guidance for the caretaker. " - "Do not mention that you are an AI or reference hidden chain of thought. " - f"Current scene: {json.dumps(scene_summary)}\n" - f"Caretaker question: {question}" - ) - - -class _Handler(BaseHTTPRequestHandler): - def _send_json(self, status: int, payload: dict) -> None: - body = json.dumps(payload).encode("utf-8") - self.send_response(status) - self.send_header("Content-Type", "application/json; charset=utf-8") - self.send_header("Content-Length", str(len(body))) - self.send_header("Access-Control-Allow-Origin", "*") - self.send_header("Access-Control-Allow-Methods", "GET, OPTIONS") - self.send_header("Access-Control-Allow-Headers", "Content-Type") - self.end_headers() - self.wfile.write(body) - - def do_OPTIONS(self) -> None: # noqa: N802 - self.send_response(204) - self.send_header("Access-Control-Allow-Origin", "*") - self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") - self.send_header("Access-Control-Allow-Headers", "Content-Type") - self.end_headers() - - def do_GET(self) -> None: # noqa: N802 - if self.path in {"/", "/health"}: - self._send_json(200, {"status": "ok"}) - return - - if self.path == "/latest": - payload = get_latest_output() - if not payload: - self._send_json(200, {"status": "empty"}) - return - self._send_json(200, payload) - return - - self._send_json(404, {"detail": "Not found"}) - - def do_POST(self) -> None: # noqa: N802 - if self.path != "/ask": - self._send_json(404, {"detail": "Not found"}) - return - - content_length = int(self.headers.get("Content-Length", "0")) - raw_body = self.rfile.read(content_length).decode("utf-8") if content_length else "" - content_type = self.headers.get("Content-Type", "application/json") - - if content_type.startswith("application/json"): - try: - payload = json.loads(raw_body or "{}") - except json.JSONDecodeError: - self._send_json(400, {"detail": "Invalid JSON"}) - return - question = (payload.get("question") or "").strip() - else: - form = parse_qs(raw_body) - question = (form.get("question", [""])[0] or "").strip() - - if not question: - self._send_json(400, {"detail": "Question is required"}) - return - - scene = get_latest_scene() - question_key = _normalize_question(question) - cache_key = f"{_scene_signature(scene)}::{question_key}" - - with _cache_lock: - cached = _ask_cache.get(cache_key) - if cached is not None: - self._send_json(200, cached) - return - - try: - response = _client.models.generate_content( - model="gemini-2.5-flash", - contents=_build_caretaker_prompt(question, scene), - config=types.GenerateContentConfig( - system_instruction=( - "You answer caretakers with short, concrete suggestions. " - "Focus on safety, reassurance, and immediate next steps. " - "Keep the response short and speak directly to the caretaker." - ), - response_mime_type="application/json", - response_schema=CaretakerAdvice, - temperature=0.2, - ), - ) - result = response.parsed.model_dump() - set_latest_output(result) - with _cache_lock: - _ask_cache[cache_key] = result - self._send_json(200, result) - except Exception as error: - self._send_json(500, {"detail": str(error)}) - - def log_message(self, format: str, *args) -> None: # silence console noise - return - - -def start_http_server(host: str, port: int) -> ThreadingHTTPServer: - server = ThreadingHTTPServer((host, port), _Handler) - return server \ No newline at end of file diff --git a/llm-service/main.py b/llm-service/main.py deleted file mode 100644 index 1c7de05..0000000 --- a/llm-service/main.py +++ /dev/null @@ -1,41 +0,0 @@ -"""LLM service entrypoint. - -Run this after starting vision_service: - - export GEMINI_API_KEY=... - export VISION_WS_URL=ws://localhost:8000/ws - python main.py -""" - -import asyncio -import os -from threading import Thread - -from http_server import start_http_server -from perception import PerceptionStateTracker - - -def main() -> None: - if not os.environ.get("GEMINI_API_KEY"): - print("Set GEMINI_API_KEY before running.") - raise SystemExit(1) - - http_host = os.environ.get("LLM_HTTP_HOST", "127.0.0.1") - http_port = int(os.environ.get("LLM_HTTP_PORT", "8001")) - - server = start_http_server(http_host, http_port) - server_thread = Thread(target=server.serve_forever, daemon=True) - server_thread.start() - print(f"LLM latest-output server listening on http://{http_host}:{http_port}") - - tracker = PerceptionStateTracker() - - try: - asyncio.run(tracker.listen_and_process()) - finally: - server.shutdown() - server.server_close() - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/llm-service/mock_vision.py b/llm-service/mock_vision.py deleted file mode 100644 index f4fadb1..0000000 --- a/llm-service/mock_vision.py +++ /dev/null @@ -1,42 +0,0 @@ -import asyncio -import json -import time -from websockets.asyncio.server import serve - -async def generate_frames(websocket): - print("Connection established with Perception Engine.") - - # 1. Timeline of simulated visual telemetry frames - scenarios = [ - # Frame 1 & 2: Baseline empty room - {"person": {"recognized": False, "name": "Unknown", "relationship": "", "note": ""}, "objects": []}, - {"person": {"recognized": False, "name": "Unknown", "relationship": "", "note": ""}, "objects": []}, - - # Frame 3 & 4: Sarah walks in - {"person": {"recognized": True, "name": "Sarah", "relationship": "Daughter", "note": "Visits on weekends"}, "objects": []}, - {"person": {"recognized": True, "name": "Sarah", "relationship": "Daughter", "note": "Visits on weekends"}, "objects": []}, - - # Frame 5 & 6: Sarah hands over a medicine bottle - {"person": {"recognized": True, "name": "Sarah", "relationship": "Daughter", "note": "Visits on weekends"}, "objects": [{"label": "Medicine Bottle"}]}, - {"person": {"recognized": True, "name": "Sarah", "relationship": "Daughter", "note": "Visits on weekends"}, "objects": [{"label": "Medicine Bottle"}]}, - - # Frame 7: Room empties out again - {"person": {"recognized": False, "name": "Unknown", "relationship": "", "note": ""}, "objects": []} - ] - - for frame in scenarios: - frame["timestamp"] = int(time.time()) - await websocket.send(json.dumps(frame)) - print("Frame dispatched from camera feed...") - await asyncio.sleep(2) # Emit telemetry at standard intervals - -async def main(): - async with serve(generate_frames, "localhost", 8000) as server: - print("Local Mock Vision Server running on ws://localhost:8000") - await server.serve_forever() - -if __name__ == "__main__": - try: - asyncio.run(main()) - except KeyboardInterrupt: - print("\nStopping Mock Vision Server.") \ No newline at end of file diff --git a/llm-service/perception.py b/llm-service/perception.py deleted file mode 100644 index ac9d623..0000000 --- a/llm-service/perception.py +++ /dev/null @@ -1,135 +0,0 @@ -import os -import json -import asyncio -import websockets -from typing import Optional, List -from pydantic import BaseModel, Field -from dotenv import load_dotenv -from google import genai -from google.genai import types -from store import set_latest_output, set_latest_scene - - -load_dotenv() - -VISION_WS_URL = os.environ.get("VISION_WS_URL", "ws://localhost:8000/ws") -VISION_RECONNECT_DELAY = float(os.environ.get("VISION_RECONNECT_DELAY", "2.0")) - -# 1. Structured output for the Frontend -class MemoryCardOutput(BaseModel): - card_title: str = Field(description="Large, simple title text for the frontend display (e.g., 'Sarah (Daughter)').") - card_body: str = Field(description="Max 2 short sentences reinforcing context (e.g., 'She visits every weekend. She loves you.').") - voice_guidance: str = Field(description="The exact text to convert to audio. Must be short, exceptionally gentle, comforting, and spoken directly to the patient.") - -class PerceptionStateTracker: - def __init__(self): - self.client = genai.Client() - self.model = "gemini-2.5-flash" - - # State tracking variables to prevent redundant LLM fires - self.current_person_name: Optional[str] = None - self.current_objects: List[str] = [] - self.current_scene_signature: Optional[str] = None - self.last_output: Optional[MemoryCardOutput] = None - - def _should_trigger_llm(self, new_person: dict, new_objects: list) -> bool: - """Determines if the scene has changed enough to warrant a new cue.""" - new_name = new_person["name"] if new_person.get("recognized") else "Unknown" - new_obj_labels = sorted([obj["label"] for obj in new_objects]) - new_scene_signature = json.dumps( - { - "recognized": bool(new_person.get("recognized")), - "face_detected": bool(new_person.get("face_detected")), - "name": new_name, - "relationship": new_person.get("relationship", ""), - "note": new_person.get("note", ""), - "objects": new_obj_labels, - }, - sort_keys=True, - ) - - if new_scene_signature != self.current_scene_signature: - self.current_person_name = new_name - self.current_objects = new_obj_labels - self.current_scene_signature = new_scene_signature - return True - return False - - async def generate_orientation_cue(self, person: dict, objects: list) -> MemoryCardOutput: - """Generates the structured memory card data using Gemini.""" - # Compile vision context dynamically - vision_context = f"Person detected: {person['name']} ({person['relationship']}). Note: {person['note']}.\n" if person['recognized'] else "Person detected: Unrecognized face.\n" - if objects: - vision_context += "Objects currently visible in frame: " + ", ".join(o["label"] for o in objects) - - # Patient Baseline/RAG context injection point - # (Kept flat here, but this is where your static memory books attach) - patient_profile = "Patient is Arthur, advanced Alzheimer's. Gets easily startled by sudden changes. Needs reassurance." - - response = self.client.models.generate_content( - model=self.model, - contents=f"Patient Profile: {patient_profile}\nLive Camera Feed Data:\n{vision_context}", - config=types.GenerateContentConfig( - system_instruction=( - "You are a compassionate, real-time memory assistant for a person with dementia. " - "Your job is to read environmental data from their camera and output a clear, visual " - "memory card specification and spoken audio script to orient them gently. " - "Rule 1: Keep UI text extremely simple and large. " - "Rule 2: Voice guidance must be spoken to the patient, calm, conversational, and slow. " - "Never say 'Based on the camera feed' or look clinical. Act like a loving, unseen companion." - ), - response_mime_type="application/json", - response_schema=MemoryCardOutput, - temperature=0.3, - ) - ) - result = response.parsed - self.last_output = result - return result - - async def listen_and_process(self): - print(f"Connecting to live vision websocket stream at {VISION_WS_URL}...") - - while True: - try: - async with websockets.connect(VISION_WS_URL) as ws: - print("Connected to vision service.") - async for msg in ws: - try: - data = json.loads(msg) - set_latest_scene(data) - person = data.get("person", {"recognized": False, "name": "Unknown", "relationship": "", "note": ""}) - objects = data.get("objects", []) - - # Dedup the 1Hz stream to protect LLM calls - if self._should_trigger_llm(person, objects): - print(f"\n[Scene Shift Detected] Processing telemetry timestamp: {data.get('timestamp')}") - - # Call Gemini to compute memory parameters - output: MemoryCardOutput = await self.generate_orientation_cue(person, objects) - - # --- ROUTE TO FRONTEND --- - # Here you push this clean payload down to your mobile app or frontend display - print("\n>>> BROADCASTING TO FRONTEND:") - output_data = output.model_dump() - print(json.dumps(output_data, indent=2)) - set_latest_output(output_data) - # e.g., await self.frontend_websocket.send(output.model_dump_json()) - else: - # Scene is stable, keep the previous LLM card. - if self.last_output is not None: - set_latest_output(self.last_output.model_dump()) - - except Exception as e: - print(f"Error handling frame: {e}") - except Exception as e: - print(f"Vision connection lost ({e}). Reconnecting in {VISION_RECONNECT_DELAY:.1f}s...") - await asyncio.sleep(VISION_RECONNECT_DELAY) - -if __name__ == "__main__": - if not os.environ.get("GEMINI_API_KEY"): - print("Set GEMINI_API_KEY before running.") - exit(1) - - tracker = PerceptionStateTracker() - asyncio.run(tracker.listen_and_process()) \ No newline at end of file diff --git a/llm-service/requirements.txt b/llm-service/requirements.txt deleted file mode 100644 index 93131eb..0000000 --- a/llm-service/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -google-genai>=1.0.0 -pydantic>=2.0.0 -websockets -python-dotenv>=1.0.0 \ No newline at end of file diff --git a/llm-service/store.py b/llm-service/store.py deleted file mode 100644 index fdeb735..0000000 --- a/llm-service/store.py +++ /dev/null @@ -1,31 +0,0 @@ -from __future__ import annotations - -from threading import Lock -from typing import Any - - -_lock = Lock() -_latest_output: dict[str, Any] = {} -_latest_scene: dict[str, Any] = {} - - -def set_latest_output(output: dict[str, Any]) -> None: - with _lock: - _latest_output.clear() - _latest_output.update(output) - - -def get_latest_output() -> dict[str, Any]: - with _lock: - return dict(_latest_output) - - -def set_latest_scene(scene: dict[str, Any]) -> None: - with _lock: - _latest_scene.clear() - _latest_scene.update(scene) - - -def get_latest_scene() -> dict[str, Any]: - with _lock: - return dict(_latest_scene) \ No newline at end of file diff --git a/offline-chatbot/README.md b/offline-chatbot/README.md new file mode 100644 index 0000000..a59bbad --- /dev/null +++ b/offline-chatbot/README.md @@ -0,0 +1,101 @@ +# Offline chatbot + +A fully offline memory assistant: Streamlit for the UI, a local Ollama model for +generation, Chroma for retrieval. Nothing here calls a cloud API. + +**This is a separate prototype.** It shares no code, ports, or data with +`vision_service` and `backend`. There is no camera, no face recognition, and no +connection to the live pipeline — it answers questions from documents and +caregiver-entered notes instead of from a video scene. It lives in this +repository because it explores the same problem from a different angle. + +--- + +## Prerequisites + +Python 3.10+, and **a running Ollama daemon** with the `llama3.2` model pulled: + +```bash +# https://ollama.com/download +ollama pull llama3.2 +ollama serve # must stay running on http://localhost:11434 +``` + +> **If Ollama is not running, the app does not tell you.** The LLM call is +> wrapped in a bare `except` (`app.py:653`), so a connection failure is +> indistinguishable from "no relevant memory found" — the patient just sees +> *"I do not remember that right now."* If answers seem uniformly blank, check +> the daemon first. + +The first run also downloads the `all-MiniLM-L6-v2` embedding model from Hugging +Face. That one download is the only network access the app ever makes; after it, +the app runs with no internet at all. + +## Install + +```bash +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +``` + +On Windows use `.venv\Scripts\activate` instead. + +This pulls in `torch`, `transformers`, and `chromadb` — expect a multi-hundred-MB +install. + +## Run + +Two Streamlit apps share one data directory. Run each in its own terminal: + +```bash +# Caregiver admin UI — add people, objects, routines, reminders; upload documents +python -m streamlit run admin.py --server.port 8502 + +# Patient chat UI +python -m streamlit run app.py --server.port 8501 +``` + +Start with the admin app. Until something has been added and indexed, the chat +app has nothing to retrieve and will answer every question with the fallback +message. + +`ingest.py` is a standalone CLI batch loader — an alternative to the admin app's +upload page for bulk PDF/TXT ingestion. + +--- + +## How it works + +1. The admin app writes structured entries to `memory_data/memories.json` and + indexes them, along with any uploaded PDFs and text files, into a Chroma + store at `memory_data/chroma_memory/`. +2. A patient question runs a similarity search over that store — top 4 results, + discarding anything below a `0.35` relevance score. +3. If the top result is a person, routine, object, or reminder, a **templated** + answer is returned directly. This is the common case, and it involves **no + LLM call at all**. +4. Only when no template applies does the app make a single Ollama call, + constrained to the retrieved context. +5. The answer is rejected and replaced with the fallback message if it contains + hedging language ("I think", "probably", "as an AI"), on the theory that an + uncertain answer is worse than no answer for this audience. + +All paths derive from the file's own location, so the directory can be moved or +cloned anywhere. + +## Data and privacy + +`memory_data/` holds caregiver-entered personal information and the vector index +built from it. It is gitignored and never leaves your machine. Delete the +directory to erase everything; both apps recreate it empty on next start. + +See the repository's [PRIVACY.md](../PRIVACY.md) and +[DISCLAIMER.md](../DISCLAIMER.md) — the disclaimer applies to this app in full. + +## Known limitations + +- **Silent failures.** Ollama being down, a retrieval error, and a genuinely + unknown question all produce the same fallback message. +- **No authentication.** The admin app is a full CRUD interface on the patient's + memory data, reachable by anyone who can open port 8502. +- **No tests.** diff --git a/offline-chatbot/admin.py b/offline-chatbot/admin.py new file mode 100755 index 0000000..0ca6fcb --- /dev/null +++ b/offline-chatbot/admin.py @@ -0,0 +1,1224 @@ +import os +import re +import json +import uuid +import base64 +from datetime import date +from pathlib import Path + +import streamlit as st +from langchain_chroma import Chroma +from langchain_community.document_loaders import PyPDFLoader, TextLoader +from langchain_community.embeddings import HuggingFaceEmbeddings +from langchain_core.documents import Document +from langchain_text_splitters import RecursiveCharacterTextSplitter + + +# ========================================================= +# PAGE CONFIGURATION +# ========================================================= + +st.set_page_config( + page_title="Caregiver Memory Admin", + layout="wide", + initial_sidebar_state="expanded" +) + + +# ========================================================= +# PROJECT PATHS +# ========================================================= + +BASE_DIR = str(Path(__file__).resolve().parent) + +DATA_DIR = os.path.join(BASE_DIR, "memory_data") +PHOTO_DIR = os.path.join(DATA_DIR, "photos") +CHROMA_DIR = os.path.join(DATA_DIR, "chroma_memory") +UPLOAD_DIR = os.path.join(BASE_DIR, "caregiver_documents") + +MEMORY_JSON_PATH = os.path.join(DATA_DIR, "memories.json") + +LOGO_PATH = "ai.png" + +for folder in [DATA_DIR, PHOTO_DIR, CHROMA_DIR, UPLOAD_DIR]: + os.makedirs(folder, exist_ok=True) + + +# ========================================================= +# UI STYLING +# Keeps same dark style/colors as your current chatbot direction. +# ========================================================= + +def get_base64_image(image_path): + if not os.path.exists(image_path): + return "" + + try: + with open(image_path, "rb") as image_file: + return base64.b64encode(image_file.read()).decode("utf-8") + except OSError: + return "" + + + + +def escape_html(text): + text = str(text) + return ( + text.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + .replace("'", "'") + ) + +logo_base64 = get_base64_image(LOGO_PATH) + +background_logo_css = "" + +if logo_base64: + background_logo_css = f""" + .stApp::before {{ + content: ""; + position: fixed; + top: 52%; + left: 61%; + width: 680px; + height: 680px; + transform: translate(-50%, -50%); + background-image: url("data:image/png;base64,{logo_base64}"); + background-repeat: no-repeat; + background-position: center; + background-size: contain; + opacity: 0.035; + pointer-events: none; + z-index: 0; + }} + + section[data-testid="stSidebar"]::after {{ + content: ""; + position: absolute; + left: 50%; + bottom: 3.5rem; + width: 155px; + height: 155px; + transform: translateX(-50%); + background-image: url("data:image/png;base64,{logo_base64}"); + background-repeat: no-repeat; + background-position: center; + background-size: contain; + opacity: 0.08; + pointer-events: none; + z-index: 0; + }} + """ + +st.markdown( + f""" + + """, + unsafe_allow_html=True +) + + +# ========================================================= +# JSON STORAGE +# ========================================================= + +def empty_store(): + return { + "people": [], + "objects": [], + "routines": [], + "reminders": [], + "documents": [] + } + + +def load_store(): + if not os.path.exists(MEMORY_JSON_PATH): + return empty_store() + + try: + with open(MEMORY_JSON_PATH, "r", encoding="utf-8") as file: + data = json.load(file) + + base = empty_store() + + for key in base: + if key not in data or not isinstance(data[key], list): + data[key] = [] + + return data + + except (OSError, json.JSONDecodeError): + return empty_store() + + +def save_store(data): + with open(MEMORY_JSON_PATH, "w", encoding="utf-8") as file: + json.dump(data, file, ensure_ascii=False, indent=2) + + +def safe_filename(filename): + filename = os.path.basename(str(filename)).strip() + return re.sub(r'[<>:"/\\|?*]', "_", filename) + + +def save_photo(uploaded_photo, prefix): + if uploaded_photo is None: + return "" + + extension = os.path.splitext(uploaded_photo.name)[1].lower() + + if extension not in [".png", ".jpg", ".jpeg", ".webp"]: + extension = ".jpg" + + filename = f"{prefix}_{uuid.uuid4().hex}{extension}" + path = os.path.join(PHOTO_DIR, filename) + + with open(path, "wb") as file: + file.write(uploaded_photo.getbuffer()) + + return path + + +def save_document_file(uploaded_file): + filename = safe_filename(uploaded_file.name) + path = os.path.join(UPLOAD_DIR, filename) + + base_name, extension = os.path.splitext(filename) + counter = 1 + + while os.path.exists(path): + filename = f"{base_name}_{counter}{extension}" + path = os.path.join(UPLOAD_DIR, filename) + counter += 1 + + with open(path, "wb") as file: + file.write(uploaded_file.getbuffer()) + + return path, filename + + +# ========================================================= +# CHROMA HELPERS +# ========================================================= + +@st.cache_resource +def get_vector_store(): + embeddings = HuggingFaceEmbeddings( + model_name="all-MiniLM-L6-v2" + ) + + return Chroma( + persist_directory=CHROMA_DIR, + embedding_function=embeddings + ) + + +def memory_to_document(memory_type, record): + if memory_type == "people": + content = ( + f"{record.get('name', '')} is {record.get('relationship', '')}. " + f"Memory notes: {record.get('notes', '')}" + ) + + metadata = { + "source": "admin_form", + "type": "person", + "name": record.get("name", ""), + "relationship": record.get("relationship", ""), + "record_id": record.get("id", "") + } + + elif memory_type == "objects": + content = ( + f"{record.get('name', '')} is an important object. " + f"Description: {record.get('description', '')}" + ) + + metadata = { + "source": "admin_form", + "type": "object", + "name": record.get("name", ""), + "record_id": record.get("id", "") + } + + elif memory_type == "routines": + content = ( + f"{record.get('title', '')} routine at {record.get('time', '')}. " + f"Steps and notes: {record.get('steps', '')}" + ) + + metadata = { + "source": "admin_form", + "type": "routine", + "title": record.get("title", ""), + "time": record.get("time", ""), + "record_id": record.get("id", "") + } + + elif memory_type == "reminders": + content = ( + f"Reminder: {record.get('title', '')}. " + f"Date: {record.get('date', '')}. " + f"Notes: {record.get('notes', '')}" + ) + + metadata = { + "source": "admin_form", + "type": "reminder", + "title": record.get("title", ""), + "date": record.get("date", ""), + "record_id": record.get("id", "") + } + + else: + content = str(record) + metadata = { + "source": "admin_form", + "type": memory_type, + "record_id": record.get("id", "") + } + + return Document( + page_content=content, + metadata=metadata + ) + + +def upsert_memory(vector_store, memory_type, record): + record_id = record.get("id") + + if not record_id: + return + + item_id = f"{memory_type}_{record_id}" + + try: + vector_store.delete(ids=[item_id]) + except Exception: + pass + + vector_store.add_documents( + [memory_to_document(memory_type, record)], + ids=[item_id] + ) + + +def delete_memory(vector_store, memory_type, record_id): + try: + vector_store.delete(ids=[f"{memory_type}_{record_id}"]) + except Exception: + pass + + +def clean_text(text): + text = str(text).replace("\r", "\n") + text = re.sub(r"-+\s*PAGE\s*\d+\s*-+", " ", text, flags=re.IGNORECASE) + text = re.sub(r"[━─═―_]{3,}", "\n", text) + text = re.sub(r"\n{3,}", "\n\n", text) + text = re.sub(r"[ \t]{2,}", " ", text) + return text.strip() + + +def load_uploaded_document(path): + extension = os.path.splitext(path)[1].lower() + + if extension == ".pdf": + return PyPDFLoader(path).load() + + if extension == ".txt": + return TextLoader(path, encoding="utf-8").load() + + raise ValueError("Only PDF and TXT files are supported.") + + +def ingest_document(vector_store, path, filename, document_id): + raw_docs = load_uploaded_document(path) + + full_text = clean_text( + "\n".join( + document.page_content + for document in raw_docs + if document.page_content + ) + ) + + if not full_text: + return 0 + + base_doc = Document( + page_content=full_text, + metadata={ + "source": "uploaded_document", + "filename": filename, + "document_id": document_id, + "type": "caregiver_document" + } + ) + + splitter = RecursiveCharacterTextSplitter( + chunk_size=900, + chunk_overlap=150, + separators=["\n\n", "\n", ". ", " ", ""] + ) + + chunks = splitter.split_documents([base_doc]) + ids = [f"document_{document_id}_chunk_{index}" for index in range(len(chunks))] + + if chunks: + vector_store.add_documents(chunks, ids=ids) + + return len(chunks) + + +def delete_document(vector_store, document_id): + try: + existing = vector_store.get() + ids = existing.get("ids", []) + metadatas = existing.get("metadatas", []) + + delete_ids = [ + item_id + for item_id, metadata in zip(ids, metadatas) + if metadata and metadata.get("document_id") == document_id + ] + + if delete_ids: + vector_store.delete(ids=delete_ids) + + except Exception: + pass + + +def rebuild_index(vector_store, data): + try: + existing = vector_store.get() + ids = existing.get("ids", []) + + if ids: + vector_store.delete(ids=ids) + + except Exception: + pass + + docs = [] + ids = [] + + for memory_type in ["people", "objects", "routines", "reminders"]: + for record in data.get(memory_type, []): + docs.append(memory_to_document(memory_type, record)) + ids.append(f"{memory_type}_{record.get('id')}") + + if docs: + vector_store.add_documents(docs, ids=ids) + + for record in data.get("documents", []): + path = record.get("path", "") + + if path and os.path.exists(path): + ingest_document( + vector_store, + path, + record.get("filename", ""), + record.get("id", "") + ) + + +# ========================================================= +# UI HELPERS +# ========================================================= + +def render_header(): + st.markdown( + """ +
+
Caregiver Memory Admin
+
+ Add trusted people, objects, routines, reminders, and caregiver documents. + Everything saved here becomes part of the memory assistant's trusted knowledge base. +
+
+ """, + unsafe_allow_html=True + ) + + +def render_sidebar_logo(): + if not logo_base64: + return + + st.markdown( + f""" +
+ +
+ """, + unsafe_allow_html=True + ) + + +def render_metric_card(label, value): + st.markdown( + f""" +
+
{int(value)}
+
{escape_html(label)}
+
+ """, + unsafe_allow_html=True + ) + +def render_card(memory_type, record): + label = memory_type[:-1] if memory_type.endswith("s") else memory_type + + if memory_type == "people": + name = record.get("name", "Unnamed person") + meta = record.get("relationship", "") + notes = record.get("notes", "") + + elif memory_type == "objects": + name = record.get("name", "Unnamed object") + meta = "Object" + notes = record.get("description", "") + + elif memory_type == "routines": + name = record.get("title", "Untitled routine") + meta = f"Time: {record.get('time', '')}" + notes = record.get("steps", "") + + elif memory_type == "reminders": + name = record.get("title", "Untitled reminder") + meta = f"Date: {record.get('date', '')}" + notes = record.get("notes", "") + + elif memory_type == "documents": + name = record.get("filename", "Uploaded document") + meta = f"Chunks indexed: {record.get('chunks', 0)}" + notes = record.get("path", "") + + else: + name = "Memory" + meta = "" + notes = str(record) + + st.markdown( + f""" +
+
{escape_html(label)}
+
{escape_html(name)}
+
{escape_html(meta)}
+
{escape_html(notes)}
+
+ """, + unsafe_allow_html=True + ) + + +# ========================================================= +# FORM SECTIONS +# ========================================================= + +def add_person(data, vector_store): + st.subheader("Add Person") + + with st.form("person_form", clear_on_submit=True): + name = st.text_input("Name", placeholder="Example: Sarah Ahmed") + relationship = st.text_input("Relationship", placeholder="Example: Daughter") + notes = st.text_area( + "Memory Notes", + placeholder="Example: Sarah visits every Saturday and likes gardening." + ) + photo = st.file_uploader( + "Photo (optional)", + type=["png", "jpg", "jpeg", "webp"], + key="person_photo" + ) + submitted = st.form_submit_button("Save Person") + + if submitted: + if not name.strip(): + st.warning("Please enter a name.") + return + + record = { + "id": uuid.uuid4().hex, + "name": name.strip(), + "relationship": relationship.strip(), + "notes": notes.strip(), + "photo_path": save_photo(photo, "person") + } + + data["people"].append(record) + save_store(data) + upsert_memory(vector_store, "people", record) + st.success("Person saved and indexed.") + + +def add_object(data, vector_store): + st.subheader("Add Object") + + with st.form("object_form", clear_on_submit=True): + name = st.text_input("Object Name", placeholder="Example: Blue Mug") + description = st.text_area( + "Description", + placeholder="Example: This is the blue mug used for morning tea." + ) + photo = st.file_uploader( + "Photo (optional)", + type=["png", "jpg", "jpeg", "webp"], + key="object_photo" + ) + submitted = st.form_submit_button("Save Object") + + if submitted: + if not name.strip(): + st.warning("Please enter an object name.") + return + + record = { + "id": uuid.uuid4().hex, + "name": name.strip(), + "description": description.strip(), + "photo_path": save_photo(photo, "object") + } + + data["objects"].append(record) + save_store(data) + upsert_memory(vector_store, "objects", record) + st.success("Object saved and indexed.") + + +def add_routine(data, vector_store): + st.subheader("Add Routine") + + with st.form("routine_form", clear_on_submit=True): + title = st.text_input("Routine Title", placeholder="Example: Morning Medicine") + routine_time = st.text_input("Time", placeholder="Example: 08:00 AM") + steps = st.text_area( + "Steps / Notes", + placeholder="Example:\n1. Take blood pressure medicine.\n2. Drink a glass of water." + ) + submitted = st.form_submit_button("Save Routine") + + if submitted: + if not title.strip(): + st.warning("Please enter a routine title.") + return + + record = { + "id": uuid.uuid4().hex, + "title": title.strip(), + "time": routine_time.strip(), + "steps": steps.strip() + } + + data["routines"].append(record) + save_store(data) + upsert_memory(vector_store, "routines", record) + st.success("Routine saved and indexed.") + + +def add_reminder(data, vector_store): + st.subheader("Add Reminder") + + with st.form("reminder_form", clear_on_submit=True): + title = st.text_input("Reminder Title", placeholder="Example: Doctor Appointment") + reminder_date = st.date_input("Date", value=date.today()) + notes = st.text_area( + "Notes", + placeholder="Example: Appointment with Dr. Khan at City Hospital." + ) + submitted = st.form_submit_button("Save Reminder") + + if submitted: + if not title.strip(): + st.warning("Please enter a reminder title.") + return + + record = { + "id": uuid.uuid4().hex, + "title": title.strip(), + "date": str(reminder_date), + "notes": notes.strip() + } + + data["reminders"].append(record) + save_store(data) + upsert_memory(vector_store, "reminders", record) + st.success("Reminder saved and indexed.") + + +def upload_documents(data, vector_store): + st.subheader("Upload Caregiver Documents") + + st.markdown( + "Upload TXT or PDF files such as family notes, daily routines, medication schedules, or object descriptions." + ) + + uploaded_files = st.file_uploader( + "Upload PDF/TXT files", + type=["pdf", "txt"], + accept_multiple_files=True + ) + + if st.button("Save and Index Uploaded Documents"): + if not uploaded_files: + st.warning("Please choose at least one PDF or TXT file.") + return + + count = 0 + + for uploaded_file in uploaded_files: + document_id = uuid.uuid4().hex + + try: + path, filename = save_document_file(uploaded_file) + chunks = ingest_document(vector_store, path, filename, document_id) + + data["documents"].append({ + "id": document_id, + "filename": filename, + "path": path, + "chunks": chunks + }) + + count += 1 + + except Exception as error: + st.error(f"Could not ingest {uploaded_file.name}: {error}") + + save_store(data) + + if count: + st.success(f"{count} document(s) uploaded and indexed.") + + +def view_memories(data, vector_store): + st.subheader("Stored Memories") + + labels = { + "people": "People", + "objects": "Objects", + "routines": "Routines", + "reminders": "Reminders", + "documents": "Uploaded Documents" + } + + for memory_type, label in labels.items(): + st.markdown(f"### {label}") + + records = data.get(memory_type, []) + + if not records: + st.info(f"No {label.lower()} saved yet.") + continue + + for record in records: + col1, col2 = st.columns([5, 1]) + + with col1: + render_card(memory_type, record) + + with col2: + if st.button("Delete", key=f"delete_{memory_type}_{record.get('id')}"): + data[memory_type] = [ + item + for item in data[memory_type] + if item.get("id") != record.get("id") + ] + + save_store(data) + + if memory_type == "documents": + delete_document(vector_store, record.get("id")) + else: + delete_memory(vector_store, memory_type, record.get("id")) + + st.success("Deleted successfully.") + st.rerun() + + +# ========================================================= +# MAIN +# ========================================================= + +vector_store = get_vector_store() +data = load_store() + +total_sources = ( + len(data["people"]) + + len(data["objects"]) + + len(data["routines"]) + + len(data["reminders"]) + + len(data["documents"]) +) + +with st.sidebar: + render_sidebar_logo() + + st.markdown( + """ +
Memory Admin
+
+ Manage trusted memories for the assistant. +
+ """, + unsafe_allow_html=True + ) + + st.markdown("
Workspace
", unsafe_allow_html=True) + + page = st.radio( + "Section", + [ + "Dashboard", + "Add Person", + "Add Object", + "Add Routine", + "Add Reminder", + "Upload Documents", + "View Memories" + ], + label_visibility="collapsed" + ) + + st.markdown( + f""" +
+
{total_sources}
+
Saved memory sources
+
+ """, + unsafe_allow_html=True + ) + + if st.button("Rebuild Memory Index", use_container_width=True): + rebuild_index(vector_store, data) + st.success("Memory index rebuilt.") + + st.markdown( + """ +
+ Tip: add memories through forms for structured data, or upload PDF/TXT files for existing caregiver notes. +
+ """, + unsafe_allow_html=True + ) + +render_header() + +if page == "Dashboard": + col1, col2, col3, col4, col5 = st.columns(5) + + with col1: + render_metric_card("People", len(data["people"])) + with col2: + render_metric_card("Objects", len(data["objects"])) + with col3: + render_metric_card("Routines", len(data["routines"])) + with col4: + render_metric_card("Reminders", len(data["reminders"])) + with col5: + render_metric_card("Documents", len(data["documents"])) + + st.markdown("
", unsafe_allow_html=True) + + st.markdown( + """ +
+

Suggested Demo Flow

+
    +
  1. Add a familiar person such as a daughter, son, spouse, or caregiver.
    2. +
  2. Add a daily routine such as morning medicine.
    3. +
  3. Add a meaningful object such as a favorite mug or walking stick.
    4. +
  4. Upload a family notes or routine document.
    5. +
  5. Open the patient app and ask: "Who is Sarah?" or "What medicine do I take in the morning?"
    6. +
+
+ """, + unsafe_allow_html=True + ) + +elif page == "Add Person": + add_person(data, vector_store) + +elif page == "Add Object": + add_object(data, vector_store) + +elif page == "Add Routine": + add_routine(data, vector_store) + +elif page == "Add Reminder": + add_reminder(data, vector_store) + +elif page == "Upload Documents": + upload_documents(data, vector_store) + +elif page == "View Memories": + view_memories(data, vector_store) diff --git a/offline-chatbot/ai.png b/offline-chatbot/ai.png new file mode 100755 index 0000000..87f13eb Binary files /dev/null and b/offline-chatbot/ai.png differ diff --git a/offline-chatbot/app.py b/offline-chatbot/app.py new file mode 100755 index 0000000..8df7eca --- /dev/null +++ b/offline-chatbot/app.py @@ -0,0 +1,855 @@ +import os +import re +import json +import base64 +from pathlib import Path + +import streamlit as st +from langchain_chroma import Chroma +from langchain_community.embeddings import HuggingFaceEmbeddings +from langchain_community.llms import Ollama +from langchain_core.prompts import ChatPromptTemplate + + +# ========================================================= +# PAGE CONFIGURATION +# ========================================================= + +st.set_page_config( + page_title="Dementia Memory Assistant", + layout="wide", + initial_sidebar_state="expanded" +) + + +# ========================================================= +# PROJECT PATHS +# ========================================================= + +BASE_DIR = str(Path(__file__).resolve().parent) + +DATA_DIR = os.path.join(BASE_DIR, "memory_data") +CHROMA_DIR = os.path.join(DATA_DIR, "chroma_memory") +MEMORY_JSON_PATH = os.path.join(DATA_DIR, "memories.json") + +os.makedirs(DATA_DIR, exist_ok=True) +os.makedirs(CHROMA_DIR, exist_ok=True) + +LOGO_PATH = "ai.png" + +FALLBACK_MSG = "I do not remember that right now. Please ask your caregiver." +WELCOME_MSG = "Hello. I am here to help you remember people, objects, routines, and reminders." + +MIN_RELEVANCE_SCORE = 0.35 +MAX_RETRIEVED_DOCS = 4 + + +# ========================================================= +# BASIC HELPERS +# ========================================================= + +def get_base64_image(image_path): + if not os.path.exists(image_path): + return "" + + try: + with open(image_path, "rb") as image_file: + return base64.b64encode(image_file.read()).decode("utf-8") + except OSError: + return "" + + +def escape_html(text): + text = str(text) + + return ( + text.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + .replace("'", "'") + ) + + +logo_base64 = get_base64_image(LOGO_PATH) + + +# ========================================================= +# LIGHT CHATGPT-LIKE UI +# ========================================================= + +background_logo_css = "" + +if logo_base64: + background_logo_css = f""" + .stApp::before {{ + content: ""; + position: fixed; + top: 52%; + left: 61%; + width: 690px; + height: 690px; + transform: translate(-50%, -50%); + background-image: url("data:image/png;base64,{logo_base64}"); + background-repeat: no-repeat; + background-position: center; + background-size: contain; + opacity: 0.035; + pointer-events: none; + z-index: 0; + }} + + section[data-testid="stSidebar"]::after {{ + content: ""; + position: absolute; + left: 50%; + bottom: 3.5rem; + width: 155px; + height: 155px; + transform: translateX(-50%); + background-image: url("data:image/png;base64,{logo_base64}"); + background-repeat: no-repeat; + background-position: center; + background-size: contain; + opacity: 0.075; + pointer-events: none; + z-index: 0; + }} + """ + +st.markdown( + f""" + + """, + unsafe_allow_html=True +) + + +# ========================================================= +# MEMORY STORE / QUICK QUESTION HELPERS +# ========================================================= + +def load_memory_store(): + empty = { + "people": [], + "objects": [], + "routines": [], + "reminders": [], + "documents": [] + } + + if not os.path.exists(MEMORY_JSON_PATH): + return empty + + try: + with open(MEMORY_JSON_PATH, "r", encoding="utf-8") as file: + data = json.load(file) + + for key in empty: + if key not in data or not isinstance(data[key], list): + data[key] = [] + + return data + + except Exception: + return empty + + +def build_quick_questions(memory_store): + questions = [] + + if memory_store.get("people"): + name = memory_store["people"][0].get("name", "").strip() + if name: + questions.append(f"Who is {name}?") + + if memory_store.get("routines"): + title = memory_store["routines"][0].get("title", "").strip() + if title: + questions.append(f"What is my {title} routine?") + + if memory_store.get("objects"): + name = memory_store["objects"][0].get("name", "").strip() + if name: + questions.append(f"What is {name}?") + + if memory_store.get("reminders"): + questions.append("What reminders do I have?") + + defaults = [ + "Who visits me often?", + "What do I do in the morning?", + "What medicine do I take?", + "Who helps me with appointments?" + ] + + for item in defaults: + if len(questions) >= 4: + break + if item not in questions: + questions.append(item) + + return questions[:4] + + +def queue_question(question): + st.session_state.pending_question = question + + +# ========================================================= +# RAG INITIALIZATION +# ========================================================= + +@st.cache_resource +def initialize_rag(): + embeddings = HuggingFaceEmbeddings( + model_name="all-MiniLM-L6-v2" + ) + + llm = Ollama( + model="llama3.2", + temperature=0.0, + base_url="http://localhost:11434" + ) + + vector_store = Chroma( + persist_directory=CHROMA_DIR, + embedding_function=embeddings + ) + + return llm, vector_store + + +def retrieve_memories(vector_store, query): + if vector_store is None: + return [] + + query = str(query).strip() + + if not query: + return [] + + try: + results = vector_store.similarity_search_with_relevance_scores( + query, + k=MAX_RETRIEVED_DOCS + ) + except Exception: + return [] + + accepted = [] + + for document, score in results: + if score < MIN_RELEVANCE_SCORE: + continue + + accepted.append((document, score)) + + accepted.sort(key=lambda item: item[1], reverse=True) + + return [ + document + for document, _ in accepted + ] + + +def build_context(documents): + context_parts = [] + + for index, document in enumerate(documents, start=1): + source = document.metadata.get("source", "memory") + memory_type = document.metadata.get("type", "memory") + filename = document.metadata.get("filename", "") + + header = f"Memory {index} | Type: {memory_type} | Source: {source}" + + if filename: + header += f" | File: {filename}" + + context_parts.append( + f"{header}\n{document.page_content}" + ) + + return "\n\n".join(context_parts) + + +def clean_answer(text): + text = str(text).replace("\r", "\n") + text = re.sub(r"\n{3,}", "\n\n", text) + return text.strip() + + +def get_simple_memory_answer(user_question, documents): + if not documents: + return "" + + question = str(user_question).lower() + top_document = documents[0] + metadata = top_document.metadata + content = top_document.page_content.strip() + memory_type = metadata.get("type", "") + + if memory_type == "person" and any(word in question for word in ["who", "name", "person"]): + name = metadata.get("name", "").strip() + relationship = metadata.get("relationship", "").strip() + + if name and relationship: + return f"This is {name}, your {relationship}. {content}" + + if memory_type in {"routine", "object", "reminder"}: + return content + + return "" + + +def generate_answer(llm, user_question, context): + prompt = ChatPromptTemplate.from_messages([ + ( + "system", + "You are a gentle memory assistant for a person with dementia. " + "Answer only using the provided caregiver memory context. " + "Do not guess. Do not invent names, medical instructions, dates, or relationships. " + "Keep the answer short, calm, and reassuring. " + "Use simple language. " + "If the context does not contain the answer, say exactly: " + "'I do not remember that right now. Please ask your caregiver.'" + ), + ( + "human", + "Caregiver memory context:\n{context}\n\nQuestion:\n{question}" + ) + ]) + + chain = prompt | llm + + try: + response = chain.invoke({ + "context": context, + "question": user_question + }) + + answer = clean_answer(response) + + if not answer: + return FALLBACK_MSG + + unsafe_phrases = [ + "i think", + "probably", + "maybe", + "usually, people", + "in general", + "as an ai", + "based on general" + ] + + lowered = answer.lower() + + if any(phrase in lowered for phrase in unsafe_phrases): + return FALLBACK_MSG + + return answer + + except Exception: + return FALLBACK_MSG + + +def render_memory_card(documents): + if not documents: + return + + top_document = documents[0] + memory_type = escape_html(top_document.metadata.get("type", "memory")) + source = escape_html(top_document.metadata.get("source", "memory")) + filename = escape_html(top_document.metadata.get("filename", "")) + + source_line = source + + if filename: + source_line += f" | {filename}" + + preview = escape_html(top_document.page_content.strip()) + + if len(preview) > 430: + preview = preview[:430].rstrip() + "..." + + st.markdown( + f""" +
+
Retrieved Memory: {memory_type}
+
+ {preview} +
+ Source: {source_line} +
+
+ """, + unsafe_allow_html=True + ) + + +def render_sidebar_logo(): + if not logo_base64: + return + + st.markdown( + f""" +
+ +
+ """, + unsafe_allow_html=True + ) + + +# ========================================================= +# SESSION STATE +# ========================================================= + +if "messages" not in st.session_state: + st.session_state.messages = [ + { + "role": "assistant", + "content": WELCOME_MSG + } + ] + +if "pending_question" not in st.session_state: + st.session_state.pending_question = "" + + +memory_store = load_memory_store() + + +# ========================================================= +# SIDEBAR +# ========================================================= + +with st.sidebar: + render_sidebar_logo() + + st.markdown( + """ +
Memory Assistant
+
+ Ask calm, simple questions using trusted caregiver memories. +
+ """, + unsafe_allow_html=True + ) + + total_sources = ( + len(memory_store.get("people", [])) + + len(memory_store.get("objects", [])) + + len(memory_store.get("routines", [])) + + len(memory_store.get("reminders", [])) + + len(memory_store.get("documents", [])) + ) + + st.markdown( + f""" +
+
{total_sources}
+
Saved memory sources
+
+ """, + unsafe_allow_html=True + ) + + if st.button("New Conversation", use_container_width=True): + st.session_state.messages = [ + { + "role": "assistant", + "content": WELCOME_MSG + } + ] + st.rerun() + + st.markdown( + """ +
+ This assistant only answers from saved caregiver knowledge. If it cannot find the answer, it will ask you to check with the caregiver. +
+ """, + unsafe_allow_html=True + ) + + +# ========================================================= +# MAIN APP +# ========================================================= + +llm, vector_store = initialize_rag() + +st.markdown( + """ +
+
Dementia Memory Assistant
+
A calm assistant that answers only from trusted caregiver memories.
+
+ """, + unsafe_allow_html=True +) + +quick_questions = build_quick_questions(memory_store) + +if quick_questions: + st.markdown("
Quick questions
", unsafe_allow_html=True) + cols = st.columns(2) + + for index, question in enumerate(quick_questions): + with cols[index % 2]: + st.button( + question, + key=f"quick_question_{index}", + use_container_width=True, + on_click=queue_question, + args=(question,) + ) + +st.markdown("---") + +for message in st.session_state.messages: + with st.chat_message(message["role"]): + st.markdown(message["content"]) + +user_question = st.session_state.pop("pending_question", "") + +if not user_question: + user_question = st.chat_input( + "Ask a memory question, for example: Who is Sarah?" + ) + +if user_question: + user_question = str(user_question).strip() + + if user_question: + st.session_state.messages.append({ + "role": "user", + "content": user_question + }) + + with st.chat_message("user"): + st.markdown(user_question) + + documents = retrieve_memories(vector_store, user_question) + + if documents: + render_memory_card(documents) + simple_answer = get_simple_memory_answer(user_question, documents) + + if simple_answer: + answer = simple_answer + else: + context = build_context(documents) + answer = generate_answer(llm, user_question, context) + else: + answer = FALLBACK_MSG + + with st.chat_message("assistant"): + st.markdown(answer) + + st.session_state.messages.append({ + "role": "assistant", + "content": answer + }) diff --git a/offline-chatbot/ingest.py b/offline-chatbot/ingest.py new file mode 100755 index 0000000..7b9d369 --- /dev/null +++ b/offline-chatbot/ingest.py @@ -0,0 +1,125 @@ +import os +import re +import uuid +from pathlib import Path + +from langchain_chroma import Chroma +from langchain_community.document_loaders import PyPDFLoader, TextLoader +from langchain_community.embeddings import HuggingFaceEmbeddings +from langchain_core.documents import Document +from langchain_text_splitters import RecursiveCharacterTextSplitter + + +# ========================================================= +# PROJECT PATHS +# ========================================================= + +BASE_DIR = str(Path(__file__).resolve().parent) + +DATA_DIR = os.path.join(BASE_DIR, "memory_data") +CHROMA_DIR = os.path.join(DATA_DIR, "chroma_memory") +UPLOAD_DIR = os.path.join(BASE_DIR, "caregiver_documents") + +for folder in [DATA_DIR, CHROMA_DIR, UPLOAD_DIR]: + os.makedirs(folder, exist_ok=True) + + +def clean_text(text): + text = str(text).replace("\r", "\n") + text = re.sub(r"-+\s*PAGE\s*\d+\s*-+", " ", text, flags=re.IGNORECASE) + text = re.sub(r"[━─═―_]{3,}", "\n", text) + text = re.sub(r"\n{3,}", "\n\n", text) + text = re.sub(r"[ \t]{2,}", " ", text) + return text.strip() + + +def load_file(path): + extension = os.path.splitext(path)[1].lower() + + if extension == ".pdf": + return PyPDFLoader(path).load() + + if extension == ".txt": + return TextLoader(path, encoding="utf-8").load() + + return [] + + +def main(): + embeddings = HuggingFaceEmbeddings( + model_name="all-MiniLM-L6-v2" + ) + + vector_store = Chroma( + persist_directory=CHROMA_DIR, + embedding_function=embeddings + ) + + splitter = RecursiveCharacterTextSplitter( + chunk_size=900, + chunk_overlap=150, + separators=["\n\n", "\n", ". ", " ", ""] + ) + + supported_extensions = {".pdf", ".txt"} + + files = [ + filename + for filename in os.listdir(UPLOAD_DIR) + if os.path.splitext(filename)[1].lower() in supported_extensions + ] + + if not files: + print("No PDF/TXT files found in caregiver_documents.") + return + + total_chunks = 0 + + for filename in files: + file_path = os.path.join(UPLOAD_DIR, filename) + document_id = uuid.uuid4().hex + + print(f"Ingesting: {filename}") + + raw_docs = load_file(file_path) + + full_text = clean_text( + "\n".join( + document.page_content + for document in raw_docs + if document.page_content + ) + ) + + if not full_text: + print(f"Skipped empty file: {filename}") + continue + + base_document = Document( + page_content=full_text, + metadata={ + "source": "batch_ingest", + "filename": filename, + "document_id": document_id, + "type": "caregiver_document" + } + ) + + chunks = splitter.split_documents([base_document]) + + ids = [ + f"batch_document_{document_id}_chunk_{index}" + for index in range(len(chunks)) + ] + + if chunks: + vector_store.add_documents(chunks, ids=ids) + total_chunks += len(chunks) + + print(f"Indexed {len(chunks)} chunk(s).") + + print(f"Done. Total chunks indexed: {total_chunks}") + + +if __name__ == "__main__": + main() diff --git a/offline-chatbot/requirements.txt b/offline-chatbot/requirements.txt new file mode 100755 index 0000000..6f7bcd4 --- /dev/null +++ b/offline-chatbot/requirements.txt @@ -0,0 +1,15 @@ +streamlit>=1.35.0 +chromadb>=0.5.0 +langchain>=0.2.0 +langchain-community>=0.2.0 +langchain-chroma>=0.1.0 +langchain-text-splitters>=0.2.0 +sentence-transformers>=2.7.0 +huggingface-hub>=0.23.0 +transformers>=4.41.0 +torch>=2.2.0 +pypdf>=4.2.0 +pillow>=10.3.0 +python-dotenv>=1.0.1 +numpy>=1.26.0 +pandas>=2.2.0 diff --git a/offline-chatbot/user.png b/offline-chatbot/user.png new file mode 100755 index 0000000..6bb4e24 Binary files /dev/null and b/offline-chatbot/user.png differ diff --git a/rag-service/data/patient_profile.json b/patient_profile.json similarity index 100% rename from rag-service/data/patient_profile.json rename to patient_profile.json diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..cb933ad --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +# Tool configuration only. This repository is a set of standalone services, not +# an installable package, so there is deliberately no [project] table. + +[tool.ruff] +line-length = 120 +target-version = "py310" + +# Backend modules import each other flatly (`import store`, `from models import ...`) +# because each service runs with its own directory as the working directory. Listing +# it here is what lets the import sorter tell those apart from third-party packages. +src = [".", "backend"] + +extend-exclude = [ + ".venv", + # Neither of these is lint-clean yet, and neither is exercised by CI. Removing + # them from this list is a good first contribution. + "offline-chatbot", # standalone Streamlit prototype, unrelated to the pipeline + "vision_service", # heavy CV stack; needs a camera to run, so CI skips it too +] + +[tool.ruff.lint] +# Default rules (pycodestyle errors + pyflakes) plus import sorting and a few +# bug-catchers. Kept deliberately small: the point is to catch real mistakes, +# not to enforce a house style on a prototype. +select = ["E4", "E7", "E9", "F", "I", "B", "UP"] +ignore = [ + "B008", # FastAPI's Depends()/Query() defaults are function calls by design +] + +[tool.ruff.lint.isort] +known-first-party = [ + "gemini_client", "main", "mock_llm", "mock_vision", "models", + "patient_memory", "perception", "store", +] + +[tool.ruff.lint.per-file-ignores] +# Tests must set MOCK_LLM and sys.path before importing any backend module +# (gemini_client exits at import time without a key), so imports come after code. +"backend/tests/*" = ["E402"] + +[tool.pytest.ini_options] +testpaths = ["backend/tests"] +addopts = "-q" diff --git a/rag-service/.env.example b/rag-service/.env.example deleted file mode 100644 index 4bde800..0000000 --- a/rag-service/.env.example +++ /dev/null @@ -1,3 +0,0 @@ -GEMINI_API_KEY=replace_me -VISION_WS_URL=ws://localhost:8000/ws -FRONTEND_API_URL=http://localhost:8080/api/cue diff --git a/rag-service/README.md b/rag-service/README.md deleted file mode 100644 index 9445405..0000000 --- a/rag-service/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# Dementia Memory Assistant - -This completes the pipeline: - -```text -Webcam / Photo - ↓ -Vision Service → JSON { person, objects } - ↓ -LLM / RAG → memory card text + voice answer - ↓ -Frontend → shows card, plays audio -``` - -## Files - -- `mock_vision.py` — local simulated vision websocket service. -- `perception_service.py` — consumes vision frames, deduplicates scene changes, calls Gemini, and posts cues to the frontend API. -- `app/main.py` — FastAPI app with frontend websocket broadcasting and caregiver-analysis endpoint. -- `app/models.py` — shared Pydantic schemas. -- `app/patient_memory.py` — simple JSON-backed RAG placeholder. -- `data/patient_profile.json` — editable patient profile and family memory. -- `static/*` — browser frontend. It displays the card and uses Web Speech API for audio playback. - -## Setup - -```bash -python -m venv .venv -source .venv/bin/activate -pip install -r requirements.txt -cp .env.example .env -# edit .env and set GEMINI_API_KEY -``` - -## Run in three terminals - -Terminal 1: -```bash -python mock_vision.py -``` - -Terminal 2: -```bash -uvicorn app.main:app --host 0.0.0.0 --port 8080 -``` - -Terminal 3: -```bash -python perception_service.py -``` - -Open: - -```text -http://localhost:8080 -``` - -## Real camera integration later - -Replace `mock_vision.py` with a real service that emits the same JSON shape: - -```json -{ - "person": { - "recognized": true, - "name": "Sarah", - "relationship": "Daughter", - "note": "Visits on weekends" - }, - "objects": [{ "label": "Medicine Bottle" }], - "timestamp": 1710000000 -} -``` diff --git a/rag-service/app/frontend_hub.py b/rag-service/app/frontend_hub.py deleted file mode 100644 index c77e492..0000000 --- a/rag-service/app/frontend_hub.py +++ /dev/null @@ -1,28 +0,0 @@ -import json -from fastapi import WebSocket - - -class FrontendHub: - def __init__(self): - self.connections: set[WebSocket] = set() - self.latest_payload: dict | None = None - - async def connect(self, websocket: WebSocket): - await websocket.accept() - self.connections.add(websocket) - if self.latest_payload: - await websocket.send_json(self.latest_payload) - - def disconnect(self, websocket: WebSocket): - self.connections.discard(websocket) - - async def broadcast(self, payload: dict): - self.latest_payload = payload - stale: list[WebSocket] = [] - for ws in self.connections: - try: - await ws.send_text(json.dumps(payload)) - except Exception: - stale.append(ws) - for ws in stale: - self.disconnect(ws) diff --git a/rag-service/app/main.py b/rag-service/app/main.py deleted file mode 100644 index 54d5334..0000000 --- a/rag-service/app/main.py +++ /dev/null @@ -1,110 +0,0 @@ -import os -from pathlib import Path -from dotenv import load_dotenv -from fastapi import FastAPI, WebSocket, WebSocketDisconnect -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import FileResponse -from fastapi.staticfiles import StaticFiles -from google import genai -from google.genai import types - -from app.frontend_hub import FrontendHub -from app.models import BehavioralAnalysis, CaregiverInput, MemoryCardOutput, RAGChatRequest, RAGChatResponse -from app.rag_chatbot import RAGMemoryChatbot - -load_dotenv() - -app = FastAPI(title="Dementia Memory Assistant") -hub = FrontendHub() - -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - -STATIC_DIR = Path(__file__).resolve().parent.parent / "static" -app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static") - - -@app.get("/") -async def home(): - return FileResponse(STATIC_DIR / "index.html") - - -@app.get("/health") -async def health(): - return {"ok": True} - - -@app.websocket("/ws/frontend") -async def frontend_socket(websocket: WebSocket): - await hub.connect(websocket) - try: - while True: - await websocket.receive_text() - except WebSocketDisconnect: - hub.disconnect(websocket) - - -@app.post("/api/cue") -async def receive_memory_cue(cue: MemoryCardOutput): - payload = cue.model_dump() - await hub.broadcast(payload) - return {"ok": True, "broadcast": payload} - - -@app.post("/api/rag/orientation", response_model=RAGChatResponse) -async def rag_orientation_chat(payload: RAGChatRequest): - if not os.environ.get("GEMINI_API_KEY"): - fallback = MemoryCardOutput( - card_title="Configuration needed", - card_body="The RAG chatbot needs a Gemini API key before it can generate cards.", - voice_guidance="The assistant is not connected yet. Please ask the caregiver to check the setup.", - ) - return RAGChatResponse( - output=fallback, - debug={ - "retrieved_context": "GEMINI_API_KEY is missing, so retrieval/LLM generation did not run.", - "llm_input": payload.message, - }, - ) - - chatbot = RAGMemoryChatbot() - result = chatbot.generate_structured_output(payload) - await hub.broadcast(result.output.model_dump()) - return result - - -@app.post("/api/caregiver/analyze", response_model=BehavioralAnalysis) -async def analyze_caregiver_input(payload: CaregiverInput): - if not os.environ.get("GEMINI_API_KEY"): - return BehavioralAnalysis( - category="Configuration Error", - observed_triggers=["Missing GEMINI_API_KEY"], - clinical_rationale="The caregiver analysis model cannot run until the API key is configured.", - actionable_intervention="Set GEMINI_API_KEY in your environment or .env file, then restart the server.", - is_crisis=False, - ) - - client = genai.Client() - patient_profile = ( - "Patient: Arthur. Diagnosed with Alzheimer's. Needs calm, short, non-clinical reassurance." - ) - response = client.models.generate_content( - model="gemini-2.5-flash", - contents=f"Context: {patient_profile}\nInput: {payload.message}", - config=types.GenerateContentConfig( - system_instruction=( - "You are a clinical expert system in dementia caregiving. Analyze the user's input. " - "Prioritize de-escalation, behavioral redirection, and validating caregiver stress. " - "Never suggest medical prescriptions or changing drug dosages." - ), - response_mime_type="application/json", - response_schema=BehavioralAnalysis, - temperature=0.1, - ), - ) - return response.parsed diff --git a/rag-service/app/models.py b/rag-service/app/models.py deleted file mode 100644 index d78e29b..0000000 --- a/rag-service/app/models.py +++ /dev/null @@ -1,53 +0,0 @@ -from typing import List -from pydantic import BaseModel, Field - - -class PersonPayload(BaseModel): - recognized: bool = False - name: str = "Unknown" - relationship: str = "" - note: str = "" - - -class ObjectPayload(BaseModel): - label: str - - -class VisionFrame(BaseModel): - person: PersonPayload = Field(default_factory=PersonPayload) - objects: List[ObjectPayload] = Field(default_factory=list) - timestamp: int | None = None - - -class MemoryCardOutput(BaseModel): - card_title: str = Field(description="Large, simple title text for the frontend display.") - card_body: str = Field(description="Max 2 short sentences reinforcing context.") - voice_guidance: str = Field(description="Text to speak to the patient.") - - -class CaregiverInput(BaseModel): - message: str - - -class RAGChatRequest(BaseModel): - message: str = Field(description="Caregiver/user situation or question to ground with patient memory before calling the LLM.") - person: PersonPayload | None = None - objects: List[ObjectPayload] = Field(default_factory=list) - - -class RAGDebugContext(BaseModel): - retrieved_context: str - llm_input: str - - -class RAGChatResponse(BaseModel): - output: MemoryCardOutput - debug: RAGDebugContext - - -class BehavioralAnalysis(BaseModel): - category: str - observed_triggers: List[str] - clinical_rationale: str - actionable_intervention: str - is_crisis: bool diff --git a/rag-service/app/rag_chatbot.py b/rag-service/app/rag_chatbot.py deleted file mode 100644 index 2441673..0000000 --- a/rag-service/app/rag_chatbot.py +++ /dev/null @@ -1,58 +0,0 @@ -from google import genai -from google.genai import types - -from app.models import MemoryCardOutput, RAGChatRequest, RAGChatResponse, RAGDebugContext -from app.patient_memory import PatientMemoryStore - - -class RAGMemoryChatbot: - """RAG pipeline: user/vision info -> retrieve patient memory -> LLM -> structured output.""" - - def __init__(self, memory_store: PatientMemoryStore | None = None): - self.client = genai.Client() - self.model = "gemini-2.5-flash" - self.memory = memory_store or PatientMemoryStore() - - def _build_query(self, request: RAGChatRequest) -> str: - person_text = "" - if request.person: - person_text = ( - f"Person: {request.person.name} {request.person.relationship} " - f"{request.person.note} recognized={request.person.recognized}." - ) - object_text = "Objects: " + ", ".join(obj.label for obj in request.objects) if request.objects else "" - return f"{request.message}\n{person_text}\n{object_text}".strip() - - def generate_structured_output(self, request: RAGChatRequest) -> RAGChatResponse: - retrieval_query = self._build_query(request) - retrieved_context = self.memory.retrieve_text(retrieval_query, top_k=5) - - llm_input = ( - "Retrieved patient memory context:\n" - f"{retrieved_context}\n\n" - "Live/user-provided situation:\n" - f"{retrieval_query}" - ) - - response = self.client.models.generate_content( - model=self.model, - contents=llm_input, - config=types.GenerateContentConfig( - system_instruction=( - "You are the structured-output generation step in a dementia memory assistant. " - "Use ONLY the retrieved patient memory context and live situation. " - "Return a MemoryCardOutput JSON object for the frontend. " - "Keep card text very short, gentle, reassuring, and non-clinical. " - "The voice guidance must speak directly to the patient. " - "Never mention RAG, retrieval, JSON, camera feed, analysis, or clinical labels." - ), - response_mime_type="application/json", - response_schema=MemoryCardOutput, - temperature=0.2, - ), - ) - - return RAGChatResponse( - output=response.parsed, - debug=RAGDebugContext(retrieved_context=retrieved_context, llm_input=llm_input), - ) diff --git a/rag-service/perception_service.py b/rag-service/perception_service.py deleted file mode 100644 index e2aaaed..0000000 --- a/rag-service/perception_service.py +++ /dev/null @@ -1,90 +0,0 @@ -import os -import json -import asyncio -from typing import Optional - -import httpx -import websockets -from dotenv import load_dotenv -from google import genai -from google.genai import types - -from app.models import MemoryCardOutput -from app.patient_memory import PatientMemoryStore - -load_dotenv() - - -class PerceptionStateTracker: - def __init__(self): - self.client = genai.Client() - self.model = "gemini-2.5-flash" - self.memory = PatientMemoryStore() - self.current_person_name: Optional[str] = None - self.current_objects: list[str] = [] - self.frontend_api_url = os.getenv("FRONTEND_API_URL", "http://localhost:8080/api/cue") - - def _should_trigger_llm(self, new_person: dict, new_objects: list) -> bool: - new_name = new_person["name"] if new_person.get("recognized") else "Unknown" - new_obj_labels = sorted([obj["label"] for obj in new_objects]) - if new_name != self.current_person_name or new_obj_labels != self.current_objects: - self.current_person_name = new_name - self.current_objects = new_obj_labels - return True - return False - - async def generate_orientation_cue(self, person: dict, objects: list) -> MemoryCardOutput: - person = self.memory.enrich_person(person) - if person.get("recognized"): - vision_context = f"Person detected: {person['name']} ({person.get('relationship', '')}). Note: {person.get('note', '')}.\n" - else: - vision_context = "Person detected: Unrecognized face or no familiar person present.\n" - - if objects: - vision_context += "Objects visible: " + ", ".join(o["label"] for o in objects) - - response = self.client.models.generate_content( - model=self.model, - contents=f"Patient Profile / Memory Context: {self.memory.profile_text()}\nLive Camera Feed Data:\n{vision_context}", - config=types.GenerateContentConfig( - system_instruction=( - "You are a compassionate, real-time memory assistant for a person with dementia. " - "Output a clear visual memory card and a spoken audio script. " - "Keep the card extremely simple and reassuring. Voice guidance must be calm, slow, direct, and non-clinical. " - "Never say 'camera feed', 'detected', or 'analysis' to the patient." - ), - response_mime_type="application/json", - response_schema=MemoryCardOutput, - temperature=0.3, - ), - ) - return response.parsed - - async def send_to_frontend(self, output: MemoryCardOutput): - async with httpx.AsyncClient(timeout=10) as client: - await client.post(self.frontend_api_url, json=output.model_dump()) - - async def listen_and_process(self): - uri = os.getenv("VISION_WS_URL", "ws://localhost:8000/ws") - print(f"Connecting to live vision websocket stream at {uri}...") - async with websockets.connect(uri) as ws: - async for msg in ws: - try: - data = json.loads(msg) - person = data.get("person", {"recognized": False, "name": "Unknown", "relationship": "", "note": ""}) - objects = data.get("objects", []) - - if self._should_trigger_llm(person, objects): - print(f"\n[Scene Shift Detected] timestamp={data.get('timestamp')}") - output = await self.generate_orientation_cue(person, objects) - print(json.dumps(output.model_dump(), indent=2)) - await self.send_to_frontend(output) - except Exception as exc: - print(f"Error handling frame: {exc}") - - -if __name__ == "__main__": - if not os.environ.get("GEMINI_API_KEY"): - print("Set GEMINI_API_KEY before running.") - raise SystemExit(1) - asyncio.run(PerceptionStateTracker().listen_and_process()) diff --git a/rag-service/run_all.sh b/rag-service/run_all.sh deleted file mode 100755 index f0604b2..0000000 --- a/rag-service/run_all.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -python mock_vision.py & -VISION_PID=$! -uvicorn app.main:app --host 0.0.0.0 --port 8080 & -API_PID=$! -sleep 2 -python perception_service.py - -trap 'kill $VISION_PID $API_PID 2>/dev/null || true' EXIT diff --git a/rag-service/static/app.js b/rag-service/static/app.js deleted file mode 100644 index 843db3a..0000000 --- a/rag-service/static/app.js +++ /dev/null @@ -1,160 +0,0 @@ -let lastVoiceText = ""; -let reconnectTimer = null; - -const els = { - status: document.getElementById("connection-status"), - title: document.getElementById("card-title"), - body: document.getElementById("card-body"), - voice: document.getElementById("voice-text"), - updated: document.getElementById("last-updated"), - avatar: document.getElementById("avatar-initial"), - replay: document.getElementById("replay"), - demo: document.getElementById("demo-cue"), - caregiverInput: document.getElementById("caregiver-input"), - analyze: document.getElementById("analyze"), - analysisCard: document.getElementById("analysis-card"), - ragInput: document.getElementById("rag-input"), - ragGenerate: document.getElementById("rag-generate"), - retrievalCard: document.getElementById("retrieval-card"), -}; - -function setStatus(text, state) { - els.status.className = `status-pill ${state || ""}`.trim(); - els.status.lastChild.textContent = ` ${text}`; -} - -function speak(text) { - lastVoiceText = text || ""; - if (!lastVoiceText || !("speechSynthesis" in window)) return; - - window.speechSynthesis.cancel(); - const utterance = new SpeechSynthesisUtterance(lastVoiceText); - utterance.rate = 0.82; - utterance.pitch = 0.95; - utterance.volume = 1; - window.speechSynthesis.speak(utterance); -} - -function initials(title) { - const clean = (title || "").replace(/[^a-zA-Z\s]/g, "").trim(); - if (!clean) return "♡"; - return clean.split(/\s+/).slice(0, 2).map(word => word[0]).join("").toUpperCase(); -} - -function applyCue(cue) { - const title = cue.card_title || "A gentle reminder"; - const body = cue.card_body || "You are safe. Take your time."; - const voice = cue.voice_guidance || body; - - els.title.textContent = title; - els.body.textContent = body; - els.voice.textContent = voice; - els.avatar.textContent = initials(title); - els.updated.textContent = `Updated ${new Date().toLocaleTimeString([], { hour: "numeric", minute: "2-digit" })}`; - - speak(voice); -} - -function connectFrontendSocket() { - const protocol = location.protocol === "https:" ? "wss" : "ws"; - const ws = new WebSocket(`${protocol}://${location.host}/ws/frontend`); - - ws.onopen = () => setStatus("Live", "connected"); - ws.onmessage = (event) => applyCue(JSON.parse(event.data)); - ws.onerror = () => setStatus("Connection issue", "disconnected"); - ws.onclose = () => { - setStatus("Reconnecting", "disconnected"); - clearTimeout(reconnectTimer); - reconnectTimer = setTimeout(connectFrontendSocket, 1500); - }; -} - - -function renderRetrieval(result) { - const context = result?.debug?.retrieved_context || "No retrieved context returned."; - const output = result?.output; - els.retrievalCard.className = "retrieval-card"; - els.retrievalCard.innerHTML = ` -

Retrieved context sent to the LLM

- 
${escapeHtml(context)}
- `; - if (output) applyCue(output); -} - -function renderAnalysis(result) { - const triggers = (result.observed_triggers || []).map(item => `
  • ${escapeHtml(item)}
    • `).join(""); - els.analysisCard.className = "analysis-card"; - els.analysisCard.innerHTML = ` -

    ${escapeHtml(result.category || "Caregiver guidance")}

    - ${result.is_crisis ? '

    Crisis risk detected. Prioritize immediate safety.

    ' : ""} -

    Rationale: ${escapeHtml(result.clinical_rationale || "No rationale returned.")}

    -

    Try this: ${escapeHtml(result.actionable_intervention || "No intervention returned.")}

    - ${triggers ? `

    Observed triggers:

      ${triggers}
    ` : ""} - `; -} - -function escapeHtml(value) { - return String(value) - .replaceAll("&", "&") - .replaceAll("<", "<") - .replaceAll(">", ">") - .replaceAll('"', """) - .replaceAll("'", "'"); -} - -els.replay.addEventListener("click", () => speak(lastVoiceText)); - -els.demo.addEventListener("click", async () => { - const cue = { - card_title: "Sarah, your daughter", - card_body: "Sarah is here with you. She visits often and cares about you very much.", - voice_guidance: "Hi Arthur, Sarah is here with you. You are safe, and she is happy to see you." - }; - applyCue(cue); - await fetch("/api/cue", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(cue) - }).catch(() => {}); -}); - - -els.ragGenerate.addEventListener("click", async () => { - const message = els.ragInput.value.trim(); - if (!message) return; - - els.retrievalCard.className = "retrieval-card"; - els.retrievalCard.innerHTML = "

    Retrieving patient memory and generating structured output...

    "; - - try { - const response = await fetch("/api/rag/orientation", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ message }) - }); - renderRetrieval(await response.json()); - } catch (error) { - els.retrievalCard.innerHTML = "

    Could not run RAG right now. Make sure the backend is running.

    "; - } -}); - -els.analyze.addEventListener("click", async () => { - const message = els.caregiverInput.value.trim(); - if (!message) return; - - els.analysisCard.className = "analysis-card"; - els.analysisCard.innerHTML = "

    Analyzing...

    "; - - try { - const response = await fetch("/api/caregiver/analyze", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ message }) - }); - renderAnalysis(await response.json()); - } catch (error) { - els.analysisCard.innerHTML = "

    Could not analyze right now. Make sure the backend is running.

    "; - } -}); - -connectFrontendSocket(); diff --git a/rag-service/static/index.html b/rag-service/static/index.html deleted file mode 100644 index a86d4e3..0000000 --- a/rag-service/static/index.html +++ /dev/null @@ -1,113 +0,0 @@ - - - - - - Memory Assistant - - - - - - -
    -
    - -
    -
    -
    -

    Dementia Memory Assistant

    -

    Gentle orientation, in real time.

    -
    -
    - - Connecting -
    -
    - -
    -
    -
    - Memory Card - Waiting for camera -
    - - - -

    Waiting for camera...

    -

    When the scene changes, a calm reminder will appear here.

    - -
    -
    -

    Voice guidance

    -

    No spoken message yet.

    -
    - -
    -
    - - -
    - - -
    -
    -
    -

    RAG Chatbot

    -

    Turn patient memory into a structured card

    -
    - Retrieval → LLM → JSON -
    - -
    -
    - - -
    -
    -

    Retrieved patient context will appear here before the LLM output.

    -
    -
    -
    - -
    -
    -
    -

    Caregiver Support

    -

    Need help with a situation?

    -
    - Non-medical guidance -
    - -
    - - -
    - -
    -

    The analysis result will appear here.

    -
    -
    -
    - - - - diff --git a/rag-service/static/style.css b/rag-service/static/style.css deleted file mode 100644 index c5b6099..0000000 --- a/rag-service/static/style.css +++ /dev/null @@ -1,401 +0,0 @@ -:root { - --bg: #f6f3ee; - --card: rgba(255, 255, 255, 0.82); - --card-solid: #ffffff; - --text: #1f2933; - --muted: #6b7280; - --line: rgba(31, 41, 51, 0.12); - --accent: #5865f2; - --accent-dark: #414ac4; - --warm: #f59e0b; - --safe: #16a34a; - --danger: #dc2626; - --shadow: 0 24px 70px rgba(31, 41, 51, 0.14); - --radius-xl: 32px; - --radius-lg: 22px; - --radius-md: 16px; -} - -* { box-sizing: border-box; } - -body { - margin: 0; - min-height: 100vh; - font-family: Inter, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; - color: var(--text); - background: - radial-gradient(circle at top left, rgba(88, 101, 242, 0.18), transparent 36rem), - radial-gradient(circle at bottom right, rgba(245, 158, 11, 0.16), transparent 34rem), - var(--bg); - overflow-x: hidden; -} - -button, textarea { font: inherit; } -button { cursor: pointer; } - -.ambient { - position: fixed; - border-radius: 999px; - filter: blur(4px); - opacity: 0.28; - pointer-events: none; -} -.ambient-one { - width: 18rem; - height: 18rem; - right: -6rem; - top: 7rem; - background: #8b5cf6; -} -.ambient-two { - width: 14rem; - height: 14rem; - left: -4rem; - bottom: 7rem; - background: #fbbf24; -} - -.app-shell { - width: min(1120px, calc(100% - 32px)); - margin: 0 auto; - padding: 38px 0 56px; -} - -.topbar { - display: flex; - justify-content: space-between; - align-items: flex-start; - gap: 24px; - margin-bottom: 28px; -} - -.kicker { - margin: 0 0 8px; - color: var(--accent); - font-size: 0.78rem; - font-weight: 800; - letter-spacing: 0.12em; - text-transform: uppercase; -} - -.topbar h1 { - max-width: 720px; - margin: 0; - font-size: clamp(2.2rem, 6vw, 4.8rem); - line-height: 0.95; - letter-spacing: -0.07em; -} - -.status-pill { - display: inline-flex; - align-items: center; - gap: 9px; - padding: 12px 16px; - border: 1px solid var(--line); - border-radius: 999px; - background: rgba(255, 255, 255, 0.72); - color: var(--muted); - font-weight: 700; - white-space: nowrap; - box-shadow: 0 12px 35px rgba(31, 41, 51, 0.08); -} - -.status-dot { - width: 10px; - height: 10px; - border-radius: 999px; - background: var(--warm); - box-shadow: 0 0 0 5px rgba(245, 158, 11, 0.16); -} -.status-pill.connected .status-dot { - background: var(--safe); - box-shadow: 0 0 0 5px rgba(22, 163, 74, 0.14); -} -.status-pill.disconnected .status-dot { - background: var(--danger); - box-shadow: 0 0 0 5px rgba(220, 38, 38, 0.12); -} - -.hero-grid { - display: grid; - grid-template-columns: minmax(0, 1fr) 340px; - gap: 22px; - align-items: stretch; -} - -.memory-card, .caregiver-panel, .side-panel { - border: 1px solid rgba(255, 255, 255, 0.7); - background: var(--card); - box-shadow: var(--shadow); - backdrop-filter: blur(18px); -} - -.memory-card { - min-height: 560px; - padding: clamp(24px, 4vw, 44px); - border-radius: var(--radius-xl); - display: flex; - flex-direction: column; - justify-content: space-between; -} - -.card-topline, .panel-heading { - display: flex; - justify-content: space-between; - align-items: center; - gap: 14px; -} - -.badge, .soft-tag { - display: inline-flex; - padding: 8px 12px; - border-radius: 999px; - background: rgba(88, 101, 242, 0.10); - color: var(--accent-dark); - font-size: 0.78rem; - font-weight: 800; -} - -#last-updated { - color: var(--muted); - font-size: 0.9rem; - font-weight: 700; -} - -.person-avatar { - width: clamp(108px, 18vw, 160px); - height: clamp(108px, 18vw, 160px); - margin: 32px auto 18px; - border-radius: 44px; - display: grid; - place-items: center; - background: - linear-gradient(145deg, rgba(88, 101, 242, 0.16), rgba(245, 158, 11, 0.18)), - #fff; - color: var(--accent); - font-size: clamp(3rem, 7vw, 5.4rem); - font-weight: 800; - box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.9), 0 22px 50px rgba(88, 101, 242, 0.16); -} - -.memory-card h2 { - margin: 0; - text-align: center; - font-size: clamp(2.3rem, 7vw, 5.6rem); - line-height: 0.94; - letter-spacing: -0.075em; -} - -#card-body { - max-width: 660px; - margin: 22px auto 34px; - color: #4b5563; - text-align: center; - font-size: clamp(1.18rem, 2vw, 1.65rem); - line-height: 1.45; -} - -.voice-box { - display: flex; - align-items: center; - justify-content: space-between; - gap: 18px; - padding: 20px; - border: 1px solid var(--line); - border-radius: 24px; - background: rgba(255, 255, 255, 0.72); -} -.voice-label { - margin: 0 0 6px; - color: var(--muted); - font-size: 0.8rem; - font-weight: 800; - letter-spacing: 0.08em; - text-transform: uppercase; -} -#voice-text { - margin: 0; - font-weight: 650; - line-height: 1.45; -} -.round-button { - flex: 0 0 auto; - width: 56px; - height: 56px; - border: 0; - border-radius: 999px; - background: var(--accent); - color: #fff; - font-weight: 900; - box-shadow: 0 14px 28px rgba(88, 101, 242, 0.28); -} - -.side-panel { - padding: 22px; - border-radius: var(--radius-xl); - display: flex; - flex-direction: column; - gap: 16px; -} -.mini-card { - display: flex; - gap: 14px; - padding: 18px; - border: 1px solid var(--line); - border-radius: var(--radius-lg); - background: rgba(255, 255, 255, 0.62); -} -.mini-icon { - font-size: 1.6rem; -} -.mini-card h3 { - margin: 0 0 6px; - font-size: 1rem; -} -.mini-card p { - margin: 0; - color: var(--muted); - line-height: 1.45; -} -.secondary-button, #analyze { - border: 0; - border-radius: 18px; - background: #1f2937; - color: white; - font-weight: 800; - padding: 15px 18px; - box-shadow: 0 18px 36px rgba(31, 41, 55, 0.18); -} -.secondary-button { - margin-top: auto; - width: 100%; -} - -.caregiver-panel { - margin-top: 22px; - padding: clamp(22px, 4vw, 32px); - border-radius: var(--radius-xl); -} -.panel-heading h2 { - margin: 0; - font-size: clamp(1.5rem, 3vw, 2.4rem); - letter-spacing: -0.04em; -} -.input-row { - display: grid; - grid-template-columns: 1fr 150px; - gap: 14px; - margin-top: 22px; -} -textarea { - width: 100%; - min-height: 130px; - resize: vertical; - border: 1px solid var(--line); - border-radius: 20px; - padding: 18px; - background: rgba(255, 255, 255, 0.82); - color: var(--text); - outline: none; - line-height: 1.5; -} -textarea:focus { - border-color: rgba(88, 101, 242, 0.45); - box-shadow: 0 0 0 5px rgba(88, 101, 242, 0.10); -} -#analyze { align-self: stretch; } -.analysis-card { - margin-top: 16px; - padding: 20px; - border: 1px solid var(--line); - border-radius: 20px; - background: rgba(255, 255, 255, 0.64); -} -.analysis-card.is-empty { - color: var(--muted); -} -.analysis-card h3 { - margin: 0 0 12px; - font-size: 1.1rem; -} -.analysis-card p { - margin: 8px 0; - line-height: 1.5; -} -.analysis-card ul { - margin: 8px 0 0; - padding-left: 20px; -} -.crisis { - color: var(--danger); - font-weight: 900; -} -.empty-state { margin: 0; } - -@media (max-width: 900px) { - .topbar, .card-topline, .panel-heading { align-items: flex-start; } - .topbar, .hero-grid, .input-row { grid-template-columns: 1fr; } - .hero-grid { display: grid; } - .side-panel { order: -1; } - .topbar, .card-topline, .panel-heading { flex-direction: column; } - #analyze { min-height: 58px; } -} - -@media (max-width: 560px) { - .app-shell { width: min(100% - 20px, 1120px); padding-top: 18px; } - .memory-card { min-height: 500px; } - .voice-box { align-items: flex-start; } -} - -.rag-panel { - margin-top: 28px; - padding: 28px; - border: 1px solid rgba(255, 255, 255, 0.72); - border-radius: 30px; - background: rgba(255, 255, 255, 0.70); - box-shadow: 0 24px 70px rgba(79, 70, 229, 0.10); - backdrop-filter: blur(18px); -} - -.rag-layout { - display: grid; - grid-template-columns: minmax(0, 0.95fr) minmax(0, 1.05fr); - gap: 18px; - align-items: stretch; -} - -.input-row.stacked { - display: flex; - flex-direction: column; -} - -.input-row.stacked textarea { - min-height: 140px; -} - -.retrieval-card { - min-height: 170px; - padding: 18px; - border-radius: 22px; - background: rgba(255, 255, 255, 0.78); - border: 1px solid rgba(148, 163, 184, 0.30); -} - -.retrieval-card h3 { - margin: 0 0 10px; -} - -.retrieval-card pre { - margin: 0; - white-space: pre-wrap; - line-height: 1.5; - color: #334155; - font-family: inherit; - font-size: 0.92rem; -} - -@media (max-width: 850px) { - .rag-layout { - grid-template-columns: 1fr; - } -} diff --git a/vision_service/README.md b/vision_service/README.md index cd616af..2b8c674 100644 --- a/vision_service/README.md +++ b/vision_service/README.md @@ -1,6 +1,6 @@ # Vision Service — Internals -Face recognition and object detection backend. Runs fully offline on a MacBook. +Face recognition and object detection backend. Runs fully offline on macOS, Linux, and Windows — no image, frame, or embedding is ever transmitted anywhere. --- @@ -12,6 +12,10 @@ pip install -r requirements.txt python main.py ``` +Serves on . A webcam is required. The first run downloads the InsightFace and YOLO weights; after that it needs no network at all. + +The bind address defaults to `127.0.0.1` because **no endpoint here is authenticated** — anyone who can reach the port can pull a live camera still from `GET /frame`, enrol a face, or delete a registered person. Override with `VISION_HOST` / `VISION_PORT` only on a network you trust. + --- ## Folder structure @@ -35,7 +39,7 @@ vision_service/ │ └── websocket_handler.py # WS /ws + connection manager ├── static/ │ └── register.html # face registration UI -└── train/ # custom model training (see Fine-Tune.md inside) +└── train/ # custom model training scripts (optional) ``` --- @@ -78,11 +82,15 @@ enable_terminal_log = True # print results to terminal ## Embeddings storage -Stored in `storage/data/embeddings.json`. Plain JSON — human-readable, no database needed. +Stored in `storage/data/embeddings.json`. Plain JSON — human-readable, no database needed. The file is gitignored. Structure: per person, a list of raw embeddings (preserved for fallback matching) and a pre-computed average embedding (used for fast lookup). Average is recomputed from the raw list on every write and on startup — it's never persisted separately. -To wipe all registered faces: `rm storage/data/embeddings.json` +**These embeddings are biometric identifiers.** Register only people who have knowingly agreed, and read [../PRIVACY.md](../PRIVACY.md) before collecting any. There is no encryption at rest, no audit log, and no retention limit. + +Remove one person: `curl -X DELETE http://localhost:8000/people/NAME` + +Wipe all registered faces: `rm storage/data/embeddings.json` — no backup, no recovery. --- diff --git a/vision_service/main.py b/vision_service/main.py index 474ac3b..94b3fae 100644 --- a/vision_service/main.py +++ b/vision_service/main.py @@ -2,11 +2,15 @@ Vision Service — entry point. Run with: - uvicorn main:app --host 0.0.0.0 --port 8000 --reload + uvicorn main:app --host 127.0.0.1 --port 8000 --reload Or via the helper script: python main.py +No endpoint here is authenticated and GET /frame serves live camera stills, so +the default bind is 127.0.0.1. Override with VISION_HOST / VISION_PORT only on a +network you trust. + Toggle UI / terminal logging in config.py: enable_ui = True / False enable_terminal_log = True / False @@ -14,6 +18,7 @@ import asyncio import logging import time +from pathlib import Path from threading import Lock from concurrent.futures import ThreadPoolExecutor from contextlib import asynccontextmanager @@ -344,7 +349,8 @@ async def lifespan(app: FastAPI): app.include_router(rec_router, tags=["Recognition"]) app.include_router(reg_router, tags=["Registration"]) app.include_router(ws_router, tags=["WebSocket"]) -app.mount("/static", StaticFiles(directory="static"), name="static") +# Anchored to this file, not the CWD, so the service starts from any directory. +app.mount("/static", StaticFiles(directory=Path(__file__).parent / "static"), name="static") @app.get("/frame") @@ -356,5 +362,13 @@ async def frame(): if __name__ == "__main__": + import os + import uvicorn - uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True) + + # Defaults to localhost: none of these endpoints are authenticated, and + # /frame serves live camera stills. Set VISION_HOST=0.0.0.0 only on a + # network you trust. + host = os.environ.get("VISION_HOST", "127.0.0.1") + port = int(os.environ.get("VISION_PORT", "8000")) + uvicorn.run("main:app", host=host, port=port, reload=True) diff --git a/vision_service/static/register.html b/vision_service/static/register.html index 467a76b..6c01fe3 100644 --- a/vision_service/static/register.html +++ b/vision_service/static/register.html @@ -33,6 +33,9 @@ .progress { font-size: 0.85rem; color: #888; } .progress span { color: #0f766e; font-weight: 600; } + .consent { font-size: 0.82rem; line-height: 1.5; padding: 0.7rem 0.85rem; border-radius: 10px; background: #fff7ed; border: 1px solid #fed7aa; color: #7c2d12; } + .consent code { background: rgba(124, 45, 18, 0.08); padding: 0 0.25rem; border-radius: 4px; } + button { padding: 0.7rem 1.2rem; border: none; border-radius: 8px; font-size: 0.95rem; font-weight: 600; cursor: pointer; transition: opacity 0.2s; } button:disabled { opacity: 0.4; cursor: not-allowed; } #captureBtn { background: #2563eb; color: #fff; width: 100%; } @@ -86,6 +89,12 @@

    Register New Person

    Photos captured: 0 / 5

    +

    + Consent required. This stores a face embedding — biometric data — on this + machine. Only register someone who has knowingly agreed. Delete anytime from the list below. + See PRIVACY.md in the repository. +

    + @@ -212,22 +221,50 @@

    Registered People

    const data = await res.json(); const people = data.people || []; if (people.length === 0) { - container.innerHTML = 'No one registered yet.'; + container.replaceChildren(emptyNote('No one registered yet.', '#888')); return; } - container.innerHTML = people.map(p => ` -
    -
    - ${p.name} -
    ${p.relationship}${p.note ? ' · ' + p.note : ''} · ${p.num_embeddings} photo(s)
    -
    - -
    `).join(''); + // Built as DOM nodes rather than an innerHTML string: a name is caregiver-entered + // text and must never be parsed as markup or spliced into an inline handler. + container.replaceChildren(...people.map(personCard)); } catch { - container.innerHTML = 'Could not load people list.'; + container.replaceChildren(emptyNote('Could not load people list.', '#f87171')); } } +function personCard(p) { + const card = document.createElement('div'); + card.className = 'person-card'; + + const info = document.createElement('div'); + info.className = 'person-info'; + + const name = document.createElement('strong'); + name.textContent = p.name; + + const rel = document.createElement('div'); + rel.className = 'rel'; + const note = p.note ? ` · ${p.note}` : ''; + rel.textContent = `${p.relationship}${note} · ${p.num_embeddings} photo(s)`; + + info.append(name, rel); + + const del = document.createElement('button'); + del.className = 'del-btn'; + del.textContent = 'Delete'; + del.addEventListener('click', () => deletePerson(p.name)); + + card.append(info, del); + return card; +} + +function emptyNote(text, color) { + const em = document.createElement('em'); + em.style.color = color; + em.textContent = text; + return em; +} + async function deletePerson(name) { if (!confirm(`Delete ${name}?`)) return; const res = await fetch(`${API}/people/${encodeURIComponent(name)}`, { method: 'DELETE' }); diff --git a/vision_service/yolov8n.pt b/vision_service/yolov8n.pt deleted file mode 100644 index 0db4ca4..0000000 Binary files a/vision_service/yolov8n.pt and /dev/null differ