fix: the classifier matches whole words; one feeling word fires one signal

CHANGELOG.md

@@ -41,6 +41,13 @@ a decision with visible consequences.
   limit, the Entities screen (662), is split (its three dialogs moved to
   `components/EntityModals.jsx`). No behaviour change.
 
+- **The classifier matches whole words.** A memory type came from substring hits, so "again"
+  in "against" made a memory `watch`, "sad" in "Sadie" or "crusade" made it `phase`, and "train"
+  in "constraint" counted as a domain word. Phrases now match as whole words (a curly apostrophe
+  counts as a straight one). One feeling word also fired two phase signals, because the two
+  emotion lists overlap; a phase now has two signals, a feeling and a timeframe, so "sad this
+  week" is `medium`, not `high`. The unused `NEGATION_WORDS` list is gone.
+
 - **Merging entities keeps lineage and the graph whole.** The startup dedup and the manual
   merge (`POST /entity/merge`) now share one routine: lineage links to the removed entity move to
   the kept one, an edge between the two (a would-be self-loop) or one the kept entity already has

services/api/app/services/classifier.py

@@ -18,6 +18,9 @@
 2 -> medium, 3+ -> high.
 """
 
+import re
+from functools import cache
+
 # priority order used to break ties between types with equal signal counts
 TYPE_PRIORITY = ["watch", "fact", "phase"]
 
@@ -61,12 +64,17 @@ def normalize_memory_type(memory_type: str | None) -> str | None:
 SUPPORT_EMOTIONAL_WORDS = ["sad", "upset", "anxious", "stressed", "overwhelmed", "hurt", "struggling", "hard time"]
 SUPPORT_TIMEFRAMES = ["today", "right now", "this week"]
 
-# words that would negate an otherwise-matching contradiction signal
-NEGATION_WORDS = ["not", "no longer", "stopped", "don't", "never"]
+
+@cache
+def _phrase(phrase: str) -> re.Pattern:
+    # A whole word or phrase: no letter or digit right before or after it ("again" is not in
+    # "against", "sad" is not in "Sadie"). Spaces inside a phrase match any run of whitespace.
+    body = r"\s+".join(re.escape(part) for part in phrase.split())
+    return re.compile(rf"(?<![^\W_]){body}(?![^\W_])")
 
 
 def _count_hits(lower: str, phrases: list[str]) -> int:
-    return sum(1 for p in phrases if p in lower)
+    return sum(1 for p in phrases if _phrase(p).search(lower))
 
 
 def _score_preference(lower: str) -> int:
@@ -80,9 +88,9 @@ def _score_preference(lower: str) -> int:
 
 
 def _score_identity(lower: str) -> int:
-    if any(w in lower for w in TEMPORARY_LANGUAGE):
+    if _count_hits(lower, TEMPORARY_LANGUAGE):
         return 0
-    return sum(1 for p in IDENTITY_PHRASES if p in lower)
+    return _count_hits(lower, IDENTITY_PHRASES)
 
 
 def _score_humor(lower: str) -> int:
@@ -98,28 +106,18 @@ def _score_fact(lower: str) -> int:
     return _score_preference(lower) + _score_identity(lower) + _score_humor(lower)
 
 
-def _score_temporary(lower: str) -> int:
-    signals = 0
-    if _count_hits(lower, TEMPORARY_TIMEFRAMES) > 0:
-        signals += 1
-    if _count_hits(lower, EMOTIONAL_LANGUAGE) > 0:
-        signals += 1
-    return signals
-
-
-def _score_support(lower: str) -> int:
+def _score_phase(lower: str) -> int:
+    # Two independent signals: a feeling word, and a word that says it is for now. The two old
+    # lists of each overlap ("sad", "right now" sit in both), so they are joined, and one word
+    # fires one signal, not two.
     signals = 0
-    if _count_hits(lower, SUPPORT_EMOTIONAL_WORDS) > 0:
+    if _count_hits(lower, EMOTIONAL_LANGUAGE + SUPPORT_EMOTIONAL_WORDS) > 0:
         signals += 1
-    if _count_hits(lower, SUPPORT_TIMEFRAMES) > 0:
+    if _count_hits(lower, TEMPORARY_TIMEFRAMES + SUPPORT_TIMEFRAMES) > 0:
         signals += 1
     return signals
 
 
-def _score_phase(lower: str) -> int:
-    return _score_temporary(lower) + _score_support(lower)
-
-
 def _score_watch(lower: str, source_type: str) -> int:
     signals = 0
     if _count_hits(lower, HARMFUL_BEHAVIOR_INDICATORS) > 0:
@@ -138,7 +136,7 @@ def _confidence_for(signal_count: int) -> str:
 
 
 def classify_memory(text: str, source_type: str = "user") -> dict:
-    lower = text.lower()
+    lower = text.lower().replace(chr(0x2019), "'")  # a curly apostrophe (typed on a phone) counts as a straight one
 
     scores = {
         "watch": _score_watch(lower, source_type),

services/api/tests/test_classifier.py

@@ -16,8 +16,10 @@
         ("I am a night owl, I'm a developer", "user", "fact", "medium"),
         ("I always love dark humor, deadpan", "user", "fact", "medium"),
         ("I am feeling tired today", "user", "phase", "low"),
-        ("I'm feeling sad and overwhelmed this week", "user", "phase", "high"),
-        ("I am a person and I am sad today", "user", "phase", "high"),
+        # A feeling and a timeframe are two signals; one word never fires two (it used to).
+        ("I'm feeling sad and overwhelmed this week", "user", "phase", "medium"),
+        ("I am a person and I am sad today", "user", "phase", "medium"),
+        ("I'm sad", "user", "phase", "low"),
         ("I keep doing it again, can't stop", "user", "watch", "low"),
         ("the meeting is at 3", "user", "context", "medium"),
         ("the meeting is at 3", "soulgate_inferred", "watch", "low"),
@@ -75,3 +77,24 @@ def test_legacy_types_map_to_current_ones():
 def test_legacy_map_targets_are_current_types():
     for _legacy_type, current_type in LEGACY_TYPE_MAP.items():
         assert current_type in CURRENT_MEMORY_TYPES
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "I argued against the plan",  # "again" inside "against"
+        "Sadie trains in the morning",  # "sad" inside "Sadie"
+        "the constraint is the budget",  # "train" inside "constraint"
+        "a crusade for cleaner hallways",  # "sad" in "crusade", "always" in "hallways"
+    ],
+)
+def test_a_phrase_inside_another_word_does_not_count(text):
+    assert classify_memory(text, "user") == {"memory_type": "context", "confidence": "medium",
+                                             "summary": text}
+
+
+def test_a_curly_apostrophe_counts_as_a_straight_one():
+    straight = classify_memory("I can't stop checking my phone", "user")
+    curly = classify_memory("I can" + chr(0x2019) + "t stop checking my phone", "user")
+    assert straight == {**curly, "summary": straight["summary"]}
+    assert straight["memory_type"] == "watch"