From 6a25eaecbd3c6bdab03889122facf050271c4c56 Mon Sep 17 00:00:00 2001 From: alexeybe1kin <210597588+alexeybe1kin@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:40:00 +0200 Subject: [PATCH 1/2] fix: the classifier matches whole words; one feeling word fires one signal --- CHANGELOG.md | 7 +++++ services/api/app/services/classifier.py | 42 ++++++++++++------------- services/api/tests/test_classifier.py | 27 ++++++++++++++-- 3 files changed, 52 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fe2293..e0d6642 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ a decision with visible consequences. ## Unreleased +- **The classifier matches whole words.** A memory type came from substring hits, so "again" + in "against" made a memory `watch`, "sad" in "Sadie" or "crusade" made it `phase`, and "train" + in "constraint" counted as a domain word. Phrases now match as whole words (a curly apostrophe + counts as a straight one). One feeling word also fired two phase signals, because the two + emotion lists overlap; a phase now has two signals, a feeling and a timeframe, so "sad this + week" is `medium`, not `high`. The unused `NEGATION_WORDS` list is gone. + - **"Don't remember" keeps no words.** A conversation message the owner asked not to keep ("don't remember ...", "не запоминай ...") used to be stored word for word as evidence (only the memory was skipped). Its evidence now holds no text (`normalized_payload` is diff --git a/services/api/app/services/classifier.py b/services/api/app/services/classifier.py index 003f72e..69dc99c 100644 --- a/services/api/app/services/classifier.py +++ b/services/api/app/services/classifier.py @@ -18,6 +18,9 @@ 2 -> medium, 3+ -> high. """ +import re +from functools import cache + # priority order used to break ties between types with equal signal counts TYPE_PRIORITY = ["watch", "fact", "phase"] @@ -61,12 +64,17 @@ def normalize_memory_type(memory_type: str | None) -> str | None: SUPPORT_EMOTIONAL_WORDS = ["sad", "upset", "anxious", "stressed", "overwhelmed", "hurt", "struggling", "hard time"] SUPPORT_TIMEFRAMES = ["today", "right now", "this week"] -# words that would negate an otherwise-matching contradiction signal -NEGATION_WORDS = ["not", "no longer", "stopped", "don't", "never"] + +@cache +def _phrase(phrase: str) -> re.Pattern: + # A whole word or phrase: no letter or digit right before or after it ("again" is not in + # "against", "sad" is not in "Sadie"). Spaces inside a phrase match any run of whitespace. + body = r"\s+".join(re.escape(part) for part in phrase.split()) + return re.compile(rf"(? int: - return sum(1 for p in phrases if p in lower) + return sum(1 for p in phrases if _phrase(p).search(lower)) def _score_preference(lower: str) -> int: @@ -80,9 +88,9 @@ def _score_preference(lower: str) -> int: def _score_identity(lower: str) -> int: - if any(w in lower for w in TEMPORARY_LANGUAGE): + if _count_hits(lower, TEMPORARY_LANGUAGE): return 0 - return sum(1 for p in IDENTITY_PHRASES if p in lower) + return _count_hits(lower, IDENTITY_PHRASES) def _score_humor(lower: str) -> int: @@ -98,28 +106,18 @@ def _score_fact(lower: str) -> int: return _score_preference(lower) + _score_identity(lower) + _score_humor(lower) -def _score_temporary(lower: str) -> int: - signals = 0 - if _count_hits(lower, TEMPORARY_TIMEFRAMES) > 0: - signals += 1 - if _count_hits(lower, EMOTIONAL_LANGUAGE) > 0: - signals += 1 - return signals - - -def _score_support(lower: str) -> int: +def _score_phase(lower: str) -> int: + # Two independent signals: a feeling word, and a word that says it is for now. The two old + # lists of each overlap ("sad", "right now" sit in both), so they are joined, and one word + # fires one signal, not two. signals = 0 - if _count_hits(lower, SUPPORT_EMOTIONAL_WORDS) > 0: + if _count_hits(lower, EMOTIONAL_LANGUAGE + SUPPORT_EMOTIONAL_WORDS) > 0: signals += 1 - if _count_hits(lower, SUPPORT_TIMEFRAMES) > 0: + if _count_hits(lower, TEMPORARY_TIMEFRAMES + SUPPORT_TIMEFRAMES) > 0: signals += 1 return signals -def _score_phase(lower: str) -> int: - return _score_temporary(lower) + _score_support(lower) - - def _score_watch(lower: str, source_type: str) -> int: signals = 0 if _count_hits(lower, HARMFUL_BEHAVIOR_INDICATORS) > 0: @@ -138,7 +136,7 @@ def _confidence_for(signal_count: int) -> str: def classify_memory(text: str, source_type: str = "user") -> dict: - lower = text.lower() + lower = text.lower().replace(chr(0x2019), "'") # a curly apostrophe (typed on a phone) counts as a straight one scores = { "watch": _score_watch(lower, source_type), diff --git a/services/api/tests/test_classifier.py b/services/api/tests/test_classifier.py index e7eff7e..671d23e 100644 --- a/services/api/tests/test_classifier.py +++ b/services/api/tests/test_classifier.py @@ -16,8 +16,10 @@ ("I am a night owl, I'm a developer", "user", "fact", "medium"), ("I always love dark humor, deadpan", "user", "fact", "medium"), ("I am feeling tired today", "user", "phase", "low"), - ("I'm feeling sad and overwhelmed this week", "user", "phase", "high"), - ("I am a person and I am sad today", "user", "phase", "high"), + # A feeling and a timeframe are two signals; one word never fires two (it used to). + ("I'm feeling sad and overwhelmed this week", "user", "phase", "medium"), + ("I am a person and I am sad today", "user", "phase", "medium"), + ("I'm sad", "user", "phase", "low"), ("I keep doing it again, can't stop", "user", "watch", "low"), ("the meeting is at 3", "user", "context", "medium"), ("the meeting is at 3", "soulgate_inferred", "watch", "low"), @@ -75,3 +77,24 @@ def test_legacy_types_map_to_current_ones(): def test_legacy_map_targets_are_current_types(): for _legacy_type, current_type in LEGACY_TYPE_MAP.items(): assert current_type in CURRENT_MEMORY_TYPES + + +@pytest.mark.parametrize( + "text", + [ + "I argued against the plan", # "again" inside "against" + "Sadie trains in the morning", # "sad" inside "Sadie" + "the constraint is the budget", # "train" inside "constraint" + "a crusade for cleaner hallways", # "sad" in "crusade", "always" in "hallways" + ], +) +def test_a_phrase_inside_another_word_does_not_count(text): + assert classify_memory(text, "user") == {"memory_type": "context", "confidence": "medium", + "summary": text} + + +def test_a_curly_apostrophe_counts_as_a_straight_one(): + straight = classify_memory("I can't stop checking my phone", "user") + curly = classify_memory("I can" + chr(0x2019) + "t stop checking my phone", "user") + assert straight == {**curly, "summary": straight["summary"]} + assert straight["memory_type"] == "watch" From 0882ae096201c365bae7a498a14f2812d7ab7d03 Mon Sep 17 00:00:00 2001 From: alexeybe1kin <210597588+alexeybe1kin@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:26:22 +0200 Subject: [PATCH 2/2] changelog: place entry apart from other open PRs --- CHANGELOG.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ade084d..5b00a54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,6 @@ a decision with visible consequences. ## Unreleased -- **The classifier matches whole words.** A memory type came from substring hits, so "again" - in "against" made a memory `watch`, "sad" in "Sadie" or "crusade" made it `phase`, and "train" - in "constraint" counted as a domain word. Phrases now match as whole words (a curly apostrophe - counts as a straight one). One feeling word also fired two phase signals, because the two - emotion lists overlap; a phase now has two signals, a feeling and a timeframe, so "sad this - week" is `medium`, not `high`. The unused `NEGATION_WORDS` list is gone. - - **A lockout stops wrong keys, never the right one.** After five wrong keys from one address, every key from that address was refused for 300 s, so one misconfigured caller behind a proxy or the host network locked out the rest, and failures never expired. Now the key is checked @@ -48,6 +41,13 @@ a decision with visible consequences. limit, the Entities screen (662), is split (its three dialogs moved to `components/EntityModals.jsx`). No behaviour change. +- **The classifier matches whole words.** A memory type came from substring hits, so "again" + in "against" made a memory `watch`, "sad" in "Sadie" or "crusade" made it `phase`, and "train" + in "constraint" counted as a domain word. Phrases now match as whole words (a curly apostrophe + counts as a straight one). One feeling word also fired two phase signals, because the two + emotion lists overlap; a phase now has two signals, a feeling and a timeframe, so "sad this + week" is `medium`, not `high`. The unused `NEGATION_WORDS` list is gone. + - **Merging entities keeps lineage and the graph whole.** The startup dedup and the manual merge (`POST /entity/merge`) now share one routine: lineage links to the removed entity move to the kept one, an edge between the two (a would-be self-loop) or one the kept entity already has