From cf9cb22111b1d0fe227d7b11008f037ac49df227 Mon Sep 17 00:00:00 2001 From: Devin Phillip Davis <354534380+quantum-osiris@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:37:35 -0500 Subject: [PATCH] fix(claims): flag only asserted claims in replies; keep honest rows in training flagged_in_reply() now checks only register entries with explicit `asserts` patterns (currently OSIRIS_ON_QPU), and a sentence that reports a claim as refuted, failed or untrue is not an assertion. Previously, naming a topic (e.g. "retiring constants like theta_lock") flagged the reply and dropped it from training. Ledger dry run (~/.osiris/living/exchanges.jsonl): 921 kept, 1 left out (a test exchange). Full suite: 1803 passed, 2 failed; both failures pre-exist on main (test_exec_policy symlink case; GitHubBridge auth test expects the old osiris-dnalang account). Co-Authored-By: Claude Opus 5.5 --- osiris_cli/claims.py | 48 +++++++++++++++++++++++++++++++++++--------- osiris_cli/train.py | 23 +++++++++++++++++---- tests/test_claims.py | 28 +++++++++++++++++++++++--- tests/test_train.py | 17 ++++++++++++++++ 4 files changed, 100 insertions(+), 16 deletions(-) diff --git a/osiris_cli/claims.py b/osiris_cli/claims.py index 3c111a8f..fac90557 100644 --- a/osiris_cli/claims.py +++ b/osiris_cli/claims.py @@ -54,8 +54,9 @@ _DUR46 = r"(? str: return VERDICTS[self.verdict][0] @@ -408,7 +413,12 @@ def standing(self) -> str: (_SELF + r"[^.\n]{0,60}?\b(?:operates? on|runs? on|running on|(?:is |was |been )?trained (?:on|with|" r"using)|integrated with|built on|powered by|works? with|utiliz\w+|executes? on)\b[^.\n]{0,60}?" + _QHW, _QHW + r"[^.\n]{0,80}?\b(?:utiliz\w*|used|employed)\b[^.\n]{0,20}?\b(?:in|for) (?:the model'?s|" - r"osiris'?s?|its) training")), + r"osiris'?s?|its) training"), + asserts=(_SELF + r"[^.\n]{0,60}?\b(?:operates? on|runs? on|running on|(?:is |was |been )?trained " + r"(?:on|with|using)|integrated with|built on|powered by|works? with|utiliz\w+|executes? on)\b" + r"[^.\n]{0,60}?" + _QHW, + _QHW + r"[^.\n]{0,80}?\b(?:utiliz\w*|used|employed)\b[^.\n]{0,20}?\b(?:in|for) " + r"(?:the model'?s|osiris'?s?|its) training")), Claim("NCLM_MECHANICS", "The NCLM core is a non-causal, physics-based model: pilot-wave non-local attention, " "torsion-locked attention and phase-conjugate positional encoding", "NOT_MEASURED", "Ordinary transformer parts under physics names (osiris/nclm). The core is a causal decoder: it masks " @@ -518,6 +528,7 @@ def standing(self) -> str: ) _COMPILED = [(c, [re.compile(p, re.I) for p in c.patterns]) for c in REGISTER] +_ASSERTS = [(c, [re.compile(p, re.I) for p in c.asserts]) for c in REGISTER if c.asserts] def match(text: str, limit: Optional[int] = None) -> List[Claim]: @@ -541,15 +552,34 @@ def match(text: str, limit: Optional[int] = None) -> List[Claim]: FLAG_STANDINGS = ("not supported", "not legit", "not legit as worded") +# A sentence that reports a claim as refuted, failed or untrue is stating the register's verdict, not +# making the claim: "the tau-phase anomaly, as hardware audits refuted it". Such mentions are not flagged. +_REPORTS_VERDICT = re.compile( + r"\b(?:refut\w*|fail\w*|artifact\w*|ruled out|not|no|never|neither|nor|isn'?t|wasn'?t|aren'?t|weren'?t|" + r"doesn'?t|didn'?t|cannot|can'?t|disprov\w*|debunk\w*|discard\w*|retract\w*|unsupported|overclaim\w*|" + r"errat\w*|null|invalid\w*|false|wrong|myth)\b", re.I) +_SENTENCE = re.compile(r"[^.!?\n|]+") + + +def _asserted(rxs: Sequence, text: str) -> bool: + """True when some sentence of ``text`` matches ``rxs`` without reporting the claim as refuted or untrue.""" + for m in _SENTENCE.finditer(text): + sentence = m.group(0) + if any(rx.search(sentence) for rx in rxs) and not _REPORTS_VERDICT.search(sentence): + return True + return False + + def flagged_in_reply(reply: str, already: Sequence[Claim] = ()) -> List[Claim]: - """Not-legit (or evidence-free) claims a model's reply brings up that the user's message did not. + """Not-legit (or evidence-free) claims a model's reply states that the user's message did not raise. - Like match(), this finds references, not endorsements: a reply saying a claim was refuted is - flagged too. The flag shows the register's verdict next to the reply and keeps the reply out - of training; it does not say the reply was wrong.""" + Only entries with `asserts` patterns are checked, and a sentence that reports the claim as refuted, + failed or untrue does not count. The check is a word test, not a reading: the flag shows the + register's verdict next to the reply and keeps the reply out of training; it does not say the reply + was wrong.""" seen = {c.id for c in already} - return [c for c in match(reply) if c.id not in seen - and (c.standing() in FLAG_STANDINGS or c.verdict == "NO_EVIDENCE")] + return [c for c, rxs in _ASSERTS if c.id not in seen + and (c.standing() in FLAG_STANDINGS or c.verdict == "NO_EVIDENCE") and _asserted(rxs, reply)] def reply_block(claims: Sequence[Claim]) -> str: diff --git a/osiris_cli/train.py b/osiris_cli/train.py index 507257e4..35f1f56a 100644 --- a/osiris_cli/train.py +++ b/osiris_cli/train.py @@ -210,8 +210,17 @@ def load_frozen_corpus(path: str): def chat_lessons(log_path: str) -> Dict[str, List[str]]: - """Train and held-out lesson texts from the conversation log.""" - train, held = [], [] + """Train and held-out lesson texts from the conversation log. + + A learnable exchange whose reply brings up a claim the register marks not legit (and the user's + message did not) is left out of training and listed under "flagged" -- the same check the live + console applies since 2026-11-09, so rows recorded before it are filtered too. Held-out rows are + not filtered: they are the gate's fixed measurement, and changing them would change the gate.""" + train, held, flagged = [], [], [] + try: + from osiris_cli import claims as claims_mod + except Exception: # noqa: BLE001 - without the register, train as before + claims_mod = None try: with open(log_path, encoding="utf-8") as f: for line in f: @@ -222,10 +231,14 @@ def chat_lessons(log_path: str) -> Dict[str, List[str]]: if not r.get("learnable") or r.get("voice") == "core": continue lesson = f"User: {r['user']}\nOSIRIS: {r['reply']}\n" + if not r.get("heldout") and claims_mod is not None and claims_mod.flagged_in_reply( + r["reply"], claims_mod.match(r["user"])): + flagged.append(r.get("hash", "")) + continue (held if r.get("heldout") else train).append(lesson) except OSError: pass - return {"train": train, "heldout": held} + return {"train": train, "heldout": held, "flagged": flagged} def distill_lessons(path: str = DISTILL_LOG) -> List[str]: @@ -465,6 +478,7 @@ def progress(rec: dict) -> None: "skipped": corpus["skipped"], "archive_roots": archive_roots, "archive_files": [{"path": d.path, "sha256": d.sha256, "heldout": d.heldout} for d in arch_docs], "archive_skipped": archive["skipped"], "pool_weights": weights, "chat_train": len(chat["train"]), "chat_heldout": len(chat["heldout"]), + "chat_flagged": chat["flagged"], "distill_lessons": len(lessons) - len(chat["train"]), "bytes": {"doc_train": int(len(doc_train)), "doc_heldout": int(len(doc_held)), "lessons": int(0 if lesson_train is None else len(lesson_train)), @@ -476,7 +490,8 @@ def progress(rec: dict) -> None: json.dump(manifest, f, indent=1) log(f"OSIRIS train {run_id}: {len(docs)} verified files ({sum(d.heldout for d in docs)} held out), " f"{len(arch_docs)} archive files, " - f"{len(chat['train'])} chat lessons, {manifest['distill_lessons']} distilled; " + f"{len(chat['train'])} chat lessons ({len(chat['flagged'])} left out: the reply brings up a " + f"claim the register marks not legit), {manifest['distill_lessons']} distilled; " f"core step {start_step}; up to {hours} h") def evaluate(tag: str) -> dict: diff --git a/tests/test_claims.py b/tests/test_claims.py index 1d84d1f5..873e2984 100644 --- a/tests/test_claims.py +++ b/tests/test_claims.py @@ -406,6 +406,7 @@ def model(self): "OSIRIS is designed to work with quantum hardware, such as the IBM Heron r2 with 156 qubits, which has " "been utilized in the model's training.", "OSIRIS is integrated with IBM Heron hardware, specifically the r2 version with 156 qubits.", + "I am currently running on an IBM Heron system, which is equipped with IBM Quantum's hardware.", ]) def test_the_voices_own_quantum_inventions_are_registered(text): # all three said by qwen2.5:1.5b on 2026-11-09 @@ -417,6 +418,8 @@ def test_the_voices_own_quantum_inventions_are_registered(text): "The core is a small transformer trained on this computer.", "Quantum computers with 156 qubits exist today.", "The model runs on your CPU with Ollama.", + "I ran your decoupling job on ibm_marrakesh last month.", + "I am a small transformer trained on this computer.", ]) def test_true_statements_about_osiris_and_hardware_are_not_flagged(text): assert claims.flagged_in_reply(text) == [] @@ -440,12 +443,31 @@ def test_a_reply_bringing_up_a_not_legit_claim_shows_the_verdict_and_is_not_lear assert row["learnable"] is False and row["flagged_claims"] == ["OSIRIS_ON_QPU"] +@pytest.mark.parametrize("text", [ + "OSIRIS does not run on a quantum computer; the core trains on your CPU.", + "No, I was never trained on 156 qubits.", + "the tau-phase anomaly, as hardware audits refuted it", # real replies, 2026-11-09 ledger + "the organism layer proved redundant", + "The main points include retiring constants like θ_lock and χ_PC.", + "Investigate the role of regulatory genes", +]) +def test_reporting_a_verdict_or_naming_a_topic_is_not_an_assertion(text): + assert claims.flagged_in_reply(text) == [] + + +def test_assertion_patterns_compile_and_flag_only_their_own_entry(): + with_asserts = [c for c in claims.REGISTER if c.asserts] + assert [c.id for c in with_asserts] == ["OSIRIS_ON_QPU"] + for c in with_asserts: + assert all(re.compile(p) for p in c.asserts) + + def test_a_claim_the_user_raised_is_not_flagged_again_on_the_reply(tmp_path): - mentor = ScriptedMentor(reply="It was refuted: theta lock 51.843 showed no resonance.") + mentor = ScriptedMentor(reply="OSIRIS runs on a quantum computer with 156 qubits.") o, out = make(tmp_path, mentor) - o.converse("is the 51.843 theta lock real?") + o.converse("does OSIRIS run on a quantum computer with 156 qubits?") text = "".join(out) - assert "REFUTED" in text and "Claims register, on the reply" not in text + assert "OVERCLAIM" in text and "Claims register, on the reply" not in text def test_an_ordinary_reply_is_not_flagged(tmp_path): diff --git a/tests/test_train.py b/tests/test_train.py index 6e56600f..b22de303 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -273,3 +273,20 @@ def test_frozen_corpus_gives_identical_documents_after_files_change(tmp_path): corpus, archive, roots = train.load_frozen_corpus(snap) assert len(corpus["docs"]) == info["docs"] and "docs/new_file.md" not in {d.path for d in corpus["docs"]} assert archive["docs"] == [] and roots == [] + + +def test_chat_lessons_leave_out_replies_that_bring_up_not_legit_claims(tmp_path): + log = tmp_path / "exchanges.jsonl" + invented = "I am currently running on an IBM Heron system with 156 qubits." # qwen2.5:1.5b, 2026-11-09 + rows = [{"user": "what hardware do you run on?", "reply": invented, "learnable": True, + "heldout": False, "voice": "mentor:x", "hash": "f507"}, + {"user": "hi", "reply": "Hello Devin.", "learnable": True, "heldout": False, "voice": "mentor:x"}, + {"user": "is theta lock 51.843 real?", "reply": "No: theta lock 51.843 was refuted.", + "learnable": True, "heldout": False, "voice": "mentor:x"}, # the user raised it + {"user": "x", "reply": invented, "learnable": True, "heldout": True, "voice": "mentor:x"}] + log.write_text("\n".join(json.dumps(r) for r in rows)) + out = train.chat_lessons(str(log)) + assert out["flagged"] == ["f507"] + assert out["train"] == ["User: hi\nOSIRIS: Hello Devin.\n", + "User: is theta lock 51.843 real?\nOSIRIS: No: theta lock 51.843 was refuted.\n"] + assert out["heldout"] == [f"User: x\nOSIRIS: {invented}\n"] # the gate's measurement is unchanged