Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 39 additions & 9 deletions osiris_cli/claims.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,9 @@
_DUR46 = r"(?<![\d.,])46(?:\.(?:0+|9\d*))?\s*(?:[µμ]s(?:ec)?|us|micro-?\s?seconds?|microsec)\b"
_CTX46 = r"(?:τ|\btau|revival|anomal\w*|oscillat\w*|period|phase|φ)"
_TAU46 = rf"{_CTX46}[^\n;]{{0,40}}?{_DUR46}|{_DUR46}[^\n;]{{0,40}}?{_CTX46}"
# OSIRIS_ON_QPU: OSIRIS (or "the model", "the core") as the subject, quantum hardware as the object.
_SELF = r"\b(?:the osiris model|osiris|the model|the core|nclm)\b"
# OSIRIS_ON_QPU: OSIRIS (or "the model", "the core", or the voice's own "I") as the subject, quantum hardware
# as the object. "I am currently running on an IBM Heron system" was missed without the first person (2026-11-09).
_SELF = r"\b(?:the osiris model|osiris|the model|the core|nclm|i(?: am|'m| was)?|my (?:core|model|weights))\b"
_QHW = r"(?:\b\d+[\s-]*qubits?\b|quantum (?:computer|hardware|processor|computing engine)s?|\bibm heron\b|" \
r"\bheron r2\b|\bqpu\b)"

Expand All @@ -70,6 +71,10 @@ class Claim:
patterns: Tuple[str, ...]
testable: str = ""
checks: Tuple[Tuple[str, Tuple], ...] = ()
# Patterns for the claim being *stated* (subject, verb, object), used to check a model's reply.
# `patterns` find the topic, which is right for showing a verdict on a message and wrong for judging
# a reply: on the 2026-11-09 ledger they flagged 35 honest or merely topical replies for 1 invention.
asserts: Tuple[str, ...] = ()

def standing(self) -> str:
return VERDICTS[self.verdict][0]
Expand Down Expand Up @@ -408,7 +413,12 @@ def standing(self) -> str:
(_SELF + r"[^.\n]{0,60}?\b(?:operates? on|runs? on|running on|(?:is |was |been )?trained (?:on|with|"
r"using)|integrated with|built on|powered by|works? with|utiliz\w+|executes? on)\b[^.\n]{0,60}?" + _QHW,
_QHW + r"[^.\n]{0,80}?\b(?:utiliz\w*|used|employed)\b[^.\n]{0,20}?\b(?:in|for) (?:the model'?s|"
r"osiris'?s?|its) training")),
r"osiris'?s?|its) training"),
asserts=(_SELF + r"[^.\n]{0,60}?\b(?:operates? on|runs? on|running on|(?:is |was |been )?trained "
r"(?:on|with|using)|integrated with|built on|powered by|works? with|utiliz\w+|executes? on)\b"
r"[^.\n]{0,60}?" + _QHW,
_QHW + r"[^.\n]{0,80}?\b(?:utiliz\w*|used|employed)\b[^.\n]{0,20}?\b(?:in|for) "
r"(?:the model'?s|osiris'?s?|its) training")),
Claim("NCLM_MECHANICS", "The NCLM core is a non-causal, physics-based model: pilot-wave non-local attention, "
"torsion-locked attention and phase-conjugate positional encoding", "NOT_MEASURED",
"Ordinary transformer parts under physics names (osiris/nclm). The core is a causal decoder: it masks "
Expand Down Expand Up @@ -518,6 +528,7 @@ def standing(self) -> str:
)

_COMPILED = [(c, [re.compile(p, re.I) for p in c.patterns]) for c in REGISTER]
_ASSERTS = [(c, [re.compile(p, re.I) for p in c.asserts]) for c in REGISTER if c.asserts]


def match(text: str, limit: Optional[int] = None) -> List[Claim]:
Expand All @@ -541,15 +552,34 @@ def match(text: str, limit: Optional[int] = None) -> List[Claim]:
FLAG_STANDINGS = ("not supported", "not legit", "not legit as worded")


# A sentence that reports a claim as refuted, failed or untrue is stating the register's verdict, not
# making the claim: "the tau-phase anomaly, as hardware audits refuted it". Such mentions are not flagged.
_REPORTS_VERDICT = re.compile(
r"\b(?:refut\w*|fail\w*|artifact\w*|ruled out|not|no|never|neither|nor|isn'?t|wasn'?t|aren'?t|weren'?t|"
r"doesn'?t|didn'?t|cannot|can'?t|disprov\w*|debunk\w*|discard\w*|retract\w*|unsupported|overclaim\w*|"
r"errat\w*|null|invalid\w*|false|wrong|myth)\b", re.I)
_SENTENCE = re.compile(r"[^.!?\n|]+")


def _asserted(rxs: Sequence, text: str) -> bool:
"""True when some sentence of ``text`` matches ``rxs`` without reporting the claim as refuted or untrue."""
for m in _SENTENCE.finditer(text):
sentence = m.group(0)
if any(rx.search(sentence) for rx in rxs) and not _REPORTS_VERDICT.search(sentence):
return True
return False


def flagged_in_reply(reply: str, already: Sequence[Claim] = ()) -> List[Claim]:
"""Not-legit (or evidence-free) claims a model's reply brings up that the user's message did not.
"""Not-legit (or evidence-free) claims a model's reply states that the user's message did not raise.

Like match(), this finds references, not endorsements: a reply saying a claim was refuted is
flagged too. The flag shows the register's verdict next to the reply and keeps the reply out
of training; it does not say the reply was wrong."""
Only entries with `asserts` patterns are checked, and a sentence that reports the claim as refuted,
failed or untrue does not count. The check is a word test, not a reading: the flag shows the
register's verdict next to the reply and keeps the reply out of training; it does not say the reply
was wrong."""
seen = {c.id for c in already}
return [c for c in match(reply) if c.id not in seen
and (c.standing() in FLAG_STANDINGS or c.verdict == "NO_EVIDENCE")]
return [c for c, rxs in _ASSERTS if c.id not in seen
and (c.standing() in FLAG_STANDINGS or c.verdict == "NO_EVIDENCE") and _asserted(rxs, reply)]


def reply_block(claims: Sequence[Claim]) -> str:
Expand Down
23 changes: 19 additions & 4 deletions osiris_cli/train.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,8 +210,17 @@ def load_frozen_corpus(path: str):


def chat_lessons(log_path: str) -> Dict[str, List[str]]:
"""Train and held-out lesson texts from the conversation log."""
train, held = [], []
"""Train and held-out lesson texts from the conversation log.

A learnable exchange whose reply brings up a claim the register marks not legit (and the user's
message did not) is left out of training and listed under "flagged" -- the same check the live
console applies since 2026-11-09, so rows recorded before it are filtered too. Held-out rows are
not filtered: they are the gate's fixed measurement, and changing them would change the gate."""
train, held, flagged = [], [], []
try:
from osiris_cli import claims as claims_mod
except Exception: # noqa: BLE001 - without the register, train as before
claims_mod = None
try:
with open(log_path, encoding="utf-8") as f:
for line in f:
Expand All @@ -222,10 +231,14 @@ def chat_lessons(log_path: str) -> Dict[str, List[str]]:
if not r.get("learnable") or r.get("voice") == "core":
continue
lesson = f"User: {r['user']}\nOSIRIS: {r['reply']}\n"
if not r.get("heldout") and claims_mod is not None and claims_mod.flagged_in_reply(
r["reply"], claims_mod.match(r["user"])):
flagged.append(r.get("hash", ""))
continue
(held if r.get("heldout") else train).append(lesson)
except OSError:
pass
return {"train": train, "heldout": held}
return {"train": train, "heldout": held, "flagged": flagged}


def distill_lessons(path: str = DISTILL_LOG) -> List[str]:
Expand Down Expand Up @@ -465,6 +478,7 @@ def progress(rec: dict) -> None:
"skipped": corpus["skipped"], "archive_roots": archive_roots,
"archive_files": [{"path": d.path, "sha256": d.sha256, "heldout": d.heldout} for d in arch_docs],
"archive_skipped": archive["skipped"], "pool_weights": weights, "chat_train": len(chat["train"]), "chat_heldout": len(chat["heldout"]),
"chat_flagged": chat["flagged"],
"distill_lessons": len(lessons) - len(chat["train"]),
"bytes": {"doc_train": int(len(doc_train)), "doc_heldout": int(len(doc_held)),
"lessons": int(0 if lesson_train is None else len(lesson_train)),
Expand All @@ -476,7 +490,8 @@ def progress(rec: dict) -> None:
json.dump(manifest, f, indent=1)
log(f"OSIRIS train {run_id}: {len(docs)} verified files ({sum(d.heldout for d in docs)} held out), "
f"{len(arch_docs)} archive files, "
f"{len(chat['train'])} chat lessons, {manifest['distill_lessons']} distilled; "
f"{len(chat['train'])} chat lessons ({len(chat['flagged'])} left out: the reply brings up a "
f"claim the register marks not legit), {manifest['distill_lessons']} distilled; "
f"core step {start_step}; up to {hours} h")

def evaluate(tag: str) -> dict:
Expand Down
28 changes: 25 additions & 3 deletions tests/test_claims.py
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,7 @@ def model(self):
"OSIRIS is designed to work with quantum hardware, such as the IBM Heron r2 with 156 qubits, which has "
"been utilized in the model's training.",
"OSIRIS is integrated with IBM Heron hardware, specifically the r2 version with 156 qubits.",
"I am currently running on an IBM Heron system, which is equipped with IBM Quantum's hardware.",
])
def test_the_voices_own_quantum_inventions_are_registered(text):
# all three said by qwen2.5:1.5b on 2026-11-09
Expand All @@ -417,6 +418,8 @@ def test_the_voices_own_quantum_inventions_are_registered(text):
"The core is a small transformer trained on this computer.",
"Quantum computers with 156 qubits exist today.",
"The model runs on your CPU with Ollama.",
"I ran your decoupling job on ibm_marrakesh last month.",
"I am a small transformer trained on this computer.",
])
def test_true_statements_about_osiris_and_hardware_are_not_flagged(text):
assert claims.flagged_in_reply(text) == []
Expand All @@ -440,12 +443,31 @@ def test_a_reply_bringing_up_a_not_legit_claim_shows_the_verdict_and_is_not_lear
assert row["learnable"] is False and row["flagged_claims"] == ["OSIRIS_ON_QPU"]


@pytest.mark.parametrize("text", [
"OSIRIS does not run on a quantum computer; the core trains on your CPU.",
"No, I was never trained on 156 qubits.",
"the tau-phase anomaly, as hardware audits refuted it", # real replies, 2026-11-09 ledger
"the organism layer proved redundant",
"The main points include retiring constants like θ_lock and χ_PC.",
"Investigate the role of regulatory genes",
])
def test_reporting_a_verdict_or_naming_a_topic_is_not_an_assertion(text):
assert claims.flagged_in_reply(text) == []


def test_assertion_patterns_compile_and_flag_only_their_own_entry():
with_asserts = [c for c in claims.REGISTER if c.asserts]
assert [c.id for c in with_asserts] == ["OSIRIS_ON_QPU"]
for c in with_asserts:
assert all(re.compile(p) for p in c.asserts)


def test_a_claim_the_user_raised_is_not_flagged_again_on_the_reply(tmp_path):
mentor = ScriptedMentor(reply="It was refuted: theta lock 51.843 showed no resonance.")
mentor = ScriptedMentor(reply="OSIRIS runs on a quantum computer with 156 qubits.")
o, out = make(tmp_path, mentor)
o.converse("is the 51.843 theta lock real?")
o.converse("does OSIRIS run on a quantum computer with 156 qubits?")
text = "".join(out)
assert "REFUTED" in text and "Claims register, on the reply" not in text
assert "OVERCLAIM" in text and "Claims register, on the reply" not in text


def test_an_ordinary_reply_is_not_flagged(tmp_path):
Expand Down
17 changes: 17 additions & 0 deletions tests/test_train.py
Original file line number Diff line number Diff line change
Expand Up @@ -273,3 +273,20 @@ def test_frozen_corpus_gives_identical_documents_after_files_change(tmp_path):
corpus, archive, roots = train.load_frozen_corpus(snap)
assert len(corpus["docs"]) == info["docs"] and "docs/new_file.md" not in {d.path for d in corpus["docs"]}
assert archive["docs"] == [] and roots == []


def test_chat_lessons_leave_out_replies_that_bring_up_not_legit_claims(tmp_path):
log = tmp_path / "exchanges.jsonl"
invented = "I am currently running on an IBM Heron system with 156 qubits." # qwen2.5:1.5b, 2026-11-09
rows = [{"user": "what hardware do you run on?", "reply": invented, "learnable": True,
"heldout": False, "voice": "mentor:x", "hash": "f507"},
{"user": "hi", "reply": "Hello Devin.", "learnable": True, "heldout": False, "voice": "mentor:x"},
{"user": "is theta lock 51.843 real?", "reply": "No: theta lock 51.843 was refuted.",
"learnable": True, "heldout": False, "voice": "mentor:x"}, # the user raised it
{"user": "x", "reply": invented, "learnable": True, "heldout": True, "voice": "mentor:x"}]
log.write_text("\n".join(json.dumps(r) for r in rows))
out = train.chat_lessons(str(log))
assert out["flagged"] == ["f507"]
assert out["train"] == ["User: hi\nOSIRIS: Hello Devin.\n",
"User: is theta lock 51.843 real?\nOSIRIS: No: theta lock 51.843 was refuted.\n"]
assert out["heldout"] == [f"User: x\nOSIRIS: {invented}\n"] # the gate's measurement is unchanged
Loading