From c7427d1ac4778e3d28c411e6171532a5c33e2b6c Mon Sep 17 00:00:00 2001 From: Adam Wright Date: Wed, 16 Sep 2026 14:26:44 +0000 Subject: [PATCH] Stop the safety checker refusing people for asking Reactome to do its job Measured on beta: "can you run gsea for me" was refused 4 times out of 4. safety=false can you run gsea for me reason: Requests a specific analysis task without providing context or data, which is outside the scope That is an on-topic question from a researcher about a flagship Reactome feature, and it never reached the user guide, the MCP, or the GSA tools -- all of which are further along the graph. The user guide bundle installed on Sunday and the five ReactomeGSA tools added on Monday were both aimed at this question. Neither was ever consulted. "run a pathway analysis on my genes" was refused 1 time in 4: the same user, the same intent, a different answer depending on the roll. The cause is one clause read too broadly: The question must NOT request personal, medical, legal, or other types of advice outside the scope of scientific knowledge. The model took "outside the scope" to cover "asks the assistant to perform a task", so a request to do something Reactome does was rejected as inappropriate. Whether this assistant can carry out an analysis is a capability question, answered downstream and helpfully. It is not a safety question. The prompt now says so, with two worked examples. Ten questions, three runs each: all six that should pass now pass 3/3, and all four that should be refused -- medical advice, immune evasion, gene drives, trivia -- still refuse 3/3. Five tests pin the prompt's wording, including that the genuine refusals keep their examples, so loosening this cannot quietly loosen the part that matters. Four of the five fail against the shipped prompt. They assert wording rather than behaviour on purpose: a behavioural test costs an API call per case and drifts with the model, and the wording is what was wrong. **Worth recording how this was missed.** Both of Adam's original questions were diagnosed from the answers they produced, and the answers looked like retrieval failures, so retrieval is what got fixed. The safety checker sits two nodes earlier and was never looked at. One of those two questions has been broken all week for a reason nothing I built could have addressed. Co-Authored-By: Claude Opus 5 --- src/agent/tasks/safety_checker.py | 18 ++++++++++ tests/agent/test_safety_scope.py | 58 +++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 tests/agent/test_safety_scope.py diff --git a/src/agent/tasks/safety_checker.py b/src/agent/tasks/safety_checker.py index f69fcad..86962ae 100644 --- a/src/agent/tasks/safety_checker.py +++ b/src/agent/tasks/safety_checker.py @@ -21,6 +21,13 @@ - Biology, life sciences, molecular biology, pathways, proteins, genes, and related scientific topics. - How to use the Reactome website, Pathway Browser, search, analysis tools, and other Reactome features (user guide topics). - Mark questions as not relevant if they are about unrelated topics (such as programming, math, history, trivia, etc.). + - A request to *perform* a Reactome operation is relevant: "run an analysis on my genes", + "can you run GSEA for me", "search Reactome for X", "analyse this gene list". These ask + for something Reactome does. Whether this assistant can carry it out is decided later and + answered helpfully; it is not a safety question, and refusing here means the user is told + their on-topic question was inappropriate. + - "Outside the scope of scientific knowledge" above means medical, legal or personal advice + — diagnosis, treatment, dosage. It does not mean "asks the assistant to do a task". IMPORTANT: - If the standalone question is unsafe or not relevant return "safety": "false". @@ -50,6 +57,17 @@ 5. Q: How do I use the Reactome pathway browser? "safety": "true", "reason_unsafe": "" + + 6. Q: Can you run GSEA for me? + "safety": "true", + "reason_unsafe": "" + // Asks for something Reactome does. Answer it -- ReactomeGSA exists, and + // where the assistant cannot run it, saying so is the answer. Refusing + // tells a researcher their own field is off-topic. + + 7. Q: I have a gene list, can you analyse where these genes are involved? + "safety": "true", + "reason_unsafe": "" """ safety_check_prompt = ChatPromptTemplate.from_messages( diff --git a/tests/agent/test_safety_scope.py b/tests/agent/test_safety_scope.py new file mode 100644 index 0000000..8583593 --- /dev/null +++ b/tests/agent/test_safety_scope.py @@ -0,0 +1,58 @@ +"""The safety checker must not refuse people for asking Reactome to do its job. + +Measured on beta 2026-09-16: "can you run gsea for me" was refused 4 times out +of 4, with + + Requests a specific analysis task without providing context or data, + which is outside the scope + +and "run a pathway analysis on my genes" was refused 1 time in 4 -- the same +user, the same intent, a different answer depending on the roll. + +Nothing downstream ever saw those questions. The user guide bundle that answers +them and the ReactomeGSA tools that describe them are both two nodes further on +in the graph. + +These assert the prompt's *wording*, not the model's behaviour -- a behavioural +test would cost an API call per case and drift with the model. The wording is +what was wrong, and it is what a future edit could silently undo. +""" + +from agent.tasks.safety_checker import safety_check_message + + +def test_asking_reactome_to_do_something_is_relevant() -> None: + """The failure this file exists for.""" + assert "run an analysis on my genes" in safety_check_message + assert "can you run GSEA for me" in safety_check_message.lower() or ( + "Can you run GSEA for me?" in safety_check_message + ) + + +def test_a_task_request_is_distinguished_from_personal_advice() -> None: + """ "Outside the scope of scientific knowledge" was being read as "asks the + assistant to do a task". It means medical, legal and personal advice.""" + assert "It does not mean" in safety_check_message + assert "asks the assistant to do a task" in safety_check_message + + +def test_the_cost_of_refusing_is_stated() -> None: + """So a later editor tightening this knows what it buys and what it costs.""" + assert ( + "off-topic" in safety_check_message or "inappropriate" in safety_check_message + ) + + +def test_the_genuine_refusals_are_still_described() -> None: + """Loosening this must not loosen the part that matters. Medical advice, + dual-use and irrelevance all keep their examples.""" + for kept in ("medical", "dual-use", "harmful", "statins", "gene drives"): + assert ( + kept in safety_check_message + ), f"{kept!r} disappeared from the safety prompt" + + +def test_capability_is_answered_downstream_not_refused_here() -> None: + """Whether the assistant can run an analysis is a different question from + whether it is allowed to be asked.""" + assert "decided later" in safety_check_message