From 47bb1faa346cbc294d811cfbb019338c61a4e8aa Mon Sep 17 00:00:00 2001 From: Adam Wright Date: Wed, 16 Sep 2026 15:17:08 +0000 Subject: [PATCH] Prefer the Reactome tool that needs no installation Asked "can you run gsea for me", beta answered with ReactomeFIViz -- the Cytoscape plugin -- and gave steps for installing Cytoscape. ReactomeGSA is a web tool at reactome.org/gsa that needs nothing installed, and is what someone asking that question wants. Not a retrieval failure. The top four passages were the ReactomeGSA page: 1. [analysis/gsa] 2. [analysis/gsa] 3. [analysis/gsa] 4. [analysis/gsa] 5. [reactome-fiviz] The FIViz page arrives fifth because the rephrase expands "gsea" into "Gene Set Enrichment Analysis (GSEA)", and that page carries a "Perform GSEA Analysis" menu item. The model then led with it, because the prompt asks for clear actionable steps and FIViz has by far the most step-by-step text. That is the general trap, and the prompt now names it: the most detailed instructions are usually for the most involved tool. Where several Reactome tools do the same job, lead with the one needing least setup -- web before desktop application, plugin or R package -- and offer the rest as alternatives. Both phrasings now lead with ReactomeGSA and say it needs no installation. Checked that this did not simply bury FIViz: asked for by name, or asked about Cytoscape, it still answers with FIViz and its install steps. Co-Authored-By: Claude Opus 5 --- src/retrievers/userguide/prompt.py | 3 +++ tests/agent/test_safety_scope.py | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/retrievers/userguide/prompt.py b/src/retrievers/userguide/prompt.py index 9ec0d63..046298c 100644 --- a/src/retrievers/userguide/prompt.py +++ b/src/retrievers/userguide/prompt.py @@ -15,6 +15,9 @@ - Use a clear display name (page title or section title). - If multiple excerpts support the same fact, cite them together (space-separated). 3. How-to focus: Give clear, actionable steps when the user asks how to perform a task. Name UI elements accurately (buttons, panels, tabs) as they appear in the context. + - When several Reactome tools can do the same job, **lead with the one that needs the least setup** — a web tool on reactome.org before a desktop application, a plugin, or an R package. Mention the others afterwards as alternatives. + - Concretely: gene set analysis is **ReactomeGSA** (web, at reactome.org/gsa, nothing to install). ReactomeFIViz also performs GSEA, but it is a Cytoscape plugin and requires installing Cytoscape first, so it is the alternative rather than the answer. + - Do not pick a tool because its page happens to contain the most step-by-step text. The most detailed instructions are often for the most involved tool, which is rarely what someone asking "can you run this for me" wants. 4. Tone and style: - Write in a clear, friendly, and conversational tone. - Use accessible language; avoid unnecessary jargon. diff --git a/tests/agent/test_safety_scope.py b/tests/agent/test_safety_scope.py index 8583593..33789b6 100644 --- a/tests/agent/test_safety_scope.py +++ b/tests/agent/test_safety_scope.py @@ -56,3 +56,25 @@ def test_capability_is_answered_downstream_not_refused_here() -> None: """Whether the assistant can run an analysis is a different question from whether it is allowed to be asked.""" assert "decided later" in safety_check_message + + +def test_the_user_guide_prefers_the_tool_that_needs_no_install() -> None: + """Asked "can you run gsea for me", beta answered with ReactomeFIViz -- + the Cytoscape plugin -- rather than ReactomeGSA, the web tool. + + Not a retrieval failure: the top four passages were the ReactomeGSA page. + The FIViz page arrived fifth (the rephrase expands "gsea" to "Gene Set + Enrichment Analysis", and that page has a "Perform GSEA Analysis" menu + item), and the model led with it because the prompt asks for actionable + steps and FIViz has the most step-by-step text. + + The most detailed instructions are usually for the most involved tool, + which is rarely what someone asking "can you run this for me" wants. + """ + from retrievers.userguide.prompt import userguide_system_prompt + + assert "least setup" in userguide_system_prompt + assert "ReactomeGSA" in userguide_system_prompt + assert "Cytoscape" in userguide_system_prompt + # The trap that caused it, named so a later edit does not reintroduce it. + assert "most step-by-step text" in userguide_system_prompt