fix: filler words never shrink a research query to nothing

toolgate/executors/research_base.py

@@ -83,6 +83,14 @@ def _cooldown_provider(provider: str, seconds: int) -> None:
 }
 
 
+
+def topic_terms(query: str, stop_words: set[str], min_length: int = 3) -> list[str]:
+    """The query's words without filler, in order. When every word is filler ("email
+    software"), all of them: the owner's query never shrinks to nothing."""
+    words = list(dict.fromkeys(term for term in re.findall(r"[a-z0-9]+", query.lower()) if len(term) >= min_length))
+    return [word for word in words if word not in stop_words] or words
+
+
 REDDIT_QUERY_STOP_WORDS = QUERY_STOP_WORDS - {
     "business", "businesses", "manual", "spreadsheet", "spreadsheets", "workflow", "workflows",
 }

toolgate/executors/research_forums.py

@@ -21,6 +21,7 @@
     _normalize,
     _require_provider_ready,
     json_body,
+    topic_terms,
 )
 from toolgate.executors.research_text import clean_text, extract_html_text
 from toolgate.executors.research_web import _searx, _tavily
@@ -213,10 +214,7 @@ def _reddit_subreddit_tavily(query: str, source: str, limit: int, recency_days:
 def _discourse(query: str, _source: str, limit: int, recency_days: int, timeout: float) -> list[dict]:
     """Search approved public automation communities through Discourse JSON."""
     deadline = time.monotonic() + max(1.0, timeout)
-    terms = [
-        term for term in dict.fromkeys(re.findall(r"[a-z0-9]+", query.lower()))
-        if len(term) >= 3 and term not in QUERY_STOP_WORDS
-    ][:4]
+    terms = topic_terms(query, QUERY_STOP_WORDS)[:4]
     if not terms:
         return []
     after = (datetime.now(UTC) - timedelta(days=recency_days)).date().isoformat()

toolgate/executors/research_sites.py

@@ -15,6 +15,7 @@
     ResearchError,
     _normalize,
     json_body,
+    topic_terms,
 )
 from toolgate.executors.research_text import clean_text, extract_html_text
 
@@ -49,10 +50,7 @@ def _github_repositories(query: str, _source: str, limit: int, recency_days: int
     headers = {"Accept": "application/vnd.github+json", "User-Agent": "ToolGateResearch/2.0"}
     with contextlib.suppress(KeyError):
         headers["Authorization"] = f"Bearer {vault.get_key('GITHUB_TOKEN')}"
-    terms = [
-        term for term in dict.fromkeys(re.findall(r"[a-z0-9]+", query.lower()))
-        if len(term) >= 3 and term not in QUERY_STOP_WORDS
-    ][:3]
+    terms = topic_terms(query, QUERY_STOP_WORDS)[:3]
     if not terms:
         return []
     request_args = {
@@ -241,10 +239,7 @@ def _appstore_reviews(query: str, _source: str, limit: int, recency_days: int, t
 
 def _appstore_catalog(query: str, _source: str, limit: int, _recency_days: int, timeout: float) -> list[dict]:
     """Resolve a bounded market query to public iOS app identities."""
-    terms = {
-        term for term in re.findall(r"[a-z0-9]+", query.lower())
-        if len(term) >= 3 and term not in QUERY_STOP_WORDS
-    }
+    terms = set(topic_terms(query, QUERY_STOP_WORDS))
     if not terms:
         raise ResearchError("App Store catalog query needs a concrete market or workflow term")
     response = httpx.get(
@@ -377,10 +372,7 @@ def _producthunt(query: str, _source: str, limit: int, recency_days: int, timeou
     if not control_plane.settings().get("producthunt_commercial_use_approved", False):
         raise ResearchError("Product Hunt API business-use approval has not been confirmed")
     token = vault.get_key("PRODUCTHUNT_TOKEN")
-    terms = [
-        term for term in dict.fromkeys(re.findall(r"[a-z0-9]+", query.lower()))
-        if len(term) >= 3 and term not in QUERY_STOP_WORDS
-    ][:4]
+    terms = topic_terms(query, QUERY_STOP_WORDS)[:4]
     if not terms:
         raise ResearchError("Product Hunt query has no useful topic terms")
     endpoint = "https://api.producthunt.com/v2/api/graphql"

toolgate/tests/test_research_topic_terms.py

@@ -0,0 +1,52 @@
+"""Filler words never shrink the owner's query to nothing (hunt toolgate-4 F4)."""
+
+from unittest.mock import Mock
+
+import pytest
+
+from toolgate.core import control_plane
+from toolgate.executors import research_base, research_sites
+
+
+@pytest.fixture(autouse=True)
+def gate(tmp_path, monkeypatch):
+    monkeypatch.setattr(control_plane, "DB_PATH", tmp_path / "gate.db")
+
+
+def answer(payload: dict) -> Mock:
+    response = Mock(status_code=200, content=b"{}")
+    response.raise_for_status.return_value = None
+    response.json.return_value = payload
+    return response
+
+
+def test_filler_is_dropped_but_a_query_of_only_filler_is_kept():
+    stop = research_base.QUERY_STOP_WORDS
+    assert research_base.topic_terms("the invoice tool for teams", stop) == ["invoice"]
+    assert research_base.topic_terms("email software", stop) == ["email", "software"]
+    assert research_base.topic_terms("a to", stop) == []
+
+
+def test_github_repositories_searches_a_query_made_of_filler_words(monkeypatch):
+    sent = []
+
+    def get(_url, **kwargs):
+        sent.append(kwargs["params"]["q"])
+        return answer({"items": []})
+
+    monkeypatch.setattr(research_sites.vault, "get_key", Mock(side_effect=KeyError("GITHUB_TOKEN")))
+    monkeypatch.setattr(research_sites.httpx, "get", get)
+    research_sites._github_repositories("email software", "github_repositories", 5, 365, 5)
+    assert sent and sent[0].startswith("email software ")
+
+
+def test_appstore_catalog_accepts_a_query_made_of_filler_words(monkeypatch):
+    sent = []
+
+    def get(_url, **kwargs):
+        sent.append(kwargs["params"]["term"])
+        return answer({"results": []})
+
+    monkeypatch.setattr(research_sites.httpx, "get", get)
+    assert research_sites._appstore_catalog("small team tools", "appstore_catalog", 5, 3650, 5) == []
+    assert sent == ["small team tools"]