diff --git a/toolgate/executors/research_base.py b/toolgate/executors/research_base.py index bb4243f..f80d95c 100644 --- a/toolgate/executors/research_base.py +++ b/toolgate/executors/research_base.py @@ -83,6 +83,14 @@ def _cooldown_provider(provider: str, seconds: int) -> None: } + +def topic_terms(query: str, stop_words: set[str], min_length: int = 3) -> list[str]: + """The query's words without filler, in order. When every word is filler ("email + software"), all of them: the owner's query never shrinks to nothing.""" + words = list(dict.fromkeys(term for term in re.findall(r"[a-z0-9]+", query.lower()) if len(term) >= min_length)) + return [word for word in words if word not in stop_words] or words + + REDDIT_QUERY_STOP_WORDS = QUERY_STOP_WORDS - { "business", "businesses", "manual", "spreadsheet", "spreadsheets", "workflow", "workflows", } diff --git a/toolgate/executors/research_forums.py b/toolgate/executors/research_forums.py index 3e05f47..440a495 100644 --- a/toolgate/executors/research_forums.py +++ b/toolgate/executors/research_forums.py @@ -21,6 +21,7 @@ _normalize, _require_provider_ready, json_body, + topic_terms, ) from toolgate.executors.research_text import clean_text, extract_html_text from toolgate.executors.research_web import _searx, _tavily @@ -213,10 +214,7 @@ def _reddit_subreddit_tavily(query: str, source: str, limit: int, recency_days: def _discourse(query: str, _source: str, limit: int, recency_days: int, timeout: float) -> list[dict]: """Search approved public automation communities through Discourse JSON.""" deadline = time.monotonic() + max(1.0, timeout) - terms = [ - term for term in dict.fromkeys(re.findall(r"[a-z0-9]+", query.lower())) - if len(term) >= 3 and term not in QUERY_STOP_WORDS - ][:4] + terms = topic_terms(query, QUERY_STOP_WORDS)[:4] if not terms: return [] after = (datetime.now(UTC) - timedelta(days=recency_days)).date().isoformat() diff --git a/toolgate/executors/research_sites.py b/toolgate/executors/research_sites.py index fdc7874..6edfafc 100644 --- a/toolgate/executors/research_sites.py +++ b/toolgate/executors/research_sites.py @@ -15,6 +15,7 @@ ResearchError, _normalize, json_body, + topic_terms, ) from toolgate.executors.research_text import clean_text, extract_html_text @@ -49,10 +50,7 @@ def _github_repositories(query: str, _source: str, limit: int, recency_days: int headers = {"Accept": "application/vnd.github+json", "User-Agent": "ToolGateResearch/2.0"} with contextlib.suppress(KeyError): headers["Authorization"] = f"Bearer {vault.get_key('GITHUB_TOKEN')}" - terms = [ - term for term in dict.fromkeys(re.findall(r"[a-z0-9]+", query.lower())) - if len(term) >= 3 and term not in QUERY_STOP_WORDS - ][:3] + terms = topic_terms(query, QUERY_STOP_WORDS)[:3] if not terms: return [] request_args = { @@ -241,10 +239,7 @@ def _appstore_reviews(query: str, _source: str, limit: int, recency_days: int, t def _appstore_catalog(query: str, _source: str, limit: int, _recency_days: int, timeout: float) -> list[dict]: """Resolve a bounded market query to public iOS app identities.""" - terms = { - term for term in re.findall(r"[a-z0-9]+", query.lower()) - if len(term) >= 3 and term not in QUERY_STOP_WORDS - } + terms = set(topic_terms(query, QUERY_STOP_WORDS)) if not terms: raise ResearchError("App Store catalog query needs a concrete market or workflow term") response = httpx.get( @@ -377,10 +372,7 @@ def _producthunt(query: str, _source: str, limit: int, recency_days: int, timeou if not control_plane.settings().get("producthunt_commercial_use_approved", False): raise ResearchError("Product Hunt API business-use approval has not been confirmed") token = vault.get_key("PRODUCTHUNT_TOKEN") - terms = [ - term for term in dict.fromkeys(re.findall(r"[a-z0-9]+", query.lower())) - if len(term) >= 3 and term not in QUERY_STOP_WORDS - ][:4] + terms = topic_terms(query, QUERY_STOP_WORDS)[:4] if not terms: raise ResearchError("Product Hunt query has no useful topic terms") endpoint = "https://api.producthunt.com/v2/api/graphql" diff --git a/toolgate/tests/test_research_topic_terms.py b/toolgate/tests/test_research_topic_terms.py new file mode 100644 index 0000000..2f5ee65 --- /dev/null +++ b/toolgate/tests/test_research_topic_terms.py @@ -0,0 +1,52 @@ +"""Filler words never shrink the owner's query to nothing (hunt toolgate-4 F4).""" + +from unittest.mock import Mock + +import pytest + +from toolgate.core import control_plane +from toolgate.executors import research_base, research_sites + + +@pytest.fixture(autouse=True) +def gate(tmp_path, monkeypatch): + monkeypatch.setattr(control_plane, "DB_PATH", tmp_path / "gate.db") + + +def answer(payload: dict) -> Mock: + response = Mock(status_code=200, content=b"{}") + response.raise_for_status.return_value = None + response.json.return_value = payload + return response + + +def test_filler_is_dropped_but_a_query_of_only_filler_is_kept(): + stop = research_base.QUERY_STOP_WORDS + assert research_base.topic_terms("the invoice tool for teams", stop) == ["invoice"] + assert research_base.topic_terms("email software", stop) == ["email", "software"] + assert research_base.topic_terms("a to", stop) == [] + + +def test_github_repositories_searches_a_query_made_of_filler_words(monkeypatch): + sent = [] + + def get(_url, **kwargs): + sent.append(kwargs["params"]["q"]) + return answer({"items": []}) + + monkeypatch.setattr(research_sites.vault, "get_key", Mock(side_effect=KeyError("GITHUB_TOKEN"))) + monkeypatch.setattr(research_sites.httpx, "get", get) + research_sites._github_repositories("email software", "github_repositories", 5, 365, 5) + assert sent and sent[0].startswith("email software ") + + +def test_appstore_catalog_accepts_a_query_made_of_filler_words(monkeypatch): + sent = [] + + def get(_url, **kwargs): + sent.append(kwargs["params"]["term"]) + return answer({"results": []}) + + monkeypatch.setattr(research_sites.httpx, "get", get) + assert research_sites._appstore_catalog("small team tools", "appstore_catalog", 5, 3650, 5) == [] + assert sent == ["small team tools"]