diff --git a/toolgate/executors/research_web.py b/toolgate/executors/research_web.py index 12819c5..132b6de 100644 --- a/toolgate/executors/research_web.py +++ b/toolgate/executors/research_web.py @@ -81,6 +81,25 @@ def _community_snapshot(item: dict, source: str, timeout: float) -> dict: return item +def _searx_time_range(recency_days: int) -> dict: + """SearXNG's narrowest window that still covers recency_days (it knows day/week/month/year).""" + for days, name in ((1, "day"), (7, "week"), (31, "month"), (366, "year")): + if recency_days <= days: + return {"time_range": name} + return {} + + +def _older_than(item: dict, recency_days: int) -> bool: + """True when the row carries a publication date before the requested window.""" + try: + published = datetime.fromisoformat(str(item.get("published_at")).replace("Z", "+00:00")) + except ValueError: + return False + if published.tzinfo is None: + published = published.replace(tzinfo=UTC) + return published < datetime.now(UTC) - timedelta(days=recency_days) + + def _searx(query: str, source: str, limit: int, recency_days: int, timeout: float) -> list[dict]: deadline = time.monotonic() + max(1.0, timeout) source_domain = SOURCE_DOMAINS[source] @@ -96,7 +115,7 @@ def _searx(query: str, source: str, limit: int, recency_days: int, timeout: floa f"{base_url}/search", params={"q": scoped_query, "format": "json", "language": tool_settings.search_language(), "safesearch": tool_settings.safe_search(), "categories": "general", - **({"time_range": "day" if recency_days <= 1 else "month"} if recency_days <= 45 else {})}, + **_searx_time_range(recency_days)}, headers=headers, timeout=timeout, ) @@ -107,6 +126,9 @@ def _searx(query: str, source: str, limit: int, recency_days: int, timeout: floa if len(results) >= limit: break item = _normalize(row.get("title"), row.get("url"), row.get("content"), source, row.get("publishedDate")) + # SearXNG's windows are coarse (a week for "last 3 days"), so dated rows are checked too. + if item and _older_than(item, recency_days): + continue hostname = urlsplit(item["url"]).hostname.lower() if item and urlsplit(item["url"]).hostname else "" if source == "general" and not domain and any( hostname == blocked or hostname.endswith(f".{blocked}") diff --git a/toolgate/tests/test_searx_recency.py b/toolgate/tests/test_searx_recency.py new file mode 100644 index 0000000..b2082b4 --- /dev/null +++ b/toolgate/tests/test_searx_recency.py @@ -0,0 +1,34 @@ +"""A SearXNG search keeps the owner's recency window (hunt toolgate-4 F3).""" + +from datetime import UTC, datetime, timedelta +from unittest.mock import Mock, patch + +import pytest + +from toolgate.core import control_plane +from toolgate.executors import research_base, research_web + + +@pytest.mark.parametrize(("days", "window"), [ + (1, "day"), (3, "week"), (7, "week"), (14, "month"), (31, "month"), (90, "year"), (400, None), +]) +def test_the_narrowest_searxng_window_that_covers_the_request(days, window): + assert research_web._searx_time_range(days).get("time_range") == window + + +def test_rows_dated_before_the_window_are_dropped(tmp_path, monkeypatch): + monkeypatch.setattr(control_plane, "DB_PATH", tmp_path / "gate.db") + now = datetime.now(UTC) + answer = Mock(content=b"{}", raise_for_status=Mock()) + answer.json.return_value = {"results": [ + {"title": "Old", "url": "https://a.example.com/old", "content": "old", + "publishedDate": (now - timedelta(days=20)).isoformat()}, + {"title": "New", "url": "https://a.example.com/new", "content": "new", + "publishedDate": (now - timedelta(days=2)).isoformat()}, + {"title": "Undated", "url": "https://a.example.com/undated", "content": "undated"}, + ]} + with patch.object(research_base, "_public_https_url", return_value=True), \ + patch.object(research_web.httpx, "get", return_value=answer) as get: + rows = research_web._searx("invoice tools", "general", 5, 7, 5) + assert get.call_args.kwargs["params"]["time_range"] == "week" + assert [row["title"] for row in rows] == ["New", "Undated"]