diff --git a/.agents/skills/do-web-doc-resolver/scripts/_cascade.py b/.agents/skills/do-web-doc-resolver/scripts/_cascade.py new file mode 100644 index 00000000..ec453d0f --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/_cascade.py @@ -0,0 +1,194 @@ +"""Shared cascade resolution logic for query and URL resolution.""" + +import asyncio +import logging +import time +from collections.abc import Callable, Generator +from dataclasses import asdict +from typing import Any + +import scripts.cache_negative +import scripts.quality +from scripts.circuit_breaker import CircuitBreakerRegistry +from scripts.models import ( + ErrorType, + ProviderType, + ReadonlyResolverProtocol, + ResolvedResult, + ResolveMetrics, +) +from scripts.routing import ResolutionBudget +from scripts.routing_memory import RoutingMemory +from scripts.utils import _detect_error_type, _get_cache + +logger = logging.getLogger(__name__) + + +def cascade_stream( + target: str, + cascade_map: dict[str, tuple[ProviderType, ReadonlyResolverProtocol]], + eligible: list[str], + budget: ResolutionBudget, + metrics: ResolveMetrics, + routing_memory: RoutingMemory, + circuit_breakers: CircuitBreakerRegistry, + semantic_cache_store: Callable[[str, dict], bool], + routing_key: str, + result_builder: Callable[[Any, str, str, ResolveMetrics, float], dict[str, Any]] | None = None, + skip_providers: set[str] | None = None, + content_acceptable: Callable[[Any, ProviderType], bool] | None = None, + target_key: str = "query", +) -> Generator[dict[str, Any]]: + skip = skip_providers or set() + cache = _get_cache() + _accept = content_acceptable or (lambda q, pt: q.acceptable) + + # Mutable state shared with inner async function + state: dict[str, Any] = {"best_free_result": None} + + async def _run_cascade() -> dict[str, Any] | None: + active_tasks: dict[asyncio.Task, tuple[str, ProviderType, float]] = {} + + for i, p_name in enumerate(eligible): + if p_name in skip: + continue + pt, func = cascade_map[p_name] + + if pt.is_paid() and state["best_free_result"]: + score = state["best_free_result"].get("score", 0.0) + if score >= budget.min_free_quality_to_skip_paid: + metrics.quality_gate = {"passed": True, "score": score} + state["best_free_result"]["metrics"] = asdict(metrics) + semantic_cache_store(target, state["best_free_result"]) + return dict(state["best_free_result"]) + + if not budget.can_try(is_paid=pt.is_paid()): + if budget.stop_reason in ("paid_disabled", "max_paid_attempts"): + continue + break + if scripts.cache_negative.should_skip_from_negative_cache(cache, target, p_name): + continue + if circuit_breakers.is_open(p_name): + continue + + logger.info("Starting probe: %s", p_name) + start_time_probe = time.time() + task = asyncio.create_task(asyncio.to_thread(func)) + active_tasks[task] = (p_name, pt, start_time_probe) + threshold = routing_memory.get_p75_latency(routing_key, p_name) / 1000.0 + + while active_tasks: + elapsed = time.time() - start_time_probe + if i < len(eligible) - 1 and elapsed >= threshold: + break + + # Calculate timeout: use remaining time until threshold, or None if no threshold + remaining = threshold - elapsed if i < len(eligible) - 1 else None + done, _ = await asyncio.wait( + active_tasks.keys(), + timeout=remaining, + return_when=asyncio.FIRST_COMPLETED, + ) + found_final = False + for task_done in list(done): + if task_done not in active_tasks: + continue + p_name_done, pt_done, s_time = active_tasks.pop(task_done) + latency = int((time.time() - s_time) * 1000) + budget.record_attempt(is_paid=pt_done.is_paid(), latency_ms=latency) + try: + res = task_done.result() + except Exception as e: + err_type = _detect_error_type(e) + if err_type not in ( + ErrorType.AUTH_ERROR, + ErrorType.SSRF_BLOCKED, + ErrorType.BOT_CHALLENGE, + ): + circuit_breakers.record_failure(p_name_done) + metrics.record_provider(pt_done, latency, False) + continue + if res: + content = res.content if isinstance(res, ResolvedResult) else str(res) + q_score = scripts.quality.score_content(content) + if _accept(q_score, pt_done): + circuit_breakers.record_success(p_name_done) + metrics.record_provider(pt_done, latency, True) + routing_memory.record( + routing_key, p_name_done, True, latency, q_score.score + ) + + if result_builder: + result_dict = result_builder( + res, target, p_name_done, metrics, q_score.score + ) + elif isinstance(res, ResolvedResult): + res.metrics, res.score = metrics, q_score.score + result_dict = res.to_dict() + else: + result_dict = { + "source": p_name_done, + "content": content, + "metrics": asdict(metrics), + "score": q_score.score, + } + + if pt_done.is_paid(): + semantic_cache_store(target, result_dict) + return result_dict + else: + if not state["best_free_result"] or q_score.score > state[ + "best_free_result" + ].get("score", 0.0): + state["best_free_result"] = result_dict + + if q_score.score >= budget.min_free_quality_to_skip_paid: + metrics.quality_gate = {"passed": True, "score": q_score.score} + result_dict["metrics"] = asdict(metrics) + semantic_cache_store(target, result_dict) + return dict(result_dict) + else: + scripts.cache_negative.write_negative_cache( + cache, target, p_name_done, "thin_content" + ) + routing_memory.record( + routing_key, p_name_done, False, latency, q_score.score + ) + else: + circuit_breakers.record_failure(p_name_done) + metrics.record_provider(pt_done, latency, False) + + if found_final: + return None + if done: + break + if not active_tasks: + break + + if state["best_free_result"]: + return dict(state["best_free_result"]) + return None + + # Run async cascade + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None + + if loop and loop.is_running(): + import concurrent.futures + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + result = pool.submit(asyncio.run, _run_cascade()).result() + else: + result = asyncio.run(_run_cascade()) + + if result: + yield result + else: + yield { + "source": "none", + target_key: target, + "content": "Failed", + "error": f"No resolution method available. Stop reason: {budget.stop_reason}", + } diff --git a/.agents/skills/do-web-doc-resolver/scripts/_cascade_async.py b/.agents/skills/do-web-doc-resolver/scripts/_cascade_async.py new file mode 100644 index 00000000..d20a9bb5 --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/_cascade_async.py @@ -0,0 +1,254 @@ +"""Async cascade resolution logic for query and URL resolution.""" + +import asyncio +import logging +import time +from collections.abc import AsyncGenerator, Callable +from dataclasses import asdict +from typing import Any + +import scripts.cache_negative +import scripts.quality +from scripts.circuit_breaker import CircuitBreakerRegistry +from scripts.models import ( + ErrorType, + ProviderType, + ReadonlyResolverProtocol, + ResolvedResult, + ResolveMetrics, +) +from scripts.routing import ResolutionBudget +from scripts.routing_memory import RoutingMemory +from scripts.utils import _detect_error_type, _get_cache + +logger = logging.getLogger(__name__) + +# Early exit threshold: if quality score exceeds this, return immediately +# without trying more providers. This trades potential marginal quality +# gains for significant latency reduction. +EXCELLENT_QUALITY_THRESHOLD = 0.85 + + +async def cascade_stream_async( + target: str, + cascade_map: dict[str, tuple[ProviderType, ReadonlyResolverProtocol]], + eligible: list[str], + budget: ResolutionBudget, + metrics: ResolveMetrics, + routing_memory: RoutingMemory, + circuit_breakers: CircuitBreakerRegistry, + semantic_cache_store: Callable[[str, dict], bool], + routing_key: str, + result_builder: Callable[[Any, str, str, ResolveMetrics, float], dict[str, Any]] | None = None, + skip_providers: set[str] | None = None, + content_acceptable: Callable[[Any, ProviderType], bool] | None = None, + target_key: str = "query", +) -> AsyncGenerator[dict[str, Any]]: + """Async version of cascade_stream using asyncio tasks with true parallel launch.""" + skip = skip_providers or set() + cache = _get_cache() + _accept = content_acceptable or (lambda q, pt: q.acceptable) + + async def _store(key: str, result: dict) -> None: + """Persist to the semantic cache off the event loop. + + Encoding is CPU-bound, so doing it inline would block every other + coroutine in the process. + """ + try: + await asyncio.to_thread(semantic_cache_store, key, result) + except Exception as exc: # pragma: no cover - cache writes are best-effort + logger.debug("Semantic cache store failed for %s: %s", key, exc) + + # Pre-filter eligible providers and prepare tasks + tasks_to_launch: list[tuple[str, ProviderType, Callable]] = [] + for p_name in eligible: + if p_name in skip: + continue + pt, func = cascade_map[p_name] + + if not budget.can_try(is_paid=pt.is_paid()): + if budget.stop_reason in ("paid_disabled", "max_paid_attempts"): + continue + break + if scripts.cache_negative.should_skip_from_negative_cache(cache, target, p_name): + continue + if circuit_breakers.is_open(p_name): + continue + + tasks_to_launch.append((p_name, pt, func)) + + if not tasks_to_launch: + yield { + "source": "none", + target_key: target, + "content": "Failed", + "error": f"No providers available. Stop reason: {budget.stop_reason}", + } + return + + # Free providers are launched together; paid providers are held back until + # the free tier has actually failed to clear the quality gate. Launching + # everything at once burned paid credits on every async resolution even + # when a free provider would have satisfied the gate. + free_tasks = [t for t in tasks_to_launch if not t[1].is_paid()] + paid_tasks = [t for t in tasks_to_launch if t[1].is_paid()] + + active_tasks: dict[asyncio.Task, tuple[str, ProviderType, float]] = {} + best_free_result: dict[str, Any] | None = None + paid_launched = False + deadline = ( + time.time() + budget.max_total_latency_ms / 1000.0 if budget.max_total_latency_ms else None + ) + + def _launch(batch: list[tuple[str, ProviderType, Callable]]) -> None: + for p_name, pt, func in batch: + logger.info("Starting parallel probe: %s", p_name) + start_time_probe = time.time() + task = asyncio.create_task(func()) + active_tasks[task] = (p_name, pt, start_time_probe) + + _launch(free_tasks) + if not paid_tasks: + paid_launched = True + elif not free_tasks: + # Nothing free to wait on (e.g. a paid-only plan or profile): the paid + # tier is the only option, so it goes first. + _launch(paid_tasks) + paid_launched = True + + try: + while active_tasks: + # Wait for any task to complete, bounded by the remaining latency + # budget so a hung provider cannot stall past the profile deadline. + timeout = None + if deadline is not None: + timeout = deadline - time.time() + if timeout <= 0: + logger.info("Latency budget exhausted; cancelling remaining probes") + break + done, _ = await asyncio.wait( + active_tasks.keys(), + timeout=timeout, + return_when=asyncio.FIRST_COMPLETED, + ) + if not done: + # Budget expired with nothing to show for it. + break + + for task_done in list(done): + if task_done not in active_tasks: + continue + p_name_done, pt_done, s_time = active_tasks.pop(task_done) + latency = int((time.time() - s_time) * 1000) + budget.record_attempt(is_paid=pt_done.is_paid(), latency_ms=latency) + try: + res = task_done.result() + except Exception as e: + err_type = _detect_error_type(e) + if err_type not in ( + ErrorType.AUTH_ERROR, + ErrorType.SSRF_BLOCKED, + ErrorType.BOT_CHALLENGE, + ): + circuit_breakers.record_failure(p_name_done) + metrics.record_provider(pt_done, latency, False) + continue + if res: + content = res.content if isinstance(res, ResolvedResult) else str(res) + q_score = scripts.quality.score_content(content) + if _accept(q_score, pt_done): + circuit_breakers.record_success(p_name_done) + metrics.record_provider(pt_done, latency, True) + routing_memory.record( + routing_key, p_name_done, True, latency, q_score.score + ) + + if result_builder: + result_dict = result_builder( + res, target, p_name_done, metrics, q_score.score + ) + elif isinstance(res, ResolvedResult): + res.metrics, res.score = metrics, q_score.score + result_dict = res.to_dict() + else: + result_dict = { + "source": p_name_done, + "content": content, + "metrics": asdict(metrics), + "score": q_score.score, + } + + if pt_done.is_paid(): + # Paid provider succeeded - return immediately + await _store(target, result_dict) + yield result_dict + return + else: + if not best_free_result or q_score.score > best_free_result.get( + "score", 0.0 + ): + best_free_result = result_dict + + # Early exit: excellent quality + if q_score.score >= EXCELLENT_QUALITY_THRESHOLD: + logger.info( + "Early exit: excellent quality %.2f from %s", + q_score.score, + p_name_done, + ) + metrics.quality_gate = { + "passed": True, + "score": q_score.score, + "early_exit": True, + } + result_dict["metrics"] = asdict(metrics) + await _store(target, result_dict) + yield result_dict + return + + # Quality gate: skip paid if free result is good enough + if q_score.score >= budget.min_free_quality_to_skip_paid: + metrics.quality_gate = {"passed": True, "score": q_score.score} + result_dict["metrics"] = asdict(metrics) + await _store(target, result_dict) + yield result_dict + return + else: + scripts.cache_negative.write_negative_cache( + cache, target, p_name_done, "thin_content" + ) + routing_memory.record( + routing_key, p_name_done, False, latency, q_score.score + ) + else: + circuit_breakers.record_failure(p_name_done) + metrics.record_provider(pt_done, latency, False) + + # Free tier exhausted without clearing the gate: escalate to paid. + if not paid_launched and not active_tasks: + best_score = best_free_result.get("score", 0.0) if best_free_result else 0.0 + if best_score < budget.min_free_quality_to_skip_paid: + logger.info( + "Free tier best score %.2f below gate %.2f; launching paid", + best_score, + budget.min_free_quality_to_skip_paid, + ) + _launch(paid_tasks) + paid_launched = True + + finally: + for task in active_tasks: + task.cancel() + + if best_free_result: + best_free_result["metrics"] = asdict(metrics) + await _store(target, best_free_result) + yield best_free_result + else: + yield { + "source": "none", + target_key: target, + "content": "Failed", + "error": f"No resolution method available. Stop reason: {budget.stop_reason}", + } diff --git a/.agents/skills/do-web-doc-resolver/scripts/_query_resolve.py b/.agents/skills/do-web-doc-resolver/scripts/_query_resolve.py new file mode 100644 index 00000000..b330336b --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/_query_resolve.py @@ -0,0 +1,133 @@ +"""Query resolution - resolve_query and resolve_query_stream.""" + +import logging +from collections.abc import Generator +from typing import Any + +import scripts.routing +from scripts._cascade import cascade_stream +from scripts.models import Profile, ProviderType, ReadonlyResolverProtocol, ResolveMetrics +from scripts.providers_impl import ( + resolve_with_duckduckgo, + resolve_with_exa, + resolve_with_exa_mcp, + resolve_with_mistral_websearch, + resolve_with_serper, + resolve_with_tavily, +) +from scripts.semantic_cache import get_semantic_cache +from scripts.state import circuit_breakers as _circuit_breakers +from scripts.state import routing_memory as _routing_memory + +logger = logging.getLogger(__name__) + + +def _check_semantic_cache(query_or_url: str) -> dict[str, Any] | None: + """Check semantic cache for similar query/URL.""" + cache = get_semantic_cache() + if cache is None: + return None + + try: + entry = cache.query(query_or_url) + if entry: + logger.info( + f"Semantic cache hit for '{query_or_url[:50]}...' (similarity: {entry.similarity:.3f})" + ) + result = dict(entry.result) + result["semantic_cache_hit"] = True + result["semantic_similarity"] = entry.similarity + result["semantic_original_query"] = entry.query + return result + except Exception as e: + logger.debug(f"Semantic cache check failed: {e}") + + return None + + +def _store_in_semantic_cache(query_or_url: str, result: dict[str, Any]) -> bool: + """Store a successful result in the semantic cache.""" + cache = get_semantic_cache() + if cache is None: + return False + + if result.get("source") == "none" or result.get("semantic_cache_hit"): + return False + + try: + return cache.store(query_or_url, result) + except Exception as e: + logger.debug(f"Failed to store in semantic cache: {e}") + return False + + +__all__ = [ + "resolve_query", + "resolve_query_stream", +] + + +def resolve_query( + query: str, + max_chars: int = 8000, + skip_providers: set[str] | None = None, + profile: Profile = Profile.BALANCED, +) -> dict[str, Any]: + for result in resolve_query_stream(query, max_chars, skip_providers, profile): + if result.get("source") != "partial": + return result + return {"source": "none", "query": query, "content": "Failed"} + + +def resolve_query_stream( + query: str, + max_chars: int = 8000, + skip_providers: set[str] | None = None, + profile: Profile = Profile.BALANCED, +) -> Generator[dict[str, Any]]: + skip = skip_providers or set() + + cached_result = _check_semantic_cache(query) + if cached_result: + cached_result["query"] = query + yield cached_result + return + + metrics = ResolveMetrics() + budget_data = scripts.routing.PROFILE_BUDGETS.get( + profile.value, scripts.routing.PROFILE_BUDGETS["balanced"] + ) + budget = scripts.routing.ResolutionBudget( + max_provider_attempts=int(budget_data["max_provider_attempts"]), + max_paid_attempts=int(budget_data["max_paid_attempts"]), + max_total_latency_ms=int(budget_data["max_total_latency_ms"]), + min_free_quality_to_skip_paid=float(budget_data.get("min_free_quality_to_skip_paid", 0.70)), + allow_paid=bool(budget_data["allow_paid"]), + ) + provider_names = scripts.routing.plan_provider_order( + target=query, is_url=False, skip_providers=skip, routing_memory=_routing_memory + ) + cascade_map: dict[str, tuple[ProviderType, ReadonlyResolverProtocol]] = { + "exa_mcp": (ProviderType.EXA_MCP, lambda: resolve_with_exa_mcp(query, max_chars)), + "exa": (ProviderType.EXA, lambda: resolve_with_exa(query, max_chars)), + "tavily": (ProviderType.TAVILY, lambda: resolve_with_tavily(query, max_chars)), + "serper": (ProviderType.SERPER, lambda: resolve_with_serper(query, max_chars)), + "duckduckgo": (ProviderType.DUCKDUCKGO, lambda: resolve_with_duckduckgo(query, max_chars)), + "mistral_websearch": ( + ProviderType.MISTRAL_WEBSEARCH, + lambda: resolve_with_mistral_websearch(query, max_chars), + ), + } + eligible = [p for p in provider_names if p in cascade_map] + + yield from cascade_stream( + target=query, + cascade_map=cascade_map, + eligible=eligible, + budget=budget, + metrics=metrics, + routing_memory=_routing_memory, + circuit_breakers=_circuit_breakers, + semantic_cache_store=_store_in_semantic_cache, + routing_key=query, + ) diff --git a/.agents/skills/do-web-doc-resolver/scripts/_routing_utils.py b/.agents/skills/do-web-doc-resolver/scripts/_routing_utils.py new file mode 100644 index 00000000..b465aaa1 --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/_routing_utils.py @@ -0,0 +1,17 @@ +import logging + +logger = logging.getLogger(__name__) + +DEFAULT_PROVIDER_STATS = { + "success": 0, + "failure": 0, + "avg_latency_ms": 0.0, + "avg_quality": 0.0, + "last_attempted": None, +} + + +def compute_p75_latency(avg_latency_ms: float, default: int = 3000) -> int: + if avg_latency_ms == 0: + return default + return int(avg_latency_ms * 1.5) diff --git a/.agents/skills/do-web-doc-resolver/scripts/_url_resolve.py b/.agents/skills/do-web-doc-resolver/scripts/_url_resolve.py new file mode 100644 index 00000000..c952d90d --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/_url_resolve.py @@ -0,0 +1,212 @@ +"""URL resolution - resolve_url and resolve_url_stream.""" + +import logging +from collections.abc import Generator +from dataclasses import asdict +from typing import Any + +import scripts.routing +from scripts._cascade import cascade_stream +from scripts.constants import PROVIDER_TIERS +from scripts.models import ( + FetchTier, + Profile, + ProviderType, + ReadonlyResolverProtocol, + ResolvedResult, + ResolveMetrics, +) +from scripts.providers_impl import ( + resolve_with_docling, + resolve_with_duckduckgo, + resolve_with_firecrawl, + resolve_with_jina, + resolve_with_mistral_browser, + resolve_with_ocr, + resolve_with_stealth, + resolve_with_visual_clip, +) +from scripts.semantic_cache import get_semantic_cache +from scripts.state import circuit_breakers as _circuit_breakers +from scripts.state import routing_memory as _routing_memory +from scripts.utils import ( + compact_content, + fetch_llms_txt, + fetch_url_content, +) + +logger = logging.getLogger(__name__) + + +def _check_semantic_cache(query_or_url: str) -> dict[str, Any] | None: + """Check semantic cache for similar query/URL.""" + cache = get_semantic_cache() + if cache is None: + return None + + try: + entry = cache.query(query_or_url) + if entry: + logger.info( + f"Semantic cache hit for '{query_or_url[:50]}...' (similarity: {entry.similarity:.3f})" + ) + result = dict(entry.result) + result["semantic_cache_hit"] = True + result["semantic_similarity"] = entry.similarity + result["semantic_original_query"] = entry.query + return result + except Exception as e: + logger.debug(f"Semantic cache check failed: {e}") + + return None + + +def _store_in_semantic_cache(query_or_url: str, result: dict[str, Any]) -> bool: + """Store a successful result in the semantic cache.""" + cache = get_semantic_cache() + if cache is None: + return False + + if result.get("source") == "none" or result.get("semantic_cache_hit"): + return False + + try: + return cache.store(query_or_url, result) + except Exception as e: + logger.debug(f"Failed to store in semantic cache: {e}") + return False + + +__all__ = [ + "resolve_url", + "resolve_url_stream", +] + + +def resolve_url( + url: str, + max_chars: int = 8000, + profile: Profile = Profile.BALANCED, + query: str | None = None, + skip_providers: set[str] | None = None, +) -> dict[str, Any]: + for result in resolve_url_stream( + url, max_chars, profile, query=query, skip_providers=skip_providers + ): + if result.get("source") != "partial": + return result + return {"source": "none", "url": url, "content": "Failed"} + + +def resolve_url_stream( + url: str, + max_chars: int = 8000, + profile: Profile = Profile.BALANCED, + query: str | None = None, + skip_providers: set[str] | None = None, +) -> Generator[dict[str, Any]]: + logger.info(f"Resolving URL: {url}") + + cached_result = _check_semantic_cache(url) + if cached_result: + cached_result["url"] = url + yield cached_result + return + + metrics = ResolveMetrics() + budget_data = scripts.routing.PROFILE_BUDGETS.get( + profile.value, scripts.routing.PROFILE_BUDGETS["balanced"] + ) + budget = scripts.routing.ResolutionBudget( + max_provider_attempts=int(budget_data["max_provider_attempts"]), + max_paid_attempts=int(budget_data["max_paid_attempts"]), + max_total_latency_ms=int(budget_data["max_total_latency_ms"]), + min_free_quality_to_skip_paid=float(budget_data.get("min_free_quality_to_skip_paid", 0.70)), + allow_paid=bool(budget_data["allow_paid"]), + ) + + if any(url.lower().endswith(ext) for ext in [".pdf", ".docx", ".pptx"]): + res = resolve_with_docling(url, max_chars) + if res: + res.metrics = metrics + yield res.to_dict() + return + if any(url.lower().endswith(ext) for ext in [".png", ".jpg", ".jpeg"]): + res = resolve_with_ocr(url, max_chars) + if res: + res.metrics = metrics + yield res.to_dict() + return + + provider_names = scripts.routing.plan_provider_order( + target=url, is_url=True, routing_memory=_routing_memory, skip_providers=skip_providers + ) + cascade_map: dict[str, tuple[ProviderType, ReadonlyResolverProtocol]] = { + "llms_txt": (ProviderType.LLMS_TXT, lambda: fetch_llms_txt(url)), + "jina": (ProviderType.JINA, lambda: resolve_with_jina(url, max_chars)), + "firecrawl": (ProviderType.FIRECRAWL, lambda: resolve_with_firecrawl(url, max_chars)), + "direct_fetch": ( + ProviderType.DIRECT_FETCH, + lambda: fetch_url_content(url, max_chars=max_chars), + ), + "mistral_browser": ( + ProviderType.MISTRAL_BROWSER, + lambda: resolve_with_mistral_browser(url, max_chars), + ), + "visual_clip": ( + ProviderType.VISUAL_CLIP, + lambda: resolve_with_visual_clip(url, max_chars, query=query), + ), + "duckduckgo": (ProviderType.DUCKDUCKGO, lambda: resolve_with_duckduckgo(url, max_chars)), + "stealth": ( + ProviderType.DIRECT_FETCH, + lambda: resolve_with_stealth(url, max_chars), + ), + } + + domain = scripts.routing.extract_domain(url) + + def _sort_by_tier(provider_name: str) -> int: + val = PROVIDER_TIERS.get(provider_name, FetchTier.PAID_BROWSER) + return int(val) + + eligible = sorted( + [p for p in provider_names if p in cascade_map], + key=_sort_by_tier, + ) + + def _url_result_builder(res, target_url, p_name, met, score): + if isinstance(res, ResolvedResult): + res.metrics, res.score = met, score + return res.to_dict() + elif p_name == "llms_txt": + return { + "source": "llms.txt", + "url": target_url, + "content": compact_content(str(res), max_chars), + "metrics": asdict(met), + "score": score, + } + else: + return { + "source": p_name, + "url": target_url, + "content": str(res), + "metrics": asdict(met), + "score": score, + } + + yield from cascade_stream( + target=url, + cascade_map=cascade_map, + eligible=eligible, + budget=budget, + metrics=metrics, + routing_memory=_routing_memory, + circuit_breakers=_circuit_breakers, + semantic_cache_store=_store_in_semantic_cache, + routing_key=domain or "any", + result_builder=_url_result_builder, + content_acceptable=lambda q, pt: q.acceptable or pt == ProviderType.LLMS_TXT, + target_key="url", + ) diff --git a/.agents/skills/do-web-doc-resolver/scripts/_url_resolve_async.py b/.agents/skills/do-web-doc-resolver/scripts/_url_resolve_async.py new file mode 100644 index 00000000..a7a7fc0b --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/_url_resolve_async.py @@ -0,0 +1,206 @@ +"""Async URL resolution - resolve_url_async and resolve_url_stream_async.""" + +import asyncio +import logging +from collections.abc import AsyncGenerator +from dataclasses import asdict +from typing import Any + +import scripts.routing +from scripts._cascade_async import cascade_stream_async +from scripts.constants import PROVIDER_TIERS +from scripts.models import FetchTier, Profile, ProviderType, ResolvedResult, ResolveMetrics +from scripts.providers.jina import resolve_with_jina_async +from scripts.providers.visual_clip import resolve_with_visual_clip_async +from scripts.providers_impl import ( + resolve_with_duckduckgo, + resolve_with_firecrawl, + resolve_with_mistral_browser, + resolve_with_stealth, +) +from scripts.semantic_cache import get_semantic_cache +from scripts.state import circuit_breakers as _circuit_breakers +from scripts.state import routing_memory as _routing_memory +from scripts.utils import compact_content +from scripts.utils.fetch import fetch_url_content + +logger = logging.getLogger(__name__) + + +async def resolve_url_async( + url: str, + max_chars: int = 8000, + profile: Profile = Profile.BALANCED, + query: str | None = None, + skip_providers: set[str] | None = None, +) -> dict[str, Any]: + """Async version of resolve_url.""" + async for result in resolve_url_stream_async( + url, max_chars, profile, query=query, skip_providers=skip_providers + ): + if result.get("source") != "partial": + return result + return {"source": "none", "url": url, "content": "Failed"} + + +async def resolve_url_stream_async( + url: str, + max_chars: int = 8000, + profile: Profile = Profile.BALANCED, + query: str | None = None, + skip_providers: set[str] | None = None, +) -> AsyncGenerator[dict[str, Any]]: + """Async generator version of resolve_url_stream.""" + logger.info(f"Resolving URL async: {url}") + + # Embedding encode (and the first-call model load) is CPU/IO heavy, so it + # must not run on the event loop thread where it would stall every other + # concurrent task in the process. + cached_result = await asyncio.to_thread(_check_semantic_cache, url) + if cached_result: + cached_result["url"] = url + yield cached_result + return + + metrics = ResolveMetrics() + budget_data = scripts.routing.PROFILE_BUDGETS.get( + profile.value, scripts.routing.PROFILE_BUDGETS["balanced"] + ) + budget = scripts.routing.ResolutionBudget( + max_provider_attempts=int(budget_data["max_provider_attempts"]), + max_paid_attempts=int(budget_data["max_paid_attempts"]), + max_total_latency_ms=int(budget_data["max_total_latency_ms"]), + min_free_quality_to_skip_paid=float(budget_data.get("min_free_quality_to_skip_paid", 0.70)), + allow_paid=bool(budget_data["allow_paid"]), + ) + + provider_names = scripts.routing.plan_provider_order( + target=url, is_url=True, routing_memory=_routing_memory, skip_providers=skip_providers + ) + + # Async provider map — native async + asyncio.to_thread() bridges for sync providers + cascade_map: dict[str, tuple[ProviderType, Any]] = { + "jina": (ProviderType.JINA, lambda: resolve_with_jina_async(url, max_chars)), + "visual_clip": ( + ProviderType.VISUAL_CLIP, + lambda: resolve_with_visual_clip_async(url, max_chars, query=query), + ), + "stealth": ( + ProviderType.DIRECT_FETCH, + lambda: resolve_with_stealth(url, max_chars), + ), + "firecrawl": ( + ProviderType.FIRECRAWL, + lambda: asyncio.to_thread(resolve_with_firecrawl, url, max_chars), + ), + "direct_fetch": ( + ProviderType.DIRECT_FETCH, + lambda: asyncio.to_thread(fetch_url_content, url, max_chars), + ), + "mistral_browser": ( + ProviderType.MISTRAL_BROWSER, + lambda: asyncio.to_thread(resolve_with_mistral_browser, url, max_chars), + ), + "duckduckgo": ( + ProviderType.DUCKDUCKGO, + lambda: asyncio.to_thread(resolve_with_duckduckgo, query or url, max_chars), + ), + } + + domain = scripts.routing.extract_domain(url) + + def _sort_by_tier(provider_name: str) -> int: + val = PROVIDER_TIERS.get(provider_name, FetchTier.PAID_BROWSER) + return int(val) + + eligible = sorted( + [p for p in provider_names if p in cascade_map], + key=_sort_by_tier, + ) + + if not eligible: + # Fall back to sync cascade for providers not yet converted + logger.info("No async providers eligible, falling back to sync cascade") + from scripts._url_resolve import resolve_url_stream + + for result in resolve_url_stream( + url, max_chars, profile, query=query, skip_providers=skip_providers + ): + yield result + return + + def _url_result_builder(res, target_url, p_name, met, score): + if isinstance(res, ResolvedResult): + res.metrics, res.score = met, score + return res.to_dict() + elif p_name == "llms_txt": + return { + "source": "llms.txt", + "url": target_url, + "content": compact_content(str(res), max_chars), + "metrics": asdict(met), + "score": score, + } + else: + return { + "source": p_name, + "url": target_url, + "content": str(res), + "metrics": asdict(met), + "score": score, + } + + async for result in cascade_stream_async( + target=url, + cascade_map=cascade_map, + eligible=eligible, + budget=budget, + metrics=metrics, + routing_memory=_routing_memory, + circuit_breakers=_circuit_breakers, + semantic_cache_store=_store_in_semantic_cache, + routing_key=domain or "any", + result_builder=_url_result_builder, + content_acceptable=lambda q, pt: q.acceptable or pt == ProviderType.LLMS_TXT, + target_key="url", + ): + yield result + + +def _check_semantic_cache(query_or_url: str) -> dict[str, Any] | None: + """Check semantic cache for similar query/URL.""" + cache = get_semantic_cache() + if cache is None: + return None + + try: + entry = cache.query(query_or_url) + if entry: + logger.info( + f"Semantic cache hit for '{query_or_url[:50]}...' (similarity: {entry.similarity:.3f})" + ) + result = dict(entry.result) + result["semantic_cache_hit"] = True + result["semantic_similarity"] = entry.similarity + result["semantic_original_query"] = entry.query + return result + except Exception as e: + logger.debug(f"Semantic cache check failed: {e}") + + return None + + +def _store_in_semantic_cache(query_or_url: str, result: dict[str, Any]) -> bool: + """Store a successful result in the semantic cache.""" + cache = get_semantic_cache() + if cache is None: + return False + + if result.get("source") == "none" or result.get("semantic_cache_hit"): + return False + + try: + return cache.store(query_or_url, result) + except Exception as e: + logger.debug(f"Failed to store in semantic cache: {e}") + return False diff --git a/.agents/skills/do-web-doc-resolver/scripts/cli.py b/.agents/skills/do-web-doc-resolver/scripts/cli.py new file mode 100755 index 00000000..850e8698 --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/cli.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +""" +CLI entrypoint for Web Doc Resolver. +""" + +import argparse +import asyncio +import json +import logging +import os + +# Persist learned per-domain provider preferences across CLI runs (AUDIT #25). +# Must be set before importing scripts.state (via scripts.resolve), which builds +# the routing-memory singleton at import time. +from scripts.constants import CACHE_DIR + +os.environ.setdefault("DO_WDR_ROUTING_MEMORY_PATH", os.path.join(CACHE_DIR, "routing_memory.json")) + +from scripts.models import Profile, ProviderType # noqa: E402 +from scripts.resolve import ( # noqa: E402 + MAX_CHARS, + is_url, + resolve_direct, + resolve_query_stream, + resolve_url_stream_async, + resolve_with_order, +) + + +async def _async_main(args): + """Async main function for URL resolution.""" + profile = Profile(args.profile) + skip = set(args.skip) if args.skip else None + + if args.provider: + results = [resolve_direct(args.input, ProviderType(args.provider), args.max_chars)] + elif args.providers_order: + order = [ProviderType(p.strip()) for p in args.providers_order.split(",")] + results = [resolve_with_order(args.input, order, args.max_chars)] + else: + if is_url(args.input): + # Use async URL resolver + results = [] + async for res in resolve_url_stream_async( + args.input, args.max_chars, profile, skip_providers=skip + ): + results.append(res) + else: + results = list(resolve_query_stream(args.input, args.max_chars, skip, profile)) + return results + + +def main(): + parser = argparse.ArgumentParser(description="Web Doc Resolver") + parser.add_argument("input", nargs="?", help="URL or query") + parser.add_argument("--max-chars", type=int, default=MAX_CHARS) + parser.add_argument("--json", action="store_true") + parser.add_argument( + "--profile", type=str, choices=[p.value for p in Profile], default="balanced" + ) + parser.add_argument("--skip", action="append") + parser.add_argument("--provider", type=str, choices=[p.value for p in ProviderType]) + parser.add_argument("--providers-order", type=str) + parser.add_argument("--log-level", default="INFO") + args = parser.parse_args() + logging.basicConfig(level=getattr(logging, args.log_level)) + if not args.input: + parser.error("Input required") + + # Use asyncio.run for URL resolution + if is_url(args.input) and not args.provider and not args.providers_order: + results = asyncio.run(_async_main(args)) + else: + profile = Profile(args.profile) + skip = set(args.skip) if args.skip else None + if args.provider: + results = [resolve_direct(args.input, ProviderType(args.provider), args.max_chars)] + elif args.providers_order: + order = [ProviderType(p.strip()) for p in args.providers_order.split(",")] + results = [resolve_with_order(args.input, order, args.max_chars)] + else: + results = list(resolve_query_stream(args.input, args.max_chars, skip, profile)) + + final_result = None + for res in results: + if not args.json and res.get("source") != "partial": + print(f"--- Source: {res.get('source')} ---") + print(res.get("content", "")[:500] + "...") + final_result = res + if args.json: + print( + json.dumps( + final_result, + indent=2, + default=lambda o: o.__dict__ if hasattr(o, "__dict__") else str(o), + ) + ) + else: + print("\n=== FINAL RESULT ===") + if final_result: + print(final_result.get("content", "")) + + +if __name__ == "__main__": + main() diff --git a/.agents/skills/do-web-doc-resolver/scripts/providers/__init__.py b/.agents/skills/do-web-doc-resolver/scripts/providers/__init__.py new file mode 100644 index 00000000..8c8bd018 --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/providers/__init__.py @@ -0,0 +1,71 @@ +""" +Provider implementations for the Web Doc Resolver. +""" + +import threading +import time + +from scripts.providers.docling import resolve_with_docling, resolve_with_ocr +from scripts.providers.duckduckgo import resolve_with_duckduckgo +from scripts.providers.exa import resolve_with_exa, resolve_with_exa_mcp +from scripts.providers.firecrawl import resolve_with_firecrawl +from scripts.providers.jina import resolve_with_jina +from scripts.providers.mistral import resolve_with_mistral_browser, resolve_with_mistral_websearch +from scripts.providers.serper import resolve_with_serper +from scripts.providers.stealth import resolve_with_stealth +from scripts.providers.tavily import resolve_with_tavily +from scripts.providers.visual_clip import ( + resolve_with_visual_clip, + resolve_with_visual_clip_async, +) + +# Rate limiting functions +_rate_limits: dict[str, float] = {} +_rate_limits_lock = threading.Lock() + + +def _is_rate_limited(provider: str) -> bool: + with _rate_limits_lock: + if provider in _rate_limits: + if time.time() < _rate_limits[provider]: + return True + del _rate_limits[provider] + return False + + +def _set_rate_limit(provider: str, cooldown: int = 60): + with _rate_limits_lock: + _rate_limits[provider] = time.time() + cooldown + + +def _clear_rate_limits() -> None: + with _rate_limits_lock: + _rate_limits.clear() + + +# Exported names for both internal use and tests +is_rate_limited = _is_rate_limited +set_rate_limit = _set_rate_limit + +__all__ = [ + "resolve_with_jina", + "resolve_with_exa", + "resolve_with_exa_mcp", + "resolve_with_tavily", + "resolve_with_serper", + "resolve_with_duckduckgo", + "resolve_with_firecrawl", + "resolve_with_mistral_browser", + "resolve_with_mistral_websearch", + "resolve_with_docling", + "resolve_with_ocr", + "resolve_with_visual_clip", + "resolve_with_stealth", + "resolve_with_visual_clip_async", + "_is_rate_limited", + "_set_rate_limit", + "_clear_rate_limits", + "_rate_limits", + "is_rate_limited", + "set_rate_limit", +] diff --git a/.agents/skills/do-web-doc-resolver/scripts/providers/docling.py b/.agents/skills/do-web-doc-resolver/scripts/providers/docling.py new file mode 100644 index 00000000..eff9b360 --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/providers/docling.py @@ -0,0 +1,96 @@ +""" +Docling and OCR provider implementations. +""" + +import asyncio +import logging +import subprocess + +from scripts.models import ResolvedResult +from scripts.utils import is_safe_url +from scripts.utils.async_http import is_safe_url as async_is_safe_url + +logger = logging.getLogger(__name__) + + +async def resolve_with_docling_async(url: str, max_chars: int) -> ResolvedResult | None: + """Async version of Docling resolver using asyncio subprocess.""" + if not async_is_safe_url(url): + logger.warning("SSRF blocked: %s", url) + return None + try: + proc = await asyncio.create_subprocess_exec( + "docling", + "--format", + "markdown", + url, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=60) + if proc.returncode == 0: + return ResolvedResult(source="docling", content=stdout.decode()[:max_chars], url=url) + except TimeoutError: + logger.warning("Docling timed out for URL: %s", url) + except (OSError, subprocess.SubprocessError) as e: + logger.warning("Docling resolution failed: %s: %s", type(e).__name__, e) + return None + + +async def resolve_with_ocr_async(url: str, max_chars: int) -> ResolvedResult | None: + """Async version of OCR resolver using asyncio subprocess.""" + if not async_is_safe_url(url): + logger.warning("SSRF blocked: %s", url) + return None + try: + proc = await asyncio.create_subprocess_exec( + "tesseract", + url, + "stdout", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=30) + if proc.returncode == 0: + return ResolvedResult( + source="ocr-tesseract", content=stdout.decode()[:max_chars], url=url + ) + except TimeoutError: + logger.warning("OCR timed out for URL: %s", url) + except (OSError, subprocess.SubprocessError) as e: + logger.warning("OCR resolution failed: %s: %s", type(e).__name__, e) + return None + + +def resolve_with_docling(url: str, max_chars: int) -> ResolvedResult | None: + if not is_safe_url(url): + logger.warning("SSRF blocked: %s", url) + return None + try: + res = subprocess.run( + ["docling", "--format", "markdown", url], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + if res.returncode == 0: + return ResolvedResult(source="docling", content=res.stdout[:max_chars], url=url) + except (subprocess.SubprocessError, OSError) as e: + logger.warning("Docling resolution failed: %s: %s", type(e).__name__, e) + return None + + +def resolve_with_ocr(url: str, max_chars: int) -> ResolvedResult | None: + if not is_safe_url(url): + logger.warning("SSRF blocked: %s", url) + return None + try: + res = subprocess.run( + ["tesseract", url, "stdout"], capture_output=True, text=True, timeout=30, check=False + ) + if res.returncode == 0: + return ResolvedResult(source="ocr-tesseract", content=res.stdout[:max_chars], url=url) + except (subprocess.SubprocessError, OSError) as e: + logger.warning("OCR resolution failed: %s: %s", type(e).__name__, e) + return None diff --git a/.agents/skills/do-web-doc-resolver/scripts/providers/duckduckgo.py b/.agents/skills/do-web-doc-resolver/scripts/providers/duckduckgo.py new file mode 100644 index 00000000..c874a2f7 --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/providers/duckduckgo.py @@ -0,0 +1,77 @@ +""" +DuckDuckGo provider implementation. +""" + +import asyncio +import logging + +from scripts.constants import DDG_RESULTS, MAX_CHARS +from scripts.models import ResolvedResult +from scripts.utils import _get_from_cache, _save_to_cache + +logger = logging.getLogger(__name__) + + +async def resolve_with_duckduckgo_async( + query: str, max_chars: int = MAX_CHARS +) -> ResolvedResult | None: + """Async version of DuckDuckGo resolver.""" + cached = _get_from_cache(query, "duckduckgo") + if cached: + return ResolvedResult(**cached) + from scripts.providers_impl import _is_rate_limited + + if _is_rate_limited("duckduckgo"): + logger.debug("DuckDuckGo skipped: rate limited") + return None + try: + from ddgs import DDGS + + def _sync_search(): + with DDGS() as ddgs: + return list(ddgs.text(query, max_results=DDG_RESULTS)) + + from scripts.utils.thread_pool import get_shared_pool + + loop = asyncio.get_event_loop() + results = await loop.run_in_executor(get_shared_pool(), _sync_search) + if not results: + logger.warning("DuckDuckGo returned no results for query: %s", query) + return None + content = "\n\n---\n\n".join( + [f"## {r.get('title', '')}\n\n{r.get('body', '')}" for r in results] + ) + result = ResolvedResult(source="duckduckgo", content=content[:max_chars], query=query) + _save_to_cache(query, "duckduckgo", result.to_dict()) + return result + except Exception as e: + logger.warning("DuckDuckGo resolution failed: %s: %s", type(e).__name__, e) + return None + + +def resolve_with_duckduckgo(query: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: + cached = _get_from_cache(query, "duckduckgo") + if cached: + return ResolvedResult(**cached) + from scripts.providers_impl import _is_rate_limited + + if _is_rate_limited("duckduckgo"): + logger.debug("DuckDuckGo skipped: rate limited") + return None + try: + from ddgs import DDGS + + with DDGS() as ddgs: + results = list(ddgs.text(query, max_results=DDG_RESULTS)) + if not results: + logger.warning("DuckDuckGo returned no results for query: %s", query) + return None + content = "\n\n---\n\n".join( + [f"## {r.get('title', '')}\n\n{r.get('body', '')}" for r in results] + ) + result = ResolvedResult(source="duckduckgo", content=content[:max_chars], query=query) + _save_to_cache(query, "duckduckgo", result.to_dict()) + return result + except Exception as e: + logger.warning("DuckDuckGo resolution failed: %s: %s", type(e).__name__, e) + return None diff --git a/.agents/skills/do-web-doc-resolver/scripts/providers/exa.py b/.agents/skills/do-web-doc-resolver/scripts/providers/exa.py new file mode 100644 index 00000000..c9e7cb83 --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/providers/exa.py @@ -0,0 +1,220 @@ +""" +Exa provider implementation. +""" + +import asyncio +import json +import logging +import os + +import httpx + +from scripts.constants import EXA_RESULTS, MAX_CHARS +from scripts.models import ResolvedResult +from scripts.utils import _get_from_cache, _save_to_cache, get_session +from scripts.utils.async_http import get_async_client + +logger = logging.getLogger(__name__) + + +def _get_result_text(r: object) -> str | None: + highlight = getattr(r, "highlight", None) + if highlight: + return str(highlight) + highlights = getattr(r, "highlights", None) + if highlights: + return highlights[0] if isinstance(highlights, list) else str(highlights) + text = getattr(r, "text", None) + return str(text) if text else None + + +async def resolve_with_exa_mcp_async( + query: str, max_chars: int = MAX_CHARS +) -> ResolvedResult | None: + """Async version of Exa MCP resolver using httpx.""" + cached = _get_from_cache(query, "exa_mcp") + if cached: + return ResolvedResult(**cached) + from scripts.providers import _is_rate_limited + + if _is_rate_limited("exa_mcp"): + return None + try: + mcp_request = { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "web_search_exa", "arguments": {"query": query, "numResults": 8}}, + } + client = await get_async_client() + response = await client.post( + "https://mcp.exa.ai/mcp", + json=mcp_request, + headers={"Accept": "application/json, text/event-stream"}, + timeout=25, + ) + if response.status_code != 200: + logger.warning("Exa MCP HTTP %s for query: %s", response.status_code, query) + return None + for line in response.text.split("\n"): + if line.startswith("data: "): + data = json.loads(line[6:]) + if data.get("result") and data["result"].get("content"): + content = data["result"]["content"][0].get("text", "") + if not content: + logger.warning("Exa MCP returned empty content for query: %s", query) + return None + result = ResolvedResult( + source="exa_mcp", content=content[:max_chars], query=query + ) + _save_to_cache(query, "exa_mcp", result.to_dict()) + return result + logger.warning("Exa MCP returned no usable content for query: %s", query) + except json.JSONDecodeError as e: + logger.warning("Exa MCP JSON parse failed: %s", e) + except httpx.RequestError as e: + logger.warning("Exa MCP resolution failed: %s: %s", type(e).__name__, e) + return None + + +async def resolve_with_exa_async(query: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: + """Async version of Exa resolver.""" + cached = _get_from_cache(query, "exa") + if cached: + return ResolvedResult(**cached) + api_key = os.getenv("EXA_API_KEY") + if not api_key: + logger.debug("Exa skipped: no API key") + return None + from scripts.providers import _is_rate_limited, _set_rate_limit + + if _is_rate_limited("exa"): + logger.debug("Exa skipped: rate limited") + return None + try: + from exa_py import Exa + + def _sync_search(): + client = Exa(api_key) + return client.search_and_contents( + query, use_autoprompt=True, highlights=True, num_results=EXA_RESULTS + ) + + from scripts.utils.thread_pool import get_shared_pool + + loop = asyncio.get_event_loop() + res = await loop.run_in_executor(get_shared_pool(), _sync_search) + if not res or not res.results: + logger.warning("Exa returned no results for query: %s", query) + return None + extracted = [_get_result_text(r) for r in res.results] + content = "\n\n---\n\n".join([t for t in extracted if t]) + if not content: + logger.warning("Exa returned empty content for query: %s", query) + return None + result = ResolvedResult(source="exa", content=content[:max_chars], query=query) + _save_to_cache(query, "exa", result.to_dict()) + return result + except Exception as e: + status = getattr(e, "status_code", None) + if status == 401: + logger.warning("Exa failed: 401 Unauthorized — API key may be invalid or expired") + elif status == 429: + logger.warning("Exa failed: 429 Rate limited — setting cooldown") + _set_rate_limit("exa") + elif status == 403: + logger.warning("Exa failed: 403 Forbidden — %s", e) + else: + logger.warning("Exa resolution failed: %s: %s", type(e).__name__, e) + return None + + +def resolve_with_exa_mcp(query: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: + cached = _get_from_cache(query, "exa_mcp") + if cached: + return ResolvedResult(**cached) + from scripts.providers import _is_rate_limited + + if _is_rate_limited("exa_mcp"): + return None + try: + mcp_request = { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "web_search_exa", "arguments": {"query": query, "numResults": 8}}, + } + client = get_session() + response = client.post( + "https://mcp.exa.ai/mcp", + json=mcp_request, + headers={"Accept": "application/json, text/event-stream"}, + timeout=25, + ) + if response.status_code != 200: + logger.warning("Exa MCP HTTP %s for query: %s", response.status_code, query) + return None + for line in response.text.split("\n"): + if line.startswith("data: "): + data = json.loads(line[6:]) + if data.get("result") and data["result"].get("content"): + content = data["result"]["content"][0].get("text", "") + if not content: + logger.warning("Exa MCP returned empty content for query: %s", query) + return None + result = ResolvedResult( + source="exa_mcp", content=content[:max_chars], query=query + ) + _save_to_cache(query, "exa_mcp", result.to_dict()) + return result + logger.warning("Exa MCP returned no usable content for query: %s", query) + except json.JSONDecodeError as e: + logger.warning("Exa MCP JSON parse failed: %s", e) + except httpx.RequestError as e: + logger.warning("Exa MCP resolution failed: %s: %s", type(e).__name__, e) + return None + + +def resolve_with_exa(query: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: + cached = _get_from_cache(query, "exa") + if cached: + return ResolvedResult(**cached) + api_key = os.getenv("EXA_API_KEY") + if not api_key: + logger.debug("Exa skipped: no API key") + return None + from scripts.providers import _is_rate_limited, _set_rate_limit + + if _is_rate_limited("exa"): + logger.debug("Exa skipped: rate limited") + return None + try: + from exa_py import Exa + + client = Exa(api_key) + res = client.search_and_contents( + query, use_autoprompt=True, highlights=True, num_results=EXA_RESULTS + ) + if not res or not res.results: + logger.warning("Exa returned no results for query: %s", query) + return None + extracted = [_get_result_text(r) for r in res.results] + content = "\n\n---\n\n".join([t for t in extracted if t]) + if not content: + logger.warning("Exa returned empty content for query: %s", query) + return None + result = ResolvedResult(source="exa", content=content[:max_chars], query=query) + _save_to_cache(query, "exa", result.to_dict()) + return result + except Exception as e: + status = getattr(e, "status_code", None) + if status == 401: + logger.warning("Exa failed: 401 Unauthorized — API key may be invalid or expired") + elif status == 429: + logger.warning("Exa failed: 429 Rate limited — setting cooldown") + _set_rate_limit("exa") + elif status == 403: + logger.warning("Exa failed: 403 Forbidden — %s", e) + else: + logger.warning("Exa resolution failed: %s: %s", type(e).__name__, e) + return None diff --git a/.agents/skills/do-web-doc-resolver/scripts/providers/firecrawl.py b/.agents/skills/do-web-doc-resolver/scripts/providers/firecrawl.py new file mode 100644 index 00000000..3683a30c --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/providers/firecrawl.py @@ -0,0 +1,113 @@ +""" +Firecrawl provider implementation. +""" + +import asyncio +import logging +import os + +from scripts.constants import MAX_CHARS +from scripts.models import ResolvedResult +from scripts.utils import _get_from_cache, _save_to_cache, is_safe_url +from scripts.utils.async_http import is_safe_url as async_is_safe_url + +logger = logging.getLogger(__name__) + + +async def resolve_with_firecrawl_async( + url: str, max_chars: int = MAX_CHARS +) -> ResolvedResult | None: + """Async version of Firecrawl resolver.""" + if not async_is_safe_url(url): + logger.warning("SSRF blocked: %s", url) + return None + cached = _get_from_cache(url, "firecrawl") + if cached: + return ResolvedResult(**cached) + api_key = os.getenv("FIRECRAWL_API_KEY") + if not api_key: + logger.debug("Firecrawl skipped: no API key") + return None + from scripts.providers import _is_rate_limited, _set_rate_limit + + if _is_rate_limited("firecrawl"): + logger.debug("Firecrawl skipped: rate limited") + return None + try: + from firecrawl import Firecrawl + + def _sync_scrape(): + app = Firecrawl(api_key=api_key) + return app.scrape(url, formats=["markdown"]) + + from scripts.utils.thread_pool import get_shared_pool + + loop = asyncio.get_event_loop() + res = await loop.run_in_executor(get_shared_pool(), _sync_scrape) + if not res or not hasattr(res, "markdown"): + logger.warning("Firecrawl returned no markdown for URL: %s", url) + return None + markdown = res.markdown + if not markdown: + logger.warning("Firecrawl returned empty markdown for URL: %s", url) + return None + result = ResolvedResult(source="firecrawl", content=markdown[:max_chars], url=url) + _save_to_cache(url, "firecrawl", result.to_dict()) + return result + except Exception as e: + status = getattr(e, "status_code", None) + if status == 401: + logger.warning("Firecrawl failed: 401 Unauthorized — API key may be invalid or expired") + elif status == 429: + logger.warning("Firecrawl failed: 429 Rate limited — setting cooldown") + _set_rate_limit("firecrawl") + elif status == 403: + logger.warning("Firecrawl failed: 403 Forbidden — %s", e) + else: + logger.warning("Firecrawl resolution failed: %s: %s", type(e).__name__, e) + return None + + +def resolve_with_firecrawl(url: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: + if not is_safe_url(url): + logger.warning("SSRF blocked: %s", url) + return None + cached = _get_from_cache(url, "firecrawl") + if cached: + return ResolvedResult(**cached) + api_key = os.getenv("FIRECRAWL_API_KEY") + if not api_key: + logger.debug("Firecrawl skipped: no API key") + return None + from scripts.providers import _is_rate_limited, _set_rate_limit + + if _is_rate_limited("firecrawl"): + logger.debug("Firecrawl skipped: rate limited") + return None + try: + from firecrawl import Firecrawl + + app = Firecrawl(api_key=api_key) + res = app.scrape(url, formats=["markdown"]) + if not res or not hasattr(res, "markdown"): + logger.warning("Firecrawl returned no markdown for URL: %s", url) + return None + markdown = res.markdown + if not markdown: + logger.warning("Firecrawl returned empty markdown for URL: %s", url) + return None + result = ResolvedResult(source="firecrawl", content=markdown[:max_chars], url=url) + _save_to_cache(url, "firecrawl", result.to_dict()) + return result + except Exception as e: + status = getattr(e, "status_code", None) + if status == 401: + logger.warning("Firecrawl failed: 401 Unauthorized — API key may be invalid or expired") + elif status == 429: + logger.warning("Firecrawl failed: 429 Rate limited — setting cooldown") + _set_rate_limit("firecrawl") + elif status == 403: + logger.warning("Firecrawl failed: 403 Forbidden — %s", e) + else: + logger.warning("Firecrawl resolution failed: %s: %s", type(e).__name__, e) + return None diff --git a/.agents/skills/do-web-doc-resolver/scripts/providers/jina.py b/.agents/skills/do-web-doc-resolver/scripts/providers/jina.py new file mode 100644 index 00000000..2c0cb197 --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/providers/jina.py @@ -0,0 +1,101 @@ +""" +Jina provider implementation. +""" + +import logging + +import httpx + +from scripts.constants import DEFAULT_TIMEOUT, MAX_CHARS, MIN_CHARS +from scripts.models import ResolvedResult +from scripts.utils import _get_from_cache, _save_to_cache, get_session, is_safe_url +from scripts.utils.async_http import get_async_client +from scripts.utils.async_http import is_safe_url as async_is_safe_url + +logger = logging.getLogger(__name__) + + +async def resolve_with_jina_async(url: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: + """Async version of Jina resolver using httpx.""" + if not async_is_safe_url(url): + logger.warning("SSRF blocked: %s", url) + return None + cached = _get_from_cache(url, "jina") + if cached: + return ResolvedResult(**cached) + from scripts.providers import _is_rate_limited, _set_rate_limit + + if _is_rate_limited("jina"): + return None + try: + client = await get_async_client() + response = await client.get( + f"https://r.jina.ai/{url}", + timeout=DEFAULT_TIMEOUT, + headers={"Accept": "text/markdown"}, + ) + if response.status_code == 429: + logger.warning("Jina rate limited — setting cooldown") + _set_rate_limit("jina") + return None + if response.status_code in (401, 403): + logger.warning("Jina auth error: HTTP %s for %s", response.status_code, url) + return None + if response.status_code != 200: + logger.warning("Jina HTTP %s for %s", response.status_code, url) + return None + content = response.text.strip() + if len(content) < MIN_CHARS: + logger.warning( + "Jina returned insufficient content (%s chars) for %s", len(content), url + ) + return None + result = ResolvedResult(source="jina", content=content[:max_chars], url=url) + _save_to_cache(url, "jina", result.to_dict()) + return result + except (httpx.RequestError, OSError, TimeoutError) as e: + logger.warning("Jina resolution failed: %s: %s", type(e).__name__, e) + return None + + +def resolve_with_jina(url: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: + """Sync version of Jina resolver using httpx (backward compatible).""" + if not is_safe_url(url): + logger.warning("SSRF blocked: %s", url) + return None + cached = _get_from_cache(url, "jina") + if cached: + return ResolvedResult(**cached) + from scripts.providers import _is_rate_limited, _set_rate_limit + + if _is_rate_limited("jina"): + return None + try: + client = get_session() + response = client.get( + f"https://r.jina.ai/{url}", + timeout=DEFAULT_TIMEOUT, + headers={"Accept": "text/markdown"}, + ) + if response.status_code == 429: + logger.warning("Jina rate limited — setting cooldown") + _set_rate_limit("jina") + return None + if response.status_code in (401, 403): + logger.warning("Jina auth error: HTTP %s for %s", response.status_code, url) + return None + if response.status_code != 200: + logger.warning("Jina HTTP %s for %s", response.status_code, url) + return None + content = response.text.strip() + if len(content) < MIN_CHARS: + logger.warning( + "Jina returned insufficient content (%s chars) for %s", len(content), url + ) + return None + result = ResolvedResult(source="jina", content=content[:max_chars], url=url) + _save_to_cache(url, "jina", result.to_dict()) + return result + except (httpx.RequestError, OSError, TimeoutError) as e: + logger.warning("Jina resolution failed: %s: %s", type(e).__name__, e) + return None diff --git a/.agents/skills/do-web-doc-resolver/scripts/providers/mistral.py b/.agents/skills/do-web-doc-resolver/scripts/providers/mistral.py new file mode 100644 index 00000000..06c6ece1 --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/providers/mistral.py @@ -0,0 +1,298 @@ +""" +Mistral provider implementation. +""" + +import asyncio +import logging +import os + +from scripts.constants import MAX_CHARS +from scripts.models import ResolvedResult +from scripts.utils import _get_from_cache, _save_to_cache, is_safe_url +from scripts.utils.async_http import is_safe_url as async_is_safe_url + +logger = logging.getLogger(__name__) + + +async def resolve_with_mistral_browser_async( + url: str, max_chars: int = MAX_CHARS +) -> ResolvedResult | None: + """Async version of Mistral browser resolver.""" + if not async_is_safe_url(url): + logger.warning("SSRF blocked: %s", url) + return None + cached = _get_from_cache(url, "mistral_browser") + if cached: + return ResolvedResult(**cached) + api_key = os.getenv("MISTRAL_API_KEY") + if not api_key: + logger.debug("Mistral browser skipped: no API key") + return None + from scripts.providers import _is_rate_limited, _set_rate_limit + + if _is_rate_limited("mistral"): + logger.debug("Mistral browser skipped: rate limited") + return None + try: + from mistralai.client import Mistral + + def _sync_extract(): + client = Mistral(api_key=api_key) + agent = client.beta.agents.create( + model="mistral-small-latest", + name="url-extractor", + instructions="Extract and summarize content from web pages. Return clean markdown.", + tools=[{"type": "web_search"}], # type: ignore[arg-type] + ) + try: + result = client.beta.conversations.start( + agent_id=agent.id, + inputs=f"Extract the main content from this URL and return it as markdown: {url}", + ) + content = "" + for entry in result.outputs: + if hasattr(entry, "content") and entry.content is not None: + if isinstance(entry.content, str): + content += entry.content + elif isinstance(entry.content, list): + for chunk in entry.content: + if hasattr(chunk, "text") and chunk.text: + content += chunk.text + elif isinstance(chunk, str): + content += chunk + return content + finally: + try: + client.beta.agents.delete(agent_id=agent.id) + except Exception as e: + logger.warning("Mistral browser agent cleanup failed: %s", e) + + from scripts.utils.thread_pool import get_shared_pool + + loop = asyncio.get_event_loop() + content = await loop.run_in_executor(get_shared_pool(), _sync_extract) + if not content: + logger.warning("Mistral browser returned empty content for URL: %s", url) + return None + + resolved = ResolvedResult(source="mistral-browser", content=content[:max_chars], url=url) + _save_to_cache(url, "mistral_browser", resolved.to_dict()) + return resolved + except Exception as e: + status = getattr(e, "status_code", None) + if status == 401: + logger.warning( + "Mistral browser failed: 401 Unauthorized — API key may be invalid or expired" + ) + elif status == 429: + logger.warning("Mistral browser failed: 429 Rate limited — setting cooldown") + _set_rate_limit("mistral") + elif status == 403: + logger.warning("Mistral browser failed: 403 Forbidden — %s", e) + else: + logger.warning("Mistral browser failed: %s: %s", type(e).__name__, e) + return None + + +async def resolve_with_mistral_websearch_async( + query: str, max_chars: int = MAX_CHARS +) -> ResolvedResult | None: + """Async version of Mistral websearch resolver.""" + cached = _get_from_cache(query, "mistral_websearch") + if cached: + return ResolvedResult(**cached) + api_key = os.getenv("MISTRAL_API_KEY") + if not api_key: + logger.debug("Mistral websearch skipped: no API key") + return None + from scripts.providers import _is_rate_limited, _set_rate_limit + + if _is_rate_limited("mistral"): + logger.debug("Mistral websearch skipped: rate limited") + return None + try: + from mistralai.client import Mistral + from mistralai.client.models import UserMessage + + def _sync_search(): + client = Mistral(api_key=api_key) + return client.chat.complete( + model="mistral-small-latest", + messages=[UserMessage(content=f"Search: {query}")], # type: ignore[arg-type] + ) + + from scripts.utils.thread_pool import get_shared_pool + + loop = asyncio.get_event_loop() + resp = await loop.run_in_executor(get_shared_pool(), _sync_search) + content = "" + if resp.choices and resp.choices[0].message and resp.choices[0].message.content: + msg_content = resp.choices[0].message.content + if isinstance(msg_content, str): + content = msg_content + elif isinstance(msg_content, list): + for chunk in msg_content: + if hasattr(chunk, "text") and chunk.text: + content += chunk.text + elif isinstance(chunk, str): + content += chunk + if not content: + logger.warning("Mistral websearch returned empty content for query: %s", query) + return None + result = ResolvedResult( + source="mistral-websearch", content=content[:max_chars], query=query + ) + _save_to_cache(query, "mistral_websearch", result.to_dict()) + return result + except Exception as e: + status = getattr(e, "status_code", None) + if status == 401: + logger.warning( + "Mistral websearch failed: 401 Unauthorized — API key may be invalid or expired" + ) + elif status == 429: + logger.warning("Mistral websearch failed: 429 Rate limited — setting cooldown") + _set_rate_limit("mistral") + elif status == 403: + logger.warning("Mistral websearch failed: 403 Forbidden — %s", e) + else: + logger.warning("Mistral websearch failed: %s: %s", type(e).__name__, e) + return None + + +def resolve_with_mistral_browser(url: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: + if not is_safe_url(url): + logger.warning("SSRF blocked: %s", url) + return None + cached = _get_from_cache(url, "mistral_browser") + if cached: + return ResolvedResult(**cached) + api_key = os.getenv("MISTRAL_API_KEY") + if not api_key: + logger.debug("Mistral browser skipped: no API key") + return None + from scripts.providers import _is_rate_limited, _set_rate_limit + + if _is_rate_limited("mistral"): + logger.debug("Mistral browser skipped: rate limited") + return None + try: + from mistralai.client import Mistral + + client = Mistral(api_key=api_key) + + # Create an agent with web_search tool + agent = client.beta.agents.create( + model="mistral-small-latest", + name="url-extractor", + instructions="Extract and summarize content from web pages. Return clean markdown.", + tools=[{"type": "web_search"}], # type: ignore[arg-type] + ) + + try: + # Start conversation to extract the URL + result = client.beta.conversations.start( + agent_id=agent.id, + inputs=f"Extract the main content from this URL and return it as markdown: {url}", + ) + + content = "" + for entry in result.outputs: + if hasattr(entry, "content") and entry.content is not None: + # In newer mistralai, content might be a list of chunks + if isinstance(entry.content, str): + content += entry.content + elif isinstance(entry.content, list): + for chunk in entry.content: + if hasattr(chunk, "text") and chunk.text: + content += chunk.text + elif isinstance(chunk, str): + content += chunk + + if not content: + logger.warning("Mistral browser returned empty content for URL: %s", url) + return None + + resolved = ResolvedResult( + source="mistral-browser", content=content[:max_chars], url=url + ) + _save_to_cache(url, "mistral_browser", resolved.to_dict()) + return resolved + finally: + # Clean up the agent + try: + client.beta.agents.delete(agent_id=agent.id) + except Exception as e: + logger.warning("Mistral browser agent cleanup failed: %s", e) + except Exception as e: + status = getattr(e, "status_code", None) + if status == 401: + logger.warning( + "Mistral browser failed: 401 Unauthorized — API key may be invalid or expired" + ) + elif status == 429: + logger.warning("Mistral browser failed: 429 Rate limited — setting cooldown") + _set_rate_limit("mistral") + elif status == 403: + logger.warning("Mistral browser failed: 403 Forbidden — %s", e) + else: + logger.warning("Mistral browser failed: %s: %s", type(e).__name__, e) + return None + + +def resolve_with_mistral_websearch(query: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: + cached = _get_from_cache(query, "mistral_websearch") + if cached: + return ResolvedResult(**cached) + api_key = os.getenv("MISTRAL_API_KEY") + if not api_key: + logger.debug("Mistral websearch skipped: no API key") + return None + from scripts.providers import _is_rate_limited, _set_rate_limit + + if _is_rate_limited("mistral"): + logger.debug("Mistral websearch skipped: rate limited") + return None + try: + from mistralai.client import Mistral + from mistralai.client.models import UserMessage + + client = Mistral(api_key=api_key) + resp = client.chat.complete( + model="mistral-small-latest", + messages=[UserMessage(content=f"Search: {query}")], # type: ignore[arg-type] + ) + content = "" + if resp.choices and resp.choices[0].message and resp.choices[0].message.content: + msg_content = resp.choices[0].message.content + if isinstance(msg_content, str): + content = msg_content + elif isinstance(msg_content, list): + # Handle list of chunks if necessary + for chunk in msg_content: + if hasattr(chunk, "text") and chunk.text: + content += chunk.text + elif isinstance(chunk, str): + content += chunk + if not content: + logger.warning("Mistral websearch returned empty content for query: %s", query) + return None + result = ResolvedResult( + source="mistral-websearch", content=content[:max_chars], query=query + ) + _save_to_cache(query, "mistral_websearch", result.to_dict()) + return result + except Exception as e: + status = getattr(e, "status_code", None) + if status == 401: + logger.warning( + "Mistral websearch failed: 401 Unauthorized — API key may be invalid or expired" + ) + elif status == 429: + logger.warning("Mistral websearch failed: 429 Rate limited — setting cooldown") + _set_rate_limit("mistral") + elif status == 403: + logger.warning("Mistral websearch failed: 403 Forbidden — %s", e) + else: + logger.warning("Mistral websearch failed: %s: %s", type(e).__name__, e) + return None diff --git a/.agents/skills/do-web-doc-resolver/scripts/providers/serper.py b/.agents/skills/do-web-doc-resolver/scripts/providers/serper.py new file mode 100644 index 00000000..00c682f5 --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/providers/serper.py @@ -0,0 +1,139 @@ +""" +Serper provider implementation. +""" + +import logging +import os + +import httpx + +from scripts.constants import DEFAULT_TIMEOUT, MAX_CHARS +from scripts.models import ResolvedResult +from scripts.utils import _get_from_cache, _save_to_cache, get_session +from scripts.utils.async_http import get_async_client + +logger = logging.getLogger(__name__) + + +async def resolve_with_serper_async( + query: str, max_chars: int = MAX_CHARS +) -> ResolvedResult | None: + """Async version of Serper resolver using httpx.""" + cached = _get_from_cache(query, "serper") + if cached: + return ResolvedResult(**cached) + api_key = os.getenv("SERPER_API_KEY") + if not api_key: + logger.debug("Serper skipped: no API key") + return None + from scripts.providers import _is_rate_limited, _set_rate_limit + + if _is_rate_limited("serper"): + logger.debug("Serper skipped: rate limited") + return None + try: + client = await get_async_client() + response = await client.post( + "https://google.serper.dev/search", + headers={ + "X-API-KEY": api_key, + "Content-Type": "application/json", + }, + json={"q": query, "num": 5}, + timeout=DEFAULT_TIMEOUT, + ) + if response.status_code == 429: + logger.warning("Serper rate limited — setting 1hr cooldown") + _set_rate_limit("serper", 3600) + return None + if response.status_code in (401, 403): + logger.warning( + "Serper auth error: HTTP %s — API key may be invalid", response.status_code + ) + return None + if response.status_code != 200: + logger.warning("Serper HTTP %s for query: %s", response.status_code, query) + return None + data = response.json() + organic = data.get("organic", []) + if not organic: + logger.warning("Serper returned no organic results for query: %s", query) + return None + parts = [] + for r in organic: + title = r.get("title", "") + link = r.get("link", "") + snippet = r.get("snippet", "") + if title and snippet: + parts.append(f"## {title}\n\n{snippet}\n\n[{link}]({link})") + if not parts: + logger.warning("Serper returned no usable snippets for query: %s", query) + return None + content = "\n\n---\n\n".join(parts) + result = ResolvedResult(source="serper", content=content[:max_chars], query=query) + _save_to_cache(query, "serper", result.to_dict()) + return result + except (httpx.RequestError, OSError, TimeoutError) as e: + logger.warning("Serper resolution failed: %s: %s", type(e).__name__, e) + return None + + +def resolve_with_serper(query: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: + """Search via Serper (Google Search API). Free tier: 2500 credits.""" + cached = _get_from_cache(query, "serper") + if cached: + return ResolvedResult(**cached) + api_key = os.getenv("SERPER_API_KEY") + if not api_key: + logger.debug("Serper skipped: no API key") + return None + from scripts.providers import _is_rate_limited, _set_rate_limit + + if _is_rate_limited("serper"): + logger.debug("Serper skipped: rate limited") + return None + try: + client = get_session() + response = client.post( + "https://google.serper.dev/search", + headers={ + "X-API-KEY": api_key, + "Content-Type": "application/json", + }, + json={"q": query, "num": 5}, + timeout=DEFAULT_TIMEOUT, + ) + if response.status_code == 429: + logger.warning("Serper rate limited — setting 1hr cooldown") + _set_rate_limit("serper", 3600) + return None + if response.status_code in (401, 403): + logger.warning( + "Serper auth error: HTTP %s — API key may be invalid", response.status_code + ) + return None + if response.status_code != 200: + logger.warning("Serper HTTP %s for query: %s", response.status_code, query) + return None + data = response.json() + organic = data.get("organic", []) + if not organic: + logger.warning("Serper returned no organic results for query: %s", query) + return None + parts = [] + for r in organic: + title = r.get("title", "") + link = r.get("link", "") + snippet = r.get("snippet", "") + if title and snippet: + parts.append(f"## {title}\n\n{snippet}\n\n[{link}]({link})") + if not parts: + logger.warning("Serper returned no usable snippets for query: %s", query) + return None + content = "\n\n---\n\n".join(parts) + result = ResolvedResult(source="serper", content=content[:max_chars], query=query) + _save_to_cache(query, "serper", result.to_dict()) + return result + except (httpx.RequestError, OSError, TimeoutError) as e: + logger.warning("Serper resolution failed: %s: %s", type(e).__name__, e) + return None diff --git a/.agents/skills/do-web-doc-resolver/scripts/providers/stealth.py b/.agents/skills/do-web-doc-resolver/scripts/providers/stealth.py new file mode 100644 index 00000000..762d0603 --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/providers/stealth.py @@ -0,0 +1,28 @@ +"""Stealth fetch provider — placeholder for anti-bot escalation tier. + +TODO: Implement using playwright-stealth or curl-impersonate. +Returns ``None`` so the cascade skips it cleanly. Returning an empty +``ResolvedResult`` here would be truthy: the cascade would score it 0.0 and +write a negative-cache "thin_content" entry that suppresses the STEALTH tier +slot for the whole TTL. +""" + +import logging + +from scripts.models import ResolvedResult + +logger = logging.getLogger(__name__) + + +def resolve_with_stealth(url: str, max_chars: int) -> ResolvedResult | None: + """Stealth browser fetch (anti-bot bypass). + + Returns ``None`` until a concrete implementation is chosen; the cascade + treats ``None`` as a provider miss and moves on to the next tier. + """ + logger.debug( + "Stealth provider not yet implemented — skipping %s " + "(candidates: playwright-stealth, curl-impersonate, camoufox)", + url, + ) + return None diff --git a/.agents/skills/do-web-doc-resolver/scripts/providers/tavily.py b/.agents/skills/do-web-doc-resolver/scripts/providers/tavily.py new file mode 100644 index 00000000..9c18d29b --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/providers/tavily.py @@ -0,0 +1,100 @@ +""" +Tavily provider implementation. +""" + +import asyncio +import logging +import os + +from scripts.constants import MAX_CHARS, TAVILY_RESULTS +from scripts.models import ResolvedResult +from scripts.utils import _get_from_cache, _save_to_cache + +logger = logging.getLogger(__name__) + + +async def resolve_with_tavily_async( + query: str, max_chars: int = MAX_CHARS +) -> ResolvedResult | None: + """Async version of Tavily resolver.""" + cached = _get_from_cache(query, "tavily") + if cached: + return ResolvedResult(**cached) + api_key = os.getenv("TAVILY_API_KEY") + if not api_key: + logger.debug("Tavily skipped: no API key") + return None + from scripts.providers import _is_rate_limited, _set_rate_limit + + if _is_rate_limited("tavily"): + logger.debug("Tavily skipped: rate limited") + return None + try: + from tavily import TavilyClient + + from scripts.utils.thread_pool import get_shared_pool + + def _sync_search(): + client = TavilyClient(api_key=api_key) + return client.search(query, max_results=TAVILY_RESULTS) + + loop = asyncio.get_event_loop() + res = await loop.run_in_executor(get_shared_pool(), _sync_search) + if not res or not res.get("results"): + logger.warning("Tavily returned no results for query: %s", query) + return None + content = "\n\n---\n\n".join([f"## {r['title']}\n\n{r['content']}" for r in res["results"]]) + result = ResolvedResult(source="tavily", content=content[:max_chars], query=query) + _save_to_cache(query, "tavily", result.to_dict()) + return result + except Exception as e: + status = getattr(e, "status_code", None) + if status == 401: + logger.warning("Tavily failed: 401 Unauthorized — API key may be invalid or expired") + elif status == 429: + logger.warning("Tavily failed: 429 Rate limited — setting cooldown") + _set_rate_limit("tavily") + elif status == 403: + logger.warning("Tavily failed: 403 Forbidden — %s", e) + else: + logger.warning("Tavily resolution failed: %s: %s", type(e).__name__, e) + return None + + +def resolve_with_tavily(query: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: + cached = _get_from_cache(query, "tavily") + if cached: + return ResolvedResult(**cached) + api_key = os.getenv("TAVILY_API_KEY") + if not api_key: + logger.debug("Tavily skipped: no API key") + return None + from scripts.providers import _is_rate_limited, _set_rate_limit + + if _is_rate_limited("tavily"): + logger.debug("Tavily skipped: rate limited") + return None + try: + from tavily import TavilyClient + + client = TavilyClient(api_key=api_key) + res = client.search(query, max_results=TAVILY_RESULTS) + if not res or not res.get("results"): + logger.warning("Tavily returned no results for query: %s", query) + return None + content = "\n\n---\n\n".join([f"## {r['title']}\n\n{r['content']}" for r in res["results"]]) + result = ResolvedResult(source="tavily", content=content[:max_chars], query=query) + _save_to_cache(query, "tavily", result.to_dict()) + return result + except Exception as e: + status = getattr(e, "status_code", None) + if status == 401: + logger.warning("Tavily failed: 401 Unauthorized — API key may be invalid or expired") + elif status == 429: + logger.warning("Tavily failed: 429 Rate limited — setting cooldown") + _set_rate_limit("tavily") + elif status == 403: + logger.warning("Tavily failed: 403 Forbidden — %s", e) + else: + logger.warning("Tavily resolution failed: %s: %s", type(e).__name__, e) + return None diff --git a/.agents/skills/do-web-doc-resolver/scripts/providers/visual_clip.py b/.agents/skills/do-web-doc-resolver/scripts/providers/visual_clip.py new file mode 100644 index 00000000..0b35993c --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/providers/visual_clip.py @@ -0,0 +1,118 @@ +""" +Visual CLIP provider implementation. +""" + +import logging +import os + +from scripts.constants import MAX_CHARS +from scripts.models import ResolvedResult +from scripts.utils import _get_from_cache, _save_to_cache, is_safe_url +from scripts.visual_resolver import VisualResolver + +logger = logging.getLogger(__name__) + +_visual_resolver: "VisualResolver | None" = None + + +def get_visual_resolver() -> VisualResolver: + """Lazy initialization of the VisualResolver.""" + global _visual_resolver + if _visual_resolver is None: + _visual_resolver = VisualResolver() + return _visual_resolver + + +def _is_api_available() -> bool: + """Check if required API keys for VLM are present.""" + return bool(os.getenv("MISTRAL_API_KEY") or os.getenv("OPENROUTER_API_KEY")) + + +def resolve_with_visual_clip( + url: str, max_chars: int = MAX_CHARS, query: str | None = None +) -> ResolvedResult | None: + """ + Resolve a URL using the Visual CLIP resolver (sync). + """ + if not is_safe_url(url): + logger.warning("SSRF blocked: %s", url) + return None + + if not _is_api_available(): + logger.debug("Visual CLIP skipped: no API key") + return None + + resolver = get_visual_resolver() + if not resolver.is_available(): + logger.debug("Visual CLIP skipped: dependencies missing or disabled") + return None + + cached = _get_from_cache(url, "visual_clip") + if cached: + return ResolvedResult(**cached) + + effective_query = query or "Extract the main content of this page." + + try: + res = resolver.resolve(url, effective_query) + if not res or res.content is None: + return None + + result = ResolvedResult( + source="visual_clip", + content=res.content[:max_chars], + url=url, + query=effective_query, + score=res.score, + metadata=res.metadata, + ) + _save_to_cache(url, "visual_clip", result.to_dict()) + return result + except Exception as e: + logger.warning("Visual CLIP resolution failed: %s", e) + return None + + +async def resolve_with_visual_clip_async( + url: str, max_chars: int = MAX_CHARS, query: str | None = None +) -> ResolvedResult | None: + """ + Resolve a URL using the Visual CLIP resolver (async). + """ + if not is_safe_url(url): + logger.warning("SSRF blocked: %s", url) + return None + + if not _is_api_available(): + logger.debug("Visual CLIP skipped: no API key") + return None + + resolver = get_visual_resolver() + if not resolver.is_available(): + logger.debug("Visual CLIP skipped: dependencies missing or disabled") + return None + + cached = _get_from_cache(url, "visual_clip") + if cached: + return ResolvedResult(**cached) + + effective_query = query or "Extract the main content of this page." + + try: + res = await resolver.resolve_async(url, effective_query) + if not res or res.content is None: + return None + + result = ResolvedResult( + source="visual_clip", + content=res.content[:max_chars], + url=url, + query=effective_query, + score=res.score, + metadata=res.metadata, + ) + _save_to_cache(url, "visual_clip", result.to_dict()) + return result + except Exception as e: + logger.warning("Visual CLIP resolution failed: %s", e) + return None diff --git a/.agents/skills/do-web-doc-resolver/scripts/quality.py b/.agents/skills/do-web-doc-resolver/scripts/quality.py index ff8db32f..5a497f59 100644 --- a/.agents/skills/do-web-doc-resolver/scripts/quality.py +++ b/.agents/skills/do-web-doc-resolver/scripts/quality.py @@ -2,6 +2,7 @@ Heuristics for scoring the quality of resolved content. """ +import re from dataclasses import dataclass # Quality scoring penalties @@ -168,12 +169,28 @@ def _compute_bonuses(score: float, has_frontmatter: bool, has_anchors: bool) -> return score +# Markdown inline links plus bare autolinks, used to infer link presence when the +# caller does not supply an explicit link list. +_MARKDOWN_LINK_RE = re.compile(r"\[[^\]]*\]\(\s*?\s*\)") +_AUTOLINK_RE = re.compile(r"\s]+>") + + +def extract_links(markdown: str) -> list[str]: + """Return the URLs referenced by markdown link syntax.""" + if not isinstance(markdown, str) or not markdown: + return [] + return _MARKDOWN_LINK_RE.findall(markdown) + _AUTOLINK_RE.findall(markdown) + + def score_content(markdown: str, links: list[str] | None = None) -> QualityScore: if not isinstance(markdown, str): return QualityScore(0.0, True, True, False, False, False) text = (markdown or "").strip() - links = links or [] + # The cascades score raw provider content without passing links, so infer + # them here. Without this, `missing_links` was always true and every result + # took a flat PENALTY_MISSING_LINKS hit, skewing every downstream threshold. + links = extract_links(text) if links is None else links too_short = len(text) < THRESHOLD_MIN_CHARS missing_links = len(links) == 0 diff --git a/.agents/skills/do-web-doc-resolver/scripts/routing_memory.py b/.agents/skills/do-web-doc-resolver/scripts/routing_memory.py index 3385cf9b..54675e83 100644 --- a/.agents/skills/do-web-doc-resolver/scripts/routing_memory.py +++ b/.agents/skills/do-web-doc-resolver/scripts/routing_memory.py @@ -31,7 +31,6 @@ def __init__(self, path: str | os.PathLike[str] | None = None) -> None: ) self._lock = threading.RLock() self._path = Path(path) if path is not None else None - self._dirty = False self._last_save = 0.0 if self._path is not None: self._load_from_disk() @@ -68,7 +67,6 @@ def _save_to_disk_unlocked(self) -> None: with tmp.open("w", encoding="utf-8") as fh: json.dump(data, fh, sort_keys=True) tmp.replace(self._path) - self._dirty = False self._last_save = time.time() except OSError as e: logger.warning("Failed to save routing memory to %s: %s", self._path, e) @@ -99,10 +97,10 @@ def record( stats["failure"] = f + 1 # Throttled auto-persist so a running CLI retains learned preferences. - if self._path is not None and ( - self._dirty is False or time.time() - self._last_save >= SAVE_INTERVAL_SECONDS - ): - self._dirty = True + # Only the elapsed interval gates the write. A `_dirty` flag would not: + # `_save_to_disk_unlocked()` resets it, so testing it here evaluated to + # True on every call and wrote JSON to disk on every single record(). + if self._path is not None and time.time() - self._last_save >= SAVE_INTERVAL_SECONDS: self._save_to_disk_unlocked() def get_domain_stats(self, provider: str, domain: str) -> dict[str, Any] | None: @@ -182,4 +180,3 @@ def get_p75_latency(self, domain: str, provider: str, default: int = 3000) -> in def clear(self) -> None: with self._lock: self.domain_stats.clear() - self._dirty = False diff --git a/.agents/skills/do-web-doc-resolver/scripts/semantic_cache.py b/.agents/skills/do-web-doc-resolver/scripts/semantic_cache.py new file mode 100644 index 00000000..ca3f8419 --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/semantic_cache.py @@ -0,0 +1,448 @@ +""" +Semantic Cache implementation using sqlite-vec + sentence-transformers. +""" + +import json +import logging +import os +import struct +import threading +import time +from dataclasses import dataclass, field +from typing import Any, cast + +try: + from pysqlite3 import dbapi2 as sqlite3 +except ImportError: + import sqlite3 + +from scripts.constants import ( + SEMANTIC_CACHE_MAX_ENTRIES, + SEMANTIC_CACHE_THRESHOLD, +) + +logger = logging.getLogger(__name__) +DEFAULT_MODEL = "all-MiniLM-L6-v2" + + +@dataclass +class SemanticCacheEntry: + query: str + result: dict[str, Any] + timestamp: float = field(default_factory=time.time) + similarity: float = 0.0 + + def to_dict(self) -> dict[str, Any]: + return { + "query": self.query, + "result": self.result, + "timestamp": self.timestamp, + "similarity": self.similarity, + } + + +class SemanticCache: + @staticmethod + def normalize_text(text: str, filter_stop_words: bool = False) -> str: + import re + + tokens = [w for w in re.split(r"[^a-zA-Z0-9]", text) if w] + is_url = text.startswith("http://") or text.startswith("https://") + if is_url: + url_stop = { + "https", + "http", + "www", + "html", + "htm", + "php", + "asp", + "aspx", + "jsp", + "docs", + "api", + "index", + } + tokens = [w for w in tokens if w.lower() not in url_stop] + elif filter_stop_words: + stop = { + "docs", + "documentation", + "guide", + "tutorial", + "reference", + "ref", + "lib", + "library", + "std", + "standard", + "for", + "of", + "the", + "a", + "an", + "and", + "programming", + "language", + "module", + "api", + } + tokens = [w for w in tokens if w.lower() not in stop] + if not tokens: + return " ".join(text.lower().split()) + return " ".join(sorted(w.lower() for w in tokens)) + + def __init__( + self, + cache_dir: str | None = None, + threshold: float = SEMANTIC_CACHE_THRESHOLD, + max_entries: int = SEMANTIC_CACHE_MAX_ENTRIES, + model_name: str = DEFAULT_MODEL, + ) -> None: + self.enabled = False + self._model: Any = None + self._model_name = model_name + self.threshold = threshold + self.max_entries = max_entries + self._embedding_dimension: int | None = None + + if cache_dir is None: + cache_dir = os.path.expanduser( + os.getenv( + "WEB_RESOLVER_SEMANTIC_CACHE_DIR", "~/.cache/do-web-doc-resolver/semantic" + ) + ) + self.cache_dir = cache_dir + os.makedirs(self.cache_dir, exist_ok=True) + self.db_path = os.path.join(self.cache_dir, "semantic_cache.db") + self._conn_lock = threading.RLock() + + try: + self._init_db() + self._init_model() + self.enabled = True + logger.info("Semantic cache initialized at %s", self.db_path) + except Exception as e: + logger.warning("Semantic cache initialization failed: %s. Cache disabled.", e) + self.enabled = False + + def _init_db(self) -> None: + self._conn = sqlite3.connect(self.db_path, check_same_thread=False) + self._conn.row_factory = sqlite3.Row + vec_loaded = False + try: + import sqlite_vec + + self._conn.enable_load_extension(True) + sqlite_vec.load(self._conn) + self._conn.enable_load_extension(False) + vec_loaded = True + logger.debug("sqlite-vec extension loaded successfully") + except ImportError: + logger.warning("sqlite-vec not installed, trying dynamic loading") + except Exception as e: + logger.warning("Failed to load sqlite-vec via Python API: %s", e) + + if not vec_loaded: + try: + self._conn.enable_load_extension(True) + for lib in [ + "libsqlite_vec.so", + "libsqlite_vec.dylib", + "sqlite_vec.so", + "sqlite_vec.dylib", + "libsqlite_vec", + ]: + try: + self._conn.execute(f"SELECT load_extension('{lib}')") + vec_loaded = True + logger.debug("Loaded sqlite-vec from %s", lib) + break + except sqlite3.OperationalError: + continue + self._conn.enable_load_extension(False) + except Exception as e: + logger.warning("Failed to load sqlite-vec dynamically: %s", e) + + if not vec_loaded: + raise RuntimeError("sqlite-vec extension could not be loaded") + + self._conn.execute(""" + CREATE TABLE IF NOT EXISTS cache_entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + query TEXT UNIQUE NOT NULL, + result_json TEXT NOT NULL, + timestamp REAL NOT NULL, + access_count INTEGER DEFAULT 1, + last_accessed REAL NOT NULL + ) + """) + self._conn.commit() + + def _init_model(self) -> None: + self._model = None + self._model_loading = False + + def _load_model(self) -> Any: + if self._model is None and not self._model_loading: + self._model_loading = True + try: + from sentence_transformers import SentenceTransformer + + logger.info("Loading sentence-transformers model: %s", self._model_name) + self._model = SentenceTransformer(self._model_name) + self._embedding_dimension = self._model.get_embedding_dimension() + logger.info("Model loaded. Embedding dimension: %s", self._embedding_dimension) + self._create_vector_table() + except Exception as e: + logger.error("Failed to load embedding model: %s", e) + raise + finally: + self._model_loading = False + return self._model + + def _create_vector_table(self) -> None: + if self._embedding_dimension is None: + return + with self._conn_lock: + self._conn.execute(f""" + CREATE VIRTUAL TABLE IF NOT EXISTS vec_cache USING vec0( + embedding float[{self._embedding_dimension}] + ) + """) + self._conn.commit() + + def _embedding_to_blob(self, embedding: list[float]) -> bytes: + return struct.pack(f"<{len(embedding)}f", *embedding) + + def _compute_embedding(self, text: str) -> list[float]: + model = self._load_model() + if model is None: + raise RuntimeError("Embedding model not available") + normalized = self.normalize_text(text, True) + embedding = model.encode(normalized, convert_to_numpy=True, normalize_embeddings=True) + return cast(list[float], embedding.tolist()) + + def query(self, query_str: str) -> SemanticCacheEntry | None: + if not self.enabled: + return None + try: + normalized = self.normalize_text(query_str, False) + with self._conn_lock: + cursor = self._conn.execute( + "SELECT id, query, result_json, timestamp FROM cache_entries WHERE query = ?", + (normalized,), + ) + row = cursor.fetchone() + if row: + self._conn.execute( + "UPDATE cache_entries SET access_count = access_count + 1, last_accessed = ? WHERE id = ?", + (time.time(), row["id"]), + ) + self._conn.commit() + return SemanticCacheEntry( + query=row["query"], + result=json.loads(row["result_json"]), + timestamp=row["timestamp"], + similarity=1.0, + ) + + query_embedding = self._compute_embedding(query_str) + embedding_blob = self._embedding_to_blob(query_embedding) + + with self._conn_lock: + cursor = self._conn.execute( + """ + SELECT ce.id, ce.query, ce.result_json, ce.timestamp, vc.distance + FROM vec_cache vc + JOIN cache_entries ce ON ce.id = vc.rowid + WHERE embedding MATCH ? + AND k = 1 + """, + (embedding_blob,), + ) + row = cursor.fetchone() + if row is None: + return None + + distance = row["distance"] + if distance is None: + distance = 2.0 + similarity = 1.0 - (distance * distance / 2.0) + if similarity < self.threshold: + return None + + self._conn.execute( + "UPDATE cache_entries SET access_count = access_count + 1, last_accessed = ? WHERE id = ?", + (time.time(), row["id"]), + ) + self._conn.commit() + + return SemanticCacheEntry( + query=row["query"], + result=json.loads(row["result_json"]), + timestamp=row["timestamp"], + similarity=similarity, + ) + except Exception as e: + logger.warning("Semantic cache query failed: %s", e) + return None + + def store(self, query_str: str, result: dict[str, Any]) -> bool: + if not self.enabled: + return False + try: + normalized = self.normalize_text(query_str, False) + embedding = self._compute_embedding(query_str) + embedding_blob = self._embedding_to_blob(embedding) + + with self._conn_lock: + cursor = self._conn.execute( + """ + SELECT ce.id, ce.result_json, vc.distance + FROM vec_cache vc + JOIN cache_entries ce ON ce.id = vc.rowid + WHERE embedding MATCH ? + AND k = 5 + """, + (embedding_blob,), + ) + for row in cursor.fetchall(): + distance = row["distance"] + if distance is None: + distance = 2.0 + similarity = 1.0 - (distance * distance / 2.0) + if similarity > 0.995: + logger.info("Skipping store: similarity %.4f", similarity) + return True + if similarity > 0.98 and row["result_json"] == json.dumps(result): + logger.info( + "Skipping store: identical result with similarity %.4f", similarity + ) + return True + + with self._conn_lock: + cursor = self._conn.execute( + "SELECT id FROM cache_entries WHERE query = ?", (normalized,) + ) + old_row = cursor.fetchone() + if old_row: + old_id = old_row["id"] + self._conn.execute("DELETE FROM vec_cache WHERE rowid = ?", (old_id,)) + self._conn.execute("DELETE FROM cache_entries WHERE id = ?", (old_id,)) + + cursor = self._conn.execute( + "INSERT INTO cache_entries (query, result_json, timestamp, last_accessed) VALUES (?, ?, ?, ?)", + (normalized, json.dumps(result), time.time(), time.time()), + ) + entry_id = cursor.lastrowid + self._conn.execute( + "INSERT INTO vec_cache (rowid, embedding) VALUES (?, ?)", + (entry_id, embedding_blob), + ) + self._conn.commit() + self._maybe_evict() + return True + except Exception as e: + logger.warning("Failed to store in semantic cache: %s", e) + return False + + def _maybe_evict(self) -> None: + with self._conn_lock: + try: + cursor = self._conn.execute("SELECT COUNT(*) as count FROM cache_entries") + count = cursor.fetchone()["count"] + if count > self.max_entries: + to_delete = count - self.max_entries + cursor = self._conn.execute( + "SELECT id FROM cache_entries ORDER BY last_accessed ASC, access_count ASC LIMIT ?", + (to_delete,), + ) + ids_to_delete = [row["id"] for row in cursor.fetchall()] + for entry_id in ids_to_delete: + self._conn.execute("DELETE FROM vec_cache WHERE rowid = ?", (entry_id,)) + self._conn.execute("DELETE FROM cache_entries WHERE id = ?", (entry_id,)) + self._conn.commit() + logger.info("Evicted %d old semantic cache entries", len(ids_to_delete)) + except Exception as e: + logger.warning("Cache eviction failed: %s", e) + + def close(self) -> None: + if hasattr(self, "_conn") and self._conn: + with self._conn_lock: + self._conn.close() + self._conn = None + + def clear(self) -> bool: + if not self.enabled: + return False + try: + with self._conn_lock: + self._conn.execute("DELETE FROM vec_cache") + self._conn.execute("DELETE FROM cache_entries") + self._conn.commit() + return True + except Exception as e: + logger.warning("Failed to clear semantic cache: %s", e) + return False + + def stats(self) -> dict[str, Any]: + if not self.enabled: + return {"enabled": False} + try: + with self._conn_lock: + cursor = self._conn.execute("SELECT COUNT(*) as count FROM cache_entries") + total_entries = cursor.fetchone()["count"] + cursor = self._conn.execute( + "SELECT AVG(access_count) as avg_access FROM cache_entries" + ) + avg_access = cursor.fetchone()["avg_access"] or 0 + return { + "enabled": True, + "total_entries": total_entries, + "max_entries": self.max_entries, + "threshold": self.threshold, + "model": self._model_name, + "embedding_dimension": self._embedding_dimension, + "avg_access_count": round(avg_access, 2), + "db_path": self.db_path, + } + except Exception as e: + return {"enabled": True, "error": str(e)} + + def __enter__(self) -> "SemanticCache": + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + self.close() + + +_semantic_cache_instance: SemanticCache | None = None +_semantic_cache_lock = threading.Lock() + + +def get_semantic_cache() -> SemanticCache | None: + global _semantic_cache_instance + if _semantic_cache_instance is None: + with _semantic_cache_lock: + if _semantic_cache_instance is None: + if os.environ.get("DO_WDR_SEMANTIC_CACHE", "1") != "1": + return None + try: + _semantic_cache_instance = SemanticCache( + threshold=SEMANTIC_CACHE_THRESHOLD, max_entries=SEMANTIC_CACHE_MAX_ENTRIES + ) + if not _semantic_cache_instance.enabled: + return None + except Exception as e: + logger.warning("Failed to initialize semantic cache: %s", e) + return None + return _semantic_cache_instance if _semantic_cache_instance.enabled else None + + +def reset_semantic_cache() -> None: + global _semantic_cache_instance + with _semantic_cache_lock: + if _semantic_cache_instance: + _semantic_cache_instance.close() + _semantic_cache_instance = None diff --git a/.agents/skills/do-web-doc-resolver/scripts/visual_resolver.py b/.agents/skills/do-web-doc-resolver/scripts/visual_resolver.py new file mode 100644 index 00000000..2d6db0da --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/visual_resolver.py @@ -0,0 +1,465 @@ +import argparse +import asyncio +import atexit +import base64 +import io +import json +import logging +import os +import sys +import time +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from typing import Any, Optional, cast + +import numpy as np + +logger = logging.getLogger(__name__) + + +def _import_pil() -> Any: + try: + import PIL.Image + + return PIL.Image + except ImportError as e: + raise ImportError( + "Pillow is required for visual resolution. Install with: pip install Pillow" + ) from e + + +def _import_playwright() -> Any: + try: + from playwright.async_api import async_playwright + + return async_playwright + except ImportError as e: + raise ImportError( + "playwright is required for visual resolution. Install with: pip install playwright" + ) from e + + +def _import_sentence_transformers() -> Any: + try: + from sentence_transformers import SentenceTransformer + + return SentenceTransformer + except ImportError as e: + raise ImportError( + "sentence-transformers is required for visual resolution. Install with: pip install sentence-transformers" + ) from e + + +@dataclass +class VisualConfig: + clip_threshold: float = 0.22 + viewport_width: int = 1280 + viewport_height: int = 900 + page_timeout_ms: int = 12_000 + scroll_frames: int = 3 + caption_model: str = "qwen/qwen2.5-vl-7b-instruct:free" + caption_max_tokens: int = 512 + caption_timeout_s: float = 30.0 + enabled: bool = True + clip_model_name: str = "clip-ViT-B-32" + + @classmethod + def from_toml(cls, toml_path: str = "config.toml") -> "VisualConfig": + config_dict = {} + if os.path.exists(toml_path): + try: + import tomllib + + with open(toml_path, "rb") as f: + full_config = tomllib.load(f) + config_dict = full_config.get("visual", {}) + except Exception as e: + logger.debug("Failed to load %s: %s", toml_path, e) + + # Environment overrides + env_caption = os.getenv("DO_WDR_VISUAL_CAPTION") + if env_caption == "0": + config_dict["caption_model"] = "" + + env_threshold = os.getenv("DO_WDR_VISUAL_THRESHOLD") + if env_threshold: + try: + config_dict["clip_threshold"] = float(env_threshold) + except ValueError: + logger.warning("Invalid DO_WDR_VISUAL_THRESHOLD: %s", env_threshold) + + env_timeout = os.getenv("DO_WDR_VISUAL_TIMEOUT") + if env_timeout: + try: + config_dict["caption_timeout_s"] = float(env_timeout) + except ValueError: + logger.warning("Invalid DO_WDR_VISUAL_TIMEOUT: %s", env_timeout) + + res = cls() + for k, v in config_dict.items(): + if hasattr(res, k): + setattr(res, k, v) + return res + + +class ScreenshotEngine: + def __init__(self, cfg: VisualConfig): + self.cfg = cfg + self.playwright: Any = None + self.browser: Any = None + self.context: Any = None + + async def _ensure_browser(self) -> None: + if self.browser: + return + pw = _import_playwright() + self.playwright = await pw().__aenter__() + self.browser = await self.playwright.chromium.launch( + headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"] + ) + self.context = await self.browser.new_context( + viewport={"width": self.cfg.viewport_width, "height": self.cfg.viewport_height} + ) + + async def capture(self, url: str) -> list[Any]: + """Returns cfg.scroll_frames PIL images.""" + PIL_Image = _import_pil() + try: + await self._ensure_browser() + if self.context is None: + return [] + page = await self.context.new_page() + frames = [] + try: + await page.goto(url, timeout=self.cfg.page_timeout_ms, wait_until="networkidle") + + for i in range(self.cfg.scroll_frames): + if i > 0: + await page.evaluate(f"window.scrollBy(0, {self.cfg.viewport_height})") + await asyncio.sleep(0.3) # settle + + screenshot_bytes = await page.screenshot(type="png") + img = PIL_Image.open(io.BytesIO(screenshot_bytes)).convert("RGB") + frames.append(img) + finally: + await page.close() + return frames + except Exception as e: + logger.warning("Capture failed for %s: %s", url, e) + return [] + + async def close(self) -> None: + if self.browser: + await self.browser.close() + self.browser = None + if self.playwright: + await self.playwright.__aexit__(None, None, None) + self.playwright = None + + +class ClipEncoder: + _instance: Optional["ClipEncoder"] = None + + def __init__(self, model_name: str = "clip-ViT-B-32"): + SentenceTransformer = _import_sentence_transformers() + self.model = SentenceTransformer(model_name) + + @classmethod + def get_instance(cls, model_name: str = "clip-ViT-B-32") -> "ClipEncoder": + if cls._instance is None: + cls._instance = cls(model_name) + return cls._instance + + def encode_image(self, img: Any) -> np.ndarray: + """Returns L2-normalised 512-d float32 vector.""" + vec: np.ndarray = self.model.encode(img) # type: ignore[no-any-return] + if not isinstance(vec, np.ndarray): + vec = np.array(vec) + return vec / (np.linalg.norm(vec) + 1e-10) # type: ignore[no-any-return] + + def encode_text(self, text: str) -> np.ndarray: + """Returns L2-normalised 512-d float32 vector.""" + vec: np.ndarray = self.model.encode(text) # type: ignore[no-any-return] + if not isinstance(vec, np.ndarray): + vec = np.array(vec) + return vec / (np.linalg.norm(vec) + 1e-10) # type: ignore[no-any-return] + + def similarity(self, a: np.ndarray, b: np.ndarray) -> float: + """Cosine similarity clipped to [0, 1].""" + sim = float(np.dot(a, b)) + return max(0.0, min(1.0, sim)) + + def best_frame(self, frames: list[Any], query_vec: np.ndarray) -> tuple[Any, float]: + """Returns (highest-similarity frame, its score).""" + if not frames: + return None, 0.0 + + best_img = frames[0] + max_sim = -1.0 + + for img in frames: + img_vec = self.encode_image(img) + sim = self.similarity(img_vec, query_vec) + if sim > max_sim: + max_sim = sim + best_img = img + + return best_img, max_sim + + +class VlmCaptioner: + def __init__(self, model: str, max_tokens: int, timeout_s: float = 30.0): + self.model = model + self.max_tokens = max_tokens + self.timeout_s = timeout_s + self.system_prompt = ( + "You are a document analysis assistant. Describe the page content, " + "focusing on the query context. Output compact GitHub Flavored Markdown." + ) + + def caption(self, img: Any, query: str) -> str: + """Route: Ollama if OLLAMA_BASE_URL set, OpenRouter if key set, else fallback.""" + if not self.model: + return self._fallback_caption(img, query) + + if os.getenv("OLLAMA_BASE_URL"): + try: + return self._ollama_caption(img, query) + except Exception as e: + logger.warning("Ollama caption failed: %s", e) + + if os.getenv("OPENROUTER_API_KEY"): + try: + return self._openrouter_caption(img, query) + except Exception as e: + logger.warning("OpenRouter caption failed: %s", e) + + return self._fallback_caption(img, query) + + def _image_to_data_url(self, img: Any) -> str: + """Base64-encode PNG to data:image/png;base64,... URI.""" + buffered = io.BytesIO() + img.save(buffered, format="PNG") + img_str = base64.b64encode(buffered.getvalue()).decode() + return f"data:image/png;base64,{img_str}" + + def _openrouter_caption(self, img: Any, query: str) -> str: + """POST to https://openrouter.ai/api/v1/chat/completions with vision payload.""" + api_key = os.getenv("OPENROUTER_API_KEY") + data_url = self._image_to_data_url(img) + + payload = { + "model": self.model, + "messages": [ + {"role": "system", "content": self.system_prompt}, + { + "role": "user", + "content": [ + {"type": "text", "text": f"Query: {query}"}, + {"type": "image_url", "image_url": {"url": data_url}}, + ], + }, + ], + "max_tokens": self.max_tokens, + } + + req = urllib.request.Request( + "https://openrouter.ai/api/v1/chat/completions", + data=json.dumps(payload).encode(), + headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, + method="POST", + ) + + with urllib.request.urlopen(req, timeout=self.timeout_s) as response: + res_data = json.loads(response.read().decode()) + return cast(str, res_data["choices"][0]["message"]["content"].strip()) + + def _ollama_caption(self, img: Any, query: str) -> str: + """POST to {OLLAMA_BASE_URL}/api/generate with base64 image.""" + base_url = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434").rstrip("/") + buffered = io.BytesIO() + img.save(buffered, format="PNG") + img_base64 = base64.b64encode(buffered.getvalue()).decode() + + payload = { + "model": self.model, + "prompt": f"{self.system_prompt}\n\nQuery: {query}", + "images": [img_base64], + "stream": False, + "options": {"num_predict": self.max_tokens}, + } + + req = urllib.request.Request( + f"{base_url}/api/generate", + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + + with urllib.request.urlopen(req, timeout=self.timeout_s) as response: + res_data = json.loads(response.read().decode()) + return cast(str, res_data["response"].strip()) + + def _fallback_caption(self, img: Any, query: str) -> str: + """Returns structured Markdown label with image dimensions and query.""" + width, height = img.size + return f"### Visual Analysis (Fallback)\n\n- **Dimensions**: {width}x{height}\n- **Query**: {query}\n- **Status**: No VLM captioning available (check DO_WDR_VISUAL_CAPTION or API keys)." + + +@dataclass +class ProviderResult: + content: str | None + score: float + provider: str + latency_ms: float = 0.0 + metadata: dict = field(default_factory=dict) + + def to_dict(self) -> dict: + return { + "content": self.content, + "score": self.score, + "provider": self.provider, + "latency_ms": self.latency_ms, + "metadata": self.metadata, + } + + +class VisualResolver: + PROVIDER_NAME = "visual_clip" + + def __init__(self, cfg: VisualConfig | None = None): + self.cfg = cfg or VisualConfig.from_toml() + self.engine = ScreenshotEngine(self.cfg) + self.encoder: ClipEncoder | None = None + self.captioner = VlmCaptioner( + self.cfg.caption_model, self.cfg.caption_max_tokens, self.cfg.caption_timeout_s + ) + + def is_available(self) -> bool: + """Returns False if cfg.enabled=False or any required import fails.""" + if not self.cfg.enabled: + return False + try: + _import_pil() + _import_playwright() + _import_sentence_transformers() + return True + except ImportError: + return False + + def resolve(self, url: str, query: str) -> ProviderResult | None: + """Sync wrapper: asyncio.run(_resolve_async).""" + return asyncio.run(self._resolve_async(url, query)) + + async def resolve_async(self, url: str, query: str) -> ProviderResult | None: + """Async variant for callers already in an async context.""" + return await self._resolve_async(url, query) + + async def _resolve_async(self, url: str, query: str) -> ProviderResult | None: + """ + Pipeline: + 1. capture(url) -> list[PIL.Image] + 2. encode_text(query) -> query_vec + 3. best_frame(frames, query_vec) -> (best_img, clip_score) + 4. if clip_score < threshold: return ProviderResult(content=None, score=...) + 5. captioner.caption(best_img, query) -> markdown + 6. return ProviderResult(content=markdown, score=clip_score, ...) + """ + start_time = time.time() + try: + if not self.is_available(): + return None + + if self.encoder is None: + self.encoder = ClipEncoder.get_instance(self.cfg.clip_model_name) + + frames = await self.engine.capture(url) + if not frames or self.encoder is None: + return None + + query_vec = self.encoder.encode_text(query) + best_img, clip_score = self.encoder.best_frame(frames, query_vec) + + metadata = { + "clip_score": clip_score, + "frames_captured": len(frames), + "frame_size": best_img.size if best_img else None, + "caption_model": self.cfg.caption_model, + "url": url, + } + + if clip_score < self.cfg.clip_threshold: + latency = (time.time() - start_time) * 1000 + return ProviderResult( + content=None, + score=clip_score, + provider=self.PROVIDER_NAME, + latency_ms=latency, + metadata=metadata, + ) + + content = self.captioner.caption(best_img, query) + latency = (time.time() - start_time) * 1000 + + return ProviderResult( + content=content, + score=clip_score, + provider=self.PROVIDER_NAME, + latency_ms=latency, + metadata=metadata, + ) + except Exception as e: + logger.error("VisualResolver failed: %s", e) + return None + + async def close(self) -> None: + """Release Playwright browser.""" + await self.engine.close() + + +def main(): + parser = argparse.ArgumentParser(description="Visual Resolver CLI") + parser.add_argument("url", help="URL to capture") + parser.add_argument("query", help="Query context for captioning") + parser.add_argument("--threshold", type=float, help="Override CLIP threshold") + parser.add_argument("--no-caption", action="store_true", help="Disable captioning") + parser.add_argument("--json", action="store_true", help="Output in JSON format") + + args = parser.parse_args() + + cfg = VisualConfig.from_toml() + if args.threshold is not None: + cfg.clip_threshold = args.threshold + if args.no_caption: + cfg.caption_model = "" + + resolver = VisualResolver(cfg) + if not resolver.is_available(): + print( + "Error: Dependencies missing (Pillow, playwright, or sentence-transformers).", + file=sys.stderr, + ) + sys.exit(1) + + atexit.register(lambda: asyncio.run(resolver.close())) + + result = resolver.resolve(args.url, args.query) + + if result: + if args.json: + print(json.dumps(result.to_dict(), indent=2)) + else: + if result.content: + print(result.content) + else: + print( + f"No content found (CLIP score: {result.score:.4f} < threshold: {cfg.clip_threshold})" + ) + else: + print("Resolution failed.", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c4ec6de..c56de59d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,15 @@ jobs: run: | python scripts/validate_skill_symlink.py + - name: Validate skill mirror is in sync + run: | + # The standalone skill embeds a byte-for-byte copy of scripts/. Without + # this gate the copy silently drifts and the skill stops importing — + # which is exactly how _query_resolve, _url_resolve, semantic_cache, + # cli and providers/ went missing before. sync_skill.py discovers the + # file set from disk, so a new module is picked up automatically. + python scripts/sync_skill.py --check + validate-docs: name: Validate Documentation runs-on: ubuntu-latest diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 04b36f40..03d61931 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -213,7 +213,21 @@ jobs: node-version: ${{ env.NODE_VERSION }} - name: Run npm audit (web) - run: cd web && npm ci && npm audit --audit-level=high + run: | + cd web && npm ci + # Shipped dependencies: nothing below high. + npm audit --omit=dev --audit-level=high + # Dev toolchain: nothing below critical. + # + # GHSA-vfj7-8cjw-p6xm (CVE-2026-93687) is the one high advisory in the + # dev tree: braces 3.0.3 stack-exhaustion DoS, no patched release + # upstream (micromatch/braces PR #72 is still open), so it cannot be + # upgraded away. It is dev-only -- `npm ls braces --omit=dev` is empty -- + # reached solely via @next/eslint-plugin-next -> fast-glob -> + # micromatch -> braces, and only when ESLint expands glob patterns + # from this repo's own eslint.config.mjs. No path from untrusted input. + # Same shape as the RUSTSEC-2026-0258 ignore below. + npm audit --audit-level=critical - name: Set up Rust uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 diff --git a/agents-docs/ISSUES.md b/agents-docs/ISSUES.md index ca79e21a..487885a1 100644 --- a/agents-docs/ISSUES.md +++ b/agents-docs/ISSUES.md @@ -43,9 +43,36 @@ - **Action Taken**: Deprioritized serper in the routing logic. - **Status**: Monitoring for stability. +## Accepted Risk: braces stack-exhaustion DoS (dev-only, unpatched) + +- **Date**: 2026-10-03 +- **Advisory**: [GHSA-vfj7-8cjw-p6xm](https://github.com/advisories/GHSA-vfj7-8cjw-p6xm) / CVE-2026-93687, CVSS 8.7 High, EPSS 0.74% +- **Issue**: `npm audit --audit-level=high` fails on `braces` 3.0.3 — recursive AST walkers have no depth guard, so a deeply nested brace pattern exhausts the call stack. Affects `<= 3.0.3`; **patched versions: None** (`micromatch/braces` PR #72 still open), so it cannot be upgraded away. +- **Exposure**: dev-only. `npm ls braces --omit=dev` is empty. The chain is `@next/eslint-plugin-next@16.3.6 → fast-glob@3.3.1 → micromatch@4.0.8 → braces@3.0.3`, reached only when ESLint expands glob patterns from this repo's own `eslint.config.mjs`. There is no path from an HTTP request or any untrusted input to `braces()`. +- **Rejected**: `npm audit fix --force` downgrades `@next/eslint-plugin-next` 16.3.6 → 14.2.35, a breaking major downgrade that drops Next 16 lint rules. +- **Action Taken**: `.github/workflows/security-scan.yml` gates shipped dependencies with `npm audit --omit=dev --audit-level=high` and dev toolchain with `--audit-level=critical`. Runtime exposure is still fully gated at high; only dev deps are relaxed. +- **Status**: Accepted until upstream ships a patched `braces`. Re-evaluate when PR #72 releases or when `@next/eslint-plugin-next` drops `fast-glob`. +- **Prevention**: Revisit this gate whenever a new `high` advisory appears in the dev tree — do not lower the dev threshold again to absorb a second one. + ## Semantic Health Audit: September 2026 - **Date**: 2026-09-14 - **Summary**: Executed `do-wdr` CLI benchmark audit across 5 standard documentation URLs (`docs.python.org/3/library/os.html`, `doc.rust-lang.org/std/fs/index.html`, `developer.mozilla.org/en-US/docs/Web/JavaScript`, `docs.python.org/3/library/sys.html`, `doc.rust-lang.org/std/path/struct.Path.html`). - **Results**: Semantic cache hit rate = 100% (5/5), Cache hit latency = 1ms (well within the < 200ms threshold), Quality synthesis score = 1.0 (exceeds the >= 0.85 threshold). - **Status**: Healthy. Python-Rust bridge integration operating with zero bottlenecks; embedding retrieval logic and cache redundancy pruning remain fully optimized. + +## Known Gap: session cookie is not authorization + +- **Date**: 2026-10-03 +- **Issue**: `DELETE /api/cache` and `DELETE /api/records` require a `ui-session` cookie, but the cookie value is client-supplied and unverified. Anyone can obtain one by calling any endpoint that sets it, so the check stops anonymous `curl` and nothing more. +- **Why it is still worth having**: the realistic abuse is an unauthenticated script, and the cookie check is the cheapest thing that raises the bar. It is not an access control boundary and should not be described as one. +- **Follow-up**: sign the session value server-side (HMAC over a random secret, or a platform-issued session) so the check becomes real authorization. `hasSessionCookie` in `app/api/records/route.ts` carries the same caveat in code. +- **Related**: the records store in `lib/records.ts` is a single process-global map with no per-session partitioning, so `GET /api/records` returns every user's records and `GET /api/records?q=` searches across them. Partitioning is a schema change and is out of scope for the current work. +- **Status**: Open. + +## Known Gap: request-body limits are per-handler + +- **Date**: 2026-10-03 +- **Issue**: `readJsonWithLimit` in `web/lib/body-limit.ts` bounds the body for handlers that call it (currently `POST /api/records`). Other handlers still use bare `request.json()`. +- **Why**: App Router has no `bodyParser.sizeLimit` equivalent, so the cap has to be applied in code at each entry point. +- **Status**: Open. Apply to `POST /api/resolve` and `POST /api/history` when they are next touched. diff --git a/cli/src/bias_scorer.rs b/cli/src/bias_scorer.rs index b2f91e74..d06dd041 100644 --- a/cli/src/bias_scorer.rs +++ b/cli/src/bias_scorer.rs @@ -17,6 +17,17 @@ fn contains_ignore_ascii_case(haystack: &str, needle: &str) -> bool { .any(|window| window.eq_ignore_ascii_case(needle_bytes)) } +/// Check whether `host` is `site` itself or a subdomain of it. +/// +/// Substring matching would award trust to hosts like `github.com.evil.example`, +/// so the boundary has to land on a dot. +fn is_domain_or_subdomain(host: &str, site: &str) -> bool { + let host = host.to_ascii_lowercase(); + let site = site.to_ascii_lowercase(); + + host == site || host.ends_with(&format!(".{site}")) +} + /// Score a result based on domain trust and content quality pub fn score_result(url: &str, content: &str) -> f64 { let mut score: f64 = 0.5; @@ -31,7 +42,10 @@ pub fn score_result(url: &str, content: &str) -> f64 { } let news_sites = ["nytimes.com", "bbc.co.uk", "reuters.com", "theguardian.com"]; - if news_sites.iter().any(|&site| domain.contains(site)) { + if news_sites + .iter() + .any(|&site| is_domain_or_subdomain(domain, site)) + { score += 0.1; } @@ -46,7 +60,10 @@ pub fn score_result(url: &str, content: &str) -> f64 { "react.dev", "python.org", ]; - if dev_sites.iter().any(|&site| domain.contains(site)) { + if dev_sites + .iter() + .any(|&site| is_domain_or_subdomain(domain, site)) + { score += 0.2; } } @@ -81,3 +98,42 @@ pub fn score_result(url: &str, content: &str) -> f64 { score.clamp(0.0, 1.0) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_subdomain_match() { + assert!(is_domain_or_subdomain("github.com", "github.com")); + assert!(is_domain_or_subdomain("gist.github.com", "github.com")); + assert!(is_domain_or_subdomain("GITHUB.COM", "github.com")); + } + + #[test] + fn test_suffix_lookalike_rejected() { + // A host that merely contains a trusted domain must not earn trust. + assert!(!is_domain_or_subdomain( + "github.com.evil.example", + "github.com" + )); + assert!(!is_domain_or_subdomain( + "notstackoverflow.com", + "stackoverflow.com" + )); + } + + #[test] + fn test_lookalike_host_scores_lower_than_real() { + // 600 words of filler satisfies the word-count bonus for both hosts, so + // any difference comes purely from the domain trust heuristic. + let content = "word ".repeat(600); + let real = score_result("https://github.com/foo/bar", &content); + let spoofed = score_result("https://github.com.evil.example/foo", &content); + + assert!( + real > spoofed, + "real host ({real}) should outscore lookalike ({spoofed})" + ); + } +} diff --git a/cli/src/config/merge.rs b/cli/src/config/merge.rs new file mode 100644 index 00000000..153dc423 --- /dev/null +++ b/cli/src/config/merge.rs @@ -0,0 +1,61 @@ +//! Per-field merge helpers. +//! +//! Each helper encodes one rule for "did the incoming layer actually set this +//! value?". Keeping them together makes the rules auditable in one read +//! instead of scattered through `Config::merge`, and keeps `mod.rs` under the +//! 500-line source limit documented in AGENTS.md. + +use std::collections::HashMap; + +use super::ExplicitBools; + +pub(super) fn merge_value(target: &mut T, value: T, default: T) { + if value != default { + *target = value; + } +} + +pub(super) fn merge_string(target: &mut String, value: String) { + merge_value(target, value, "info".to_string()); +} + +/// Merge a boolean that is `true` by default. +/// +/// Plain "override when true" merging makes it impossible to turn a +/// default-on feature off from a config file, because an absent key +/// deserializes to the same `true` as an explicit `enabled = true`. Presence +/// is therefore tracked in [`ExplicitBools`] at parse time and the merge only +/// applies keys the source actually specified. +pub(super) fn merge_explicit_bool( + target: &mut bool, + value: bool, + key: &str, + explicit: &ExplicitBools, +) { + if explicit.contains(key) { + *target = value; + } else if value { + *target = true; + } +} + +pub(super) fn merge_option(target: &mut Option, value: Option) { + if value.is_some() { + *target = value; + } +} + +pub(super) fn merge_vec(target: &mut Vec, value: Vec) { + if !value.is_empty() { + *target = value; + } +} + +pub(super) fn merge_map(target: &mut HashMap, value: HashMap) +where + K: Eq + std::hash::Hash, +{ + for (name, provider_config) in value { + target.entry(name).or_insert(provider_config); + } +} diff --git a/cli/src/config/mod.rs b/cli/src/config/mod.rs index 4cd62280..a197c844 100644 --- a/cli/src/config/mod.rs +++ b/cli/src/config/mod.rs @@ -8,7 +8,9 @@ use thiserror::Error; use defaults::*; mod defaults; +mod merge; mod parsing; +use merge::{merge_explicit_bool, merge_map, merge_option, merge_string, merge_value, merge_vec}; pub use defaults::RoutingProfileConfig; pub use defaults::routing_profile_defaults; @@ -21,8 +23,54 @@ pub enum ConfigError { ParseError(#[from] toml::de::Error), } +/// Boolean config keys whose TOML presence must be tracked so that +/// `enabled = false` can override a `true` default. +/// +/// This is a hand-maintained list, so **adding a default-on boolean config +/// option means adding its dotted path here too.** `test_config_file_can_disable_ +/// default_on_features` covers every entry and will not catch a new field that +/// was never registered, so treat a new `enabled` flag as requiring both a +/// registry entry and a test assertion. +const EXPLICIT_BOOL_KEYS: &[&str] = &[ + "semantic_cache.enabled", + "cache.synthesis.enabled", + "routing.prewarm.enabled", + "disable_routing_memory", +]; + +/// Set of explicitly-present boolean keys, tracked as `a.b.c` dotted paths. +pub type ExplicitBools = std::collections::BTreeSet; + +/// Resolve a dotted path (e.g. `routing.prewarm.enabled`) inside a TOML table. +fn lookup_dotted<'a>(table: &'a toml::Table, key: &str) -> Option<&'a toml::Value> { + let mut segments = key.split('.'); + let first = segments.next()?; + let mut cursor = table.get(first)?; + + for segment in segments { + cursor = cursor.as_table()?.get(segment)?; + } + + Some(cursor) +} + +/// Record which of [`EXPLICIT_BOOL_KEYS`] the parsed TOML actually specified. +fn collect_explicit_bools(table: &toml::Table) -> ExplicitBools { + let mut out = ExplicitBools::new(); + for key in EXPLICIT_BOOL_KEYS { + if lookup_dotted(table, key).is_some_and(toml::Value::is_bool) { + out.insert((*key).to_string()); + } + } + out +} + #[derive(Debug, Clone, Deserialize)] pub struct Config { + /// Not user-facing: records which boolean keys the source config set + /// explicitly, so `enabled = false` can override a `true` default. + #[serde(skip)] + pub explicit_bools: ExplicitBools, #[serde(default = "default_max_chars")] pub max_chars: usize, #[serde(default = "default_min_chars")] @@ -177,6 +225,7 @@ impl Default for Config { min_chars: default_min_chars(), exa_results: default_exa_results(), tavily_results: default_tavily_results(), + explicit_bools: ExplicitBools::new(), output_limit: default_output_limit(), log_level: "info".to_string(), skip_providers: Vec::new(), @@ -202,13 +251,29 @@ impl Default for Config { impl Config { pub fn from_file(path: impl AsRef) -> Result { let content = std::fs::read_to_string(path.as_ref())?; - let file_config: Config = toml::from_str(&content)?; + + // Parse twice: once to learn which boolean keys the file actually set, + // and once to build the typed config. + let value: toml::Value = toml::from_str(&content)?; + let explicit_bools = value + .as_table() + .map(collect_explicit_bools) + .unwrap_or_default(); + + let mut file_config: Config = toml::from_str(&content)?; + file_config.explicit_bools = explicit_bools; + let mut config = Config::default(); config.merge(file_config); Ok(config) } pub fn merge(&mut self, other: Config) { + // Carry forward which boolean keys were explicit, so a merged-in file + // keeps its ability to turn a `true` default off through later merges. + self.explicit_bools + .extend(other.explicit_bools.iter().cloned()); + merge_value(&mut self.max_chars, other.max_chars, default_max_chars()); merge_value(&mut self.min_chars, other.min_chars, default_min_chars()); merge_value( @@ -249,9 +314,11 @@ impl Config { other.circuit_breaker_cooldown_secs, default_circuit_breaker_cooldown(), ); - merge_bool( + merge_explicit_bool( &mut self.semantic_cache.enabled, other.semantic_cache.enabled, + "semantic_cache.enabled", + &other.explicit_bools, ); merge_value( &mut self.semantic_cache.path, @@ -324,9 +391,22 @@ impl Config { &mut self.routing.min_free_quality_to_skip_paid, other.routing.min_free_quality_to_skip_paid, ); - merge_bool( + merge_explicit_bool( &mut self.routing.prewarm.enabled, other.routing.prewarm.enabled, + "routing.prewarm.enabled", + &other.explicit_bools, + ); + merge_explicit_bool( + &mut self.cache.synthesis.enabled, + other.cache.synthesis.enabled, + "cache.synthesis.enabled", + &other.explicit_bools, + ); + merge_value( + &mut self.cache.synthesis.ttl, + other.cache.synthesis.ttl, + default_synthesis_cache_ttl(), ); merge_value( &mut self.routing.prewarm.top_n_domains, @@ -341,9 +421,11 @@ impl Config { merge_option(&mut self.max_provider_attempts, other.max_provider_attempts); merge_option(&mut self.max_paid_attempts, other.max_paid_attempts); merge_option(&mut self.max_total_latency_ms, other.max_total_latency_ms); - merge_bool( + merge_explicit_bool( &mut self.disable_routing_memory, other.disable_routing_memory, + "disable_routing_memory", + &other.explicit_bools, ); merge_map(&mut self.providers, other.providers); } @@ -386,90 +468,8 @@ impl Config { } } -fn merge_value(target: &mut T, value: T, default: T) { - if value != default { - *target = value; - } -} - -fn merge_string(target: &mut String, value: String) { - merge_value(target, value, "info".to_string()); -} - -fn merge_bool(target: &mut bool, value: bool) { - if value { - *target = value; - } -} - -fn merge_option(target: &mut Option, value: Option) { - if value.is_some() { - *target = value; - } -} - -fn merge_vec(target: &mut Vec, value: Vec) { - if !value.is_empty() { - *target = value; - } -} - -fn merge_map(target: &mut HashMap, value: HashMap) -where - K: Eq + std::hash::Hash, -{ - for (name, provider_config) in value { - target.entry(name).or_insert(provider_config); - } -} - +// Unit tests live in their own module file so this one stays under the +// 500-line source limit documented in AGENTS.md. #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_default_config() { - let config = Config::default(); - assert_eq!(config.max_chars, 8000); - assert_eq!(config.min_chars, 200); - assert_eq!(config.exa_results, 5); - assert_eq!(config.tavily_results, 3); - assert_eq!(config.output_limit, 10); - } - - #[test] - fn test_api_key_lookup() { - let config = Config::default(); - assert!(config.api_key("unknown").is_none()); - } - - #[test] - fn test_skip_providers() { - let config = Config { - skip_providers: vec!["exa".to_string(), "tavily".to_string()], - ..Default::default() - }; - - assert!(config.is_skipped("exa")); - assert!(config.is_skipped("tavily")); - assert!(!config.is_skipped("firecrawl")); - } - - #[test] - fn test_get_ttl() { - let config = Config::default(); - assert_eq!(config.get_ttl("firecrawl"), 21600); - assert_eq!(config.get_ttl("exa"), 14400); - assert_eq!(config.get_ttl("exa_mcp"), 14400); - assert_eq!(config.get_ttl("tavily"), 14400); - assert_eq!(config.get_ttl("serper"), 7200); - assert_eq!(config.get_ttl("jina"), 7200); - assert_eq!(config.get_ttl("mistral"), 28800); - assert_eq!(config.get_ttl("mistral_browser"), 28800); - assert_eq!(config.get_ttl("mistral_websearch"), 28800); - assert_eq!(config.get_ttl("duckduckgo"), 3600); - assert_eq!(config.get_ttl("llms_txt"), 28800); - assert_eq!(config.get_ttl("synthesis"), 43200); - assert_eq!(config.get_ttl("unknown"), 3600); - } -} +#[path = "tests.rs"] +mod tests; diff --git a/cli/src/config/tests.rs b/cli/src/config/tests.rs new file mode 100644 index 00000000..1a5aad72 --- /dev/null +++ b/cli/src/config/tests.rs @@ -0,0 +1,140 @@ +use super::*; + +#[test] +fn test_default_config() { + let config = Config::default(); + assert_eq!(config.max_chars, 8000); + assert_eq!(config.min_chars, 200); + assert_eq!(config.exa_results, 5); + assert_eq!(config.tavily_results, 3); + assert_eq!(config.output_limit, 10); +} + +#[test] +fn test_api_key_lookup() { + let config = Config::default(); + assert!(config.api_key("unknown").is_none()); +} + +#[test] +fn test_skip_providers() { + let config = Config { + skip_providers: vec!["exa".to_string(), "tavily".to_string()], + ..Default::default() + }; + + assert!(config.is_skipped("exa")); + assert!(config.is_skipped("tavily")); + assert!(!config.is_skipped("firecrawl")); +} + +#[test] +fn test_get_ttl() { + let config = Config::default(); + assert_eq!(config.get_ttl("firecrawl"), 21600); + assert_eq!(config.get_ttl("exa"), 14400); + assert_eq!(config.get_ttl("exa_mcp"), 14400); + assert_eq!(config.get_ttl("tavily"), 14400); + assert_eq!(config.get_ttl("serper"), 7200); + assert_eq!(config.get_ttl("jina"), 7200); + assert_eq!(config.get_ttl("mistral"), 28800); + assert_eq!(config.get_ttl("mistral_browser"), 28800); + assert_eq!(config.get_ttl("mistral_websearch"), 28800); + assert_eq!(config.get_ttl("duckduckgo"), 3600); + assert_eq!(config.get_ttl("llms_txt"), 28800); + assert_eq!(config.get_ttl("synthesis"), 43200); + assert_eq!(config.get_ttl("unknown"), 3600); +} + +/// Regression test: `enabled = false` in a config file must survive the +/// merge. Previously `merge_bool` only applied `true`, so every +/// default-on feature could be turned off by env var or CLI flag but never +/// by the config file itself. +#[test] +fn test_config_file_can_disable_default_on_features() { + let dir = std::env::temp_dir().join(format!("wdr-config-merge-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("create temp dir"); + let path = dir.join("disable.toml"); + + std::fs::write( + &path, + r#" +max_chars = 1234 + +[semantic_cache] +enabled = false + +[cache.synthesis] +enabled = false + +[routing.prewarm] +enabled = false +"#, + ) + .expect("write config"); + + let config = Config::from_file(&path).expect("parse config"); + + assert!( + !config.semantic_cache.enabled, + "semantic_cache must be disabled" + ); + assert!( + !config.cache.synthesis.enabled, + "synthesis cache must be disabled" + ); + assert!(!config.routing.prewarm.enabled, "prewarm must be disabled"); + // Non-boolean fields must still merge normally. + assert_eq!(config.max_chars, 1234); + + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_dir(&dir); +} + +/// `disable_routing_memory` is default-off, so the presence tracking only +/// has to make an explicit `true` survive the merge. Covered here because it +/// shares `EXPLICIT_BOOL_KEYS` with the default-on flags and would otherwise +/// be the one untested entry. +#[test] +fn test_explicit_true_survives_merge_for_default_off_flag() { + let dir = std::env::temp_dir().join(format!("wdr-config-flag-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("create temp dir"); + let path = dir.join("flag.toml"); + + std::fs::write(&path, "disable_routing_memory = true\n").expect("write config"); + let on = Config::from_file(&path).expect("parse config"); + assert!(on.disable_routing_memory); + + // Absent must leave the default alone in both directions. + std::fs::write(&path, "log_level = \"debug\"\n").expect("write config"); + let absent = Config::from_file(&path).expect("parse config"); + assert!(!absent.disable_routing_memory); + + // Explicitly false must not resurrect a true default. + std::fs::write(&path, "disable_routing_memory = false\n").expect("write config"); + let off = Config::from_file(&path).expect("parse config"); + assert!(!off.disable_routing_memory); + + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_dir(&dir); +} + +/// A config file that omits these keys must not disable them. +#[test] +fn test_absent_bools_keep_defaults() { + let dir = std::env::temp_dir().join(format!("wdr-config-absent-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("create temp dir"); + let path = dir.join("minimal.toml"); + + std::fs::write(&path, "log_level = \"debug\"\n").expect("write config"); + + let config = Config::from_file(&path).expect("parse config"); + + assert!(config.semantic_cache.enabled); + assert!(config.cache.synthesis.enabled); + assert!(config.routing.prewarm.enabled); + assert_eq!(config.log_level, "debug"); + + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_dir(&dir); +} diff --git a/cli/src/output.rs b/cli/src/output.rs index 6445af84..ab2faf11 100644 --- a/cli/src/output.rs +++ b/cli/src/output.rs @@ -13,6 +13,10 @@ pub struct JsonOutput<'a> { pub content: Option<&'a str>, pub score: f64, pub metrics: Option<&'a crate::metrics::ResolveMetrics>, + /// Failure reason. `None` for successful resolutions. Skipped when absent + /// so success payloads stay byte-compatible with earlier versions. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option<&'a str>, } impl<'a> JsonOutput<'a> { @@ -24,17 +28,20 @@ impl<'a> JsonOutput<'a> { content: result.content.as_deref(), score: result.score, metrics: result.metrics.as_ref(), + error: None, } } - /// Create error output - pub fn error(_msg: &'a str) -> Self { + /// Create error output. The message is surfaced in the `error` field so + /// `--json` consumers get the reason instead of having to scrape stderr. + pub fn error(msg: &'a str) -> Self { Self { source: "none", url: "", content: None, score: 0.0, metrics: None, + error: Some(msg), } } diff --git a/cli/src/semantic_cache/mod.rs b/cli/src/semantic_cache/mod.rs index 4cb93969..67cf5b7f 100644 --- a/cli/src/semantic_cache/mod.rs +++ b/cli/src/semantic_cache/mod.rs @@ -73,12 +73,16 @@ pub struct SemanticCache { #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct SemanticCacheConfig { /// Enable semantic cache + #[serde(default = "default_semantic_cache_enabled")] pub enabled: bool, /// Path to cache database + #[serde(default = "default_semantic_cache_path")] pub path: String, /// Similarity threshold (0.0 - 1.0) + #[serde(default = "default_semantic_cache_threshold")] pub threshold: f32, /// Maximum entries + #[serde(default = "default_semantic_cache_max_entries")] pub max_entries: usize, /// Tiered TTL configuration (injected from Config) #[serde(skip)] @@ -110,13 +114,29 @@ impl SemanticCacheConfig { } } +fn default_semantic_cache_enabled() -> bool { + true +} + +fn default_semantic_cache_path() -> String { + ".do-wdr_cache".to_string() +} + +fn default_semantic_cache_threshold() -> f32 { + 0.85 +} + +fn default_semantic_cache_max_entries() -> usize { + 10000 +} + impl Default for SemanticCacheConfig { fn default() -> Self { Self { - enabled: true, - path: ".do-wdr_cache".to_string(), - threshold: 0.85, - max_entries: 10000, + enabled: default_semantic_cache_enabled(), + path: default_semantic_cache_path(), + threshold: default_semantic_cache_threshold(), + max_entries: default_semantic_cache_max_entries(), ttls: None, } } diff --git a/plans/22-codebase-improvement-2026-10.md b/plans/22-codebase-improvement-2026-10.md new file mode 100644 index 00000000..43d86cc7 --- /dev/null +++ b/plans/22-codebase-improvement-2026-10.md @@ -0,0 +1,250 @@ +# Plan 22 — Full-Repo Improvement Sweep (2026-10-02) + +> Source: four parallel deep-dive analyses (Python core, Rust CLI, Web UI, Tests/CI) +> run on `main` @ `76b1b5c` after v0.3.10. +> Target landing: `plans/22-codebase-improvement-2026-10.md` + `plans/README.md` row. + +## Goal + +Close every gap surfaced by the 2026-10 analysis: correctness bugs, security +hardening, cross-runtime parity, feature gaps, DRY consolidation, file-size +violations, and CI enforcement. Items verified by direct inspection are marked +**[V]**. + +## Wave A — P0 Correctness & Security + +| ID | Area | File(s) | Issue | Action | +|----|------|---------|-------|--------| +| A1 | Skill | `scripts/sync_skill.py:28-40` | `SYNC_FILES` stale — mirrored skill imports `_query_resolve`, `_url_resolve`, `_url_resolve_async`, `providers` pkg, `semantic_cache` that don't exist in mirror; standalone skill unusable | Add missing modules + `providers/` package to SYNC_FILES; add mirror-import smoke test | +| A2 | Py-async | `scripts/_cascade_async.py:52-95` | Async cascade launches ALL eligible providers (free+paid) in parallel; sync quality-gate/cost control bypassed; `asyncio.wait` has no timeout so latency budgets never enforced | Staggered hedged launch: free tier first, paid only when `min_free_quality_to_skip_paid` unmet; add `max_total_latency_ms` timeout | +| A3 | Web | `web/app/page.tsx:322` **[V]** | Reads `data.quality_score` but API returns `quality` object (`route.ts:288,294`); score always null | `setQualityScore(data.quality?.score ?? null)` | +| A4 | Web | `web/middleware.ts:20` + `web/app/api/resolve/route.ts:209` | Rate limit counted twice → effective 15/min not 30/min | Count in one layer only (keep middleware) | +| A5 | Web | `web/app/api/ui-state/route.ts:24-33`, `web/lib/ui-state.ts:153-162` | API keys synced unthrottled every keystroke; stored plaintext server-side keyed by `sha256(IP+UA)`; unauthenticated GET exposes keys to shared-NAT users | Keep keys localStorage-only; drop server key sync; debounce persistence | +| A6 | Web | `web/app/api/cache/route.ts:8-11`, `web/app/api/records/route.ts:35-38` | Unauthenticated, unrate-limited DELETE endpoints wipe server state | Require session token or remove endpoints | +| A7 | Rust | `cli/src/output.rs:31-39` | `JsonOutput::error()` takes `_msg` and discards it; JSON consumers never see the error | Add `error: Option` field and render it | +| A8 | Rust | `cli/src/config/mod.rs:399-403` | `merge_bool` only overrides on `true` — TOML can never disable `semantic_cache`/`prewarm`/`synthesis` | Presence-tracking merge (`Option` + serde default) | +| A9 | Rust | `cli/src/bias_scorer.rs:29,34,46` | `domain.contains(site)` — `github.com.evil.example` earns dev-domain trust bonus | Exact-host / public-suffix match | +| A10 | Py | `scripts/routing_memory.py:101-106` **[V]** | `_dirty is False or ...` → `_save_to_disk_unlocked()` resets `_dirty`, so every `record()` writes JSON to disk; 5s throttle never throttles; hot-path disk churn | Save on interval only: `time.time() - self._last_save >= SAVE_INTERVAL_SECONDS` | +| A11 | Py | `scripts/providers/stealth.py:1-31` **[V]** | TODO stub returns truthy empty `ResolvedResult` (not None) → cascade scores it 0.0 and writes negative-cache entry suppressing the tier slot | Return `None` or remove from `PROVIDER_TIERS`/cascade maps until implemented | +| A12 | Rust | `cli/src/providers/shared_client.rs:16` | AGENTS.md advertises `DEFAULT_MAX_RETRIES=3` but the Rust HTTP client has zero retries; the doc comment claims "retry configuration" that does not exist | Add bounded retry/backoff middleware or fix the comment (parity gap, see F9) | + +A12 deferred to Wave F9 with the Python retry work so both runtimes land together. + +## Wave B — P1 High-Impact Fixes + +| ID | Area | File(s) | Issue | Action | +|----|------|---------|-------|--------| +| B1 | Py | `scripts/_url_resolve_async.py:56`, `_cascade_async.py:142-192` | Blocking semantic-cache/embedding work on event loop (sentence-transformers encode stalls all tasks; matches SEMANTIC_HEALTH_ISSUE.md) | Wrap in `asyncio.to_thread()`; background model preload | +| B2 | Py | `scripts/_cascade.py:113`, `_cascade_async.py:117` vs `quality.py:152,179` | `score_content()` called without `links` → `missing_links` always True → every result takes -0.10 penalty; quality gates effectively +0.10 | Extract markdown links inside `score_content` when `links is None` | +| B3 | Py | `scripts/quality.py:171` | No query-relevance signal; pristine-but-irrelevant content scores high | Add `query: str \| None` param; blend embedding/term-overlap relevance | +| B4 | Rust | `cli/src/resolver/mod.rs:77-79`, `startup.rs:27-30` | Routing memory / negative cache / circuit breakers process-local — memory never accumulates, prewarm always early-returns (dead), breakers reset per run | Persist to `.do-wdr_cache` state file | +| B5 | Rust | `cli/src/resolver/mod.rs:210-220` | Synthesis error aborts command; deterministic-merge fallback only on missing key, not on Mistral failure | Fall back on `Err` too | +| B6 | Rust | `cli/src/providers/llms_txt.rs:65-69` | Port dropped when building `scheme://host/llms.txt` | Include port from parsed URL | +| B7 | Rust | `cli/src/link_validator.rs:38` | HEAD-only probing drops valid links on 405/501 sites | Retry with GET or `Range: bytes=0-0` | +| B8 | Rust | `cli/src/compaction.rs:60-68` | Fast path compares bytes to `max_chars`, slow path truncates chars | Unify on chars | +| B9 | Rust | `cli/src/resolver/url.rs:56-59` | `url.ends_with(".pdf")` misses `.PDF`, `file.pdf?v=2` | Case-insensitive match on path component | +| B10 | Web | `web/app/page.tsx:381-389` | History load re-runs query; provider outage clobbers stored result while stale result stays visible | Show stored result; make re-run explicit | +| B11 | Web | `web/app/page.tsx:222` vs `web/lib/resolvers/index.ts:35` | Client `startsWith("http")` vs server regex — mode label can disagree with execution | Share one `isUrl` util | +| B12 | Web | `web/app/components/History.tsx:201` | `>+${n}` renders literal `+$` | Fix template | +| B13 | Py | `scripts/constants.py:48` vs `quality.py:21` | `MIN_CHARS=200` (Jina gate) vs `THRESHOLD_MIN_CHARS=500` (quality) — 300-char results pass then always fail | Align thresholds | +| B14 | Py | `scripts/providers/*` (all except blocks) | Providers swallow error causes; cascade/negative-cache can't distinguish no-key vs transient vs bot-challenge | Typed `ProviderResult`/error return | + +**Wave A + early Wave B items: DONE.** Verified per-item below. + +| ID | Change | Verification | +|----|--------|--------------| +| A1 | `sync_skill.py` now syncs `_cascade*`, `_query_resolve`, `_url_resolve*`, `_routing_utils`, `semantic_cache`, `cli`, `visual_resolver`, and the whole `providers/` package; standalone skill imports resolve | `python -m scripts.resolve` import check in the skill dir ✅ | +| A2 | Free tier launches first; paid escalates only when the free tier fails the gate; `max_total_latency_ms` enforced as a deadline | `tests/test_cascade_async.py` (8 tests) ✅ | +| A3 | `page.tsx` reads `data.quality?.score` | typecheck + web unit tests ✅ | +| A4 | Duplicate `checkRateLimit` removed from `app/api/resolve/route.ts`; middleware is the single gate | `tests/api/resolve-rate-limit.test.ts` rewritten against middleware ✅ | +| A5 | `apiKeys` stripped in `/api/ui-state` POST **and** GET, and never sent from the client | `tests/api/ui-state-route.test.ts` new case ✅ | +| A6 | DELETE on `/api/cache` and `/api/records` requires the `ui-session` cookie; records POST gains zod validation with size caps | `tests/api/records-route.test.ts` (new, 7 tests) ✅ | +| A7 | `JsonOutput::error()` now surfaces the message in an `error` field | `cargo test` ✅ | +| A8 | Presence-tracked TOML booleans (`ExplicitBools`) so `enabled = false` overrides a `true` default; `SemanticCacheConfig` fields gained serde defaults so partial tables parse | 2 new `config::tests` cases + `cargo test` ✅ | +| A9 | `bias_scorer` uses exact-host/subdomain matching instead of substring | 3 new `bias_scorer` tests ✅ | +| A10 | `RoutingMemory.record()` writes to disk on the interval only | full pytest ✅ | +| A11 | `resolve_with_stealth` returns `None` instead of a truthy empty result | full pytest ✅ | +| B1 | Semantic-cache check and stores moved off the event loop via `asyncio.to_thread` | full pytest ✅ | +| B2 | `score_content` infers markdown links when none are passed, removing the systematic -0.10 penalty | `tests/test_quality_links.py` (new, 6 tests) ✅ | + +`cargo fmt` and `cargo clippy --all-targets -D warnings` clean. One pre-existing +failure remains and is unrelated to this work: `semantic_cache::tests:: +test_database_failure` (verified failing on pristine `HEAD` in the same +container). Rust was verified in `rust:1.90` because no host toolchain exists. + +## Wave C — Feature Additions + +| ID | Area | Feature | Notes | +|----|------|---------|-------| +| C1 | Py | **New keyless query providers** — Brave Search (2k/mo free), Wikipedia/CirrusSearch (no key), Mojeek or SearxNG JSON, Google CSE (100/day) | De-risks DDG/Serper instability (ISSUES.md recurring); follow zero-arg-lambda pattern; register in web constants too | +| C2 | Web | **Streaming cascade UX** — SSE/NDJSON per-provider events ("Trying Tavily… 1.2s ✗"), progressive render in deep-research mode | Replaces 60s blocking POST + "Fetching…" | +| C3 | Web | **Structured API response + provider trace** — return `{results, trace, budget, cache_hit}`; stop regex-parsing markdown client-side (`lib/results.ts`) | Route already collects trace via `getProviderSummary()`/`budget.getState()` then discards | +| C4 | Web | **`/api/providers` single source of truth** — serve ids/labels/free-paid/profiles/key-status; derive UI | Provider lists hardcoded in 7 places (constants.ts, routing.ts, resolvers/index.ts, route.ts ×2, key-status, settings, help) | +| C5 | Web | **Real persistence** — Vercel KV/Upstash for history, cache, rate-limit, ui-state (all in-memory Maps; history broken in prod) | Comment at `app/api/history/route.ts:3` already promises this | +| C6 | Web | **Export/share** — .md/.json download, copy-as-JSON, permalink `?q=&profile=&providers=` | Trivial on existing state | +| C7 | Web | **Theme toggle** — `UIState.theme` already round-trips but never read; `layout.tsx:34` hardcodes dark; dark-mode Playwright project exists but never runs | Read theme; honor `prefers-color-scheme` | +| C8 | Web | **Cost estimate display** — map provider → est. cost/request; budget already tracks `paidAttempts` | MetadataBar | +| C9 | Web | **Provider comparison view** — side-by-side cards per provider w/ overlap stats (deep-research currently concatenates) | `extractNormalizedUrls` exists | +| C10 | Rust | **Shell completions** — `clap_complete` + `completions ` subcommand | ~40 lines | +| C11 | Rust | **Distinct exit codes** — map `ResolverError` taxonomy: NotFound→2, RateLimit→3, Quota→4, Auth→5 | Error enum already has variants | +| C12 | Rust | **`cache clear` / `cache drop `** — `SemanticCache::remove` exists, just expose | | +| C13 | Rust | **Streaming/progress events** — NDJSON `{"event":"provider_start",...}` or `--progress` stderr lines | `RoutingDecision` vec has the data | +| C14 | Rust | **stdin input** — `do-wdr resolve -` | | +| C15 | Rust | **Release profile** — `lto="thin"`, `codegen-units=1`, `strip="symbols"`; startup-time win on cache-hit fast path | | +| C16 | Rust | **Parallel `resolve_aggregated`** — race top 2-3 free providers with JoinSet, first acceptable wins | Currently sequential | +| C17 | Web/Rust | **Markdown rendering** — `react-markdown@^10` declared but never imported; render raw view as markdown or drop dep | | +| C18 | Web | **History upgrades** — export JSON/CSV, clear-all, pagination, restore quality/maxChars on load | | +| C19 | Web | **PWA completion** — manifest references nonexistent `/icon-192.png`/`/icon-512.png`; add icons + SW or delete manifest | | +| C20 | Web | **AbortController + Cancel button** during up-to-60s resolves | | + +## Wave D — Architecture & DRY + +| ID | Area | File(s) | Action | +|----|------|---------|--------| +| D1 | Py | `_cascade.py` / `_cascade_async.py` / `_query_resolve.py:25-61` / `_url_resolve.py:41-77` / `_url_resolve_async.py:167-203` | Consolidate 3× copy-pasted semantic-cache helpers into `semantic_cache.py` single `check()`/`store()`; `resolve.py:75-87` double-computes embeddings on miss | +| D2 | Py | `providers/jina.py`, `serper.py`, `exa.py`, `tavily.py`, `firecrawl.py`, `mistral.py` | Each duplicates sync+async bodies verbatim; extract shared core taking a client factory (~halves each file) | +| D3 | Py | `utils/http.py:76-153` vs `utils/async_http.py:62-137` | Dedupe `is_safe_url`/`_normalize_host`/`_getaddrinfo_cached` (~80 lines each); async `is_safe_url` is pure-sync | +| D4 | Py | `_url_resolve.py:178-197` vs `_url_resolve_async.py:129-148` | Dedupe verbatim `_url_result_builder` | +| D5 | Py | `_url_resolve_async.py:79-105` | Async parity: add `llms_txt`, wire `docling`/`ocr` async file-type shortcuts, add `resolve_query_stream_async`, store final best-free result in semantic cache (sync misses it), port `EXCELLENT_QUALITY_THRESHOLD` early-exit, enforce budget mid-flight | +| D6 | Py | `providers/tavily.py:80`, `exa.py:98,194`, `firecrawl.py:40,90`, `mistral.py:182,260` | Cache SDK clients per API key (thread-safe) instead of per-call construction | +| D7 | Py | `utils/fetch.py:20-25` | Drop HEAD pre-flight in `fetch_url_content` (double round-trip; `response.url` suffices) | +| D8 | Py | `utils/cache.py:124-129`, `32-70`, `105-108` | Remove `_get_cache_proxy` (per-op import of `scripts.resolve`; never assigned) and dead `coalesce_request` (or wire it in for thundering-herd) | +| D9 | Py | `utils/async_http.py:28,31-51` | Actually acquire `_client_lock` (AsyncClient leak race) | +| D10 | Py | `semantic_cache.py:174-179, 350-368, 315-322` | Add TTL/max-age invalidation (timestamp stored, never read); fix similarity-dedup JSON key-ordering | +| D11 | Py | `visual_resolver.py:193-208` | Batch frame encoding (`encode` accepts a list) | +| D12 | Py | `models.py:35-51`, `utils/urls.py:118-137`, `cache_negative.py:58-71`, `constants.py:12-13,73`, `resolve.py:90-129` | Remove dead code: `Profile.is_provider_allowed`/`max_hops`, `score_result`, `should_skip_from_bot_challenge_cache` (cache never populated), `ENABLE_SEMANTIC_CACHE`, empty TYPE_CHECKING, re-export bloat; fix `found_final` unreachable branch (`_cascade.py:92,161`) | +| D13 | Py | `doc_validator.py:336-343`, `_routing_utils.py` | Remove `fix_rust_architecture` stub (always returns 0); fold 17-line `_routing_utils` into `routing_memory` | +| D14 | Py | `scripts/cli.py:35-50,74-82` | Dedupe async/sync dispatch if/elif chain | +| D15 | Rust | `cli/src/resolver/url.rs` (532) + `resolver/query/mod.rs` (504) | Extract shared cascade engine (param'd over async attempt fn) — kills ~300 duplicated lines, fixes 2 file-size violations, removes subtle drift (min_free_quality override) | +| D16 | Rust | `cli/src/providers/direct_fetch.rs` (612) | Move `decode_entities` + `strip_html`/`StriperState` to `providers/html_strip.rs` | +| D17 | Rust | `cli/src/semantic_cache/ops.rs` (521) | Split `normalize_text`/`encode_query` into own module | +| D18 | Rust | `cli/src/types.rs:189-199`, `output.rs:50-76`, `semantic_cache/mod.rs:31-42`, `routing.rs:9,59` | Remove dead code: `is_fast()`, `TextOutput`, `CacheEntry`, `PreflightResult.confidence`, `PlannedProvider.skip_reason` | +| D19 | Rust | `resolver/mod.rs:33` + `synthesis.rs:24` | Unify duplicate link-regex statics | +| D20 | Rust | `cli/Cargo.toml` | Drop unused deps: `sha2`, dev `mockall`, clap `env` feature, `futures` (JoinSet), `once_cell` (LazyLock); trim tokio features; `#[command(version)]` instead of hardcoded "0.3.10" (`cli.rs:11`) | +| D21 | Rust | `providers/serper.rs:47-82`, `.do_wdr_state.toml` | Move CWD-relative state to `$XDG_STATE_HOME/do-wdr/`; wrap `save_credits` sync IO in `spawn_blocking` | +| D22 | Rust | `main.rs:197,202` | Config loaded twice per invocation | +| D23 | Rust | `url.rs:337`, `query/mod.rs:330`, `output.rs:9-16` | `validate_links` blocks response on up to 10 HEADs but `validated_links` isn't in JSON output — make opt-in (`--validate-links`) or expose | +| D24 | Rust | `main.rs:156-161` | `--metrics-json` before content on stdout breaks parsing — send to stderr or `--metrics-file` | +| D25 | Rust | `url.rs:233-244` | Skips recorded as `success:false, latency:0` skew failure-rate metrics — record as skip | +| D26 | Rust | `providers/mistral_websearch.rs:122-128` | Fabricated pseudo-URL treated as real source by bias/caching — mark synthetic | +| D27 | Rust | `providers/docling.rs:29,42`, `ocr.rs` | `is_available()` hardcoded true burns attempts when tools missing — actually probe | +| D28 | Rust | stray `cli/text` artifact | Delete (accidental `> text` redirect) | +| D29 | Web | `web/app/page.tsx` (514 — **violates 500 limit**) | Decompose god component: extract reducer/context + hooks (prewarm, history, keys); MainContent takes ~30 props incl. unused setters | +| D30 | Web | `lib/results.ts:45-47` | Remove `nextjs.org` special case from generic canonicalizer | +| D31 | Web | `lib/records.ts:149-182` | Browser `fetch("/api/history")` inside server lib — split client/server modules | +| D32 | Web | `page.tsx:155-167` | Full result markdown written to localStorage on every query keystroke — debounce | +| D33 | Web | `app/api/history/route.ts:27-31` vs `ui-state/route.ts:24-33` | Unify session identity (httpOnly cookie vs sha256(IP+UA)) | +| D34 | Web | `middleware.ts:11-17` | Matcher runs on all API traffic but only acts on POST /api/resolve — narrow | +| D35 | Web | `app/api/history`, `app/api/records` POST | Add zod validation + size caps (unbounded `result` string = memory-flood vector); validate provider ids against known list (currently silent null → "No results") | +| D36 | Web | `lib/validation.ts:225` vs `Sidebar.tsx:153-157` | maxChars bounds mismatch (zod 100–50000 vs slider 1000–32000) | +| D37 | Web | `lib/validation.ts:264` | `validateResolveRequest` swallows zod detail — surface message | +| D38 | Web | `next.config.mjs:14-37` vs `vercel.json:10-20` | Duplicate security-header config — one source; drop deprecated `X-XSS-Protection` | +| D39 | Web | `lib/resolvers/index.ts:58` vs `routing.ts:70` | Dedupe disagreeing paid-provider sets; remove unused export | +| D40 | Web | `lib/errors.ts` | Wire `classifyError`/`formatErrorForDisplay` user hints into `page.tsx:308-311` (currently raw `${provider}: ${error}`) | +| D41 | Web | `app/help/page.tsx:53-59`, `web/README.md` | Fix cascade drift (Serper missing), "Next.js 15" → 16, phantom word count | +| D42 | Web | `app/api/analytics/route.ts:36-39` | `cacheHitRate` computed as `score>=0.8` ratio — store real hit info or rename | +| D43 | Web | `app/page.tsx:170-209` | Silent prewarm fires 5 POSTs on mount consuming rate budget — gate behind setting / batch | +| D44 | Web | `eslint.config.mjs:35-37` | Re-enable `no-unused-vars`/`no-explicit-any`; delete accumulated dead exports | +| D45 | Web | `app/css.d.ts`, `.ignore` | Remove vestigial files | +| D46 | Web | `playwright.config.ts:11`, `package.json:17` | E2E defaults to **production** URL — default to local; prevent accidental prod mutation | + +## Wave E — Testing & CI + +| ID | Area | Action | +|----|------|--------| +| E1 | Py | Unit tests for `_cascade_async.py` (~200 lines, zero direct tests): parallel launch, 0.85 early-exit, paid-skip gate, exception→breaker, cancellation, best-free fallback | +| E2 | Py | Make conftest's `plan_provider_order` override opt-in (marker/fixture); real routing (adaptive reorder, tier sort, probabilistic skip) is dead code in every resolve test | +| E3 | CI | `--cov-fail-under=75` (ratchet to 85) + Codecov upload + PR comments; coverage.xml already generated but gates nothing | +| E4 | CI | Weekly perf-regression workflow: `cargo bench` + `pytest -m benchmark`, fail on >20% regression vs baseline — codifies documented <200ms / ≥0.85 semantic-health thresholds (currently hand-run) | +| E5 | CI | `RUST_TEST_THREADS=1` (libsql `Once` poisoning) via `cli/.cargo/config.toml` — currently missing from ci.yml/release.yml | +| E6 | CI | Release smoke test: on publish, download each asset, run `do-wdr --help` + no-key resolve on all 3 OSes | +| E7 | CI | Fix live-test vacuous passes: fail job when expected secret absent (typo'd name = green CI) | +| E8 | CI | Add concurrency group to `ci-integration.yml`; split 3× Python matrix (lint once, full tests on 3.12) | +| E9 | CI | Run `dark-mode` Playwright project + web vitest coverage in ci-ui.yml | +| E10 | Rust | Tests for untested modules: `bias_scorer`, `circuit_breaker`, `negative_cache`, `routing_memory`, `routing`, `startup`, `link_validator`, `semantic_cache/ops+synthesis`, `config/defaults+parsing`, `docling`, `ocr` | +| E11 | Rust | Replace `tests/quality_gate.rs:5-47` re-implementation with tests against the real gate function | +| E12 | Web | Dedupe `tests/api/route.test.ts` local helper re-implementations (own `isUrl`) — import production code | +| E13 | Web | Extract shared `mockAppState` helper (duplicated in 5 e2e specs); `test.skip` instead of silent `return` | +| E14 | Gate | `scripts/quality_gate.sh`: enforce markdownlint (remove `\|\| true`); run web unit tests; extend 500-line check beyond `scripts/*.py` to `web/` + `cli/src/` | +| E15 | Ops | Serper 403 root cause: add key-expiry detection to `monitor_providers.py` (3 consecutive monthly auto-deprioritizations = symptom-only treatment) | +| E16 | Py | Unify nested-loop workarounds on shared thread pool (`_cascade.py:178-184`, `utils/http.py:255-261` spawn fresh executors per call); route `asyncio.to_thread` through `get_shared_pool()` | + +## Wave F — Polish & Docs + +| ID | Area | Action | +|----|------|--------| +| F1 | Web a11y | Result-card headings (URL-titled cards aren't headings); move `aria-live` off Copy button to status region; `aria-busy` on loading region; visible (non-tooltip) provider key hints; replace full-screen backdrop `