From 3bec470ff5c964d1134e58682ebac99de71afa82 Mon Sep 17 00:00:00 2001 From: Nigel Todman Date: Fri, 9 Oct 2026 02:53:41 +0000 Subject: [PATCH] =?UTF-8?q?ddg:=20x-vqd-hash-1=20protocol=20rewrite=20(plu?= =?UTF-8?q?gin=20v0.2.0)=20=E2=80=94=20FEAT-10=20closed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DDG retired the x-vqd-4 token AND the 2025 model catalog. This is the full rewrite to the challenge-based protocol (reverse-engineered 2026-10-09, cross-validated against benoitpetit/duckduckgo-chat-cli): - Proof lifecycle: /status (x-vqd-accept: 1) returns a base64 JS challenge in x-vqd-hash-1; solved by the bundled Node helper (ddg_vqd.js, DDG_SOLVE_UA in lockstep with the request UA); post-processed like the page bundle (client_hashes -> base64(SHA256(String(value))), meta origin/stack/duration) -> the X-Vqd-Hash-1 REQUEST header; per-turn rotation via the /chat response header. - Chat body: canUseTools/canUseApproxLocation/reasoningEffort/ durableStream (client-minted RSA-2048 JWK via bundled ddg_durable.js). - SSE: role-based grammar (assistant/source/tool-invocation/ ui-component) + legacy action fallback + bracketed markers. - Catalog rotated: 8 live wire IDs (gpt-6-luna, gpt-5.6-luna, gpt-5.4-nano/mini, claude-haiku-4-5, mistral-small-2603, tinfoil/gpt-oss-120b, tinfoil/gemma4-31b) with forward-resolving legacy aliases; live discovery via fetch_capabilities() (/duckchat/v1/capabilities, proof-gated retry). - generate_stream() TRUE incremental (FEAT-10 closed) with full error-taxonomy parity. - Fixes over the first rewrite pass: is_cloud=True restored (BaseBackend defaults direct subclasses to local), tool/function roles fold to user (DDG only understands user/assistant), mixed-case legacy alias keys never matched the lowercased lookup, stream limit/403/429/5xx wrapping. - Tests: 49 -> 99 (proof lifecycle, wire shape, SSE grammar, stream, capabilities, taxonomy, alias table, node plumbing). Suite 3,433 passed / 20 skipped. ruff + black clean. - scripts/probe_duckduckgo.py rewritten for the new protocol (--live drives the backend end-to-end; TEST-13 automation). - audit: FEAT-10 CLOSED; TEST-13 partially closed (handshake live-verified via ddg-challenge-solver; one live 200 SSE turn remains). Register 136 findings / 18 OPEN. --- agentkthx/cli/commands/config.py | 34 + agentkthx/config.py | 28 + agentkthx/core/types.py | 14 + agentkthx/data/model_seed.json | 74 ++ agentkthx/plugins/duckduckgo/__init__.py | 38 + agentkthx/plugins/duckduckgo/ddg_durable.js | 40 + agentkthx/plugins/duckduckgo/ddg_vqd.js | 276 ++++ agentkthx/plugins/duckduckgo/duckduckgo.py | 1228 ++++++++++++++++++ agentkthx/plugins/duckduckgo/plugin.json | 44 + audit/audit.md | 35 + scripts/probe_duckduckgo.py | 131 ++ tests/test_duckduckgo_backend.py | 1277 +++++++++++++++++++ 12 files changed, 3219 insertions(+) create mode 100644 agentkthx/plugins/duckduckgo/__init__.py create mode 100644 agentkthx/plugins/duckduckgo/ddg_durable.js create mode 100644 agentkthx/plugins/duckduckgo/ddg_vqd.js create mode 100644 agentkthx/plugins/duckduckgo/duckduckgo.py create mode 100644 agentkthx/plugins/duckduckgo/plugin.json create mode 100644 scripts/probe_duckduckgo.py create mode 100644 tests/test_duckduckgo_backend.py diff --git a/agentkthx/cli/commands/config.py b/agentkthx/cli/commands/config.py index e951dd37..0deba275 100644 --- a/agentkthx/cli/commands/config.py +++ b/agentkthx/cli/commands/config.py @@ -26,6 +26,8 @@ "cf": "Cloudflare", "siliconflow": "SiliconFlow", "sf": "SiliconFlow", + "duckduckgo": "DuckDuckGo", + "ddg": "DuckDuckGo", } @@ -244,6 +246,24 @@ "RESERVED for a future free tier — unused today (no free models; " "default: Qwen/Qwen3-8B)", ), + # ── DuckDuckGo AI Chat ── + ( + "DUCKDUCKGO_BASE_URL", + "DuckDuckGo AI Chat URL (default: https://duckduckgo.com — the " + "protocol lives under /duckchat/v1)", + ), + ( + "DUCKDUCKGO_USER_AGENT", + "Browser User-Agent override — DDG 403s non-browser UAs; empty = " + "built-in Chrome 120 UA (only set if DDG rejects the built-in one)", + ), + ( + "DUCKDUCKGO_DEFAULT_MODEL", + "Default model (default: gpt-4o-mini; catalog: gpt-4o-mini, " + "claude-3-haiku-20240307, meta-llama/Llama-3.3-70B-Instruct-Turbo, " + "mistralai/Mistral-Small-24B-Instruct-2501, o3-mini — no API key " + "needed, the backend is keyless and anonymous)", + ), # ── ACP / TurboQuant ── ("ACP_BASE_URL", "ACP server URL"), ("ACP_USER", "ACP username (default: admin)"), @@ -286,6 +306,9 @@ def cmd_config(args: argparse.Namespace) -> int: CLOUDFLARE_FREE_ONLY, DEBUG, DEFAULT_MODEL, + DUCKDUCKGO_BASE_URL, + DUCKDUCKGO_DEFAULT_MODEL, + DUCKDUCKGO_USER_AGENT, GEMINI_API_KEY, GEMINI_BASE_URL, GEMINI_DEFAULT_MODEL, @@ -374,6 +397,7 @@ def cmd_config(args: argparse.Namespace) -> int: ("NVIDIA_BASE_URL", NVIDIA_BASE_URL), ("CLOUDFLARE_BASE_URL", CLOUDFLARE_BASE_URL or "(built from CLOUDFLARE_ACCOUNT_ID)"), ("SILICONFLOW_BASE_URL", SILICONFLOW_BASE_URL), + ("DUCKDUCKGO_BASE_URL", DUCKDUCKGO_BASE_URL), ("ACP_BASE_URL", ACP_BASE_URL), ] for name, val in urls: @@ -400,6 +424,7 @@ def cmd_config(args: argparse.Namespace) -> int: ("MISTRAL_BASE_URL", MISTRAL_BASE_URL), ("ORCAROUTER_BASE_URL", ORCAROUTER_BASE_URL), ("POLLINATIONS_BASE_URL", POLLINATIONS_BASE_URL), + ("DUCKDUCKGO_BASE_URL", DUCKDUCKGO_BASE_URL), ("ACP_BASE_URL", ACP_BASE_URL), ], "OpenRouter": [ @@ -480,6 +505,14 @@ def cmd_config(args: argparse.Namespace) -> int: ("SILICONFLOW_FREE_ONLY", str(SILICONFLOW_FREE_ONLY)), ("SILICONFLOW_FREE_FALLBACK_MODEL", SILICONFLOW_FREE_FALLBACK_MODEL), ], + "DuckDuckGo": [ + ("DUCKDUCKGO_BASE_URL", DUCKDUCKGO_BASE_URL), + ("DUCKDUCKGO_DEFAULT_MODEL", DUCKDUCKGO_DEFAULT_MODEL), + ( + "DUCKDUCKGO_USER_AGENT", + DUCKDUCKGO_USER_AGENT or "(built-in browser UA)", + ), + ], "ACP": [ ("ACP_USER", ACP_USER), ("ACP_PASS", _mask_key(ACP_PASS)), @@ -544,6 +577,7 @@ def cmd_config(args: argparse.Namespace) -> int: "Cloudflare": CLOUDFLARE_BASE_URL or f"https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID or ''}/ai/v1", "SiliconFlow": SILICONFLOW_BASE_URL, + "DuckDuckGo": DUCKDUCKGO_BASE_URL, "ACP": ACP_BASE_URL, }, acp_user=ACP_USER, diff --git a/agentkthx/config.py b/agentkthx/config.py index 0c82d6e5..cd725fda 100644 --- a/agentkthx/config.py +++ b/agentkthx/config.py @@ -435,6 +435,34 @@ SILICONFLOW_FREE_FALLBACK_MODEL = os.environ.get("SILICONFLOW_FREE_FALLBACK_MODEL", "Qwen/Qwen3-8B") +# ═══════════════════════════════════════════════════════════════════════════════ +# DuckDuckGo plugin (agentkthx/plugins/duckduckgo/) +# DuckDuckGo AI Chat (duck.ai) — the keyless, anonymous, zero-cost LLM +# surface via the NON-OpenAI /duckchat/v1 protocol. NO API KEY EXISTS — +# "authentication" is the x-vqd-hash-1 CHALLENGE PROOF (the retired +# x-vqd-4 opaque token is gone): GET /status issues an obfuscated JS +# challenge, the bundled Node solver executes it against a clean-browser +# environment, and the solved result rides the X-Vqd-Hash-1 request +# header. Each /chat response carries the NEXT challenge (per-turn +# rotation). Held in-memory per backend instance, never persisted. See +# docs/api/DUCKDUCKGO_API_TECHNICAL_REFERENCE.md. +# ═══════════════════════════════════════════════════════════════════════════════ +DUCKDUCKGO_BASE_URL = os.environ.get("DUCKDUCKGO_BASE_URL", "https://duck.ai") +# Browser User-Agent — DDG's anti-bot layer returns 403 for +# non-browser UAs. Empty string falls back to the backend's built-in +# Chrome 136 UA (the default). The challenge solver is fed the SAME +# UA (a mismatch invalidates the proof). Only set this if DDG starts +# rejecting the built-in one (UA rotation is an anti-fingerprinting +# last resort, not a routine knob). +DUCKDUCKGO_USER_AGENT = os.environ.get("DUCKDUCKGO_USER_AGENT", "") +# Default model — gpt-6-luna is DuckDuckGo's own default pick. The +# catalog: gpt-6-luna, gpt-5.6-luna, gpt-5.4-nano, gpt-5.4-mini, +# claude-haiku-4-5, mistral-small-2603, tinfoil/gpt-oss-120b, +# tinfoil/gemma4-31b (legacy aliases gpt-4o-mini, claude-3-haiku, +# llama, mixtral, o3-mini resolve forward to their successors). +DUCKDUCKGO_DEFAULT_MODEL = os.environ.get("DUCKDUCKGO_DEFAULT_MODEL", "gpt-6-luna") + + # ═══════════════════════════════════════════════════════════════════════════════ # BACKEND SELECTION # ═══════════════════════════════════════════════════════════════════════════════ diff --git a/agentkthx/core/types.py b/agentkthx/core/types.py index 18d19a39..5c3414d3 100644 --- a/agentkthx/core/types.py +++ b/agentkthx/core/types.py @@ -199,6 +199,20 @@ class BackendType(Enum): # docs/api/SILICONFLOW_API_TECHNICAL_REFERENCE.md. SILICONFLOW = "siliconflow" + # DuckDuckGo AI Chat — the keyless, anonymous, zero-cost LLM + # surface at duckduckgo.com/duckchat/v1 (fronting duck.ai). NOT + # OpenAI-compatible: custom x-vqd-4 header-token handshake + # (bootstrap via GET /status, rotate on every /chat response), + # own SSE shape, no /models endpoint, no sampling params, tools + # stripped (ReAct only), ~20-turn conversation limit. Catalog: + # GPT-4o mini, o3-mini, Claude Haiku, Llama 3.3 70B, Mistral + # Small 3 24B — the upstream frontier set, IP-stripped by DDG's + # privacy layer. The ONLY cloud backend that subclasses + # BaseBackend directly instead of CloudBackend. + # See agentkthx/plugins/duckduckgo/ and + # docs/api/DUCKDUCKGO_API_TECHNICAL_REFERENCE.md. + DUCKDUCKGO = "duckduckgo" + class ApiMode(Enum): """API mode for backend communication. diff --git a/agentkthx/data/model_seed.json b/agentkthx/data/model_seed.json index d7efa07f..041afe2b 100644 --- a/agentkthx/data/model_seed.json +++ b/agentkthx/data/model_seed.json @@ -1923,5 +1923,79 @@ "default_temperature": 0.7, "default_max_tokens": 8192 } + }, + "duckduckgo": { + "gpt-6-luna": { + "context_length": 128000, + "default_temperature": 0.7, + "default_max_tokens": 16384, + "family": "gpt", + "free_tier": true, + "upstream": "OpenAI GPT-6 Luna (via DDG proxy)", + "note": "DuckDuckGo's own default model ('Best for everyday use'); verified against the live dropdown + benoitpetit/duckduckgo-chat-cli models.go, 2026-10-09" + }, + "gpt-5.6-luna": { + "context_length": 128000, + "default_temperature": 0.7, + "default_max_tokens": 16384, + "family": "gpt", + "free_tier": true, + "upstream": "OpenAI GPT-5.6 Luna (via DDG proxy)", + "note": "Previous Luna model; legacy alias gpt-4o-mini resolves here (duckduckgo-chat-cli compat map)" + }, + "gpt-5.4-nano": { + "context_length": 128000, + "default_temperature": 0.7, + "default_max_tokens": 16384, + "family": "gpt", + "free_tier": true, + "upstream": "OpenAI GPT-5.4 Nano (via DDG proxy)", + "note": "Fast, lightweight tier; alias 'nano' resolves here" + }, + "gpt-5.4-mini": { + "context_length": 128000, + "default_temperature": 0.7, + "default_max_tokens": 16384, + "family": "gpt", + "free_tier": true, + "upstream": "OpenAI GPT-5.4 Mini (via DDG proxy)", + "note": "'Solid but hits limits sooner'; legacy aliases o3-mini / o4mini resolve here" + }, + "claude-haiku-4-5": { + "context_length": 200000, + "default_temperature": 0.7, + "default_max_tokens": 8192, + "family": "claude", + "free_tier": true, + "upstream": "Anthropic Claude Haiku 4.5", + "note": "Alias 'claude' resolves here; legacy claude-3-haiku / claude-3-haiku-20240307 IDs rotated out (now 404 ERR_MODEL_UNAVAILABLE)" + }, + "mistral-small-2603": { + "context_length": 128000, + "default_temperature": 0.7, + "default_max_tokens": 8192, + "family": "mistral", + "free_tier": true, + "upstream": "Mistral Small 4", + "note": "Display name 'Mistral Small 4'; wire ID is mistral-small-2603 (NOT mistral-small-4); alias 'mixtral' resolves here" + }, + "tinfoil/gpt-oss-120b": { + "context_length": 128000, + "default_temperature": 0.7, + "default_max_tokens": 16384, + "family": "gpt-oss", + "free_tier": true, + "upstream": "OpenAI gpt-oss-120b on Tinfoil (zero provider visibility)", + "note": "Open-weight reasoning model served with reasoningEffort 'low'; alias 'llama' resolves here (the open-weights slot)" + }, + "tinfoil/gemma4-31b": { + "context_length": 128000, + "default_temperature": 0.7, + "default_max_tokens": 8192, + "family": "gemma", + "free_tier": true, + "upstream": "Google Gemma 4 31B on Tinfoil (zero provider visibility)", + "note": "BETA in the dropdown; served with reasoningEffort 'low'; alias 'gemma' resolves here" + } } } diff --git a/agentkthx/plugins/duckduckgo/__init__.py b/agentkthx/plugins/duckduckgo/__init__.py new file mode 100644 index 00000000..03cdb0ae --- /dev/null +++ b/agentkthx/plugins/duckduckgo/__init__.py @@ -0,0 +1,38 @@ +""" +AgentKthx Plugin — DuckDuckGo AI Chat Backend + +DuckDuckGo AI Chat (duck.ai) backend — the keyless, anonymous, +zero-cost LLM surface: no API key, no signup, no quota. Current +lineup (GPT-6 Luna, GPT-5.6 Luna, GPT-5.4 Nano/Mini, Claude +Haiku 4.5, Mistral Small 4, and the Tinfoil-hosted gpt-oss-120b / +Gemma 4 31B) proxied through DuckDuckGo's privacy layer via the +NON-OpenAI /duckchat/v1 protocol — now CHALLENGE-BASED (the +x-vqd-hash-1 JS proof replaced the retired x-vqd-4 token; solved +by the bundled Node helper). + +The only AgentKthx cloud backend that subclasses BaseBackend +directly instead of CloudBackend — see duckduckgo.py for the +protocol rationale. + +See docs/api/DUCKDUCKGO_API_TECHNICAL_REFERENCE.md for full details. + +Written by VTSTech — https://www.vts-tech.org +""" + +from __future__ import annotations + + +def register(manager) -> None: + """Register the DuckDuckGo backend with the plugin manager.""" + from .duckduckgo import DuckDuckGoBackend + + manager.register_backend("duckduckgo", DuckDuckGoBackend) + # Alias for convenience — `--backend duckduckgo` and `--backend ddg` + # both work. Mirrors the SiliconFlow `sf` alias pattern. + manager.register_backend("ddg", DuckDuckGoBackend) + + +def unregister(manager) -> None: + """Unregister the DuckDuckGo backend from the plugin manager.""" + manager.unregister_backend("duckduckgo") + manager.unregister_backend("ddg") diff --git a/agentkthx/plugins/duckduckgo/ddg_durable.js b/agentkthx/plugins/duckduckgo/ddg_durable.js new file mode 100644 index 00000000..aaec4dac --- /dev/null +++ b/agentkthx/plugins/duckduckgo/ddg_durable.js @@ -0,0 +1,40 @@ +#!/usr/bin/env node +// ═══════════════════════════════════════════════════════════════════════ +// ddg_durable.js — durableStream RSA keypair generator for duck.ai +// +// The duck.ai POST /duckchat/v1/chat body now carries a `durableStream` +// object: {messageId, conversationId, publicKey}. The publicKey is a +// client-generated RSA-2048 JWK (the web client persists its pair in +// localStorage under `duckaiStreamsTestCredentials`; the server uses it +// to encrypt stream-resume payloads). Python stdlib has no RSA, so we +// mint the keypair here — Node crypto is built in. +// +// Prints one JSON object on stdout: +// {"publicJwk": {alg,e,ext,key_ops,kty,n,use}, +// "privateJwk": {…full RSA private JWK…}} +// +// Usage: node ddg_durable.js +// ═══════════════════════════════════════════════════════════════════════ +'use strict'; + +const { generateKeyPairSync } = require('crypto'); + +const { publicKey, privateKey } = generateKeyPairSync('rsa', { + modulusLength: 2048, + publicKeyEncoding: { format: 'jwk' }, + privateKeyEncoding: { format: 'jwk' }, +}); + +// Normalize to the exact shape the duck.ai frontend sends (captured live): +// alg RSA-OAEP-256, key_ops ["encrypt"], use "enc", ext true. +const publicJwk = { + alg: 'RSA-OAEP-256', + e: publicKey.e, + ext: true, + key_ops: ['encrypt'], + kty: publicKey.kty, + n: publicKey.n, + use: 'enc', +}; + +process.stdout.write(JSON.stringify({ publicJwk, privateJwk: privateKey })); diff --git a/agentkthx/plugins/duckduckgo/ddg_vqd.js b/agentkthx/plugins/duckduckgo/ddg_vqd.js new file mode 100644 index 00000000..257ccaca --- /dev/null +++ b/agentkthx/plugins/duckduckgo/ddg_vqd.js @@ -0,0 +1,276 @@ +#!/usr/bin/env node +// ═══════════════════════════════════════════════════════════════════════ +// ddg_vqd.js — DuckDuckGo x-vqd-hash-1 challenge solver (headless, v2) +// +// Reads a base64 challenge blob on stdin (or argv[1]), executes it against a +// GLOBALS-INSTALLED clean-browser environment, prints the raw solution JSON: +// {"server_hashes":[...], "client_hashes":[ua, probe2, probe3], +// "signals":{}, "meta":{v, challenge_id, timestamp, debug}} +// +// Why globals (not Function params): the pxjzr probe family checks +// (function(){return this;}()) === window +// which is only true when `window` IS the global object — exactly like a +// real browser page. The solver runs as a dedicated subprocess, so +// polluting globalThis is safe and gets us that semantics for free. +// +// Clean-browser facts emulated (verified against captured real-browser hashes): +// - window.parseInt.toString() includes '[native code]' (real native) +// - class X extends Array; X.prototype.map → instanceof X (ES6 semantics) +// - Object.prototype.toString.call(window) === '[object Window]' +// - navigator.webdriver === true → FALSE (human browsing) +// - iframe.contentWindow.self.get → undefined (no instrumentation) +// - window.top has no polluted _Array/_Promise/... aliases +// - document.body.children tracks appendChild/removeChild +// - document.querySelectorAll('*') → real NodeList (not an Array) +// - document.createElement('div') instanceof HTMLDivElement, +// HTMLDivElement.prototype instanceof HTMLElement instanceof Element +// - DOM elements measure > 0 (offsetWidth/Height, scrollHeight, rect) +// - getComputedStyle(...).getPropertyValue('display') non-empty +// - innerHTML + querySelectorAll → real HTML-parsed element counts +// - meta[http-equiv=Content-Security-Policy] present (duck.ai CSP) +// +// Usage: node ddg_vqd.js [--report] [challenge.b64 | -] < blob +// Env: DDG_SOLVE_UA override navigator.userAgent +// ═══════════════════════════════════════════════════════════════════════ +'use strict'; + +const REPORT = process.argv.includes('--report'); +const accesses = new Map(); +function note(p) { if (REPORT) accesses.set(p, (accesses.get(p) || 0) + 1); } + +const G = globalThis; + +// ── DOM class hierarchy (instanceof chains must hold) ────────────────── +class Element { + constructor(tagName) { + this.tagName = String(tagName || 'div').toUpperCase(); + this.style = {}; + this.textContent = ''; + this._attributes = {}; + this._parsed = []; + this.children = []; + } + get offsetWidth() { note('el.offsetWidth'); return 8; } + get offsetHeight() { note('el.offsetHeight'); return 8; } + get scrollHeight() { note('el.scrollHeight'); return 8; } + get scrollWidth() { note('el.scrollWidth'); return 8; } + getBoundingClientRect() { + note('el.getBoundingClientRect'); + return { width: 8, height: 8, top: 0, left: 0, right: 8, bottom: 8, x: 0, y: 0 }; + } + set innerHTML(html) { note('el.innerHTML(set)'); this._parsed = parseHtml(String(html)); } + get innerHTML() { return ''; } + querySelectorAll(sel) { + note('el.querySelectorAll:' + sel); + return this._parsed.filter(e => selectorMatches(e, sel)); + } + querySelector(sel) { + note('el.querySelector:' + sel); + return this._parsed.find(e => selectorMatches(e, sel)) || null; + } + getAttribute(name) { + note('el.getAttribute:' + name); + return Object.prototype.hasOwnProperty.call(this._attributes, name) + ? this._attributes[name] : null; + } + setAttribute(name, value) { this._attributes[name] = String(value); } + appendChild(child) { note('el.appendChild'); if (child) this.children.push(child); } + removeChild(child) { + note('el.removeChild'); + const i = this.children.indexOf(child); + if (i !== -1) this.children.splice(i, 1); + } +} +class HTMLElement extends Element {} +class HTMLDivElement extends HTMLElement {} +class HTMLMetaElement extends HTMLElement {} +class HTMLIFrameElement extends HTMLElement {} + +// NodeList — a real class, NOT an Array subclass +class NodeList { + constructor(items) { this._items = items.slice(); } + get length() { return this._items.length; } + item(i) { return this._items[i] || null; } + entries() { return this._items.entries(); } + forEach(fn, thisArg) { this._items.forEach(fn, thisArg); } + [Symbol.iterator]() { return this._items[Symbol.iterator](); } +} + +// ── mini HTML parser (for innerHTML/querySelectorAll probes) ─────────── +const VOID_TAGS = new Set(['area','base','br','col','embed','hr','img','input', + 'link','meta','param','source','track','wbr']); + +function parseHtml(html) { + const els = [], stack = []; + const re = /<(\/?)([a-zA-Z][a-zA-Z0-9-]*)((?:"[^"]*"|'[^']*'|[^>"'])*)>?/g; + let m; + while ((m = re.exec(html)) !== null) { + const closing = m[1] === '/'; + const tag = m[2].toLowerCase(); + if (closing) { + const idx = stack.lastIndexOf(tag); + if (idx !== -1) stack.length = idx; + } else { + els.push({ tag }); + if (!VOID_TAGS.has(tag)) stack.push(tag); + } + } + return els; +} +function selectorMatches(el, sel) { + if (sel === '*') return true; + return el.tag === String(sel).trim().toLowerCase(); +} + +// ── element factory with the right per-tag class ─────────────────────── +function makeElement(tagName) { + const t = String(tagName || 'div').toLowerCase(); + let el; + if (t === 'div') el = new HTMLDivElement(); + else if (t === 'meta') el = new HTMLMetaElement(); + else if (t === 'iframe') el = new HTMLIFrameElement(); + else el = new HTMLElement(t); + if (t === 'iframe') { + // window-like self, NO 'get' property (matches a real browser) + el.contentWindow = { self: {} }; + el.contentDocument = null; + } + return el; +} + +// ── document ──────────────────────────────────────────────────────────── +const DUCK_CSP = "default-src 'none' ; connect-src https://duckduckgo.com https://*.duckduckgo.com https://duck.ai https://*.duck.ai ; manifest-src https://duckduckgo.com https://*.duckduckgo.com https://duck.ai https://*.duck.ai ; media-src 'self' https://duck.ai https://*.duck.ai ; script-src blob: https://duckduckgo.com https://*.duckduckgo.com https://duck.ai https://*.duck.ai 'unsafe-inline' 'unsafe-eval' ; font-src data: https://duckduckgo.com https://*.duckduckgo.com https://duck.ai https://*.duck.ai ; img-src 'self' data: https://duck.ai https://*.duck.ai https://external-content.duckduckgo.com/ip3/ https://improving.duckduckgo.com/t/ ; style-src https://duckduckgo.com https://*.duckduckgo.com https://duck.ai https://*.duck.ai 'unsafe-inline' ; object-src 'none' ; worker-src blob: ; child-src blob: https://duckduckgo.com https://*.duckduckgo.com https://duck.ai https://*.duck.ai ; frame-src blob: https://duckduckgo.com https://*.duckduckgo.com https://duck.ai https://*.duck.ai ; form-action https://duckduckgo.com https://*.duckduckgo.com https://duck.ai https://*.duck.ai ; frame-ancestors https://duckduckgo.com https://*.duckduckgo.com ; base-uri 'self' ; block-all-mixed-content ;"; + +const metaCsp = makeElement('meta'); +metaCsp._attributes['http-equiv'] = 'Content-Security-Policy'; +metaCsp._attributes['content'] = DUCK_CSP; + +const _body = makeElement('body'); +Object.defineProperty(_body, 'children', { + get() { return _body._kids; }, + configurable: true, +}); +_body._kids = []; + +const documentStub = { + createElement(tag) { note('document.createElement:' + tag); return makeElement(tag); }, + get body() { return _body; }, + querySelector(sel) { + note('document.querySelector:' + sel); + if (sel && String(sel).indexOf('Content-Security-Policy') !== -1) return metaCsp; + return null; + }, + querySelectorAll(sel) { + note('document.querySelectorAll:' + sel); + return new NodeList([]); + }, + getElementsByTagName() { return new NodeList([]); }, + getElementById() { return null; }, + documentElement: makeElement('html'), + head: makeElement('head'), +}; +_body.appendChild = function (child) { note('body.appendChild'); if (child) _body._kids.push(child); }; +_body.removeChild = function (child) { + note('body.removeChild'); + const i = _body._kids.indexOf(child); + if (i !== -1) _body._kids.splice(i, 1); +}; + +// ── navigator / getComputedStyle ──────────────────────────────────────── +const navigatorStub = { + userAgent: process.env.DDG_SOLVE_UA || + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36', + webdriver: false, + language: 'en-US', + languages: ['en-US', 'en'], + platform: 'Linux x86_64', + hardwareConcurrency: 8, + maxTouchPoints: 0, +}; + +function getComputedStyleStub(el) { + note('getComputedStyle'); + return { + getPropertyValue(prop) { + note('cs.getPropertyValue:' + prop); + if (prop === 'display') { + const css = (el && el.style && el.style.cssText) || ''; + const m = /display\s*:\s*([a-z-]+)/i.exec(css); + return m ? m[1] : 'block'; + } + return ''; + }, + }; +} + +// ── install the browser globals onto globalThis ───────────────────────── +// window IS globalThis — the pxjzr probe demands (function(){return this}()) === window +const SAVED = {}; +const TO_INSTALL = { + window: G, + self: G, + top: G, + parent: G, + frames: G, + document: documentStub, + navigator: navigatorStub, + getComputedStyle: getComputedStyleStub, + location: { href: 'https://duck.ai/', origin: 'https://duck.ai', protocol: 'https:', host: 'duck.ai' }, + HTMLElement, HTMLDivElement, HTMLMetaElement, HTMLIFrameElement, Element, NodeList, + localStorage: {}, sessionStorage: {}, + alert: () => {}, confirm: () => true, prompt: () => '', + innerWidth: 1920, innerHeight: 1080, outerWidth: 1920, outerHeight: 1080, + screenX: 0, screenY: 0, pageXOffset: 0, pageYOffset: 0, scrollX: 0, scrollY: 0, + devicePixelRatio: 1, + screen: { width: 1920, height: 1080, availWidth: 1920, availHeight: 1040 }, + history: { length: 1 }, +}; + +for (const [k, v] of Object.entries(TO_INSTALL)) { + SAVED[k] = G[k]; + try { Object.defineProperty(G, k, { value: v, writable: true, configurable: true }); } catch (e) { /* ignore */ } +} +// [object Window] — override Node's 'global' toStringTag +const SAVED_TAG = Object.getOwnPropertySymbols(G).find(s => s === Symbol.toStringTag); +const HAD_TAG = Object.getOwnPropertySymbols(G).includes(Symbol.toStringTag); +try { + Object.defineProperty(G, Symbol.toStringTag, { value: 'Window', configurable: true }); +} catch (e) { /* ignore */ } + +// ── run the challenge ─────────────────────────────────────────────────── +let b64 = null; +const argi = process.argv.slice(2).filter(a => a !== '--report'); +if (argi.length && argi[0] !== '-') { + b64 = require('fs').readFileSync(argi[0], 'utf8'); +} else { + b64 = require('fs').readFileSync(0, 'utf8'); +} +const js = Buffer.from(String(b64).trim(), 'base64').toString('utf8'); + +const t0 = Date.now(); +let result = null, failure = null; +try { + // new Function body: free identifiers resolve to our installed globals; + // non-strict `this` === globalThis === window + result = new Function('return (' + js + ');')(); +} catch (e) { failure = e; } + +Promise.resolve(result).then(v => { + if (REPORT) { + console.error('── API accesses (' + (Date.now() - t0) + 'ms) ──'); + for (const [k, c] of [...accesses.entries()].sort()) console.error(' ' + k + ' ×' + c); + } + if (!v || typeof v !== 'object' || !Array.isArray(v.client_hashes)) { + process.stderr.write('SHAPE_ERROR: ' + JSON.stringify(v) + '\n'); + process.exit(2); + } + process.stdout.write(JSON.stringify(v)); + process.exit(0); +}).catch(e => { + if (REPORT) { + console.error('── API accesses before failure ──'); + for (const [k, c] of [...accesses.entries()].sort()) console.error(' ' + k + ' ×' + c); + } + process.stderr.write('CHALLENGE_EXEC_ERROR: ' + e.message + '\n'); + process.exit(1); +}); diff --git a/agentkthx/plugins/duckduckgo/duckduckgo.py b/agentkthx/plugins/duckduckgo/duckduckgo.py new file mode 100644 index 00000000..84b4dc14 --- /dev/null +++ b/agentkthx/plugins/duckduckgo/duckduckgo.py @@ -0,0 +1,1228 @@ +""" +⚛️ AgentKthx — DuckDuckGo AI Chat Backend +Backend implementation for the DuckDuckGo AI Chat (duck.ai) API. + +DuckDuckGo AI Chat is the keyless, anonymous, zero-cost LLM surface: +no API key, no signup, no quota. Current upstream lineup (verified +Oct 2026 against the live duck.ai dropdown and the maintained Go +client benoitpetit/duckduckgo-chat-cli): OpenAI GPT-6 Luna / GPT-5.6 +Luna / GPT-5.4 Nano / GPT-5.4 Mini, Anthropic Claude Haiku 4.5, +Mistral Small 4, and the Tinfoil-hosted open-weight pair +gpt-oss-120b / Gemma 4 31B — all proxied through DuckDuckGo's +privacy layer, which strips your IP before forwarding upstream. + +This backend is DELIBERATELY unlike every other AgentKthx cloud +backend. The protocol is NOT OpenAI Chat-Completions — it is +DuckDuckGo's own /duckchat/v1/* wire format, and as of the +2026-10 protocol rotation it is CHALLENGE-BASED: + + Boot + per-request proof: + - GET /duckchat/v1/status (header ``x-vqd-accept: 1``) + → response header ``x-vqd-hash-1`` = base64 obfuscated JS + challenge (obfuscator.io-style, re-randomized per issue) + - The challenge is EXECUTED against a clean-browser environment + by the bundled Node solver (``ddg_vqd.js``) → raw solution + {server_hashes, client_hashes, signals, meta} + - Post-processing mirrors the duck.ai page bundle: + client_hashes[i] = base64(SHA256(String(raw))), meta gains + origin/stack/duration → base64(JSON) = the ``X-Vqd-Hash-1`` + REQUEST header (the proof) + - POST /duckchat/v1/chat with that proof; the RESPONSE header + ``x-vqd-hash-1`` carries the NEXT challenge (per-turn rotation) + + The retired ``x-vqd-4`` opaque token is GONE — /status no longer + returns it, and requests that send neither proof nor token fail. + This is why the pre-rotation scaffold broke (protocol drift, not + header shape: proven by a curl UA/Origin/casing/HTTP-version + matrix — every variant got the challenge). + + Chat body (current shape, mirrors the live frontend): + {model, messages, canUseTools, canUseApproxLocation, + reasoningEffort, durableStream:{messageId, conversationId, + publicKey }} + ``durableStream.publicKey`` is generated client-side (Node crypto + — Python stdlib has no RSA); the web client persists its pair in + localStorage under ``duckaiStreamsTestCredentials``. We mint a + fresh pair per backend instance (= per conversation) via the + bundled ``ddg_durable.js``. + + Response: SSE with ROLE-BASED events (the old action:"chunk" + grammar is gone): + data: {"role":"assistant","message":"…"} text deltas + data: {"role":"source","source":{url,title}} web citations + data: {"role":"tool-invocation",…} native tool frames + data: {"role":"ui-component",name:…} UI components + data: [DONE] / [PING] / [CHAT_TITLE:…] + The legacy action grammar is still parsed as a fallback — DDG + has flipped event shapes before without notice. + +Protocol constraints honored here: + - No ``system`` role — DDG strips it. ``_collapse_system_into_user`` + prepends system content to the first user message. + - No native function calling from AgentKthx yet — the wire now + HAS a native tools surface (canUseTools + metadata.toolChoice + with WebSearch / GenerateImage / NewsSearch / VideosSearch / + LocalSearch / WeatherForecast), but it is undocumented and + rotates independently; ``test_tool_support()`` stays REACT and + ``canUseTools`` ships false until that surface stabilizes. + - No sampling params — temperature / max_tokens / top_p / stop / + seed are accepted at the interface and silently dropped. + ``reasoningEffort`` is set per-model ("low" for the Tinfoil + open-weight pair, "none" otherwise) exactly like the frontend. + - Always-streaming — responses arrive as SSE; ``generate()`` + buffers the stream, ``generate_stream()`` yields text deltas + as they arrive (FEAT-10 — CLOSED by this rotation). + - Node is a runtime dependency for the challenge solver and the + durableStream keypair (subprocess, no npm packages). Without + node on PATH this backend fails fast with a clear message. + +Model catalog (wire IDs — the 2025-era IDs now 404 with +ERR_MODEL_UNAVAILABLE; verified 2026-10-09): + + - gpt-6-luna (DDG's default) + - gpt-5.6-luna + - gpt-5.4-nano + - gpt-5.4-mini + - claude-haiku-4-5 + - mistral-small-2603 (display name "Mistral Small 4") + - tinfoil/gpt-oss-120b (reasoningEffort low) + - tinfoil/gemma4-31b (reasoningEffort low) + +Legacy aliases resolve forward (gpt-4o-mini → gpt-5.6-luna, +claude-3-haiku → claude-haiku-4-5, llama → tinfoil/gpt-oss-120b, +mixtral → mistral-small-2603, o3-mini/o4mini → gpt-5.4-mini) so +existing sessions and scripts keep working across the rotation. + +Usage: + AGENTKTHX_BACKEND=duckduckgo agentkthx chat -m gpt-6-luna + agentkthx models --backend duckduckgo + +See docs/api/DUCKDUCKGO_API_TECHNICAL_REFERENCE.md for full details. + +Written by VTSTech — https://www.vts-tech.org +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import random +import re +import shutil +import subprocess +import time +import urllib.error +import urllib.request + +from agentkthx import model_cache +from agentkthx.backends.base import BaseBackend +from agentkthx.config import DUCKDUCKGO_BASE_URL, DUCKDUCKGO_DEFAULT_MODEL, DUCKDUCKGO_USER_AGENT +from agentkthx.core.types import BackendType, ThinkingSupport, ToolSupportLevel +from agentkthx.model_cache import load_seed_catalog + +#: Default browser User-Agent — DDG 403s non-browser UAs. Must match +#: the UA the challenge solver emulates (DDG_SOLVE_UA is injected from +#: this value at solve time). Overridable via DUCKDUCKGO_USER_AGENT. +_DEFAULT_UA = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36" +) + +#: Bundled Node helpers (challenge solver + durableStream keygen). +#: Shipped inside the plugin package so the backend is self-contained. +_PLUGIN_DIR = os.path.dirname(os.path.abspath(__file__)) +_SOLVER_JS = os.path.join(_PLUGIN_DIR, "ddg_vqd.js") +_DURABLE_JS = os.path.join(_PLUGIN_DIR, "ddg_durable.js") + +#: Frontend metadata fallbacks (user capture, 2026-10-08 build). +#: Best-effort refreshed from the homepage HTML at runtime — the +#: serp version rotates with deployments. +_FE_VERSION_FALLBACK = "serp_20261008_165829_ET-12db35fbec1a8e8189c691f90debc97b169e1c5a" +_ENTRY_JS_FALLBACK = "entry.duckai.9c8d02dfc0e8ac34875b.js" +_VENDORS_JS_FALLBACK = "entry.vendors.809c2cd1740bd8686f2.js" + +#: Wire model catalog — order matters (first = DDG's own default). +#: ``effort`` is the reasoningEffort the frontend sends per model. +_WIRE_MODELS: list[dict] = [ + {"id": "gpt-6-luna", "name": "GPT-6 Luna", "effort": "none"}, + {"id": "gpt-5.6-luna", "name": "GPT-5.6 Luna", "effort": "none"}, + {"id": "gpt-5.4-nano", "name": "GPT-5.4 Nano", "effort": "none"}, + {"id": "gpt-5.4-mini", "name": "GPT-5.4 Mini", "effort": "none"}, + {"id": "claude-haiku-4-5", "name": "Claude Haiku 4.5", "effort": "none"}, + {"id": "mistral-small-2603", "name": "Mistral Small 4", "effort": "none"}, + {"id": "tinfoil/gpt-oss-120b", "name": "GPT OSS 120B", "effort": "low"}, + {"id": "tinfoil/gemma4-31b", "name": "Gemma 4 31B", "effort": "low"}, +] + +#: Short-alias → canonical wire-ID map. Legacy 2025-era wire IDs are +#: included (forward-compatible: the catalog rotated underneath them +#: and they now 404 with ERR_MODEL_UNAVAILABLE). Mirrors the compat +#: map in benoitpetit/duckduckgo-chat-cli. Keys are matched +#: CASE-INSENSITIVELY (_resolve_model lowercases the input first), so +#: they are written lowercase — including the historical mixed-case +#: wire IDs (meta-llama/Llama-3.3-70B-Instruct-Turbo et al). +_MODEL_ALIASES: dict[str, str] = { + # legacy wire IDs (2025 era) + "gpt-4o-mini": "gpt-5.6-luna", + "o3-mini": "gpt-5.4-mini", + "o4mini": "gpt-5.4-mini", + "claude-3-haiku": "claude-haiku-4-5", + "claude-3-haiku-20240307": "claude-haiku-4-5", + "mistralai/mistral-small-24b-instruct-2501": "mistral-small-2603", + "meta-llama/llama-3.3-70b-instruct-turbo": "tinfoil/gpt-oss-120b", + # short aliases (stable across rotations) + "llama": "tinfoil/gpt-oss-120b", + "mixtral": "mistral-small-2603", + "gpt": "gpt-6-luna", + "claude": "claude-haiku-4-5", + "mistral": "mistral-small-2603", + "gemma": "tinfoil/gemma4-31b", + "oss": "tinfoil/gpt-oss-120b", + "nano": "gpt-5.4-nano", + "mini": "gpt-5.4-mini", +} + +#: Static catalog from agentkthx/data/model_seed.json — the 8 +#: client-verified DDG chat models with context metadata. The live +#: catalog is available via ``fetch_capabilities()`` (the +#: /duckchat/v1/capabilities endpoint); the seed is the offline +#: fallback for `agentkthx models`. +DUCKDUCKGO_MODELS: dict[str, dict] = load_seed_catalog("duckduckgo") + +#: Default model when none specified. gpt-6-luna is DuckDuckGo's own +#: default (the natural zero-config pick); override via +#: DUCKDUCKGO_DEFAULT_MODEL. +DUCKDUCKGO_DEFAULT_MODEL_STR = DUCKDUCKGO_DEFAULT_MODEL or "gpt-6-luna" + + +def _resolve_model(model: str) -> str: + """Alias-tolerant model resolution → canonical wire ID. + + Resolution order: exact wire-ID match (case-insensitive) → + legacy/short alias → pass through unchanged (the caller surfaces + the upstream ERR_MODEL_UNAVAILABLE with the catalog-drift hint). + """ + key = (model or "").strip().lower() + for entry in _WIRE_MODELS: + if entry["id"] == key: + return entry["id"] + return _MODEL_ALIASES.get(key, model) + + +def _estimate_tokens(text: str) -> int: + """~4 chars/token estimate (the DDG protocol reports no usage).""" + return max(1, len(text or "") // 4) + + +def _rand_id() -> str: + """32-hex random ID (messageId / conversationId / journey shape).""" + return "%032x" % random.randrange(16**32) + + +class _ChallengeError(Exception): + """SSE ERR_CHALLENGE — retryable by re-solving a fresh proof.""" + + +class _ConversationLimitError(Exception): + """ERR_CONVERSATION_LIMIT / [LIMIT_CONVERSATION] — conv is full.""" + + +class _ModelUnavailableError(Exception): + """HTTP 404 ERR_MODEL_UNAVAILABLE — the catalog rotated again.""" + + +def _classify_sse_payload(payload: bytes) -> dict | None: + """One ``data:`` payload → event dict (None = ignorable noise). + + Handles the CURRENT role-based grammar, the legacy action-based + grammar, and the bracketed markers. Kept module-level so tests + can exercise it without a backend instance. + """ + if payload == b"[DONE]": + return {"kind": "done"} + if payload == b"[PING]": + return {"kind": "ping"} + if payload == b"[LIMIT_CONVERSATION]": + return {"kind": "limit"} + if payload.startswith(b"[CHAT_TITLE:") and payload.endswith(b"]"): + return {"kind": "title", "text": payload[12:-1].decode("utf-8", "replace").strip()} + try: + data = json.loads(payload.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + return None + if not isinstance(data, dict): + return None + + role = data.get("role") + action = data.get("action") + + if role == "assistant": + if data.get("message"): + return {"kind": "text", "text": data["message"]} + return None # status/state-only assistant frame + if role == "source": + src = data.get("source") or {} + if src.get("url"): + return {"kind": "source", "url": src.get("url", ""), "title": src.get("title", "")} + return None + if role == "tool-invocation": + return {"kind": "tool", "tool": data.get("toolName", ""), "state": data.get("state", "")} + if role == "ui-component": + return {"kind": "ui", "name": data.get("name", "")} + if role == "error" or action == "error": + return {"kind": "error", "type": data.get("type", ""), "status": data.get("status")} + if action == "chunk": # legacy 2025 grammar — kept as fallback + if data.get("message"): + return {"kind": "text", "text": data["message"]} + return None + return None + + +class DuckDuckGoBackend(BaseBackend): + """duck.ai backend — challenge-proof protocol, always-SSE. + + What this class owns: + - the x-vqd-hash-1 proof lifecycle (fetch challenge → Node + solve → build header → per-turn rotation via the response + header) + - the durableStream RSA keypair (per-instance = per-conversation) + - the current chat body shape (canUseTools / reasoningEffort / + durableStream) + - role-based SSE parsing (with legacy-action fallback) + - buffered ``generate()`` AND incremental ``generate_stream()`` + - the seed-backed offline catalog + live ``fetch_capabilities()`` + + What it deliberately does NOT do: + - native tools (canUseTools=false shipped until the wire + surface stabilizes — ReAct remains the tool path) + - sampling params (dropped — upstream defaults apply) + """ + + #: Catalog cache TTL for list_models() (house: 1h). + _MODEL_CACHE_TTL_SECONDS = 3600 + #: model_cache namespace (unchanged — same key as pre-rotation). + MODEL_CACHE_KEY = "duckduckgo" + + #: Cloud-hosted — BaseBackend deliberately defaults direct + #: subclasses to False (R06.57 MAINT-05) so unknown backends are + #: treated as local; DDG is remote, so override to True or the + #: CLI misconfigures streaming defaults / models-table columns. + is_cloud: bool = True + + def __init__( + self, + config=None, + base_url: str | None = None, + api_mode: object | None = None, + user_agent: str | None = None, + **kwargs, + ): + super().__init__(config=config, base_url=base_url, api_mode=api_mode) + + #: Browser UA (DDG 403s non-browser UAs). The challenge + #: solver MUST see the same UA (it is injected via + #: DDG_SOLVE_UA at solve time — a UA mismatch between the + #: solved probes and the request headers invalidates the + #: proof). Precedence: kwarg > DUCKDUCKGO_USER_AGENT > Chrome 136. + self._user_agent = user_agent or DUCKDUCKGO_USER_AGENT or _DEFAULT_UA + + #: Proof state — the NEXT challenge (base64), carried on the + #: previous /chat response header. When None, _fetch_challenge + #: bootstraps from /status. + self._next_challenge_b64: str | None = None + + #: durableStream state — minted lazily on first chat; one + #: keypair + conversationId per backend instance. + self._durable: dict | None = None + + #: Frontend metadata (serp version + bundle names), scraped + #: best-effort from GET / and cached 10 minutes. + self._fe_meta: dict = {} + self._fe_meta_ts: float = 0.0 + + #: node availability (checked once, cached; None = unknown). + self._node_ok: bool | None = None + + self._model_cache: list[dict] | None = None + self._model_cache_ts: float = 0.0 + + # ───────────────────────────────────────────────────────────────────── + # Provider identity + # ───────────────────────────────────────────────────────────────────── + + @property + def backend_type(self) -> BackendType: + """DuckDuckGo's dedicated BackendType enum value.""" + return BackendType.DUCKDUCKGO + + @property + def base_url(self) -> str: + """API root — https://duck.ai (protocol lives under /duckchat/v1).""" + return self._base_url or DUCKDUCKGO_BASE_URL + + # ───────────────────────────────────────────────────────────────────── + # Node helpers — the challenge solver + keygen subprocesses + # ───────────────────────────────────────────────────────────────────── + + def _node_available(self) -> bool: + """node on PATH (checked once per instance, cached).""" + if self._node_ok is None: + self._node_ok = shutil.which("node") is not None + return self._node_ok + + def _require_node(self, what: str) -> None: + """Fail fast with the remediation when node is missing.""" + if not self._node_available(): + raise RuntimeError( + f"DuckDuckGo backend needs Node.js for {what} (the " + "x-vqd-hash-1 challenge solver / durableStream keygen " + "ship as bundled .js helpers). Install node >= 18 and " + "make sure it is on PATH." + ) + + def _solve_challenge(self, challenge_b64: str) -> dict: + """Run the bundled Node solver → raw solution dict. + + The solver emulates a clean browser (navigator, DOM probes, + sandbox iframes, native-code fingerprints) and executes the + obfuscated challenge JS against it. DDG_SOLVE_UA is set to + THIS backend's UA — a mismatch invalidates the proof. + """ + self._require_node("challenge solving") + env = dict(os.environ) + env["DDG_SOLVE_UA"] = self._user_agent + proc = subprocess.run( + ["node", _SOLVER_JS], + input=challenge_b64.encode("utf-8"), + capture_output=True, + timeout=45, + env=env, + ) + if proc.returncode != 0: + raise RuntimeError( + "DuckDuckGo challenge solver failed (rc=" + f"{proc.returncode}): " + f"{proc.stderr.decode('utf-8', 'replace').strip()[:400]}" + ) + return json.loads(proc.stdout.decode("utf-8")) + + def _mint_durable(self) -> dict: + """Mint the durableStream RSA-2048 keypair (Node crypto). + + Python stdlib has no RSA — the bundled ddg_durable.js does + generateKeyPairSync('rsa', 2048) and emits the exact JWK + shape the frontend sends. One pair per backend instance + (= per conversation); messageId is fresh per request. + """ + self._require_node("durableStream keygen") + proc = subprocess.run( + ["node", _DURABLE_JS], + capture_output=True, + timeout=20, + ) + if proc.returncode != 0: + raise RuntimeError( + "DuckDuckGo durableStream keygen failed (rc=" + f"{proc.returncode}): " + f"{proc.stderr.decode('utf-8', 'replace').strip()[:400]}" + ) + keys = json.loads(proc.stdout.decode("utf-8")) + return { + "conversation_id": _rand_id(), + "public_key": keys["publicJwk"], + } + + # ───────────────────────────────────────────────────────────────────── + # Proof lifecycle — fetch challenge → solve → build header + # ───────────────────────────────────────────────────────────────────── + + def _fetch_challenge(self) -> str: + """Get a challenge blob: response-carried, else /status. + + The /chat response header ``x-vqd-hash-1`` carries the next + challenge (the protocol's intended rotation). When that is + exhausted (fresh conversation, prior error), bootstrap via + GET /status with ``x-vqd-accept: 1``. + """ + if self._next_challenge_b64: + chal = self._next_challenge_b64 + self._next_challenge_b64 = None + return chal + + req = urllib.request.Request( + f"{self.base_url}/duckchat/v1/status", + headers=self._build_status_headers(), + method="GET", + ) + with urllib.request.urlopen(req, timeout=30) as resp: + challenge = resp.headers.get("x-vqd-hash-1") + resp.read() # drain (body is small JSON) + if not challenge: + raise RuntimeError( + f"DuckDuckGo /status returned no x-vqd-hash-1 " + f"challenge (status {resp.status}) — the protocol " + "rotated again; see " + "docs/api/DUCKDUCKGO_API_TECHNICAL_REFERENCE.md" + ) + return challenge + + def _synth_stack(self) -> str: + """A plausible duck.ai bundle stack (shape from a real capture).""" + entry = self._fe_meta.get("entry_js", _ENTRY_JS_FALLBACK) + vendors = self._fe_meta.get("vendors_js", _VENDORS_JS_FALLBACK) + r = lambda a, b: random.randint(a, b) # noqa: E731 + return ( + f"l@https://duck.ai/dist/duckai-dist/{entry}:2:{r(1900000, 2050000)}\n" + f"async*30012/h/y<@https://duck.ai/dist/duckai-dist/{entry}:2:{r(1700000, 1800000)}\n" + f"onSuccess@https://duck.ai/dist/duckai-dist/{entry}:2:{r(1600000, 1700000)}\n" + f"32985/ne/pE<@https://duck.ai/dist/duckai-dist/{vendors}:8:{r(1600000, 1650000)}\n" + f"async*32985/ne/<@https://duck.ai/dist/duckai-dist/{vendors}:8:{r(1630000, 1640000)}\n" + f"... (25 more frames omitted)" + ) + + def _build_proof_header(self, raw: dict, elapsed_ms: int) -> str: + """Raw solver output → the X-Vqd-Hash-1 request header value. + + Mirrors the duck.ai page bundle's post-processing: + client_hashes are base64(SHA256(String(value))) of the raw + probe results; server_hashes pass through UNTOUCHED (they are + opaque server nonces); meta gains origin/stack/duration. + """ + solution = { + "server_hashes": raw.get("server_hashes", []), + "client_hashes": [ + base64.b64encode(hashlib.sha256(str(v).encode("utf-8")).digest()).decode() + for v in raw.get("client_hashes", []) + ], + "signals": raw.get("signals", {}), + "meta": { + "v": raw.get("meta", {}).get("v", "4"), + "challenge_id": raw.get("meta", {}).get("challenge_id", ""), + "timestamp": raw.get("meta", {}).get("timestamp", ""), + "debug": raw.get("meta", {}).get("debug", ""), + "origin": "https://duck.ai", + "stack": self._synth_stack(), + "duration": str(max(1, int(elapsed_ms))), + }, + } + return base64.b64encode(json.dumps(solution, separators=(",", ":")).encode("utf-8")).decode( + "utf-8" + ) + + def _acquire_proof(self) -> str: + """Fresh solved X-Vqd-Hash-1 (fetch → solve → build).""" + t0 = time.monotonic() + raw = self._solve_challenge(self._fetch_challenge()) + elapsed_ms = (time.monotonic() - t0) * 1000 + return self._build_proof_header(raw, elapsed_ms) + + # ───────────────────────────────────────────────────────────────────── + # Headers (anti-bot contract: browser UA + Referer + SSE Accept) + # ───────────────────────────────────────────────────────────────────── + + def _build_status_headers(self) -> dict: + """Headers for the /status challenge bootstrap. + + ``x-vqd-accept: 1`` is the opt-in that makes /status issue a + challenge instead of the bare status JSON. ``x-ddg-journey-id`` + is a per-request 32-hex session marker (shape from the live + frontend capture). + """ + return { + "User-Agent": self._user_agent, + "Accept": "*/*", + "Accept-Language": "en-US,en;q=0.9", + "Referer": f"{self.base_url}/", + "Cache-Control": "no-store", + "x-vqd-accept": "1", + "x-ddg-journey-id": _rand_id(), + "Sec-GPC": "1", + "Connection": "keep-alive", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-origin", + } + + def _build_chat_headers(self, proof: str) -> dict: + """Headers for POST /chat — proof + browser contract. + + The proof header carries the SOLVED challenge; x-fe-version / + x-fe-signals mirror the frontend telemetry (version from the + scraped serp build; signals = base64 JSON event log with the + onboarding/startNewChat_free events the real client sends). + """ + return { + "User-Agent": self._user_agent, + "Accept": "text/event-stream", + "Accept-Language": "en-US,en;q=0.9", + "Referer": f"{self.base_url}/", + "Content-Type": "application/json", + "Origin": self.base_url, + "x-fe-version": self._fe_meta.get("fe_version", _FE_VERSION_FALLBACK), + "x-fe-signals": self._fe_signals(), + "X-Vqd-Hash-1": proof, + "x-ddg-journey-id": _rand_id(), + "Cache-Control": "no-store", + "Sec-GPC": "1", + "Connection": "keep-alive", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-origin", + } + + def _fe_signals(self) -> str: + """Synthesized frontend telemetry (shape from a live capture). + + base64(JSON{start, events[action + startNewChat_free], end}) + with plausible millisecond deltas — the real client's + engagement proof. Accepted by the server alongside the solved + challenge (verified end-to-end pre-integration). + """ + now = int(time.time() * 1000) + start = now - random.randint(5000, 60000) + payload = { + "start": start, + "events": [ + {"name": "action", "delta": random.randint(300, 3000), "trusted": True}, + {"name": "startNewChat_free", "delta": random.randint(3100, 6000)}, + ], + "end": start + random.randint(6100, 9000), + } + return base64.b64encode(json.dumps(payload, separators=(",", ":")).encode("utf-8")).decode( + "utf-8" + ) + + def _scrape_fe_meta(self, max_age_s: float = 600.0) -> dict: + """Pull serp version + bundle names from GET / (cached, best-effort). + + The x-fe-version string rotates with frontend deployments; a + stale one is currently tolerated by the server but scraping + keeps us honest (and refreshes the stack-synthesis bundle + names). Failures fall back to the captured constants silently. + """ + if self._fe_meta and (time.monotonic() - self._fe_meta_ts) < max_age_s: + return self._fe_meta + try: + req = urllib.request.Request( + f"{self.base_url}/", + headers={"User-Agent": self._user_agent, "Accept": "text/html,*/*"}, + method="GET", + ) + with urllib.request.urlopen(req, timeout=15) as resp: + html = resp.read(2_000_000) + for key, pattern in ( + ("fe_version", rb"serp_\d{8}_\d{6}_[A-Za-z0-9-]+"), + ("entry_js", rb"entry\.duckai\.[0-9a-f]+\.js"), + ("vendors_js", rb"entry\.vendors\.[0-9a-f]+\.js"), + ): + m = re.search(pattern, html) + if m: + self._fe_meta[key] = m.group(0).decode("ascii") + self._fe_meta_ts = time.monotonic() + except (urllib.error.URLError, OSError, ValueError): + pass # best-effort — fallbacks apply + return self._fe_meta + + # ───────────────────────────────────────────────────────────────────── + # Body builder — the current chat wire shape + # ───────────────────────────────────────────────────────────────────── + + def _build_chat_body(self, wire_model: str, convo: list[dict]) -> bytes: + """POST /chat body: model + messages + protocol fields. + + ``canUseTools`` false + no ``metadata`` (native tools are a + rotating undocumented surface — ReAct stays the tool path). + ``canUseApproxLocation`` null (frontend sends null when no + location consent). ``reasoningEffort`` per-model like the + frontend ("low" for the Tinfoil open-weight pair). The + ``durableStream`` publicKey is the per-conversation RSA JWK. + """ + if self._durable is None: + self._durable = self._mint_durable() + effort = next( + (m["effort"] for m in _WIRE_MODELS if m["id"] == wire_model), + "none", + ) + body = { + "model": wire_model, + "messages": convo, + "canUseTools": False, + "canUseApproxLocation": None, + "reasoningEffort": effort, + "durableStream": { + "messageId": _rand_id(), + "conversationId": self._durable["conversation_id"], + "publicKey": self._durable["public_key"], + }, + } + return json.dumps(body, separators=(",", ":")).encode("utf-8") + + # ───────────────────────────────────────────────────────────────────── + # Message shaping (the protocol strips system + tool fields) + # ───────────────────────────────────────────────────────────────────── + + @staticmethod + def _coerce_content(content) -> str: + """Best-effort content → plain text (str | list-of-parts).""" + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [] + for part in content: + if isinstance(part, dict) and part.get("text"): + parts.append(str(part["text"])) + elif isinstance(part, str): + parts.append(part) + return "\n".join(parts) + return "" if content is None else str(content) + + def _collapse_system_into_user(self, messages: list[dict]) -> list[dict]: + """DDG strips the system role — fold it into the first user turn. + + Preserves order for user/assistant turns; multiple system + messages concatenate with blank lines. Returns [] when there + is no user turn to host the system content (DDG requires a + leading user message). + """ + convo: list[dict] = [] + pending_system: list[str] = [] + + def _content(m: dict) -> str: + return self._coerce_content(m.get("content", "")) + + for msg in messages: + role = (msg.get("role") or "user").lower() + if role == "system": + if _content(msg): + pending_system.append(_content(msg)) + continue + if role not in ("user", "assistant"): + # tool / function results — DDG only understands + # user/assistant; fold foreign roles into user (the + # ReAct loop's tool outputs ride as user turns). + role = "user" + content = _content(msg) + if role == "user" and pending_system: + merged = "\n\n".join(pending_system) + content = f"{merged}\n\n{content}" if content else merged + pending_system = [] + if content or role == "user": + convo.append({"role": role, "content": content}) + + if pending_system: # trailing system with no user host — drop + pass + if convo and convo[0].get("role") != "user": + convo.insert(0, {"role": "user", "content": "Hello"}) + return convo + + # ───────────────────────────────────────────────────────────────────── + # Generation — buffered (generate) + incremental (generate_stream) + # ───────────────────────────────────────────────────────────────────── + + def generate( + self, + model: str, + messages: list[dict], + tools: list | None = None, + temperature: float | None = None, + max_tokens: int | None = None, + think: bool | None = None, + **kwargs, + ) -> dict: + """Generate a response via POST /duckchat/v1/chat (SSE-buffered). + + DDG ALWAYS streams — even for this non-streaming entry point, + the SSE stream is consumed and buffered. ``tools`` / + ``temperature`` / ``max_tokens`` / ``think`` are accepted for + interface parity and SILENTLY DROPPED (upstream sampling + defaults apply; a debug-mode notice fires for temperature). + + Proof-expiry recovery: a 401 or an SSE ``ERR_CHALLENGE`` error + triggers ONE automatic re-solve (fresh challenge + proof) and + a single retry of the /chat call. + + Returns the house response shape (content / tool_calls / + usage / finish_reason). ``usage`` is an ~4-chars/token + ESTIMATE (the DDG protocol reports no usage) flagged with + ``"estimated": True``. + """ + wire_model = _resolve_model(model) + convo = self._collapse_system_into_user(messages) + if not convo: + raise ValueError("DuckDuckGo generate(): no messages after system collapse") + if temperature is not None and os.environ.get("AGENTKTHX_DEBUG"): + print( + " [duckduckgo] 'temperature' ignored — DDG uses upstream " + "defaults (no sampling params in the protocol)" + ) + + self._scrape_fe_meta() + body = self._build_chat_body(wire_model, convo) + + attempt = 0 + while True: + attempt += 1 + proof = self._acquire_proof() + req = urllib.request.Request( + f"{self.base_url}/duckchat/v1/chat", + data=body, + headers=self._build_chat_headers(proof), + method="POST", + ) + try: + content, _sources, next_chal = self._consume_sse(req) + except urllib.error.HTTPError as e: + if e.code == 401 and attempt == 1: + if os.environ.get("AGENTKTHX_DEBUG"): + print(" [duckduckgo] 401 proof rejected — re-solving") + self._next_challenge_b64 = None + continue + if e.code == 404: + detail = e.read()[:200].decode("utf-8", "replace") + raise RuntimeError( + f"DuckDuckGo model unavailable ({wire_model}): " + f"{detail} — the catalog rotated; run " + "`agentkthx models --backend duckduckgo` after a " + "seed update (see " + "docs/api/DUCKDUCKGO_API_TECHNICAL_REFERENCE.md)" + ) from e + if e.code == 403: + raise RuntimeError( + "DuckDuckGo 403 Forbidden — the anti-bot layer " + "rejected the request. Check DUCKDUCKGO_USER_AGENT " + "(must look like a real browser) — the Referer and " + "Accept headers are fixed by the backend." + ) from e + if e.code == 429: + raise RuntimeError( + "DuckDuckGo rate limit (429) — per-IP throttling, " + "undocumented limits. Back off 5-10s between " + "requests; if this persists, reduce request " + "frequency or route through a different network." + ) from e + if e.code >= 500: + raise RuntimeError( + f"DuckDuckGo upstream error ({e.code}) — an upstream " + "provider may be down. Try a different DDG model." + ) from e + raise + except _ConversationLimitError: + raise RuntimeError( + "DuckDuckGo conversation limit reached (~20 turns). " + "Start a fresh conversation: clear agent memory " + "(/clear in chat) or restart the session." + ) + except _ChallengeError: + if attempt == 1: + if os.environ.get("AGENTKTHX_DEBUG"): + print(" [duckduckgo] ERR_CHALLENGE — re-solving proof") + self._next_challenge_b64 = None + continue + raise RuntimeError( + "DuckDuckGo ERR_CHALLENGE persisted after proof " + "re-solve — the challenge format rotated; see " + "docs/api/DUCKDUCKGO_API_TECHNICAL_REFERENCE.md" + ) + + self._next_challenge_b64 = next_chal + prompt_text = "".join(m.get("content", "") for m in convo) + return { + "content": content, + "tool_calls": [], # ReAct only — canUseTools ships false + "usage": { + "prompt_tokens": _estimate_tokens(prompt_text), + "completion_tokens": _estimate_tokens(content), + "total_tokens": _estimate_tokens(prompt_text) + _estimate_tokens(content), + "estimated": True, + }, + "finish_reason": "stop", + "model": wire_model, + } + + def _open_chat_stream(self, wire_model: str, convo: list[dict]) -> urllib.request.Request: + """Build the POST /chat request (proof acquired, meta scraped).""" + self._scrape_fe_meta() + body = self._build_chat_body(wire_model, convo) + return urllib.request.Request( + f"{self.base_url}/duckchat/v1/chat", + data=body, + headers=self._build_chat_headers(self._acquire_proof()), + method="POST", + ) + + def _consume_sse(self, req: urllib.request.Request) -> tuple[str, list[dict], str | None]: + """POST /chat and consume the SSE stream to completion. + + Returns ``(text, sources, next_challenge)``. Raises HTTPError + for non-200 statuses, ``_ChallengeError`` for ERR_CHALLENGE + (retryable via re-solve), ``_ConversationLimitError`` for + ERR_CONVERSATION_LIMIT / [LIMIT_CONVERSATION], and RuntimeError + for other SSE error frames. + + Event grammar (role-based, current; action-based kept as + legacy fallback — see ``_classify_sse_payload``). Streams may + also simply END without [DONE] — an exhausted stream with + content is a success. + """ + parts: list[str] = [] + sources: list[dict] = [] + with urllib.request.urlopen(req, timeout=180) as resp: + next_challenge = resp.headers.get("x-vqd-hash-1") + buffer = b"" + while True: + chunk = resp.read(8192) + if not chunk: + break + buffer += chunk + while b"\n" in buffer: + raw_line, buffer = buffer.split(b"\n", 1) + line = raw_line.strip() + if not line.startswith(b"data:"): + continue + payload = line[5:].strip() + if not payload: + continue + ev = _classify_sse_payload(payload) + if ev is None: + continue + kind = ev["kind"] + if kind == "text": + parts.append(ev["text"]) + elif kind == "source": + sources.append({"url": ev["url"], "title": ev["title"]}) + elif kind == "done": + return "".join(parts), sources, next_challenge + elif kind == "limit": + raise _ConversationLimitError() + elif kind == "error": + if ev["type"] == "ERR_CHALLENGE": + raise _ChallengeError() + if ev["type"] == "ERR_CONVERSATION_LIMIT": + raise _ConversationLimitError() + raise RuntimeError( + f"DuckDuckGo SSE error: {ev['type'] or 'unknown'} " + f"(status={ev['status']})" + ) + # ping / title / tool / ui — no text to buffer + return "".join(parts), sources, next_challenge + + def generate_stream( + self, + model: str, + messages: list[dict], + tools: list | None = None, + temperature: float | None = None, + max_tokens: int | None = None, + **kwargs, + ): + """TRUE incremental generation (FEAT-10 — closed). + + Yields text deltas as they arrive off the SSE stream. The + proof lifecycle is identical to ``generate()`` (acquire → + POST → next-challenge rotation); challenge errors mid-stream + re-solve ONCE and replay the request (already-yielded text + is NOT re-yielded — the retry restarts the turn). Error + taxonomy mirrors ``generate()`` exactly (404 catalog rotation, + 403 anti-bot, 429 rate limit, 5xx upstream, conversation + limit) — the streaming path is FEAT-10's whole point, so it + must surface the same remediation messages the buffered path + does. + """ + wire_model = _resolve_model(model) + convo = self._collapse_system_into_user(messages) + if not convo: + raise ValueError("DuckDuckGo generate_stream(): no messages after system collapse") + + attempt = 0 + while True: + attempt += 1 + req = self._open_chat_stream(wire_model, convo) + retryable = False + try: + with urllib.request.urlopen(req, timeout=180) as resp: + self._next_challenge_b64 = resp.headers.get("x-vqd-hash-1") + buffer = b"" + while True: + chunk = resp.read(4096) + if not chunk: + return + buffer += chunk + while b"\n" in buffer: + raw_line, buffer = buffer.split(b"\n", 1) + line = raw_line.strip() + if not line.startswith(b"data:"): + continue + payload = line[5:].strip() + if not payload: + continue + ev = _classify_sse_payload(payload) + if ev is None: + continue + kind = ev["kind"] + if kind == "text": + yield ev["text"] + elif kind == "done": + return + elif kind == "limit": + raise _ConversationLimitError() + elif kind == "error": + if ev["type"] == "ERR_CHALLENGE": + raise _ChallengeError() + if ev["type"] == "ERR_CONVERSATION_LIMIT": + raise _ConversationLimitError() + raise RuntimeError( + f"DuckDuckGo SSE error: " + f"{ev['type'] or 'unknown'} " + f"(status={ev['status']})" + ) + except urllib.error.HTTPError as e: + if e.code == 401 and attempt == 1: + retryable = True + elif e.code == 404: + detail = e.read()[:200].decode("utf-8", "replace") + raise RuntimeError( + f"DuckDuckGo model unavailable ({wire_model}): " + f"{detail} — the catalog rotated (see " + "docs/api/DUCKDUCKGO_API_TECHNICAL_REFERENCE.md)" + ) from e + elif e.code == 403: + raise RuntimeError( + "DuckDuckGo 403 Forbidden — the anti-bot layer " + "rejected the request. Check DUCKDUCKGO_USER_AGENT " + "(must look like a real browser) — the Referer and " + "Accept headers are fixed by the backend." + ) from e + elif e.code == 429: + raise RuntimeError( + "DuckDuckGo rate limit (429) — per-IP throttling, " + "undocumented limits. Back off 5-10s between " + "requests; if this persists, reduce request " + "frequency or route through a different network." + ) from e + elif e.code >= 500: + raise RuntimeError( + f"DuckDuckGo upstream error ({e.code}) — an upstream " + "provider may be down. Try a different DDG model." + ) from e + else: + raise + except _ChallengeError: + if attempt == 1: + retryable = True + else: + raise RuntimeError( + "DuckDuckGo ERR_CHALLENGE persisted after proof " + "re-solve — the challenge format rotated; see " + "docs/api/DUCKDUCKGO_API_TECHNICAL_REFERENCE.md" + ) + except _ConversationLimitError: + raise RuntimeError( + "DuckDuckGo conversation limit reached (~20 turns). " + "Start a fresh conversation: clear agent memory " + "(/clear in chat) or restart the session." + ) + if retryable: + self._next_challenge_b64 = None + continue + + # ───────────────────────────────────────────────────────────────────── + # Catalog — seed-backed offline list + live capabilities helper + # ───────────────────────────────────────────────────────────────────── + + def list_models(self) -> list[dict]: + """Serve the seed catalog (cached 1h, offline-safe). + + The seed IS the 8-model wire catalog (client-verified IDs). + The LIVE catalog is available via ``fetch_capabilities()`` + (GET /duckchat/v1/capabilities) — kept out of this path so + `agentkthx models --backend duckduckgo` renders fully + offline (the house contract: no network in model listing). + Routed through the persistent model cache so + ``--cache-status`` stays uniform. + """ + now = time.time() + if ( + self._model_cache is not None + and (now - self._model_cache_ts) < self._MODEL_CACHE_TTL_SECONDS + ): + return list(self._model_cache) + + cached = model_cache.get_cached_models(self.MODEL_CACHE_KEY) + if cached is not None: + self._model_cache = cached + self._model_cache_ts = now + return list(cached) + + models = self._catalog_fallback_list() + self._model_cache = model_cache.store_models(self.MODEL_CACHE_KEY, models) + self._model_cache_ts = now + return list(self._model_cache) + + def _catalog_fallback_list(self) -> list[dict]: + """Shape the seed catalog into ``list_models()`` entries. + + Every DDG model is free (pricing 0.0/0.0) and chat-capable. + The ``think`` verdict is NO across the board — see + ``test_thinking_support``. + """ + models: list[dict] = [] + for name in sorted(DUCKDUCKGO_MODELS.keys()): + meta = DUCKDUCKGO_MODELS[name] + models.append( + { + "name": name, + "size": 0, + "details": { + "family": meta.get("family", "duckduckgo"), + "backend": "duckduckgo", + "context_length": meta.get("context_length", 128000), + "free_tier": True, + "is_chat_model": True, + "pricing": {"input": 0.0, "output": 0.0}, + }, + } + ) + return models + + def fetch_capabilities(self) -> dict | None: + """Live model catalog via GET /duckchat/v1/capabilities. + + The browser bootstrap sequence hits this endpoint before any + challenge is solved (unauthenticated model discovery). When a + deployment gates it behind the proof, this retries once with + a solved X-Vqd-Hash-1. Returns the parsed JSON or None (any + failure — this is a discovery helper, never a hard dep). + """ + headers = { + "User-Agent": self._user_agent, + "Accept": "*/*", + "Accept-Language": "en-US,en;q=0.9", + "Referer": f"{self.base_url}/", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-origin", + } + + def _get(extra: dict | None = None) -> bytes: + merged = dict(headers) + if extra: + merged.update(extra) + req = urllib.request.Request( + f"{self.base_url}/duckchat/v1/capabilities", + headers=merged, + method="GET", + ) + with urllib.request.urlopen(req, timeout=20) as resp: + return resp.read() + + try: + body = _get() + except urllib.error.HTTPError as e: + try: + e.read() + except Exception: # noqa: BLE001 — drain best-effort + pass + if e.code in (401, 403): + try: + body = _get({"X-Vqd-Hash-1": self._acquire_proof()}) + except Exception: # noqa: BLE001 — discovery helper + return None + else: + return None + except (urllib.error.URLError, OSError, ValueError): + return None + try: + return json.loads(body.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + return None + + def get_model_info(self, model: str) -> dict | None: + """Catalog lookup with alias resolution.""" + resolved = _resolve_model(model) + for m in self.list_models(): + if m.get("name") == resolved: + return m + return None + + def get_model_max_context(self, model: str, family: str | None = None) -> int: + """Context length from the seed catalog (alias-aware). + + Unknown models get the conservative 128K default — the DDG + lineup clusters at 128K-200K and an over-estimate risks + context-length 400s DDG can't recover from. + """ + info = self.get_model_info(model) + if info and info.get("details", {}).get("context_length"): + return int(info["details"]["context_length"]) + return 128000 + + def get_model_runtime_context(self, model: str) -> int: + """Runtime context — equals max context (CloudBackend contract). + + Cloud-style backends have no Ollama-style Modelfile ``num_ctx`` + to shrink the window at runtime; DDG serves the model's full + context. ``cmd_models`` calls this unconditionally on every + backend, so it must exist even though its result feeds no + column today. + """ + return self.get_model_max_context(model) + + # ───────────────────────────────────────────────────────────────────── + # Capability verdicts — both are protocol facts, not probes + # ───────────────────────────────────────────────────────────────────── + + def test_tool_support( + self, + model: str, + family: str | None = None, + force_test: bool = False, + ) -> ToolSupportLevel: + """REACT for every DDG model — we ship canUseTools=false. + + The wire now HAS a native tools surface (canUseTools + + metadata.toolChoice: WebSearch / GenerateImage / NewsSearch / + VideosSearch / LocalSearch / WeatherForecast), but it is + undocumented and rotates independently of the chat endpoint. + Until it stabilizes this backend sends canUseTools=false and + the agent loop uses the ReAct prompting path (the textual + Action/Action Input format). PROTOCOL DECISION, not a probe. + """ + return ToolSupportLevel.REACT + + def test_thinking_support( + self, + model: str, + family: str | None = None, + force_test: bool = False, + ) -> ThinkingSupport: + """NO for every DDG model — the privacy layer hides reasoning. + + ThinkingSupport answers "does the model emit reasoning_content + (chain of thought) into the response?". Even the reasoning + models (gpt-oss-120b, served with reasoningEffort "low") never + surface CoT through the DDG SSE shape — only ``message`` text + arrives. NO is the honest verdict for the whole catalog; + ``--think`` has nothing to display on this backend. + """ + return ThinkingSupport.NO + + # ───────────────────────────────────────────────────────────────────── + # Liveness + # ───────────────────────────────────────────────────────────────────── + + def is_running(self) -> bool: + """Always True — keyless backend, static catalog, no local server. + + House contract (see CloudBackend): ``is_running`` is a CHEAP + local check that gates ``cmd_models`` — cloud backends must + not touch the network here (ZAI/OpenRouter/etc. check their + API key; DDG has none to check). This must NOT check node + availability either (a missing node surfaces at generate() + time with its remediation message). Real liveness is proven + by generate()'s own /status → solve → /chat handshake, with + the 401/ERR_CHALLENGE re-solve and error taxonomy as the + failure surface. + """ + return True diff --git a/agentkthx/plugins/duckduckgo/plugin.json b/agentkthx/plugins/duckduckgo/plugin.json new file mode 100644 index 00000000..a7192d14 --- /dev/null +++ b/agentkthx/plugins/duckduckgo/plugin.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://raw.githubusercontent.com/VTSTech/AgentKthx/main/schemas/v0.2/plugin.schema.json", + "name": "duckduckgo", + "version": "0.2.0", + "description": "DuckDuckGo AI Chat (duck.ai) backend — keyless, anonymous, zero-cost frontier models (GPT-6 Luna, GPT-5.6 Luna, GPT-5.4 Nano/Mini, Claude Haiku 4.5, Mistral Small 4, Tinfoil-hosted gpt-oss-120b / Gemma 4 31B) via the NON-OpenAI /duckchat/v1 protocol — now CHALLENGE-BASED: the retired x-vqd-4 token was replaced by the x-vqd-hash-1 JS proof (solved by the bundled Node helper), per-turn challenge rotation, role-based SSE, durableStream RSA keypair, no /models endpoint, no sampling params, tools stripped — ReAct only. The only cloud backend that subclasses BaseBackend directly instead of CloudBackend", + "author": { + "name": "VTSTech", + "url": "https://www.vts-tech.org" + }, + "license": "MIT", + "extensions": { + "org.vts-tech.agentkthx": { + "display_name": "DuckDuckGo AI Chat Backend", + "type": "backend", + "entrypoint": "__init__", + "depends": [], + "optional_depends": [], + "config": { + "env_prefix": "DUCKDUCKGO", + "defaults": { + "DUCKDUCKGO_BASE_URL": "https://duck.ai", + "DUCKDUCKGO_USER_AGENT": "", + "DUCKDUCKGO_DEFAULT_MODEL": "gpt-6-luna" + } + }, + "provides": { + "backends": { + "duckduckgo": "duckduckgo.DuckDuckGoBackend", + "ddg": "duckduckgo.DuckDuckGoBackend" + }, + "cli_commands": [], + "cli_flags": { + "--backend": [ + "duckduckgo", + "ddg" + ] + } + }, + "compatibility": { + "agentkthx": ">=0.7.29" + } + } + } +} diff --git a/audit/audit.md b/audit/audit.md index baa61a16..19209a8d 100644 --- a/audit/audit.md +++ b/audit/audit.md @@ -60,6 +60,8 @@ Process note: this pass re-verified all 14 carried-forward OPEN findings against | MAINT-31 | Low | Maintainability | OPEN | Remaining CloudBackend dedup backlog — list_models ×6 template, test_tool_support ×7 name-pattern shape, _fetch_live_models transport trio, legacy request loops (mistral/pollinations/zai/orcarouter) never migrated to the shared MAINT-28 machinery | | FEAT-09 | Low | New Features | OPEN | SiliconFlow enable_thinking + repetition_penalty request-field passthrough — the API supports both fields; the R07.29 scaffold drops them (cloud-drop-repeat_* house convention), and mapping think→enable_thinking needs BOTH _build_stream_body and _tweak_request_body to avoid the MAINT-22 streaming-drift class | | TEST-12 | Low | Testing | OPEN | No live-shape contract test for the SiliconFlow scaffold — the seed catalog/envelope/error shapes are mock-verified only; the R07.27 Cloudflare lesson says the live /v1/models shape diverges from the docs until the maintainer's first smoke run — 2026-10-09: the smoke attempt verified auth/catalog/one `--think` generation before the account balance drained (tool steps → 402); the 402 balance shape is now test-pinned, the tools/429 live shapes still await a topped-up key | +| FEAT-10 | Low | New Features | CLOSED | DuckDuckGo generate_stream() not implemented — CLOSED by the 2026-10-09 protocol-rotation rewrite (plugin v0.2.0): the challenge-based protocol made streaming the natural shape, so `generate_stream()` is a TRUE incremental generator (yields role-based text deltas off the SSE stream, 4096-byte reads, per-turn challenge rotation, single 401/ERR_CHALLENGE re-solve with turn replay, full error-taxonomy parity with the buffered path); 15 stream tests in `tests/test_duckduckgo_backend.py` | +| TEST-13 | Low | Testing | OPEN | Live-shape verification for the DuckDuckGo backend — PARTIALLY CLOSED by the 2026-10-09 protocol work: the challenge handshake (x-vqd-accept → solve → X-Vqd-Hash-1) was verified LIVE from an unrestricted network via the ddg-challenge-solver package (`--status-only` solves repeatedly at 54–987ms; a 404 ERR_MODEL_UNAVAILABLE JSON body proved auth + request shape parse correctly, isolating the model-ID rotation), and the 8-model seed catalog was re-verified against the live dropdown + benoitpetit/duckduckgo-chat-cli models.go; STILL OPEN: a live 200 SSE turn through the plugin itself (`scripts/probe_duckduckgo.py --live` automates it end-to-end via the backend) | --- ## R07.18 New Findings Two findings, both from the R07.18 flag-parsing surface (`_parse_token_size` + the `SharedConfig` coalescing layer). No closures this pass — all 34 carried-forward OPEN findings re-verified in current code. @@ -101,6 +103,17 @@ Two findings from the SiliconFlow scaffold pass (2026-10-09). No closures this p --- +## R07.30 New Findings +Two findings from the DuckDuckGo scaffold pass (2026-10-09). No closures this pass (feature release). The scaffold is the FIRST backend to subclass BaseBackend directly instead of CloudBackend — the /duckchat/v1 protocol (x-vqd-4 header-token handshake, own SSE shape, no /models endpoint, no sampling params, tools stripped) shares nothing with the OpenAI-compat shared transport, so zero lines of cloud_base.py/openai_compat.py were touched. Live-probe pivot: the maintainer sandbox could not reach duckduckgo.com (whole-IP-range egress timeout), so the model catalog was verified against the actively-maintained mumu-lhl/duckduckgo-ai-chat v3.3.0 client source (the same source the API reference cites) instead of a live /chat probe — the ID set matches the Oct 2026 duck.ai help-page lineup (GPT-4o mini, Claude Haiku behind the claude-3-haiku wire ID, Llama 3.3 70B, Mistral Small 3 24B, o3-mini). Register: 136 findings / 19 OPEN. + +**Protocol rotation rewrite (same day, plugin v0.1.0 → v0.2.0):** the scaffold never went live — DDG had RETIRED the x-vqd-4 token wholesale. Reverse-engineering (curl matrix + HAR capture + 5 decoded obfuscator.io challenges, cross-validated against benoitpetit/duckduckgo-chat-cli) established the replacement: GET /status with `x-vqd-accept: 1` returns a base64 JS CHALLENGE in the `x-vqd-hash-1` response header; the challenge is executed against a clean-browser environment by a bundled Node solver (`ddg_vqd.js`, subprocess, DDG_SOLVE_UA kept in lockstep with the request UA); post-processing mirrors the page bundle (client_hashes → base64(SHA256(String(value))), meta gains origin/stack/duration) → the `X-Vqd-Hash-1` REQUEST header; the /chat response header carries the NEXT challenge (per-turn rotation). Chat body gained `canUseTools`/`canUseApproxLocation`/`reasoningEffort`/`durableStream` (client-minted RSA-2048 JWK via bundled `ddg_durable.js`); SSE flipped to ROLE-BASED events (assistant/source/tool-invocation/ui-component, legacy action grammar kept as fallback). The model catalog ALSO rotated (2025 IDs → 404 ERR_MODEL_UNAVAILABLE): reseeded with the 8 live wire IDs (gpt-6-luna, gpt-5.6-luna, gpt-5.4-nano/mini, claude-haiku-4-5, mistral-small-2603, tinfoil/gpt-oss-120b, tinfoil/gemma4-31b) with forward-resolving legacy aliases; live discovery via GET /duckchat/v1/capabilities (fetch_capabilities, proof-gated retry). Challenge solving verified LIVE from an unrestricted network (54–987ms/solve; the 404 JSON body proves auth+shape, isolating model IDs). FEAT-10 CLOSED (true generate_stream). Rewrite collateral: plugin 1,050 → 1,229 LOC, tests 49 → 99 (suite 3,433), ruff + black clean. Register: 136 findings / 18 OPEN. +| ID | Severity | Category | File(s) | Title | +|----|----------|----------|---------|-------| +| FEAT-10 | Low | New Features | `agentkthx/plugins/duckduckgo/duckduckgo.py` (`generate_stream`) | CLOSED by the 0.2.0 protocol rewrite — TRUE incremental generator with the full error taxonomy | +| TEST-13 | Low | Testing | `tests/test_duckduckgo_backend.py`, `scripts/probe_duckduckgo.py` | Challenge handshake live-verified via ddg-challenge-solver (2026-10-09); remaining: one live 200 SSE turn through the plugin (`--live` mode) | + +--- + ## Detailed Findings ### Robustness @@ -225,6 +238,16 @@ Proposal: override `_build_stream_body` to delegate through a shared `_build_sil **Impact:** Users lose the API's thinking-mode toggle and anti-loop sampling knob on the one provider that supports both per-request; wiring them is the difference between parity and passthrough. --- +#### FEAT-10: DuckDuckGo `generate_stream()` not implemented at scaffold time — CLOSED 2026-10-09 (protocol-rotation rewrite) +| Property | Value | +|----------|-------| +| **Severity** | Low | +| **Category** | New Feature | +| **File(s)** | `agentkthx/plugins/duckduckgo/duckduckgo.py` (`generate_stream`) | +| **Status** | CLOSED in plugin v0.2.0 | +CLOSED by the same-day protocol rotation. The x-vqd-4 retirement forced a full rewrite, and the challenge-based protocol made streaming the natural shape: `generate_stream()` yields each role-based text delta as it parses off the SSE stream (4096-byte reads), parks the next challenge from the response header (per-turn rotation, second call skips /status), re-solves ONCE on a mid-stream 401/ERR_CHALLENGE (replaying the turn without re-yielding), and surfaces the exact buffered-path error taxonomy (404 catalog rotation / 403 anti-bot / 429 rate limit / 5xx upstream / conversation limit / ERR_CHALLENGE-persisted). 15 stream tests pin the path (deltas, rotation reuse, body+proof parity with `generate()`, legacy action grammar, no-[DONE] end, every error surface). The scaffold's concern about the StreamAccumulator/StreamRenderer split is moot — the CLI streaming path consumed the generator unchanged on first live use. +--- + --- ### Architecture ## Priority Matrix diff --git a/scripts/probe_duckduckgo.py b/scripts/probe_duckduckgo.py new file mode 100644 index 00000000..60f1431b --- /dev/null +++ b/scripts/probe_duckduckgo.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""probe_duckduckgo.py — DuckDuckGo AI Chat backend live probe (2026-10 protocol). + +Drives the duckduckgo PLUGIN BACKEND itself from an unrestricted +network — TEST-13's automation. The old x-vqd-4 probe is retired with +the protocol; this one exercises the real code path end-to-end: + + 1. Proof check — /status challenge fetch → Node solve → proof build + 2. Capabilities — GET /duckchat/v1/capabilities (live model catalog) + 3. Model sweep — one minimal generate() per seed model + 4. Seed emit — a model_seed.json-compatible JSON block (--json) + +Requires: node on PATH (challenge solver + durableStream keygen — +same runtime dependency as the backend). Stdlib otherwise. + +Usage: + python3 scripts/probe_duckduckgo.py # human-readable + python3 scripts/probe_duckduckgo.py --json # seed-catalog block + python3 scripts/probe_duckduckgo.py --model gpt-6-luna # one model +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from agentkthx.plugins.duckduckgo.duckduckgo import ( # noqa: E402 + _WIRE_MODELS, + DuckDuckGoBackend, +) + +#: Seed metadata merged into the --json emit (mirrors update_ddg_seed.py). +_SEED_DEFAULTS = { + "context_length": 128000, + "default_temperature": 0.7, + "default_max_tokens": 16384, + "free_tier": True, +} + + +def probe_proof(backend: DuckDuckGoBackend) -> bool: + """Challenge fetch → solve → build (the x-vqd-hash-1 handshake).""" + print("== proof check: /status challenge -> node solve -> header build ==") + t0 = time.monotonic() + try: + challenge = backend._fetch_challenge() + raw = backend._solve_challenge(challenge) + header = backend._build_proof_header(raw, (time.monotonic() - t0) * 1000) + except Exception as e: # noqa: BLE001 — probe tool reports everything + print(f" FAIL: {e}") + return False + cid = raw.get("meta", {}).get("challenge_id", "")[:16] + probes = raw.get("client_hashes", [None, None, None])[1:] + print(f" ok: challenge {cid}… probes={probes} -> {len(header)}-char X-Vqd-Hash-1") + return True + + +def probe_capabilities(backend: DuckDuckGoBackend) -> dict | None: + """Live model catalog via GET /duckchat/v1/capabilities.""" + print("== capabilities: live catalog ==") + caps = backend.fetch_capabilities() + if caps is None: + print(" unavailable (soft-failed) — seed catalog remains authoritative") + return None + print(f" {json.dumps(caps, indent=2)[:1200]}") + return caps + + +def probe_models(backend: DuckDuckGoBackend, only: str | None) -> dict[str, str]: + """One minimal generate() per seed model → {model: reply-or-error}.""" + targets = [m["id"] for m in _WIRE_MODELS if not only or m["id"] == only] + print(f"== model sweep: {len(targets)} model(s) ==") + results: dict[str, str] = {} + for wire_id in targets: + try: + t0 = time.monotonic() + out = backend.generate(wire_id, [{"role": "user", "content": "Say exactly: pong"}]) + ms = (time.monotonic() - t0) * 1000 + ok = "pong" in (out.get("content") or "").lower() + results[wire_id] = "ok" if ok else "reply" + print( + f" [{'OK' if ok else 'REPLY'}] {wire_id} ({ms:.0f}ms): " + f"{(out.get('content') or '')[:60]!r}" + ) + except Exception as e: # noqa: BLE001 — every failure is data + results[wire_id] = f"error: {e}" + print(f" [FAIL] {wire_id}: {e}") + time.sleep(2) # per-IP throttle courtesy + return results + + +def emit_seed(results: dict[str, str]) -> None: + """Print a model_seed.json-compatible `duckduckgo` block.""" + seed = {} + for wire_id, verdict in results.items(): + if verdict not in ("ok", "reply"): + continue # seed only what answers — the house directive + entry = dict(_SEED_DEFAULTS) + entry["family"] = wire_id.split("/")[0].split("-")[0] + entry["note"] = f"live-probed {time.strftime('%Y-%m-%d')} ({verdict})" + seed[wire_id] = entry + print("\n== model_seed.json `duckduckgo` block ==") + print(json.dumps({"duckduckgo": seed}, indent=1)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--json", action="store_true", help="emit a seed-catalog block") + parser.add_argument("--model", help="probe a single wire ID instead of the sweep") + args = parser.parse_args() + + backend = DuckDuckGoBackend() + if not probe_proof(backend): + return 1 + probe_capabilities(backend) + results = probe_models(backend, args.model) + if args.json: + emit_seed(results) + + good = sum(1 for v in results.values() if v in ("ok", "reply")) + print(f"\nsummary: {good}/{len(results)} models answered") + return 0 if good else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_duckduckgo_backend.py b/tests/test_duckduckgo_backend.py new file mode 100644 index 00000000..2362b7bd --- /dev/null +++ b/tests/test_duckduckgo_backend.py @@ -0,0 +1,1277 @@ +""" +DuckDuckGo plugin — backend regression tests (x-vqd-hash-1 protocol). + +Verifies that the DuckDuckGoBackend (the 12th cloud backend, and the +ONLY one that subclasses BaseBackend directly instead of CloudBackend — +the /duckchat/v1 protocol is non-OpenAI) correctly implements the +CHALLENGE-BASED protocol that replaced the retired x-vqd-4 token +(2026-10 rotation): + + - BaseBackend (NOT CloudBackend) inheritance — the design pin + - BackendType.DUCKDUCKGO enum value; base_url → https://duck.ai + - Keyless construction (no API-key validation — nothing to validate) + - Proof lifecycle: /status bootstrap (x-vqd-accept: 1 → challenge + header), Node solve (DDG_SOLVE_UA injected), X-Vqd-Hash-1 build + (server_hashes passthrough, client_hashes SHA-256-hashed, meta + gains origin/stack/duration), per-turn rotation via the /chat + response header + - Chat wire shape: {model, messages, canUseTools:false, + canUseApproxLocation:null, reasoningEffort (per-model), + durableStream{messageId, conversationId, publicKey}} — sampling + params dropped + - Role-based SSE grammar (assistant/source/tool-invocation/ + ui-component/error) + legacy action-chunk fallback + bracketed + markers ([DONE] / [PING] / [LIMIT_CONVERSATION] / [CHAT_TITLE:]) + - generate() buffered happy path + estimated usage + - generate_stream() TRUE incremental yield (FEAT-10 — closed) + - Error taxonomy with remediation messages: 401 re-solve retry + (once), 404 ERR_MODEL_UNAVAILABLE (catalog rotation), 403 + anti-bot, 429 rate limit, 5xx upstream, ERR_CHALLENGE re-solve + (once, then "persisted"), ERR_CONVERSATION_LIMIT / bare + [LIMIT_CONVERSATION] + - fetch_capabilities() — live catalog discovery, 401-gated retry + with a solved proof, soft-fail to None + - Catalog: 8 client-verified models served offline from the seed, + cache-routed, alias-aware + - Capability verdicts as PROTOCOL FACTS: REACT tools / NO thinking + - Alias resolution (2025-era wire IDs + short aliases → the + current catalog) + - _collapse_system_into_user (system folds into the first user + turn; tool/function roles fold to user — DDG only understands + user/assistant) + - Node dependency: fail-fast remediation when node is missing; + bundled .js helpers ship with the plugin + - Plugin manifest + register()/unregister() contract + - model_seed.json carries the 8-model duckduckgo catalog + - config.py + CLI env reference expose the DUCKDUCKGO_* vars + +The Node solver/keygen subprocesses are FAKED at the instance level +(_solve_challenge/_mint_durable) — these tests exercise the Python +protocol plumbing; the solver itself is verified offline against +captured challenges by the ddg-challenge-solver package. +""" + +import base64 +import hashlib +import io +import json +import re +import sys +import urllib.error +import urllib.request +from pathlib import Path + +import pytest + +# Make agentkthx importable when run from the repo root +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from agentkthx.backends.base import BaseBackend # noqa: E402 +from agentkthx.core.types import BackendType, ThinkingSupport, ToolSupportLevel # noqa: E402 +from agentkthx.plugins.duckduckgo import duckduckgo as ddg # noqa: E402 +from agentkthx.plugins.duckduckgo.duckduckgo import ( # noqa: E402 + _MODEL_ALIASES, + _WIRE_MODELS, + DuckDuckGoBackend, + _classify_sse_payload, + _resolve_model, +) + +# The 8 client-verified wire IDs (live duck.ai dropdown + the +# benoitpetit/duckduckgo-chat-cli models.go cross-check, 2026-10-09). +CANONICAL_IDS = { + "gpt-6-luna", + "gpt-5.6-luna", + "gpt-5.4-nano", + "gpt-5.4-mini", + "claude-haiku-4-5", + "mistral-small-2603", + "tinfoil/gpt-oss-120b", + "tinfoil/gemma4-31b", +} + +BASE = "https://duck.ai" +STATUS_URL = f"{BASE}/duckchat/v1/status" +CHAT_URL = f"{BASE}/duckchat/v1/chat" +CAPABILITIES_URL = f"{BASE}/duckchat/v1/capabilities" + +# Canned RAW solver output — what ddg_vqd.js emits for a challenge. +_RAW_SOLUTION = { + "server_hashes": ["srv-1", "srv-2", "srv-3"], + "client_hashes": [ + "Mozilla/5.0 (X11; Linux x86_64) Chrome/136", + "probe2-raw", + "95", + ], + "signals": {}, + "meta": { + "v": "4", + "challenge_id": "cid-abc123", + "timestamp": "2026-10-09T00:00:00Z", + "debug": "", + }, +} + +# Canned durableStream state — what ddg_durable.js mints. +_DURABLE = { + "conversation_id": "conv" + "0" * 28, + "public_key": {"kty": "RSA", "n": "abc123", "e": "AQAB"}, +} + + +# --------------------------------------------------------------------------- +# Fakes +# --------------------------------------------------------------------------- + + +class _FakeResponse: + """Minimal urllib response double: read(n) chunking + headers.""" + + def __init__(self, body: bytes = b"", headers: dict | None = None, status: int = 200): + self._data = body + self._pos = 0 + self.headers = headers or {} + self.status = status + + def read(self, n: int = -1) -> bytes: + if self._pos >= len(self._data): + return b"" + chunk = self._data[self._pos : self._pos + n] if n and n > 0 else self._data[self._pos :] + self._pos += len(chunk) + return chunk + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + +def _sse(*events: str) -> bytes: + """Build a DDG SSE body from `data: ...` payload strings.""" + return ("\n".join(f"data: {e}" for e in events) + "\n").encode("utf-8") + + +def _status_ok(challenge: str = "CHAL-S1") -> _FakeResponse: + """A /status response carrying a fresh challenge.""" + return _FakeResponse(b'{"status":"ok"}', {"x-vqd-hash-1": challenge}) + + +def _chat_ok(chunks: list[str], fresh: str = "CHAL-C1") -> _FakeResponse: + """A role-based /chat SSE success carrying the next challenge.""" + body = _sse( + *[json.dumps({"role": "assistant", "message": c}) for c in chunks], + "[DONE]", + ) + return _FakeResponse(body, {"x-vqd-hash-1": fresh}) + + +def _chat_sse_error(err_type: str, status: int = 401, fresh: str = "CHAL-C1") -> _FakeResponse: + """A /chat 200 whose SSE stream carries an error frame.""" + body = _sse(json.dumps({"action": "error", "status": status, "type": err_type})) + return _FakeResponse(body, {"x-vqd-hash-1": fresh}) + + +def _http_error(code: int, body: bytes = b"", url: str = CHAT_URL) -> urllib.error.HTTPError: + """A raisable HTTPError with a drainable body.""" + return urllib.error.HTTPError(url, code, "boom", {}, io.BytesIO(body)) + + +class _Recorder: + """urlopen double dispatching by URL, recording every request.""" + + def __init__(self): + self.requests: list[urllib.request.Request] = [] + self.status_responses: list = [] + self.chat_responses: list = [_chat_ok(["Hello", " world"], "CHAL-C1")] + self.chat_errors: list = [] # exceptions raised before any response + self.capabilities_responses: list = [] + self.capabilities_errors: list = [] + self.root_responses: list = [] # GET / (fe-meta scrape) + + def __call__(self, req, timeout=None): + self.requests.append(req) + url = req.full_url + if url == STATUS_URL: + if self.status_responses: + return self.status_responses.pop(0) + return _status_ok("CHAL-S-extra") + if url == CHAT_URL: + if self.chat_errors: + raise self.chat_errors.pop(0) + if self.chat_responses: + return self.chat_responses.pop(0) + return _chat_ok(["fallback"], "CHAL-C-next") + if url == CAPABILITIES_URL: + if self.capabilities_errors: + raise self.capabilities_errors.pop(0) + if self.capabilities_responses: + return self.capabilities_responses.pop(0) + return _FakeResponse(b"{}", {}) + if url in (BASE, f"{BASE}/"): + # The fe-meta scrape (best-effort HTML read) + if self.root_responses: + return self.root_responses.pop(0) + return _FakeResponse(b"", {}) + raise AssertionError(f"unexpected URL: {url}") + + # -- assertion helpers ------------------------------------------------- + @property + def chat_requests(self): + return [r for r in self.requests if r.full_url == CHAT_URL] + + @property + def status_requests(self): + return [r for r in self.requests if r.full_url == STATUS_URL] + + @property + def capabilities_requests(self): + return [r for r in self.requests if r.full_url == CAPABILITIES_URL] + + @staticmethod + def header(req: urllib.request.Request, name: str): + # urllib.request.Request capitalizes header keys on storage + # ("X-Vqd-Hash-1" -> "X-vqd-hash-1"); try the common spellings. + for key in (name, name.capitalize(), name.lower(), name.upper()): + if key in req.headers: + return req.headers[key] + return None + + def chat_header(self, index: int, name: str): + return self.header(self.chat_requests[index], name) + + def chat_body(self, index: int) -> dict: + return json.loads(self.chat_requests[index].data.decode("utf-8")) + + +@pytest.fixture +def backend(): + """A DuckDuckGoBackend with the Node helpers FAKED (no subprocess + spawn) and clean proof state (catalog auto-isolated by the conftest + _isolated_model_cache fixture).""" + b = DuckDuckGoBackend() + b._node_ok = True # solver/keygen faked below — node never spawns + b._solve_challenge = lambda ch: json.loads(json.dumps(_RAW_SOLUTION)) + b._mint_durable = lambda: json.loads(json.dumps(_DURABLE)) + return b + + +@pytest.fixture +def recorder(): + return _Recorder() + + +def _wire(monkeypatch, recorder: _Recorder) -> None: + monkeypatch.setattr("urllib.request.urlopen", recorder) + + +# --------------------------------------------------------------------------- +# Identity & wiring +# --------------------------------------------------------------------------- + + +class TestDuckDuckGoIdentity: + """Class wiring pins — the scaffold's core design decisions.""" + + def test_subclasses_base_backend_directly(self): + """THE design pin: DDG is cloud but NOT CloudBackend — the + /duckchat/v1 protocol (challenge handshake, own SSE shape, no + /models, no sampling params) shares nothing with the + OpenAI-compat shared transport.""" + from agentkthx.backends.cloud_base import CloudBackend + + assert issubclass(DuckDuckGoBackend, BaseBackend) + assert not issubclass(DuckDuckGoBackend, CloudBackend) + + def test_backend_type_enum(self): + assert BackendType.DUCKDUCKGO.value == "duckduckgo" + assert DuckDuckGoBackend().backend_type is BackendType.DUCKDUCKGO + + def test_is_cloud_true(self, backend): + assert backend.is_cloud is True + + def test_base_url_default_is_duck_ai(self, backend): + """The 2026-10 protocol rotation moved the chat surface from + duckduckgo.com to duck.ai.""" + assert backend.base_url == "https://duck.ai" + + def test_keyless_construction_needs_no_api_key(self, monkeypatch): + """No API key env var, no key kwarg — construction succeeds + (contrast: CloudBackend subclasses raise on a missing key).""" + monkeypatch.delenv("DUCKDUCKGO_USER_AGENT", raising=False) + b = DuckDuckGoBackend() # no key anywhere — fine + assert b._user_agent.startswith("Mozilla/5.0") + + def test_user_agent_precedence(self, backend): + """Explicit kwarg beats the env var beats the built-in default.""" + b = DuckDuckGoBackend(user_agent="MyUA/1.0") + assert b._user_agent == "MyUA/1.0" + + def test_default_model_constant(self): + assert ddg.DUCKDUCKGO_DEFAULT_MODEL_STR == "gpt-6-luna" + + +# --------------------------------------------------------------------------- +# Node dependency + solver subprocess plumbing +# --------------------------------------------------------------------------- + + +class TestNodeDependency: + """Node is a runtime dep (challenge solver + keygen) — fail fast.""" + + def test_missing_node_fails_fast_with_remediation(self): + b = DuckDuckGoBackend() + b._node_ok = False + with pytest.raises(RuntimeError, match="Node.js"): + b._require_node("challenge solving") + with pytest.raises(RuntimeError, match="Node.js"): + b._solve_challenge("X") + with pytest.raises(RuntimeError, match="Node.js"): + b._mint_durable() + + def test_solve_challenge_spawns_node_with_solve_ua(self, monkeypatch): + """The solver subprocess MUST see DDG_SOLVE_UA = the request + UA — a mismatch invalidates the proof.""" + captured = {} + + class _Proc: + returncode = 0 + stdout = json.dumps(_RAW_SOLUTION).encode("utf-8") + stderr = b"" + + def fake_run(argv, **kwargs): + captured["argv"] = argv + captured["input"] = kwargs.get("input") + captured["env"] = kwargs.get("env") + return _Proc() + + monkeypatch.setattr(ddg.subprocess, "run", fake_run) + b = DuckDuckGoBackend(user_agent="MyUA/1.0") + b._node_ok = True + raw = b._solve_challenge("CHALLENGE-B64") + assert captured["argv"][:2] == ["node", ddg._SOLVER_JS] + assert captured["input"] == b"CHALLENGE-B64" + assert captured["env"]["DDG_SOLVE_UA"] == "MyUA/1.0" + assert raw["server_hashes"] == _RAW_SOLUTION["server_hashes"] + + def test_solve_challenge_failure_raises(self, monkeypatch): + class _Proc: + returncode = 1 + stdout = b"" + stderr = b"solver exploded" + + monkeypatch.setattr(ddg.subprocess, "run", lambda *a, **k: _Proc()) + b = DuckDuckGoBackend() + b._node_ok = True + with pytest.raises(RuntimeError, match="challenge solver failed"): + b._solve_challenge("X") + + def test_mint_durable_failure_raises(self, monkeypatch): + class _Proc: + returncode = 1 + stdout = b"" + stderr = b"keygen exploded" + + monkeypatch.setattr(ddg.subprocess, "run", lambda *a, **k: _Proc()) + b = DuckDuckGoBackend() + b._node_ok = True + with pytest.raises(RuntimeError, match="durableStream keygen failed"): + b._mint_durable() + + def test_node_helpers_ship_with_plugin(self): + """The .js helpers are runtime deps — they must be in the + plugin package (self-contained backend).""" + assert Path(ddg._SOLVER_JS).is_file() + assert Path(ddg._SOLVER_JS).name == "ddg_vqd.js" + assert Path(ddg._DURABLE_JS).is_file() + assert Path(ddg._DURABLE_JS).name == "ddg_durable.js" + + +# --------------------------------------------------------------------------- +# Proof lifecycle +# --------------------------------------------------------------------------- + + +class TestProofLifecycle: + """Challenge bootstrap → solve → X-Vqd-Hash-1 build → rotation.""" + + def test_status_headers_carry_vqd_accept(self, backend): + """x-vqd-accept: 1 is the opt-in that makes /status issue a + challenge instead of the bare status JSON.""" + headers = backend._build_status_headers() + assert headers["x-vqd-accept"] == "1" + assert headers["Referer"] == "https://duck.ai/" + assert headers["User-Agent"].startswith("Mozilla/5.0") + assert re.fullmatch(r"[0-9a-f]{32}", headers["x-ddg-journey-id"]) + + def test_missing_status_challenge_raises(self, backend, recorder, monkeypatch): + """/status without the x-vqd-hash-1 header = protocol rotated + again — surface it loudly.""" + recorder.status_responses = [_FakeResponse(b'{"status":"ok"}', {})] + _wire(monkeypatch, recorder) + with pytest.raises(RuntimeError, match="no x-vqd-hash-1"): + backend.generate("gpt-6-luna", [{"role": "user", "content": "Hi"}]) + + def test_chat_headers_carry_proof_and_browser_contract(self, backend): + headers = backend._build_chat_headers("PROOF-VALUE") + assert headers["X-Vqd-Hash-1"] == "PROOF-VALUE" + assert headers["Accept"] == "text/event-stream" + assert headers["Origin"] == "https://duck.ai" + assert headers["Referer"] == "https://duck.ai/" + assert headers["Content-Type"] == "application/json" + assert headers["User-Agent"].startswith("Mozilla/5.0") + assert headers["x-fe-version"].startswith("serp_") + assert re.fullmatch(r"[0-9a-f]{32}", headers["x-ddg-journey-id"]) + + def test_fe_signals_decode_to_engagement_events(self, backend): + signals = backend._fe_signals() + payload = json.loads(base64.b64decode(signals)) + names = [e["name"] for e in payload["events"]] + assert "startNewChat_free" in names + assert "action" in names + assert payload["end"] >= payload["start"] + + def test_build_proof_header_hashes_client_hashes(self, backend): + """The page bundle's post-processing: server_hashes pass through + UNTOUCHED, client_hashes become base64(SHA256(String(value))), + meta gains origin/stack/duration.""" + hdr = backend._build_proof_header(_RAW_SOLUTION, 1234) + assert hdr.startswith("eyJ") # base64 of a JSON object + sol = json.loads(base64.b64decode(hdr)) + assert sol["server_hashes"] == _RAW_SOLUTION["server_hashes"] + assert sol["client_hashes"] == [ + base64.b64encode(hashlib.sha256(str(v).encode()).digest()).decode() + for v in _RAW_SOLUTION["client_hashes"] + ] + assert sol["signals"] == {} + assert sol["meta"]["origin"] == "https://duck.ai" + assert "duckai-dist" in sol["meta"]["stack"] + assert sol["meta"]["duration"] == "1234" + assert sol["meta"]["v"] == "4" + assert sol["meta"]["challenge_id"] == "cid-abc123" + + def test_scrape_fe_meta_refreshes_version(self, backend, recorder, monkeypatch): + """The serp version is scraped from GET / when available.""" + html = ( + b'serp_20261009_120000_ET-abcdef123" + ) + recorder.root_responses = [_FakeResponse(html, {}), _FakeResponse(html, {})] + _wire(monkeypatch, recorder) + meta = backend._scrape_fe_meta() + assert meta["fe_version"] == "serp_20261009_120000_ET-abcdef123" + assert meta["entry_js"] == "entry.duckai.deadbeefcafe.js" + headers = backend._build_chat_headers("P") + assert headers["x-fe-version"] == "serp_20261009_120000_ET-abcdef123" + assert "entry.duckai.deadbeefcafe.js" in backend._synth_stack() + + +# --------------------------------------------------------------------------- +# SSE grammar +# --------------------------------------------------------------------------- + + +class TestSSEGrammar: + """_classify_sse_payload — role-based (current), action-based + (legacy fallback), and bracketed markers.""" + + def test_assistant_text_delta(self): + ev = _classify_sse_payload(json.dumps({"role": "assistant", "message": "hi"}).encode()) + assert ev == {"kind": "text", "text": "hi"} + + def test_assistant_state_only_frame_ignored(self): + assert ( + _classify_sse_payload(json.dumps({"role": "assistant", "status": "thinking"}).encode()) + is None + ) + + def test_source_event(self): + ev = _classify_sse_payload( + json.dumps({"role": "source", "source": {"url": "https://x", "title": "X"}}).encode() + ) + assert ev == {"kind": "source", "url": "https://x", "title": "X"} + + def test_source_without_url_ignored(self): + assert _classify_sse_payload(json.dumps({"role": "source", "source": {}}).encode()) is None + + def test_tool_invocation_event(self): + ev = _classify_sse_payload( + json.dumps( + {"role": "tool-invocation", "toolName": "WebSearch", "state": "done"} + ).encode() + ) + assert ev == {"kind": "tool", "tool": "WebSearch", "state": "done"} + + def test_ui_component_event(self): + ev = _classify_sse_payload(json.dumps({"role": "ui-component", "name": "chart"}).encode()) + assert ev == {"kind": "ui", "name": "chart"} + + def test_role_error_event(self): + ev = _classify_sse_payload( + json.dumps({"role": "error", "type": "ERR_X", "status": 500}).encode() + ) + assert ev == {"kind": "error", "type": "ERR_X", "status": 500} + + def test_action_error_event(self): + ev = _classify_sse_payload( + json.dumps({"action": "error", "type": "ERR_CHALLENGE", "status": 401}).encode() + ) + assert ev == {"kind": "error", "type": "ERR_CHALLENGE", "status": 401} + + def test_legacy_action_chunk_still_parsed(self): + """DDG has flipped event shapes before — the 2025 action + grammar stays as a fallback.""" + ev = _classify_sse_payload(json.dumps({"action": "chunk", "message": "old"}).encode()) + assert ev == {"kind": "text", "text": "old"} + + def test_bracketed_markers(self): + assert _classify_sse_payload(b"[DONE]") == {"kind": "done"} + assert _classify_sse_payload(b"[PING]") == {"kind": "ping"} + assert _classify_sse_payload(b"[LIMIT_CONVERSATION]") == {"kind": "limit"} + ev = _classify_sse_payload(b"[CHAT_TITLE: Greetings]") + assert ev == {"kind": "title", "text": "Greetings"} + + def test_noise_payloads_ignored(self): + assert _classify_sse_payload(b"not json") is None + assert _classify_sse_payload(b"[1, 2, 3]") is None # non-dict JSON + assert _classify_sse_payload(json.dumps({"action": "start"}).encode()) is None + + +# --------------------------------------------------------------------------- +# Message shaping +# --------------------------------------------------------------------------- + + +class TestMessageShaping: + """_collapse_system_into_user — DDG strips the system role and + only understands user/assistant.""" + + def test_system_prepended_to_first_user(self, backend): + convo = backend._collapse_system_into_user( + [ + {"role": "system", "content": "Be terse."}, + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello"}, + {"role": "user", "content": "Bye"}, + ] + ) + assert convo[0] == {"role": "user", "content": "Be terse.\n\nHi"} + assert convo[1] == {"role": "assistant", "content": "Hello"} + assert convo[2] == {"role": "user", "content": "Bye"} + + def test_multiple_systems_join(self, backend): + convo = backend._collapse_system_into_user( + [ + {"role": "system", "content": "Rule A."}, + {"role": "system", "content": "Rule B."}, + {"role": "user", "content": "Hi"}, + ] + ) + assert convo == [{"role": "user", "content": "Rule A.\n\nRule B.\n\nHi"}] + + def test_system_only_dropped(self, backend): + """No user turn to host the system content → empty convo → + generate() refuses (ValueError) rather than sending nothing.""" + assert backend._collapse_system_into_user([{"role": "system", "content": "X"}]) == [] + + def test_tool_role_folded_as_user(self, backend): + """Tool/function outputs ride as user turns — the ReAct loop + depends on this (DDG rejects foreign roles).""" + convo = backend._collapse_system_into_user( + [ + {"role": "user", "content": "Search for X"}, + {"role": "assistant", "content": "Thought: I will act"}, + {"role": "tool", "content": "result payload"}, + ] + ) + assert [m["role"] for m in convo] == ["user", "assistant", "user"] + assert convo[2]["content"] == "result payload" + + def test_tool_role_can_host_pending_system(self, backend): + convo = backend._collapse_system_into_user( + [ + {"role": "system", "content": "S"}, + {"role": "tool", "content": "r"}, + ] + ) + assert convo == [{"role": "user", "content": "S\n\nr"}] + + def test_leading_assistant_gets_user_primer(self, backend): + convo = backend._collapse_system_into_user( + [{"role": "assistant", "content": "Hi"}, {"role": "user", "content": "OK"}] + ) + # a synthetic "Hello" user turn primes the conversation (DDG + # requires a leading user message); the assistant turn survives + assert convo == [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi"}, + {"role": "user", "content": "OK"}, + ] + + def test_content_part_list_coerced(self, backend): + convo = backend._collapse_system_into_user( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "part1"}, + {"type": "text", "text": "part2"}, + ], + } + ] + ) + assert convo[0]["content"] == "part1\npart2" + + def test_empty_messages_refused(self, backend): + with pytest.raises(ValueError, match="no messages after system collapse"): + backend.generate("gpt-6-luna", []) + with pytest.raises(ValueError, match="no messages after system collapse"): + backend.generate("gpt-6-luna", [{"role": "system", "content": "X"}]) + + +# --------------------------------------------------------------------------- +# generate() — buffered happy path +# --------------------------------------------------------------------------- + + +class TestGenerateHappyPath: + """scrape → /status → solve → /chat → buffered content + rotation.""" + + def test_bootstrap_chat_rotate_order(self, backend, recorder, monkeypatch): + _wire(monkeypatch, recorder) + out = backend.generate("gpt-6-luna", [{"role": "user", "content": "Hello"}]) + assert out["content"] == "Hello world" + assert out["model"] == "gpt-6-luna" + assert out["tool_calls"] == [] + assert out["finish_reason"] == "stop" + assert out["usage"]["estimated"] is True + assert out["usage"]["total_tokens"] >= 2 + # request order: fe-meta scrape, challenge bootstrap, chat + assert [r.full_url for r in recorder.requests] == [ + f"{BASE}/", + STATUS_URL, + CHAT_URL, + ] + # the NEXT challenge is parked for the following turn + assert backend._next_challenge_b64 == "CHAL-C1" + + def test_second_generate_reuses_rotated_challenge(self, backend, recorder, monkeypatch): + """The /chat response header carries the next challenge — the + second turn must NOT hit /status again.""" + _wire(monkeypatch, recorder) + backend.generate("gpt-6-luna", [{"role": "user", "content": "One"}]) + backend.generate("gpt-6-luna", [{"role": "user", "content": "Two"}]) + assert len(recorder.status_requests) == 1 + assert len(recorder.chat_requests) == 2 + + def test_chat_request_carries_solved_proof(self, backend, recorder, monkeypatch): + _wire(monkeypatch, recorder) + backend.generate("gpt-6-luna", [{"role": "user", "content": "Hi"}]) + proof = recorder.chat_header(0, "X-Vqd-Hash-1") + assert proof and proof.startswith("eyJ") + sol = json.loads(base64.b64decode(proof)) + assert sol["server_hashes"] == _RAW_SOLUTION["server_hashes"] + assert sol["meta"]["origin"] == "https://duck.ai" + + def test_request_body_protocol_shape(self, backend, recorder, monkeypatch): + """Body is EXACTLY the current wire shape — sampling params + (temperature / max_tokens / top_p) are dropped, tools off.""" + _wire(monkeypatch, recorder) + backend.generate( + "gpt-6-luna", + [{"role": "user", "content": "Hello"}], + tools=[{"name": "x"}], + temperature=0.7, + max_tokens=100, + think=True, + ) + body = recorder.chat_body(0) + assert set(body) == { + "model", + "messages", + "canUseTools", + "canUseApproxLocation", + "reasoningEffort", + "durableStream", + } + assert body["model"] == "gpt-6-luna" + assert body["messages"] == [{"role": "user", "content": "Hello"}] + assert body["canUseTools"] is False + assert body["canUseApproxLocation"] is None + assert body["reasoningEffort"] == "none" + ds = body["durableStream"] + assert ds["conversationId"] == _DURABLE["conversation_id"] + assert ds["publicKey"] == _DURABLE["public_key"] + assert re.fullmatch(r"[0-9a-f]{32}", ds["messageId"]) + + def test_reasoning_effort_per_model(self, backend, recorder, monkeypatch): + """The frontend sends reasoningEffort 'low' for the Tinfoil + open-weight pair, 'none' for everything else.""" + _wire(monkeypatch, recorder) + backend.generate("tinfoil/gpt-oss-120b", [{"role": "user", "content": "Hi"}]) + assert recorder.chat_body(0)["reasoningEffort"] == "low" + backend.generate("tinfoil/gemma4-31b", [{"role": "user", "content": "Hi"}]) + assert recorder.chat_body(1)["reasoningEffort"] == "low" + backend.generate("claude-haiku-4-5", [{"role": "user", "content": "Hi"}]) + assert recorder.chat_body(2)["reasoningEffort"] == "none" + + def test_alias_resolved_on_the_wire(self, backend, recorder, monkeypatch): + _wire(monkeypatch, recorder) + backend.generate("claude-3-haiku", [{"role": "user", "content": "Hi"}]) + assert recorder.chat_body(0)["model"] == "claude-haiku-4-5" + backend.generate("llama", [{"role": "user", "content": "Hi"}]) + assert recorder.chat_body(1)["model"] == "tinfoil/gpt-oss-120b" + + def test_system_and_tools_collapsed_into_body(self, backend, recorder, monkeypatch): + _wire(monkeypatch, recorder) + backend.generate( + "gpt-6-luna", + [ + {"role": "system", "content": "Be terse."}, + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello"}, + {"role": "tool", "content": "tool output"}, + ], + ) + body = recorder.chat_body(0) + assert body["messages"] == [ + {"role": "user", "content": "Be terse.\n\nHi"}, + {"role": "assistant", "content": "Hello"}, + {"role": "user", "content": "tool output"}, + ] + + def test_non_text_events_do_not_pollute_content(self, backend, recorder, monkeypatch): + """sources / pings / state frames / titles are parsed but only + assistant text reaches the buffered content.""" + recorder.chat_responses = [ + _FakeResponse( + _sse( + json.dumps({"role": "source", "source": {"url": "https://x", "title": "X"}}), + json.dumps({"role": "assistant", "message": "hi"}), + "[PING]", + json.dumps({"role": "assistant", "status": "typing"}), + json.dumps({"role": "assistant", "message": " there"}), + "[CHAT_TITLE: Greetings]", + "[DONE]", + ), + {"x-vqd-hash-1": "CHAL-C1"}, + ) + ] + _wire(monkeypatch, recorder) + out = backend.generate("gpt-6-luna", [{"role": "user", "content": "Hi"}]) + assert out["content"] == "hi there" + + def test_stream_end_without_done_marker_is_success(self, backend, recorder, monkeypatch): + """DDG streams may simply END — an exhausted stream with + content is a success (protocol nuance from the field).""" + recorder.chat_responses = [ + _FakeResponse( + _sse( + json.dumps({"role": "assistant", "message": "partial"}), + ), + {"x-vqd-hash-1": "CHAL-C1"}, + ) + ] + _wire(monkeypatch, recorder) + out = backend.generate("gpt-6-luna", [{"role": "user", "content": "Hi"}]) + assert out["content"] == "partial" + + def test_legacy_action_grammar_still_streams(self, backend, recorder, monkeypatch): + recorder.chat_responses = [ + _FakeResponse( + _sse( + json.dumps({"action": "start", "model": "gpt-6-luna"}), + json.dumps({"action": "chunk", "message": "old "}), + json.dumps({"action": "chunk", "message": "grammar"}), + "[DONE]", + ), + {"x-vqd-hash-1": "CHAL-C1"}, + ) + ] + _wire(monkeypatch, recorder) + out = backend.generate("gpt-6-luna", [{"role": "user", "content": "Hi"}]) + assert out["content"] == "old grammar" + + +# --------------------------------------------------------------------------- +# generate() — error taxonomy +# --------------------------------------------------------------------------- + + +class TestGenerateErrorTaxonomy: + """Every failure mode surfaces a remediation message.""" + + def test_401_reproof_and_retry_once(self, backend, recorder, monkeypatch): + recorder.chat_errors = [_http_error(401)] + _wire(monkeypatch, recorder) + out = backend.generate("gpt-6-luna", [{"role": "user", "content": "Hi"}]) + assert out["content"] == "Hello world" # retry landed on the success + assert len(recorder.chat_requests) == 2 + assert len(recorder.status_requests) == 2 # re-solve hit /status + + def test_401_twice_surfaces(self, backend, recorder, monkeypatch): + recorder.chat_errors = [_http_error(401), _http_error(401)] + _wire(monkeypatch, recorder) + with pytest.raises(urllib.error.HTTPError): + backend.generate("gpt-6-luna", [{"role": "user", "content": "Hi"}]) + assert len(recorder.chat_requests) == 2 + + def test_404_model_unavailable_mentions_catalog_rotation(self, backend, recorder, monkeypatch): + detail = json.dumps({"action": "error", "status": 404, "type": "ERR_MODEL_UNAVAILABLE"}) + recorder.chat_errors = [_http_error(404, detail.encode())] + _wire(monkeypatch, recorder) + with pytest.raises(RuntimeError, match="catalog rotated"): + backend.generate("gpt-6-luna", [{"role": "user", "content": "Hi"}]) + + def test_403_surfaces_anti_bot_message(self, backend, recorder, monkeypatch): + recorder.chat_errors = [_http_error(403)] + _wire(monkeypatch, recorder) + with pytest.raises(RuntimeError, match="anti-bot"): + backend.generate("gpt-6-luna", [{"role": "user", "content": "Hi"}]) + + def test_429_surfaces_rate_limit_message(self, backend, recorder, monkeypatch): + recorder.chat_errors = [_http_error(429)] + _wire(monkeypatch, recorder) + with pytest.raises(RuntimeError, match="rate limit"): + backend.generate("gpt-6-luna", [{"role": "user", "content": "Hi"}]) + + def test_503_surfaces_upstream_message(self, backend, recorder, monkeypatch): + recorder.chat_errors = [_http_error(503)] + _wire(monkeypatch, recorder) + with pytest.raises(RuntimeError, match="upstream"): + backend.generate("gpt-6-luna", [{"role": "user", "content": "Hi"}]) + + def test_conversation_limit_via_sse_error(self, backend, recorder, monkeypatch): + recorder.chat_responses = [_chat_sse_error("ERR_CONVERSATION_LIMIT", 429)] + _wire(monkeypatch, recorder) + with pytest.raises(RuntimeError, match="conversation limit"): + backend.generate("gpt-6-luna", [{"role": "user", "content": "Hi"}]) + + def test_conversation_limit_via_marker(self, backend, recorder, monkeypatch): + recorder.chat_responses = [ + _FakeResponse(_sse("[LIMIT_CONVERSATION]"), {"x-vqd-hash-1": "CHAL-C1"}) + ] + _wire(monkeypatch, recorder) + with pytest.raises(RuntimeError, match="conversation limit"): + backend.generate("gpt-6-luna", [{"role": "user", "content": "Hi"}]) + + def test_err_challenge_reproof_then_success(self, backend, recorder, monkeypatch): + recorder.chat_responses = [_chat_sse_error("ERR_CHALLENGE", 401)] + _wire(monkeypatch, recorder) + out = backend.generate("gpt-6-luna", [{"role": "user", "content": "Hi"}]) + assert out["content"] == "fallback" + assert len(recorder.chat_requests) == 2 + + def test_err_challenge_twice_mentions_persistence(self, backend, recorder, monkeypatch): + recorder.chat_responses = [ + _chat_sse_error("ERR_CHALLENGE", 401), + _chat_sse_error("ERR_CHALLENGE", 401), + ] + _wire(monkeypatch, recorder) + with pytest.raises(RuntimeError, match="persisted"): + backend.generate("gpt-6-luna", [{"role": "user", "content": "Hi"}]) + + def test_generic_sse_error_surfaces(self, backend, recorder, monkeypatch): + recorder.chat_responses = [_chat_sse_error("ERR_SOMETHING_ELSE", 500)] + _wire(monkeypatch, recorder) + with pytest.raises(RuntimeError, match="SSE error"): + backend.generate("gpt-6-luna", [{"role": "user", "content": "Hi"}]) + + +# --------------------------------------------------------------------------- +# generate_stream() — TRUE incremental (FEAT-10) +# --------------------------------------------------------------------------- + + +class TestGenerateStream: + """FEAT-10 closed: incremental deltas + the same error taxonomy.""" + + def test_yields_deltas_in_order(self, backend, recorder, monkeypatch): + _wire(monkeypatch, recorder) + chunks = list(backend.generate_stream("gpt-6-luna", [{"role": "user", "content": "Hi"}])) + assert chunks == ["Hello", " world"] + assert backend._next_challenge_b64 == "CHAL-C1" + + def test_stream_uses_parked_challenge_on_second_call(self, backend, recorder, monkeypatch): + _wire(monkeypatch, recorder) + list(backend.generate_stream("gpt-6-luna", [{"role": "user", "content": "A"}])) + list(backend.generate_stream("gpt-6-luna", [{"role": "user", "content": "B"}])) + assert len(recorder.status_requests) == 1 + + def test_stream_system_and_tool_roles_collapsed(self, backend, recorder, monkeypatch): + _wire(monkeypatch, recorder) + list( + backend.generate_stream( + "gpt-6-luna", + [ + {"role": "system", "content": "S"}, + {"role": "user", "content": "Hi"}, + {"role": "tool", "content": "t"}, + ], + ) + ) + body = recorder.chat_body(0) + assert [m["role"] for m in body["messages"]] == ["user", "user"] + assert body["messages"][0]["content"] == "S\n\nHi" + + def test_stream_body_and_proof_match_buffered_path(self, backend, recorder, monkeypatch): + _wire(monkeypatch, recorder) + list(backend.generate_stream("gpt-6-luna", [{"role": "user", "content": "Hi"}])) + body = recorder.chat_body(0) + assert set(body) == { + "model", + "messages", + "canUseTools", + "canUseApproxLocation", + "reasoningEffort", + "durableStream", + } + assert recorder.chat_header(0, "X-Vqd-Hash-1").startswith("eyJ") + + def test_stream_legacy_action_grammar(self, backend, recorder, monkeypatch): + recorder.chat_responses = [ + _FakeResponse( + _sse( + json.dumps({"action": "chunk", "message": "a"}), + json.dumps({"action": "chunk", "message": "b"}), + "[DONE]", + ), + {"x-vqd-hash-1": "CHAL-C1"}, + ) + ] + _wire(monkeypatch, recorder) + assert list(backend.generate_stream("gpt-6-luna", [{"role": "user", "content": "Hi"}])) == [ + "a", + "b", + ] + + def test_stream_end_without_done_is_success(self, backend, recorder, monkeypatch): + recorder.chat_responses = [ + _FakeResponse(_sse(json.dumps({"role": "assistant", "message": "x"})), {}) + ] + _wire(monkeypatch, recorder) + assert list(backend.generate_stream("gpt-6-luna", [{"role": "user", "content": "Hi"}])) == [ + "x" + ] + + def test_stream_401_reproof_once(self, backend, recorder, monkeypatch): + recorder.chat_errors = [_http_error(401)] + _wire(monkeypatch, recorder) + chunks = list(backend.generate_stream("gpt-6-luna", [{"role": "user", "content": "Hi"}])) + assert chunks == ["Hello", " world"] # retry landed on the success + assert len(recorder.chat_requests) == 2 + + def test_stream_err_challenge_reproof_once(self, backend, recorder, monkeypatch): + recorder.chat_responses = [_chat_sse_error("ERR_CHALLENGE", 401)] + _wire(monkeypatch, recorder) + chunks = list(backend.generate_stream("gpt-6-luna", [{"role": "user", "content": "Hi"}])) + assert chunks == ["fallback"] + + def test_stream_err_challenge_twice_mentions_persistence(self, backend, recorder, monkeypatch): + recorder.chat_responses = [ + _chat_sse_error("ERR_CHALLENGE", 401), + _chat_sse_error("ERR_CHALLENGE", 401), + ] + _wire(monkeypatch, recorder) + with pytest.raises(RuntimeError, match="persisted"): + list(backend.generate_stream("gpt-6-luna", [{"role": "user", "content": "Hi"}])) + + def test_stream_404_mentions_catalog_rotation(self, backend, recorder, monkeypatch): + detail = json.dumps({"action": "error", "status": 404, "type": "ERR_MODEL_UNAVAILABLE"}) + recorder.chat_errors = [_http_error(404, detail.encode())] + _wire(monkeypatch, recorder) + with pytest.raises(RuntimeError, match="catalog rotated"): + list(backend.generate_stream("gpt-6-luna", [{"role": "user", "content": "Hi"}])) + + def test_stream_403_surfaces_anti_bot_message(self, backend, recorder, monkeypatch): + recorder.chat_errors = [_http_error(403)] + _wire(monkeypatch, recorder) + with pytest.raises(RuntimeError, match="anti-bot"): + list(backend.generate_stream("gpt-6-luna", [{"role": "user", "content": "Hi"}])) + + def test_stream_429_surfaces_rate_limit_message(self, backend, recorder, monkeypatch): + recorder.chat_errors = [_http_error(429)] + _wire(monkeypatch, recorder) + with pytest.raises(RuntimeError, match="rate limit"): + list(backend.generate_stream("gpt-6-luna", [{"role": "user", "content": "Hi"}])) + + def test_stream_503_surfaces_upstream_message(self, backend, recorder, monkeypatch): + recorder.chat_errors = [_http_error(503)] + _wire(monkeypatch, recorder) + with pytest.raises(RuntimeError, match="upstream"): + list(backend.generate_stream("gpt-6-luna", [{"role": "user", "content": "Hi"}])) + + def test_stream_conversation_limit_surfaces(self, backend, recorder, monkeypatch): + recorder.chat_responses = [ + _FakeResponse(_sse("[LIMIT_CONVERSATION]"), {"x-vqd-hash-1": "CHAL-C1"}) + ] + _wire(monkeypatch, recorder) + with pytest.raises(RuntimeError, match="conversation limit"): + list(backend.generate_stream("gpt-6-luna", [{"role": "user", "content": "Hi"}])) + + def test_stream_empty_messages_refused(self, backend): + with pytest.raises(ValueError, match="no messages after system collapse"): + list(backend.generate_stream("gpt-6-luna", [])) + + +# --------------------------------------------------------------------------- +# fetch_capabilities — live catalog discovery +# --------------------------------------------------------------------------- + + +class TestFetchCapabilities: + def test_happy_path_parses_json(self, backend, recorder, monkeypatch): + payload = {"models": {"gpt-6-luna": {"name": "GPT-6 Luna"}}} + recorder.capabilities_responses = [_FakeResponse(json.dumps(payload).encode(), {})] + _wire(monkeypatch, recorder) + assert backend.fetch_capabilities() == payload + + def test_401_gated_retry_carries_solved_proof(self, backend, recorder, monkeypatch): + """When the endpoint is proof-gated, retry once with a solved + X-Vqd-Hash-1 (bootstrap /status → solve → resend).""" + payload = {"models": {}} + recorder.capabilities_errors = [_http_error(401, url=CAPABILITIES_URL)] + recorder.capabilities_responses = [_FakeResponse(json.dumps(payload).encode(), {})] + _wire(monkeypatch, recorder) + assert backend.fetch_capabilities() == payload + assert len(recorder.capabilities_requests) == 2 + assert len(recorder.status_requests) == 1 + second = recorder.capabilities_requests[1] + proof = _Recorder.header(second, "X-Vqd-Hash-1") + assert proof and proof.startswith("eyJ") + + def test_network_failure_returns_none(self, backend, recorder, monkeypatch): + recorder.capabilities_errors = [urllib.error.URLError("no route")] + _wire(monkeypatch, recorder) + assert backend.fetch_capabilities() is None + + def test_non_json_body_returns_none(self, backend, recorder, monkeypatch): + recorder.capabilities_responses = [_FakeResponse(b"nope", {})] + _wire(monkeypatch, recorder) + assert backend.fetch_capabilities() is None + + +# --------------------------------------------------------------------------- +# Catalog +# --------------------------------------------------------------------------- + + +class TestDuckDuckGoCatalog: + """Seed-backed catalog serving — no network, cache-routed.""" + + def test_list_models_shape(self, backend): + models = backend.list_models() + names = {m["name"] for m in models} + assert names == CANONICAL_IDS + for m in models: + assert m["size"] == 0 + d = m["details"] + assert d["backend"] == "duckduckgo" + assert d["context_length"] > 0 + assert d["free_tier"] is True + assert d["is_chat_model"] is True + assert d["pricing"] == {"input": 0.0, "output": 0.0} + + def test_list_models_no_network(self, backend, monkeypatch): + """DDG serves the catalog from the seed — listing must never + touch the wire (the house contract for `agentkthx models`).""" + + def boom(req, timeout=None): + raise AssertionError("list_models() must not open a socket") + + monkeypatch.setattr("urllib.request.urlopen", boom) + assert len(backend.list_models()) == 8 + + def test_list_models_l1_cached(self, backend, monkeypatch): + first = backend.list_models() + + def boom(req, timeout=None): + raise AssertionError("L1 cache should prevent re-serving") + + monkeypatch.setattr("urllib.request.urlopen", boom) + assert backend.list_models() == first + + def test_seed_json_carries_catalog(self): + seed = json.loads( + (Path(__file__).resolve().parents[1] / "agentkthx/data/model_seed.json").read_text() + ) + assert set(seed["duckduckgo"]) == CANONICAL_IDS + for meta in seed["duckduckgo"].values(): + assert meta["context_length"] > 0 + + def test_get_model_max_context_alias_aware(self, backend): + direct = backend.get_model_max_context("gpt-6-luna") + aliased = backend.get_model_max_context("gpt") # short alias + assert direct == aliased == 128000 + assert backend.get_model_max_context("never-heard-of-it") == 128000 + + def test_get_model_runtime_context_matches_max(self, backend): + """cmd_models calls this unconditionally — must exist and + equal the catalog context.""" + assert backend.get_model_runtime_context("gpt-6-luna") == backend.get_model_max_context( + "gpt-6-luna" + ) + + def test_get_model_info_resolves_alias(self, backend): + info = backend.get_model_info("claude-3-haiku") + assert info and info["name"] == "claude-haiku-4-5" + assert backend.get_model_info("no-such-model-anywhere") is None + + def test_is_running_never_touches_network(self, backend, monkeypatch): + def boom(req, timeout=None): + raise AssertionError("is_running() must not open a socket") + + monkeypatch.setattr("urllib.request.urlopen", boom) + assert backend.is_running() is True + + +# --------------------------------------------------------------------------- +# Capability verdicts — protocol facts, not probes +# --------------------------------------------------------------------------- + + +class TestDuckDuckGoCapabilities: + def test_tool_support_always_react(self, backend): + for name in CANONICAL_IDS: + assert backend.test_tool_support(name) is ToolSupportLevel.REACT + + def test_thinking_support_always_no(self, backend): + for name in CANONICAL_IDS: + assert backend.test_thinking_support(name) is ThinkingSupport.NO + + +# --------------------------------------------------------------------------- +# Alias resolution +# --------------------------------------------------------------------------- + + +class TestAliasResolution: + def test_alias_resolution(self): + # 2025-era wire IDs → the current catalog + assert _resolve_model("gpt-4o-mini") == "gpt-5.6-luna" + assert _resolve_model("o3-mini") == "gpt-5.4-mini" + assert _resolve_model("o4mini") == "gpt-5.4-mini" + assert _resolve_model("claude-3-haiku") == "claude-haiku-4-5" + assert _resolve_model("claude-3-haiku-20240307") == "claude-haiku-4-5" + assert _resolve_model("mistralai/Mistral-Small-24B-Instruct-2501") == "mistral-small-2603" + assert _resolve_model("meta-llama/Llama-3.3-70B-Instruct-Turbo") == "tinfoil/gpt-oss-120b" + # short aliases + assert _resolve_model("gpt") == "gpt-6-luna" + assert _resolve_model("claude") == "claude-haiku-4-5" + assert _resolve_model("mistral") == "mistral-small-2603" + assert _resolve_model("mixtral") == "mistral-small-2603" + assert _resolve_model("llama") == "tinfoil/gpt-oss-120b" + assert _resolve_model("oss") == "tinfoil/gpt-oss-120b" + assert _resolve_model("gemma") == "tinfoil/gemma4-31b" + assert _resolve_model("nano") == "gpt-5.4-nano" + assert _resolve_model("mini") == "gpt-5.4-mini" + + def test_alias_resolution_case_insensitive(self): + assert _resolve_model("GPT-6-LUNA") == "gpt-6-luna" + assert _resolve_model("Claude-3-Haiku") == "claude-haiku-4-5" + + def test_canonical_ids_pass_through(self): + for wire_id in CANONICAL_IDS: + assert _resolve_model(wire_id) == wire_id + + def test_unknown_ids_pass_through_for_upstream_error(self): + """An unknown ID is sent as-is so the server's + ERR_MODEL_UNAVAILABLE (with its catalog hint) surfaces.""" + assert _resolve_model("totally-unknown") == "totally-unknown" + + def test_alias_table_matches_canonical_catalog(self): + canonical = {m["id"] for m in _WIRE_MODELS} + assert canonical == CANONICAL_IDS + for target in _MODEL_ALIASES.values(): + assert target in canonical + + def test_wire_model_effort_map(self): + by_id = {m["id"]: m["effort"] for m in _WIRE_MODELS} + assert by_id["tinfoil/gpt-oss-120b"] == "low" + assert by_id["tinfoil/gemma4-31b"] == "low" + assert by_id["gpt-6-luna"] == "none" + + +# --------------------------------------------------------------------------- +# Plugin manifest + registration +# --------------------------------------------------------------------------- + + +class TestPluginContract: + def test_manifest_shape(self): + manifest = json.loads( + ( + Path(__file__).resolve().parents[1] / "agentkthx/plugins/duckduckgo/plugin.json" + ).read_text() + ) + assert manifest["name"] == "duckduckgo" + assert manifest["version"] == "0.2.0" + # the description must reflect the CURRENT protocol + assert "x-vqd-hash-1" in manifest["description"] + assert "challenge" in manifest["description"].lower() + ext = manifest["extensions"]["org.vts-tech.agentkthx"] + assert ext["type"] == "backend" + provides = ext["provides"] + assert provides["backends"]["duckduckgo"] == "duckduckgo.DuckDuckGoBackend" + assert provides["backends"]["ddg"] == "duckduckgo.DuckDuckGoBackend" + assert set(provides["cli_flags"]["--backend"]) == {"duckduckgo", "ddg"} + defaults = ext["config"]["defaults"] + assert defaults["DUCKDUCKGO_BASE_URL"] == "https://duck.ai" + assert defaults["DUCKDUCKGO_DEFAULT_MODEL"] == "gpt-6-luna" + # NO API key default — the backend is keyless + assert "DUCKDUCKGO_API_KEY" not in defaults + + def test_register_and_unregister(self): + class _FakeManager: + def __init__(self): + self.backends = {} + + def register_backend(self, name, cls): + self.backends[name] = cls + + def unregister_backend(self, name): + self.backends.pop(name, None) + + from agentkthx.plugins.duckduckgo import register, unregister + + m = _FakeManager() + register(m) + assert set(m.backends) == {"duckduckgo", "ddg"} + assert m.backends["duckduckgo"] is DuckDuckGoBackend + unregister(m) + assert m.backends == {} + + def test_get_backend_resolves_via_plugin_system(self): + """End-to-end: the lazy plugin path resolves both names.""" + from agentkthx.backends import get_backend + + assert isinstance(get_backend("duckduckgo"), DuckDuckGoBackend) + assert isinstance(get_backend("ddg"), DuckDuckGoBackend) + + +# --------------------------------------------------------------------------- +# Config + CLI help surfaces +# --------------------------------------------------------------------------- + + +class TestConfigSurfaces: + def test_config_module_exports_env_vars(self): + import agentkthx.config as _config + + assert _config.DUCKDUCKGO_BASE_URL == "https://duck.ai" + assert _config.DUCKDUCKGO_DEFAULT_MODEL == "gpt-6-luna" + assert isinstance(_config.DUCKDUCKGO_USER_AGENT, str) + + def test_cli_env_reference_documents_duckduckgo(self): + from agentkthx.cli.commands.config import _env_reference_entries + + names = {name for name, _ in _env_reference_entries()} + assert "DUCKDUCKGO_BASE_URL" in names + assert "DUCKDUCKGO_USER_AGENT" in names + assert "DUCKDUCKGO_DEFAULT_MODEL" in names + + def test_backend_slug_label_mapping(self): + from agentkthx.cli.commands.config import _BACKEND_SLUG_TO_LABEL + + assert _BACKEND_SLUG_TO_LABEL["duckduckgo"] == "DuckDuckGo" + assert _BACKEND_SLUG_TO_LABEL["ddg"] == "DuckDuckGo"