diff --git a/docs/configuration.md b/docs/configuration.md index ef912b8..d2290d2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -203,6 +203,16 @@ challenges or guarantee a Search Assist answer. Entity questions can also fall back to a matching Wikipedia summary; procedural and opinion questions cannot use an unrelated encyclopedia summary as an answer. +The lightweight Search Assist stage allows up to four seconds; the browser +stage allows up to ten seconds including startup and reserves time for the +remaining search fallback. Both respect the overall `timeout_seconds` limit. +For complex comparisons on small VPS hosts, 20 seconds is a useful setting. +Only one browser lookup runs at a time across networks; waiting for that slot +also counts against the request deadline. Cost/comparison and other open-ended +questions skip generic Instant Answer/Wikidata entity summaries and use the +web-result fallback when Search Assist has no answer. Page challenges and +provider outages can still prevent an answer even when a personal browser works. + `define` uses the public English dictionary service with an independent English Wiktionary fallback. The primary request is capped at two seconds or one third of the remaining deadline, leaving time for fallback. Wiktionary responses link diff --git a/docs/plugins.md b/docs/plugins.md index f1ce96b..1c739e8 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -1346,6 +1346,14 @@ unavailable or presents an anti-automation challenge, open-ended, procedural, opinion, comparison, and controversy questions also use the bounded Bing public-search fallback described below. +Price and comparison questions such as `!ask how much do dirt bikes cost vs GoKarts?` +use Search Assist or attributed web excerpts, never a generic entity description +as a substitute for the comparison. A bounded comparison retry normalizes `vs` +and compound word boundaries while retaining both subjects. Browser extraction +waits for answer text with a source link instead of stopping at the empty Search +Assist heading, and supports expanded answer cards. Browser requests share one +slot across networks to limit memory use on small hosts. + The plugin is configured under `plugins.ask`: ~~~yaml diff --git a/plugins/ask.go b/plugins/ask.go index 9a21e72..27df97f 100644 --- a/plugins/ask.go +++ b/plugins/ask.go @@ -248,7 +248,7 @@ func (p *Ask) findSource(ctx context.Context, question string, cfg bot.PluginCon var deferredWebResult askSource deferredWebResultFound := false if cfg.Bool("search_assist_enabled", true) { - stepCtx, cancel := askStepContext(ctx, 1500*time.Millisecond) + stepCtx, cancel := askStepContext(ctx, 4*time.Second) if source, ok := askDuckDuckGoSearchAssist(stepCtx, question); ok { cancel() return source, true @@ -257,7 +257,7 @@ func (p *Ask) findSource(ctx context.Context, question string, cfg bot.PluginCon if cfg.Bool("search_assist_browser_enabled", true) { fetchResults := cfg.Bool("search_results_enabled", true) webResultTried = fetchResults - stepCtx, cancel = askStepContext(ctx, 5*time.Second) + stepCtx, cancel = askBrowserStepContext(ctx) if source, ok := askDuckDuckGoRenderedSearchAssist(stepCtx, question, cfg.String("browser_path", ""), fetchResults); ok { cancel() if source.Provider == "search_result" { @@ -271,7 +271,7 @@ func (p *Ask) findSource(ctx context.Context, question string, cfg bot.PluginCon } } } - if cfg.Bool("duckduckgo_enabled", true) { + if cfg.Bool("duckduckgo_enabled", true) && !askNeedsWebResultAnswer(question) { stepCtx, cancel := askStepContext(ctx, 1600*time.Millisecond) if source, ok := askDuckDuckGoWithRetry(stepCtx, question); ok { if refined, ok := refineFocusedAskSource(question, source); ok { @@ -311,7 +311,7 @@ func (p *Ask) findSource(ctx context.Context, question string, cfg bot.PluginCon } cancel() } - if cfg.Bool("wikidata_fallback", true) && focused != "" { + if cfg.Bool("wikidata_fallback", true) && focused != "" && !askNeedsWebResultAnswer(question) { stepCtx, cancel := askStepContext(ctx, 2500*time.Millisecond) switch { case askNeedsRelationshipAnswer(question): @@ -364,6 +364,23 @@ func askStepContext(parent context.Context, max time.Duration) (context.Context, return context.WithTimeout(parent, max) } +// Reserve a small, fixed tail for public search. Halving every stage's +// remaining time gave Chromium too little time to start and render an answer. +func askBrowserStepContext(parent context.Context) (context.Context, context.CancelFunc) { + budget := 10 * time.Second + if deadline, ok := parent.Deadline(); ok { + remaining := time.Until(deadline) + reserve := 2 * time.Second + if remaining/4 < reserve { + reserve = remaining / 4 + } + if remaining-reserve < budget { + budget = remaining - reserve + } + } + return context.WithTimeout(parent, budget) +} + func askLocalAnswer(question string) (string, bool) { normalized := strings.ToLower(strings.TrimSpace(question)) normalized = strings.Trim(normalized, " \t\r\n?%!.") @@ -472,9 +489,13 @@ func askNeedsTemporalAnswer(question string) bool { func askNeedsWebResultAnswer(question string) bool { lower := strings.ToLower(strings.TrimSpace(question)) + if askComparisonPattern.MatchString(lower) { + return true + } for _, phrase := range []string{ "why ", "why is ", "why are ", "why was ", "why were ", "how does ", "how do ", "how can ", "how should ", + "how much ", "what does it cost", "what is the price", "what are the prices", "what makes ", "what do people think", "why do people ", "why do some ", "what are the disadvantages", "what are the benefits", "what are the alternatives", "is it true", "is it worth", "is it safe", "should i ", "should we ", @@ -657,9 +678,9 @@ type duckDuckGoSearchAssistPayload struct { } `json:"instantAnswers"` } -var askSearchAssistScriptPattern = regexp.MustCompile(`(?s)]*id=["']deep_preload_script["'][^>]*src=["']([^"']+)["']`) var askSearchAssistOpinionPattern = regexp.MustCompile(`(?i)^why\s+should\s+(?:someone|somebody|people|users?|we|you|they)\s+not\s+use\s+(.+?)\s*[?!.]*$`) var askSearchAssistGenrePattern = regexp.MustCompile(`(?i)^what\s+(?:(?:music|musical)\s+)?genre\s+is\s+(?:the\s+)?(?:band|artist|group)\s+(.+?)\s*[?!.]*$`) +var askComparisonPattern = regexp.MustCompile(`(?i)\b(?:vs\.?|versus|compare|comparison|compared)\b|\bdifference\s+between\b`) type askRenderedSearchAssistData struct { Text string `json:"text"` @@ -674,19 +695,36 @@ type askRenderedSearchResult struct { const askRenderedSearchAssistScript = `() => { const textOf = (element) => (element.innerText || element.textContent || "").trim(); - const headers = Array.from(document.querySelectorAll("body *")).filter((element) => { - const text = textOf(element); - return text === "Search Assist" || text.startsWith("Search Assist\n"); - }); - if (!headers.length) return null; - let card = headers.sort((a, b) => textOf(a).length - textOf(b).length)[0]; - while (card.parentElement && textOf(card.parentElement).length < 3500) card = card.parentElement; - const links = Array.from(card.querySelectorAll("a[href]")) - .map((anchor) => anchor.href) - .filter((href) => /^https?:\/\//i.test(href)); - const clone = card.cloneNode(true); - clone.querySelectorAll("a, button, [role=button]").forEach((element) => element.remove()); - return {text: textOf(clone), links}; + const headers = Array.from(document.querySelectorAll("body *")) + .filter((element) => textOf(element) === "Search Assist" && + !Array.from(element.children).some(child => textOf(child) === "Search Assist")); + for (const header of headers) { + let card = header.parentElement; + for (let depth = 0; card && depth < 8; depth++, card = card.parentElement) { + // Never collect the surrounding search results or a navigation shell. + if (card === document.body || card.querySelector('[data-testid="result-title-a"], .result__a')) break; + const links = Array.from(card.querySelectorAll("a[href]")).map(a => a.href).filter(href => { + try { + const u = new URL(href); + return u.protocol === "https:" && u.hostname !== "duckduckgo.com" && !u.hostname.endsWith(".duckduckgo.com"); + } catch (_) { return false; } + }); + if (!links.length) continue; + const clone = card.cloneNode(true); + clone.querySelectorAll("a, button, [role=button], script, style, svg").forEach(element => element.remove()); + clone.querySelectorAll("*").forEach(element => { + if (element.children.length === 0 && textOf(element) === "Search Assist") element.remove(); + }); + clone.querySelectorAll("p, div, li, h1, h2, h3, tr, br").forEach(element => element.append("\n")); + const text = textOf(clone).split("\n").map(line => line.trim()).filter(line => + line && !/^(Search Assist|More|Generating[.…]*|Searching[.…]*)$/i.test(line) && + !/^(Auto-generated based on|Was this helpful)/i.test(line)).join("\n"); + // A heading, loading state, or empty card is not a completed answer. + if (!text) continue; + return {text: text.slice(0, 12000), links: links.slice(0, 8)}; + } + } + return null; }` // DuckDuckGo changes its result-card markup periodically. The selector list @@ -769,9 +807,28 @@ func askSearchAssistQueryVariants(question string) []string { queries = append(queries, "what genres does "+subject+" have?") } } + if len(queries) == 1 && askComparisonPattern.MatchString(original) { + // Preserve the question and both subjects. Normalize word boundaries + // and the common comparison abbreviation without inventing a new query. + var expanded strings.Builder + var previous rune + for _, r := range original { + if unicode.IsUpper(r) && unicode.IsLower(previous) { + expanded.WriteByte(' ') + } + expanded.WriteRune(r) + previous = r + } + normalized := askVersusPattern.ReplaceAllString(strings.ToLower(expanded.String()), "versus") + if normalized != strings.ToLower(original) { + queries = append(queries, normalized) + } + } return queries } +var askVersusPattern = regexp.MustCompile(`(?i)\bvs\b\.?`) + func askDuckDuckGoSearchAssistOnce(ctx context.Context, question, fallbackURL string) (askSource, bool) { pageURL := duckDuckGoSearchURL(question) req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil) @@ -793,11 +850,7 @@ func askDuckDuckGoSearchAssistOnce(ctx context.Context, question, fallbackURL st if err != nil { return askSource{}, false } - matches := askSearchAssistScriptPattern.FindSubmatch(page) - if len(matches) < 2 { - return askSource{}, false - } - assistURL := html.UnescapeString(string(matches[1])) + assistURL := askSearchAssistPreloadURL(page, pageURL) assistParsed, parseErr := url.Parse(assistURL) if parseErr != nil || assistParsed.Scheme != "https" || !isDuckDuckGoHost(assistParsed.Hostname()) || assistParsed.User != nil || assistParsed.Port() != "" { return askSource{}, false @@ -825,6 +878,42 @@ func askDuckDuckGoSearchAssistOnce(ctx context.Context, question, fallbackURL st return parseDuckDuckGoSearchAssist(string(assistBody), fallbackURL) } +func askSearchAssistPreloadURL(page []byte, pageURL string) string { + base, err := url.Parse(pageURL) + if err != nil { + return "" + } + tokens := xhtml.NewTokenizer(bytes.NewReader(page)) + for { + switch tokens.Next() { + case xhtml.ErrorToken: + return "" + case xhtml.StartTagToken, xhtml.SelfClosingTagToken: + token := tokens.Token() + if token.Data != "script" { + continue + } + var id, src string + for _, attr := range token.Attr { + if attr.Key == "id" { + id = attr.Val + } + if attr.Key == "src" { + src = attr.Val + } + } + if id != "deep_preload_script" || src == "" { + continue + } + ref, err := url.Parse(src) + if err != nil { + return "" + } + return base.ResolveReference(ref).String() + } + } +} + func askDuckDuckGoRenderedSearchAssist(ctx context.Context, question, browserPath string, fetchResults bool) (askSource, bool) { executable := strings.TrimSpace(browserPath) if executable != "" { @@ -842,6 +931,14 @@ func askDuckDuckGoRenderedSearchAssist(ctx context.Context, question, browserPat return askSource{}, false } } + // Chromium is expensive on small VPS hosts. Requests from multiple IRC + // networks must share one bounded browser slot, including shutdown. + select { + case askBrowserSlots <- struct{}{}: + defer func() { <-askBrowserSlots }() + case <-ctx.Done(): + return askSource{}, false + } queries := askSearchAssistQueryVariants(question) for index := 0; index < 2; index++ { @@ -859,8 +956,10 @@ func askDuckDuckGoRenderedSearchAssist(ctx context.Context, question, browserPat return askSource{}, false } +var askBrowserSlots = make(chan struct{}, 1) + func askDuckDuckGoRenderedSearchAssistOnce(ctx context.Context, question, fallbackURL, executable string, fetchResults bool) (askSource, bool) { - browserTimeout := 4 * time.Second + browserTimeout := 10 * time.Second if deadline, ok := ctx.Deadline(); ok { remaining := time.Until(deadline) if remaining <= 0 { @@ -890,12 +989,15 @@ func askDuckDuckGoRenderedSearchAssistOnce(ctx context.Context, question, fallba if err := chromedp.Run(browserCtx, chromedp.Navigate(duckDuckGoSearchURL(question))); err != nil { return askSource{}, false } - assistTimeout := 2 * time.Second + // Navigation and browser startup have already consumed some of the budget. + // Wait for an actual answer, not merely the Search Assist header. + deadline, _ := runCtx.Deadline() + assistTimeout := time.Until(deadline) if fetchResults { - assistTimeout = 1500 * time.Millisecond + assistTimeout -= 750 * time.Millisecond } - if browserTimeout < assistTimeout { - assistTimeout = browserTimeout + if assistTimeout <= 0 { + return askSource{}, false } if err := chromedp.Run(browserCtx, chromedp.PollFunction(askRenderedSearchAssistScript, &result, chromedp.WithPollingInterval(100*time.Millisecond), @@ -914,7 +1016,7 @@ func askDuckDuckGoRenderedSearchAssistOnce(ctx context.Context, question, fallba return askSource{}, false } var results []askRenderedSearchResult - resultTimeout := browserTimeout - assistTimeout + resultTimeout := time.Until(deadline) if resultTimeout <= 0 { return askSource{}, false } @@ -1306,19 +1408,12 @@ func parseRenderedSearchAssist(result askRenderedSearchAssistData, fallbackURL s } func parseDuckDuckGoSearchAssist(body, fallbackURL string) (askSource, bool) { - const prefix = "DDG.deep.deepPayload = " - const suffix = ";DDG.deep.bn=" - start := strings.Index(body, prefix) - if start < 0 { - return askSource{}, false - } - start += len(prefix) - end := strings.Index(body[start:], suffix) - if end < 0 { + assignment := askSearchAssistPayloadPattern.FindStringIndex(body) + if assignment == nil { return askSource{}, false } var payload duckDuckGoSearchAssistPayload - if err := json.Unmarshal([]byte(body[start:start+end]), &payload); err != nil { + if err := json.NewDecoder(strings.NewReader(body[assignment[1]:])).Decode(&payload); err != nil { return askSource{}, false } for _, instantAnswer := range payload.InstantAnswers { @@ -1338,6 +1433,8 @@ func parseDuckDuckGoSearchAssist(body, fallbackURL string) (askSource, bool) { return askSource{}, false } +var askSearchAssistPayloadPattern = regexp.MustCompile(`\bDDG\.deep\.deepPayload\s*=\s*`) + func askDuckDuckGo(ctx context.Context, question string) (askSource, bool) { endpoint := "https://api.duckduckgo.com/?q=" + url.QueryEscape(question) + "&format=json&no_html=1&skip_disambig=1" req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) diff --git a/plugins/ask_resilience_test.go b/plugins/ask_resilience_test.go new file mode 100644 index 0000000..b2b38e2 --- /dev/null +++ b/plugins/ask_resilience_test.go @@ -0,0 +1,167 @@ +package plugins + +import ( + "context" + "net/http" + "os" + "reflect" + "strings" + "testing" + "time" + + "github.com/chromedp/chromedp" + "github.com/variablenix/GoBot/bot" +) + +func TestAskCostComparisonsNeedWebAnswers(t *testing.T) { + for _, question := range []string{ + "how much do dirt bikes cost vs GoKarts?", "how much does a piano cost?", + "compare buses and trains", "electric cars versus petrol cars", + "What is the difference between Linux and BSD?", "price of bikes vs. karts", + } { + if !askNeedsWebResultAnswer(question) { + t.Errorf("not classified as web question: %q", question) + } + } + for _, question := range []string{"what is Linux?", "who created Arch Linux?", "when was Debian released?"} { + if askNeedsWebResultAnswer(question) { + t.Errorf("entity question misclassified: %q", question) + } + } + got := askSearchAssistQueryVariants("how much do dirt bikes cost vs GoKarts?") + want := []string{"how much do dirt bikes cost vs GoKarts?", "how much do dirt bikes cost versus go karts?"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("variants=%q", got) + } +} + +func TestSearchAssistPreloadMarkupVariants(t *testing.T) { + for _, markup := range []string{ + ``, + "", + } { + if got := askSearchAssistPreloadURL([]byte(markup), "https://duckduckgo.com/?q=test"); got != "https://duckduckgo.com/assist.js?q=a&x=b" { + t.Fatalf("preload=%q", got) + } + } + if got := askSearchAssistPreloadURL([]byte(``), "https://duckduckgo.com/"); got != "" { + t.Fatal("unrelated script accepted") + } +} + +func TestSearchAssistPayloadWhitespaceAndTrailingCode(t *testing.T) { + for _, body := range []string{ + `DDG.deep.deepPayload={"instantAnswers":[{"data":{"answer":"A sourced comparison.","sources":[]}}]};differentCallback();`, + "DDG.deep.deepPayload\n=\n{\"instantAnswers\":[{\"data\":{\"answer\":\"A sourced comparison.\"}}]};", + } { + got, ok := parseDuckDuckGoSearchAssist(body, "https://duckduckgo.com/?q=test") + if !ok || got.Summary != "A sourced comparison." { + t.Fatalf("%+v %v", got, ok) + } + } + if _, ok := parseDuckDuckGoSearchAssist(`DDG.deep.deepPayload={broken`, ""); ok { + t.Fatal("malformed JSON accepted") + } +} + +func TestAskComparisonRecoversOnNormalizedRetry(t *testing.T) { + old := askHTTPClient + t.Cleanup(func() { askHTTPClient = old }) + var queries []string + askHTTPClient = &http.Client{Transport: newPluginRoundTripper(func(r *http.Request) (*http.Response, error) { + if r.URL.Path == "/assist.js" { + return newPluginResponse(200, `DDG.deep.deepPayload={"instantAnswers":[{"data":{"answer":"Compare equivalent dirt bikes and go-karts, including maintenance costs.","sources":[{"article":{"link":"https://example.org/comparison"}}]}}]};callback();`), nil + } + queries = append(queries, r.URL.Query().Get("q")) + if len(queries) == 1 { + return newPluginResponse(200, `No answer yet`), nil + } + return newPluginResponse(200, ``), nil + })} + got, ok := (&Ask{}).findSource(t.Context(), "how much do dirt bikes cost vs GoKarts?", bot.PluginConfig{}) + if !ok || got.Provider != "search_assist" || got.URL != "https://example.org/comparison" { + t.Fatalf("retry failed: %+v %v", got, ok) + } + if !reflect.DeepEqual(queries, []string{"how much do dirt bikes cost vs GoKarts?", "how much do dirt bikes cost versus go karts?"}) { + t.Fatalf("unexpected requests: %q", queries) + } +} + +func TestAskComparisonDoesNotReturnEntityDescription(t *testing.T) { + old := askHTTPClient + t.Cleanup(func() { askHTTPClient = old }) + askHTTPClient = &http.Client{Transport: newPluginRoundTripper(func(r *http.Request) (*http.Response, error) { + t.Fatalf("entity provider should not be called for a cost comparison: %s", r.URL.Host) + return nil, context.Canceled + })} + cfg := bot.PluginConfig{"search_assist_enabled": false, "search_assist_browser_enabled": false, "search_results_enabled": false} + if _, ok := (&Ask{}).findSource(t.Context(), "how much do dirt bikes cost vs GoKarts?", cfg); ok { + t.Fatal("generic answer substituted for comparison") + } +} + +func TestAskBrowserBudgetLeavesFallbackTime(t *testing.T) { + parent, cancel := context.WithTimeout(t.Context(), 8*time.Second) + defer cancel() + ctx, done := askBrowserStepContext(parent) + defer done() + deadline, _ := ctx.Deadline() + remaining := time.Until(deadline) + if remaining < 5*time.Second || remaining > 6*time.Second { + t.Fatalf("browser budget=%s", remaining) + } +} + +// Run with GOBOT_TEST_BROWSER=/path/to/chromium. This fixture runs entirely +// offline and verifies the DOM code against a delayed, expanded answer card. +func TestAskRenderedCardWaitsForAnswer(t *testing.T) { + executable := os.Getenv("GOBOT_TEST_BROWSER") + if executable == "" { + t.Skip("set GOBOT_TEST_BROWSER for offline browser fixture") + } + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + allocator, stop := chromedp.NewExecAllocator(ctx, append(chromedp.DefaultExecAllocatorOptions[:], chromedp.ExecPath(executable))...) + defer stop() + page, closePage := chromedp.NewContext(allocator) + defer closePage() + var before *askRenderedSearchAssistData + var answer askRenderedSearchAssistData + const fixture = `document.body.innerHTML='
Search Assist

'` + if err := chromedp.Run(page, chromedp.Navigate("about:blank"), chromedp.Evaluate(fixture, nil), chromedp.Evaluate("("+askRenderedSearchAssistScript+")()", &before)); err != nil { + t.Fatal(err) + } + if before != nil { + t.Fatalf("empty card accepted: %+v", before) + } + const delayed = `setTimeout(() => {document.getElementById('answer').textContent='Dirt bikes and go-karts have different purchase and ownership costs. Compare equivalent models before deciding.';document.getElementById('sources').innerHTML='Source';const details=document.createElement('div');details.textContent='Additional comparison details. '.repeat(180);document.getElementById('card').append(details);}, 1800)` + if err := chromedp.Run(page, chromedp.Evaluate(delayed, nil), chromedp.PollFunction(askRenderedSearchAssistScript, &answer, chromedp.WithPollingInterval(100*time.Millisecond), chromedp.WithPollingTimeout(4*time.Second))); err != nil { + t.Fatal(err) + } + if !strings.Contains(answer.Text, "purchase and ownership") || strings.Contains(answer.Text, "Unrelated") || len(answer.Links) != 1 { + t.Fatalf("wrong card: %+v", answer) + } + // A short, cited answer is valid too: do not impose a prose-length floor. + if err := chromedp.Run(page, chromedp.Evaluate(`document.getElementById('card').innerHTML='
Search Assist

2002

Source'`, nil), chromedp.Evaluate("("+askRenderedSearchAssistScript+")()", &answer)); err != nil { + t.Fatal(err) + } + if answer.Text != "2002" { + t.Fatalf("short answer lost: %+v", answer) + } +} + +func TestLiveAskCostComparison(t *testing.T) { + if os.Getenv("GOBOT_LIVE_LOOKUPS") != "1" { + t.Skip("opt-in live provider smoke test") + } + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + source, ok := (&Ask{}).findSource(ctx, "how much do dirt bikes cost vs GoKarts?", bot.PluginConfig{}) + if !ok { + t.Fatal("no comparison answer") + } + t.Logf("provider=%s source=%s answer=%s", source.Provider, source.URL, source.Summary) + if source.Provider != "search_assist" && source.Provider != "search_result" { + t.Fatalf("wrong provider for comparison: %s", source.Provider) + } +}