diff --git a/config.yaml b/config.yaml index 820e481..c2f2c88 100644 --- a/config.yaml +++ b/config.yaml @@ -104,7 +104,7 @@ plugins: wikidata_fallback: true max_length: 360 max_response_chars: 240 - timeout_seconds: 8 + timeout_seconds: 15 cooldown_seconds: 15 cache_seconds: 300 wikipedia: {enabled: true, max_summary_length: 300} diff --git a/docs/configuration.md b/docs/configuration.md index cec732a..ef912b8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -133,7 +133,7 @@ plugins: wikidata_fallback: true max_length: 360 max_response_chars: 240 - timeout_seconds: 8 + timeout_seconds: 15 cooldown_seconds: 15 cache_seconds: 300 grab: {enabled: true, max_length: 320, max_quotes_per_user: 20} @@ -194,7 +194,21 @@ source-grounded, sanitized, one-line, and bounded by `max_length` and in memory for `cache_seconds` (default five minutes), keyed by normalized question text, so repeated questions do not depend on a second provider race. The cache is in memory and is cleared when GoBot restarts. No credentials are required. -`define` uses the public English dictionary service, `reddit` uses Reddit's public post and +The default overall `ask` timeout is 15 seconds (maximum 30). Each provider +reserves time for later fallbacks. Existing configurations explicitly using 8 +seconds keep that limit; increasing it to 15–20 seconds gives the browser and +fallbacks more time. Chromium must be installed separately for browser lookup +(Debian: `sudo apt-get install chromium`). A browser does not bypass provider +challenges or guarantee a Search Assist answer. Entity questions can also fall +back to a matching Wikipedia summary; procedural and opinion questions cannot +use an unrelated encyclopedia summary as an answer. + +`define` uses the public English dictionary service with an independent English +Wiktionary fallback. The primary request is capped at two seconds or one third +of the remaining deadline, leaving time for fallback. Wiktionary responses link +to the source entry and strip HTML. An unavailable lookup may mean missing +coverage or a provider failure, rather than proof that a word does not exist. +`reddit` uses Reddit's public post and subreddit JSON endpoints with an RSS fallback, and `horoscope` uses a public daily horoscope API. `foods` and `sports` use local text files only. `fun` uses local operator-editable text catalogs under `data/fun`, and `weapons` uses the diff --git a/docs/plugins.md b/docs/plugins.md index ae5d467..f1ce96b 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -836,7 +836,8 @@ message/command counters. It is useful for a quick IRC-side health check. ## Definitions -Look up one English word using the public Free Dictionary API: +Look up one English word using the public Free Dictionary API, with an English +Wiktionary fallback when the primary service is unreachable or lacks the entry: ~~~text !define resilient @@ -848,6 +849,11 @@ GoBot returns the first concise definition and part of speech in one bounded message. No API key is required. The lookup is English-only and third-party text is cleaned before it is sent to IRC. +Proper nouns such as `!define Linux` can be found through Wiktionary. Fallback +answers include the entry link. Coverage is not universal; failure messages +distinguish an unavailable lookup from a definitive statement that no word +exists. Use `!wiki Linux` for an encyclopedia summary. + ## Calculator and unit conversion Calculator expressions are evaluated locally with a small parser; GoBot never @@ -1354,7 +1360,7 @@ plugins: wikidata_fallback: true max_length: 360 max_response_chars: 240 - timeout_seconds: 8 + timeout_seconds: 15 cooldown_seconds: 15 cache_seconds: 300 ~~~ diff --git a/plugins/ask.go b/plugins/ask.go index 45f87f5..9a21e72 100644 --- a/plugins/ask.go +++ b/plugins/ask.go @@ -136,7 +136,7 @@ func (p *Ask) Handle(b *bot.Bot, m bot.Message) bool { return true } - timeout := cfg.Int("timeout_seconds", 8) + timeout := cfg.Int("timeout_seconds", 15) if timeout < 1 { timeout = 1 } @@ -257,7 +257,7 @@ func (p *Ask) findSource(ctx context.Context, question string, cfg bot.PluginCon if cfg.Bool("search_assist_browser_enabled", true) { fetchResults := cfg.Bool("search_results_enabled", true) webResultTried = fetchResults - stepCtx, cancel = askStepContext(ctx, 3200*time.Millisecond) + stepCtx, cancel = askStepContext(ctx, 5*time.Second) if source, ok := askDuckDuckGoRenderedSearchAssist(stepCtx, question, cfg.String("browser_path", ""), fetchResults); ok { cancel() if source.Provider == "search_result" { @@ -343,10 +343,21 @@ func (p *Ask) findSource(ctx context.Context, question string, cfg bot.PluginCon } cancel() } + // An encyclopedia summary is useful only for entity questions, not as a + // substitute for an opinion, procedure, relationship, or date answer. + if cfg.Bool("wikidata_fallback", true) && focused != "" && !askNeedsWebResultAnswer(question) && !askNeedsRelationshipAnswer(question) && !askNeedsTemporalAnswer(question) { + if article, ok := wikipediaSummary(ctx, focused); ok { + return askSource{Title: article.Title, Summary: cleanExternalText(article.Extract), URL: article.ContentURLs.Desktop.Page, Provider: "wikipedia"}, true + } + } return askSource{}, false } func askStepContext(parent context.Context, max time.Duration) (context.Context, context.CancelFunc) { + // Leave part of the remaining budget for later independent providers. + if deadline, ok := parent.Deadline(); ok && time.Until(deadline)/2 < max { + max = time.Until(deadline) / 2 + } if max <= 0 { return context.WithCancel(parent) } @@ -787,7 +798,8 @@ func askDuckDuckGoSearchAssistOnce(ctx context.Context, question, fallbackURL st return askSource{}, false } assistURL := html.UnescapeString(string(matches[1])) - if !validHTTPURL(assistURL) { + assistParsed, parseErr := url.Parse(assistURL) + if parseErr != nil || assistParsed.Scheme != "https" || !isDuckDuckGoHost(assistParsed.Hostname()) || assistParsed.User != nil || assistParsed.Port() != "" { return askSource{}, false } assistReq, err := http.NewRequestWithContext(ctx, http.MethodGet, assistURL, nil) @@ -1809,7 +1821,10 @@ func newAskWebHTTPClient() *http.Client { return &http.Client{ Timeout: 6 * time.Second, Transport: transport, - CheckRedirect: func(req *http.Request, _ []*http.Request) error { + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return fmt.Errorf("too many redirects") + } if !validPublicHTTPURL(req.URL.String()) { return fmt.Errorf("refusing non-public redirect") } diff --git a/plugins/define.go b/plugins/define.go index 41d198c..d751bc0 100644 --- a/plugins/define.go +++ b/plugins/define.go @@ -10,6 +10,8 @@ import ( "strings" "time" + xhtml "golang.org/x/net/html" + "github.com/variablenix/GoBot/bot" "github.com/variablenix/GoBot/storage" ) @@ -36,21 +38,25 @@ func (p *Define) Handle(b *bot.Bot, m bot.Message) bool { ctx, cancel := context.WithTimeout(context.Background(), definitionTimeout(p.cfg)) defer cancel() - entry, ok := dictionaryEntry(ctx, term) + entry, ok := lookupDefinition(ctx, term) if !ok { - b.Send(m.ReplyTarget(), ircColor(ircRed, "no English definition found")) + b.Send(m.ReplyTarget(), ircColor(ircRed, "No definition available: the word may be missing or the dictionary services unavailable. Try !wiki "+cleanExternalText(term))) return true } maxLength := p.cfg.Int("max_length", 240) - if maxLength < 80 { - maxLength = 80 + if maxLength < 80 || maxLength > 400 { + maxLength = 240 } definition := truncateRunes(cleanExternalText(entry.Definition), maxLength) part := cleanExternalText(entry.PartOfSpeech) if part != "" { part = " (" + part + ")" } - b.Send(m.ReplyTarget(), fmt.Sprintf("📖 %s%s: %s", cleanExternalText(entry.Word), part, definition)) + message := fmt.Sprintf("📖 %s%s: %s", cleanExternalText(entry.Word), part, definition) + if entry.URL != "" { + message += " — " + entry.URL + } + b.Send(m.ReplyTarget(), message) return true } @@ -58,6 +64,60 @@ type dictionaryEntryResult struct { Word string PartOfSpeech string Definition string + URL string +} + +// Reserve time for an independent provider when the primary is unreachable. +func lookupDefinition(ctx context.Context, term string) (dictionaryEntryResult, bool) { + budget := 2 * time.Second + if deadline, ok := ctx.Deadline(); ok && time.Until(deadline)/3 < budget { + budget = time.Until(deadline) / 3 + } + primary, cancel := context.WithTimeout(ctx, budget) + entry, ok := dictionaryEntry(primary, term) + cancel() + if ok { + return entry, true + } + return wiktionaryEntry(ctx, term) +} + +func wiktionaryEntry(ctx context.Context, term string) (dictionaryEntryResult, bool) { + endpoint := "https://en.wiktionary.org/api/rest_v1/page/definition/" + url.PathEscape(term) + req, err := wikipediaRequest(ctx, endpoint) + if err != nil { + return dictionaryEntryResult{}, false + } + res, err := apiHTTPClient.Do(req) + if err != nil { + return dictionaryEntryResult{}, false + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return dictionaryEntryResult{}, false + } + var entries map[string][]struct { + PartOfSpeech string `json:"partOfSpeech"` + Definitions []struct { + Definition string `json:"definition"` + } `json:"definitions"` + } + if json.NewDecoder(io.LimitReader(res.Body, 512*1024)).Decode(&entries) != nil { + return dictionaryEntryResult{}, false + } + for _, entry := range entries["en"] { + for _, definition := range entry.Definitions { + node, err := xhtml.Parse(strings.NewReader(definition.Definition)) + if err != nil { + continue + } + text := cleanExternalText(askHTMLText(node)) + if text != "" { + return dictionaryEntryResult{Word: term, PartOfSpeech: entry.PartOfSpeech, Definition: text, URL: "https://en.wiktionary.org/wiki/" + url.PathEscape(term) + "#English"}, true + } + } + } + return dictionaryEntryResult{}, false } func dictionaryEntry(ctx context.Context, term string) (dictionaryEntryResult, bool) { diff --git a/plugins/lookup_resilience_test.go b/plugins/lookup_resilience_test.go new file mode 100644 index 0000000..182027f --- /dev/null +++ b/plugins/lookup_resilience_test.go @@ -0,0 +1,170 @@ +package plugins + +import ( + "context" + "fmt" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/variablenix/GoBot/bot" + "go.uber.org/zap" +) + +func TestDefinitionFallback(t *testing.T) { + old := apiHTTPClient + t.Cleanup(func() { apiHTTPClient = old }) + for _, status := range []int{http.StatusNotFound, http.StatusServiceUnavailable, http.StatusTooManyRequests} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + apiHTTPClient = &http.Client{Transport: newPluginRoundTripper(func(r *http.Request) (*http.Response, error) { + if r.URL.Host == "api.dictionaryapi.dev" { + return newPluginResponse(status, `{}`), nil + } + if r.URL.EscapedPath() != "/api/rest_v1/page/definition/Linux" { + t.Fatalf("unexpected path %s", r.URL.EscapedPath()) + } + return newPluginResponse(200, `{"fr":[{"definitions":[{"definition":"wrong language"}]}],"en":[{"partOfSpeech":"Proper noun","definitions":[{"definition":""},{"definition":"A Unix-like operating system."}]}]}`), nil + })} + got, ok := lookupDefinition(t.Context(), "Linux") + if !ok || got.Definition != "A Unix-like operating system." || !strings.HasSuffix(got.URL, "Linux#English") { + t.Fatalf("%+v %v", got, ok) + } + }) + } +} + +func TestDefinitionTimeoutLeavesFallbackBudget(t *testing.T) { + old := apiHTTPClient + t.Cleanup(func() { apiHTTPClient = old }) + apiHTTPClient = &http.Client{Transport: newPluginRoundTripper(func(r *http.Request) (*http.Response, error) { + if r.URL.Host == "api.dictionaryapi.dev" { + <-r.Context().Done() + return nil, r.Context().Err() + } + if r.Context().Err() != nil { + t.Fatal("fallback inherited expired primary context") + } + return newPluginResponse(200, `{"en":[{"definitions":[{"definition":"A greeting."}]}]}`), nil + })} + ctx, cancel := context.WithTimeout(t.Context(), 300*time.Millisecond) + defer cancel() + if _, ok := lookupDefinition(ctx, "hello"); !ok { + t.Fatal("timeout prevented fallback") + } +} + +func TestDefinitionKeepsPrimarySuccess(t *testing.T) { + old := apiHTTPClient + t.Cleanup(func() { apiHTTPClient = old }) + apiHTTPClient = &http.Client{Transport: newPluginRoundTripper(func(r *http.Request) (*http.Response, error) { + if r.URL.Host != "api.dictionaryapi.dev" { + t.Fatal("unnecessary fallback") + } + return newPluginResponse(200, `[{"word":"hello","meanings":[{"partOfSpeech":"interjection","definitions":[{"definition":"A greeting."}]}]}]`), nil + })} + got, ok := lookupDefinition(t.Context(), "hello") + if !ok || got.Definition != "A greeting." || got.URL != "" { + t.Fatalf("%+v %v", got, ok) + } +} + +func TestWiktionaryRejectsInvalidPayload(t *testing.T) { + old := apiHTTPClient + t.Cleanup(func() { apiHTTPClient = old }) + for _, body := range []string{`not json`, `{}`, `{"fr":[{"definitions":[{"definition":"bonjour"}]}]}`, `{"en":[{"definitions":[{"definition":""}]}]}`} { + apiHTTPClient = &http.Client{Transport: newPluginRoundTripper(func(r *http.Request) (*http.Response, error) { return newPluginResponse(200, body), nil })} + if _, ok := wiktionaryEntry(t.Context(), "hello"); ok { + t.Fatalf("accepted %s", body) + } + } +} + +func TestSearchAssistRejectsForeignPreload(t *testing.T) { + old := askHTTPClient + t.Cleanup(func() { askHTTPClient = old }) + calls := 0 + askHTTPClient = &http.Client{Transport: newPluginRoundTripper(func(r *http.Request) (*http.Response, error) { + calls++ + return newPluginResponse(200, ``), nil + })} + if _, ok := askDuckDuckGoSearchAssistOnce(t.Context(), "hello", "https://duckduckgo.com/"); ok || calls != 1 { + t.Fatal("untrusted preload was followed") + } +} + +func TestAskStepReservesTime(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + step, done := askStepContext(ctx, 5*time.Second) + defer done() + deadline, _ := step.Deadline() + if time.Until(deadline) > 600*time.Millisecond { + t.Fatal("step can exhaust entire parent budget") + } +} + +func TestWikipediaNegativeLengthDoesNotPanic(t *testing.T) { + old := apiHTTPClient + t.Cleanup(func() { apiHTTPClient = old }) + apiHTTPClient = &http.Client{Transport: newPluginRoundTripper(func(r *http.Request) (*http.Response, error) { + return newPluginResponse(200, `{"title":"Linux","extract":"Linux is an operating system.","content_urls":{"desktop":{"page":"https://en.wikipedia.org/wiki/Linux"}}}`), nil + })} + p := &Wikipedia{} + p.Init(bot.PluginConfig{"max_summary_length": -1}, nil) + b := bot.New(bot.Config{CommandPrefix: "!"}, nil, nil, zap.NewNop()) + if !p.Handle(b, bot.Message{Text: "!wiki Linux", Nick: "tester", Target: "#test", IsChannel: true}) { + t.Fatal("command not handled") + } +} + +func TestAskWikipediaFallback(t *testing.T) { + oldAPI, oldAsk := apiHTTPClient, askHTTPClient + t.Cleanup(func() { apiHTTPClient, askHTTPClient = oldAPI, oldAsk }) + askHTTPClient = &http.Client{Transport: newPluginRoundTripper(func(r *http.Request) (*http.Response, error) { return newPluginResponse(503, `{}`), nil })} + calls := 0 + apiHTTPClient = &http.Client{Transport: newPluginRoundTripper(func(r *http.Request) (*http.Response, error) { + calls++ + return newPluginResponse(200, `{"title":"Linux","extract":"Linux is an operating system.","content_urls":{"desktop":{"page":"https://en.wikipedia.org/wiki/Linux"}}}`), nil + })} + cfg := bot.PluginConfig{"search_assist_enabled": false, "search_assist_browser_enabled": false, "search_results_enabled": false, "duckduckgo_enabled": false} + got, ok := (&Ask{}).findSource(t.Context(), "what is Linux?", cfg) + if !ok || got.Provider != "wikipedia" { + t.Fatalf("unexpected fallback: %+v, %v", got, ok) + } + calls = 0 + if _, ok := (&Ask{}).findSource(t.Context(), "why is Linux popular?", cfg); ok || calls != 0 { + t.Fatal("entity summary used for opinion question") + } +} + +// Explicitly opt in on the deployment host. Normal CI is deterministic and +// does not depend on third-party availability or rate limits. +func TestLiveLookupProviders(t *testing.T) { + if os.Getenv("GOBOT_LIVE_LOOKUPS") != "1" { + t.Skip("opt-in live provider smoke test") + } + for _, term := range []string{"Linux", "hello", "resilient"} { + t.Run("define/"+term, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second) + defer cancel() + result, ok := lookupDefinition(ctx, term) + if !ok { + t.Fatal("no definition") + } + t.Logf("%s: %s", term, result.Definition) + }) + } + for _, q := range []string{"what is Linux?", "who created Arch Linux?", "when was Debian first released?", "why is the sky blue?"} { + t.Run("ask/"+q, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second) + defer cancel() + result, ok := (&Ask{}).findSource(ctx, q, bot.PluginConfig{}) + if !ok { + t.Fatal("no answer") + } + t.Logf("provider=%s answer=%s", result.Provider, result.Summary) + }) + } +} diff --git a/plugins/wikipedia.go b/plugins/wikipedia.go index d47c354..2cb22c5 100644 --- a/plugins/wikipedia.go +++ b/plugins/wikipedia.go @@ -36,13 +36,16 @@ func (p *Wikipedia) Handle(b *bot.Bot, m bot.Message) bool { b.Send(m.ReplyTarget(), "I couldn't find that Wikipedia article.") return true } - summary := strings.Join(strings.Fields(x.Extract), " ") + summary := cleanExternalText(x.Extract) if summary == "" { b.Send(m.ReplyTarget(), "I couldn't find a summary for that Wikipedia article.") return true } r := []rune(summary) max := p.cfg.Int("max_summary_length", 300) + if max < 1 || max > 1000 { + max = 300 + } if len(r) > max { summary = string(r[:max]) + "…" }