diff --git a/include/mcp/tools/web_search.hpp b/include/mcp/tools/web_search.hpp index 3891c0b..952a841 100644 --- a/include/mcp/tools/web_search.hpp +++ b/include/mcp/tools/web_search.hpp @@ -25,6 +25,8 @@ #pragma once #include +#include +#include #include #include #include @@ -39,15 +41,32 @@ struct WebSearchDialValue { std::string label; }; +// What a dial's value is. A list dial stores one of its `values`; a text +// dial stores what the user typed, checked against its kind and max_bytes. +enum class WebSearchText : std::uint8_t { + None, // a list dial + Line, // one line of printable UTF-8 ("San Francisco,California") + HttpsUrl, // one https:// URL with no spaces +}; + // One extra knob a service honours (freshness, region, ...). A service -// declares at most three. `values[0]` is always the default (id ""). +// declares at most kMaxWebSearchDials. For a list dial `values[0]` is always +// the default (id ""). A text dial has no values; "" is its default. struct WebSearchDial { std::string id; std::string label; std::string help; - std::vector values; + std::vector values; // empty for a text dial + WebSearchText text = WebSearchText::None; + std::size_t max_bytes = 0; // text dials only }; +// Whether `value` may be sent for `d`: a listed id, or text of the dial's +// kind within max_bytes. "" (the default) is never sent, so it is not ok. +// Hosts can check what a user typed with the same rule the request uses. +[[nodiscard]] bool web_search_dial_value_ok(const WebSearchDial& d, + std::string_view value) noexcept; + struct WebSearchService { std::string id; // stable, stored by hosts: "brave-free", "exa-api" std::string label; // shown to people and in the tool's "[via ...]" line @@ -56,9 +75,13 @@ struct WebSearchService { std::string note; // one honest line about limits and caveats int max_count = 20; // most results one request can ask for bool language = false; // honours WebSearchPlan::language - std::vector dials; // 0..3 + std::vector dials; // 0..kMaxWebSearchDials }; +// The most dials one service declares, so a host can size its settings +// page. Tavily needs all seven. +inline constexpr std::size_t kMaxWebSearchDials = 7; + // Every service, in menu order: the Free ones first, then the API-key ones. [[nodiscard]] const std::vector& web_search_services(); @@ -69,9 +92,9 @@ struct WebSearchService { struct WebSearchSlot { std::string service; // WebSearchService::id std::string api_key; // empty for Free services - // dial id -> value id. A dial that is absent, or whose value is not one - // the service lists, is ignored -- a stale setting can never put arbitrary - // text into a request. + // dial id -> value. A dial that is absent, or whose value fails + // web_search_dial_value_ok, is ignored -- a stale setting can never put + // arbitrary text into a request. std::vector> dials; }; diff --git a/src/tools/web_search_services.cpp b/src/tools/web_search_services.cpp index 998c3c9..7571b48 100644 --- a/src/tools/web_search_services.cpp +++ b/src/tools/web_search_services.cpp @@ -47,6 +47,14 @@ std::vector dwmy(std::string d, std::string w, V(std::move(m), "Past month"), V(std::move(y), "Past year")}; } +// The same, led by Past hour, for the services that filter down to the hour. +std::vector hdwmy(std::string h, std::string d, std::string w, + std::string m, std::string y) { + auto v = dwmy(std::move(d), std::move(w), std::move(m), std::move(y)); + v.insert(v.begin(), V(std::move(h), "Past hour")); + return v; +} + // ISO 3166-1 alpha-2, upper case. The shared short list: enough to cover // where most users are, without a 250-row dropdown nobody scrolls. WebSearchDial country_dial(std::string help) { @@ -62,6 +70,62 @@ WebSearchDial country_dial(std::string help) { }}; } +// Firecrawl's documented categories, the same for both tiers. Only one is +// sent, which also keeps `gov` alone as Firecrawl requires. +WebSearchDial firecrawl_category_dial() { + return WebSearchDial{"category", "Category", + "Restrict to one kind of source. Developer: code repositories and " + "documentation sites. Government: laws, regulations and court opinions.", { + V("", "Any"), V("developer", "Developer"), V("research", "Research"), + V("pdf", "PDF"), V("gov", "Government")}}; +} + +// Firecrawl folds date sorting into tbs: sbd:1 alone, or joined to the +// freshness window as "sbd:1,qdr:w" (its documented combined form). +WebSearchDial firecrawl_sort_dial() { + return WebSearchDial{"sort", "Sort", + "Newest first sorts by date, inside the Freshness window if one is set.", { + V("", "Relevance (default)"), V("date", "Newest first")}}; +} + +// Firecrawl's location is free text; its reference pairs it with country. +WebSearchDial firecrawl_location_dial() { + WebSearchDial d{"location", "Location", + "A place to search from, e.g. San Francisco,California,United States. " + "Works best with Country set too.", {}}; + d.text = WebSearchText::Line; + d.max_bytes = 200; + return d; +} + +WebSearchDial on_off_dial(std::string id, std::string label, std::string help) { + return WebSearchDial{std::move(id), std::move(label), std::move(help), { + V("", "Off (default)"), V("true", "On")}}; +} + +WebSearchDial tavily_depth_dial(std::string help) { + return WebSearchDial{"depth", "Depth", std::move(help), { + V("", "Basic (default)"), V("advanced", "Advanced"), + V("fast", "Fast"), V("ultra-fast", "Ultra fast")}}; +} + +// The dials both Tavily tiers share after topic, freshness and depth. The +// key-less mode documents "all parameters", so the free tier gets them too. +void add_tavily_filters(WebSearchService& s) { + s.dials.push_back(on_off_dial("exact_match", "Exact phrases", + "Only results that contain the query's \"quoted phrases\" as written, " + "with no synonyms.")); + s.dials.push_back(WebSearchDial{"language_filter", "Language match", + "Uses the web search language. Boost ranks it first; only drops every " + "other language. Not sent when no language is set.", { + V("", "Boost (default)"), V("only", "Only this language")}}); + s.dials.push_back(country_dial("Boosts results from this country. Sent only " + "with the General topic, the only one Tavily allows it with.")); + s.dials.push_back(on_off_dial("safe_search", "Safe search", + "Filters adult content. The fast and ultra-fast depths do not support " + "it, so it is not sent with them.")); +} + std::vector make_catalogue() { std::vector c; @@ -87,13 +151,42 @@ std::vector make_catalogue() { s.max_count = 30; s.dials.push_back(WebSearchDial{"region", "Region", "DuckDuckGo's kl parameter. Skews which country's results rank first.", { + // Every region on DuckDuckGo's URL-parameters help page, in its + // order. us-en is the default, so it is the "" row, not a second one. V("", "United States (default)"), V("wt-wt", "No region"), - V("uk-en", "United Kingdom"), V("au-en", "Australia"), - V("nz-en", "New Zealand"), V("ca-en", "Canada"), - V("in-en", "India"), V("de-de", "Germany"), - V("fr-fr", "France"), V("es-es", "Spain"), - V("it-it", "Italy"), V("br-pt", "Brazil"), - V("jp-jp", "Japan"), + V("xa-ar", "Arabia"), V("xa-en", "Arabia (en)"), + V("ar-es", "Argentina"), V("au-en", "Australia"), + V("at-de", "Austria"), V("be-fr", "Belgium (fr)"), + V("be-nl", "Belgium (nl)"), V("br-pt", "Brazil"), + V("bg-bg", "Bulgaria"), V("ca-en", "Canada"), + V("ca-fr", "Canada (fr)"), V("ct-ca", "Catalan"), + V("cl-es", "Chile"), V("cn-zh", "China"), + V("co-es", "Colombia"), V("hr-hr", "Croatia"), + V("cz-cs", "Czech Republic"), V("dk-da", "Denmark"), + V("ee-et", "Estonia"), V("fi-fi", "Finland"), + V("fr-fr", "France"), V("de-de", "Germany"), + V("gr-el", "Greece"), V("hk-tzh", "Hong Kong"), + V("hu-hu", "Hungary"), V("in-en", "India"), + V("id-id", "Indonesia"), V("id-en", "Indonesia (en)"), + V("ie-en", "Ireland"), V("il-he", "Israel"), + V("it-it", "Italy"), V("jp-jp", "Japan"), + V("kr-kr", "Korea"), V("lv-lv", "Latvia"), + V("lt-lt", "Lithuania"), V("xl-es", "Latin America"), + V("my-ms", "Malaysia"), V("my-en", "Malaysia (en)"), + V("mx-es", "Mexico"), V("nl-nl", "Netherlands"), + V("nz-en", "New Zealand"), V("no-no", "Norway"), + V("pe-es", "Peru"), V("ph-en", "Philippines"), + V("ph-tl", "Philippines (tl)"), V("pl-pl", "Poland"), + V("pt-pt", "Portugal"), V("ro-ro", "Romania"), + V("ru-ru", "Russia"), V("sg-en", "Singapore"), + V("sk-sk", "Slovak Republic"), V("sl-sl", "Slovenia"), + V("za-en", "South Africa"), V("es-es", "Spain"), + V("se-sv", "Sweden"), V("ch-de", "Switzerland (de)"), + V("ch-fr", "Switzerland (fr)"), V("ch-it", "Switzerland (it)"), + V("tw-tzh", "Taiwan"), V("th-th", "Thailand"), + V("tr-tr", "Turkey"), V("ua-uk", "Ukraine"), + V("uk-en", "United Kingdom"), V("ue-es", "United States (es)"), + V("ve-es", "Venezuela"), V("vn-vi", "Vietnam"), }}); s.dials.push_back(WebSearchDial{"safe", "Safe search", "DuckDuckGo's kp parameter.", { @@ -113,6 +206,10 @@ std::vector make_catalogue() { s.dials.push_back(freshness_dial("time_range", "Only pages from this window (Tavily's time_range).", dwmy("day", "week", "month", "year"))); + s.dials.push_back(tavily_depth_dial( + "Advanced reads more of each page and spends 2 of the key-less " + "budget's credits; fast and ultra-fast trade relevance for speed.")); + add_tavily_filters(s); c.push_back(std::move(s)); } { @@ -123,12 +220,13 @@ std::vector make_catalogue() { s.max_count = 100; s.dials.push_back(freshness_dial("freshness", "Only pages from this window (Firecrawl's tbs=qdr:).", - dwmy("qdr:d", "qdr:w", "qdr:m", "qdr:y"))); - s.dials.push_back(WebSearchDial{"category", "Category", - "Restrict to one kind of site.", { - V("", "Any"), V("github", "GitHub"), V("research", "Research"), - V("pdf", "PDF"), V("developer", "Developer docs")}}); + hdwmy("qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y"))); + s.dials.push_back(firecrawl_category_dial()); s.dials.push_back(country_dial("Firecrawl's country parameter.")); + s.dials.push_back(on_off_dial("safe", "Safe search", + "Filters explicit content (Firecrawl's safe).")); + s.dials.push_back(firecrawl_sort_dial()); + s.dials.push_back(firecrawl_location_dial()); c.push_back(std::move(s)); } @@ -144,7 +242,22 @@ std::vector make_catalogue() { dwmy("pd", "pw", "pm", "py"))); s.dials.push_back(WebSearchDial{"safesearch", "Safe search", "", { V("", "Moderate (default)"), V("off", "Off"), V("strict", "Strict")}}); - s.dials.push_back(country_dial("Where results come from.")); + // Brave's default is US; ALL is its documented "no country" value. + auto country = country_dial("Where results come from."); + country.values.insert(country.values.begin() + 1, V("ALL", "All regions")); + s.dials.push_back(std::move(country)); + s.dials.push_back(WebSearchDial{"spellcheck", "Spell correction", + "When on, Brave searches its corrected query instead of the one " + "sent, which can change identifiers and error strings.", { + V("", "On (default)"), V("false", "Off")}}); + // One hosted Goggle; Brave advises hosted files over inline rules. + WebSearchDial goggle{"goggles", "Goggle", + "Re-ranks results by a Goggle's rules: the https URL of a Goggle " + "file on GitHub, GitLab or a Gist, registered at " + "search.brave.com/goggles/create.", {}}; + goggle.text = WebSearchText::HttpsUrl; + goggle.max_bytes = 2048; + s.dials.push_back(std::move(goggle)); c.push_back(std::move(s)); } { @@ -153,14 +266,13 @@ std::vector make_catalogue() { s.signup = "https://app.tavily.com"; s.note = "Tavily's search API. 'Advanced' costs 2 credits a search."; s.max_count = 20; s.language = true; - s.dials.push_back(WebSearchDial{"depth", "Depth", - "Basic is 1 credit; advanced is 2 and reads more of each page.", { - V("", "Basic (default)"), V("advanced", "Advanced"), - V("fast", "Fast"), V("ultra-fast", "Ultra fast")}}); + s.dials.push_back(tavily_depth_dial( + "Basic is 1 credit; advanced is 2 and reads more of each page.")); s.dials.push_back(WebSearchDial{"topic", "Topic", "", { V("", "General"), V("news", "News"), V("finance", "Finance")}}); s.dials.push_back(freshness_dial("time_range", "Only pages from this window.", dwmy("day", "week", "month", "year"))); + add_tavily_filters(s); c.push_back(std::move(s)); } { @@ -170,11 +282,14 @@ std::vector make_catalogue() { s.note = "Firecrawl's search API. 2 credits a search."; s.max_count = 100; s.dials.push_back(freshness_dial("freshness", - "Only pages from this window.", dwmy("qdr:d", "qdr:w", "qdr:m", "qdr:y"))); - s.dials.push_back(WebSearchDial{"category", "Category", "", { - V("", "Any"), V("github", "GitHub"), V("research", "Research"), - V("pdf", "PDF"), V("developer", "Developer docs")}}); + "Only pages from this window.", + hdwmy("qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y"))); + s.dials.push_back(firecrawl_category_dial()); s.dials.push_back(country_dial("Firecrawl's country parameter.")); + s.dials.push_back(on_off_dial("safe", "Safe search", + "Filters explicit content (Firecrawl's safe).")); + s.dials.push_back(firecrawl_sort_dial()); + s.dials.push_back(firecrawl_location_dial()); c.push_back(std::move(s)); } { @@ -184,16 +299,23 @@ std::vector make_catalogue() { s.note = "Exa's neural search. Highlights are returned as the snippet."; s.max_count = 100; s.dials.push_back(WebSearchDial{"type", "Mode", - "Fast and instant trade depth for speed.", { + "Fast and instant trade depth for speed. Deep lite, deep and deep " + "reasoning research and synthesise: about 4, 4-15 and 12-40 seconds, " + "so deep reasoning can outlast a host's request timeout.", { V("", "Auto (default)"), V("fast", "Fast"), V("instant", "Instant"), - V("deep", "Deep")}}); + V("deep-lite", "Deep lite"), V("deep", "Deep"), + V("deep-reasoning", "Deep reasoning")}}); s.dials.push_back(WebSearchDial{"category", "Category", "Restrict to one kind of source.", { V("", "Any"), V("news", "News"), V("publication", "Publications"), V("personal site", "Personal sites")}}); s.dials.push_back(freshness_dial("freshness", "Only pages published within this window (sent as startPublishedDate).", - dwmy("day", "week", "month", "year"))); + hdwmy("hour", "day", "week", "month", "year"))); + s.dials.push_back(on_off_dial("moderation", "Safe search", + "Filters unsafe content (Exa's moderation).")); + s.dials.push_back(country_dial( + "The searcher's country (Exa's userLocation); localises results.")); c.push_back(std::move(s)); } { @@ -204,7 +326,8 @@ std::vector make_catalogue() { s.max_count = 20; s.language = true; s.dials.push_back(country_dial("Google's gl parameter.")); s.dials.push_back(freshness_dial("freshness", - "Google's tbs=qdr: time filter.", dwmy("qdr:d", "qdr:w", "qdr:m", "qdr:y"))); + "Google's tbs=qdr: time filter.", + hdwmy("qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y"))); s.dials.push_back(WebSearchDial{"autocorrect", "Spell correction", "", { V("", "On (default)"), V("false", "Off")}}); c.push_back(std::move(s)); @@ -213,15 +336,16 @@ std::vector make_catalogue() { WebSearchService s; s.id = "perplexity-api"; s.label = "Perplexity API key"; s.needs_key = true; s.signup = "https://docs.perplexity.ai"; - s.note = "Perplexity's Search API (ranked results, not an answer). " - "'Never return' sites are left in the query as -site: here, " - "because its domain-filter syntax for excluding is not documented."; + s.note = "Perplexity's Search API (ranked results, not an answer)."; s.max_count = 20; s.language = true; s.dials.push_back(WebSearchDial{"recency", "Freshness", "Only pages from this window (search_recency_filter).", { V("", "Any time"), V("hour", "Past hour"), V("day", "Past day"), V("week", "Past week"), V("month", "Past month"), V("year", "Past year")}}); s.dials.push_back(country_dial("Perplexity's country parameter.")); + s.dials.push_back(WebSearchDial{"search_type", "Mode", + "Fast answers sooner with a lighter web search.", { + V("", "Web (default)"), V("fast", "Fast")}}); c.push_back(std::move(s)); } return c; @@ -316,10 +440,10 @@ std::string join_strings(const json& arr, std::string_view sep) { return out; } -// ISO 8601 UTC, `days` before `now`. -std::string iso_days_ago(std::chrono::system_clock::time_point now, int days) { +// ISO 8601 UTC, `hours` before `now`. +std::string iso_hours_ago(std::chrono::system_clock::time_point now, int hours) { const std::time_t t = std::chrono::system_clock::to_time_t( - now - std::chrono::hours(24 * days)); + now - std::chrono::hours(hours)); std::tm tm{}; #ifdef _WIN32 gmtime_s(&tm, &t); @@ -374,11 +498,12 @@ std::vector> browser_headers( return h; } -int days_for(std::string_view v) { - if (v == "day") return 1; - if (v == "week") return 7; - if (v == "month") return 30; - if (v == "year") return 365; +int hours_for(std::string_view v) { + if (v == "hour") return 1; + if (v == "day") return 24; + if (v == "week") return 24 * 7; + if (v == "month") return 24 * 30; + if (v == "year") return 24 * 365; return 0; } @@ -413,16 +538,42 @@ HttpRequest req_ddg_free(const WebSearchQuery& q) { return r; } +// Tavily's country takes a lower-case English name from its own list, not +// an ISO code. Covers country_dial's codes; anything else is not sent. +std::string tavily_country(std::string_view code) { + static constexpr std::pair names[] = { + {"US", "united states"}, {"GB", "united kingdom"}, {"NZ", "new zealand"}, + {"AU", "australia"}, {"CA", "canada"}, {"DE", "germany"}, + {"FR", "france"}, {"ES", "spain"}, {"IT", "italy"}, + {"NL", "netherlands"}, {"BR", "brazil"}, {"IN", "india"}, + {"JP", "japan"}}; + for (const auto& [c, n] : names) + if (c == code) return std::string{n}; + return {}; +} + // Tavily, keyless or keyed. [D] POST /search ; [L] keyless = the header -// X-Tavily-Access-Mode: keyless, no Authorization. +// X-Tavily-Access-Mode: keyless, no Authorization. Three dials depend on +// others, per the reference: country only with topic general, safe_search +// not with the fast depths, filter_by_language only with a language (else +// a 400). HttpRequest req_tavily(const WebSearchQuery& q, std::string_view key) { json b = {{"query", q.text}, {"max_results", q.count}, {"include_answer", false}, {"include_raw_content", false}}; if (!q.exclude.empty()) b["exclude_domains"] = q.exclude; if (!q.language.empty()) b["language"] = q.language; - if (auto v = q.dial("topic"); !v.empty()) b["topic"] = std::string{v}; + const std::string_view topic = q.dial("topic"); + const std::string_view depth = q.dial("depth"); + if (!topic.empty()) b["topic"] = std::string{topic}; if (auto v = q.dial("time_range"); !v.empty()) b["time_range"] = std::string{v}; - if (auto v = q.dial("depth"); !v.empty()) b["search_depth"] = std::string{v}; + if (!depth.empty()) b["search_depth"] = std::string{depth}; + if (q.dial("exact_match") == "true") b["exact_match"] = true; + if (q.dial("language_filter") == "only" && !q.language.empty()) + b["filter_by_language"] = true; + if (auto c = tavily_country(q.dial("country")); !c.empty() && topic.empty()) + b["country"] = c; + if (q.dial("safe_search") == "true" && depth != "fast" && depth != "ultra-fast") + b["safe_search"] = true; std::vector> h; if (key.empty()) h.push_back({"x-tavily-access-mode", "keyless"}); else h.push_back({"authorization", "Bearer " + std::string{key}}); @@ -430,21 +581,27 @@ HttpRequest req_tavily(const WebSearchQuery& q, std::string_view key) { } // Firecrawl, keyless or keyed. [D]+[L] POST /v2/search ; limit 1-100 ; -// tbs=qdr:d|w|m|y ; categories is a list of strings ; excludeDomains takes -// bare hostnames. Keyless = no Authorization header at all. +// tbs=qdr:h|d|w|m|y, sbd:1 sorts by date and joins a window with a comma ; +// categories is a list of strings ; excludeDomains takes bare hostnames ; +// safe is a boolean. Keyless = no Authorization header at all. HttpRequest req_firecrawl(const WebSearchQuery& q, std::string_view key) { json b = {{"query", q.text}, {"limit", q.count}}; if (!q.exclude.empty()) b["excludeDomains"] = q.exclude; - if (auto v = q.dial("freshness"); !v.empty()) b["tbs"] = std::string{v}; + std::string tbs{q.dial("freshness")}; + if (q.dial("sort") == "date") tbs = tbs.empty() ? "sbd:1" : "sbd:1," + tbs; + if (!tbs.empty()) b["tbs"] = std::move(tbs); if (auto v = q.dial("category"); !v.empty()) b["categories"] = json::array({std::string{v}}); if (auto v = q.dial("country"); !v.empty()) b["country"] = std::string{v}; + if (q.dial("safe") == "true") b["safe"] = true; + if (auto v = q.dial("location"); !v.empty()) b["location"] = std::string{v}; std::vector> h; if (!key.empty()) h.push_back({"authorization", "Bearer " + std::string{key}}); return json_post("https://api.firecrawl.dev/v2/search", b, std::move(h)); } // Brave API. [D] GET /res/v1/web/search ; header X-Subscription-Token ; -// count 1-20 ; text_decorations=false drops the highlight markers. +// count 1-20 ; text_decorations=false drops the highlight markers ; +// spellcheck=false searches the query as sent. HttpRequest req_brave_api(const WebSearchQuery& q, std::string_view key) { HttpRequest r; r.method = "GET"; @@ -459,22 +616,29 @@ HttpRequest req_brave_api(const WebSearchQuery& q, std::string_view key) { r.url += "&freshness=" + url_escape(v); if (auto v = q.dial("safesearch"); !v.empty()) r.url += "&safesearch=" + url_escape(v); + if (q.dial("spellcheck") == "false") + r.url += "&spellcheck=false"; + if (auto v = q.dial("goggles"); !v.empty()) + r.url += "&goggles=" + url_escape(v); r.headers.push_back({"accept", "application/json"}); r.headers.push_back({"x-subscription-token", std::string{key}}); return r; } // Exa. [D] POST /search ; x-api-key ; numResults 1-100 ; contents.highlights -// puts excerpts on each result ; startPublishedDate is an ISO 8601 instant. +// puts excerpts on each result ; startPublishedDate is an ISO 8601 instant ; +// userLocation takes the bare ISO alpha-2 string form ; moderation is boolean. HttpRequest req_exa(const WebSearchQuery& q, std::string_view key) { json b = {{"query", q.text}, {"numResults", q.count}, {"contents", {{"highlights", true}}}}; if (!q.exclude.empty()) b["excludeDomains"] = q.exclude; if (auto v = q.dial("type"); !v.empty()) b["type"] = std::string{v}; if (auto v = q.dial("category"); !v.empty()) b["category"] = std::string{v}; - if (int d = days_for(q.dial("freshness")); d > 0 + if (int h = hours_for(q.dial("freshness")); h > 0 && q.now != std::chrono::system_clock::time_point{}) // host gave the time - b["startPublishedDate"] = iso_days_ago(q.now, d); + b["startPublishedDate"] = iso_hours_ago(q.now, h); + if (q.dial("moderation") == "true") b["moderation"] = true; + if (auto v = q.dial("country"); !v.empty()) b["userLocation"] = std::string{v}; return json_post("https://api.exa.ai/search", b, {{"x-api-key", std::string{key}}}); } @@ -492,12 +656,21 @@ HttpRequest req_serper(const WebSearchQuery& q, std::string_view key) { // Perplexity Search. [D] POST /search ; Bearer ; max_results 1-20 without a // tier ; country is ISO alpha-2 ; search_language_filter is a list of ISO 639-1 -// codes ; search_recency_filter is hour|day|week|month|year. +// codes ; search_recency_filter is hour|day|week|month|year ; search_type is +// web|fast|people (people is not offered: it searches for people, not pages) ; +// search_domain_filter takes at most 20 hosts, a "-" in front excluding one, +// and is either all allow or all deny (only deny is sent). HttpRequest req_perplexity(const WebSearchQuery& q, std::string_view key) { json b = {{"query", q.text}, {"max_results", q.count}}; if (!q.language.empty()) b["search_language_filter"] = json::array({q.language}); if (auto v = q.dial("country"); !v.empty()) b["country"] = std::string{v}; if (auto v = q.dial("recency"); !v.empty()) b["search_recency_filter"] = std::string{v}; + if (auto v = q.dial("search_type"); !v.empty()) b["search_type"] = std::string{v}; + if (!q.exclude.empty()) { + json deny = json::array(); + for (const auto& host : q.exclude) deny.push_back("-" + host); + b["search_domain_filter"] = std::move(deny); + } return json_post("https://api.perplexity.ai/search", b, {{"authorization", "Bearer " + std::string{key}}}); } @@ -529,6 +702,63 @@ std::vector collect(const json& arr, const char* url_key, return out; } +// Percent-encode an id for a URL path, keeping '/' (a DOI and an old-style +// arXiv id such as hep-th/9901001 both carry one). +std::string path_escape(std::string_view s) { + std::string out; + for (std::size_t from = 0;;) { + const auto slash = s.find('/', from); + out += url_escape(s.substr(from, slash == std::string_view::npos + ? std::string_view::npos : slash - from)); + if (slash == std::string_view::npos) return out; + out += '/'; + from = slash + 1; + } +} + +// A Firecrawl paper record carries ids, not a link. The four primaryId forms +// Firecrawl documents become the paper's own page; anything else falls back +// to Firecrawl's paper endpoint by the record's canonical paperId, which +// answers for every record. "" when the record has neither. +std::string paper_url(const json& r) { + const std::string primary = str_at(r, "primaryId"); + if (const auto colon = primary.find(':'); + colon != std::string::npos && colon + 1 < primary.size()) { + const std::string ns = lower(primary.substr(0, colon)); + std::string id = primary.substr(colon + 1); + if (ns == "arxiv") return "https://arxiv.org/abs/" + path_escape(id); + if (ns == "pmid") return "https://pubmed.ncbi.nlm.nih.gov/" + url_escape(id) + "/"; + if (ns == "pmcid") { + // PMC writes its ids as PMC1234567; whether Firecrawl keeps the + // prefix is not documented, so both spellings are accepted. + if (lower(id.substr(0, 3)) != "pmc") id = "PMC" + id; + return "https://pmc.ncbi.nlm.nih.gov/articles/" + url_escape(id) + "/"; + } + if (ns == "doi") return "https://doi.org/" + path_escape(id); + } + if (const std::string paper = str_at(r, "paperId"); !paper.empty()) + return "https://api.firecrawl.dev/v2/search/research/papers/" + url_escape(paper); + return {}; +} + +// Firecrawl's data.research: paper records (title, abstract, ids). +std::vector collect_papers(const json& arr, int count) { + std::vector out; + if (!arr.is_array()) return out; + for (const auto& r : arr) { + if (static_cast(out.size()) >= count) break; + if (!r.is_object()) continue; + SearchHit h; + h.url = paper_url(r); + if (h.url.empty() || h.url.size() > 2048) continue; // never a cut-off link + h.title = tidy(str_at(r, "title"), 300); + if (h.title.empty()) h.title = h.url; + h.snippet = tidy(str_at(r, "abstract"), 500); + out.push_back(std::move(h)); + } + return out; +} + } // namespace std::string_view WebSearchQuery::dial(std::string_view id) const noexcept { @@ -603,10 +833,8 @@ validate_dials(const WebSearchService& svc, const WebSearchSlot& slot) { std::vector> out; for (const auto& d : svc.dials) { for (const auto& [id, val] : slot.dials) { - if (id != d.id || val.empty()) continue; - const bool known = std::any_of(d.values.begin(), d.values.end(), - [&](const WebSearchDialValue& v) { return v.id == val; }); - if (known) out.emplace_back(id, val); + if (id != d.id) continue; + if (web_search_dial_value_ok(d, val)) out.emplace_back(id, val); break; } } @@ -620,7 +848,7 @@ bool is_html_service(std::string_view id) noexcept { bool has_native_exclude(std::string_view id) noexcept { return id == "tavily-free" || id == "tavily-api" || id == "firecrawl-free" || id == "firecrawl-api" - || id == "exa-api"; + || id == "exa-api" || id == "perplexity-api"; } std::string clean_api_key(std::string_view key) { @@ -739,8 +967,16 @@ JsonHits parse_json_hits(std::string_view service_id, std::string_view body, out.hits = collect(*need(field(j, "results")), "url", "title", {"content"}, count); } else if (service_id == "firecrawl-free" || service_id == "firecrawl-api") { - out.hits = collect(*need(field(field(j, "data"), "web")), "url", "title", - {"description"}, count); + // Web results, then paper records. From 2026-11-16 the research + // category answers in data.research and no longer in data.web, so + // either container makes a readable reply. + const json& data = field(j, "data"); + const json& web = field(data, "web"); + const json& papers = field(data, "research"); + if (!web.is_array() && !papers.is_array()) out.ok = false; + out.hits = collect(web, "url", "title", {"description"}, count); + for (auto& h : collect_papers(papers, count - static_cast(out.hits.size()))) + out.hits.push_back(std::move(h)); } else if (service_id == "exa-api") { out.hits = collect(*need(field(j, "results")), "url", "title", {"highlights", "summary", "text"}, count); @@ -765,6 +1001,26 @@ JsonHits parse_json_hits(std::string_view service_id, std::string_view body, namespace mcp::tools { +bool web_search_dial_value_ok(const WebSearchDial& d, std::string_view value) noexcept { + if (value.empty()) return false; + if (d.text == WebSearchText::None) + return std::any_of(d.values.begin(), d.values.end(), + [&](const WebSearchDialValue& v) { return v.id == value; }); + if (value.size() > d.max_bytes || !util::is_valid_utf8(value)) return false; + // No control characters in either kind: the value lands in a URL or a + // JSON string, and a newline could start a second Goggles rule. + const bool url = d.text == WebSearchText::HttpsUrl; + for (char ch : value) { + const auto u = static_cast(ch); + if (u < 0x20 || u == 0x7f || (url && u == ' ')) return false; + } + if (url) { + constexpr std::string_view kScheme = "https://"; + if (value.size() <= kScheme.size() || !value.starts_with(kScheme)) return false; + } + return true; +} + const std::vector& web_search_services() { static const std::vector c = detail::make_catalogue(); return c; diff --git a/src/tools/web_search_services.hpp b/src/tools/web_search_services.hpp index 0254a6a..13c2610 100644 --- a/src/tools/web_search_services.hpp +++ b/src/tools/web_search_services.hpp @@ -52,7 +52,8 @@ enum class WebSearchFailure { // unchanged with an empty list, so we never send an empty query. struct SiteSplit { std::string query; - std::vector exclude; // lower-case hosts, at most 20 + std::vector exclude; // lower-case hosts, at most 20 (also + // Perplexity's limit); more stay in `query` }; [[nodiscard]] SiteSplit split_site_exclusions(std::string_view query); @@ -70,8 +71,9 @@ struct WebSearchQuery { [[nodiscard]] std::string_view dial(std::string_view id) const noexcept; }; -// Keep only the dial choices `svc` actually offers, with a value it lists, -// and drop the default. Anything else is discarded rather than forwarded. +// Keep only the dials `svc` actually offers, with a value that passes +// web_search_dial_value_ok, and drop the default. Anything else is discarded +// rather than forwarded. [[nodiscard]] std::vector> validate_dials(const WebSearchService& svc, const WebSearchSlot& slot); diff --git a/tests/web_search_services_test.cpp b/tests/web_search_services_test.cpp index 9f72f57..6e272e5 100644 --- a/tests/web_search_services_test.cpp +++ b/tests/web_search_services_test.cpp @@ -47,6 +47,14 @@ std::string header(const HttpRequest& r, std::string_view name) { return {}; } +// The value ids one dial offers, in menu order ("" first). +std::vector dial_ids(std::string_view service, std::string_view dial) { + std::vector out; + for (const auto& d : svc(service).dials) + if (d.id == dial) for (const auto& v : d.values) out.push_back(v.id); + return out; +} + class Scripted : public HttpClient { public: // First rule whose needle is in the URL wins. @@ -97,9 +105,15 @@ TEST_CASE("web search: search catalogue is well formed") { CHECK(!s.id.empty()); CHECK(!s.label.empty()); CHECK(s.max_count >= 1); - CHECK(s.dials.size() <= 3); // the pane has three dial rows + CHECK(s.dials.size() <= kMaxWebSearchDials); CHECK(s.needs_key == !s.signup.empty()); for (const auto& d : s.dials) { + INFO(d.id); + if (d.text != WebSearchText::None) { // free text: no list, a bound + CHECK(d.values.empty()); + CHECK(d.max_bytes > 0); + continue; + } REQUIRE(!d.values.empty()); CHECK(d.values.front().id.empty()); // [0] is "send nothing" } @@ -138,6 +152,93 @@ TEST_CASE("web search: dials: only listed values reach a request") { CHECK(validate_dials(svc("ddg-free"), d).empty()); } +TEST_CASE("web search: dials offer each vendor's documented values") { + using V = std::vector; + const V hourly_tbs{"", "qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y"}; + const V categories{"", "developer", "research", "pdf", "gov"}; + const V exa_types{"", "fast", "instant", "deep-lite", "deep", "deep-reasoning"}; + const V exa_ages{"", "hour", "day", "week", "month", "year"}; + // Firecrawl's four categories; github is no longer one. + for (const char* id : {"firecrawl-free", "firecrawl-api"}) { + INFO(id); + CHECK(dial_ids(id, "category") == categories); + CHECK(dial_ids(id, "freshness") == hourly_tbs); + WebSearchSlot old{id, {}, {{"category", "github"}}}; + CHECK(validate_dials(svc(id), old).empty()); + } + CHECK(dial_ids("exa-api", "type") == exa_types); + CHECK(dial_ids("exa-api", "freshness") == exa_ages); + CHECK(dial_ids("serper-api", "freshness") == hourly_tbs); + // Brave's "no country" sits right under the default. + const auto brave = dial_ids("brave-api", "country"); + REQUIRE(brave.size() >= 2); + CHECK(brave[1] == "ALL"); + // DuckDuckGo: all 68 documented regions, us-en being the "" default. + const auto kl = dial_ids("ddg-free", "region"); + CHECK(kl.size() == 68); + CHECK(std::find(kl.begin(), kl.end(), "us-en") == kl.end()); + for (const char* r : {"wt-wt", "nl-nl", "xl-es", "ue-es", "tw-tzh", "vn-vi"}) + CHECK(std::find(kl.begin(), kl.end(), r) != kl.end()); + // No dial lists a value twice. + for (const auto& s : web_search_services()) + for (const auto& d : s.dials) { + V ids = dial_ids(s.id, d.id); + std::sort(ids.begin(), ids.end()); + const std::string where = s.id + "." + d.id; + CHECK(std::adjacent_find(ids.begin(), ids.end()) == ids.end(), where); + } +} + +TEST_CASE("web search: the new dials pass validation and fill the rows") { + auto dials = [](std::string_view id) { return svc(id).dials.size(); }; + CHECK(dials("tavily-free") == kMaxWebSearchDials); + CHECK(dials("tavily-api") == kMaxWebSearchDials); + CHECK(dials("firecrawl-free") == 6); + CHECK(dials("firecrawl-api") == 6); + CHECK(dials("brave-api") == 5); + CHECK(dials("exa-api") == 5); + CHECK(dials("perplexity-api") == 3); + WebSearchSlot t{"tavily-free", {}, {{"exact_match", "true"}, {"language_filter", "only"}, + {"country", "JP"}, {"safe_search", "true"}, + {"depth", "ultra-fast"}}}; + CHECK(validate_dials(svc("tavily-free"), t).size() == 5); + WebSearchSlot f{"firecrawl-api", {}, {{"sort", "date"}, {"safe", "true"}}}; + CHECK(validate_dials(svc("firecrawl-api"), f).size() == 2); + WebSearchSlot e{"exa-api", {}, {{"moderation", "true"}, {"country", "GB"}}}; + CHECK(validate_dials(svc("exa-api"), e).size() == 2); +} + +TEST_CASE("web search: text dials take checked free text") { + const WebSearchDial* goggle = nullptr; + for (const auto& d : svc("brave-api").dials) if (d.id == "goggles") goggle = &d; + REQUIRE(goggle != nullptr); + const std::string url = "https://raw.githubusercontent.com/brave/goggles-quickstart/" + "main/goggles/tech_blogs.goggle"; + CHECK(web_search_dial_value_ok(*goggle, url)); + CHECK(!web_search_dial_value_ok(*goggle, "")); + CHECK(!web_search_dial_value_ok(*goggle, "https://")); + CHECK(!web_search_dial_value_ok(*goggle, "http://example.com/a.goggle")); + CHECK(!web_search_dial_value_ok(*goggle, "$boost=3,site=dev.to")); // inline rules + CHECK(!web_search_dial_value_ok(*goggle, "https://a.com/x y")); + CHECK(!web_search_dial_value_ok(*goggle, "https://a.com/x\n$discard")); + CHECK(!web_search_dial_value_ok(*goggle, "https://a.com/" + std::string(2048, 'x'))); + + const WebSearchDial* where = nullptr; + for (const auto& d : svc("firecrawl-free").dials) if (d.id == "location") where = &d; + REQUIRE(where != nullptr); + CHECK(web_search_dial_value_ok(*where, "San Francisco,California,United States")); + CHECK(web_search_dial_value_ok(*where, "Z\xc3\xbcrich")); // UTF-8 is fine + CHECK(!web_search_dial_value_ok(*where, "Z\xfcrich")); // Latin-1 is not + CHECK(!web_search_dial_value_ok(*where, "line one\r\nline two")); + CHECK(!web_search_dial_value_ok(*where, std::string(201, 'a'))); + + // validate_dials applies the same rule, so a bad value never reaches a slot. + WebSearchSlot ok{"brave-api", {}, {{"goggles", url}}}; + CHECK(validate_dials(svc("brave-api"), ok).size() == 1); + WebSearchSlot bad{"brave-api", {}, {{"goggles", "javascript:alert(1)"}}}; + CHECK(validate_dials(svc("brave-api"), bad).empty()); +} + TEST_CASE("web search: keys are checked before they become a header") { CHECK(clean_api_key(" tvly-abc123 \n") == "tvly-abc123"); CHECK(clean_api_key("abc\r\nx-evil: 1").empty()); @@ -158,9 +259,16 @@ TEST_CASE("web search: -site: becomes a native filter only where the service has auto only = split_site_exclusions("-site:reddit.com"); CHECK(only.query == "-site:reddit.com"); CHECK(only.exclude.empty()); + // At most 20 lifted out (Perplexity's limit); the rest stay in the text. + std::string many = "q"; + for (int i = 0; i < 21; ++i) many += " -site:s" + std::to_string(i) + ".test"; + auto capped = split_site_exclusions(many); + CHECK(capped.exclude.size() == 20); + CHECK(capped.query == "q -site:s20.test"); CHECK(has_native_exclude("tavily-free")); CHECK(has_native_exclude("exa-api")); + CHECK(has_native_exclude("perplexity-api")); CHECK(!has_native_exclude("brave-free")); CHECK(!has_native_exclude("serper-api")); } @@ -206,6 +314,42 @@ TEST_CASE("web search: Tavily request: keyless header vs bearer key, exclusions CHECK(keyed.url.find("tvly-k") == std::string::npos); } +TEST_CASE("web search: Tavily filters follow the reference's rules") { + auto body = [](std::string_view id, const WebSearchQuery& q) { + return json::parse(build_web_search_request(svc(id), "", q).body); + }; + auto q = query(); q.language = "de"; + q.dials = {{"exact_match", "true"}, {"language_filter", "only"}, + {"country", "NZ"}, {"safe_search", "true"}, {"depth", "advanced"}}; + for (const char* id : {"tavily-free", "tavily-api"}) { + INFO(id); + auto b = body(id, q); + CHECK(b["exact_match"] == true); + CHECK(b["filter_by_language"] == true); + CHECK(b["country"] == "new zealand"); // Tavily wants the name + CHECK(b["safe_search"] == true); + CHECK(b["search_depth"] == "advanced"); // key-less now offers depth too + } + // country is for the general topic only. + q.dials = {{"country", "NZ"}, {"topic", "news"}}; + CHECK(!body("tavily-free", q).contains("country")); + // safe_search is not supported with the fast depths. + for (const char* depth : {"fast", "ultra-fast"}) { + q.dials = {{"safe_search", "true"}, {"depth", depth}}; + CHECK(!body("tavily-free", q).contains("safe_search"), depth); + } + // filter_by_language without a language is a 400, so it is not sent. + q.language.clear(); + q.dials = {{"language_filter", "only"}}; + CHECK(!body("tavily-free", q).contains("filter_by_language")); + // Defaults send nothing new. + q.dials = {}; + const auto plain = body("tavily-free", q); + for (const char* k : {"exact_match", "filter_by_language", "country", + "safe_search", "search_depth"}) + CHECK(!plain.contains(k), k); +} + TEST_CASE("web search: Firecrawl request") { auto q = query(); q.exclude = {"github.com"}; q.dials = {{"freshness", "qdr:m"}, {"category", "research"}, {"country", "NZ"}}; @@ -218,8 +362,23 @@ TEST_CASE("web search: Firecrawl request") { CHECK(b["tbs"] == "qdr:m"); CHECK(b["categories"] == json::array({"research"})); CHECK(b["country"] == "NZ"); + CHECK(!b.contains("safe")); CHECK(header(build_web_search_request(svc("firecrawl-api"), "fc-1", q), "authorization") == "Bearer fc-1"); + + // Sorting by date joins the window in tbs; alone it is just sbd:1. + q.dials = {{"freshness", "qdr:w"}, {"sort", "date"}, {"safe", "true"}, + {"location", "Germany"}}; + for (const char* id : {"firecrawl-free", "firecrawl-api"}) { + INFO(id); + auto sb = json::parse(build_web_search_request(svc(id), "k", q).body); + CHECK(sb["tbs"] == "sbd:1,qdr:w"); + CHECK(sb["safe"] == true); + CHECK(sb["location"] == "Germany"); + } + q.dials = {{"sort", "date"}}; + CHECK(json::parse(build_web_search_request(svc("firecrawl-free"), "", q).body) + ["tbs"] == "sbd:1"); } TEST_CASE("web search: Brave API request keeps the key out of the URL") { @@ -239,6 +398,16 @@ TEST_CASE("web search: Brave API request keeps the key out of the URL") { q.language = "zh"; CHECK(build_web_search_request(svc("brave-api"), "k", q).url .find("search_lang=zh-hans") != std::string::npos); + // Spell correction off is sent; on is Brave's default and sends nothing. + q.dials = {{"spellcheck", "false"}, {"country", "ALL"}}; + const auto url = build_web_search_request(svc("brave-api"), "k", q).url; + CHECK(url.find("&spellcheck=false") != std::string::npos); + CHECK(url.find("&country=ALL") != std::string::npos); + CHECK(url.find("goggles") == std::string::npos); + // A Goggle URL is percent-encoded into its own parameter. + q.dials = {{"goggles", "https://g.example/a b.goggle?x=1&y"}}; + CHECK(build_web_search_request(svc("brave-api"), "k", q).url.find( + "&goggles=https%3A%2F%2Fg.example%2Fa%20b.goggle%3Fx%3D1%26y") != std::string::npos); } TEST_CASE("web search: Exa, Serper and Perplexity requests") { @@ -254,6 +423,18 @@ TEST_CASE("web search: Exa, Serper and Perplexity requests") { CHECK(eb["type"] == "fast"); CHECK(eb["category"] == "news"); CHECK(eb["startPublishedDate"] == "2025-10-02T08:53:20.000Z"); + q.dials = {{"freshness", "hour"}}; + CHECK(json::parse(build_web_search_request(svc("exa-api"), "k", q).body) + ["startPublishedDate"] == "2025-10-09T07:53:20.000Z"); + q.dials = {{"type", "deep-reasoning"}}; + CHECK(json::parse(build_web_search_request(svc("exa-api"), "k", q).body) + ["type"] == "deep-reasoning"); + q.dials = {{"moderation", "true"}, {"country", "NZ"}}; + auto em = json::parse(build_web_search_request(svc("exa-api"), "k", q).body); + CHECK(em["moderation"] == true); + CHECK(em["userLocation"] == "NZ"); + CHECK(!eb.contains("moderation")); + CHECK(!eb.contains("userLocation")); q.dials = {{"country", "NZ"}, {"freshness", "qdr:d"}, {"autocorrect", "false"}}; auto s = build_web_search_request(svc("serper-api"), "sp-k", q); @@ -276,6 +457,17 @@ TEST_CASE("web search: Exa, Serper and Perplexity requests") { CHECK(pb["country"] == "NZ"); CHECK(pb["search_recency_filter"] == "month"); CHECK(pb["search_language_filter"] == json::array({"fr"})); + CHECK(!pb.contains("search_type")); + CHECK(!pb.contains("search_domain_filter")); + q.dials = {{"search_type", "fast"}}; + CHECK(json::parse(build_web_search_request(svc("perplexity-api"), "k", q).body) + ["search_type"] == "fast"); + // Excluded hosts become the deny form of its domain filter. + q.dials = {}; + q.exclude = {"reddit.com", "pinterest.com"}; + const json deny = json::array({"-reddit.com", "-pinterest.com"}); + CHECK(json::parse(build_web_search_request(svc("perplexity-api"), "k", q).body) + ["search_domain_filter"] == deny); } TEST_CASE("web search: reply parsers") { @@ -332,6 +524,55 @@ TEST_CASE("web search: reply parsers") { CHECK(!parse_json_hits("firecrawl-free", R"({"success":false,"error":"x"})", 5).ok); } +TEST_CASE("web search: Firecrawl paper records each get a link") { + // The record shape is the documented research example. + auto p = parse_json_hits("firecrawl-api", + R"({"success":true,"data":{"research":[)" + R"({"paperId":"2014215642691656232","primaryId":"arxiv:2105.05233",)" + R"("ids":{"arxiv":["2105.05233"]},"title":"Diffusion Models Beat GANs on Image Synthesis",)" + R"("abstract":"We show that diffusion models can achieve image sample quality.","score":0.016},)" + R"({"paperId":"1","primaryId":"pmid:31452104","title":"P"},)" + R"({"paperId":"2","primaryId":"pmcid:PMC6824467","title":"C"},)" + R"({"paperId":"3","primaryId":"pmcid:6824467","title":"C2"},)" + R"({"paperId":"4","primaryId":"doi:10.1038/s41586-021-03819-2","title":"D"},)" + R"({"paperId":"5","primaryId":"arxiv:hep-th/9901001","title":"Old arXiv"},)" + R"({"paperId":"6","primaryId":"doi:10.1000/a b","title":"Odd DOI"},)" + R"({"paperId":"7","primaryId":"openalex:W2963403868","title":"Unknown kind"},)" + R"({"paperId":"8","title":"No primary id"},)" + R"({"title":"Nothing to link"})" + R"(]}})", 20); + REQUIRE(p.ok); + REQUIRE(p.hits.size() == 9); // the last record has no id at all + CHECK(p.hits[0].url == "https://arxiv.org/abs/2105.05233"); + CHECK(p.hits[0].title == "Diffusion Models Beat GANs on Image Synthesis"); + CHECK(p.hits[0].snippet == "We show that diffusion models can achieve image sample quality."); + CHECK(p.hits[1].url == "https://pubmed.ncbi.nlm.nih.gov/31452104/"); + CHECK(p.hits[2].url == "https://pmc.ncbi.nlm.nih.gov/articles/PMC6824467/"); + CHECK(p.hits[3].url == "https://pmc.ncbi.nlm.nih.gov/articles/PMC6824467/"); + CHECK(p.hits[4].url == "https://doi.org/10.1038/s41586-021-03819-2"); + CHECK(p.hits[5].url == "https://arxiv.org/abs/hep-th/9901001"); + CHECK(p.hits[6].url == "https://doi.org/10.1000/a%20b"); + // A kind of id Firecrawl does not document, or none: its own paper page. + CHECK(p.hits[7].url == "https://api.firecrawl.dev/v2/search/research/papers/7"); + CHECK(p.hits[8].url == "https://api.firecrawl.dev/v2/search/research/papers/8"); + + // Web results come first, and the count covers both lists. + auto mixed = parse_json_hits("firecrawl-free", + R"({"success":true,"data":{"web":[{"url":"https://a.test/","title":"A"}],)" + R"("research":[{"paperId":"1","primaryId":"pmid:1","title":"P1"},)" + R"({"paperId":"2","primaryId":"pmid:2","title":"P2"}]}})", 2); + REQUIRE(mixed.ok); + REQUIRE(mixed.hits.size() == 2); + CHECK(mixed.hits[0].url == "https://a.test/"); + CHECK(mixed.hits[1].url == "https://pubmed.ncbi.nlm.nih.gov/1/"); + + // Either list makes a readable reply; an empty one is "no results". + auto empty = parse_json_hits("firecrawl-free", R"({"success":true,"data":{"research":[]}})", 5); + CHECK(empty.ok); + CHECK(empty.hits.empty()); + CHECK(!parse_json_hits("firecrawl-free", R"({"success":true,"data":{}})", 5).ok); +} + TEST_CASE("web search: failures are told apart") { const auto& brave = svc("brave-free"); const auto& tavily = svc("tavily-api"); @@ -445,6 +686,15 @@ TEST_CASE("web search: -site: reaches a native filter, and stays in the text els run(http2, p2, "rust async -site:reddit.com"); REQUIRE(http2->seen.size() == 1); CHECK(json::parse(http2->seen[0].body)["q"] == "rust async -site:reddit.com"); + + auto http3 = std::make_shared(); + http3->rules.push_back({"api.perplexity.ai", reply(200, R"({"results":[]})")}); + WebSearchPlan p3; p3.slots = {WebSearchSlot{"perplexity-api", "k", {}}}; + run(http3, p3, "rust async -site:reddit.com"); + REQUIRE(http3->seen.size() == 1); + auto pb = json::parse(http3->seen[0].body); + CHECK(pb["query"] == "rust async"); + CHECK(pb["search_domain_filter"] == json::array({"-reddit.com"})); } TEST_CASE("web search: at most three slots are tried") {