From 2ec79df571c083774a39e90e129de87fb110e037 Mon Sep 17 00:00:00 2001 From: sail3r <82087684+sail3r@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:35:00 +1300 Subject: [PATCH 1/5] web_search: dials offer every value their vendor documents Several dials offered a short or stale list. Each list now matches the vendor's current reference. - Firecrawl categories are developer, research, pdf and gov. github is gone from the docs (GitHub search moved into the developer index), so it leaves both Firecrawl tiers; a stored "github" is dropped by validate_dials and the request goes out with no category. - Past hour (qdr:h) for Firecrawl and Serper. Brave, Tavily and DuckDuckGo document nothing shorter than a day, so they keep their lists; Perplexity already had it. - Exa freshness gains Past hour too. startPublishedDate is now computed in hours rather than days, which is the same instant for the old values. - Exa mode lists all six types: auto, fast, instant, deep-lite, deep, deep-reasoning. Exa quotes deep-reasoning at 12-40 seconds, so the help text says it can outlast a host's request timeout. - Brave country offers ALL, Brave's documented "no country" value; left at its default Brave searches as US. - DuckDuckGo region lists all 68 regions on its URL-parameters page, in the page's order. us-en stays the default row. A service may now declare up to kMaxWebSearchDials (7) dials, the number Tavily will need; the catalogue test checks against the constant instead of the old three. --- include/mcp/tools/web_search.hpp | 9 ++- src/tools/web_search_services.cpp | 111 +++++++++++++++++++++-------- tests/web_search_services_test.cpp | 53 +++++++++++++- 3 files changed, 140 insertions(+), 33 deletions(-) diff --git a/include/mcp/tools/web_search.hpp b/include/mcp/tools/web_search.hpp index 3891c0b..bd3c756 100644 --- a/include/mcp/tools/web_search.hpp +++ b/include/mcp/tools/web_search.hpp @@ -25,6 +25,7 @@ #pragma once #include +#include #include #include #include @@ -40,7 +41,7 @@ struct WebSearchDialValue { }; // One extra knob a service honours (freshness, region, ...). A service -// declares at most three. `values[0]` is always the default (id ""). +// declares at most kMaxWebSearchDials. `values[0]` is always the default (id ""). struct WebSearchDial { std::string id; std::string label; @@ -56,9 +57,13 @@ struct WebSearchService { std::string note; // one honest line about limits and caveats int max_count = 20; // most results one request can ask for bool language = false; // honours WebSearchPlan::language - std::vector dials; // 0..3 + std::vector dials; // 0..kMaxWebSearchDials }; +// The most dials one service declares, so a host can size its settings +// page. Tavily needs all seven. +inline constexpr std::size_t kMaxWebSearchDials = 7; + // Every service, in menu order: the Free ones first, then the API-key ones. [[nodiscard]] const std::vector& web_search_services(); diff --git a/src/tools/web_search_services.cpp b/src/tools/web_search_services.cpp index 998c3c9..1aca902 100644 --- a/src/tools/web_search_services.cpp +++ b/src/tools/web_search_services.cpp @@ -47,6 +47,14 @@ std::vector dwmy(std::string d, std::string w, V(std::move(m), "Past month"), V(std::move(y), "Past year")}; } +// The same, led by Past hour, for the services that filter down to the hour. +std::vector hdwmy(std::string h, std::string d, std::string w, + std::string m, std::string y) { + auto v = dwmy(std::move(d), std::move(w), std::move(m), std::move(y)); + v.insert(v.begin(), V(std::move(h), "Past hour")); + return v; +} + // ISO 3166-1 alpha-2, upper case. The shared short list: enough to cover // where most users are, without a 250-row dropdown nobody scrolls. WebSearchDial country_dial(std::string help) { @@ -62,6 +70,16 @@ WebSearchDial country_dial(std::string help) { }}; } +// Firecrawl's documented categories, the same for both tiers. Only one is +// sent, which also keeps `gov` alone as Firecrawl requires. +WebSearchDial firecrawl_category_dial() { + return WebSearchDial{"category", "Category", + "Restrict to one kind of source. Developer: code repositories and " + "documentation sites. Government: laws, regulations and court opinions.", { + V("", "Any"), V("developer", "Developer"), V("research", "Research"), + V("pdf", "PDF"), V("gov", "Government")}}; +} + std::vector make_catalogue() { std::vector c; @@ -87,13 +105,42 @@ std::vector make_catalogue() { s.max_count = 30; s.dials.push_back(WebSearchDial{"region", "Region", "DuckDuckGo's kl parameter. Skews which country's results rank first.", { + // Every region on DuckDuckGo's URL-parameters help page, in its + // order. us-en is the default, so it is the "" row, not a second one. V("", "United States (default)"), V("wt-wt", "No region"), - V("uk-en", "United Kingdom"), V("au-en", "Australia"), - V("nz-en", "New Zealand"), V("ca-en", "Canada"), - V("in-en", "India"), V("de-de", "Germany"), - V("fr-fr", "France"), V("es-es", "Spain"), - V("it-it", "Italy"), V("br-pt", "Brazil"), - V("jp-jp", "Japan"), + V("xa-ar", "Arabia"), V("xa-en", "Arabia (en)"), + V("ar-es", "Argentina"), V("au-en", "Australia"), + V("at-de", "Austria"), V("be-fr", "Belgium (fr)"), + V("be-nl", "Belgium (nl)"), V("br-pt", "Brazil"), + V("bg-bg", "Bulgaria"), V("ca-en", "Canada"), + V("ca-fr", "Canada (fr)"), V("ct-ca", "Catalan"), + V("cl-es", "Chile"), V("cn-zh", "China"), + V("co-es", "Colombia"), V("hr-hr", "Croatia"), + V("cz-cs", "Czech Republic"), V("dk-da", "Denmark"), + V("ee-et", "Estonia"), V("fi-fi", "Finland"), + V("fr-fr", "France"), V("de-de", "Germany"), + V("gr-el", "Greece"), V("hk-tzh", "Hong Kong"), + V("hu-hu", "Hungary"), V("in-en", "India"), + V("id-id", "Indonesia"), V("id-en", "Indonesia (en)"), + V("ie-en", "Ireland"), V("il-he", "Israel"), + V("it-it", "Italy"), V("jp-jp", "Japan"), + V("kr-kr", "Korea"), V("lv-lv", "Latvia"), + V("lt-lt", "Lithuania"), V("xl-es", "Latin America"), + V("my-ms", "Malaysia"), V("my-en", "Malaysia (en)"), + V("mx-es", "Mexico"), V("nl-nl", "Netherlands"), + V("nz-en", "New Zealand"), V("no-no", "Norway"), + V("pe-es", "Peru"), V("ph-en", "Philippines"), + V("ph-tl", "Philippines (tl)"), V("pl-pl", "Poland"), + V("pt-pt", "Portugal"), V("ro-ro", "Romania"), + V("ru-ru", "Russia"), V("sg-en", "Singapore"), + V("sk-sk", "Slovak Republic"), V("sl-sl", "Slovenia"), + V("za-en", "South Africa"), V("es-es", "Spain"), + V("se-sv", "Sweden"), V("ch-de", "Switzerland (de)"), + V("ch-fr", "Switzerland (fr)"), V("ch-it", "Switzerland (it)"), + V("tw-tzh", "Taiwan"), V("th-th", "Thailand"), + V("tr-tr", "Turkey"), V("ua-uk", "Ukraine"), + V("uk-en", "United Kingdom"), V("ue-es", "United States (es)"), + V("ve-es", "Venezuela"), V("vn-vi", "Vietnam"), }}); s.dials.push_back(WebSearchDial{"safe", "Safe search", "DuckDuckGo's kp parameter.", { @@ -123,11 +170,8 @@ std::vector make_catalogue() { s.max_count = 100; s.dials.push_back(freshness_dial("freshness", "Only pages from this window (Firecrawl's tbs=qdr:).", - dwmy("qdr:d", "qdr:w", "qdr:m", "qdr:y"))); - s.dials.push_back(WebSearchDial{"category", "Category", - "Restrict to one kind of site.", { - V("", "Any"), V("github", "GitHub"), V("research", "Research"), - V("pdf", "PDF"), V("developer", "Developer docs")}}); + hdwmy("qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y"))); + s.dials.push_back(firecrawl_category_dial()); s.dials.push_back(country_dial("Firecrawl's country parameter.")); c.push_back(std::move(s)); } @@ -144,7 +188,10 @@ std::vector make_catalogue() { dwmy("pd", "pw", "pm", "py"))); s.dials.push_back(WebSearchDial{"safesearch", "Safe search", "", { V("", "Moderate (default)"), V("off", "Off"), V("strict", "Strict")}}); - s.dials.push_back(country_dial("Where results come from.")); + // Brave's default is US; ALL is its documented "no country" value. + auto country = country_dial("Where results come from."); + country.values.insert(country.values.begin() + 1, V("ALL", "All regions")); + s.dials.push_back(std::move(country)); c.push_back(std::move(s)); } { @@ -170,10 +217,9 @@ std::vector make_catalogue() { s.note = "Firecrawl's search API. 2 credits a search."; s.max_count = 100; s.dials.push_back(freshness_dial("freshness", - "Only pages from this window.", dwmy("qdr:d", "qdr:w", "qdr:m", "qdr:y"))); - s.dials.push_back(WebSearchDial{"category", "Category", "", { - V("", "Any"), V("github", "GitHub"), V("research", "Research"), - V("pdf", "PDF"), V("developer", "Developer docs")}}); + "Only pages from this window.", + hdwmy("qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y"))); + s.dials.push_back(firecrawl_category_dial()); s.dials.push_back(country_dial("Firecrawl's country parameter.")); c.push_back(std::move(s)); } @@ -184,16 +230,19 @@ std::vector make_catalogue() { s.note = "Exa's neural search. Highlights are returned as the snippet."; s.max_count = 100; s.dials.push_back(WebSearchDial{"type", "Mode", - "Fast and instant trade depth for speed.", { + "Fast and instant trade depth for speed. Deep lite, deep and deep " + "reasoning research and synthesise: about 4, 4-15 and 12-40 seconds, " + "so deep reasoning can outlast a host's request timeout.", { V("", "Auto (default)"), V("fast", "Fast"), V("instant", "Instant"), - V("deep", "Deep")}}); + V("deep-lite", "Deep lite"), V("deep", "Deep"), + V("deep-reasoning", "Deep reasoning")}}); s.dials.push_back(WebSearchDial{"category", "Category", "Restrict to one kind of source.", { V("", "Any"), V("news", "News"), V("publication", "Publications"), V("personal site", "Personal sites")}}); s.dials.push_back(freshness_dial("freshness", "Only pages published within this window (sent as startPublishedDate).", - dwmy("day", "week", "month", "year"))); + hdwmy("hour", "day", "week", "month", "year"))); c.push_back(std::move(s)); } { @@ -204,7 +253,8 @@ std::vector make_catalogue() { s.max_count = 20; s.language = true; s.dials.push_back(country_dial("Google's gl parameter.")); s.dials.push_back(freshness_dial("freshness", - "Google's tbs=qdr: time filter.", dwmy("qdr:d", "qdr:w", "qdr:m", "qdr:y"))); + "Google's tbs=qdr: time filter.", + hdwmy("qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y"))); s.dials.push_back(WebSearchDial{"autocorrect", "Spell correction", "", { V("", "On (default)"), V("false", "Off")}}); c.push_back(std::move(s)); @@ -316,10 +366,10 @@ std::string join_strings(const json& arr, std::string_view sep) { return out; } -// ISO 8601 UTC, `days` before `now`. -std::string iso_days_ago(std::chrono::system_clock::time_point now, int days) { +// ISO 8601 UTC, `hours` before `now`. +std::string iso_hours_ago(std::chrono::system_clock::time_point now, int hours) { const std::time_t t = std::chrono::system_clock::to_time_t( - now - std::chrono::hours(24 * days)); + now - std::chrono::hours(hours)); std::tm tm{}; #ifdef _WIN32 gmtime_s(&tm, &t); @@ -374,11 +424,12 @@ std::vector> browser_headers( return h; } -int days_for(std::string_view v) { - if (v == "day") return 1; - if (v == "week") return 7; - if (v == "month") return 30; - if (v == "year") return 365; +int hours_for(std::string_view v) { + if (v == "hour") return 1; + if (v == "day") return 24; + if (v == "week") return 24 * 7; + if (v == "month") return 24 * 30; + if (v == "year") return 24 * 365; return 0; } @@ -472,9 +523,9 @@ HttpRequest req_exa(const WebSearchQuery& q, std::string_view key) { if (!q.exclude.empty()) b["excludeDomains"] = q.exclude; if (auto v = q.dial("type"); !v.empty()) b["type"] = std::string{v}; if (auto v = q.dial("category"); !v.empty()) b["category"] = std::string{v}; - if (int d = days_for(q.dial("freshness")); d > 0 + if (int h = hours_for(q.dial("freshness")); h > 0 && q.now != std::chrono::system_clock::time_point{}) // host gave the time - b["startPublishedDate"] = iso_days_ago(q.now, d); + b["startPublishedDate"] = iso_hours_ago(q.now, h); return json_post("https://api.exa.ai/search", b, {{"x-api-key", std::string{key}}}); } diff --git a/tests/web_search_services_test.cpp b/tests/web_search_services_test.cpp index 9f72f57..4fe8c3d 100644 --- a/tests/web_search_services_test.cpp +++ b/tests/web_search_services_test.cpp @@ -47,6 +47,14 @@ std::string header(const HttpRequest& r, std::string_view name) { return {}; } +// The value ids one dial offers, in menu order ("" first). +std::vector dial_ids(std::string_view service, std::string_view dial) { + std::vector out; + for (const auto& d : svc(service).dials) + if (d.id == dial) for (const auto& v : d.values) out.push_back(v.id); + return out; +} + class Scripted : public HttpClient { public: // First rule whose needle is in the URL wins. @@ -97,7 +105,7 @@ TEST_CASE("web search: search catalogue is well formed") { CHECK(!s.id.empty()); CHECK(!s.label.empty()); CHECK(s.max_count >= 1); - CHECK(s.dials.size() <= 3); // the pane has three dial rows + CHECK(s.dials.size() <= kMaxWebSearchDials); CHECK(s.needs_key == !s.signup.empty()); for (const auto& d : s.dials) { REQUIRE(!d.values.empty()); @@ -138,6 +146,43 @@ TEST_CASE("web search: dials: only listed values reach a request") { CHECK(validate_dials(svc("ddg-free"), d).empty()); } +TEST_CASE("web search: dials offer each vendor's documented values") { + using V = std::vector; + const V hourly_tbs{"", "qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y"}; + const V categories{"", "developer", "research", "pdf", "gov"}; + const V exa_types{"", "fast", "instant", "deep-lite", "deep", "deep-reasoning"}; + const V exa_ages{"", "hour", "day", "week", "month", "year"}; + // Firecrawl's four categories; github is no longer one. + for (const char* id : {"firecrawl-free", "firecrawl-api"}) { + INFO(id); + CHECK(dial_ids(id, "category") == categories); + CHECK(dial_ids(id, "freshness") == hourly_tbs); + WebSearchSlot old{id, {}, {{"category", "github"}}}; + CHECK(validate_dials(svc(id), old).empty()); + } + CHECK(dial_ids("exa-api", "type") == exa_types); + CHECK(dial_ids("exa-api", "freshness") == exa_ages); + CHECK(dial_ids("serper-api", "freshness") == hourly_tbs); + // Brave's "no country" sits right under the default. + const auto brave = dial_ids("brave-api", "country"); + REQUIRE(brave.size() >= 2); + CHECK(brave[1] == "ALL"); + // DuckDuckGo: all 68 documented regions, us-en being the "" default. + const auto kl = dial_ids("ddg-free", "region"); + CHECK(kl.size() == 68); + CHECK(std::find(kl.begin(), kl.end(), "us-en") == kl.end()); + for (const char* r : {"wt-wt", "nl-nl", "xl-es", "ue-es", "tw-tzh", "vn-vi"}) + CHECK(std::find(kl.begin(), kl.end(), r) != kl.end()); + // No dial lists a value twice. + for (const auto& s : web_search_services()) + for (const auto& d : s.dials) { + V ids = dial_ids(s.id, d.id); + std::sort(ids.begin(), ids.end()); + const std::string where = s.id + "." + d.id; + CHECK(std::adjacent_find(ids.begin(), ids.end()) == ids.end(), where); + } +} + TEST_CASE("web search: keys are checked before they become a header") { CHECK(clean_api_key(" tvly-abc123 \n") == "tvly-abc123"); CHECK(clean_api_key("abc\r\nx-evil: 1").empty()); @@ -254,6 +299,12 @@ TEST_CASE("web search: Exa, Serper and Perplexity requests") { CHECK(eb["type"] == "fast"); CHECK(eb["category"] == "news"); CHECK(eb["startPublishedDate"] == "2025-10-02T08:53:20.000Z"); + q.dials = {{"freshness", "hour"}}; + CHECK(json::parse(build_web_search_request(svc("exa-api"), "k", q).body) + ["startPublishedDate"] == "2025-10-09T07:53:20.000Z"); + q.dials = {{"type", "deep-reasoning"}}; + CHECK(json::parse(build_web_search_request(svc("exa-api"), "k", q).body) + ["type"] == "deep-reasoning"); q.dials = {{"country", "NZ"}, {"freshness", "qdr:d"}, {"autocorrect", "false"}}; auto s = build_web_search_request(svc("serper-api"), "sp-k", q); From 3e6d8e6f3447bc769ba63981ebb532d39f8d7875 Mon Sep 17 00:00:00 2001 From: sail3r <82087684+sail3r@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:39:28 +1300 Subject: [PATCH 2/5] web_search: more quality dials for Tavily, Brave, Firecrawl, Exa, Perplexity Each is a documented request field that changes which results come back, not what they cost or how much text they carry. Tavily, both tiers (the key-less mode documents "all parameters"): - Exact phrases: exact_match, so quoted phrases match as written. - Language match: filter_by_language drops other languages instead of ranking them lower. It needs a language or Tavily answers 400, so it is sent only when the host gave one. - Country: Tavily takes a lower-case English name, not an ISO code, so the shared country codes are mapped; it is sent only with the General topic, the one Tavily accepts it with. - Safe search: safe_search, which the fast and ultra-fast depths do not support, so it is left out with them. - tavily-free also gets Depth, which only tavily-api had. Brave API: Spell correction off sends spellcheck=false. With it on, Brave searches its corrected query, which rewrites identifiers and error strings. Firecrawl, both tiers: - Safe search: safe=true. - Sort: Newest first adds sbd:1 to tbs, joined to the freshness window as "sbd:1,qdr:w", the combined form the reference documents. Exa: Safe search (moderation) and Country (userLocation, the bare ISO alpha-2 string form). Perplexity: Mode, web or fast (search_type). people is left out: it searches for people, not pages. --- src/tools/web_search_services.cpp | 120 +++++++++++++++++++++++++---- tests/web_search_services_test.cpp | 83 ++++++++++++++++++++ 2 files changed, 190 insertions(+), 13 deletions(-) diff --git a/src/tools/web_search_services.cpp b/src/tools/web_search_services.cpp index 1aca902..3486910 100644 --- a/src/tools/web_search_services.cpp +++ b/src/tools/web_search_services.cpp @@ -80,6 +80,42 @@ WebSearchDial firecrawl_category_dial() { V("pdf", "PDF"), V("gov", "Government")}}; } +// Firecrawl folds date sorting into tbs: sbd:1 alone, or joined to the +// freshness window as "sbd:1,qdr:w" (its documented combined form). +WebSearchDial firecrawl_sort_dial() { + return WebSearchDial{"sort", "Sort", + "Newest first sorts by date, inside the Freshness window if one is set.", { + V("", "Relevance (default)"), V("date", "Newest first")}}; +} + +WebSearchDial on_off_dial(std::string id, std::string label, std::string help) { + return WebSearchDial{std::move(id), std::move(label), std::move(help), { + V("", "Off (default)"), V("true", "On")}}; +} + +WebSearchDial tavily_depth_dial(std::string help) { + return WebSearchDial{"depth", "Depth", std::move(help), { + V("", "Basic (default)"), V("advanced", "Advanced"), + V("fast", "Fast"), V("ultra-fast", "Ultra fast")}}; +} + +// The dials both Tavily tiers share after topic, freshness and depth. The +// key-less mode documents "all parameters", so the free tier gets them too. +void add_tavily_filters(WebSearchService& s) { + s.dials.push_back(on_off_dial("exact_match", "Exact phrases", + "Only results that contain the query's \"quoted phrases\" as written, " + "with no synonyms.")); + s.dials.push_back(WebSearchDial{"language_filter", "Language match", + "Uses the web search language. Boost ranks it first; only drops every " + "other language. Not sent when no language is set.", { + V("", "Boost (default)"), V("only", "Only this language")}}); + s.dials.push_back(country_dial("Boosts results from this country. Sent only " + "with the General topic, the only one Tavily allows it with.")); + s.dials.push_back(on_off_dial("safe_search", "Safe search", + "Filters adult content. The fast and ultra-fast depths do not support " + "it, so it is not sent with them.")); +} + std::vector make_catalogue() { std::vector c; @@ -160,6 +196,10 @@ std::vector make_catalogue() { s.dials.push_back(freshness_dial("time_range", "Only pages from this window (Tavily's time_range).", dwmy("day", "week", "month", "year"))); + s.dials.push_back(tavily_depth_dial( + "Advanced reads more of each page and spends 2 of the key-less " + "budget's credits; fast and ultra-fast trade relevance for speed.")); + add_tavily_filters(s); c.push_back(std::move(s)); } { @@ -173,6 +213,9 @@ std::vector make_catalogue() { hdwmy("qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y"))); s.dials.push_back(firecrawl_category_dial()); s.dials.push_back(country_dial("Firecrawl's country parameter.")); + s.dials.push_back(on_off_dial("safe", "Safe search", + "Filters explicit content (Firecrawl's safe).")); + s.dials.push_back(firecrawl_sort_dial()); c.push_back(std::move(s)); } @@ -192,6 +235,10 @@ std::vector make_catalogue() { auto country = country_dial("Where results come from."); country.values.insert(country.values.begin() + 1, V("ALL", "All regions")); s.dials.push_back(std::move(country)); + s.dials.push_back(WebSearchDial{"spellcheck", "Spell correction", + "When on, Brave searches its corrected query instead of the one " + "sent, which can change identifiers and error strings.", { + V("", "On (default)"), V("false", "Off")}}); c.push_back(std::move(s)); } { @@ -200,14 +247,13 @@ std::vector make_catalogue() { s.signup = "https://app.tavily.com"; s.note = "Tavily's search API. 'Advanced' costs 2 credits a search."; s.max_count = 20; s.language = true; - s.dials.push_back(WebSearchDial{"depth", "Depth", - "Basic is 1 credit; advanced is 2 and reads more of each page.", { - V("", "Basic (default)"), V("advanced", "Advanced"), - V("fast", "Fast"), V("ultra-fast", "Ultra fast")}}); + s.dials.push_back(tavily_depth_dial( + "Basic is 1 credit; advanced is 2 and reads more of each page.")); s.dials.push_back(WebSearchDial{"topic", "Topic", "", { V("", "General"), V("news", "News"), V("finance", "Finance")}}); s.dials.push_back(freshness_dial("time_range", "Only pages from this window.", dwmy("day", "week", "month", "year"))); + add_tavily_filters(s); c.push_back(std::move(s)); } { @@ -221,6 +267,9 @@ std::vector make_catalogue() { hdwmy("qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y"))); s.dials.push_back(firecrawl_category_dial()); s.dials.push_back(country_dial("Firecrawl's country parameter.")); + s.dials.push_back(on_off_dial("safe", "Safe search", + "Filters explicit content (Firecrawl's safe).")); + s.dials.push_back(firecrawl_sort_dial()); c.push_back(std::move(s)); } { @@ -243,6 +292,10 @@ std::vector make_catalogue() { s.dials.push_back(freshness_dial("freshness", "Only pages published within this window (sent as startPublishedDate).", hdwmy("hour", "day", "week", "month", "year"))); + s.dials.push_back(on_off_dial("moderation", "Safe search", + "Filters unsafe content (Exa's moderation).")); + s.dials.push_back(country_dial( + "The searcher's country (Exa's userLocation); localises results.")); c.push_back(std::move(s)); } { @@ -272,6 +325,9 @@ std::vector make_catalogue() { V("", "Any time"), V("hour", "Past hour"), V("day", "Past day"), V("week", "Past week"), V("month", "Past month"), V("year", "Past year")}}); s.dials.push_back(country_dial("Perplexity's country parameter.")); + s.dials.push_back(WebSearchDial{"search_type", "Mode", + "Fast answers sooner with a lighter web search.", { + V("", "Web (default)"), V("fast", "Fast")}}); c.push_back(std::move(s)); } return c; @@ -464,16 +520,42 @@ HttpRequest req_ddg_free(const WebSearchQuery& q) { return r; } +// Tavily's country takes a lower-case English name from its own list, not +// an ISO code. Covers country_dial's codes; anything else is not sent. +std::string tavily_country(std::string_view code) { + static constexpr std::pair names[] = { + {"US", "united states"}, {"GB", "united kingdom"}, {"NZ", "new zealand"}, + {"AU", "australia"}, {"CA", "canada"}, {"DE", "germany"}, + {"FR", "france"}, {"ES", "spain"}, {"IT", "italy"}, + {"NL", "netherlands"}, {"BR", "brazil"}, {"IN", "india"}, + {"JP", "japan"}}; + for (const auto& [c, n] : names) + if (c == code) return std::string{n}; + return {}; +} + // Tavily, keyless or keyed. [D] POST /search ; [L] keyless = the header -// X-Tavily-Access-Mode: keyless, no Authorization. +// X-Tavily-Access-Mode: keyless, no Authorization. Three dials depend on +// others, per the reference: country only with topic general, safe_search +// not with the fast depths, filter_by_language only with a language (else +// a 400). HttpRequest req_tavily(const WebSearchQuery& q, std::string_view key) { json b = {{"query", q.text}, {"max_results", q.count}, {"include_answer", false}, {"include_raw_content", false}}; if (!q.exclude.empty()) b["exclude_domains"] = q.exclude; if (!q.language.empty()) b["language"] = q.language; - if (auto v = q.dial("topic"); !v.empty()) b["topic"] = std::string{v}; + const std::string_view topic = q.dial("topic"); + const std::string_view depth = q.dial("depth"); + if (!topic.empty()) b["topic"] = std::string{topic}; if (auto v = q.dial("time_range"); !v.empty()) b["time_range"] = std::string{v}; - if (auto v = q.dial("depth"); !v.empty()) b["search_depth"] = std::string{v}; + if (!depth.empty()) b["search_depth"] = std::string{depth}; + if (q.dial("exact_match") == "true") b["exact_match"] = true; + if (q.dial("language_filter") == "only" && !q.language.empty()) + b["filter_by_language"] = true; + if (auto c = tavily_country(q.dial("country")); !c.empty() && topic.empty()) + b["country"] = c; + if (q.dial("safe_search") == "true" && depth != "fast" && depth != "ultra-fast") + b["safe_search"] = true; std::vector> h; if (key.empty()) h.push_back({"x-tavily-access-mode", "keyless"}); else h.push_back({"authorization", "Bearer " + std::string{key}}); @@ -481,21 +563,26 @@ HttpRequest req_tavily(const WebSearchQuery& q, std::string_view key) { } // Firecrawl, keyless or keyed. [D]+[L] POST /v2/search ; limit 1-100 ; -// tbs=qdr:d|w|m|y ; categories is a list of strings ; excludeDomains takes -// bare hostnames. Keyless = no Authorization header at all. +// tbs=qdr:h|d|w|m|y, sbd:1 sorts by date and joins a window with a comma ; +// categories is a list of strings ; excludeDomains takes bare hostnames ; +// safe is a boolean. Keyless = no Authorization header at all. HttpRequest req_firecrawl(const WebSearchQuery& q, std::string_view key) { json b = {{"query", q.text}, {"limit", q.count}}; if (!q.exclude.empty()) b["excludeDomains"] = q.exclude; - if (auto v = q.dial("freshness"); !v.empty()) b["tbs"] = std::string{v}; + std::string tbs{q.dial("freshness")}; + if (q.dial("sort") == "date") tbs = tbs.empty() ? "sbd:1" : "sbd:1," + tbs; + if (!tbs.empty()) b["tbs"] = std::move(tbs); if (auto v = q.dial("category"); !v.empty()) b["categories"] = json::array({std::string{v}}); if (auto v = q.dial("country"); !v.empty()) b["country"] = std::string{v}; + if (q.dial("safe") == "true") b["safe"] = true; std::vector> h; if (!key.empty()) h.push_back({"authorization", "Bearer " + std::string{key}}); return json_post("https://api.firecrawl.dev/v2/search", b, std::move(h)); } // Brave API. [D] GET /res/v1/web/search ; header X-Subscription-Token ; -// count 1-20 ; text_decorations=false drops the highlight markers. +// count 1-20 ; text_decorations=false drops the highlight markers ; +// spellcheck=false searches the query as sent. HttpRequest req_brave_api(const WebSearchQuery& q, std::string_view key) { HttpRequest r; r.method = "GET"; @@ -510,13 +597,16 @@ HttpRequest req_brave_api(const WebSearchQuery& q, std::string_view key) { r.url += "&freshness=" + url_escape(v); if (auto v = q.dial("safesearch"); !v.empty()) r.url += "&safesearch=" + url_escape(v); + if (q.dial("spellcheck") == "false") + r.url += "&spellcheck=false"; r.headers.push_back({"accept", "application/json"}); r.headers.push_back({"x-subscription-token", std::string{key}}); return r; } // Exa. [D] POST /search ; x-api-key ; numResults 1-100 ; contents.highlights -// puts excerpts on each result ; startPublishedDate is an ISO 8601 instant. +// puts excerpts on each result ; startPublishedDate is an ISO 8601 instant ; +// userLocation takes the bare ISO alpha-2 string form ; moderation is boolean. HttpRequest req_exa(const WebSearchQuery& q, std::string_view key) { json b = {{"query", q.text}, {"numResults", q.count}, {"contents", {{"highlights", true}}}}; @@ -526,6 +616,8 @@ HttpRequest req_exa(const WebSearchQuery& q, std::string_view key) { if (int h = hours_for(q.dial("freshness")); h > 0 && q.now != std::chrono::system_clock::time_point{}) // host gave the time b["startPublishedDate"] = iso_hours_ago(q.now, h); + if (q.dial("moderation") == "true") b["moderation"] = true; + if (auto v = q.dial("country"); !v.empty()) b["userLocation"] = std::string{v}; return json_post("https://api.exa.ai/search", b, {{"x-api-key", std::string{key}}}); } @@ -543,12 +635,14 @@ HttpRequest req_serper(const WebSearchQuery& q, std::string_view key) { // Perplexity Search. [D] POST /search ; Bearer ; max_results 1-20 without a // tier ; country is ISO alpha-2 ; search_language_filter is a list of ISO 639-1 -// codes ; search_recency_filter is hour|day|week|month|year. +// codes ; search_recency_filter is hour|day|week|month|year ; search_type is +// web|fast|people (people is not offered: it searches for people, not pages). HttpRequest req_perplexity(const WebSearchQuery& q, std::string_view key) { json b = {{"query", q.text}, {"max_results", q.count}}; if (!q.language.empty()) b["search_language_filter"] = json::array({q.language}); if (auto v = q.dial("country"); !v.empty()) b["country"] = std::string{v}; if (auto v = q.dial("recency"); !v.empty()) b["search_recency_filter"] = std::string{v}; + if (auto v = q.dial("search_type"); !v.empty()) b["search_type"] = std::string{v}; return json_post("https://api.perplexity.ai/search", b, {{"authorization", "Bearer " + std::string{key}}}); } diff --git a/tests/web_search_services_test.cpp b/tests/web_search_services_test.cpp index 4fe8c3d..c4cc11e 100644 --- a/tests/web_search_services_test.cpp +++ b/tests/web_search_services_test.cpp @@ -183,6 +183,25 @@ TEST_CASE("web search: dials offer each vendor's documented values") { } } +TEST_CASE("web search: the new dials pass validation and fill the rows") { + auto dials = [](std::string_view id) { return svc(id).dials.size(); }; + CHECK(dials("tavily-free") == kMaxWebSearchDials); + CHECK(dials("tavily-api") == kMaxWebSearchDials); + CHECK(dials("firecrawl-free") == 5); + CHECK(dials("firecrawl-api") == 5); + CHECK(dials("brave-api") == 4); + CHECK(dials("exa-api") == 5); + CHECK(dials("perplexity-api") == 3); + WebSearchSlot t{"tavily-free", {}, {{"exact_match", "true"}, {"language_filter", "only"}, + {"country", "JP"}, {"safe_search", "true"}, + {"depth", "ultra-fast"}}}; + CHECK(validate_dials(svc("tavily-free"), t).size() == 5); + WebSearchSlot f{"firecrawl-api", {}, {{"sort", "date"}, {"safe", "true"}}}; + CHECK(validate_dials(svc("firecrawl-api"), f).size() == 2); + WebSearchSlot e{"exa-api", {}, {{"moderation", "true"}, {"country", "GB"}}}; + CHECK(validate_dials(svc("exa-api"), e).size() == 2); +} + TEST_CASE("web search: keys are checked before they become a header") { CHECK(clean_api_key(" tvly-abc123 \n") == "tvly-abc123"); CHECK(clean_api_key("abc\r\nx-evil: 1").empty()); @@ -251,6 +270,42 @@ TEST_CASE("web search: Tavily request: keyless header vs bearer key, exclusions CHECK(keyed.url.find("tvly-k") == std::string::npos); } +TEST_CASE("web search: Tavily filters follow the reference's rules") { + auto body = [](std::string_view id, const WebSearchQuery& q) { + return json::parse(build_web_search_request(svc(id), "", q).body); + }; + auto q = query(); q.language = "de"; + q.dials = {{"exact_match", "true"}, {"language_filter", "only"}, + {"country", "NZ"}, {"safe_search", "true"}, {"depth", "advanced"}}; + for (const char* id : {"tavily-free", "tavily-api"}) { + INFO(id); + auto b = body(id, q); + CHECK(b["exact_match"] == true); + CHECK(b["filter_by_language"] == true); + CHECK(b["country"] == "new zealand"); // Tavily wants the name + CHECK(b["safe_search"] == true); + CHECK(b["search_depth"] == "advanced"); // key-less now offers depth too + } + // country is for the general topic only. + q.dials = {{"country", "NZ"}, {"topic", "news"}}; + CHECK(!body("tavily-free", q).contains("country")); + // safe_search is not supported with the fast depths. + for (const char* depth : {"fast", "ultra-fast"}) { + q.dials = {{"safe_search", "true"}, {"depth", depth}}; + CHECK(!body("tavily-free", q).contains("safe_search"), depth); + } + // filter_by_language without a language is a 400, so it is not sent. + q.language.clear(); + q.dials = {{"language_filter", "only"}}; + CHECK(!body("tavily-free", q).contains("filter_by_language")); + // Defaults send nothing new. + q.dials = {}; + const auto plain = body("tavily-free", q); + for (const char* k : {"exact_match", "filter_by_language", "country", + "safe_search", "search_depth"}) + CHECK(!plain.contains(k), k); +} + TEST_CASE("web search: Firecrawl request") { auto q = query(); q.exclude = {"github.com"}; q.dials = {{"freshness", "qdr:m"}, {"category", "research"}, {"country", "NZ"}}; @@ -263,8 +318,21 @@ TEST_CASE("web search: Firecrawl request") { CHECK(b["tbs"] == "qdr:m"); CHECK(b["categories"] == json::array({"research"})); CHECK(b["country"] == "NZ"); + CHECK(!b.contains("safe")); CHECK(header(build_web_search_request(svc("firecrawl-api"), "fc-1", q), "authorization") == "Bearer fc-1"); + + // Sorting by date joins the window in tbs; alone it is just sbd:1. + q.dials = {{"freshness", "qdr:w"}, {"sort", "date"}, {"safe", "true"}}; + for (const char* id : {"firecrawl-free", "firecrawl-api"}) { + INFO(id); + auto sb = json::parse(build_web_search_request(svc(id), "k", q).body); + CHECK(sb["tbs"] == "sbd:1,qdr:w"); + CHECK(sb["safe"] == true); + } + q.dials = {{"sort", "date"}}; + CHECK(json::parse(build_web_search_request(svc("firecrawl-free"), "", q).body) + ["tbs"] == "sbd:1"); } TEST_CASE("web search: Brave API request keeps the key out of the URL") { @@ -284,6 +352,11 @@ TEST_CASE("web search: Brave API request keeps the key out of the URL") { q.language = "zh"; CHECK(build_web_search_request(svc("brave-api"), "k", q).url .find("search_lang=zh-hans") != std::string::npos); + // Spell correction off is sent; on is Brave's default and sends nothing. + q.dials = {{"spellcheck", "false"}, {"country", "ALL"}}; + const auto url = build_web_search_request(svc("brave-api"), "k", q).url; + CHECK(url.find("&spellcheck=false") != std::string::npos); + CHECK(url.find("&country=ALL") != std::string::npos); } TEST_CASE("web search: Exa, Serper and Perplexity requests") { @@ -305,6 +378,12 @@ TEST_CASE("web search: Exa, Serper and Perplexity requests") { q.dials = {{"type", "deep-reasoning"}}; CHECK(json::parse(build_web_search_request(svc("exa-api"), "k", q).body) ["type"] == "deep-reasoning"); + q.dials = {{"moderation", "true"}, {"country", "NZ"}}; + auto em = json::parse(build_web_search_request(svc("exa-api"), "k", q).body); + CHECK(em["moderation"] == true); + CHECK(em["userLocation"] == "NZ"); + CHECK(!eb.contains("moderation")); + CHECK(!eb.contains("userLocation")); q.dials = {{"country", "NZ"}, {"freshness", "qdr:d"}, {"autocorrect", "false"}}; auto s = build_web_search_request(svc("serper-api"), "sp-k", q); @@ -327,6 +406,10 @@ TEST_CASE("web search: Exa, Serper and Perplexity requests") { CHECK(pb["country"] == "NZ"); CHECK(pb["search_recency_filter"] == "month"); CHECK(pb["search_language_filter"] == json::array({"fr"})); + CHECK(!pb.contains("search_type")); + q.dials = {{"search_type", "fast"}}; + CHECK(json::parse(build_web_search_request(svc("perplexity-api"), "k", q).body) + ["search_type"] == "fast"); } TEST_CASE("web search: reply parsers") { From f0eba67a4a1556c3acd385277d4ae81c5ffe57e2 Mon Sep 17 00:00:00 2001 From: sail3r <82087684+sail3r@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:44:29 +1300 Subject: [PATCH 3/5] web_search: free-text dials, for Brave Goggles and Firecrawl location Every dial was a fixed list of values, so a field whose value is a URL or a place name could not be offered. A dial can now be free text: WebSearchDial::text says which kind (Line or HttpsUrl), max_bytes bounds it, and values stays empty. A list dial is unchanged. web_search_dial_value_ok is the one rule for both kinds, and is public so a host can check what a user typed against the same rule the request uses. A text value must be valid UTF-8 within max_bytes with no control characters (no newline can start a second Goggles rule); an HttpsUrl must also start with https:// and carry no spaces. validate_dials now calls it, so a hand-edited value that fails is still dropped, never sent. Two text dials: - Brave API, Goggle: one hosted Goggle URL, sent as goggles=. Goggles re-rank Brave's results by the file's rules. Brave advises hosted files over inline rules, which are capped by URL length, so inline rules are not accepted. Limit 2048 bytes. - Firecrawl, both tiers, Location: free text such as "San Francisco,California,United States", sent as location=. Firecrawl advises setting Country with it. Limit 200 bytes. --- include/mcp/tools/web_search.hpp | 28 +++++++++++++--- src/tools/web_search_services.cpp | 49 +++++++++++++++++++++++++--- src/tools/web_search_services.hpp | 5 +-- tests/web_search_services_test.cpp | 52 +++++++++++++++++++++++++++--- 4 files changed, 119 insertions(+), 15 deletions(-) diff --git a/include/mcp/tools/web_search.hpp b/include/mcp/tools/web_search.hpp index bd3c756..952a841 100644 --- a/include/mcp/tools/web_search.hpp +++ b/include/mcp/tools/web_search.hpp @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -40,15 +41,32 @@ struct WebSearchDialValue { std::string label; }; +// What a dial's value is. A list dial stores one of its `values`; a text +// dial stores what the user typed, checked against its kind and max_bytes. +enum class WebSearchText : std::uint8_t { + None, // a list dial + Line, // one line of printable UTF-8 ("San Francisco,California") + HttpsUrl, // one https:// URL with no spaces +}; + // One extra knob a service honours (freshness, region, ...). A service -// declares at most kMaxWebSearchDials. `values[0]` is always the default (id ""). +// declares at most kMaxWebSearchDials. For a list dial `values[0]` is always +// the default (id ""). A text dial has no values; "" is its default. struct WebSearchDial { std::string id; std::string label; std::string help; - std::vector values; + std::vector values; // empty for a text dial + WebSearchText text = WebSearchText::None; + std::size_t max_bytes = 0; // text dials only }; +// Whether `value` may be sent for `d`: a listed id, or text of the dial's +// kind within max_bytes. "" (the default) is never sent, so it is not ok. +// Hosts can check what a user typed with the same rule the request uses. +[[nodiscard]] bool web_search_dial_value_ok(const WebSearchDial& d, + std::string_view value) noexcept; + struct WebSearchService { std::string id; // stable, stored by hosts: "brave-free", "exa-api" std::string label; // shown to people and in the tool's "[via ...]" line @@ -74,9 +92,9 @@ inline constexpr std::size_t kMaxWebSearchDials = 7; struct WebSearchSlot { std::string service; // WebSearchService::id std::string api_key; // empty for Free services - // dial id -> value id. A dial that is absent, or whose value is not one - // the service lists, is ignored -- a stale setting can never put arbitrary - // text into a request. + // dial id -> value. A dial that is absent, or whose value fails + // web_search_dial_value_ok, is ignored -- a stale setting can never put + // arbitrary text into a request. std::vector> dials; }; diff --git a/src/tools/web_search_services.cpp b/src/tools/web_search_services.cpp index 3486910..f151856 100644 --- a/src/tools/web_search_services.cpp +++ b/src/tools/web_search_services.cpp @@ -88,6 +88,16 @@ WebSearchDial firecrawl_sort_dial() { V("", "Relevance (default)"), V("date", "Newest first")}}; } +// Firecrawl's location is free text; its reference pairs it with country. +WebSearchDial firecrawl_location_dial() { + WebSearchDial d{"location", "Location", + "A place to search from, e.g. San Francisco,California,United States. " + "Works best with Country set too.", {}}; + d.text = WebSearchText::Line; + d.max_bytes = 200; + return d; +} + WebSearchDial on_off_dial(std::string id, std::string label, std::string help) { return WebSearchDial{std::move(id), std::move(label), std::move(help), { V("", "Off (default)"), V("true", "On")}}; @@ -216,6 +226,7 @@ std::vector make_catalogue() { s.dials.push_back(on_off_dial("safe", "Safe search", "Filters explicit content (Firecrawl's safe).")); s.dials.push_back(firecrawl_sort_dial()); + s.dials.push_back(firecrawl_location_dial()); c.push_back(std::move(s)); } @@ -239,6 +250,14 @@ std::vector make_catalogue() { "When on, Brave searches its corrected query instead of the one " "sent, which can change identifiers and error strings.", { V("", "On (default)"), V("false", "Off")}}); + // One hosted Goggle; Brave advises hosted files over inline rules. + WebSearchDial goggle{"goggles", "Goggle", + "Re-ranks results by a Goggle's rules: the https URL of a Goggle " + "file on GitHub, GitLab or a Gist, registered at " + "search.brave.com/goggles/create.", {}}; + goggle.text = WebSearchText::HttpsUrl; + goggle.max_bytes = 2048; + s.dials.push_back(std::move(goggle)); c.push_back(std::move(s)); } { @@ -270,6 +289,7 @@ std::vector make_catalogue() { s.dials.push_back(on_off_dial("safe", "Safe search", "Filters explicit content (Firecrawl's safe).")); s.dials.push_back(firecrawl_sort_dial()); + s.dials.push_back(firecrawl_location_dial()); c.push_back(std::move(s)); } { @@ -575,6 +595,7 @@ HttpRequest req_firecrawl(const WebSearchQuery& q, std::string_view key) { if (auto v = q.dial("category"); !v.empty()) b["categories"] = json::array({std::string{v}}); if (auto v = q.dial("country"); !v.empty()) b["country"] = std::string{v}; if (q.dial("safe") == "true") b["safe"] = true; + if (auto v = q.dial("location"); !v.empty()) b["location"] = std::string{v}; std::vector> h; if (!key.empty()) h.push_back({"authorization", "Bearer " + std::string{key}}); return json_post("https://api.firecrawl.dev/v2/search", b, std::move(h)); @@ -599,6 +620,8 @@ HttpRequest req_brave_api(const WebSearchQuery& q, std::string_view key) { r.url += "&safesearch=" + url_escape(v); if (q.dial("spellcheck") == "false") r.url += "&spellcheck=false"; + if (auto v = q.dial("goggles"); !v.empty()) + r.url += "&goggles=" + url_escape(v); r.headers.push_back({"accept", "application/json"}); r.headers.push_back({"x-subscription-token", std::string{key}}); return r; @@ -748,10 +771,8 @@ validate_dials(const WebSearchService& svc, const WebSearchSlot& slot) { std::vector> out; for (const auto& d : svc.dials) { for (const auto& [id, val] : slot.dials) { - if (id != d.id || val.empty()) continue; - const bool known = std::any_of(d.values.begin(), d.values.end(), - [&](const WebSearchDialValue& v) { return v.id == val; }); - if (known) out.emplace_back(id, val); + if (id != d.id) continue; + if (web_search_dial_value_ok(d, val)) out.emplace_back(id, val); break; } } @@ -910,6 +931,26 @@ JsonHits parse_json_hits(std::string_view service_id, std::string_view body, namespace mcp::tools { +bool web_search_dial_value_ok(const WebSearchDial& d, std::string_view value) noexcept { + if (value.empty()) return false; + if (d.text == WebSearchText::None) + return std::any_of(d.values.begin(), d.values.end(), + [&](const WebSearchDialValue& v) { return v.id == value; }); + if (value.size() > d.max_bytes || !util::is_valid_utf8(value)) return false; + // No control characters in either kind: the value lands in a URL or a + // JSON string, and a newline could start a second Goggles rule. + const bool url = d.text == WebSearchText::HttpsUrl; + for (char ch : value) { + const auto u = static_cast(ch); + if (u < 0x20 || u == 0x7f || (url && u == ' ')) return false; + } + if (url) { + constexpr std::string_view kScheme = "https://"; + if (value.size() <= kScheme.size() || !value.starts_with(kScheme)) return false; + } + return true; +} + const std::vector& web_search_services() { static const std::vector c = detail::make_catalogue(); return c; diff --git a/src/tools/web_search_services.hpp b/src/tools/web_search_services.hpp index 0254a6a..8aacf59 100644 --- a/src/tools/web_search_services.hpp +++ b/src/tools/web_search_services.hpp @@ -70,8 +70,9 @@ struct WebSearchQuery { [[nodiscard]] std::string_view dial(std::string_view id) const noexcept; }; -// Keep only the dial choices `svc` actually offers, with a value it lists, -// and drop the default. Anything else is discarded rather than forwarded. +// Keep only the dials `svc` actually offers, with a value that passes +// web_search_dial_value_ok, and drop the default. Anything else is discarded +// rather than forwarded. [[nodiscard]] std::vector> validate_dials(const WebSearchService& svc, const WebSearchSlot& slot); diff --git a/tests/web_search_services_test.cpp b/tests/web_search_services_test.cpp index c4cc11e..ecbe4e9 100644 --- a/tests/web_search_services_test.cpp +++ b/tests/web_search_services_test.cpp @@ -108,6 +108,12 @@ TEST_CASE("web search: search catalogue is well formed") { CHECK(s.dials.size() <= kMaxWebSearchDials); CHECK(s.needs_key == !s.signup.empty()); for (const auto& d : s.dials) { + INFO(d.id); + if (d.text != WebSearchText::None) { // free text: no list, a bound + CHECK(d.values.empty()); + CHECK(d.max_bytes > 0); + continue; + } REQUIRE(!d.values.empty()); CHECK(d.values.front().id.empty()); // [0] is "send nothing" } @@ -187,9 +193,9 @@ TEST_CASE("web search: the new dials pass validation and fill the rows") { auto dials = [](std::string_view id) { return svc(id).dials.size(); }; CHECK(dials("tavily-free") == kMaxWebSearchDials); CHECK(dials("tavily-api") == kMaxWebSearchDials); - CHECK(dials("firecrawl-free") == 5); - CHECK(dials("firecrawl-api") == 5); - CHECK(dials("brave-api") == 4); + CHECK(dials("firecrawl-free") == 6); + CHECK(dials("firecrawl-api") == 6); + CHECK(dials("brave-api") == 5); CHECK(dials("exa-api") == 5); CHECK(dials("perplexity-api") == 3); WebSearchSlot t{"tavily-free", {}, {{"exact_match", "true"}, {"language_filter", "only"}, @@ -202,6 +208,37 @@ TEST_CASE("web search: the new dials pass validation and fill the rows") { CHECK(validate_dials(svc("exa-api"), e).size() == 2); } +TEST_CASE("web search: text dials take checked free text") { + const WebSearchDial* goggle = nullptr; + for (const auto& d : svc("brave-api").dials) if (d.id == "goggles") goggle = &d; + REQUIRE(goggle != nullptr); + const std::string url = "https://raw.githubusercontent.com/brave/goggles-quickstart/" + "main/goggles/tech_blogs.goggle"; + CHECK(web_search_dial_value_ok(*goggle, url)); + CHECK(!web_search_dial_value_ok(*goggle, "")); + CHECK(!web_search_dial_value_ok(*goggle, "https://")); + CHECK(!web_search_dial_value_ok(*goggle, "http://example.com/a.goggle")); + CHECK(!web_search_dial_value_ok(*goggle, "$boost=3,site=dev.to")); // inline rules + CHECK(!web_search_dial_value_ok(*goggle, "https://a.com/x y")); + CHECK(!web_search_dial_value_ok(*goggle, "https://a.com/x\n$discard")); + CHECK(!web_search_dial_value_ok(*goggle, "https://a.com/" + std::string(2048, 'x'))); + + const WebSearchDial* where = nullptr; + for (const auto& d : svc("firecrawl-free").dials) if (d.id == "location") where = &d; + REQUIRE(where != nullptr); + CHECK(web_search_dial_value_ok(*where, "San Francisco,California,United States")); + CHECK(web_search_dial_value_ok(*where, "Z\xc3\xbcrich")); // UTF-8 is fine + CHECK(!web_search_dial_value_ok(*where, "Z\xfcrich")); // Latin-1 is not + CHECK(!web_search_dial_value_ok(*where, "line one\r\nline two")); + CHECK(!web_search_dial_value_ok(*where, std::string(201, 'a'))); + + // validate_dials applies the same rule, so a bad value never reaches a slot. + WebSearchSlot ok{"brave-api", {}, {{"goggles", url}}}; + CHECK(validate_dials(svc("brave-api"), ok).size() == 1); + WebSearchSlot bad{"brave-api", {}, {{"goggles", "javascript:alert(1)"}}}; + CHECK(validate_dials(svc("brave-api"), bad).empty()); +} + TEST_CASE("web search: keys are checked before they become a header") { CHECK(clean_api_key(" tvly-abc123 \n") == "tvly-abc123"); CHECK(clean_api_key("abc\r\nx-evil: 1").empty()); @@ -323,12 +360,14 @@ TEST_CASE("web search: Firecrawl request") { "authorization") == "Bearer fc-1"); // Sorting by date joins the window in tbs; alone it is just sbd:1. - q.dials = {{"freshness", "qdr:w"}, {"sort", "date"}, {"safe", "true"}}; + q.dials = {{"freshness", "qdr:w"}, {"sort", "date"}, {"safe", "true"}, + {"location", "Germany"}}; for (const char* id : {"firecrawl-free", "firecrawl-api"}) { INFO(id); auto sb = json::parse(build_web_search_request(svc(id), "k", q).body); CHECK(sb["tbs"] == "sbd:1,qdr:w"); CHECK(sb["safe"] == true); + CHECK(sb["location"] == "Germany"); } q.dials = {{"sort", "date"}}; CHECK(json::parse(build_web_search_request(svc("firecrawl-free"), "", q).body) @@ -357,6 +396,11 @@ TEST_CASE("web search: Brave API request keeps the key out of the URL") { const auto url = build_web_search_request(svc("brave-api"), "k", q).url; CHECK(url.find("&spellcheck=false") != std::string::npos); CHECK(url.find("&country=ALL") != std::string::npos); + CHECK(url.find("goggles") == std::string::npos); + // A Goggle URL is percent-encoded into its own parameter. + q.dials = {{"goggles", "https://g.example/a b.goggle?x=1&y"}}; + CHECK(build_web_search_request(svc("brave-api"), "k", q).url.find( + "&goggles=https%3A%2F%2Fg.example%2Fa%20b.goggle%3Fx%3D1%26y") != std::string::npos); } TEST_CASE("web search: Exa, Serper and Perplexity requests") { From 1e69b256ebc8296d9b90731fb4f637cefa727f60 Mon Sep 17 00:00:00 2001 From: sail3r <82087684+sail3r@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:03:16 +1300 Subject: [PATCH 4/5] web_search: Firecrawl paper records, each with a link From 2026-11-16 Firecrawl's research category searches its Research Index and answers in data.research with paper records (paperId, primaryId, ids, title, abstract, score) instead of web pages in data.web. The parser read data.web only and required it, so a research search would have been reported unreadable and the next slot tried. Both containers are read now, web results first, the count covering both. A reply with either is readable; one with neither still is not. Until the switch data.research is absent and nothing changes. A paper record has no url, and a hit without one is dropped, so one is made from primaryId. Firecrawl documents four forms, each mapped to the paper's own page: arxiv: https://arxiv.org/abs/ pmid: https://pubmed.ncbi.nlm.nih.gov// pmcid: https://pmc.ncbi.nlm.nih.gov/articles/PMC/ doi: https://doi.org/ Whether a pmcid keeps PMC's own "PMC" prefix is not documented, so both spellings give the same link. Any other form, or no primaryId, falls back to Firecrawl's paper endpoint by paperId, the canonical id every record has, so a namespace Firecrawl adds later still gets a link. Ids are percent-encoded, keeping the slash a DOI or an old arXiv id holds. The abstract is the snippet. --- src/tools/web_search_services.cpp | 69 +++++++++++++++++++++++++++++- tests/web_search_services_test.cpp | 49 +++++++++++++++++++++ 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/src/tools/web_search_services.cpp b/src/tools/web_search_services.cpp index f151856..284ccb5 100644 --- a/src/tools/web_search_services.cpp +++ b/src/tools/web_search_services.cpp @@ -697,6 +697,63 @@ std::vector collect(const json& arr, const char* url_key, return out; } +// Percent-encode an id for a URL path, keeping '/' (a DOI and an old-style +// arXiv id such as hep-th/9901001 both carry one). +std::string path_escape(std::string_view s) { + std::string out; + for (std::size_t from = 0;;) { + const auto slash = s.find('/', from); + out += url_escape(s.substr(from, slash == std::string_view::npos + ? std::string_view::npos : slash - from)); + if (slash == std::string_view::npos) return out; + out += '/'; + from = slash + 1; + } +} + +// A Firecrawl paper record carries ids, not a link. The four primaryId forms +// Firecrawl documents become the paper's own page; anything else falls back +// to Firecrawl's paper endpoint by the record's canonical paperId, which +// answers for every record. "" when the record has neither. +std::string paper_url(const json& r) { + const std::string primary = str_at(r, "primaryId"); + if (const auto colon = primary.find(':'); + colon != std::string::npos && colon + 1 < primary.size()) { + const std::string ns = lower(primary.substr(0, colon)); + std::string id = primary.substr(colon + 1); + if (ns == "arxiv") return "https://arxiv.org/abs/" + path_escape(id); + if (ns == "pmid") return "https://pubmed.ncbi.nlm.nih.gov/" + url_escape(id) + "/"; + if (ns == "pmcid") { + // PMC writes its ids as PMC1234567; whether Firecrawl keeps the + // prefix is not documented, so both spellings are accepted. + if (lower(id.substr(0, 3)) != "pmc") id = "PMC" + id; + return "https://pmc.ncbi.nlm.nih.gov/articles/" + url_escape(id) + "/"; + } + if (ns == "doi") return "https://doi.org/" + path_escape(id); + } + if (const std::string paper = str_at(r, "paperId"); !paper.empty()) + return "https://api.firecrawl.dev/v2/search/research/papers/" + url_escape(paper); + return {}; +} + +// Firecrawl's data.research: paper records (title, abstract, ids). +std::vector collect_papers(const json& arr, int count) { + std::vector out; + if (!arr.is_array()) return out; + for (const auto& r : arr) { + if (static_cast(out.size()) >= count) break; + if (!r.is_object()) continue; + SearchHit h; + h.url = paper_url(r); + if (h.url.empty() || h.url.size() > 2048) continue; // never a cut-off link + h.title = tidy(str_at(r, "title"), 300); + if (h.title.empty()) h.title = h.url; + h.snippet = tidy(str_at(r, "abstract"), 500); + out.push_back(std::move(h)); + } + return out; +} + } // namespace std::string_view WebSearchQuery::dial(std::string_view id) const noexcept { @@ -905,8 +962,16 @@ JsonHits parse_json_hits(std::string_view service_id, std::string_view body, out.hits = collect(*need(field(j, "results")), "url", "title", {"content"}, count); } else if (service_id == "firecrawl-free" || service_id == "firecrawl-api") { - out.hits = collect(*need(field(field(j, "data"), "web")), "url", "title", - {"description"}, count); + // Web results, then paper records. From 2026-11-16 the research + // category answers in data.research and no longer in data.web, so + // either container makes a readable reply. + const json& data = field(j, "data"); + const json& web = field(data, "web"); + const json& papers = field(data, "research"); + if (!web.is_array() && !papers.is_array()) out.ok = false; + out.hits = collect(web, "url", "title", {"description"}, count); + for (auto& h : collect_papers(papers, count - static_cast(out.hits.size()))) + out.hits.push_back(std::move(h)); } else if (service_id == "exa-api") { out.hits = collect(*need(field(j, "results")), "url", "title", {"highlights", "summary", "text"}, count); diff --git a/tests/web_search_services_test.cpp b/tests/web_search_services_test.cpp index ecbe4e9..6a586b8 100644 --- a/tests/web_search_services_test.cpp +++ b/tests/web_search_services_test.cpp @@ -510,6 +510,55 @@ TEST_CASE("web search: reply parsers") { CHECK(!parse_json_hits("firecrawl-free", R"({"success":false,"error":"x"})", 5).ok); } +TEST_CASE("web search: Firecrawl paper records each get a link") { + // The record shape is the documented research example. + auto p = parse_json_hits("firecrawl-api", + R"({"success":true,"data":{"research":[)" + R"({"paperId":"2014215642691656232","primaryId":"arxiv:2105.05233",)" + R"("ids":{"arxiv":["2105.05233"]},"title":"Diffusion Models Beat GANs on Image Synthesis",)" + R"("abstract":"We show that diffusion models can achieve image sample quality.","score":0.016},)" + R"({"paperId":"1","primaryId":"pmid:31452104","title":"P"},)" + R"({"paperId":"2","primaryId":"pmcid:PMC6824467","title":"C"},)" + R"({"paperId":"3","primaryId":"pmcid:6824467","title":"C2"},)" + R"({"paperId":"4","primaryId":"doi:10.1038/s41586-021-03819-2","title":"D"},)" + R"({"paperId":"5","primaryId":"arxiv:hep-th/9901001","title":"Old arXiv"},)" + R"({"paperId":"6","primaryId":"doi:10.1000/a b","title":"Odd DOI"},)" + R"({"paperId":"7","primaryId":"openalex:W2963403868","title":"Unknown kind"},)" + R"({"paperId":"8","title":"No primary id"},)" + R"({"title":"Nothing to link"})" + R"(]}})", 20); + REQUIRE(p.ok); + REQUIRE(p.hits.size() == 9); // the last record has no id at all + CHECK(p.hits[0].url == "https://arxiv.org/abs/2105.05233"); + CHECK(p.hits[0].title == "Diffusion Models Beat GANs on Image Synthesis"); + CHECK(p.hits[0].snippet == "We show that diffusion models can achieve image sample quality."); + CHECK(p.hits[1].url == "https://pubmed.ncbi.nlm.nih.gov/31452104/"); + CHECK(p.hits[2].url == "https://pmc.ncbi.nlm.nih.gov/articles/PMC6824467/"); + CHECK(p.hits[3].url == "https://pmc.ncbi.nlm.nih.gov/articles/PMC6824467/"); + CHECK(p.hits[4].url == "https://doi.org/10.1038/s41586-021-03819-2"); + CHECK(p.hits[5].url == "https://arxiv.org/abs/hep-th/9901001"); + CHECK(p.hits[6].url == "https://doi.org/10.1000/a%20b"); + // A kind of id Firecrawl does not document, or none: its own paper page. + CHECK(p.hits[7].url == "https://api.firecrawl.dev/v2/search/research/papers/7"); + CHECK(p.hits[8].url == "https://api.firecrawl.dev/v2/search/research/papers/8"); + + // Web results come first, and the count covers both lists. + auto mixed = parse_json_hits("firecrawl-free", + R"({"success":true,"data":{"web":[{"url":"https://a.test/","title":"A"}],)" + R"("research":[{"paperId":"1","primaryId":"pmid:1","title":"P1"},)" + R"({"paperId":"2","primaryId":"pmid:2","title":"P2"}]}})", 2); + REQUIRE(mixed.ok); + REQUIRE(mixed.hits.size() == 2); + CHECK(mixed.hits[0].url == "https://a.test/"); + CHECK(mixed.hits[1].url == "https://pubmed.ncbi.nlm.nih.gov/1/"); + + // Either list makes a readable reply; an empty one is "no results". + auto empty = parse_json_hits("firecrawl-free", R"({"success":true,"data":{"research":[]}})", 5); + CHECK(empty.ok); + CHECK(empty.hits.empty()); + CHECK(!parse_json_hits("firecrawl-free", R"({"success":true,"data":{}})", 5).ok); +} + TEST_CASE("web search: failures are told apart") { const auto& brave = svc("brave-free"); const auto& tavily = svc("tavily-api"); From 791957dd0bcf41f7bc62d3f88b532fb866d1612f Mon Sep 17 00:00:00 2001 From: sail3r <82087684+sail3r@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:41:51 +1300 Subject: [PATCH 5/5] web_search: Perplexity takes excluded sites as its domain filter Perplexity was the one keyed service that still got excluded sites as -site: words in the query. The catalogue note said its domain filter had no documented way to exclude. It has: the Search API's domain-filter guide and the /search reference list a "-" in front of a host as the deny form of search_domain_filter, at most 20 entries, and a list that is all allow or all deny. Whether /search honours -site: in the query text is documented nowhere. Perplexity now has a native exclude, like Tavily, Firecrawl and Exa: the -site: tokens are lifted out of the query and sent as search_domain_filter ["-reddit.com", ...]. split_site_exclusions already lifts at most 20, so the list never exceeds Perplexity's limit; a 21st and later stay in the query as before. Only the deny form is ever sent, so allow and deny are never mixed. The catalogue note no longer says excluding is undocumented. --- src/tools/web_search_services.cpp | 15 ++++++++++----- src/tools/web_search_services.hpp | 3 ++- tests/web_search_services_test.cpp | 23 +++++++++++++++++++++++ 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/tools/web_search_services.cpp b/src/tools/web_search_services.cpp index 284ccb5..7571b48 100644 --- a/src/tools/web_search_services.cpp +++ b/src/tools/web_search_services.cpp @@ -336,9 +336,7 @@ std::vector make_catalogue() { WebSearchService s; s.id = "perplexity-api"; s.label = "Perplexity API key"; s.needs_key = true; s.signup = "https://docs.perplexity.ai"; - s.note = "Perplexity's Search API (ranked results, not an answer). " - "'Never return' sites are left in the query as -site: here, " - "because its domain-filter syntax for excluding is not documented."; + s.note = "Perplexity's Search API (ranked results, not an answer)."; s.max_count = 20; s.language = true; s.dials.push_back(WebSearchDial{"recency", "Freshness", "Only pages from this window (search_recency_filter).", { @@ -659,13 +657,20 @@ HttpRequest req_serper(const WebSearchQuery& q, std::string_view key) { // Perplexity Search. [D] POST /search ; Bearer ; max_results 1-20 without a // tier ; country is ISO alpha-2 ; search_language_filter is a list of ISO 639-1 // codes ; search_recency_filter is hour|day|week|month|year ; search_type is -// web|fast|people (people is not offered: it searches for people, not pages). +// web|fast|people (people is not offered: it searches for people, not pages) ; +// search_domain_filter takes at most 20 hosts, a "-" in front excluding one, +// and is either all allow or all deny (only deny is sent). HttpRequest req_perplexity(const WebSearchQuery& q, std::string_view key) { json b = {{"query", q.text}, {"max_results", q.count}}; if (!q.language.empty()) b["search_language_filter"] = json::array({q.language}); if (auto v = q.dial("country"); !v.empty()) b["country"] = std::string{v}; if (auto v = q.dial("recency"); !v.empty()) b["search_recency_filter"] = std::string{v}; if (auto v = q.dial("search_type"); !v.empty()) b["search_type"] = std::string{v}; + if (!q.exclude.empty()) { + json deny = json::array(); + for (const auto& host : q.exclude) deny.push_back("-" + host); + b["search_domain_filter"] = std::move(deny); + } return json_post("https://api.perplexity.ai/search", b, {{"authorization", "Bearer " + std::string{key}}}); } @@ -843,7 +848,7 @@ bool is_html_service(std::string_view id) noexcept { bool has_native_exclude(std::string_view id) noexcept { return id == "tavily-free" || id == "tavily-api" || id == "firecrawl-free" || id == "firecrawl-api" - || id == "exa-api"; + || id == "exa-api" || id == "perplexity-api"; } std::string clean_api_key(std::string_view key) { diff --git a/src/tools/web_search_services.hpp b/src/tools/web_search_services.hpp index 8aacf59..13c2610 100644 --- a/src/tools/web_search_services.hpp +++ b/src/tools/web_search_services.hpp @@ -52,7 +52,8 @@ enum class WebSearchFailure { // unchanged with an empty list, so we never send an empty query. struct SiteSplit { std::string query; - std::vector exclude; // lower-case hosts, at most 20 + std::vector exclude; // lower-case hosts, at most 20 (also + // Perplexity's limit); more stay in `query` }; [[nodiscard]] SiteSplit split_site_exclusions(std::string_view query); diff --git a/tests/web_search_services_test.cpp b/tests/web_search_services_test.cpp index 6a586b8..6e272e5 100644 --- a/tests/web_search_services_test.cpp +++ b/tests/web_search_services_test.cpp @@ -259,9 +259,16 @@ TEST_CASE("web search: -site: becomes a native filter only where the service has auto only = split_site_exclusions("-site:reddit.com"); CHECK(only.query == "-site:reddit.com"); CHECK(only.exclude.empty()); + // At most 20 lifted out (Perplexity's limit); the rest stay in the text. + std::string many = "q"; + for (int i = 0; i < 21; ++i) many += " -site:s" + std::to_string(i) + ".test"; + auto capped = split_site_exclusions(many); + CHECK(capped.exclude.size() == 20); + CHECK(capped.query == "q -site:s20.test"); CHECK(has_native_exclude("tavily-free")); CHECK(has_native_exclude("exa-api")); + CHECK(has_native_exclude("perplexity-api")); CHECK(!has_native_exclude("brave-free")); CHECK(!has_native_exclude("serper-api")); } @@ -451,9 +458,16 @@ TEST_CASE("web search: Exa, Serper and Perplexity requests") { CHECK(pb["search_recency_filter"] == "month"); CHECK(pb["search_language_filter"] == json::array({"fr"})); CHECK(!pb.contains("search_type")); + CHECK(!pb.contains("search_domain_filter")); q.dials = {{"search_type", "fast"}}; CHECK(json::parse(build_web_search_request(svc("perplexity-api"), "k", q).body) ["search_type"] == "fast"); + // Excluded hosts become the deny form of its domain filter. + q.dials = {}; + q.exclude = {"reddit.com", "pinterest.com"}; + const json deny = json::array({"-reddit.com", "-pinterest.com"}); + CHECK(json::parse(build_web_search_request(svc("perplexity-api"), "k", q).body) + ["search_domain_filter"] == deny); } TEST_CASE("web search: reply parsers") { @@ -672,6 +686,15 @@ TEST_CASE("web search: -site: reaches a native filter, and stays in the text els run(http2, p2, "rust async -site:reddit.com"); REQUIRE(http2->seen.size() == 1); CHECK(json::parse(http2->seen[0].body)["q"] == "rust async -site:reddit.com"); + + auto http3 = std::make_shared(); + http3->rules.push_back({"api.perplexity.ai", reply(200, R"({"results":[]})")}); + WebSearchPlan p3; p3.slots = {WebSearchSlot{"perplexity-api", "k", {}}}; + run(http3, p3, "rust async -site:reddit.com"); + REQUIRE(http3->seen.size() == 1); + auto pb = json::parse(http3->seen[0].body); + CHECK(pb["query"] == "rust async"); + CHECK(pb["search_domain_filter"] == json::array({"-reddit.com"})); } TEST_CASE("web search: at most three slots are tried") {