diff --git a/src/tools/web_search_services.cpp b/src/tools/web_search_services.cpp index 7571b48..2d6b785 100644 --- a/src/tools/web_search_services.cpp +++ b/src/tools/web_search_services.cpp @@ -332,6 +332,24 @@ std::vector make_catalogue() { V("", "On (default)"), V("false", "Off")}}); c.push_back(std::move(s)); } + { + WebSearchService s; + s.id = "serply-api"; s.label = "Serply API key"; s.needs_key = true; + s.signup = "https://serply.io"; + s.note = "Google results as JSON."; + s.max_count = 10; + // Without gl Serply serves from whichever proxy region is free, so the + // results can change country between calls. req_serply sends gl=us + // for the default, so United States takes the first row. + auto country = country_dial("Google's gl parameter."); + country.values.erase(country.values.begin() + 1); + country.values[0].label = "United States (default)"; + s.dials.push_back(std::move(country)); + s.dials.push_back(freshness_dial("freshness", + "Google's tbs=qdr: time filter.", + hdwmy("qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y"))); + c.push_back(std::move(s)); + } { WebSearchService s; s.id = "perplexity-api"; s.label = "Perplexity API key"; s.needs_key = true; @@ -654,6 +672,27 @@ HttpRequest req_serper(const WebSearchQuery& q, std::string_view key) { {{"x-api-key", std::string{key}}}); } +// Serply. [L] GET /v1/search/?q=&num= ; X-Api-Key ; num stops at 10 ; gl and +// tbs pass through to Google ; hl does not change the results, so no language. +// gl is always sent, us by default: without it the proxy region is random. +// Cloudflare in front of it refuses some default client user-agents, so the +// request names itself. +HttpRequest req_serply(const WebSearchQuery& q, std::string_view key) { + HttpRequest r; + r.method = "GET"; + r.url = "https://api.serply.io/v1/search/?q=" + url_escape(q.text) + + "&num=" + std::to_string(q.count); + auto country = q.dial("country"); + r.url += "&gl=" + (country.empty() ? std::string{"us"} + : url_escape(lower(std::string{country}))); + if (auto v = q.dial("freshness"); !v.empty()) + r.url += "&tbs=" + url_escape(v); + r.headers.push_back({"accept", "application/json"}); + r.headers.push_back({"user-agent", "mcp-cpp"}); + r.headers.push_back({"x-api-key", std::string{key}}); + return r; +} + // Perplexity Search. [D] POST /search ; Bearer ; max_results 1-20 without a // tier ; country is ISO alpha-2 ; search_language_filter is a list of ISO 639-1 // codes ; search_recency_filter is hour|day|week|month|year ; search_type is @@ -877,6 +916,7 @@ HttpRequest build_web_search_request(const WebSearchService& svc, if (id == "brave-api") return req_brave_api(q, key); if (id == "exa-api") return req_exa(q, key); if (id == "serper-api") return req_serper(q, key); + if (id == "serply-api") return req_serply(q, key); if (id == "perplexity-api") return req_perplexity(q, key); return {}; // unreachable for a catalogue id; empty url fails the send } @@ -984,6 +1024,14 @@ JsonHits parse_json_hits(std::string_view service_id, std::string_view body, // Serper omits `organic` when Google found nothing. const json& o = field(j, "organic"); out.hits = collect(o, "link", "title", {"snippet"}, count); + } else if (service_id == "serply-api") { + // Now and then a row's `link` is an opaque token, not a URL. [L] + // Such rows are dropped before counting, so they never become hits. + json rows = json::array(); + for (const auto& r : *need(field(j, "results"))) + if (r.is_object() && str_at(r, "link").rfind("http", 0) == 0) + rows.push_back(r); + out.hits = collect(rows, "link", "title", {"description"}, count); } else if (service_id == "perplexity-api") { out.hits = collect(*need(field(j, "results")), "url", "title", {"snippet"}, count); diff --git a/tests/web_search_services_test.cpp b/tests/web_search_services_test.cpp index 6e272e5..cd5bd22 100644 --- a/tests/web_search_services_test.cpp +++ b/tests/web_search_services_test.cpp @@ -99,7 +99,7 @@ const char* kTavily = TEST_CASE("web search: search catalogue is well formed") { const auto& all = web_search_services(); - CHECK(all.size() == 10); + CHECK(all.size() == 11); for (const auto& s : all) { INFO(s.id); CHECK(!s.id.empty()); @@ -169,6 +169,7 @@ TEST_CASE("web search: dials offer each vendor's documented values") { CHECK(dial_ids("exa-api", "type") == exa_types); CHECK(dial_ids("exa-api", "freshness") == exa_ages); CHECK(dial_ids("serper-api", "freshness") == hourly_tbs); + CHECK(dial_ids("serply-api", "freshness") == hourly_tbs); // Brave's "no country" sits right under the default. const auto brave = dial_ids("brave-api", "country"); REQUIRE(brave.size() >= 2); @@ -470,6 +471,47 @@ TEST_CASE("web search: Exa, Serper and Perplexity requests") { ["search_domain_filter"] == deny); } +TEST_CASE("web search: Serply request and reply") { + auto q = query(); q.language = "fr"; + q.dials = {{"country", "NZ"}, {"freshness", "qdr:d"}}; + auto r = build_web_search_request(svc("serply-api"), "sy-k", q); + CHECK(r.method == "GET"); + CHECK(r.url == "https://api.serply.io/v1/search/?q=rust%20async&num=5" + "&gl=nz&tbs=qdr%3Ad"); + CHECK(header(r, "x-api-key") == "sy-k"); + CHECK(!header(r, "user-agent").empty()); + CHECK(r.url.find("sy-k") == std::string::npos); + CHECK(r.url.find("hl=") == std::string::npos); // no language parameter + CHECK(svc("serply-api").max_count == 10); + CHECK(!has_native_exclude("serply-api")); + + // No country set: gl=us, since without gl the proxy region is random. + // United States is the default row, not a second "US" row. + auto plain = build_web_search_request(svc("serply-api"), "sy-k", query()); + CHECK(plain.url.find("&gl=us") != std::string::npos); + const auto countries = dial_ids("serply-api", "country"); + REQUIRE(!countries.empty()); + CHECK(countries[0].empty()); + CHECK(std::find(countries.begin(), countries.end(), "US") == countries.end()); + + auto h = parse_json_hits("serply-api", + R"({"results":[{"title":"S","link":"https://s.test/","description":"sd",)" + R"("position":1,"result_type":"organic"}],"total":1})", 5); + REQUIRE(h.ok); + REQUIRE(h.hits.size() == 1); + CHECK(h.hits[0].url == "https://s.test/"); + CHECK(h.hits[0].snippet == "sd"); + // A 2xx without `results` is not the documented shape. + CHECK(!parse_json_hits("serply-api", R"({"detail":"x"})", 5).ok); + // A row whose `link` is a token, not a URL, is skipped and not counted. + auto t = parse_json_hits("serply-api", + R"({"results":[{"title":"T","link":"CAESBggA","description":"td"},)" + R"({"title":"S","link":"https://s.test/","description":"sd"}]})", 1); + REQUIRE(t.ok); + REQUIRE(t.hits.size() == 1); + CHECK(t.hits[0].url == "https://s.test/"); +} + TEST_CASE("web search: reply parsers") { auto t = parse_json_hits("tavily-free", kTavily, 10); REQUIRE(t.ok); @@ -582,6 +624,9 @@ TEST_CASE("web search: failures are told apart") { CHECK(classify_http(tavily, 401, "") == WebSearchFailure::BadKey); CHECK(classify_http(tavily, 432, "") == WebSearchFailure::QuotaExhausted); CHECK(classify_http(svc("exa-api"), 402, "") == WebSearchFailure::QuotaExhausted); + // Serply's gateway at a zero balance, expired free credits included. + CHECK(classify_http(svc("serply-api"), 402, + R"({"detail":"Insufficient prepaid credits"})") == WebSearchFailure::QuotaExhausted); // Brave's API answers a rejected token with 422, recorded live. CHECK(classify_http(svc("brave-api"), 422, R"({"error":{"code":"SUBSCRIPTION_TOKEN_INVALID"}})") == WebSearchFailure::BadKey);