From e42675dcdc5b06d62db8dec28ade0f755ddd10e3 Mon Sep 17 00:00:00 2001 From: Vito Meznaric Date: Mon, 14 Sep 2026 15:38:08 +0200 Subject: [PATCH 1/3] Limit `proxies list` to 100 proxies by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `proxies list` walked every page of the proxy list and buffered the whole thing before printing a single byte. On a residential backbone plan, where the list runs to millions of entries at ~25-60 per request, that is thousands of sequential round trips with no output and no progress — the command is indistinguishable from a hang. Default --limit to 100, matching `proxies replaced`. --limit 0 restores the previous fetch-everything behavior, and the command now points at `proxies download`, which renders a full list server-side in one request. When the limit cuts the list short, say so on stderr so the cap is never silent — stderr keeps the notice out of piped proxy lists. --- README.md | 8 ++-- internal/cmd/cmd_test.go | 88 ++++++++++++++++++++++++++++++++++++++++ internal/cmd/proxies.go | 36 +++++++++++++--- 3 files changed, 122 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 462233e..8d9721f 100644 --- a/README.md +++ b/README.md @@ -47,10 +47,10 @@ pipe you get machine-readable output with no headers or colors: ```sh # Plain address:port:username:password lines — the format most # proxy-consuming tools accept directly -webshare proxies list > proxies.txt +webshare proxies list --limit 0 > proxies.txt # CSV for tools that want structured proxy lists -webshare proxies list --format csv > proxies.csv +webshare proxies list --format csv --limit 0 > proxies.csv # JSON everywhere, for jq and friends webshare proxies list --json | jq '.[].country_code' @@ -68,7 +68,7 @@ to stderr; exit codes are `0` (success), `1` (API/network error), `2` | Command | Purpose | |---|---| -| `webshare proxies list` | List proxies as a table, txt, csv or json (`--country`, `--limit`) | +| `webshare proxies list` | List proxies as a table, txt, csv or json (`--country`, `--limit`; first 100 by default) | | `webshare proxies download` | Server-rendered proxy list; download token fetched automatically | | `webshare proxies refresh` | Replace the whole proxy list (asks for confirmation) | | `webshare proxies replaced` | Show replaced proxies and their successors | @@ -103,7 +103,7 @@ webshare proxy-url --country us --sessions 5 webshare activity list --since 15m --error '*' # Feed only French proxies to a scraper -webshare proxies list --country fr --format csv > fr.csv +webshare proxies list --country fr --format csv --limit 0 > fr.csv # Download last month's invoice webshare transactions list diff --git a/internal/cmd/cmd_test.go b/internal/cmd/cmd_test.go index 4ca311d..1c07b42 100644 --- a/internal/cmd/cmd_test.go +++ b/internal/cmd/cmd_test.go @@ -3,10 +3,12 @@ package cmd import ( "context" "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" "os" + "strconv" "strings" "testing" "time" @@ -217,3 +219,89 @@ func TestParseSince(t *testing.T) { t.Error("parseSince(soon) succeeded, want an error") } } + +// newPagedProxyServer serves totalPages pages of perPage proxies, linked by +// the envelope's next URL, and records how many pages were actually fetched. +func newPagedProxyServer(t *testing.T, totalPages, perPage int, fetched *int) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v2/proxy/list/" { + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + *fetched++ + page := 1 + if p := r.URL.Query().Get("page"); p != "" { + if n, err := strconv.Atoi(p); err == nil { + page = n + } + } + results := make([]any, 0, perPage) + for i := range perPage { + id := (page-1)*perPage + i + results = append(results, proxyFixture(fmt.Sprintf("d-%d", id), "10.0.0.1", 8000+id)) + } + body := map[string]any{ + "count": totalPages * perPage, "previous": nil, "results": results, + } + if page < totalPages { + body["next"] = fmt.Sprintf("http://%s/api/v2/proxy/list/?page=%d", r.Host, page+1) + } else { + body["next"] = nil + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(body); err != nil { + t.Errorf("encoding response: %v", err) + } + })) + t.Cleanup(server.Close) + return server +} + +func TestProxiesListStopsAtDefaultLimit(t *testing.T) { + t.Setenv("WEBSHARE_API_KEY", "k") + fetched := 0 + // 100 pages of 60 is 6000 proxies; the default limit of 100 should need + // only the first two pages. + server := newPagedProxyServer(t, 100, 60, &fetched) + out, code := runCLI(t, "proxies", "list", "--base-url", server.URL, "--format", "txt") + if code != 0 { + t.Fatalf("exit code = %d, want 0", code) + } + if got := len(strings.Split(strings.TrimSpace(out), "\n")); got != defaultProxyListLimit { + t.Errorf("printed %d proxies, want the default limit of %d", got, defaultProxyListLimit) + } + if fetched != 2 { + t.Errorf("fetched %d pages, want 2 — the default limit must stop pagination early", fetched) + } +} + +func TestProxiesListLimitZeroFetchesEveryPage(t *testing.T) { + t.Setenv("WEBSHARE_API_KEY", "k") + fetched := 0 + server := newPagedProxyServer(t, 3, 60, &fetched) + out, code := runCLI(t, "proxies", "list", "--base-url", server.URL, "--format", "txt", "--limit", "0") + if code != 0 { + t.Fatalf("exit code = %d, want 0", code) + } + if got := len(strings.Split(strings.TrimSpace(out), "\n")); got != 180 { + t.Errorf("printed %d proxies, want all 180", got) + } + if fetched != 3 { + t.Errorf("fetched %d pages, want all 3", fetched) + } +} + +func TestProxiesListShorterThanLimitIsWhole(t *testing.T) { + t.Setenv("WEBSHARE_API_KEY", "k") + fetched := 0 + server := newPagedProxyServer(t, 1, 2, &fetched) + out, code := runCLI(t, "proxies", "list", "--base-url", server.URL, "--format", "txt") + if code != 0 { + t.Fatalf("exit code = %d, want 0", code) + } + if got := len(strings.Split(strings.TrimSpace(out), "\n")); got != 2 { + t.Errorf("printed %d proxies, want both", got) + } +} diff --git a/internal/cmd/proxies.go b/internal/cmd/proxies.go index 6877bca..6897850 100644 --- a/internal/cmd/proxies.go +++ b/internal/cmd/proxies.go @@ -25,6 +25,13 @@ func newProxiesCmd(flags *rootFlags) *cobra.Command { return cmd } +// defaultProxyListLimit caps `proxies list` by default. Without a cap the +// command walks every page of the proxy list before printing anything, which +// on a residential backbone plan — where the list runs to millions of entries +// — looks like a hang. --limit 0 still fetches everything, and +// `proxies download` returns a full list in a single request. +const defaultProxyListLimit = 100 + // proxyHost returns the address to connect to: the proxy's own address in // direct mode, or the backbone host when the API returns none (residential // plans). @@ -50,9 +57,15 @@ func newProxiesListCmd(flags *rootFlags) *cobra.Command { On a terminal the result is a table. In a pipe it is plain address:port:username:password lines — the format most proxy-consuming tools -accept directly. Use --format to force txt, csv, json or table.`, +accept directly. Use --format to force txt, csv, json or table. + +Only the first 100 proxies are listed unless you raise --limit; the command +pages through the API one request at a time, and a residential backbone plan +holds far more entries than is useful to page through. --limit 0 fetches every +one of them. To export a full list, prefer "webshare proxies download", which +the server renders in a single request.`, Example: ` webshare proxies list - webshare proxies list --country us,fr > proxies.txt + webshare proxies list --country us,fr --limit 0 > proxies.txt webshare proxies list --format csv > proxies.csv curl --proxy "$(webshare proxies list --limit 1 | awk -F: '{print "http://"$3":"$4"@"$1":"$2}')" https://ipv4.webshare.io/`, Args: cobra.NoArgs, @@ -68,23 +81,34 @@ accept directly. Use --format to force txt, csv, json or table.`, for _, c := range countries { params.CountryCodeIn = append(params.CountryCodeIn, strings.ToUpper(c)) } + // Read one past the limit so we can tell a list that happens to + // end exactly at the limit from one that was cut short. var proxies []webshare.Proxy + truncated := false for proxy, err := range client.Proxies.ListAll(cmd.Context(), params) { if err != nil { return err } - proxies = append(proxies, proxy) - if limit > 0 && len(proxies) >= limit { + if limit > 0 && len(proxies) == limit { + truncated = true break } + proxies = append(proxies, proxy) } - return writeProxies(cmd, flags, proxies, format) + if err := writeProxies(cmd, flags, proxies, format); err != nil { + return err + } + if truncated { + fmt.Fprintf(os.Stderr, "stopped at --limit %d; pass --limit 0 for every proxy, "+ + "or use `webshare proxies download` to fetch the whole list in one request\n", limit) + } + return nil }, } addPlanFlag(cmd, &planID) cmd.Flags().StringVar(&mode, "mode", "direct", "connection mode: direct or backbone (residential plans need backbone)") cmd.Flags().StringSliceVar(&countries, "country", nil, "filter by country codes (e.g. us,fr)") - cmd.Flags().IntVar(&limit, "limit", 0, "maximum number of proxies (default: all)") + cmd.Flags().IntVar(&limit, "limit", defaultProxyListLimit, "maximum number of proxies (0 for all)") cmd.Flags().StringVar(&format, "format", "auto", "output format: auto, table, txt, csv or json") return cmd } From 89a5aba81ea6e8ce3eaf518a71dc739c4c5ea133 Mon Sep 17 00:00:00 2001 From: Vito Meznaric Date: Mon, 14 Sep 2026 15:38:46 +0200 Subject: [PATCH 2/3] Pass --limit 0 in the root help's full-list examples The root command's examples redirect the proxy list to a file, which now needs --limit 0 to stay a complete list. --- internal/cmd/root.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/cmd/root.go b/internal/cmd/root.go index d48de0f..36285a4 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -41,8 +41,8 @@ on the API Keys page of the Webshare dashboard. Output adapts to where it goes: tables on a terminal, tab-separated values in a pipe, and --format csv/json/txt where structured output is useful.`, Example: ` # Feed your proxy list to another tool - webshare proxies list > proxies.txt - webshare proxies list --format csv > proxies.csv + webshare proxies list --limit 0 > proxies.txt + webshare proxies list --format csv --limit 0 > proxies.csv # Authorize this machine's IP for credential-less proxy use webshare ipauth add --current From 8496961848d238fce3f7ed8d82122a0041eae242 Mon Sep 17 00:00:00 2001 From: Vito Meznaric Date: Mon, 14 Sep 2026 15:41:35 +0200 Subject: [PATCH 3/3] Bump staticcheck to 2026.2.1 so lint works on current Go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI pins staticcheck@2026.1 (v0.7.0) but installs go-version: stable, which is now Go 1.27.1. staticcheck v0.7.0 cannot read Go 1.27's export data and fails on stdlib packages before it reaches any repo code: internal error in importing "math/bits" (cannot decode "math/bits", export data version 4 is greater than maximum supported version 2) This is unrelated to any code change — it reproduces on an unmodified main. 2026.2.1 supports the current toolchain and reports nothing on this repo. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index baea15a..2f8a002 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: run: go vet ./... - name: staticcheck run: | - go install honnef.co/go/tools/cmd/staticcheck@2026.1 + go install honnef.co/go/tools/cmd/staticcheck@2026.2.1 staticcheck ./... test: