Limit proxies list to 100 proxies by default

.github/workflows/ci.yml

@@ -34,7 +34,7 @@ jobs:
         run: go vet ./...
       - name: staticcheck
         run: |
-          go install honnef.co/go/tools/cmd/staticcheck@2026.1
+          go install honnef.co/go/tools/cmd/staticcheck@2026.2.1
           staticcheck ./...
 
   test:

README.md

@@ -47,10 +47,10 @@ pipe you get machine-readable output with no headers or colors:
 ```sh
 # Plain address:port:username:password lines — the format most
 # proxy-consuming tools accept directly
-webshare proxies list > proxies.txt
+webshare proxies list --limit 0 > proxies.txt
 
 # CSV for tools that want structured proxy lists
-webshare proxies list --format csv > proxies.csv
+webshare proxies list --format csv --limit 0 > proxies.csv
 
 # JSON everywhere, for jq and friends
 webshare proxies list --json | jq '.[].country_code'
@@ -68,7 +68,7 @@ to stderr; exit codes are `0` (success), `1` (API/network error), `2`
 
 | Command | Purpose |
 |---|---|
-| `webshare proxies list` | List proxies as a table, txt, csv or json (`--country`, `--limit`) |
+| `webshare proxies list` | List proxies as a table, txt, csv or json (`--country`, `--limit`; first 100 by default) |
 | `webshare proxies download` | Server-rendered proxy list; download token fetched automatically |
 | `webshare proxies refresh` | Replace the whole proxy list (asks for confirmation) |
 | `webshare proxies replaced` | Show replaced proxies and their successors |
@@ -103,7 +103,7 @@ webshare proxy-url --country us --sessions 5
 webshare activity list --since 15m --error '*'
 
 # Feed only French proxies to a scraper
-webshare proxies list --country fr --format csv > fr.csv
+webshare proxies list --country fr --format csv --limit 0 > fr.csv
 
 # Download last month's invoice
 webshare transactions list

internal/cmd/cmd_test.go

@@ -3,10 +3,12 @@ package cmd
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -217,3 +219,89 @@ func TestParseSince(t *testing.T) {
 		t.Error("parseSince(soon) succeeded, want an error")
 	}
 }
+
+// newPagedProxyServer serves totalPages pages of perPage proxies, linked by
+// the envelope's next URL, and records how many pages were actually fetched.
+func newPagedProxyServer(t *testing.T, totalPages, perPage int, fetched *int) *httptest.Server {
+	t.Helper()
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/v2/proxy/list/" {
+			t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		*fetched++
+		page := 1
+		if p := r.URL.Query().Get("page"); p != "" {
+			if n, err := strconv.Atoi(p); err == nil {
+				page = n
+			}
+		}
+		results := make([]any, 0, perPage)
+		for i := range perPage {
+			id := (page-1)*perPage + i
+			results = append(results, proxyFixture(fmt.Sprintf("d-%d", id), "10.0.0.1", 8000+id))
+		}
+		body := map[string]any{
+			"count": totalPages * perPage, "previous": nil, "results": results,
+		}
+		if page < totalPages {
+			body["next"] = fmt.Sprintf("http://%s/api/v2/proxy/list/?page=%d", r.Host, page+1)
+		} else {
+			body["next"] = nil
+		}
+		w.Header().Set("Content-Type", "application/json")
+		if err := json.NewEncoder(w).Encode(body); err != nil {
+			t.Errorf("encoding response: %v", err)
+		}
+	}))
+	t.Cleanup(server.Close)
+	return server
+}
+
+func TestProxiesListStopsAtDefaultLimit(t *testing.T) {
+	t.Setenv("WEBSHARE_API_KEY", "k")
+	fetched := 0
+	// 100 pages of 60 is 6000 proxies; the default limit of 100 should need
+	// only the first two pages.
+	server := newPagedProxyServer(t, 100, 60, &fetched)
+	out, code := runCLI(t, "proxies", "list", "--base-url", server.URL, "--format", "txt")
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0", code)
+	}
+	if got := len(strings.Split(strings.TrimSpace(out), "\n")); got != defaultProxyListLimit {
+		t.Errorf("printed %d proxies, want the default limit of %d", got, defaultProxyListLimit)
+	}
+	if fetched != 2 {
+		t.Errorf("fetched %d pages, want 2 — the default limit must stop pagination early", fetched)
+	}
+}
+
+func TestProxiesListLimitZeroFetchesEveryPage(t *testing.T) {
+	t.Setenv("WEBSHARE_API_KEY", "k")
+	fetched := 0
+	server := newPagedProxyServer(t, 3, 60, &fetched)
+	out, code := runCLI(t, "proxies", "list", "--base-url", server.URL, "--format", "txt", "--limit", "0")
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0", code)
+	}
+	if got := len(strings.Split(strings.TrimSpace(out), "\n")); got != 180 {
+		t.Errorf("printed %d proxies, want all 180", got)
+	}
+	if fetched != 3 {
+		t.Errorf("fetched %d pages, want all 3", fetched)
+	}
+}
+
+func TestProxiesListShorterThanLimitIsWhole(t *testing.T) {
+	t.Setenv("WEBSHARE_API_KEY", "k")
+	fetched := 0
+	server := newPagedProxyServer(t, 1, 2, &fetched)
+	out, code := runCLI(t, "proxies", "list", "--base-url", server.URL, "--format", "txt")
+	if code != 0 {
+		t.Fatalf("exit code = %d, want 0", code)
+	}
+	if got := len(strings.Split(strings.TrimSpace(out), "\n")); got != 2 {
+		t.Errorf("printed %d proxies, want both", got)
+	}
+}

internal/cmd/proxies.go

@@ -25,6 +25,13 @@ func newProxiesCmd(flags *rootFlags) *cobra.Command {
 	return cmd
 }
 
+// defaultProxyListLimit caps `proxies list` by default. Without a cap the
+// command walks every page of the proxy list before printing anything, which
+// on a residential backbone plan — where the list runs to millions of entries
+// — looks like a hang. --limit 0 still fetches everything, and
+// `proxies download` returns a full list in a single request.
+const defaultProxyListLimit = 100
+
 // proxyHost returns the address to connect to: the proxy's own address in
 // direct mode, or the backbone host when the API returns none (residential
 // plans).
@@ -50,9 +57,15 @@ func newProxiesListCmd(flags *rootFlags) *cobra.Command {
 
 On a terminal the result is a table. In a pipe it is plain
 address:port:username:password lines — the format most proxy-consuming tools
-accept directly. Use --format to force txt, csv, json or table.`,
+accept directly. Use --format to force txt, csv, json or table.
+
+Only the first 100 proxies are listed unless you raise --limit; the command
+pages through the API one request at a time, and a residential backbone plan
+holds far more entries than is useful to page through. --limit 0 fetches every
+one of them. To export a full list, prefer "webshare proxies download", which
+the server renders in a single request.`,
 		Example: `  webshare proxies list
-  webshare proxies list --country us,fr > proxies.txt
+  webshare proxies list --country us,fr --limit 0 > proxies.txt
   webshare proxies list --format csv > proxies.csv
   curl --proxy "$(webshare proxies list --limit 1 | awk -F: '{print "http://"$3":"$4"@"$1":"$2}')" https://ipv4.webshare.io/`,
 		Args: cobra.NoArgs,
@@ -68,23 +81,34 @@ accept directly. Use --format to force txt, csv, json or table.`,
 			for _, c := range countries {
 				params.CountryCodeIn = append(params.CountryCodeIn, strings.ToUpper(c))
 			}
+			// Read one past the limit so we can tell a list that happens to
+			// end exactly at the limit from one that was cut short.
 			var proxies []webshare.Proxy
+			truncated := false
 			for proxy, err := range client.Proxies.ListAll(cmd.Context(), params) {
 				if err != nil {
 					return err
 				}
-				proxies = append(proxies, proxy)
-				if limit > 0 && len(proxies) >= limit {
+				if limit > 0 && len(proxies) == limit {
+					truncated = true
 					break
 				}
+				proxies = append(proxies, proxy)
 			}
-			return writeProxies(cmd, flags, proxies, format)
+			if err := writeProxies(cmd, flags, proxies, format); err != nil {
+				return err
+			}
+			if truncated {
+				fmt.Fprintf(os.Stderr, "stopped at --limit %d; pass --limit 0 for every proxy, "+
+					"or use `webshare proxies download` to fetch the whole list in one request\n", limit)
+			}
+			return nil
 		},
 	}
 	addPlanFlag(cmd, &planID)
 	cmd.Flags().StringVar(&mode, "mode", "direct", "connection mode: direct or backbone (residential plans need backbone)")
 	cmd.Flags().StringSliceVar(&countries, "country", nil, "filter by country codes (e.g. us,fr)")
-	cmd.Flags().IntVar(&limit, "limit", 0, "maximum number of proxies (default: all)")
+	cmd.Flags().IntVar(&limit, "limit", defaultProxyListLimit, "maximum number of proxies (0 for all)")
 	cmd.Flags().StringVar(&format, "format", "auto", "output format: auto, table, txt, csv or json")
 	return cmd
 }

internal/cmd/root.go

@@ -41,8 +41,8 @@ on the API Keys page of the Webshare dashboard.
 Output adapts to where it goes: tables on a terminal, tab-separated values in
 a pipe, and --format csv/json/txt where structured output is useful.`,
 		Example: `  # Feed your proxy list to another tool
-  webshare proxies list > proxies.txt
-  webshare proxies list --format csv > proxies.csv
+  webshare proxies list --limit 0 > proxies.txt
+  webshare proxies list --format csv --limit 0 > proxies.csv
 
   # Authorize this machine's IP for credential-less proxy use
   webshare ipauth add --current