From da728f56d16ea75bbef280d03fbaefdec692a09f Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 22 Oct 2026 10:38:12 -0400 Subject: [PATCH 1/4] installer: name why GitHub's API refused, and retry a refused token once without it Every failed release lookup used to print one sentence for a spent rate limit, a rejected token, a forbidden repository, an outage and a dead network, and it advised exporting GITHUB_TOKEN even when the exported token was the cause. http_get kept only the status, so the reason GitHub states on the response (x-ratelimit-remaining, x-ratelimit-reset, the body's message) was thrown away. The installer now keeps each API response's headers and body and names the refusal: the rate limit with the local time it lifts, a rejected token, access, a repository GitHub cannot find, a server fault, or an unreachable host. A token GitHub refuses with 401, 403 or 429 is set aside and the same lookup is asked once without it, because the anonymous quota is a separate bucket; when that works the install goes on with a one-line note. Downloads still never carry the token. The classifiers are pure functions between marker lines, and the tests source that block directly. Fixes #1889. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/release/install_test.go | 589 +++++++++++++++++++++++++++++-- scripts/install.sh | 289 ++++++++++++++- 2 files changed, 837 insertions(+), 41 deletions(-) diff --git a/internal/release/install_test.go b/internal/release/install_test.go index 91f5c181d1..f3f233086f 100644 --- a/internal/release/install_test.go +++ b/internal/release/install_test.go @@ -1,10 +1,12 @@ package release import ( + "bytes" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" + "net" "net/http" "net/http/httptest" "os" @@ -14,6 +16,7 @@ import ( "strings" "sync" "testing" + "time" ) type fakeRelease struct { @@ -34,11 +37,19 @@ type fakeTiming struct { published any } +type apiRefusal struct { + status int + headers map[string]string + body string + tokenOnly bool +} + type installGitHub struct { server *httptest.Server releases []string timings map[string]fakeTiming badChecksum bool + refuse *apiRefusal failAPI bool notFound bool oldRepositoryOnly bool @@ -60,6 +71,15 @@ func (github *installGitHub) serve(w http.ResponseWriter, request *http.Request) github.mu.Lock() github.requests = append(github.requests, request.Clone(request.Context())) github.mu.Unlock() + if refusal := github.refuse; refusal != nil && strings.HasPrefix(request.URL.Path, "/repos/") && + (!refusal.tokenOnly || request.Header.Get("Authorization") != "") { + for name, value := range refusal.headers { + w.Header().Set(name, value) + } + w.WriteHeader(refusal.status) + _, _ = w.Write([]byte(refusal.body)) + return + } if github.notFound { http.NotFound(w, request) return @@ -193,6 +213,7 @@ func contains(values []string, value string) bool { type installRun struct { code int output string + stderr string home string installDir string } @@ -218,7 +239,10 @@ func runInstaller(t *testing.T, github *installGitHub, arguments []string, extra "PATH=" + path, "SHELL=/bin/bash", }, extraEnv...) - output, runErr := command.CombinedOutput() + var stdout, stderr bytes.Buffer + command.Stdout = &stdout + command.Stderr = &stderr + runErr := command.Run() code := 0 if runErr != nil { var exitErr *exec.ExitError @@ -227,7 +251,7 @@ func runInstaller(t *testing.T, github *installGitHub, arguments []string, extra } code = exitErr.ExitCode() } - return installRun{code: code, output: string(output), home: home, installDir: installDir} + return installRun{code: code, output: stdout.String() + stderr.String(), stderr: stderr.String(), home: home, installDir: installDir} } func asExitError(err error, target **exec.ExitError) bool { @@ -241,7 +265,7 @@ func asExitError(err error, target **exec.ExitError) bool { func minimalPath(t *testing.T, withCurl bool) string { t.Helper() dir := t.TempDir() - commands := []string{"awk", "basename", "cat", "chmod", "cp", "dirname", "grep", "ln", "mkdir", "mktemp", "mv", "rm", "sed", "sha256sum", "shasum", "tr", "uname", "wget"} + commands := []string{"awk", "basename", "cat", "chmod", "cp", "date", "dirname", "grep", "ln", "mkdir", "mktemp", "mv", "rm", "sed", "sha256sum", "shasum", "tr", "uname", "wget"} if withCurl { commands = append(commands, "curl") } @@ -805,39 +829,42 @@ func TestInstallerHelpUnknownFlagsAndAPIFailures(t *testing.T) { } github.failAPI = true failure := runInstaller(t, github, nil) - if failure.code != 1 || !strings.Contains(failure.output, "GITHUB_TOKEN") || !strings.Contains(failure.output, "VERSION=") { + if failure.code != 1 || !strings.Contains(failure.output, "GitHub's API rate limit for this address is used up for up to an hour; export GITHUB_TOKEN to use your own, or pin VERSION=") { t.Fatalf("API failure exit %d:\n%s", failure.code, failure.output) } } -// Contract: an API refusal names rate limiting and an unreadable repository, -// while retaining both available recovery actions. +// C8: Anonymous refusals name the reason and retain the version escape hatch. func TestInstallerNamesBothReasonsWhenTheAPIRefuses(t *testing.T) { for _, test := range []struct { - name string - prepare func(*installGitHub) - wantLine bool + name string + prepare func(*installGitHub) + want string }{ - {name: "rate limit", prepare: func(github *installGitHub) { github.failAPI = true }, wantLine: true}, - {name: "private repository", prepare: func(github *installGitHub) { github.notFound = true }}, + { + name: "429 quota", prepare: func(github *installGitHub) { github.failAPI = true }, + want: "GitHub's API rate limit for this address is used up for up to an hour; export GITHUB_TOKEN to use your own, or pin VERSION=", + }, + { + name: "403 quota", prepare: func(github *installGitHub) { + github.refuse = &apiRefusal{status: 403, headers: map[string]string{"X-RateLimit-Remaining": "0"}} + }, + want: "GitHub's API rate limit for this address is used up for up to an hour; export GITHUB_TOKEN to use your own, or pin VERSION=", + }, + { + name: "private repository", prepare: func(github *installGitHub) { github.notFound = true }, + want: "GitHub's API cannot find Agent-Field/codeaf (404), which is also its answer for a repository you cannot read; export GITHUB_TOKEN, or pin VERSION=", + }, } { t.Run(test.name, func(t *testing.T) { github := newInstallGitHub(t, "dev-20260915-aaaaaaaaaaaa") test.prepare(github) run := runInstaller(t, github, []string{"--dev"}) - if run.code != 1 { - t.Fatalf("exit %d, want 1:\n%s", run.code, run.output) + if run.code != 1 || !strings.Contains(run.stderr, "codeaf: "+test.want+"\n") { + t.Fatalf("exit %d, want refusal %q:\n%s", run.code, test.want, run.output) } - for _, want := range []string{"a rate limit", "a repository you cannot read", "VERSION=", "GITHUB_TOKEN"} { - if !strings.Contains(run.output, want) { - t.Errorf("output does not contain %q:\n%s", want, run.output) - } - } - if test.wantLine { - const want = "GitHub's API could not be reached or refused (a rate limit, or a repository you cannot read?); pin VERSION=, or export GITHUB_TOKEN" - if !strings.Contains(run.output, want) { - t.Errorf("output does not contain pinned refusal:\n%s", run.output) - } + if requests := installerAPIRequests(github); len(requests) != 1 { + t.Fatalf("anonymous API requests = %d, want 1", len(requests)) } }) } @@ -1099,3 +1126,519 @@ func TestInstallerLinksIntoAFolderAlreadyOnPath(t *testing.T) { t.Fatalf("a shadowed install does not give the PATH line:\n%s", again.output) } } + +func installerAPIRequests(github *installGitHub) []*http.Request { + github.mu.Lock() + defer github.mu.Unlock() + var requests []*http.Request + for _, request := range github.requests { + if strings.HasPrefix(request.URL.Path, "/repos/") { + requests = append(requests, request) + } + } + return requests +} + +func assertInstallerAnonymousRetry(t *testing.T, github *installGitHub, token, path string) { + t.Helper() + requests := installerAPIRequests(github) + if len(requests) != 2 { + t.Fatalf("API requests = %d, want one token attempt and one anonymous retry", len(requests)) + } + for i, wantAuth := range []string{"Bearer " + token, ""} { + if requests[i].URL.RequestURI() != path || requests[i].Header.Get("Authorization") != wantAuth { + t.Errorf("request %d: %s, Authorization %q; want %s, %q", i, + requests[i].URL.RequestURI(), requests[i].Header.Get("Authorization"), path, wantAuth) + } + } +} + +// C1: Both discovery paths and both transports can use the free anonymous bucket. +func TestInstallerC1RetriesSpentTokenQuotaAnonymously(t *testing.T) { + reset := time.Now().Unix() + 60 + for _, transport := range []string{"curl", "wget"} { + t.Run(transport, func(t *testing.T) { + if _, err := exec.LookPath(transport); err != nil { + t.Skip(transport + " is not installed") + } + for _, channel := range []struct { + name, tag, path string + arguments []string + }{ + {"stable", "v1.2.3", "/repos/Agent-Field/codeaf/releases/latest", nil}, + {"dev", "dev-20260915-aaaaaaaaaaaa", "/repos/Agent-Field/codeaf/releases?per_page=100", []string{"--dev"}}, + } { + t.Run(channel.name, func(t *testing.T) { + github := newInstallGitHub(t, channel.tag) + github.refuse = &apiRefusal{ + status: 403, tokenOnly: true, + headers: map[string]string{ + "X-RateLimit-Remaining": "0", "X-RateLimit-Limit": "5000", + "X-RateLimit-Reset": fmt.Sprint(reset), "X-OAuth-Scopes": "repo, workflow", + }, + body: `{"message":"API rate limit exceeded for user ID 1."}`, + } + const token = "spent-installer-token" + run := runInstaller(t, github, channel.arguments, + "GITHUB_TOKEN="+token, "PATH="+minimalPath(t, transport == "curl"), "CODEAF_NO_MODIFY_PATH=1") + if run.code != 0 || !strings.Contains(run.output, "Installed codeaf "+channel.tag+"\n") { + t.Fatalf("exit %d without installed receipt:\n%s", run.code, run.output) + } + if _, err := os.Stat(filepath.Join(run.installDir, "codeaf")); err != nil { + t.Fatal(err) + } + if !strings.Contains(run.stderr, "codeaf: GitHub refused GITHUB_TOKEN (its rate limit is used up until ") || + !strings.Contains(run.stderr, "); continuing without it\n") || strings.Count(run.stderr, "GitHub refused GITHUB_TOKEN") != 1 { + t.Fatalf("missing recovery note:\n%s", run.output) + } + assertInstallerAnonymousRetry(t, github, token, channel.path) + for _, request := range github.requests { + if strings.Contains(request.URL.Path, "/releases/download/") && request.Header.Get("Authorization") != "" { + t.Errorf("download carries Authorization: %s", request.URL.Path) + } + } + }) + } + }) + } +} + +// C2 and C3: A second refusal explains both attempts and gives token-specific advice. +func TestInstallerC2C3NamesRejectedCredentialsAndAccess(t *testing.T) { + for _, test := range []struct { + name, message, want, absent string + status int + }{ + { + name: "C2 credentials", status: 401, message: "Bad credentials", + want: "GitHub rejected GITHUB_TOKEN (401: Bad credentials), and the request without it failed too (401: Bad credentials); replace or unset GITHUB_TOKEN, or pin VERSION=", + }, + { + name: "C3 access", status: 403, message: "Resource not accessible by personal access token", absent: "rate limit", + want: "GitHub refused GITHUB_TOKEN access to Agent-Field/codeaf (403: Resource not accessible by personal access token), and the request without it failed too (403: Resource not accessible by personal access token); check what GITHUB_TOKEN may read, or pin VERSION=", + }, + } { + t.Run(test.name, func(t *testing.T) { + github := newInstallGitHub(t, "v1.2.3") + body, err := json.Marshal(map[string]string{"message": test.message}) + if err != nil { + t.Fatal(err) + } + github.refuse = &apiRefusal{status: test.status, body: string(body)} + const token = "refused-installer-token" + run := runInstaller(t, github, nil, "GITHUB_TOKEN="+token) + if run.code != 1 || !strings.Contains(run.stderr, "codeaf: "+test.want+"\n") { + t.Fatalf("exit %d, want %q:\n%s", run.code, test.want, run.output) + } + if strings.Contains(run.output, "export GITHUB_TOKEN") || (test.absent != "" && strings.Contains(run.output, test.absent)) { + t.Fatalf("misleading recovery advice:\n%s", run.output) + } + assertInstallerAnonymousRetry(t, github, token, "/repos/Agent-Field/codeaf/releases/latest") + }) + } +} + +// C4: GitHub's reset epoch is rendered in the machine's local zone for either bucket. +func TestInstallerC4NamesQuotaResetInLocalTime(t *testing.T) { + const reset = int64(1800000000) + wait := "until " + time.Unix(reset, 0).In(time.FixedZone("IST", 19800)).Format("15:04") + " IST" + for _, withToken := range []bool{false, true} { + t.Run(fmt.Sprint("token=", withToken), func(t *testing.T) { + github := newInstallGitHub(t, "v1.2.3") + github.refuse = &apiRefusal{status: 403, headers: map[string]string{ + "X-RateLimit-Remaining": "0", "X-RateLimit-Reset": fmt.Sprint(reset), + }, body: `{"message":"API rate limit exceeded"}`} + env := []string{"TZ=IST-5:30"} + want := "GitHub's API rate limit for this address is used up " + wait + "; export GITHUB_TOKEN to use your own, or pin VERSION=" + if withToken { + env = append(env, "GITHUB_TOKEN=spent-installer-token") + want = "GitHub's API rate limit for GITHUB_TOKEN is used up " + wait + ", and the request without it failed too (this address's rate limit is used up " + wait + "); wait, or pin VERSION=" + } + run := runInstaller(t, github, nil, env...) + if run.code != 1 || !strings.Contains(run.stderr, "codeaf: "+want+"\n") { + t.Fatalf("exit %d, want %q:\n%s", run.code, want, run.output) + } + if withToken && strings.Contains(run.output, "export GITHUB_TOKEN") { + t.Fatalf("a configured token was answered with export advice:\n%s", run.output) + } + }) + } +} + +// C5 and C6: The supplying variable is named and its value stays private, even on recovery. +func TestInstallerC5C6NamesTheTokenVariableWithoutPrintingItsValue(t *testing.T) { + for _, test := range []struct { + name, variable string + env []string + }{ + {"GH_TOKEN alone", "GH_TOKEN", nil}, + {"empty GITHUB_TOKEN", "GH_TOKEN", []string{"GITHUB_TOKEN="}}, + {"GITHUB_TOKEN wins", "GITHUB_TOKEN", []string{"GH_TOKEN=unused-token"}}, + } { + t.Run(test.name, func(t *testing.T) { + for _, recover := range []bool{false, true} { + t.Run(fmt.Sprint("recover=", recover), func(t *testing.T) { + github := newInstallGitHub(t, "v1.2.3") + github.refuse = &apiRefusal{status: 401, body: `{"message":"Bad credentials"}`, tokenOnly: recover} + const token = "private-refused-installer-token" + env := append([]string{"VERBOSE=1", "CODEAF_NO_MODIFY_PATH=1"}, test.env...) + env = append(env, test.variable+"="+token) + run := runInstaller(t, github, nil, env...) + wantCode := 1 + want := "GitHub rejected " + test.variable + " (401: Bad credentials)" + if recover { + wantCode = 0 + want = "GitHub refused " + test.variable + " (401: Bad credentials); continuing without it" + } + if run.code != wantCode || !strings.Contains(run.stderr, want) { + t.Fatalf("exit %d, want %d and %q:\n%s", run.code, wantCode, want, run.output) + } + if strings.Contains(run.output, token) || strings.Contains(run.output, "unused-token") || strings.Contains(run.output, "export GITHUB_TOKEN") || + (test.variable == "GH_TOKEN" && strings.Contains(run.output, "GITHUB_TOKEN")) { + t.Fatalf("token leaked or the wrong variable was named:\n%s", run.output) + } + assertInstallerAnonymousRetry(t, github, token, "/repos/Agent-Field/codeaf/releases/latest") + }) + } + }) + } +} + +// C7: Only 401, 403 and 429 spend the single anonymous retry. +func TestInstallerC7DoesNotRetryOtherTokenFailures(t *testing.T) { + for _, status := range []int{404, 500, 502, 302, 400, 418} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + github := newInstallGitHub(t, "v1.2.3") + github.refuse = &apiRefusal{status: status, body: `{"message":"A response"}`} + run := runInstaller(t, github, []string{"--dev"}, "GITHUB_TOKEN=installer-token") + want := fmt.Sprintf("GitHub's API answered %d (A response); pin VERSION=", status) + if status == 404 { + want = "GitHub's API cannot find Agent-Field/codeaf with GITHUB_TOKEN (404), which is also its answer for a repository the token cannot read; check what GITHUB_TOKEN may read, or pin VERSION=" + } else if status >= 500 { + want = fmt.Sprintf("GitHub's API answered %d and may be having trouble; try again in a minute, or pin VERSION=", status) + } + if run.code != 1 || !strings.Contains(run.stderr, "codeaf: "+want+"\n") || strings.Contains(run.output, "export GITHUB_TOKEN") { + t.Fatalf("exit %d, want %q:\n%s", run.code, want, run.output) + } + if requests := installerAPIRequests(github); len(requests) != 1 || requests[0].Header.Get("Authorization") != "Bearer installer-token" { + t.Fatalf("API requests = %v, want one token request", requests) + } + }) + } +} + +func TestInstallerC7NamesTheUnreachableAPIHost(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + host := listener.Addr().String() + if err := listener.Close(); err != nil { + t.Fatal(err) + } + github := newInstallGitHub(t, "v1.2.3") + for _, token := range []string{"", "GITHUB_TOKEN=installer-token"} { + run := runInstaller(t, github, nil, "CODEAF_GITHUB_API=http://"+host+"/api", token) + want := "GitHub's API could not be reached at " + host + "; check the connection, or pin VERSION=" + if run.code != 1 || !strings.Contains(run.stderr, want) || strings.Contains(run.output, "export GITHUB_TOKEN") { + t.Fatalf("exit %d, want %q:\n%s", run.code, want, run.output) + } + } +} + +// D12: A retry's 404 describes both attempts, while a first 404 keeps the stable answer. +func TestInstallerKeepsStable404UnlessTheTokenWasRefused(t *testing.T) { + for _, status := range []int{401, 403, 429} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + github := newInstallGitHub(t) + github.refuse = &apiRefusal{status: status, tokenOnly: true} + run := runInstaller(t, github, nil, "GITHUB_TOKEN=installer-token") + if run.code != 1 || !strings.Contains(run.stderr, "and the request without it failed too (404);") || + strings.Contains(run.output, "no stable build") || strings.Contains(run.output, "export GITHUB_TOKEN") { + t.Fatalf("anonymous 404 lost the token refusal (exit %d):\n%s", run.code, run.output) + } + assertInstallerAnonymousRetry(t, github, "installer-token", "/repos/Agent-Field/codeaf/releases/latest") + }) + } + github := newInstallGitHub(t) + run := runInstaller(t, github, nil, "GITHUB_TOKEN=installer-token") + if run.code != 1 || !strings.Contains(run.stderr, "codeaf: no stable build has been published yet\n") || len(installerAPIRequests(github)) != 1 { + t.Fatalf("first stable 404 changed (exit %d):\n%s", run.code, run.output) + } +} + +func TestInstallerNamesAnAnswerWithoutAReleaseTag(t *testing.T) { + github := newInstallGitHub(t, "v1.2.3") + github.refuse = &apiRefusal{status: 200, body: `{}`} + run := runInstaller(t, github, nil) + if run.code != 1 || !strings.Contains(run.stderr, "codeaf: GitHub's API answered without a release tag; pin VERSION=\n") { + t.Fatalf("exit %d without the missing-tag answer:\n%s", run.code, run.output) + } +} + +func installerPureFunctions(t *testing.T) string { + t.Helper() + raw, err := os.ReadFile(filepath.Join(repositoryRoot(t), "scripts", "install.sh")) + if err != nil { + t.Fatal(err) + } + const begin = "# BEGIN api refusal: pure functions, read by internal/release/install_test.go\n" + const end = "# END api refusal\n" + _, rest, ok := strings.Cut(string(raw), begin) + if !ok || strings.Count(string(raw), begin) != 1 || strings.Count(string(raw), end) != 1 { + t.Fatal("the installer must expose exactly one pure refusal block") + } + block, _, ok := strings.Cut(rest, end) + if !ok { + t.Fatal("the pure refusal block has no end") + } + path := filepath.Join(t.TempDir(), "api-functions.sh") + if err := os.WriteFile(path, []byte(block), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func runInstallerPureFunction(t *testing.T, block, function string, arguments []string, env ...string) string { + t.Helper() + bash, err := exec.LookPath("bash") + if err != nil { + t.Skip("bash is not installed") + } + command := exec.Command(bash, append([]string{"--noprofile", "--norc", "-c", `set -eu; source "$1"; shift; "$@"`, "api-test", block, function}, arguments...)...) + command.Env = append([]string{"PATH=" + minimalPath(t, true), "TZ=IST-5:30"}, env...) + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("%s: %v\n%s", function, err, output) + } + return string(output) +} + +func installerResponseFile(t *testing.T, name, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +// C9: Classification depends only on the status, the final header block and the body. +func TestInstallerC9APIReason(t *testing.T) { + block := installerPureFunctions(t) + for _, test := range []struct{ name, status, headers, body, want string }{ + {"401 before quota", "401", "x-ratelimit-remaining: 0\n", `{"message":"rate limit"}`, "rejected"}, + {"spent quota", "403", "x-ratelimit-remaining: 0\n", "", "quota"}, + {"body quota", "403", "", `{"message":"API rate limit exceeded"}`, "quota"}, + {"secondary quota", "403", "", `{"message":"You have exceeded a secondary rate limit"}`, "quota"}, + {"case insensitive body", "403", "", `{"message":"RATE LIMIT"}`, "quota"}, + {"429", "429", "", "", "quota"}, + {"quota still available", "403", "x-ratelimit-remaining: 4999\n", "", "access"}, + {"no quota evidence", "403", "", "", "access"}, + {"404", "404", "", "", "missing"}, + {"500", "500", "", "", "server"}, + {"503", "503", "", "", "server"}, + {"no status", "", "", "", "unreachable"}, + {"418", "418", "", "", "other"}, + {"mixed case", "403", "X-RateLimit-Remaining: 0\n", "", "quota"}, + {"CRLF and wget indentation", "403", " HTTP/1.1 403 Forbidden\r\n x-RATELIMIT-remaining: 0\r\n\r\n", "", "quota"}, + {"last block spent", "403", "HTTP/1.1 301 Moved\r\nx-ratelimit-remaining: 4999\r\n\r\nHTTP/2 403\r\nX-RateLimit-Remaining: 0\r\n\r\n", "", "quota"}, + {"last block available", "403", "HTTP/1.1 301 Moved\nx-ratelimit-remaining: 0\n\nHTTP/1.1 403 Forbidden\nx-ratelimit-remaining: 4999\n\n", "", "access"}, + {"last block absent", "403", "HTTP/1.1 301 Moved\nx-ratelimit-remaining: 0\n\nHTTP/1.1 403 Forbidden\n\n", "", "access"}, + } { + t.Run(test.name, func(t *testing.T) { + headers := installerResponseFile(t, "headers", test.headers) + body := installerResponseFile(t, "body", test.body) + if got := runInstallerPureFunction(t, block, "api_reason", []string{test.status, headers, body}); got != test.want { + t.Errorf("api_reason = %q, want %q", got, test.want) + } + }) + } +} + +func TestInstallerC9APIWait(t *testing.T) { + block := installerPureFunctions(t) + const reset = int64(1800000000) + local := time.Unix(reset, 0).In(time.FixedZone("IST", 19800)).Format("15:04") + for _, test := range []struct{ name, headers, want string }{ + {"local reset", fmt.Sprintf("x-ratelimit-reset: %d\nRetry-After: 600\n", reset), "until " + local + " IST"}, + {"30 seconds", "retry-after: 30\n", "for about a minute"}, + {"600 seconds", "retry-after: 600\n", "for about 10 minutes"}, + {"61 seconds", "retry-after: 61\n", "for about a minute"}, + {"90 seconds", "retry-after: 90\n", "for about a minute"}, + {"91 seconds", "retry-after: 91\n", "for about 2 minutes"}, + {"zero seconds", "retry-after: 0\n", "for about a minute"}, + {"leading zeros", "retry-after: 0091\n", "for about 2 minutes"}, + {"neither", "", "for up to an hour"}, + {"invalid reset", "x-ratelimit-reset: tomorrow\n", "for up to an hour"}, + {"invalid reset with retry", "x-ratelimit-reset: invalid\nretry-after: 600\n", "for about 10 minutes"}, + {"invalid retry", "retry-after: 2 minutes\n", "for up to an hour"}, + {"final block only", "HTTP/1.1 301 Moved\nx-ratelimit-reset: 1800000000\n\n HTTP/1.1 403 Forbidden\r\n Retry-After: 91\r\n", "for about 2 minutes"}, + } { + t.Run(test.name, func(t *testing.T) { + headers := installerResponseFile(t, "headers", test.headers) + if got := runInstallerPureFunction(t, block, "api_wait", []string{headers}); got != test.want { + t.Errorf("api_wait = %q, want %q", got, test.want) + } + }) + } +} + +func TestInstallerC9APIMessage(t *testing.T) { + block := installerPureFunctions(t) + for _, test := range []struct{ name, body, want string }{ + {"message", `{"message":"Bad credentials"}`, "Bad credentials"}, + {"escaped quotes", `{"message":"Say \"no\", then \\ and \/"}`, `Say "no", then \ and /`}, + {"control bytes", "{\"message\":\"a\x01b\x1b[31m\tc\rd\ne\"}", "ab[31mcde"}, + {"JSON controls and unicode", `{"message":"a\nb\rc\td\u0000e\u0021é\u00e9"}`, "abcde!"}, + {"collapse spaces", `{"message":"Bad credentials"}`, "Bad credentials"}, + {"cap", `{"message":"` + strings.Repeat("a", 101) + `"}`, strings.Repeat("a", 100) + "..."}, + {"at cap", `{"message":"` + strings.Repeat("a", 100) + `"}`, strings.Repeat("a", 100)}, + {"first message", `{"message":"first","other":{"message":"second"}}`, "first"}, + {"missing", `{}`, ""}, + {"empty", `{"message":""}`, ""}, + } { + t.Run(test.name, func(t *testing.T) { + body := installerResponseFile(t, "body", test.body) + if got := runInstallerPureFunction(t, block, "api_message", []string{body}); got != test.want { + t.Errorf("api_message = %q, want %q", got, test.want) + } + }) + } +} + +// The remaining anonymous sentences distinguish access, credentials and server failures. +func TestInstallerNamesAnonymousFailuresExactly(t *testing.T) { + for _, test := range []struct { + name, body, want string + status int + }{ + {"access", `{"message":"Access denied"}`, "GitHub's API refused access to Agent-Field/codeaf (403: Access denied); pin VERSION=", 403}, + {"access without message", `{}`, "GitHub's API refused access to Agent-Field/codeaf (403); pin VERSION=", 403}, + {"credentials", `{"message":"Bad credentials"}`, "GitHub's API refused the request (401: Bad credentials); pin VERSION=", 401}, + {"server", `{}`, "GitHub's API answered 503 and may be having trouble; try again in a minute, or pin VERSION=", 503}, + {"other", `{"message":"A teapot"}`, "GitHub's API answered 418 (A teapot); pin VERSION=", 418}, + {"other without message", `{}`, "GitHub's API answered 418; pin VERSION=", 418}, + } { + t.Run(test.name, func(t *testing.T) { + github := newInstallGitHub(t, "v1.2.3") + github.refuse = &apiRefusal{status: test.status, body: test.body} + run := runInstaller(t, github, nil) + if run.code != 1 || run.stderr != "codeaf: "+test.want+"\n" { + t.Fatalf("exit %d, stderr %q; want %q", run.code, run.stderr, test.want) + } + if len(installerAPIRequests(github)) != 1 { + t.Fatal("an anonymous failure was retried") + } + }) + } +} + +// Wget's error body is kept where --content-on-error is supported. +func TestInstallerWgetKeepsTheAccessRefusalBody(t *testing.T) { + wget, err := exec.LookPath("wget") + if err != nil { + t.Skip("wget is not installed") + } + help, _ := exec.Command(wget, "--help").CombinedOutput() + if !strings.Contains(string(help), "--content-on-error") { + t.Skip("wget cannot retain error bodies") + } + github := newInstallGitHub(t, "v1.2.3") + github.refuse = &apiRefusal{status: 403, body: `{"message":"Resource not accessible by personal access token"}`} + run := runInstaller(t, github, nil, "GITHUB_TOKEN=installer-token", "PATH="+minimalPath(t, false)) + const want = "GitHub refused GITHUB_TOKEN access to Agent-Field/codeaf (403: Resource not accessible by personal access token), and the request without it failed too (403: Resource not accessible by personal access token); check what GITHUB_TOKEN may read, or pin VERSION=" + if run.code != 1 || run.stderr != "codeaf: "+want+"\n" { + t.Fatalf("exit %d, want %q:\n%s", run.code, want, run.output) + } + assertInstallerAnonymousRetry(t, github, "installer-token", "/repos/Agent-Field/codeaf/releases/latest") +} + +// D3: A wget without the advertised option must never be given that option. +func TestInstallerUsesWgetWithoutContentOnErrorSupport(t *testing.T) { + wget, err := exec.LookPath("wget") + if err != nil { + t.Skip("wget is not installed") + } + path := minimalPath(t, false) + wrapper := filepath.Join(path, "wget") + if err := os.Remove(wrapper); err != nil { + t.Fatal(err) + } + // The wrapper rejects the optional flag and otherwise uses a real transport. + script := "#!/bin/sh\n" + + "if [ \"$1\" = --help ]; then printf '%s\\n' 'minimal wget'; exit 0; fi\n" + + "for arg do if [ \"$arg\" = --content-on-error ]; then exit 64; fi; done\n" + + "exec '" + strings.ReplaceAll(wget, "'", "'\\''") + "' \"$@\"\n" + if err := os.WriteFile(wrapper, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + github := newInstallGitHub(t, "v1.2.3") + github.refuse = &apiRefusal{status: 403, tokenOnly: true, headers: map[string]string{"X-RateLimit-Remaining": "0"}} + run := runInstaller(t, github, nil, "GITHUB_TOKEN=installer-token", "PATH="+path, "CODEAF_NO_MODIFY_PATH=1") + const want = "GitHub refused GITHUB_TOKEN (its rate limit is used up for up to an hour); continuing without it" + if run.code != 0 || !strings.Contains(run.stderr, want) || !strings.Contains(run.output, "Installed codeaf v1.2.3\n") { + t.Fatalf("exit %d without the wget fallback:\n%s", run.code, run.output) + } + assertInstallerAnonymousRetry(t, github, "installer-token", "/repos/Agent-Field/codeaf/releases/latest") +} + +// D7: Failed or unparseable GNU output gives BSD date its turn; the zone may be empty. +func TestInstallerEpochRenderingTriesHostDateSyntaxInOrder(t *testing.T) { + block := installerPureFunctions(t) + for _, test := range []struct{ name, date, want, calls string }{ + {"GNU", "printf '12:34 GNU\\n'", "12:34 GNU", "-d\n"}, + {"BSD after failure", "if [ \"$1\" = -d ]; then exit 1; fi\nprintf '12:35 BSD\\n'", "12:35 BSD", "-d\n-r\n"}, + {"BSD after invalid output", "if [ \"$1\" = -d ]; then printf 'invalid\\n'; else printf '12:36 BSD\\n'; fi", "12:36 BSD", "-d\n-r\n"}, + {"empty zone", "printf '12:37 \\n'", "12:37", "-d\n"}, + } { + t.Run(test.name, func(t *testing.T) { + path := t.TempDir() + log := filepath.Join(t.TempDir(), "date-calls") + script := "#!/bin/sh\nprintf '%s\\n' \"$1\" >> \"$DATE_CALLS\"\n" + test.date + "\n" + if err := os.WriteFile(filepath.Join(path, "date"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + if got := runInstallerPureFunction(t, block, "api_epoch", []string{"1800000000"}, "PATH="+path, "DATE_CALLS="+log); got != test.want { + t.Errorf("api_epoch = %q, want %q", got, test.want) + } + calls, err := os.ReadFile(log) + if err != nil || string(calls) != test.calls { + t.Errorf("date calls = %q, %v; want %q", calls, err, test.calls) + } + }) + } +} + +func TestInstallerRunsWhenDateIsAbsent(t *testing.T) { + github := newInstallGitHub(t, "v1.2.3") + path := minimalPath(t, true) + if err := os.Remove(filepath.Join(path, "date")); err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + github.refuse = &apiRefusal{status: 429, tokenOnly: true, headers: map[string]string{"X-RateLimit-Reset": "1800000000"}} + run := runInstaller(t, github, nil, "GITHUB_TOKEN=installer-token", "PATH="+path, "CODEAF_NO_MODIFY_PATH=1") + if run.code != 0 || !strings.Contains(run.stderr, "its rate limit is used up for up to an hour") { + t.Fatalf("a missing optional date broke the install (exit %d):\n%s", run.code, run.output) + } +} + +func TestInstallerEpochRenderingFallsBackToBashWhenDateCannotRender(t *testing.T) { + bash, err := exec.LookPath("bash") + if err != nil { + t.Skip("bash is not installed") + } + probe := exec.Command(bash, "--noprofile", "--norc", "-c", `printf '%(%H:%M %Z)T' 1800000000`) + if _, err := probe.Output(); err != nil { + t.Skip("bash predates the time-format printf fallback") + } + block := installerPureFunctions(t) + path := t.TempDir() + if err := os.WriteFile(filepath.Join(path, "date"), []byte("#!/bin/sh\nprintf 'cannot render\\n' >&2\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + want := time.Unix(1800000000, 0).In(time.FixedZone("IST", 19800)).Format("15:04") + " IST" + if got := runInstallerPureFunction(t, block, "api_epoch", []string{"1800000000"}, "PATH="+path); got != want { + t.Errorf("api_epoch = %q, want %q", got, want) + } +} diff --git a/scripts/install.sh b/scripts/install.sh index b28a2d10ef..0995b7dafd 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -17,6 +17,12 @@ GITHUB_API="${CODEAF_GITHUB_API:-${AFORGE_GITHUB_API:-https://api.github.com}}" GITHUB_DOWNLOAD="${CODEAF_GITHUB_DOWNLOAD:-${AFORGE_GITHUB_DOWNLOAD:-https://github.com}}" # legacy-name # GitHub answers anonymous API calls sixty times an hour per address; a token raises that. TOKEN="${GITHUB_TOKEN:-${GH_TOKEN:-}}" +TOKEN_NAME="GH_TOKEN" +if [[ -n "${GITHUB_TOKEN:-}" ]]; then + TOKEN_NAME="GITHUB_TOKEN" +fi +TOKEN_REFUSED=0 +API_NOTE="" usage() { cat <<'EOF' @@ -46,7 +52,7 @@ Flags: Environment: CHANNEL, VERSION, CODEAF_INSTALL_NAME, CODEAF_INSTALL_DIR CODEAF_NO_MODIFY_PATH, CODEAF_NO_START, VERBOSE - GITHUB_TOKEN or GH_TOKEN: GitHub answers anonymous API calls sixty times an hour per address; a token raises that. + GITHUB_TOKEN or GH_TOKEN: GitHub answers anonymous API calls sixty times an hour per address; a token raises that. A refused token is set aside for one anonymous try. CODEAF_GITHUB_API and CODEAF_GITHUB_DOWNLOAD for mirrors and tests EOF } @@ -147,6 +153,15 @@ fail() { exit 1 } +# A recovered lookup still explains why the configured token was set aside. +note() { + if [[ -t 2 && -n "$DIM" ]]; then + printf ' %s%s%s\n' "$DIM" "$*" "$RESET" >&2 + else + printf 'codeaf: %s\n' "$*" >&2 + fi +} + usage_error() { printf 'codeaf: %s\n\n' "$*" >&2 usage >&2 @@ -255,7 +270,8 @@ case "$CHANNEL" in *) usage_error "CHANNEL must be stable, rc, dev, or staging" ;; esac -if [[ ! "$INSTALL_NAME" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then +name_pattern='^[A-Za-z0-9][A-Za-z0-9._-]*$' +if [[ ! "$INSTALL_NAME" =~ $name_pattern ]]; then usage_error "--name / CODEAF_INSTALL_NAME must match ^[A-Za-z0-9][A-Za-z0-9._-]*$" fi @@ -277,17 +293,25 @@ trap cleanup EXIT HUP INT TERM print_banner HTTP_STATUS="" +HTTP_HEADERS="" +HTTP_BODY="" +WGET_KEEPS_ERRORS="" http_get() { local url="$1" local destination="$2" local accept="${3:-application/vnd.github+json}" local authenticate="${4:-0}" local status + HTTP_HEADERS="$destination.headers" + HTTP_BODY="$destination" + # A transport failure must not leave a previous response looking current. + : > "$HTTP_HEADERS" + : > "$destination" if [[ "$VERBOSE" == "1" ]]; then printf 'codeaf: GET %s\n' "$url" fi if command -v curl >/dev/null 2>&1; then - local args=(-sSL --output "$destination" --write-out '%{http_code}' -H "Accept: ${accept}") + local args=(-sSL -D "$HTTP_HEADERS" --output "$destination" --write-out '%{http_code}' -H "Accept: ${accept}") if [[ "$authenticate" == "1" && -n "$TOKEN" ]]; then args+=(-H "Authorization: Bearer ${TOKEN}") fi @@ -302,18 +326,29 @@ http_get() { esac fi - local headers="$TMP_ROOT/http.headers" local wget_code local args=(-O "$destination" -S --header="Accept: ${accept}") if [[ "$authenticate" == "1" && -n "$TOKEN" ]]; then args+=(--header="Authorization: Bearer ${TOKEN}") fi - if wget "${args[@]}" "$url" 2> "$headers"; then + # BusyBox lacks this flag, so asking first keeps its successful downloads working. + # The help is read whole rather than piped to grep -q: under pipefail an early + # grep exit can kill wget mid-write and quietly read as "not supported". + if [[ -z "$WGET_KEEPS_ERRORS" ]]; then + WGET_KEEPS_ERRORS=0 + case "$(wget --help 2>&1 || true)" in + *--content-on-error*) WGET_KEEPS_ERRORS=1 ;; + esac + fi + if [[ "$WGET_KEEPS_ERRORS" == "1" ]]; then + args+=(--content-on-error) + fi + if wget "${args[@]}" "$url" 2> "$HTTP_HEADERS"; then wget_code=0 else wget_code=$? fi - status=$(awk '$1 ~ /^HTTP\/[0-9.]+$/ && $2 ~ /^[0-9][0-9][0-9]$/ {status=$2} END {print status}' "$headers") + status=$(awk '$1 ~ /^HTTP\/[0-9.]+$/ && $2 ~ /^[0-9][0-9][0-9]$/ {status=$2} END {print status}' "$HTTP_HEADERS") HTTP_STATUS="$status" if [[ "$wget_code" == "0" && "$status" == 2* ]]; then return 0 @@ -321,14 +356,229 @@ http_get() { return 1 } +# BEGIN api refusal: pure functions, read by internal/release/install_test.go +api_header() { + # Redirects and wget indentation must not let an earlier quota shadow the final answer. + LC_ALL=C awk -v wanted="$2" ' + { + sub(/^[ \t]+/, "") + sub(/\r$/, "") + if ($0 ~ /^HTTP\/[0-9.]+[ \t]+[0-9][0-9][0-9]/) { + value = "" + } else { + colon = index($0, ":") + if (colon && tolower(substr($0, 1, colon - 1)) == wanted) { + value = substr($0, colon + 1) + sub(/^[ \t]+/, "", value) + sub(/[ \t]+$/, "", value) + } + } + } + END {print value} + ' "$1" +} + +api_reason() { + local status="$1" headers="$2" body="$3" + case "$status" in + "") printf 'unreachable' ;; + 401) printf 'rejected' ;; + 429) printf 'quota' ;; + 403) + if [[ "$(api_header "$headers" x-ratelimit-remaining)" == "0" ]] || LC_ALL=C grep -qi 'rate limit' "$body" 2>/dev/null; then + printf 'quota' + else + printf 'access' + fi + ;; + 404) printf 'missing' ;; + 5??) printf 'server' ;; + *) printf 'other' ;; + esac +} + +api_epoch() { + local epoch="$1" rendered pattern='^[0-9][0-9]:[0-9][0-9]' digits='^[0-9]+$' + [[ "$epoch" =~ $digits ]] || return 1 + # A missing date is legal and keeps the conservative wait on minimal hosts. + command -v date >/dev/null 2>&1 || return 1 + # Each host gets its own date syntax; bash supplies the last option only where it exists. + # A BSD date once read -d as "set the kernel's daylight-saving flag", so it is + # never offered there, even though the attempt would fail without root. + case "$(uname -s 2>/dev/null)" in + Darwin|*BSD|DragonFly) ;; + *) + if rendered=$(date -d "@$epoch" '+%H:%M %Z' 2>/dev/null) && [[ "$rendered" =~ $pattern ]]; then + printf '%s' "${rendered% }" + return 0 + fi + ;; + esac + if rendered=$(date -r "$epoch" '+%H:%M %Z' 2>/dev/null) && [[ "$rendered" =~ $pattern ]]; then + printf '%s' "${rendered% }" + return 0 + fi + if (( BASH_VERSINFO[0] > 4 || (BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] >= 2) )); then + if rendered=$(printf '%(%H:%M %Z)T' "$epoch" 2>/dev/null) && [[ "$rendered" =~ $pattern ]]; then + printf '%s' "${rendered% }" + return 0 + fi + fi + return 1 +} + +api_wait() { + local headers="$1" reset rendered retry digits='^[0-9]+$' + reset=$(api_header "$headers" x-ratelimit-reset) + if [[ -n "$reset" ]] && rendered=$(api_epoch "$reset"); then + printf 'until %s' "$rendered" + return 0 + fi + retry=$(api_header "$headers" retry-after) + if [[ "$retry" =~ $digits ]]; then + LC_ALL=C awk -v seconds="$retry" 'BEGIN { + if (seconds <= 90) printf "for about a minute" + else printf "for about %.0f minutes", int((seconds + 59) / 60) + }' + else + printf 'for up to an hour' + fi +} + +api_message() { + # Only a bounded, printable message reaches the terminal, never the raw response body. + LC_ALL=C awk ' + {json = json $0 "\n"} + END { + # Spelled out rather than [[:space:]]: mawk 1.3.3 has no character classes. + if (!match(json, /"message"[ \t\r\n]*:[ \t\r\n]*"/)) exit + json = substr(json, RSTART + RLENGTH) + message = "" + for (i = 1; i <= length(json); i++) { + c = substr(json, i, 1) + if (c == "\\") { + c = substr(json, ++i, 1) + if (c == "u") { + code = 0 + for (j = 1; j <= 4; j++) { + digit = index("0123456789abcdef", tolower(substr(json, ++i, 1))) - 1 + if (digit < 0) exit + code = code * 16 + digit + } + c = "" + if (code >= 32 && code <= 126) c = sprintf("%c", code) + } else if (c != "\\" && c != "\"" && c != "/") { + c = "" + } + } else if (c == "\"") { + break + } + if (c ~ /^[ -~]$/) message = message c + } + gsub(/ +/, " ", message) + if (length(message) > 100) message = substr(message, 1, 100) "..." + printf "%s", message + } + ' "$1" +} +# END api refusal + +api_status_message() { + local status="$1" message + message=$(api_message "$2") + printf '%s' "$status" + [[ -z "$message" ]] || printf ': %s' "$message" +} + +api_short() { + local status="$1" headers="$2" body="$3" reason + reason=$(api_reason "$status" "$headers" "$body") + case "$reason" in + quota) printf "this address's rate limit is used up %s" "$(api_wait "$headers")" ;; + unreachable) printf 'GitHub could not be reached' ;; + missing|server) printf '%s' "$status" ;; + *) api_status_message "$status" "$body" ;; + esac +} + api_problem() { - fail "GitHub's API could not be reached or refused (a rate limit, or a repository you cannot read?); pin VERSION=, or export GITHUB_TOKEN" + local reason message detail host token_sentence advice + reason=$(api_reason "$HTTP_STATUS" "$HTTP_HEADERS" "$HTTP_BODY") + if [[ "$TOKEN_REFUSED" == "1" ]]; then + case "$(api_reason "$TOKEN_STATUS" "$TOKEN_HEADERS" "$TOKEN_BODY")" in + rejected) + token_sentence="GitHub rejected $TOKEN_NAME ($(api_status_message "$TOKEN_STATUS" "$TOKEN_BODY"))" + advice="replace or unset $TOKEN_NAME" + ;; + quota) + token_sentence="GitHub's API rate limit for $TOKEN_NAME is used up $(api_wait "$TOKEN_HEADERS")" + advice="wait" + ;; + access) + token_sentence="GitHub refused $TOKEN_NAME access to $REPOSITORY ($(api_status_message "$TOKEN_STATUS" "$TOKEN_BODY"))" + advice="check what $TOKEN_NAME may read" + ;; + esac + fail "$token_sentence, and the request without it failed too ($(api_short "$HTTP_STATUS" "$HTTP_HEADERS" "$HTTP_BODY")); $advice, or pin VERSION=" + fi + detail=$(api_status_message "$HTTP_STATUS" "$HTTP_BODY") + case "$reason" in + quota) fail "GitHub's API rate limit for this address is used up $(api_wait "$HTTP_HEADERS"); export GITHUB_TOKEN to use your own, or pin VERSION=" ;; + access) fail "GitHub's API refused access to $REPOSITORY ($detail); pin VERSION=" ;; + missing) + if [[ -n "$TOKEN" ]]; then + fail "GitHub's API cannot find $REPOSITORY with $TOKEN_NAME (404), which is also its answer for a repository the token cannot read; check what $TOKEN_NAME may read, or pin VERSION=" + fi + fail "GitHub's API cannot find $REPOSITORY (404), which is also its answer for a repository you cannot read; export GITHUB_TOKEN, or pin VERSION=" + ;; + rejected) fail "GitHub's API refused the request ($detail); pin VERSION=" ;; + server) fail "GitHub's API answered $HTTP_STATUS and may be having trouble; try again in a minute, or pin VERSION=" ;; + unreachable) + host="${GITHUB_API#*://}" + host="${host%%/*}" + fail "GitHub's API could not be reached at $host; check the connection, or pin VERSION=" + ;; + other) + message=$(api_message "$HTTP_BODY") + if [[ -n "$message" ]]; then + fail "GitHub's API answered $HTTP_STATUS ($message); pin VERSION=" + fi + fail "GitHub's API answered $HTTP_STATUS; pin VERSION=" + ;; + esac } release_api_get() { - local suffix="$1" - local destination="$2" - http_get "$GITHUB_API/repos/$REPOSITORY/$suffix" "$destination" "application/vnd.github+json" 1 + local suffix="$1" destination="$2" authenticate=0 short + if [[ -n "$TOKEN" && "$TOKEN_REFUSED" == "0" ]]; then + authenticate=1 + fi + if http_get "$GITHUB_API/repos/$REPOSITORY/$suffix" "$destination" "application/vnd.github+json" "$authenticate"; then + return 0 + fi + if [[ "$authenticate" != "1" ]]; then + return 1 + fi + case "$HTTP_STATUS" in + 401|403|429) ;; + *) return 1 ;; + esac + # THE REFUSED TOKEN STAYS ASIDE: a second lookup must not spend its bucket again. + TOKEN_REFUSED=1 + TOKEN_STATUS="$HTTP_STATUS" + TOKEN_HEADERS="$TMP_ROOT/token.headers" + TOKEN_BODY="$TMP_ROOT/token.json" + cp "$HTTP_HEADERS" "$TOKEN_HEADERS" + cp "$HTTP_BODY" "$TOKEN_BODY" + if ! http_get "$GITHUB_API/repos/$REPOSITORY/$suffix" "$destination" "application/vnd.github+json" 0; then + return 1 + fi + if [[ "$(api_reason "$TOKEN_STATUS" "$TOKEN_HEADERS" "$TOKEN_BODY")" == "quota" ]]; then + short="its rate limit is used up $(api_wait "$TOKEN_HEADERS")" + else + short=$(api_status_message "$TOKEN_STATUS" "$TOKEN_BODY") + fi + API_NOTE="GitHub refused $TOKEN_NAME ($short); continuing without it" } extract_tags() { @@ -338,15 +588,15 @@ extract_tags() { # Channel selection accepts exactly the grammar used by internal/update/version.go. tag_matches_channel() { - local channel="$1" - local tag="$2" + local channel="$1" tag="$2" pattern case "$channel" in - stable) [[ "$tag" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] ;; - rc) [[ "$tag" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)-rc\.([1-9][0-9]*)$ ]] ;; - dev) [[ "$tag" =~ ^dev-[0-9]{8}-[0-9a-f]{12}$ ]] ;; - staging) [[ "$tag" =~ ^staging-[0-9]{8}-[0-9a-f]{12}$ ]] ;; + stable) pattern='^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$' ;; + rc) pattern='^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)-rc\.([1-9][0-9]*)$' ;; + dev) pattern='^dev-[0-9]{8}-[0-9a-f]{12}$' ;; + staging) pattern='^staging-[0-9]{8}-[0-9a-f]{12}$' ;; *) return 1 ;; esac + [[ "$tag" =~ $pattern ]] } # GitHub's release-list order is not publish order, so the LAW carries each tag's own timestamp into channel selection. @@ -422,7 +672,7 @@ else case "$CHANNEL" in stable) if ! release_api_get "releases/latest" "$release_file"; then - if [[ "$HTTP_STATUS" == "404" ]]; then + if [[ "$HTTP_STATUS" == "404" && "$TOKEN_REFUSED" == "0" ]]; then fail "no stable build has been published yet" fi api_problem @@ -453,8 +703,11 @@ else fi spin_stop +if [[ -n "$API_NOTE" ]]; then + note "$API_NOTE" +fi if [[ -z "${TAG:-}" ]]; then - api_problem + fail "GitHub's API answered without a release tag; pin VERSION=" fi if [[ -z "$VERSION" ]] && ! tag_matches_channel "$CHANNEL" "$TAG"; then From 5818da2aa477cbdf5aea9661f3b38774bae75c53 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 22 Oct 2026 10:38:12 -0400 Subject: [PATCH 2/4] manual: say what the installer prints when GitHub refuses the release lookup The chat manual had no page for an installer that stopped on GitHub's API, and the GUIDE said a token only raises the anonymous limit. running-from-the-terminal now quotes the rate-limit, rejected-token and access sentences, the note printed when a refused token is set aside, and the VERSION pin that skips the lookup, with probes in a person's own words that must reach it. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/GUIDE.md | 2 +- .../manual/chat/running-from-the-terminal.md | 27 +++++++++++++++++++ internal/manual/chat_test.go | 6 +++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/GUIDE.md b/docs/GUIDE.md index d783f5e2f5..579b70979e 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -119,7 +119,7 @@ curl -fsSL https://agentfield.ai/get/codeaf | VERSION= bash | `--no-modify-path` | Print the PATH line without editing a shell file or linking into a folder on PATH. | | `--no-start` or `CODEAF_NO_START=1` | Do not ask to start codeaf when the install ends. | | `--verbose` | Print each GET. | -| `GITHUB_TOKEN` or `GH_TOKEN` | Raise GitHub's anonymous API limit. | +| `GITHUB_TOKEN` or `GH_TOKEN` | Raise GitHub's API limit; a token GitHub refuses is set aside for one anonymous try. | The script needs `curl` or `wget`, plus `sha256sum` or `shasum`. It downloads `checksums.txt` and refuses a sha256 mismatch. Unless `--no-modify-path` is set, it diff --git a/internal/manual/chat/running-from-the-terminal.md b/internal/manual/chat/running-from-the-terminal.md index b6c91fa103..6abf3828f9 100644 --- a/internal/manual/chat/running-from-the-terminal.md +++ b/internal/manual/chat/running-from-the-terminal.md @@ -197,6 +197,33 @@ in the repository, and the `telemetry` switch in `/settings` turns them off. running the install line again works too. With `automatic updates` on, a codeaf installed this way also installs each new stable release itself as it opens. +## The installer stopped — GitHub API rate limit used up, curl install fails with 403, 401 Bad credentials, GitHub rejected GITHUB_TOKEN, token refused, repository you cannot read + +The installer names GitHub's refusal. These are example messages; `HH:MM ZONE` +stands for the reset time GitHub supplies: + +- `GitHub's API rate limit for this address is used up until HH:MM ZONE; export GITHUB_TOKEN to use your own, or pin VERSION=` +- `GitHub rejected GITHUB_TOKEN (401: Bad credentials), and the request without it failed too (401: Bad credentials); replace or unset GITHUB_TOKEN, or pin VERSION=` +- `GitHub refused GITHUB_TOKEN access to Agent-Field/codeaf (403: Resource not accessible by personal access token), and the request without it failed too (403: Resource not accessible by personal access token); check what GITHUB_TOKEN may read, or pin VERSION=` + +A token GitHub refuses with 401, 403 or 429 is set aside for the rest of the run, +and the same lookup is asked once without it. If that works, installation +continues with a note: `GitHub refused GITHUB_TOKEN (its rate limit is used up until HH:MM ZONE); continuing without it`. +Other token failures get no retry. `GITHUB_TOKEN` supplies the token when +non-empty; otherwise `GH_TOKEN` does, and the message names `GH_TOKEN` instead. +Downloads never carry the token. + +The reset time comes from GitHub's header in this machine's local time, with +no zone suffix if the machine supplies none. Without a renderable reset, a +numeric Retry-After header gives `for about a minute` or `for about M minutes`; +with neither it says `for up to an hour`. A 403 without quota evidence means +access was refused. A 404 can also mean a repository you cannot read. Some wget +builds discard error bodies or headers, so a refusal can lack its message or +quota details. + +Pinning a complete release tag bypasses the API lookup for any of these failures: +`curl -fsSL https://agentfield.ai/get/codeaf | VERSION= bash`. + ## What is devaf — dev build beside codeaf — side by side — two versions — install under a different file name — --name `devaf` is the file name for a codeaf dev-channel build, not another product. diff --git a/internal/manual/chat_test.go b/internal/manual/chat_test.go index 168a6e6fc3..ccb145b219 100644 --- a/internal/manual/chat_test.go +++ b/internal/manual/chat_test.go @@ -30,6 +30,12 @@ func TestTheChatManualAnswersTheQuestionsPeopleAsk(t *testing.T) { question string page string }{ + // Installer refusals must reach the page that explains both API attempts. + {"installer says github api rate limit is used up", "running-from-the-terminal"}, + {"install script says GitHub rejected GITHUB_TOKEN bad credentials", "running-from-the-terminal"}, + {"curl install fails with 403 from github", "running-from-the-terminal"}, + {"why did the installer continue without my GH_TOKEN", "running-from-the-terminal"}, + // C12: Key troubleshooting must reach the self-contained restart section. {"I changed or unset my API key in the shell but codeaf still uses the old one", "starting-codeaf"}, {"your key was not accepted for this model", "starting-codeaf"}, From 62c4431d9dff129045763595568a46c167ffad74 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 22 Oct 2026 10:45:52 -0400 Subject: [PATCH 3/4] installer: refuse oversized reset and Retry-After numbers instead of inventing a wait A 38-digit x-ratelimit-reset slipped past GNU date's refusal into bash's printf fallback, which renders it as the epoch's first second, so the message named a reset time GitHub never sent. A Retry-After that long overflowed BusyBox awk into negative minutes. The reset now has at most eleven digits and Retry-After at most six, and anything longer reads as no number, which says "for up to an hour". Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/release/install_test.go | 4 ++++ scripts/install.sh | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/internal/release/install_test.go b/internal/release/install_test.go index f3f233086f..71b800b01c 100644 --- a/internal/release/install_test.go +++ b/internal/release/install_test.go @@ -1473,6 +1473,10 @@ func TestInstallerC9APIWait(t *testing.T) { {"invalid reset", "x-ratelimit-reset: tomorrow\n", "for up to an hour"}, {"invalid reset with retry", "x-ratelimit-reset: invalid\nretry-after: 600\n", "for about 10 minutes"}, {"invalid retry", "retry-after: 2 minutes\n", "for up to an hour"}, + {"oversized reset", "x-ratelimit-reset: " + strings.Repeat("9", 38) + "\n", "for up to an hour"}, + {"oversized reset with retry", "x-ratelimit-reset: " + strings.Repeat("9", 38) + "\nretry-after: 600\n", "for about 10 minutes"}, + {"oversized retry", "retry-after: " + strings.Repeat("9", 38) + "\n", "for up to an hour"}, + {"largest retry", "retry-after: 999999\n", "for about 16667 minutes"}, {"final block only", "HTTP/1.1 301 Moved\nx-ratelimit-reset: 1800000000\n\n HTTP/1.1 403 Forbidden\r\n Retry-After: 91\r\n", "for about 2 minutes"}, } { t.Run(test.name, func(t *testing.T) { diff --git a/scripts/install.sh b/scripts/install.sh index 0995b7dafd..e1e6c441f3 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -398,7 +398,9 @@ api_reason() { } api_epoch() { - local epoch="$1" rendered pattern='^[0-9][0-9]:[0-9][0-9]' digits='^[0-9]+$' + local epoch="$1" rendered pattern='^[0-9][0-9]:[0-9][0-9]' digits='^[0-9]{1,11}$' + # An epoch too long for any clock is refused here: bash's printf fallback + # would otherwise render it as the epoch's first second and invent a time. [[ "$epoch" =~ $digits ]] || return 1 # A missing date is legal and keeps the conservative wait on minimal hosts. command -v date >/dev/null 2>&1 || return 1 @@ -428,7 +430,9 @@ api_epoch() { } api_wait() { - local headers="$1" reset rendered retry digits='^[0-9]+$' + # Six digits is eleven days; a longer Retry-After overflows some awks into + # negative minutes, so it reads as no number at all. + local headers="$1" reset rendered retry digits='^[0-9]{1,6}$' reset=$(api_header "$headers" x-ratelimit-reset) if [[ -n "$reset" ]] && rendered=$(api_epoch "$reset"); then printf 'until %s' "$rendered" From d20a28b6e44c11b3792c5284bfa6fd24d65f160d Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 22 Oct 2026 10:47:55 -0400 Subject: [PATCH 4/4] changes: entry for #1890 Co-Authored-By: Claude Opus 5.5 (1M context) --- .../1890-installer-names-api-refusal.md | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 docs/changes/unreleased/1890-installer-names-api-refusal.md diff --git a/docs/changes/unreleased/1890-installer-names-api-refusal.md b/docs/changes/unreleased/1890-installer-names-api-refusal.md new file mode 100644 index 0000000000..a0fe2af6b7 --- /dev/null +++ b/docs/changes/unreleased/1890-installer-names-api-refusal.md @@ -0,0 +1,24 @@ +--- +kind: fixed +title: the installer says why GitHub's API refused, and retries once without a token GitHub refused +pr: 1890 +surface: [build, docs] +invalidates: + - >- + One sentence was believed to cover every refusal from GitHub's API: + `GitHub's API could not be reached or refused (a rate limit, or a repository + you cannot read?); pin VERSION=, or export GITHUB_TOKEN`. It is retired. + The installer now names a used-up rate limit with the local time it lifts, a + rejected token, refused access, a repository it cannot find, a server fault or + an unreachable host, and never tells someone who exported a token to export one. + - >- + A token GitHub refused ended the install. A 401, 403 or 429 on a token-bearing + lookup now sets the token aside and asks the same lookup once without it, + because the anonymous quota is a separate bucket; the install goes on with the + note `GitHub refused GITHUB_TOKEN (…); continuing without it`. +--- + +`http_get` used to keep only the status code, so the reason GitHub gives in the +same response (`x-ratelimit-remaining`, `x-ratelimit-reset`, the body's +`message`) was discarded. It now keeps the headers and body, and a pure function +of the three picks the sentence. Downloads still never carry the token.