From 63a0202212281f6d160b7448c31e87b82ddc6fcd Mon Sep 17 00:00:00 2001 From: Mike Landau Date: Thu, 22 Oct 2026 14:02:41 -0700 Subject: [PATCH] feat(pkgdef): add package definition library Add internal/pkgdef, the library behind package definitions (see specs/package-definitions.md): parsing and validation of *.pkg.json files, pkg:@ reference parsing, loading local and remote definitions (remote ones cached and pinned by content hash), version resolution from static lists, JSON indexes (via gojq) and GitHub releases, per-system artifact locking with checksums, and idempotent installs into $XDG_CACHE_HOME/devbox/pkgs///. The package is a leaf with no dependency on devpkg, lock or plugin so the wiring in a follow-up PR can build on it. The manager and script strategies defined by the spec return ErrNotImplemented for now. Co-Authored-By: Claude Fable 5.1 --- go.mod | 2 + go.sum | 4 + internal/pkgdef/checksum.go | 95 +++++ internal/pkgdef/client.go | 213 +++++++++++ internal/pkgdef/client_test.go | 558 +++++++++++++++++++++++++++++ internal/pkgdef/definition.go | 406 +++++++++++++++++++++ internal/pkgdef/definition_test.go | 192 ++++++++++ internal/pkgdef/extract.go | 142 ++++++++ internal/pkgdef/extract_test.go | 195 ++++++++++ internal/pkgdef/install.go | 361 +++++++++++++++++++ internal/pkgdef/load.go | 141 ++++++++ internal/pkgdef/lock.go | 83 +++++ internal/pkgdef/ref.go | 81 +++++ internal/pkgdef/ref_test.go | 72 ++++ internal/pkgdef/template.go | 110 ++++++ internal/pkgdef/template_test.go | 72 ++++ internal/pkgdef/version.go | 114 ++++++ internal/pkgdef/version_test.go | 81 +++++ internal/pkgdef/versions.go | 104 ++++++ specs/package-definitions.md | 4 +- vendor-hash | 2 +- 21 files changed, 3030 insertions(+), 2 deletions(-) create mode 100644 internal/pkgdef/checksum.go create mode 100644 internal/pkgdef/client.go create mode 100644 internal/pkgdef/client_test.go create mode 100644 internal/pkgdef/definition.go create mode 100644 internal/pkgdef/definition_test.go create mode 100644 internal/pkgdef/extract.go create mode 100644 internal/pkgdef/extract_test.go create mode 100644 internal/pkgdef/install.go create mode 100644 internal/pkgdef/load.go create mode 100644 internal/pkgdef/lock.go create mode 100644 internal/pkgdef/ref.go create mode 100644 internal/pkgdef/ref_test.go create mode 100644 internal/pkgdef/template.go create mode 100644 internal/pkgdef/template_test.go create mode 100644 internal/pkgdef/version.go create mode 100644 internal/pkgdef/version_test.go create mode 100644 internal/pkgdef/versions.go diff --git a/go.mod b/go.mod index 77dd5242af2..c66710cff4c 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/google/go-cmp v0.7.0 github.com/google/uuid v1.6.0 github.com/hashicorp/go-envparse v0.1.0 + github.com/itchyny/gojq v0.12.19 github.com/joho/godotenv v1.5.1 github.com/mattn/go-isatty v0.0.22 github.com/mholt/archives v0.1.5 @@ -163,6 +164,7 @@ require ( github.com/hexops/gotextdiff v1.0.3 // indirect github.com/hinshun/vt10x v0.0.0-20220301184237-5011da428d02 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/itchyny/timefmt-go v0.1.8 // indirect github.com/jgautheron/goconst v1.11.0 // indirect github.com/jjti/go-spancheck v0.6.5 // indirect github.com/juju/errors v1.0.0 // indirect diff --git a/go.sum b/go.sum index 11361f3323d..a1971307cef 100644 --- a/go.sum +++ b/go.sum @@ -453,6 +453,10 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/itchyny/gojq v0.12.19 h1:ttXA0XCLEMoaLOz5lSeFOZ6u6Q3QxmG46vfgI4O0DEs= +github.com/itchyny/gojq v0.12.19/go.mod h1:5galtVPDywX8SPSOrqjGxkBeDhSxEW1gSxoy7tn1iZY= +github.com/itchyny/timefmt-go v0.1.8 h1:1YEo1JvfXeAHKdjelbYr/uCuhkybaHCeTkH8Bo791OI= +github.com/itchyny/timefmt-go v0.1.8/go.mod h1:5E46Q+zj7vbTgWY8o5YkMeYb4I6GeWLFnetPy5oBrAI= github.com/jgautheron/goconst v1.11.0 h1:KgN90z5qXt5f0Uzf3cWXev3hfMMFUyNeKdpkSBRvLDk= github.com/jgautheron/goconst v1.11.0/go.mod h1:0p+wv1lFOiUr0IlNNT1nrm6+8DB8u2sU6KHGzFRXHDc= github.com/jjti/go-spancheck v0.6.5 h1:lmi7pKxa37oKYIMScialXUK6hP3iY5F1gu+mLBPgYB8= diff --git a/internal/pkgdef/checksum.go b/internal/pkgdef/checksum.go new file mode 100644 index 00000000000..9542d68f636 --- /dev/null +++ b/internal/pkgdef/checksum.go @@ -0,0 +1,95 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "context" + "encoding/hex" + "encoding/json" + "fmt" + "strings" +) + +// fetchChecksum returns the lowercase hex sha256 for the artifact described +// by vars, or "" when the checksum type is "none". +func (c *Client) fetchChecksum(ctx context.Context, src *Checksum, vars Vars) (string, error) { + if src == nil || src.Type == ChecksumNone { + return "", nil + } + rawURL, err := Render(src.URL, vars) + if err != nil { + return "", err + } + switch src.Type { + case ChecksumHTTPJSON: + return c.fetchChecksumHTTPJSON(ctx, src, rawURL, vars) + case ChecksumSHA256URL: + body, err := c.get(ctx, rawURL) + if err != nil { + return "", fmt.Errorf("checksum: %w", err) + } + return parseSHA256Body(string(body), vars.Filename, rawURL) + default: + return "", fmt.Errorf("checksum.type %q is not supported", src.Type) + } +} + +func (c *Client) fetchChecksumHTTPJSON(ctx context.Context, src *Checksum, rawURL string, vars Vars) (string, error) { + expr, err := Render(src.Select, vars) + if err != nil { + return "", err + } + code, err := compileJQ(expr) + if err != nil { + return "", err + } + body, err := c.get(ctx, rawURL) + if err != nil { + return "", fmt.Errorf("checksum: %w", err) + } + var input any + if err := json.Unmarshal(body, &input); err != nil { + return "", fmt.Errorf("checksum: %s: response is not JSON: %w", rawURL, err) + } + results, err := runJQ(code, expr, input) + if err != nil { + return "", fmt.Errorf("checksum: %w", err) + } + if len(results) == 0 { + return "", fmt.Errorf("checksum: %s: jq expression %q matched nothing for %s", rawURL, expr, vars.Filename) + } + return normalizeSHA256(results[0], rawURL) +} + +// parseSHA256Body extracts the digest for filename from a checksum file. +// The body is either a bare digest or a sha256sum-style list of +// " " lines (a leading "*" marks binary mode). +func parseSHA256Body(body, filename, source string) (string, error) { + fields := strings.Fields(body) + if len(fields) == 1 { + return normalizeSHA256(fields[0], source) + } + for line := range strings.SplitSeq(body, "\n") { + parts := strings.Fields(line) + if len(parts) < 2 { + continue + } + name := strings.TrimPrefix(parts[1], "*") + if name == filename || strings.HasSuffix(name, "/"+filename) { + return normalizeSHA256(parts[0], source) + } + } + return "", fmt.Errorf("checksum: %s has no entry for %s", source, filename) +} + +func normalizeSHA256(digest, source string) (string, error) { + digest = strings.ToLower(strings.TrimSpace(strings.TrimPrefix(digest, "sha256:"))) + if len(digest) != 64 { + return "", fmt.Errorf("checksum: %s: %q is not a hex sha256", source, digest) + } + if _, err := hex.DecodeString(digest); err != nil { + return "", fmt.Errorf("checksum: %s: %q is not a hex sha256", source, digest) + } + return digest, nil +} diff --git a/internal/pkgdef/client.go b/internal/pkgdef/client.go new file mode 100644 index 00000000000..f33f722a6dd --- /dev/null +++ b/internal/pkgdef/client.go @@ -0,0 +1,213 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" + + "github.com/itchyny/gojq" + "go.jetify.com/devbox/internal/build" + "go.jetify.com/devbox/internal/xdg" +) + +// ErrNotFound is returned when an HTTP resource responds with 404. +var ErrNotFound = errors.New("not found") + +// ErrNotImplemented is returned for install strategies and version sources +// that the spec defines but this version of Devbox does not implement yet. +var ErrNotImplemented = errors.New("not implemented") + +// Client resolves, locks and installs package definitions. The zero value +// is not usable; use NewClient. +type Client struct { + // HTTPClient is used for every network request. + HTTPClient *http.Client + + // CacheDir is the root under which packages are installed + // (///) and where remote definitions + // and downloads are cached. + CacheDir string + + // GitHubAPIURL is the base URL of the GitHub REST API. Tests point it + // at an httptest server. + GitHubAPIURL string + + // GitHubToken, when set, authenticates GitHub API requests to raise + // the rate limit. + GitHubToken string + + // Stderr receives the output of post_install hooks. Defaults to + // io.Discard. + Stderr io.Writer +} + +// NewClient returns a client that caches under $XDG_CACHE_HOME/devbox/pkgs +// and reads GITHUB_TOKEN (or the legacy DEVBOX_GITHUB_API_TOKEN, as runx +// packages do) for GitHub API requests. +func NewClient() *Client { + return &Client{ + HTTPClient: &http.Client{Timeout: 10 * time.Minute}, + CacheDir: xdg.CacheSubpath(filepath.Join("devbox", "pkgs")), + GitHubAPIURL: "https://api.github.com", + GitHubToken: githubTokenFromEnv(), + Stderr: io.Discard, + } +} + +func githubTokenFromEnv() string { + if token := os.Getenv("GITHUB_TOKEN"); token != "" { + return token + } + return os.Getenv("DEVBOX_GITHUB_API_TOKEN") +} + +// InstallDir returns the directory a version of a package is installed to +// on a system. It does not check whether the package is installed. +func (c *Client) InstallDir(name, version, system string) string { + return filepath.Join(c.CacheDir, name, version, system) +} + +func (c *Client) userAgent() string { + return fmt.Sprintf("Devbox/%s (%s; %s)", build.Version, runtime.GOOS, runtime.GOARCH) +} + +// get fetches a URL into memory. file:// URLs are read from disk so that +// tests and local mirrors work without a server. No credentials are ever +// attached: definitions are untrusted input and may point anywhere. +func (c *Client) get(ctx context.Context, rawURL string) ([]byte, error) { + return c.getWithAuth(ctx, rawURL, "") +} + +// getGitHubAPI fetches a GitHub REST API path (relative to GitHubAPIURL) +// with the configured token, if any. +func (c *Client) getGitHubAPI(ctx context.Context, apiPath string) ([]byte, error) { + endpoint := strings.TrimSuffix(c.GitHubAPIURL, "/") + "/" + strings.TrimPrefix(apiPath, "/") + auth := "" + if c.GitHubToken != "" { + auth = "Bearer " + c.GitHubToken + } + return c.getWithAuth(ctx, endpoint, auth) +} + +func (c *Client) getWithAuth(ctx context.Context, rawURL, authorization string) ([]byte, error) { + if p, ok := strings.CutPrefix(rawURL, "file://"); ok { + b, err := os.ReadFile(p) + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("%s: %w", rawURL, ErrNotFound) + } + return b, err + } + + body, err := c.open(ctx, rawURL, authorization) + if err != nil { + return nil, err + } + defer body.Close() + return io.ReadAll(body) +} + +// open starts a GET request and returns the response body. The caller must +// close it. Non-2xx responses are returned as errors that include the start +// of the body, which is usually a short explanation. +func (c *Client) open(ctx context.Context, rawURL, authorization string) (io.ReadCloser, error) { + if p, ok := strings.CutPrefix(rawURL, "file://"); ok { + f, err := os.Open(p) + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("%s: %w", rawURL, ErrNotFound) + } + return f, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", c.userAgent()) + if authorization != "" { + req.Header.Set("Authorization", authorization) + } + res, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + if res.StatusCode == http.StatusNotFound { + res.Body.Close() + return nil, fmt.Errorf("GET %s: %w", rawURL, ErrNotFound) + } + if res.StatusCode < 200 || res.StatusCode >= 300 { + snippet, _ := io.ReadAll(io.LimitReader(res.Body, 512)) + res.Body.Close() + msg := strings.TrimSpace(string(bytes.ToValidUTF8(snippet, nil))) + if msg == "" { + return nil, fmt.Errorf("GET %s: %s", rawURL, res.Status) + } + return nil, fmt.Errorf("GET %s: %s: %s", rawURL, res.Status, msg) + } + return res.Body, nil +} + +func compileJQ(expr string) (*gojq.Code, error) { + query, err := gojq.Parse(expr) + if err != nil { + return nil, fmt.Errorf("invalid jq expression %q: %w", expr, err) + } + code, err := gojq.Compile(query) + if err != nil { + return nil, fmt.Errorf("invalid jq expression %q: %w", expr, err) + } + return code, nil +} + +// runJQ evaluates a compiled jq expression against input and returns every +// string it emits. Numbers are formatted as strings; arrays are flattened +// one level so that ".[].version" and "[.[].version]" both work. +func runJQ(code *gojq.Code, expr string, input any) ([]string, error) { + var out []string + iter := code.Run(input) + for { + value, ok := iter.Next() + if !ok { + break + } + if err, isErr := value.(error); isErr { + return nil, fmt.Errorf("jq expression %q: %w", expr, err) + } + if arr, isArr := value.([]any); isArr { + for _, elem := range arr { + out = appendJQValue(out, elem) + } + continue + } + out = appendJQValue(out, value) + } + return out, nil +} + +func appendJQValue(out []string, v any) []string { + switch val := v.(type) { + case string: + return append(out, val) + case float64: + // encoding/json decodes every number as float64. Format without an + // exponent so that 20240101 stays "20240101" and not "2.0240101e+07". + return append(out, strconv.FormatFloat(val, 'f', -1, 64)) + case int: + return append(out, strconv.Itoa(val)) + case nil, bool: + return out + default: + return append(out, fmt.Sprint(val)) + } +} diff --git a/internal/pkgdef/client_test.go b/internal/pkgdef/client_test.go new file mode 100644 index 00000000000..fb6b7dae0d5 --- /dev/null +++ b/internal/pkgdef/client_test.go @@ -0,0 +1,558 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testSystem = "x86_64-linux" + +// fakeVendor is an httptest server that behaves like a vendor download +// site: a JSON index of versions with checksums, tarballs, a bare binary, +// a SHASUMS file and a GitHub releases endpoint. +type fakeVendor struct { + srv *httptest.Server + tarball []byte + tarSHA string + binary []byte + binSHA string + + mu sync.Mutex + hits map[string]int +} + +func (vendor *fakeVendor) hit(key string) { + vendor.mu.Lock() + defer vendor.mu.Unlock() + vendor.hits[key]++ +} + +func (vendor *fakeVendor) hitCount(key string) int { + vendor.mu.Lock() + defer vendor.mu.Unlock() + return vendor.hits[key] +} + +func newFakeVendor(t *testing.T) *fakeVendor { + t.Helper() + vendor := &fakeVendor{hits: map[string]int{}} + vendor.tarball = makeTarGz(t, map[string]string{ + "hello-1.2.0/bin/hello": "#!/bin/sh\necho hello 1.2.0\n", + "hello-1.2.0/README": "readme\n", + }) + vendor.tarSHA = sha256Hex(vendor.tarball) + vendor.binary = []byte("#!/bin/sh\necho hello binary\n") + vendor.binSHA = sha256Hex(vendor.binary) + + mux := http.NewServeMux() + mux.HandleFunc("/index.json", func(w http.ResponseWriter, _ *http.Request) { + vendor.hit("/index.json") + fmt.Fprintf(w, `[ + {"version": "v1.2.0", "stable": true, "files": [ + {"filename": "hello-1.2.0-linux-amd64.tar.gz", "sha256": %q}, + {"filename": "hello-1.2.0-darwin-arm64.tar.gz", "sha256": "%s"} + ]}, + {"version": "v1.1.0", "stable": true, "files": []}, + {"version": "v1.3.0-rc1", "stable": false, "files": []} + ]`, vendor.tarSHA, strings.Repeat("ab", 32)) + }) + mux.HandleFunc("/dl/", func(w http.ResponseWriter, r *http.Request) { + vendor.hit(r.URL.Path) + switch { + case strings.HasSuffix(r.URL.Path, ".tar.gz"): + _, _ = w.Write(vendor.tarball) + case strings.HasSuffix(r.URL.Path, "/hello"): + _, _ = w.Write(vendor.binary) + case strings.HasSuffix(r.URL.Path, "SHASUMS256.txt"): + fmt.Fprintf(w, "%s hello-1.2.0-linux-amd64.tar.gz\n%s *hello\n", vendor.tarSHA, vendor.binSHA) + case strings.HasSuffix(r.URL.Path, ".sha256"): + fmt.Fprintf(w, "%s\n", vendor.binSHA) + default: + http.NotFound(w, r) + } + }) + mux.HandleFunc("/repos/acme/hello/releases", func(w http.ResponseWriter, r *http.Request) { + vendor.hit("github") + if auth := r.Header.Get("Authorization"); auth != "" { + w.Header().Set("X-Seen-Auth", auth) + } + _, _ = io.WriteString(w, `[ + {"tag_name": "v2.0.0-beta.1", "draft": false, "prerelease": true}, + {"tag_name": "v1.5.0", "draft": true, "prerelease": false}, + {"tag_name": "v1.4.2", "draft": false, "prerelease": false}, + {"tag_name": "v1.4.1", "draft": false, "prerelease": false}, + {"tag_name": "nightly", "draft": false, "prerelease": false} + ]`) + }) + vendor.srv = httptest.NewServer(mux) + t.Cleanup(vendor.srv.Close) + return vendor +} + +func (vendor *fakeVendor) client(t *testing.T) *Client { + t.Helper() + cli := NewClient() + cli.HTTPClient = vendor.srv.Client() + cli.CacheDir = filepath.Join(t.TempDir(), "pkgs") + cli.GitHubAPIURL = vendor.srv.URL + cli.GitHubToken = "" + return cli +} + +func (vendor *fakeVendor) archiveDef(t *testing.T) *Definition { + t.Helper() + def, err := Parse([]byte(fmt.Sprintf(`{ + "schema": 1, + "name": "hello", + "platforms": ["x86_64-linux", "aarch64-darwin"], + "versions": {"type": "http_json", "url": %[1]q, "select": ".[] | select(.stable) | .version | ltrimstr(\"v\")"}, + "install": { + "type": "archive", + "url": "%[2]s/dl/hello-{{.Version}}-{{.OS}}-{{.Arch}}.tar.gz", + "checksum": {"type": "http_json", "url": %[1]q, "select": ".[].files[] | select(.filename == \"{{.Filename}}\") | .sha256"}, + "strip_components": 1 + }, + "binaries": ["hello"], + "hooks": {"post_install": ["echo installed {{.Version}} > $DEVBOX_PKG_INSTALL_DIR/hook.txt"]} + }`, vendor.srv.URL+"/index.json", vendor.srv.URL))) + require.NoError(t, err) + return def +} + +func (vendor *fakeVendor) binaryDef(t *testing.T, checksum string) *Definition { + t.Helper() + def, err := Parse([]byte(fmt.Sprintf(`{ + "schema": 1, + "name": "hello-bin", + "platforms": ["x86_64-linux"], + "versions": {"type": "static", "versions": ["1.2.0"]}, + "install": {"type": "binary", "url": "%s/dl/{{.Version}}/hello", "bin_name": "hi", %s} + }`, vendor.srv.URL, checksum))) + require.NoError(t, err) + return def +} + +func TestResolveAndLock(t *testing.T) { + vendor := newFakeVendor(t) + cli := vendor.client(t) + def := vendor.archiveDef(t) + + versions, err := cli.ListVersions(t.Context(), def) + require.NoError(t, err) + assert.ElementsMatch(t, []string{"1.2.0", "1.1.0"}, versions) + + resolved, err := cli.Resolve(t.Context(), def, "1") + require.NoError(t, err) + assert.Equal(t, "1.2.0", resolved.Version) + assert.Equal(t, map[string]Artifact{ + "x86_64-linux": {URL: vendor.srv.URL + "/dl/hello-1.2.0-linux-amd64.tar.gz", Sha256: vendor.tarSHA}, + "aarch64-darwin": {URL: vendor.srv.URL + "/dl/hello-1.2.0-darwin-arm64.tar.gz", Sha256: strings.Repeat("ab", 32)}, + }, resolved.Artifacts) + + _, err = cli.Resolve(t.Context(), def, "9") + require.ErrorIs(t, err, ErrVersionNotFound) + + // A version with no checksum entry in the index is a lock error, not a + // silently empty digest. + _, err = cli.Lock(t.Context(), def, "1.1.0") + require.Error(t, err) + assert.Contains(t, err.Error(), "matched nothing") +} + +func TestInstallArchive(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("sh is required") + } + vendor := newFakeVendor(t) + cli := vendor.client(t) + def := vendor.archiveDef(t) + art := Artifact{URL: vendor.srv.URL + "/dl/hello-1.2.0-linux-amd64.tar.gz", Sha256: vendor.tarSHA} + + assert.False(t, cli.IsInstalled(def, "1.2.0", testSystem)) + got, err := cli.Install(t.Context(), def, "1.2.0", testSystem, art) + require.NoError(t, err) + assert.Equal(t, cli.InstallDir("hello", "1.2.0", testSystem), got.Dir) + assert.Equal(t, vendor.tarSHA, got.Sha256) + assert.Equal(t, []string{filepath.Join(got.Dir, "bin")}, got.BinDirs) + assert.True(t, cli.IsInstalled(def, "1.2.0", testSystem)) + + // strip_components removed the hello-1.2.0/ directory. + content, err := os.ReadFile(filepath.Join(got.Dir, "bin", "hello")) + require.NoError(t, err) + assert.Contains(t, string(content), "echo hello 1.2.0") + _, err = os.Stat(filepath.Join(got.Dir, "README")) + require.NoError(t, err) + + // The post_install hook ran with the install dir in its environment. + hook, err := os.ReadFile(filepath.Join(got.Dir, "hook.txt")) + require.NoError(t, err) + assert.Equal(t, "installed 1.2.0\n", string(hook)) + + // The download is removed after a successful install. + _, err = os.Stat(filepath.Join(cli.CacheDir, ".downloads", "hello", "1.2.0", testSystem, "hello-1.2.0-linux-amd64.tar.gz")) + assert.True(t, os.IsNotExist(err)) + + // A second install is served from disk without touching the network. + hits := vendor.hits["/dl/hello-1.2.0-linux-amd64.tar.gz"] + again, err := cli.Install(t.Context(), def, "1.2.0", testSystem, art) + require.NoError(t, err) + assert.Equal(t, got, again) + assert.Equal(t, hits, vendor.hits["/dl/hello-1.2.0-linux-amd64.tar.gz"]) + + // Installing with a URL left empty renders it from the definition. + cli2 := vendor.client(t) + got2, err := cli2.Install(t.Context(), def, "1.2.0", testSystem, Artifact{Sha256: vendor.tarSHA}) + require.NoError(t, err) + assert.Equal(t, vendor.tarSHA, got2.Sha256) +} + +func TestInstallChecksumMismatch(t *testing.T) { + vendor := newFakeVendor(t) + cli := vendor.client(t) + def := vendor.archiveDef(t) + art := Artifact{URL: vendor.srv.URL + "/dl/hello-1.2.0-linux-amd64.tar.gz", Sha256: strings.Repeat("00", 32)} + + _, err := cli.Install(t.Context(), def, "1.2.0", testSystem, art) + require.ErrorIs(t, err, ErrChecksumMismatch) + assert.False(t, cli.IsInstalled(def, "1.2.0", testSystem)) + _, statErr := os.Stat(cli.InstallDir("hello", "1.2.0", testSystem)) + assert.True(t, os.IsNotExist(statErr), "nothing should be installed") + + // An existing install from a different artifact is also rejected. + _, err = cli.Install(t.Context(), def, "1.2.0", testSystem, Artifact{Sha256: vendor.tarSHA}) + require.NoError(t, err) + _, err = cli.Install(t.Context(), def, "1.2.0", testSystem, art) + require.ErrorIs(t, err, ErrChecksumMismatch) +} + +func TestInstallUnsupportedSystem(t *testing.T) { + vendor := newFakeVendor(t) + cli := vendor.client(t) + def := vendor.archiveDef(t) + _, err := cli.Install(t.Context(), def, "1.2.0", "aarch64-linux", Artifact{}) + require.ErrorIs(t, err, ErrUnsupportedSystem) +} + +func TestInstallBinaryTrustOnFirstUse(t *testing.T) { + vendor := newFakeVendor(t) + cli := vendor.client(t) + def := vendor.binaryDef(t, `"checksum": {"type": "none"}`) + + artifacts, err := cli.Lock(t.Context(), def, "1.2.0") + require.NoError(t, err) + assert.Equal(t, Artifact{URL: vendor.srv.URL + "/dl/1.2.0/hello"}, artifacts[testSystem], "no digest without a checksum source") + + got, err := cli.Install(t.Context(), def, "1.2.0", testSystem, artifacts[testSystem]) + require.NoError(t, err) + assert.Equal(t, vendor.binSHA, got.Sha256, "the observed digest is reported so the caller can lock it") + + bin := filepath.Join(got.Dir, "bin", "hi") + info, err := os.Stat(bin) + require.NoError(t, err) + assert.NotZero(t, info.Mode()&0o111, "binary must be executable") + content, err := os.ReadFile(bin) + require.NoError(t, err) + assert.Equal(t, vendor.binary, content) +} + +func TestChecksumSHA256URL(t *testing.T) { + vendor := newFakeVendor(t) + cli := vendor.client(t) + + // SHASUMS-style list matched by filename (with a "*" binary marker). + def := vendor.binaryDef(t, fmt.Sprintf(`"checksum": {"type": "sha256_url", "url": "%s/dl/SHASUMS256.txt"}`, vendor.srv.URL)) + artifacts, err := cli.Lock(t.Context(), def, "1.2.0") + require.NoError(t, err) + assert.Equal(t, vendor.binSHA, artifacts[testSystem].Sha256) + + // Bare digest at .sha256. + def = vendor.binaryDef(t, `"checksum": {"type": "sha256_url", "url": "{{.URL}}.sha256"}`) + artifacts, err = cli.Lock(t.Context(), def, "1.2.0") + require.NoError(t, err) + assert.Equal(t, vendor.binSHA, artifacts[testSystem].Sha256) + + // Missing checksum file is an error. + def = vendor.binaryDef(t, fmt.Sprintf(`"checksum": {"type": "sha256_url", "url": "%s/dl/missing.txt"}`, vendor.srv.URL)) + _, err = cli.Lock(t.Context(), def, "1.2.0") + require.ErrorIs(t, err, ErrNotFound) +} + +func TestParseSHA256Body(t *testing.T) { + digest := strings.Repeat("0a", 32) + got, err := parseSHA256Body(digest+"\n", "x", "src") + require.NoError(t, err) + assert.Equal(t, digest, got) + + got, err = parseSHA256Body("sha256:"+strings.ToUpper(digest), "x", "src") + require.NoError(t, err) + assert.Equal(t, digest, got) + + body := strings.Repeat("11", 32) + " other.tar.gz\n" + digest + " dist/x.tar.gz\n" + got, err = parseSHA256Body(body, "x.tar.gz", "src") + require.NoError(t, err) + assert.Equal(t, digest, got) + + _, err = parseSHA256Body(body, "nope.tar.gz", "src") + require.Error(t, err) + + _, err = parseSHA256Body("nothex", "x", "src") + require.Error(t, err) +} + +func TestListVersionsGitHub(t *testing.T) { + vendor := newFakeVendor(t) + cli := vendor.client(t) + def, err := Parse([]byte(`{ + "schema": 1, "name": "hello", "platforms": ["x86_64-linux"], + "versions": {"type": "github_releases", "repo": "acme/hello"}, + "install": {"type": "archive", "url": "u"} + }`)) + require.NoError(t, err) + + versions, err := cli.ListVersions(t.Context(), def) + require.NoError(t, err) + assert.Equal(t, []string{"1.4.2", "1.4.1"}, versions, "drafts, pre-releases and tags without the prefix are skipped") + + version, err := cli.ResolveVersion(t.Context(), def, "latest") + require.NoError(t, err) + assert.Equal(t, "1.4.2", version) + + // An empty strip_prefix keeps every tag. + empty := "" + def.Versions.StripPrefix = &empty + versions, err = cli.ListVersions(t.Context(), def) + require.NoError(t, err) + assert.Equal(t, []string{"v1.4.2", "v1.4.1", "nightly"}, versions) +} + +func TestNotImplementedStrategies(t *testing.T) { + vendor := newFakeVendor(t) + cli := vendor.client(t) + + def, err := Parse([]byte(`{ + "schema": 1, "name": "hello", "platforms": ["x86_64-linux"], + "versions": {"type": "script", "run": "echo 1"}, + "install": {"type": "script", "run": "true"} + }`)) + require.NoError(t, err) + assert.True(t, def.UsesScripts()) + + _, err = cli.ListVersions(t.Context(), def) + require.ErrorIs(t, err, ErrNotImplemented) + _, err = cli.Lock(t.Context(), def, "1") + require.ErrorIs(t, err, ErrNotImplemented) + _, err = cli.Install(t.Context(), def, "1", testSystem, Artifact{}) + require.ErrorIs(t, err, ErrNotImplemented) +} + +func TestLoadLocal(t *testing.T) { + cli := NewClient() + cli.CacheDir = t.TempDir() + projectDir := t.TempDir() + path := filepath.Join(projectDir, "defs", "hello.pkg.json") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(minimalDef), 0o644)) + + for _, source := range []string{"./defs/hello.pkg.json", path, "file://" + path} { + loaded, err := cli.Load(t.Context(), Ref{Source: source}, projectDir, "ignored-for-local") + require.NoError(t, err, source) + assert.Equal(t, "hello", loaded.Definition.Name) + assert.Equal(t, sha256Hex([]byte(minimalDef)), loaded.Hash) + assert.Equal(t, filepath.Dir(path), loaded.Dir) + } + + _, err := cli.Load(t.Context(), Ref{Source: "./missing.pkg.json"}, projectDir, "") + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestLoadRemote(t *testing.T) { + content := []byte(minimalDef) + served := content + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/hello.pkg.json" { + http.NotFound(w, r) + return + } + _, _ = w.Write(served) + })) + t.Cleanup(srv.Close) + + cli := NewClient() + cli.HTTPClient = srv.Client() + cli.CacheDir = t.TempDir() + ref := Ref{Source: srv.URL + "/hello.pkg.json", Version: "1"} + + loaded, err := cli.Load(t.Context(), ref, "", "") + require.NoError(t, err) + assert.Equal(t, "hello", loaded.Definition.Name) + assert.Equal(t, sha256Hex(content), loaded.Hash) + assert.Empty(t, loaded.Dir) + + // Pinned to the hash we just saw: served from cache even if the server + // now returns something else. + served = []byte(strings.Replace(minimalDef, `"hello"`, `"changed"`, 1)) + pinned, err := cli.Load(t.Context(), ref, "", loaded.Hash) + require.NoError(t, err) + assert.Equal(t, "hello", pinned.Definition.Name) + + // Pinned to a hash that is not cached and does not match: rejected. + _, err = cli.Load(t.Context(), ref, "", strings.Repeat("ff", 32)) + require.ErrorIs(t, err, ErrDefinitionChanged) + + // Unpinned picks up the new content. + fresh, err := cli.Load(t.Context(), ref, "", "") + require.NoError(t, err) + assert.Equal(t, "changed", fresh.Definition.Name) + + _, err = cli.Load(t.Context(), Ref{Source: srv.URL + "/missing.pkg.json"}, "", "") + require.ErrorIs(t, err, ErrNotFound) +} + +func TestGetErrors(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + _, _ = io.WriteString(w, "slow down") + })) + t.Cleanup(srv.Close) + cli := NewClient() + cli.HTTPClient = srv.Client() + + _, err := cli.get(t.Context(), srv.URL+"/x") + require.Error(t, err) + assert.Contains(t, err.Error(), "429") + assert.Contains(t, err.Error(), "slow down") +} + +func makeTarGz(t *testing.T, files map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + for name, content := range files { + mode := int64(0o644) + if strings.Contains(name, "/bin/") { + mode = 0o755 + } + require.NoError(t, tw.WriteHeader(&tar.Header{Name: name, Mode: mode, Size: int64(len(content))})) + _, err := tw.Write([]byte(content)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + return buf.Bytes() +} + +func sha256Hex(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} + +func TestGitHubTokenOnlyForGitHubAPI(t *testing.T) { + var seen []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = append(seen, r.URL.Path+"|"+r.Header.Get("Authorization")) + if strings.HasPrefix(r.URL.Path, "/repos/") { + _, _ = io.WriteString(w, `[{"tag_name": "v1.0.0"}]`) + return + } + _, _ = io.WriteString(w, `["1.0.0"]`) + })) + t.Cleanup(srv.Close) + + cli := NewClient() + cli.HTTPClient = srv.Client() + cli.CacheDir = t.TempDir() + cli.GitHubAPIURL = srv.URL + cli.GitHubToken = "secret" + + // A definition's own URLs (even on the same host as the API) never get + // the token. + def, err := Parse([]byte(fmt.Sprintf(`{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux"], + "versions": {"type": "http_json", "url": "%s/index.json", "select": ".[]"}, + "install": {"type": "archive", "url": "u"} + }`, srv.URL))) + require.NoError(t, err) + _, err = cli.ListVersions(t.Context(), def) + require.NoError(t, err) + + def.Versions = VersionSource{Type: VersionsGitHubReleases, Repo: "acme/x"} + _, err = cli.ListVersions(t.Context(), def) + require.NoError(t, err) + + assert.Equal(t, []string{"/index.json|", "/repos/acme/x/releases|Bearer secret"}, seen) +} + +func TestJQNumbersKeepTheirDigits(t *testing.T) { + code, err := compileJQ(".[]") + require.NoError(t, err) + var input any + require.NoError(t, json.Unmarshal([]byte(`[20240101, 1000000, 1.10, "1.2", true, null, [7, "8"]]`), &input)) + got, err := runJQ(code, ".[]", input) + require.NoError(t, err) + assert.Equal(t, []string{"20240101", "1000000", "1.1", "1.2", "7", "8"}, got) +} + +func TestInstallConcurrent(t *testing.T) { + vendor := newFakeVendor(t) + cli := vendor.client(t) + def := vendor.archiveDef(t) + art := Artifact{Sha256: vendor.tarSHA} + + const workers = 4 + results := make(chan error, workers) + for range workers { + go func() { + _, err := cli.Install(t.Context(), def, "1.2.0", testSystem, art) + results <- err + }() + } + for range workers { + require.NoError(t, <-results) + } + assert.True(t, cli.IsInstalled(def, "1.2.0", testSystem)) + content, err := os.ReadFile(filepath.Join(cli.InstallDir("hello", "1.2.0", testSystem), "bin", "hello")) + require.NoError(t, err) + assert.Contains(t, string(content), "echo hello 1.2.0") + assert.Equal(t, 1, vendor.hitCount("/dl/hello-1.2.0-linux-amd64.tar.gz"), "the archive is downloaded once") +} + +func TestInstallHookFailureIncludesOutput(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("sh is required") + } + vendor := newFakeVendor(t) + cli := vendor.client(t) + def := vendor.archiveDef(t) + def.Hooks.PostInstall = []string{"echo something went wrong >&2; exit 3"} + + _, err := cli.Install(t.Context(), def, "1.2.0", testSystem, Artifact{Sha256: vendor.tarSHA}) + require.Error(t, err) + assert.Contains(t, err.Error(), "exit status 3") + assert.Contains(t, err.Error(), "something went wrong") + assert.False(t, cli.IsInstalled(def, "1.2.0", testSystem), "no marker after a failed hook") + + info, err := os.Stat(cli.InstallDir("hello", "1.2.0", testSystem)) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o755), info.Mode().Perm(), "install dir is world readable") +} diff --git a/internal/pkgdef/definition.go b/internal/pkgdef/definition.go new file mode 100644 index 00000000000..30d4cb28a8c --- /dev/null +++ b/internal/pkgdef/definition.go @@ -0,0 +1,406 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +// Package pkgdef implements package definitions: declarative files that +// describe how to install a tool from the upstream vendor's binary +// distribution instead of nixpkgs. See specs/package-definitions.md for the +// format. +// +// The package is a leaf: it must not import devpkg, lock or plugin so that +// those packages can depend on it. +package pkgdef + +import ( + "encoding/json" + "fmt" + "regexp" + "slices" + "strings" + + "github.com/tailscale/hujson" +) + +// SchemaVersion is the only value of the "schema" field this version of +// Devbox understands. +const SchemaVersion = 1 + +// Scheme is the devbox.json package scheme for package definitions, +// e.g. "pkg:./go.pkg.json@1.22". +const Scheme = "pkg" + +// Prefix is Scheme followed by a colon. +const Prefix = Scheme + ":" + +// Source is the devbox.lock "source" value for packages installed from a +// package definition. +const Source = "pkg-def" + +// Version source types. +const ( + VersionsStatic = "static" + VersionsHTTPJSON = "http_json" + VersionsGitHubReleases = "github_releases" + VersionsScript = "script" +) + +// Install strategy types. +const ( + InstallArchive = "archive" + InstallBinary = "binary" + InstallManager = "manager" + InstallScript = "script" +) + +// Checksum source types. +const ( + ChecksumHTTPJSON = "http_json" + ChecksumSHA256URL = "sha256_url" + ChecksumNone = "none" +) + +// Definition is a parsed package definition file. +type Definition struct { + Schema int `json:"schema"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Homepage string `json:"homepage,omitempty"` + License string `json:"license,omitempty"` + + Platforms []string `json:"platforms"` + OSMap map[string]string `json:"os_map,omitempty"` + ArchMap map[string]string `json:"arch_map,omitempty"` + + Versions VersionSource `json:"versions"` + Install Install `json:"install"` + + Bin []string `json:"bin,omitempty"` + Binaries []string `json:"binaries,omitempty"` + + // Activation. These have the same schema as plugin.json and are + // handed to the plugin manager by the caller; pkgdef only validates + // that they are well-formed JSON. + Env map[string]string `json:"env,omitempty"` + Shell map[string]json.RawMessage `json:"shell,omitempty"` + CreateFiles map[string]string `json:"create_files,omitempty"` + + Hooks Hooks `json:"hooks,omitempty"` +} + +// VersionSource describes how to list the available versions of a package. +type VersionSource struct { + Type string `json:"type"` + + // static + Versions []string `json:"versions,omitempty"` + + // http_json + URL string `json:"url,omitempty"` + Select string `json:"select,omitempty"` + + // github_releases + Repo string `json:"repo,omitempty"` + StripPrefix *string `json:"strip_prefix,omitempty"` + + // script + Run string `json:"run,omitempty"` +} + +// Install describes how to install one version of a package on one system. +// The zero value of every field means "use the default". +type Install struct { + Type string `json:"type"` + + // archive and binary + URL string `json:"url,omitempty"` + Checksum *Checksum `json:"checksum,omitempty"` + + // archive + StripComponents *int `json:"strip_components,omitempty"` + + // binary + BinName string `json:"bin_name,omitempty"` + + // manager (planned) + Requires string `json:"requires,omitempty"` + Env map[string]string `json:"env,omitempty"` + InstallCmd string `json:"install,omitempty"` + Locate string `json:"locate,omitempty"` + + // script (planned) + Run string `json:"run,omitempty"` + + // Overrides maps a nix system to a partial Install whose non-zero + // fields replace the defaults for that system. Overrides may not + // themselves contain overrides. + Overrides map[string]*Install `json:"overrides,omitempty"` +} + +// Checksum describes where to find the sha256 of an artifact. +type Checksum struct { + Type string `json:"type"` + URL string `json:"url,omitempty"` + Select string `json:"select,omitempty"` +} + +// Hooks are shell commands run at specific points of a package's lifecycle. +type Hooks struct { + PostInstall []string `json:"post_install,omitempty"` +} + +var nameRegexp = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_.-]*$`) + +// githubRepoRegexp matches "owner/name" with exactly two segments made of +// the characters GitHub allows, so a repo can never be rewritten into a +// different API path. +var githubRepoRegexp = regexp.MustCompile(`^[A-Za-z0-9-]+/[A-Za-z0-9_.-]+$`) + +// knownSystems are the nix systems a definition may list in "platforms". +var knownSystems = []string{ + "aarch64-darwin", + "x86_64-darwin", + "aarch64-linux", + "x86_64-linux", +} + +// Parse parses and validates a package definition. The input may contain +// comments and trailing commas (hujson). +func Parse(content []byte) (*Definition, error) { + std, err := hujson.Standardize(slices.Clone(content)) + if err != nil { + return nil, fmt.Errorf("package definition is not valid JSON: %w", err) + } + def := &Definition{} + if err := json.Unmarshal(std, def); err != nil { + return nil, fmt.Errorf("package definition is not valid JSON: %w", err) + } + if err := def.Validate(); err != nil { + return nil, err + } + return def, nil +} + +// Validate checks the definition for structural errors. Parse calls it; it +// is exported for callers that build a Definition in code. +func (d *Definition) Validate() error { + var errs []string + fail := func(format string, args ...any) { + errs = append(errs, fmt.Sprintf(format, args...)) + } + + if d.Schema != SchemaVersion { + fail("schema must be %d, got %d", SchemaVersion, d.Schema) + } + if d.Name == "" { + fail("name is required") + } else if !nameRegexp.MatchString(d.Name) { + fail("name %q must match %s", d.Name, nameRegexp) + } + if len(d.Platforms) == 0 { + fail("platforms must list at least one system") + } + for _, sys := range d.Platforms { + if !slices.Contains(knownSystems, sys) { + fail("platforms: unknown system %q (want one of %s)", sys, strings.Join(knownSystems, ", ")) + } + } + for _, dir := range d.Bin { + if dir == "" || strings.HasPrefix(dir, "/") || strings.Contains(dir, "..") { + fail("bin: %q must be a relative path inside the install root", dir) + } + } + + errs = append(errs, d.Versions.validate()...) + errs = append(errs, d.Install.validate("install")...) + for sys, override := range d.Install.Overrides { + if !slices.Contains(d.Platforms, sys) { + fail("install.overrides: %q is not in platforms", sys) + } + if override == nil { + fail("install.overrides.%s must be an object", sys) + continue + } + if len(override.Overrides) > 0 { + fail("install.overrides.%s may not contain overrides", sys) + } + merged := d.Install.merge(override) + errs = append(errs, merged.validate("install.overrides."+sys)...) + } + + if len(errs) == 0 { + return nil + } + return fmt.Errorf("invalid package definition %q:\n - %s", d.Name, strings.Join(errs, "\n - ")) +} + +func (v *VersionSource) validate() []string { + var errs []string + switch v.Type { + case VersionsStatic: + if len(v.Versions) == 0 { + errs = append(errs, "versions.versions must list at least one version") + } + case VersionsHTTPJSON: + if v.URL == "" { + errs = append(errs, "versions.url is required for type http_json") + } + if v.Select == "" { + errs = append(errs, "versions.select is required for type http_json") + } else if _, err := compileJQ(v.Select); err != nil { + errs = append(errs, fmt.Sprintf("versions.select: %v", err)) + } + case VersionsGitHubReleases: + if _, repo, _ := strings.Cut(v.Repo, "/"); !githubRepoRegexp.MatchString(v.Repo) || repo == "." || repo == ".." { + errs = append(errs, "versions.repo must be of the form owner/name") + } + case VersionsScript: + if v.Run == "" { + errs = append(errs, "versions.run is required for type script") + } + case "": + errs = append(errs, "versions.type is required") + default: + errs = append(errs, fmt.Sprintf("versions.type %q is not one of %s, %s, %s, %s", + v.Type, VersionsStatic, VersionsHTTPJSON, VersionsGitHubReleases, VersionsScript)) + } + return errs +} + +func (in *Install) validate(path string) []string { + var errs []string + switch in.Type { + case InstallArchive, InstallBinary: + if in.URL == "" { + errs = append(errs, path+".url is required for type "+in.Type) + } + if in.StripComponents != nil && *in.StripComponents < 0 { + errs = append(errs, path+".strip_components must be >= 0") + } + if in.Checksum != nil { + errs = append(errs, in.Checksum.validate(path+".checksum")...) + } + case InstallManager: + if in.Requires == "" || in.InstallCmd == "" || in.Locate == "" { + errs = append(errs, path+".requires, .install and .locate are required for type manager") + } + case InstallScript: + if in.Run == "" { + errs = append(errs, path+".run is required for type script") + } + case "": + errs = append(errs, path+".type is required") + default: + errs = append(errs, fmt.Sprintf("%s.type %q is not one of %s, %s, %s, %s", + path, in.Type, InstallArchive, InstallBinary, InstallManager, InstallScript)) + } + return errs +} + +func (c *Checksum) validate(path string) []string { + var errs []string + switch c.Type { + case ChecksumHTTPJSON: + if c.URL == "" || c.Select == "" { + errs = append(errs, path+".url and .select are required for type http_json") + } else if _, err := compileJQ(c.Select); err != nil { + errs = append(errs, fmt.Sprintf("%s.select: %v", path, err)) + } + case ChecksumSHA256URL: + if c.URL == "" { + errs = append(errs, path+".url is required for type sha256_url") + } + case ChecksumNone: + case "": + errs = append(errs, path+".type is required") + default: + errs = append(errs, fmt.Sprintf("%s.type %q is not one of %s, %s, %s", + path, c.Type, ChecksumHTTPJSON, ChecksumSHA256URL, ChecksumNone)) + } + return errs +} + +// SupportsSystem reports whether the definition lists system in platforms. +func (d *Definition) SupportsSystem(system string) bool { + return slices.Contains(d.Platforms, system) +} + +// BinDirs returns the directories, relative to the install root, that +// should be added to PATH. It defaults to ["bin"]. +func (d *Definition) BinDirs() []string { + if len(d.Bin) == 0 { + return []string{"bin"} + } + return d.Bin +} + +// InstallFor returns the install strategy for system with any override for +// that system applied. +func (d *Definition) InstallFor(system string) Install { + override := d.Install.Overrides[system] + if override == nil { + merged := d.Install + merged.Overrides = nil + return merged + } + return d.Install.merge(override) +} + +// merge returns a copy of in with every non-zero field of override applied. +func (in *Install) merge(override *Install) Install { + merged := *in + merged.Overrides = nil + if override.Type != "" { + merged.Type = override.Type + } + if override.URL != "" { + merged.URL = override.URL + } + if override.Checksum != nil { + merged.Checksum = override.Checksum + } + if override.StripComponents != nil { + merged.StripComponents = override.StripComponents + } + if override.BinName != "" { + merged.BinName = override.BinName + } + if override.Requires != "" { + merged.Requires = override.Requires + } + if override.Env != nil { + merged.Env = override.Env + } + if override.InstallCmd != "" { + merged.InstallCmd = override.InstallCmd + } + if override.Locate != "" { + merged.Locate = override.Locate + } + if override.Run != "" { + merged.Run = override.Run + } + return merged +} + +// ChecksumType returns the effective checksum type, which is "none" when no +// checksum is configured. +func (in *Install) ChecksumType() string { + if in.Checksum == nil { + return ChecksumNone + } + return in.Checksum.Type +} + +// UsesScripts reports whether the definition runs shell that the user +// should be told about: script-type sources or strategies, or post-install +// hooks. +func (d *Definition) UsesScripts() bool { + if d.Versions.Type == VersionsScript || d.Install.Type == InstallScript { + return true + } + for _, o := range d.Install.Overrides { + if o != nil && o.Type == InstallScript { + return true + } + } + return len(d.Hooks.PostInstall) > 0 +} diff --git a/internal/pkgdef/definition_test.go b/internal/pkgdef/definition_test.go new file mode 100644 index 00000000000..a4796097d45 --- /dev/null +++ b/internal/pkgdef/definition_test.go @@ -0,0 +1,192 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const minimalDef = `{ + "schema": 1, + "name": "hello", + "platforms": ["aarch64-darwin", "x86_64-linux"], + "versions": {"type": "static", "versions": ["1.0.0", "1.1.0"]}, + "install": {"type": "archive", "url": "https://example.com/hello-{{.Version}}-{{.OS}}-{{.Arch}}.tar.gz"} +}` + +func TestParseGoExample(t *testing.T) { + content, err := os.ReadFile(filepath.Join("..", "..", "pkgs", "go", "go.pkg.json")) + require.NoError(t, err) + + def, err := Parse(content) + require.NoError(t, err) + assert.Equal(t, "go", def.Name) + assert.Equal(t, VersionsHTTPJSON, def.Versions.Type) + assert.Equal(t, InstallArchive, def.Install.Type) + assert.Equal(t, ChecksumHTTPJSON, def.Install.ChecksumType()) + assert.Equal(t, []string{"bin"}, def.BinDirs()) + assert.Equal(t, []string{"go", "gofmt"}, def.Binaries) + assert.False(t, def.UsesScripts()) + require.NotNil(t, def.Install.StripComponents) + assert.Equal(t, 1, *def.Install.StripComponents) +} + +func TestParseMinimalWithComments(t *testing.T) { + def, err := Parse([]byte("// comment\n" + minimalDef)) + require.NoError(t, err) + assert.Equal(t, "hello", def.Name) + assert.Equal(t, ChecksumNone, def.Install.ChecksumType()) + assert.True(t, def.SupportsSystem("x86_64-linux")) + assert.False(t, def.SupportsSystem("aarch64-linux")) +} + +func TestParseErrors(t *testing.T) { + tests := []struct { + name string + json string + want string + }{ + { + name: "not json", + json: `{`, + want: "not valid JSON", + }, + { + name: "wrong schema", + json: `{"schema": 2, "name": "x", "platforms": ["x86_64-linux"], "versions": {"type": "static", "versions": ["1"]}, "install": {"type": "archive", "url": "u"}}`, + want: "schema must be 1", + }, + { + name: "bad name", + json: `{"schema": 1, "name": "../x", "platforms": ["x86_64-linux"], "versions": {"type": "static", "versions": ["1"]}, "install": {"type": "archive", "url": "u"}}`, + want: "name \"../x\" must match", + }, + { + name: "no platforms", + json: `{"schema": 1, "name": "x", "platforms": [], "versions": {"type": "static", "versions": ["1"]}, "install": {"type": "archive", "url": "u"}}`, + want: "platforms must list at least one system", + }, + { + name: "unknown platform", + json: `{"schema": 1, "name": "x", "platforms": ["amd64-windows"], "versions": {"type": "static", "versions": ["1"]}, "install": {"type": "archive", "url": "u"}}`, + want: "unknown system \"amd64-windows\"", + }, + { + name: "missing versions type", + json: `{"schema": 1, "name": "x", "platforms": ["x86_64-linux"], "versions": {}, "install": {"type": "archive", "url": "u"}}`, + want: "versions.type is required", + }, + { + name: "bad jq", + json: `{"schema": 1, "name": "x", "platforms": ["x86_64-linux"], "versions": {"type": "http_json", "url": "u", "select": ".[ |"}, "install": {"type": "archive", "url": "u"}}`, + want: "versions.select: invalid jq expression", + }, + { + name: "github repo format", + json: `{"schema": 1, "name": "x", "platforms": ["x86_64-linux"], "versions": {"type": "github_releases", "repo": "nope"}, "install": {"type": "archive", "url": "u"}}`, + want: "versions.repo must be of the form owner/name", + }, + { + name: "github repo traversal", + json: `{"schema": 1, "name": "x", "platforms": ["x86_64-linux"], "versions": {"type": "github_releases", "repo": "owner/../../user"}, "install": {"type": "archive", "url": "u"}}`, + want: "versions.repo must be of the form owner/name", + }, + { + name: "github repo extra segment", + json: `{"schema": 1, "name": "x", "platforms": ["x86_64-linux"], "versions": {"type": "github_releases", "repo": "owner/name/extra"}, "install": {"type": "archive", "url": "u"}}`, + want: "versions.repo must be of the form owner/name", + }, + { + name: "install url missing", + json: `{"schema": 1, "name": "x", "platforms": ["x86_64-linux"], "versions": {"type": "static", "versions": ["1"]}, "install": {"type": "binary"}}`, + want: "install.url is required for type binary", + }, + { + name: "unknown install type", + json: `{"schema": 1, "name": "x", "platforms": ["x86_64-linux"], "versions": {"type": "static", "versions": ["1"]}, "install": {"type": "magic"}}`, + want: "install.type \"magic\" is not one of", + }, + { + name: "checksum type", + json: `{"schema": 1, "name": "x", "platforms": ["x86_64-linux"], "versions": {"type": "static", "versions": ["1"]}, "install": {"type": "archive", "url": "u", "checksum": {"type": "md5"}}}`, + want: "install.checksum.type \"md5\" is not one of", + }, + { + name: "override for unlisted platform", + json: `{"schema": 1, "name": "x", "platforms": ["x86_64-linux"], "versions": {"type": "static", "versions": ["1"]}, "install": {"type": "archive", "url": "u", "overrides": {"aarch64-darwin": {"url": "v"}}}}`, + want: "install.overrides: \"aarch64-darwin\" is not in platforms", + }, + { + name: "nested override", + json: `{"schema": 1, "name": "x", "platforms": ["x86_64-linux"], "versions": {"type": "static", "versions": ["1"]}, "install": {"type": "archive", "url": "u", "overrides": {"x86_64-linux": {"overrides": {"x86_64-linux": {}}}}}}`, + want: "install.overrides.x86_64-linux may not contain overrides", + }, + { + name: "bin escapes root", + json: `{"schema": 1, "name": "x", "platforms": ["x86_64-linux"], "versions": {"type": "static", "versions": ["1"]}, "install": {"type": "archive", "url": "u"}, "bin": ["../bin"]}`, + want: "bin: \"../bin\" must be a relative path", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := Parse([]byte(tt.json)) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.want) + }) + } +} + +func TestValidateReportsEveryError(t *testing.T) { + _, err := Parse([]byte(`{"schema": 1, "platforms": [], "versions": {}, "install": {}}`)) + require.Error(t, err) + msg := err.Error() + for _, want := range []string{"name is required", "platforms must list", "versions.type is required", "install.type is required"} { + assert.Contains(t, msg, want) + } + assert.Equal(t, 4, strings.Count(msg, "\n - ")) +} + +func TestInstallFor(t *testing.T) { + def, err := Parse([]byte(`{ + "schema": 1, "name": "x", "platforms": ["x86_64-linux", "aarch64-darwin"], + "versions": {"type": "static", "versions": ["1"]}, + "install": { + "type": "archive", "url": "default", "strip_components": 1, + "checksum": {"type": "sha256_url", "url": "{{.URL}}.sha256"}, + "overrides": {"aarch64-darwin": {"type": "binary", "url": "mac", "checksum": {"type": "none"}}} + } + }`)) + require.NoError(t, err) + + linux := def.InstallFor("x86_64-linux") + assert.Equal(t, InstallArchive, linux.Type) + assert.Equal(t, "default", linux.URL) + assert.Equal(t, ChecksumSHA256URL, linux.ChecksumType()) + assert.Nil(t, linux.Overrides) + + mac := def.InstallFor("aarch64-darwin") + assert.Equal(t, InstallBinary, mac.Type) + assert.Equal(t, "mac", mac.URL) + assert.Equal(t, ChecksumNone, mac.ChecksumType()) + require.NotNil(t, mac.StripComponents, "unset override fields keep the default") + assert.Equal(t, 1, *mac.StripComponents) + + // The definition itself is not mutated. + assert.Len(t, def.Install.Overrides, 1) +} + +func TestUsesScripts(t *testing.T) { + def, err := Parse([]byte(minimalDef)) + require.NoError(t, err) + assert.False(t, def.UsesScripts()) + + def.Hooks.PostInstall = []string{"true"} + assert.True(t, def.UsesScripts()) +} diff --git a/internal/pkgdef/extract.go b/internal/pkgdef/extract.go new file mode 100644 index 00000000000..10d704c4462 --- /dev/null +++ b/internal/pkgdef/extract.go @@ -0,0 +1,142 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" + + "github.com/mholt/archives" +) + +// extractArchive unpacks archivePath into dest, dropping the first +// stripComponents path elements of every entry. The format (tar with gz, +// xz, bz2 or zstd compression, or zip) is detected from the content. +// +// Entries are sanitized: paths that escape dest, absolute symlink targets +// and symlinks whose target escapes dest are rejected, and an existing +// symlink is never followed when writing a file. +func extractArchive(ctx context.Context, archivePath, dest string, stripComponents *int) error { + strip := 0 + if stripComponents != nil { + strip = *stripComponents + } + + archive, err := os.Open(archivePath) + if err != nil { + return err + } + defer archive.Close() + + format, stream, err := archives.Identify(ctx, filepath.Base(archivePath), archive) + if err != nil { + return fmt.Errorf("unrecognized archive format: %w", err) + } + extractor, ok := format.(archives.Extractor) + if !ok { + return fmt.Errorf("%s is not an archive", filepath.Base(archivePath)) + } + + handler := func(_ context.Context, info archives.FileInfo) error { + rel, ok := stripPath(info.NameInArchive, strip) + if !ok { + return nil + } + abs := filepath.Join(dest, rel) + if !isInside(dest, abs) { + return fmt.Errorf("archive entry %q escapes the destination", info.NameInArchive) + } + + mode := info.Mode() + switch { + case mode.IsDir(): + return os.MkdirAll(abs, 0o755) + case mode&os.ModeSymlink != 0: + return writeSymlink(dest, abs, info.LinkTarget, info.NameInArchive) + case mode.IsRegular(): + return writeRegular(abs, info) + default: + // Devices, sockets and the like have no place in a package. + return nil + } + } + return extractor.Extract(ctx, stream, handler) +} + +// stripPath cleans an archive path and drops its first strip components. It +// reports false when the entry should be skipped: it is one of the stripped +// directories or it has no name left. +func stripPath(name string, strip int) (string, bool) { + cleaned := path.Clean("/" + filepath.ToSlash(name)) // "/" anchors "..", stripped below + cleaned = strings.TrimPrefix(cleaned, "/") + if cleaned == "" || cleaned == "." { + return "", false + } + parts := strings.Split(cleaned, "/") + if len(parts) <= strip { + return "", false + } + return filepath.FromSlash(strings.Join(parts[strip:], "/")), true +} + +func isInside(root, p string) bool { + rel, err := filepath.Rel(root, p) + if err != nil { + return false + } + return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +func writeSymlink(dest, abs, target, name string) error { + if target == "" || filepath.IsAbs(target) { + return fmt.Errorf("archive entry %q: unsupported symlink target %q", name, target) + } + resolved := filepath.Join(filepath.Dir(abs), filepath.FromSlash(target)) + if !isInside(dest, resolved) { + return fmt.Errorf("archive entry %q: symlink target %q escapes the destination", name, target) + } + if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { + return err + } + if err := os.Remove(abs); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return os.Symlink(filepath.FromSlash(target), abs) +} + +func writeRegular(abs string, info archives.FileInfo) error { + if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { + return err + } + // Never write through a symlink that an earlier entry planted. + if existing, err := os.Lstat(abs); err == nil && existing.Mode()&os.ModeSymlink != 0 { + if err := os.Remove(abs); err != nil { + return err + } + } + + from, err := info.Open() + if err != nil { + return err + } + defer from.Close() + + // Keep the executable bits but never write files others can modify. + perm := info.Mode().Perm()&0o755 | 0o600 + to, err := os.OpenFile(abs, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + if _, err := io.Copy(to, from); err != nil { + to.Close() + return fmt.Errorf("writing %s: %w", abs, err) + } + return to.Close() +} diff --git a/internal/pkgdef/extract_test.go b/internal/pkgdef/extract_test.go new file mode 100644 index 00000000000..74c51a55df8 --- /dev/null +++ b/internal/pkgdef/extract_test.go @@ -0,0 +1,195 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type tarEntry struct { + name string + content string + linkTo string + dir bool + mode int64 + typeflag byte +} + +func writeTar(t *testing.T, path string, entries []tarEntry) { + t.Helper() + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tarw := tar.NewWriter(gzw) + for _, entry := range entries { + hdr := &tar.Header{Name: entry.name, Mode: entry.mode} + switch { + case entry.dir: + hdr.Typeflag = tar.TypeDir + case entry.linkTo != "": + hdr.Typeflag = tar.TypeSymlink + hdr.Linkname = entry.linkTo + case entry.typeflag != 0: + hdr.Typeflag = entry.typeflag + default: + hdr.Typeflag = tar.TypeReg + hdr.Size = int64(len(entry.content)) + } + if hdr.Mode == 0 { + hdr.Mode = 0o644 + } + require.NoError(t, tarw.WriteHeader(hdr)) + if hdr.Typeflag == tar.TypeReg { + _, err := tarw.Write([]byte(entry.content)) + require.NoError(t, err) + } + } + require.NoError(t, tarw.Close()) + require.NoError(t, gzw.Close()) + require.NoError(t, os.WriteFile(path, buf.Bytes(), 0o644)) +} + +func TestExtractTarWithSymlinksAndStrip(t *testing.T) { + dir := t.TempDir() + archivePath := filepath.Join(dir, "node.tar.gz") + writeTar(t, archivePath, []tarEntry{ + {name: "node-v20/", dir: true, mode: 0o755}, + {name: "node-v20/bin/", dir: true, mode: 0o755}, + {name: "node-v20/bin/node", content: "#!/bin/sh\necho node\n", mode: 0o755}, + {name: "node-v20/lib/npm/npm-cli.js", content: "js\n"}, + {name: "node-v20/bin/npm", linkTo: "../lib/npm/npm-cli.js", mode: 0o777}, + {name: "node-v20/dev/null", typeflag: tar.TypeChar}, + }) + + dest := filepath.Join(dir, "out") + strip := 1 + require.NoError(t, extractArchive(t.Context(), archivePath, dest, &strip)) + + node, err := os.Stat(filepath.Join(dest, "bin", "node")) + require.NoError(t, err) + assert.NotZero(t, node.Mode()&0o111, "executable bit is preserved") + + target, err := os.Readlink(filepath.Join(dest, "bin", "npm")) + require.NoError(t, err) + assert.Equal(t, filepath.FromSlash("../lib/npm/npm-cli.js"), target) + content, err := os.ReadFile(filepath.Join(dest, "bin", "npm")) + require.NoError(t, err, "symlink resolves inside the install") + assert.Equal(t, "js\n", string(content)) + + _, err = os.Stat(filepath.Join(dest, "node-v20")) + assert.True(t, os.IsNotExist(err), "the stripped directory is gone") + _, err = os.Lstat(filepath.Join(dest, "dev", "null")) + assert.True(t, os.IsNotExist(err), "device entries are skipped") +} + +func TestExtractZipNoStrip(t *testing.T) { + dir := t.TempDir() + archivePath := filepath.Join(dir, "tool.zip") + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, err := zw.Create("bin/tool") + require.NoError(t, err) + _, err = w.Write([]byte("tool\n")) + require.NoError(t, err) + require.NoError(t, zw.Close()) + require.NoError(t, os.WriteFile(archivePath, buf.Bytes(), 0o644)) + + dest := filepath.Join(dir, "out") + require.NoError(t, extractArchive(t.Context(), archivePath, dest, nil)) + content, err := os.ReadFile(filepath.Join(dest, "bin", "tool")) + require.NoError(t, err) + assert.Equal(t, "tool\n", string(content)) +} + +func TestExtractRejectsEscapes(t *testing.T) { + dir := t.TempDir() + + // A symlink pointing outside the destination is refused. + archivePath := filepath.Join(dir, "escape-link.tar.gz") + writeTar(t, archivePath, []tarEntry{ + {name: "pkg/bin/evil", linkTo: "../../../../etc/passwd", mode: 0o777}, + }) + err := extractArchive(t.Context(), archivePath, filepath.Join(dir, "out1"), nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "escapes the destination") + + // So is an absolute symlink target. + archivePath = filepath.Join(dir, "abs-link.tar.gz") + writeTar(t, archivePath, []tarEntry{ + {name: "pkg/bin/evil", linkTo: "/etc/passwd", mode: 0o777}, + }) + err = extractArchive(t.Context(), archivePath, filepath.Join(dir, "out2"), nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported symlink target") + + // A file that escapes via ".." lands inside the destination instead: + // the path is anchored and cleaned before use. + archivePath = filepath.Join(dir, "dotdot.tar.gz") + writeTar(t, archivePath, []tarEntry{ + {name: "../../outside.txt", content: "x"}, + }) + dest := filepath.Join(dir, "out3") + require.NoError(t, extractArchive(t.Context(), archivePath, dest, nil)) + _, err = os.Stat(filepath.Join(dest, "outside.txt")) + require.NoError(t, err) + _, err = os.Stat(filepath.Join(dir, "outside.txt")) + assert.True(t, os.IsNotExist(err)) + + // A regular file written through a planted symlink replaces the link + // rather than following it. + outside := filepath.Join(dir, "victim.txt") + require.NoError(t, os.WriteFile(outside, []byte("untouched"), 0o644)) + archivePath = filepath.Join(dir, "through-link.tar.gz") + writeTar(t, archivePath, []tarEntry{ + {name: "pkg/link", linkTo: "sub/real", mode: 0o777}, + {name: "pkg/link", content: "overwritten"}, + }) + dest = filepath.Join(dir, "out4") + require.NoError(t, extractArchive(t.Context(), archivePath, dest, nil)) + info, err := os.Lstat(filepath.Join(dest, "pkg", "link")) + require.NoError(t, err) + assert.True(t, info.Mode().IsRegular()) + content, err := os.ReadFile(outside) + require.NoError(t, err) + assert.Equal(t, "untouched", string(content)) +} + +func TestStripPath(t *testing.T) { + tests := []struct { + name string + strip int + want string + ok bool + }{ + {"go/bin/go", 1, filepath.FromSlash("bin/go"), true}, + {"./go/bin/go", 1, filepath.FromSlash("bin/go"), true}, + {"go/", 1, "", false}, + {"go", 1, "", false}, + {"bin/go", 0, filepath.FromSlash("bin/go"), true}, + {"../../x", 0, "x", true}, + {"./", 0, "", false}, + {"a/../b/c", 1, "c", true}, + } + for _, test := range tests { + got, ok := stripPath(test.name, test.strip) + assert.Equal(t, test.ok, ok, test.name) + assert.Equal(t, test.want, got, test.name) + } +} + +func TestExtractUnknownFormat(t *testing.T) { + dir := t.TempDir() + archivePath := filepath.Join(dir, "notes.txt") + require.NoError(t, os.WriteFile(archivePath, []byte("just text"), 0o644)) + err := extractArchive(t.Context(), archivePath, filepath.Join(dir, "out"), nil) + require.Error(t, err) +} diff --git a/internal/pkgdef/install.go b/internal/pkgdef/install.go new file mode 100644 index 00000000000..a1b6c50f4cd --- /dev/null +++ b/internal/pkgdef/install.go @@ -0,0 +1,361 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" +) + +// ErrUnsupportedSystem is returned when a definition does not list the +// current system in its platforms. +var ErrUnsupportedSystem = errors.New("package does not support this system") + +// ErrChecksumMismatch is returned when a downloaded artifact does not match +// the sha256 recorded in the lockfile. +var ErrChecksumMismatch = errors.New("checksum mismatch") + +// markerFile is written into an install directory once the install has +// fully completed, including post_install hooks. A directory without it is +// a failed or interrupted install and gets redone. +const markerFile = ".devbox-pkgdef.json" + +// marker is the content of markerFile. +type marker struct { + Name string `json:"name"` + Version string `json:"version"` + System string `json:"system"` + URL string `json:"url,omitempty"` + Sha256 string `json:"sha256,omitempty"` +} + +// Installed describes a completed install. +type Installed struct { + // Dir is the install root. + Dir string + + // Sha256 is the digest of the artifact that was installed. When the + // lockfile had no digest (checksum type "none"), callers should record + // this one. + Sha256 string + + // BinDirs are the absolute directories to add to PATH. + BinDirs []string +} + +// Install makes version of def available on system and returns where it +// landed. It is idempotent: a completed install is returned as is without +// any network access. Concurrent installs of the same package (from other +// processes too) are serialized with a file lock. +// +// artifact.URL may be empty, in which case the URL is rendered from the +// definition. When artifact.Sha256 is set the download must match it. +func (c *Client) Install(ctx context.Context, def *Definition, version, system string, artifact Artifact) (*Installed, error) { + if !def.SupportsSystem(system) { + return nil, fmt.Errorf("%s@%s: %w (%s; supported: %s)", + def.Name, version, ErrUnsupportedSystem, system, strings.Join(def.Platforms, ", ")) + } + dir := c.InstallDir(def.Name, version, system) + + // Fast path: no lock needed to read a finished install. + if m, ok := readMarker(dir); ok { + return c.installedFromMarker(def, version, dir, m, artifact) + } + + unlock, err := lockDir(dir) + if err != nil { + return nil, err + } + defer unlock() + + // Another process may have finished while we waited for the lock. + if m, ok := readMarker(dir); ok { + return c.installedFromMarker(def, version, dir, m, artifact) + } + return c.install(ctx, def, version, system, dir, artifact) +} + +func (c *Client) installedFromMarker(def *Definition, version, dir string, m marker, artifact Artifact) (*Installed, error) { + if artifact.Sha256 != "" && m.Sha256 != "" && !strings.EqualFold(artifact.Sha256, m.Sha256) { + return nil, fmt.Errorf("%s@%s: %w: %s was installed from an artifact with sha256 %s but the lockfile expects %s; remove the directory to reinstall", + def.Name, version, ErrChecksumMismatch, dir, m.Sha256, artifact.Sha256) + } + return c.installed(def, dir, m.Sha256), nil +} + +func (c *Client) install(ctx context.Context, def *Definition, version, system, dir string, artifact Artifact) (*Installed, error) { + install := def.InstallFor(system) + vars, err := def.TemplateVars(system, version) + if err != nil { + return nil, err + } + vars.InstallDir = dir + vars.CacheDir = c.CacheDir + + switch install.Type { + case InstallArchive, InstallBinary: + case InstallManager, InstallScript: + return nil, fmt.Errorf("%s: install.type %q: %w", def.Name, install.Type, ErrNotImplemented) + default: + return nil, fmt.Errorf("%s: install.type %q is not supported", def.Name, install.Type) + } + + if artifact.URL == "" { + artifact.URL, err = Render(install.URL, vars) + if err != nil { + return nil, err + } + } + vars = vars.withURL(artifact.URL) + + // Download next to the install dir so that a rename into place stays on + // one filesystem. + downloadDir := filepath.Join(c.CacheDir, ".downloads", def.Name, version, system) + archivePath := filepath.Join(downloadDir, vars.Filename) + sha, err := c.download(ctx, artifact.URL, archivePath) + if err != nil { + return nil, fmt.Errorf("%s@%s: %w", def.Name, version, err) + } + if artifact.Sha256 != "" && !strings.EqualFold(sha, artifact.Sha256) { + _ = os.Remove(archivePath) + return nil, fmt.Errorf("%s@%s: %w: %s has sha256 %s, lockfile expects %s", + def.Name, version, ErrChecksumMismatch, artifact.URL, sha, artifact.Sha256) + } + + // Unpack into a staging directory and rename it into place so that a + // crash never leaves a half-populated install dir behind. + if err := os.RemoveAll(dir); err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil { + return nil, err + } + staging, err := os.MkdirTemp(filepath.Dir(dir), filepath.Base(dir)+".tmp-") + if err != nil { + return nil, err + } + defer os.RemoveAll(staging) + // MkdirTemp hard-codes 0700; the install is meant to be shareable like + // its parents. + if err := os.Chmod(staging, 0o755); err != nil { + return nil, err + } + + switch install.Type { + case InstallArchive: + err = extractArchive(ctx, archivePath, staging, install.StripComponents) + case InstallBinary: + binName := install.BinName + if binName == "" { + binName = def.Name + } + err = placeBinary(archivePath, filepath.Join(staging, "bin", binName)) + } + if err != nil { + return nil, fmt.Errorf("%s@%s: unpacking %s: %w", def.Name, version, vars.Filename, err) + } + _ = os.Remove(archivePath) + + if err := os.Rename(staging, dir); err != nil { + return nil, err + } + + if err := c.runPostInstall(ctx, def, dir, vars); err != nil { + // Leave the directory in place for debugging, but without a marker + // so the next install retries. + return nil, fmt.Errorf("%s@%s: %w", def.Name, version, err) + } + + m := marker{Name: def.Name, Version: version, System: system, URL: artifact.URL, Sha256: sha} + if err := writeMarker(dir, m); err != nil { + return nil, err + } + return c.installed(def, dir, sha), nil +} + +// IsInstalled reports whether version of def is fully installed on system. +func (c *Client) IsInstalled(def *Definition, version, system string) bool { + _, ok := readMarker(c.InstallDir(def.Name, version, system)) + return ok +} + +func (c *Client) installed(def *Definition, dir, sha string) *Installed { + bins := make([]string, 0, len(def.BinDirs())) + for _, b := range def.BinDirs() { + bins = append(bins, filepath.Join(dir, b)) + } + return &Installed{Dir: dir, Sha256: sha, BinDirs: bins} +} + +// lockDir takes an exclusive advisory lock on .lock so that only one +// process installs into dir at a time. The returned function releases it. +func lockDir(dir string) (func(), error) { + if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil { + return nil, err + } + lockFile, err := os.OpenFile(dir+".lock", os.O_CREATE|os.O_RDWR, 0o644) + if err != nil { + return nil, err + } + if err := syscall.Flock(int(lockFile.Fd()), syscall.LOCK_EX); err != nil { + lockFile.Close() + return nil, fmt.Errorf("locking %s: %w", dir, err) + } + return func() { + _ = syscall.Flock(int(lockFile.Fd()), syscall.LOCK_UN) + lockFile.Close() + }, nil +} + +// download fetches rawURL to dest (unless dest already exists) and returns +// the hex sha256 of the file. +func (c *Client) download(ctx context.Context, rawURL, dest string) (string, error) { + if sha, err := fileSHA256(dest); err == nil { + return sha, nil + } + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return "", err + } + + body, err := c.open(ctx, rawURL, "") + if err != nil { + return "", err + } + defer body.Close() + + tmp, err := os.CreateTemp(filepath.Dir(dest), filepath.Base(dest)+".part-") + if err != nil { + return "", err + } + defer os.Remove(tmp.Name()) + + hasher := sha256.New() + if _, err := io.Copy(io.MultiWriter(tmp, hasher), body); err != nil { + tmp.Close() + return "", fmt.Errorf("downloading %s: %w", rawURL, err) + } + if err := tmp.Close(); err != nil { + return "", err + } + if err := os.Rename(tmp.Name(), dest); err != nil { + return "", err + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +func fileSHA256(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + hasher := sha256.New() + if _, err := io.Copy(hasher, f); err != nil { + return "", err + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +func placeBinary(src, dest string) error { + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return err + } + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + out.Close() + return err + } + return out.Close() +} + +// hookOutputLimit bounds how much hook output is kept for error messages. +const hookOutputLimit = 4096 + +func (c *Client) runPostInstall(ctx context.Context, def *Definition, dir string, vars Vars) error { + for _, hook := range def.Hooks.PostInstall { + script, err := Render(hook, vars) + if err != nil { + return err + } + // Keep the tail of the output so a failure is diagnosable even when + // Stderr is discarded. + tail := &tailBuffer{limit: hookOutputLimit} + output := io.MultiWriter(c.Stderr, tail) + + cmd := exec.CommandContext(ctx, "sh", "-c", script) + cmd.Dir = dir + cmd.Stdout = output + cmd.Stderr = output + cmd.Env = append(os.Environ(), + "DEVBOX_PKG_INSTALL_DIR="+dir, + "DEVBOX_PKG_VERSION="+vars.Version, + "DEVBOX_PKG_SYSTEM="+vars.System, + ) + if err := cmd.Run(); err != nil { + if out := strings.TrimSpace(tail.String()); out != "" { + return fmt.Errorf("post_install hook %q: %w\n%s", hook, err, out) + } + return fmt.Errorf("post_install hook %q: %w", hook, err) + } + } + return nil +} + +// tailBuffer keeps the last limit bytes written to it. +type tailBuffer struct { + buf bytes.Buffer + limit int +} + +func (t *tailBuffer) Write(p []byte) (int, error) { + t.buf.Write(p) + if t.buf.Len() > t.limit { + excess := t.buf.Len() - t.limit + t.buf.Next(excess) + } + return len(p), nil +} + +func (t *tailBuffer) String() string { + return t.buf.String() +} + +func readMarker(dir string) (marker, bool) { + b, err := os.ReadFile(filepath.Join(dir, markerFile)) + if err != nil { + return marker{}, false + } + var m marker + if err := json.Unmarshal(b, &m); err != nil { + return marker{}, false + } + return m, true +} + +func writeMarker(dir string, m marker) error { + b, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, markerFile), b, 0o644) +} diff --git a/internal/pkgdef/load.go b/internal/pkgdef/load.go new file mode 100644 index 00000000000..ed3d4b88960 --- /dev/null +++ b/internal/pkgdef/load.go @@ -0,0 +1,141 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +// ErrDefinitionChanged is returned when a remote definition no longer +// matches the hash pinned in the lockfile. +var ErrDefinitionChanged = errors.New("package definition changed") + +// Loaded is a definition together with where it came from. +type Loaded struct { + Ref Ref + Definition *Definition + + // Content is the raw file content and Hash its hex sha256, which is + // what devbox.lock records as definition_hash. It is a plain sha256 + // (not cachehash, whose algorithm may change) because the lockfile + // relies on it staying stable across Devbox versions. + Content []byte + Hash string + + // Dir is the directory containing the definition file for local + // definitions. create_files sources are resolved relative to it. It + // is empty for remote definitions. + Dir string +} + +// Load reads and parses the definition that ref points at. Relative paths +// are resolved against projectDir. +// +// pinnedHash is the definition_hash from the lockfile, or "". For remote +// definitions a pinned hash is served from the local cache without any +// network access, and a freshly fetched definition that does not match it +// is rejected with ErrDefinitionChanged. Local definitions ignore it: the +// file on disk is the source of truth and edits are expected. +func (c *Client) Load(ctx context.Context, ref Ref, projectDir, pinnedHash string) (*Loaded, error) { + if ref.IsLocal() { + return loadLocal(ref, projectDir) + } + return c.loadRemote(ctx, ref, pinnedHash) +} + +// LocalPath returns the absolute path of a local definition. It returns "" +// for remote definitions. +func (r Ref) LocalPath(projectDir string) string { + if !r.IsLocal() { + return "" + } + p := strings.TrimPrefix(r.Source, "file://") + if filepath.IsAbs(p) { + return filepath.Clean(p) + } + return filepath.Join(projectDir, p) +} + +func loadLocal(ref Ref, projectDir string) (*Loaded, error) { + path := ref.LocalPath(projectDir) + content, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("package definition %s: %w", ref.Source, os.ErrNotExist) + } + if err != nil { + return nil, err + } + def, err := Parse(content) + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return &Loaded{ + Ref: ref, + Definition: def, + Content: content, + Hash: contentHash(content), + Dir: filepath.Dir(path), + }, nil +} + +func (c *Client) loadRemote(ctx context.Context, ref Ref, pinnedHash string) (*Loaded, error) { + if pinnedHash != "" { + if content, err := os.ReadFile(c.definitionCachePath(pinnedHash)); err == nil { + if contentHash(content) == pinnedHash { + return c.parseRemote(ref, content) + } + } + } + + content, err := c.get(ctx, ref.Source) + if err != nil { + return nil, fmt.Errorf("package definition %s: %w", ref.Source, err) + } + hash := contentHash(content) + if pinnedHash != "" && hash != pinnedHash { + return nil, fmt.Errorf("package definition %s: %w: devbox.lock pins %s but the server now serves %s; run `devbox update` to accept the new definition", + ref.Source, ErrDefinitionChanged, pinnedHash, hash) + } + + loaded, err := c.parseRemote(ref, content) + if err != nil { + return nil, err + } + // Cache by content hash so future loads pinned to this hash are + // offline. Failing to cache is not fatal. + cachePath := c.definitionCachePath(hash) + if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err == nil { + _ = os.WriteFile(cachePath, content, 0o644) + } + return loaded, nil +} + +func (c *Client) parseRemote(ref Ref, content []byte) (*Loaded, error) { + def, err := Parse(content) + if err != nil { + return nil, fmt.Errorf("%s: %w", ref.Source, err) + } + return &Loaded{ + Ref: ref, + Definition: def, + Content: content, + Hash: contentHash(content), + }, nil +} + +func contentHash(content []byte) string { + sum := sha256.Sum256(content) + return hex.EncodeToString(sum[:]) +} + +func (c *Client) definitionCachePath(hash string) string { + return filepath.Join(c.CacheDir, ".definitions", hash+".json") +} diff --git a/internal/pkgdef/lock.go b/internal/pkgdef/lock.go new file mode 100644 index 00000000000..9b89149a65d --- /dev/null +++ b/internal/pkgdef/lock.go @@ -0,0 +1,83 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "context" + "fmt" + "slices" +) + +// Artifact is what gets recorded per system in devbox.lock: the exact URL +// to download and its sha256. Sha256 is empty when the definition has no +// checksum source; it is filled in by the first install (trust on first +// use). +type Artifact struct { + URL string `json:"url,omitempty"` + Sha256 string `json:"sha256,omitempty"` +} + +// Resolved is the result of resolving a version constraint: the concrete +// version and the artifact for every system the definition supports. +type Resolved struct { + Version string + Artifacts map[string]Artifact +} + +// Resolve resolves requested to a concrete version and locks the artifacts +// for every supported system. +func (c *Client) Resolve(ctx context.Context, def *Definition, requested string) (*Resolved, error) { + version, err := c.ResolveVersion(ctx, def, requested) + if err != nil { + return nil, err + } + artifacts, err := c.Lock(ctx, def, version) + if err != nil { + return nil, err + } + return &Resolved{Version: version, Artifacts: artifacts}, nil +} + +// Lock renders the artifact URL and fetches the checksum of version for +// every system in the definition's platforms. +func (c *Client) Lock(ctx context.Context, def *Definition, version string) (map[string]Artifact, error) { + artifacts := make(map[string]Artifact, len(def.Platforms)) + systems := slices.Clone(def.Platforms) + slices.Sort(systems) + for _, system := range systems { + artifact, err := c.lockSystem(ctx, def, version, system) + if err != nil { + return nil, fmt.Errorf("%s@%s (%s): %w", def.Name, version, system, err) + } + artifacts[system] = artifact + } + return artifacts, nil +} + +func (c *Client) lockSystem(ctx context.Context, def *Definition, version, system string) (Artifact, error) { + install := def.InstallFor(system) + switch install.Type { + case InstallArchive, InstallBinary: + case InstallManager, InstallScript: + return Artifact{}, fmt.Errorf("install.type %q: %w", install.Type, ErrNotImplemented) + default: + return Artifact{}, fmt.Errorf("install.type %q is not supported", install.Type) + } + + vars, err := def.TemplateVars(system, version) + if err != nil { + return Artifact{}, err + } + rawURL, err := Render(install.URL, vars) + if err != nil { + return Artifact{}, err + } + vars = vars.withURL(rawURL) + + sha, err := c.fetchChecksum(ctx, install.Checksum, vars) + if err != nil { + return Artifact{}, err + } + return Artifact{URL: rawURL, Sha256: sha}, nil +} diff --git a/internal/pkgdef/ref.go b/internal/pkgdef/ref.go new file mode 100644 index 00000000000..a4ca239a7fc --- /dev/null +++ b/internal/pkgdef/ref.go @@ -0,0 +1,81 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "fmt" + "strings" +) + +// Ref is a parsed "pkg:@" package string. +type Ref struct { + // Source locates the definition file: a path relative to the project + // directory, an absolute path, or an https:// or file:// URL. + Source string + + // Version is the requested version constraint ("latest", "1.22", + // "1.22.3"). It is empty when the package string has no "@version". + Version string +} + +// IsRef reports whether s is a package definition reference, i.e. it has +// the "pkg:" scheme. +func IsRef(s string) bool { + return strings.HasPrefix(s, Prefix) +} + +// ParseRef parses a "pkg:@" string. The version is split +// at the last "@" so that sources may themselves contain "@". +func ParseRef(raw string) (Ref, error) { + if !IsRef(raw) { + return Ref{}, fmt.Errorf("package %q: missing %q scheme", raw, Prefix) + } + rest := strings.TrimPrefix(raw, Prefix) + ref := Ref{Source: rest} + if i := strings.LastIndex(rest, "@"); i >= 0 { + ref.Source = rest[:i] + ref.Version = rest[i+1:] + } + if ref.Source == "" { + return Ref{}, fmt.Errorf("package %q: missing definition path or URL", raw) + } + if !ref.IsRemote() && !strings.HasPrefix(ref.Source, "file://") { + // Guard against schemes we don't support (github:, ssh:, ...) being + // mistaken for relative paths. A path segment before the first + // colon contains "/" or ".", a scheme does not. + if scheme, _, ok := strings.Cut(ref.Source, ":"); ok && !strings.ContainsAny(scheme, "/.") { + return Ref{}, fmt.Errorf("package %q: unsupported definition source scheme %q", raw, scheme) + } + } + return ref, nil +} + +// String formats the reference back into its devbox.json form. +func (r Ref) String() string { + s := Prefix + r.Source + if r.Version != "" { + s += "@" + r.Version + } + return s +} + +// WithVersion returns a copy of the reference with a different version. +func (r Ref) WithVersion(version string) Ref { + r.Version = version + return r +} + +// IsRemote reports whether the source is fetched over HTTP(S). +func (r Ref) IsRemote() bool { + return strings.HasPrefix(r.Source, "https://") || strings.HasPrefix(r.Source, "http://") +} + +// IsLocal reports whether the source is read from the local filesystem, +// either as a path or as a file:// URL. +func (r Ref) IsLocal() bool { + if strings.HasPrefix(r.Source, "file://") { + return true + } + return !strings.Contains(r.Source, "://") +} diff --git a/internal/pkgdef/ref_test.go b/internal/pkgdef/ref_test.go new file mode 100644 index 00000000000..54bfc5beae8 --- /dev/null +++ b/internal/pkgdef/ref_test.go @@ -0,0 +1,72 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseRef(t *testing.T) { + tests := []struct { + raw string + want Ref + local bool + wantErr string + }{ + {raw: "pkg:./go.pkg.json@1.22", want: Ref{Source: "./go.pkg.json", Version: "1.22"}, local: true}, + {raw: "pkg:./go.pkg.json", want: Ref{Source: "./go.pkg.json"}, local: true}, + {raw: "pkg:../defs/go.pkg.json@latest", want: Ref{Source: "../defs/go.pkg.json", Version: "latest"}, local: true}, + {raw: "pkg:/abs/go.pkg.json@1", want: Ref{Source: "/abs/go.pkg.json", Version: "1"}, local: true}, + {raw: "pkg:file:///tmp/go.pkg.json@1", want: Ref{Source: "file:///tmp/go.pkg.json", Version: "1"}, local: true}, + {raw: "pkg:https://example.com/go.pkg.json@1.22.3", want: Ref{Source: "https://example.com/go.pkg.json", Version: "1.22.3"}}, + { + // The version splits at the last "@" so sources may contain one. + raw: "pkg:https://user@example.com/go.pkg.json@1", + want: Ref{Source: "https://user@example.com/go.pkg.json", Version: "1"}, + }, + {raw: "go@1.22", wantErr: "missing \"pkg:\" scheme"}, + {raw: "pkg:", wantErr: "missing definition path or URL"}, + {raw: "pkg:@1.22", wantErr: "missing definition path or URL"}, + {raw: "pkg:github:owner/repo@1", wantErr: "unsupported definition source scheme \"github\""}, + } + for _, test := range tests { + t.Run(test.raw, func(t *testing.T) { + got, err := ParseRef(test.raw) + if test.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), test.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, test.want, got) + assert.Equal(t, test.raw, got.String()) + assert.Equal(t, test.local, got.IsLocal()) + assert.Equal(t, !test.local, got.IsRemote()) + }) + } +} + +func TestIsRef(t *testing.T) { + assert.True(t, IsRef("pkg:./x.pkg.json")) + assert.False(t, IsRef("runx:owner/repo")) + assert.False(t, IsRef("go@1.22")) + assert.False(t, IsRef("path:./flake")) +} + +func TestRefWithVersion(t *testing.T) { + ref := Ref{Source: "./go.pkg.json", Version: "1.22"} + assert.Equal(t, "pkg:./go.pkg.json@1.22.3", ref.WithVersion("1.22.3").String()) + assert.Equal(t, "1.22", ref.Version, "WithVersion returns a copy") +} + +func TestLocalPath(t *testing.T) { + assert.Equal(t, filepath.Join("/proj", "go.pkg.json"), Ref{Source: "./go.pkg.json"}.LocalPath("/proj")) + assert.Equal(t, "/abs/go.pkg.json", Ref{Source: "/abs/go.pkg.json"}.LocalPath("/proj")) + assert.Equal(t, "/tmp/go.pkg.json", Ref{Source: "file:///tmp/go.pkg.json"}.LocalPath("/proj")) + assert.Empty(t, Ref{Source: "https://example.com/go.pkg.json"}.LocalPath("/proj")) +} diff --git a/internal/pkgdef/template.go b/internal/pkgdef/template.go new file mode 100644 index 00000000000..06c72217cca --- /dev/null +++ b/internal/pkgdef/template.go @@ -0,0 +1,110 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "bytes" + "fmt" + "net/url" + "path" + "strings" + "text/template" +) + +// Vars are the variables available to templates in a package definition. +// See the "Template variables" section of the spec. +type Vars struct { + Version string + System string + OS string + Arch string + URL string + Filename string + + InstallDir string + CacheDir string +} + +var defaultOSMap = map[string]string{ + "darwin": "darwin", + "linux": "linux", +} + +var defaultArchMap = map[string]string{ + "x86_64": "amd64", + "aarch64": "arm64", +} + +// templateFuncs follow the sprig argument order (the string comes last) so +// that pipelines read naturally: {{ .Version | trimPrefix "v" }}. +var templateFuncs = template.FuncMap{ + "trimPrefix": func(prefix, s string) string { return strings.TrimPrefix(s, prefix) }, + "trimSuffix": func(suffix, s string) string { return strings.TrimSuffix(s, suffix) }, + "replace": func(old, replacement, s string) string { return strings.ReplaceAll(s, old, replacement) }, + "lower": strings.ToLower, + "upper": strings.ToUpper, +} + +// TemplateVars returns the template variables for installing version on +// system. URL, Filename and InstallDir are left empty; callers fill them in +// once they are known. +func (d *Definition) TemplateVars(system, version string) (Vars, error) { + arch, osName, ok := strings.Cut(system, "-") + if !ok { + return Vars{}, fmt.Errorf("invalid system %q", system) + } + vars := Vars{ + Version: version, + System: system, + OS: lookupOr(d.OSMap, osName, defaultOSMap[osName], osName), + Arch: lookupOr(d.ArchMap, arch, defaultArchMap[arch], arch), + } + return vars, nil +} + +func lookupOr(m map[string]string, key string, fallbacks ...string) string { + if v, ok := m[key]; ok { + return v + } + for _, f := range fallbacks { + if f != "" { + return f + } + } + return key +} + +// Render executes tmpl with vars. Unknown fields are an error. +func Render(tmpl string, vars Vars) (string, error) { + if !strings.Contains(tmpl, "{{") { + return tmpl, nil + } + parsed, err := template.New("pkgdef"). + Option("missingkey=error"). + Funcs(templateFuncs). + Parse(tmpl) + if err != nil { + return "", fmt.Errorf("template %q: %w", tmpl, err) + } + var buf bytes.Buffer + if err := parsed.Execute(&buf, vars); err != nil { + return "", fmt.Errorf("template %q: %w", tmpl, err) + } + return buf.String(), nil +} + +// withURL returns a copy of vars with URL and Filename set from rawURL. +func (v Vars) withURL(rawURL string) Vars { + v.URL = rawURL + v.Filename = filenameFromURL(rawURL) + return v +} + +func filenameFromURL(rawURL string) string { + parsed, err := url.Parse(rawURL) + if err != nil || parsed.Path == "" { + return path.Base(rawURL) + } + return path.Base(parsed.Path) +} diff --git a/internal/pkgdef/template_test.go b/internal/pkgdef/template_test.go new file mode 100644 index 00000000000..8b8c4078674 --- /dev/null +++ b/internal/pkgdef/template_test.go @@ -0,0 +1,72 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTemplateVarsDefaults(t *testing.T) { + def := &Definition{} + vars, err := def.TemplateVars("aarch64-darwin", "1.22.3") + require.NoError(t, err) + assert.Equal(t, Vars{Version: "1.22.3", System: "aarch64-darwin", OS: "darwin", Arch: "arm64"}, vars) + + vars, err = def.TemplateVars("x86_64-linux", "1") + require.NoError(t, err) + assert.Equal(t, "linux", vars.OS) + assert.Equal(t, "amd64", vars.Arch) + + _, err = def.TemplateVars("windows", "1") + require.Error(t, err) +} + +func TestTemplateVarsMaps(t *testing.T) { + def := &Definition{ + OSMap: map[string]string{"darwin": "macos"}, + ArchMap: map[string]string{"x86_64": "x64", "aarch64": "aarch64"}, + } + vars, err := def.TemplateVars("aarch64-darwin", "1") + require.NoError(t, err) + assert.Equal(t, "macos", vars.OS) + assert.Equal(t, "aarch64", vars.Arch) + + vars, err = def.TemplateVars("x86_64-linux", "1") + require.NoError(t, err) + assert.Equal(t, "linux", vars.OS, "unmapped OS keeps the default") + assert.Equal(t, "x64", vars.Arch) +} + +func TestRender(t *testing.T) { + vars := Vars{Version: "1.22.3", OS: "darwin", Arch: "arm64"} + + got, err := Render("https://go.dev/dl/go{{.Version}}.{{.OS}}-{{.Arch}}.tar.gz", vars) + require.NoError(t, err) + assert.Equal(t, "https://go.dev/dl/go1.22.3.darwin-arm64.tar.gz", got) + + got, err = Render(`{{ .Version | trimPrefix "1." }}-{{ upper .OS }}-{{ replace "arm" "ARM" .Arch }}-{{ trimSuffix ".3" .Version }}`, vars) + require.NoError(t, err) + assert.Equal(t, "22.3-DARWIN-ARM64-1.22", got) + + got, err = Render("no templates here", vars) + require.NoError(t, err) + assert.Equal(t, "no templates here", got) + + _, err = Render("{{.Nope}}", vars) + require.Error(t, err) + + _, err = Render("{{.Version", vars) + require.Error(t, err) +} + +func TestWithURL(t *testing.T) { + vars := Vars{}.withURL("https://go.dev/dl/go1.22.3.darwin-arm64.tar.gz?x=1") + assert.Equal(t, "https://go.dev/dl/go1.22.3.darwin-arm64.tar.gz?x=1", vars.URL) + assert.Equal(t, "go1.22.3.darwin-arm64.tar.gz", vars.Filename) + + assert.Equal(t, "hello.tgz", filenameFromURL("file:///tmp/hello.tgz")) +} diff --git a/internal/pkgdef/version.go b/internal/pkgdef/version.go new file mode 100644 index 00000000000..92fb21b5dcc --- /dev/null +++ b/internal/pkgdef/version.go @@ -0,0 +1,114 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "errors" + "fmt" + "slices" + "strconv" + "strings" + "unicode" +) + +// Latest is the version constraint that resolves to the newest stable +// version. +const Latest = "latest" + +// ErrVersionNotFound is returned when no available version satisfies the +// requested constraint. +var ErrVersionNotFound = errors.New("version not found") + +// MatchVersion picks the version from available that best satisfies +// requested, following the rules in the spec: +// +// - "latest" picks the highest non-pre-release version. +// - An exact match wins when present. +// - Otherwise requested is a prefix: "1.22" matches the highest +// "1.22.x" that is not a pre-release. +// +// A version is a pre-release when it contains a letter or a hyphen, for +// example "1.23.0-rc1" or "1.21rc2". +func MatchVersion(available []string, requested string) (string, error) { + if requested == "" { + requested = Latest + } + if slices.Contains(available, requested) { + return requested, nil + } + + best := "" + for _, v := range available { + if v == "" || IsPrerelease(v) { + continue + } + if requested != Latest && !strings.HasPrefix(v, requested+".") { + continue + } + if best == "" || CompareVersions(v, best) > 0 { + best = v + } + } + if best == "" { + return "", fmt.Errorf("%w: no version matches %q (available: %s)", + ErrVersionNotFound, requested, summarize(available)) + } + return best, nil +} + +// IsPrerelease reports whether v looks like a pre-release version. +func IsPrerelease(v string) bool { + if strings.Contains(v, "-") { + return true + } + for _, r := range v { + if unicode.IsLetter(r) { + return true + } + } + return false +} + +// CompareVersions orders two version strings. Dot-separated segments are +// compared numerically when both are numbers and lexically otherwise. A +// version that is a prefix of another sorts first ("1.22" < "1.22.1"). +func CompareVersions(a, b string) int { + as := strings.Split(a, ".") + bs := strings.Split(b, ".") + for i := range min(len(as), len(bs)) { + if c := compareSegment(as[i], bs[i]); c != 0 { + return c + } + } + return len(as) - len(bs) +} + +func compareSegment(a, b string) int { + an, aerr := strconv.Atoi(a) + bn, berr := strconv.Atoi(b) + if aerr == nil && berr == nil { + return an - bn + } + return strings.Compare(a, b) +} + +// SortVersions sorts versions from newest to oldest. +func SortVersions(versions []string) { + slices.SortStableFunc(versions, func(a, b string) int { + return CompareVersions(b, a) + }) +} + +func summarize(versions []string) string { + if len(versions) == 0 { + return "none" + } + sorted := slices.Clone(versions) + SortVersions(sorted) + const limit = 5 + if len(sorted) > limit { + return strings.Join(sorted[:limit], ", ") + fmt.Sprintf(", and %d more", len(sorted)-limit) + } + return strings.Join(sorted, ", ") +} diff --git a/internal/pkgdef/version_test.go b/internal/pkgdef/version_test.go new file mode 100644 index 00000000000..c5a56fa7545 --- /dev/null +++ b/internal/pkgdef/version_test.go @@ -0,0 +1,81 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMatchVersion(t *testing.T) { + available := []string{"1.21.0", "1.22.10", "1.22.3", "1.22.9", "1.23.0-rc1", "1.21rc2", "1.23"} + + tests := []struct { + requested string + want string + }{ + {"latest", "1.23"}, + {"", "1.23"}, + {"1.22", "1.22.10"}, + {"1.22.3", "1.22.3"}, + {"1.21", "1.21.0"}, + {"1.23", "1.23"}, + {"1.23.0-rc1", "1.23.0-rc1"}, // pre-releases only match exactly + {"1.21rc2", "1.21rc2"}, + } + for _, tt := range tests { + t.Run(tt.requested, func(t *testing.T) { + got, err := MatchVersion(available, tt.requested) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestMatchVersionNotFound(t *testing.T) { + _, err := MatchVersion([]string{"1.22.3"}, "1.24") + require.ErrorIs(t, err, ErrVersionNotFound) + assert.Contains(t, err.Error(), "1.22.3") + + _, err = MatchVersion(nil, "latest") + require.ErrorIs(t, err, ErrVersionNotFound) + + // "1.2" must not match "1.22.3" by string prefix. + _, err = MatchVersion([]string{"1.22.3"}, "1.2") + require.ErrorIs(t, err, ErrVersionNotFound) + + // Only pre-releases available: "latest" has nothing to pick. + _, err = MatchVersion([]string{"1.0.0-beta"}, "latest") + require.ErrorIs(t, err, ErrVersionNotFound) +} + +func TestCompareVersions(t *testing.T) { + assert.Negative(t, CompareVersions("1.9", "1.10")) + assert.Positive(t, CompareVersions("1.22.10", "1.22.9")) + assert.Negative(t, CompareVersions("1.22", "1.22.1")) + assert.Equal(t, 0, CompareVersions("20.11.1", "20.11.1")) + assert.Negative(t, CompareVersions("1.0a", "1.0b")) +} + +func TestSortVersions(t *testing.T) { + versions := []string{"1.9", "1.10", "2.0", "1.10.1"} + SortVersions(versions) + assert.Equal(t, []string{"2.0", "1.10.1", "1.10", "1.9"}, versions) +} + +func TestIsPrerelease(t *testing.T) { + assert.True(t, IsPrerelease("1.0.0-rc1")) + assert.True(t, IsPrerelease("1.21rc2")) + assert.True(t, IsPrerelease("2.0.0beta")) + assert.False(t, IsPrerelease("1.22.3")) + assert.False(t, IsPrerelease("20")) +} + +func TestSummarize(t *testing.T) { + assert.Equal(t, "none", summarize(nil)) + assert.Equal(t, "2.0, 1.0", summarize([]string{"1.0", "2.0"})) + assert.Equal(t, "7, 6, 5, 4, 3, and 2 more", summarize([]string{"1", "2", "3", "4", "5", "6", "7"})) +} diff --git a/internal/pkgdef/versions.go b/internal/pkgdef/versions.go new file mode 100644 index 00000000000..d24ca4f4282 --- /dev/null +++ b/internal/pkgdef/versions.go @@ -0,0 +1,104 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" +) + +// ListVersions returns every version the definition's version source knows +// about, unsorted and including pre-releases. +func (c *Client) ListVersions(ctx context.Context, def *Definition) ([]string, error) { + src := def.Versions + switch src.Type { + case VersionsStatic: + return slices.Clone(src.Versions), nil + case VersionsHTTPJSON: + return c.listVersionsHTTPJSON(ctx, src) + case VersionsGitHubReleases: + return c.listVersionsGitHub(ctx, src) + case VersionsScript: + return nil, fmt.Errorf("versions.type %q: %w", src.Type, ErrNotImplemented) + default: + return nil, fmt.Errorf("versions.type %q is not supported", src.Type) + } +} + +// ResolveVersion resolves a version constraint ("latest", "1.22", "1.22.3") +// to a concrete version using the definition's version source. +func (c *Client) ResolveVersion(ctx context.Context, def *Definition, requested string) (string, error) { + available, err := c.ListVersions(ctx, def) + if err != nil { + return "", fmt.Errorf("%s: listing versions: %w", def.Name, err) + } + version, err := MatchVersion(available, requested) + if err != nil { + return "", fmt.Errorf("%s: %w", def.Name, err) + } + return version, nil +} + +func (c *Client) listVersionsHTTPJSON(ctx context.Context, src VersionSource) ([]string, error) { + code, err := compileJQ(src.Select) + if err != nil { + return nil, err + } + body, err := c.get(ctx, src.URL) + if err != nil { + return nil, err + } + var input any + if err := json.Unmarshal(body, &input); err != nil { + return nil, fmt.Errorf("%s: response is not JSON: %w", src.URL, err) + } + return runJQ(code, src.Select, input) +} + +// githubRelease is the subset of the GitHub releases API response we need. +type githubRelease struct { + TagName string `json:"tag_name"` + Draft bool `json:"draft"` + Prerelease bool `json:"prerelease"` +} + +func (c *Client) listVersionsGitHub(ctx context.Context, src VersionSource) ([]string, error) { + owner, repo, _ := strings.Cut(src.Repo, "/") + stripPrefix := "v" + if src.StripPrefix != nil { + stripPrefix = *src.StripPrefix + } + + // The first page of 100 releases is enough for version resolution in + // practice; pagination can be added when a definition needs it. + body, err := c.getGitHubAPI(ctx, "repos/"+owner+"/"+repo+"/releases?per_page=100") + if err != nil { + return nil, fmt.Errorf("github releases for %s: %w", src.Repo, err) + } + var releases []githubRelease + if err := json.Unmarshal(body, &releases); err != nil { + return nil, fmt.Errorf("github releases for %s: response is not JSON: %w", src.Repo, err) + } + + versions := make([]string, 0, len(releases)) + for _, rel := range releases { + if rel.Draft || rel.Prerelease { + continue + } + tag := rel.TagName + if stripPrefix != "" { + if !strings.HasPrefix(tag, stripPrefix) { + continue + } + tag = strings.TrimPrefix(tag, stripPrefix) + } + if tag != "" { + versions = append(versions, tag) + } + } + return versions, nil +} diff --git a/specs/package-definitions.md b/specs/package-definitions.md index a36ddcdd488..ee30b76b6fb 100644 --- a/specs/package-definitions.md +++ b/specs/package-definitions.md @@ -231,7 +231,9 @@ variables: | `{{.InstallDir}}` | `env`, `shell`, `create_files`, `hooks` | Install root for this version and system. | | `{{.CacheDir}}` | `env`, `shell`, `hooks`, `install.env` | `$XDG_CACHE_HOME/devbox/pkgs`. | -Functions: `trimPrefix`, `trimSuffix`, `replace`, `lower`, `upper`. +Functions: `trimPrefix`, `trimSuffix`, `replace`, `lower`, `upper`. Arguments +follow the sprig order with the string last, so pipelines read naturally: +`{{ .Version | trimPrefix "v" }}`, `{{ replace "." "_" .Version }}`. ### Activation diff --git a/vendor-hash b/vendor-hash index 16b0d689af1..6f288f63528 100644 --- a/vendor-hash +++ b/vendor-hash @@ -1 +1 @@ -sha256-pA1wSCNEeGPUIXfqvrMoPtIT7NsB08PUSD4L3zq3amw= +sha256-FsBfmxOymVmiLKpMtVpHdY7bVj5TdVeqC0E9ILthyAs=