diff --git a/internal/boxcli/featureflag/pkgdefs.go b/internal/boxcli/featureflag/pkgdefs.go new file mode 100644 index 00000000000..685533e6b46 --- /dev/null +++ b/internal/boxcli/featureflag/pkgdefs.go @@ -0,0 +1,6 @@ +package featureflag + +// PkgDefs enables package definitions: installing packages from the +// vendor's binary distribution via "pkg:@" package strings. +// See specs/package-definitions.md. +var PkgDefs = disable("PKG_DEFS").EnableOnDev() diff --git a/internal/boxcli/multi/sync.go b/internal/boxcli/multi/sync.go index 10ecdf3566f..e8d29f142e5 100644 --- a/internal/boxcli/multi/sync.go +++ b/internal/boxcli/multi/sync.go @@ -49,6 +49,7 @@ func SyncLockfiles(pkgs []string) error { lockFile.Packages[key].Source = latestPkg.Source lockFile.Packages[key].Version = latestPkg.Version lockFile.Packages[key].Systems = latestPkg.Systems + lockFile.Packages[key].DefinitionHash = latestPkg.DefinitionHash changed = true } } diff --git a/internal/devbox/devbox.go b/internal/devbox/devbox.go index e5bc7472293..e5644a804c7 100644 --- a/internal/devbox/devbox.go +++ b/internal/devbox/devbox.go @@ -835,6 +835,12 @@ func (d *Devbox) computeEnv( } devboxEnvPath = envpath.JoinPathLists(devboxEnvPath, runXPaths) + pkgDefPaths, err := d.pkgDefBinPaths(ctx, false /*recordDigest*/) + if err != nil { + return nil, err + } + devboxEnvPath = envpath.JoinPathLists(append([]string{devboxEnvPath}, pkgDefPaths...)...) + pathStack := envpath.Stack(env, originalEnv) pathStack.Push(env, d.ProjectDirHash(), devboxEnvPath, envOpts.PreservePathStack) env["PATH"] = pathStack.Path(env) diff --git a/internal/devbox/packages.go b/internal/devbox/packages.go index 6c8d7319e75..1c242c3380e 100644 --- a/internal/devbox/packages.go +++ b/internal/devbox/packages.go @@ -133,7 +133,7 @@ func (d *Devbox) Add(ctx context.Context, pkgsNames []string, opts devopt.AddOpt // about not building on the current system, since user's can continue // via --exclude-platform flag. packageNameForConfig = pkg.Versioned() - } else if !versionedPkg.IsDevboxPackage { + } else if !versionedPkg.IsDevboxPackage || versionedPkg.IsPkgDef() { // This means it didn't validate and we don't want to fallback to legacy // Just propagate the error. return err @@ -482,7 +482,10 @@ func (d *Devbox) installPackages(ctx context.Context, mode installMode) error { return err } - return d.InstallRunXPackages(ctx) + if err := d.InstallRunXPackages(ctx); err != nil { + return err + } + return d.installPkgDefPackages(ctx) } func (d *Devbox) InstallRunXPackages(ctx context.Context) error { @@ -667,7 +670,7 @@ func (d *Devbox) moveAllowInsecureFromLockfile(writer io.Writer, lockfile *lock. func (d *Devbox) FixMissingStorePaths(ctx context.Context) error { packages := d.InstallablePackages() for _, pkg := range packages { - if !pkg.IsDevboxPackage || pkg.IsRunX() { + if !pkg.IsDevboxPackage || !pkg.IsNix() { continue } existingStorePaths, err := pkg.GetResolvedStorePaths() diff --git a/internal/devbox/pkgdef.go b/internal/devbox/pkgdef.go new file mode 100644 index 00000000000..30503a6f4e1 --- /dev/null +++ b/internal/devbox/pkgdef.go @@ -0,0 +1,100 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package devbox + +import ( + "context" + "strings" + + "github.com/samber/lo" + "go.jetify.com/devbox/internal/boxcli/usererr" + "go.jetify.com/devbox/internal/devpkg" + "go.jetify.com/devbox/internal/nix" + "go.jetify.com/devbox/internal/pkgdef" + "go.jetify.com/devbox/internal/ux" +) + +// installPkgDefPackages installs every package-definition package +// ("pkg:@") that is not installed yet. +func (d *Devbox) installPkgDefPackages(ctx context.Context) error { + _, err := d.pkgDefBinPaths(ctx, true /*recordDigest*/) + return err +} + +// pkgDefBinPaths returns the bin directories of every package-definition +// package, installing those that are missing. Installs are idempotent, so +// this is cheap once everything is in place and safe to call whenever the +// environment is computed. +// +// recordDigest controls whether a digest observed for a definition without +// a checksum source is written to devbox.lock. Only the install path does +// that; computing the environment must never modify the lockfile. +func (d *Devbox) pkgDefBinPaths(ctx context.Context, recordDigest bool) ([]string, error) { + paths := []string{} + for _, pkg := range lo.Filter(d.InstallablePackages(), devpkg.IsPkgDef) { + installed, err := d.installPkgDef(ctx, pkg, recordDigest) + if err != nil { + return nil, err + } + paths = append(paths, installed.BinDirs...) + } + return paths, nil +} + +func (d *Devbox) installPkgDef(ctx context.Context, pkg *devpkg.Package, recordDigest bool) (*pkgdef.Installed, error) { + locked, err := d.lockfile.Resolve(pkg.Raw) + if err != nil { + return nil, err + } + ref, err := pkgdef.ParseRef(pkg.Raw) + if err != nil { + return nil, err + } + + client := pkgdef.NewClient() + client.Stderr = d.stderr + loaded, err := client.Load(ctx, ref, d.projectDir, locked.DefinitionHash) + if err != nil { + return nil, err + } + def := loaded.Definition + + system := nix.System() + if !def.SupportsSystem(system) { + return nil, usererr.New( + "Package %s does not support %s (it supports %s). "+ + "Run `devbox add %s --exclude-platform %s` to skip it on this machine.", + pkg.Raw, system, strings.Join(def.Platforms, ", "), pkg.Raw, system, + ) + } + + artifact := pkgdef.Artifact{} + if info := locked.Systems[system]; info != nil { + artifact = pkgdef.Artifact{URL: info.URL, Sha256: info.Sha256} + } + + if !client.IsInstalled(def, locked.Version, system) { + ux.Finfof(d.stderr, "Installing %s@%s from %s\n", def.Name, locked.Version, lo.Ternary(artifact.URL != "", artifact.URL, ref.Source)) + if def.UsesScripts() { + ux.Fwarningf(d.stderr, "Package definition %s runs shell commands while installing.\n", ref.Source) + } + if artifact.Sha256 == "" { + ux.Fwarningf(d.stderr, "Package definition %s has no checksum for %s; the digest of this download will be recorded in devbox.lock.\n", ref.Source, system) + } + } + + installed, err := client.Install(ctx, def, locked.Version, system, artifact) + if err != nil { + return nil, err + } + + // Trust on first use: record the digest we observed so every other + // machine verifies against it. + if recordDigest && artifact.Sha256 == "" && installed.Sha256 != "" { + if err := d.lockfile.SetArtifactForPackage(pkg.Raw, system, installed.URL, installed.Sha256); err != nil { + return nil, err + } + } + return installed, nil +} diff --git a/internal/devbox/update.go b/internal/devbox/update.go index 5b991ae25a7..615a7b73bde 100644 --- a/internal/devbox/update.go +++ b/internal/devbox/update.go @@ -231,6 +231,15 @@ func (d *Devbox) mergeResolvedPackageToLockfile( return nil } + // A package definition can change without changing the version or the + // artifacts (for example a new env var or init hook). The lockfile pins + // the definition by hash, so the new hash must be recorded. + if existing.DefinitionHash != resolved.DefinitionHash { + useResolvedPackageInLockfile(lockfile, pkg, resolved, existing) + ux.Finfof(d.stderr, "Updated package definition for %s\n", pkg) + return nil + } + ux.Finfof(d.stderr, "Already up-to-date %s %s\n", pkg, existing.Version) return nil } diff --git a/internal/devbox/update_test.go b/internal/devbox/update_test.go index 11fa986c106..647944a3b48 100644 --- a/internal/devbox/update_test.go +++ b/internal/devbox/update_test.go @@ -384,3 +384,35 @@ func TestDescribeFlakeUpdateFormats(t *testing.T) { newNoDate := &lock.Package{Resolved: "github:numtide/flake-utils/" + newRev + "#pkg"} require.Equal(t, "abc1234 -> f456789", describeFlakeUpdate(oldNoDate, newNoDate)) } + +func TestUpdateRecordsChangedDefinitionHash(t *testing.T) { + devbox := devboxForTesting(t) + + raw := "pkg:./hello.pkg.json@1" + devPkg := devpkg.PackageFromStringWithDefaults(raw, nil) + existing := &lock.Package{ + Resolved: "pkg:./hello.pkg.json@1.2.0", + Version: "1.2.0", + Source: "pkg-def", + DefinitionHash: "old", + Systems: map[string]*lock.SystemInfo{ + "x86_64-linux": {URL: "file://vendor/hello-1.2.0.sh", Sha256: "abc"}, + }, + } + resolved := &lock.Package{ + Resolved: "pkg:./hello.pkg.json@1.2.0", + Version: "1.2.0", + Source: "pkg-def", + DefinitionHash: "new", + Systems: map[string]*lock.SystemInfo{ + "x86_64-linux": {URL: "file://vendor/hello-1.2.0.sh", Sha256: "abc"}, + }, + } + lockfile := &lock.File{ + Packages: map[string]*lock.Package{raw: existing}, + } + + err := devbox.mergeResolvedPackageToLockfile(devPkg, resolved, lockfile) + require.NoError(t, err, "update failed") + require.Equal(t, "new", lockfile.Packages[raw].DefinitionHash) +} diff --git a/internal/devconfig/config.go b/internal/devconfig/config.go index c60b1b8cd00..9d2df2edb2a 100644 --- a/internal/devconfig/config.go +++ b/internal/devconfig/config.go @@ -289,6 +289,23 @@ func (c *Config) loadRecursive( included = append(included, includable) } + // Package definitions ("pkg:" packages) carry their own env, init hook + // and create_files. They cannot include anything, so there is nothing + // to recurse into. + pkgDefs, err := plugin.GetPkgDefConfigsForPackages( + c.Root.TopLevelPackages(), + lockfile, + ) + if err != nil { + return errors.WithStack(err) + } + for _, pkgDef := range pkgDefs { + included = append(included, &Config{ + Root: pkgDef.ConfigFile, + pluginData: &pkgDef.PluginOnlyData, + }) + } + c.included = included return nil } diff --git a/internal/devpkg/package.go b/internal/devpkg/package.go index 5844b1ab86f..458203ea16f 100644 --- a/internal/devpkg/package.go +++ b/internal/devpkg/package.go @@ -25,6 +25,7 @@ import ( "go.jetify.com/devbox/internal/devpkg/pkgtype" "go.jetify.com/devbox/internal/lock" "go.jetify.com/devbox/internal/nix" + "go.jetify.com/devbox/internal/pkgdef" "go.jetify.com/devbox/internal/ux" "go.jetify.com/devbox/nix/flake" "go.jetify.com/devbox/plugins" @@ -171,6 +172,11 @@ func resolve(pkg *Package) error { if err != nil { return err } + // Package definitions resolve to "pkg:@", which is not a + // flake installable. Locking them is all that's needed here. + if pkg.IsPkgDef() { + return nil + } parsed, err := flake.ParseInstallable(resolved.Resolved) if err != nil { return err @@ -464,6 +470,14 @@ func (p *Package) Hash() string { // user always gets newest flake. sum, _ = cachehash.File(filepath.Join(p.installable.Ref.Path, "flake.nix")) } + if ref, ok := p.pkgDefRef(); ok { + // For local package definitions, hash the file so that editing it + // changes the project state and re-runs the install step (which + // redoes the install if the recipe changed). + if path := ref.LocalPath(p.lockfile.ProjectDir()); path != "" { + sum, _ = cachehash.File(path) + } + } if sum == "" { sum = cachehash.Bytes([]byte(cmp.Or(p.installable.String(), p.Raw))) @@ -501,10 +515,23 @@ func (p *Package) CanonicalName() string { if !p.IsDevboxPackage { return "" } + if ref, ok := p.pkgDefRef(); ok { + return pkgdef.Ref{Source: ref.Source}.String() + } name, _, _ := strings.Cut(p.Raw, "@") return name } +// pkgDefRef parses the package string as a package definition reference. +// Definition sources may contain "@", so these split at the last one. +func (p *Package) pkgDefRef() (pkgdef.Ref, bool) { + if !p.IsPkgDef() { + return pkgdef.Ref{}, false + } + ref, err := pkgdef.ParseRef(p.Raw) + return ref, err == nil +} + func (p *Package) Versioned() string { if p.IsDevboxPackage && !p.isVersioned() { return p.Raw + "@latest" @@ -537,6 +564,9 @@ func EnsureNixpkgsPrefetched(ctx context.Context, w io.Writer, pkgs []*Package) // ensureNixpkgsPrefetched should be called via the public EnsureNixpkgsPrefetched. // See function comment there. func (p *Package) ensureNixpkgsPrefetched(w io.Writer) error { + if !p.IsNix() { + return nil + } inCache, err := p.IsInBinaryCache() if err != nil { return err @@ -560,11 +590,17 @@ func (p *Package) version() string { if !p.IsDevboxPackage { return "" } + if ref, ok := p.pkgDefRef(); ok { + return ref.Version + } _, version, _ := strings.Cut(p.Raw, "@") return version } func (p *Package) isVersioned() bool { + if ref, ok := p.pkgDefRef(); ok { + return ref.Version != "" + } return p.IsDevboxPackage && strings.Contains(p.Raw, "@") } @@ -659,6 +695,18 @@ func (p *Package) IsRunX() bool { return pkgtype.IsRunX(p.Raw) } +// IsPkgDef reports whether the package is installed from a package +// definition ("pkg:@", see specs/package-definitions.md). +func (p *Package) IsPkgDef() bool { + return pkgtype.IsPkgDef(p.Raw) +} + +// Locked returns the package's lockfile entry, or nil if it has not been +// resolved yet. +func (p *Package) Locked() *lock.Package { + return p.lockfile.Get(p.LockfileKey()) +} + func (p *Package) IsNix() bool { return IsNix(p, 0) } @@ -682,13 +730,17 @@ func (p *Package) LockfileKey() string { } func IsNix(p *Package, _ int) bool { - return !p.IsRunX() + return !p.IsRunX() && !p.IsPkgDef() } func IsRunX(p *Package, _ int) bool { return p.IsRunX() } +func IsPkgDef(p *Package, _ int) bool { + return p.IsPkgDef() +} + func (p *Package) DocsURL() string { if p.IsRunX() { path, _, _ := strings.Cut(p.RunXPath(), "@") @@ -703,7 +755,7 @@ func (p *Package) DocsURL() string { // GetOutputNames returns the names of the nix package outputs. Outputs can be // specified in devbox.json package fields or as part of the flake reference. func (p *Package) GetOutputNames() ([]string, error) { - if p.IsRunX() { + if !p.IsNix() { return []string{}, nil } diff --git a/internal/devpkg/package_test.go b/internal/devpkg/package_test.go index b35912a0f12..15d69cfc878 100644 --- a/internal/devpkg/package_test.go +++ b/internal/devpkg/package_test.go @@ -197,6 +197,9 @@ func TestCanonicalName(t *testing.T) { {"runx:golangci/golangci-lint@latest", "runx:golangci/golangci-lint"}, {"runx:golangci/golangci-lint@v0.0.2", "runx:golangci/golangci-lint"}, {"runx:golangci/golangci-lint", "runx:golangci/golangci-lint"}, + {"pkg:./go.pkg.json@1.22", "pkg:./go.pkg.json"}, + {"pkg:./go.pkg.json", "pkg:./go.pkg.json"}, + {"pkg:https://cdn.example.com/@acme/tool.pkg.json@latest", "pkg:https://cdn.example.com/@acme/tool.pkg.json"}, {"github:NixOS/nixpkgs/12345", ""}, {"path:/to/my/file", ""}, } diff --git a/internal/devpkg/pkgtype/flake.go b/internal/devpkg/pkgtype/flake.go index b6ec9562a7d..353fb91c482 100644 --- a/internal/devpkg/pkgtype/flake.go +++ b/internal/devpkg/pkgtype/flake.go @@ -7,7 +7,7 @@ import ( ) func IsFlake(s string) bool { - if IsRunX(s) { + if IsRunX(s) || IsPkgDef(s) { return false } parsed, err := flake.ParseInstallable(s) diff --git a/internal/devpkg/pkgtype/pkgdef.go b/internal/devpkg/pkgtype/pkgdef.go new file mode 100644 index 00000000000..36933284599 --- /dev/null +++ b/internal/devpkg/pkgtype/pkgdef.go @@ -0,0 +1,14 @@ +package pkgtype + +import ( + "go.jetify.com/devbox/internal/boxcli/featureflag" + "go.jetify.com/devbox/internal/pkgdef" +) + +// IsPkgDef reports whether s is a package definition reference +// ("pkg:@", see specs/package-definitions.md). It is always +// false while the PKG_DEFS feature flag is off, in which case such strings +// fall through to the regular (and failing) nixpkgs lookup. +func IsPkgDef(s string) bool { + return featureflag.PkgDefs.Enabled() && pkgdef.IsRef(s) +} diff --git a/internal/devpkg/validation.go b/internal/devpkg/validation.go index 3ef00902e27..f09d48e9deb 100644 --- a/internal/devpkg/validation.go +++ b/internal/devpkg/validation.go @@ -9,7 +9,7 @@ import ( ) func (p *Package) ValidateExists(ctx context.Context) (bool, error) { - if p.IsRunX() { + if p.IsRunX() || p.IsPkgDef() { _, err := p.lockfile.Resolve(p.Raw) return err == nil, err } diff --git a/internal/lock/lockfile.go b/internal/lock/lockfile.go index c1f7cd34f01..2f96612f58a 100644 --- a/internal/lock/lockfile.go +++ b/internal/lock/lockfile.go @@ -81,7 +81,7 @@ func (f *File) Resolve(pkg string) (*Package, error) { locked := &Package{} _, _, versioned := searcher.ParseVersionedPackage(pkg) - if pkgtype.IsRunX(pkg) || versioned || pkgtype.IsFlake(pkg) { + if pkgtype.IsRunX(pkg) || pkgtype.IsPkgDef(pkg) || versioned || pkgtype.IsFlake(pkg) { resolved, err := f.FetchResolvedPackage(pkg, false) if err != nil { return nil, err diff --git a/internal/lock/package.go b/internal/lock/package.go index 7131aa5d417..e6c0acc2642 100644 --- a/internal/lock/package.go +++ b/internal/lock/package.go @@ -23,12 +23,22 @@ type Package struct { // Systems is keyed by the system name Systems map[string]*SystemInfo `json:"systems,omitempty"` + // DefinitionHash is the sha256 of the package definition file for + // packages with Source "pkg-def". It pins remote definitions. + DefinitionHash string `json:"definition_hash,omitempty"` + // NOTE: if you add more fields, please update SyncLockfiles } type SystemInfo struct { Outputs []Output `json:"outputs,omitempty"` + // URL and Sha256 locate and verify the vendor artifact for packages + // with Source "pkg-def". Sha256 may be empty until the first install + // records it. + URL string `json:"url,omitempty"` + Sha256 string `json:"sha256,omitempty"` + // Legacy Format StorePath string `json:"store_path,omitempty"` outputIsFromStorePath bool @@ -113,7 +123,9 @@ func (i *SystemInfo) Equals(other *SystemInfo) bool { return i == other } - return slices.Equal(i.Outputs, other.Outputs) + return slices.Equal(i.Outputs, other.Outputs) && + i.URL == other.URL && + i.Sha256 == other.Sha256 } // If we have a StorePath and no Outputs, we need to convert to the new format. diff --git a/internal/lock/pkgdef.go b/internal/lock/pkgdef.go new file mode 100644 index 00000000000..1dba97ef424 --- /dev/null +++ b/internal/lock/pkgdef.go @@ -0,0 +1,87 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package lock + +import ( + "context" + "errors" + + "go.jetify.com/devbox/internal/boxcli/usererr" + "go.jetify.com/devbox/internal/pkgdef" +) + +// resolvePkgDef resolves a "pkg:@" package string to a +// lockfile entry: the concrete version plus the artifact URL and checksum +// for every system the definition supports. +// +// When refresh is false and the package already has an entry, a remote +// definition is pinned to the entry's definition_hash so that `devbox add` +// and `devbox install` never silently pick up a changed definition. Only +// `devbox update` (refresh=true) re-fetches it. +func (f *File) resolvePkgDef(ctx context.Context, raw string, refresh bool) (*Package, error) { + ref, err := pkgdef.ParseRef(raw) + if err != nil { + return nil, usererr.WithUserMessage(err, "Invalid package %q", raw) + } + + existing := f.Packages[raw] + pinned := "" + if !refresh && existing != nil { + pinned = existing.DefinitionHash + } + + client := pkgdef.NewClient() + loaded, err := client.Load(ctx, ref, f.ProjectDir(), pinned) + if err != nil { + return nil, err + } + resolved, err := client.Resolve(ctx, loaded.Definition, ref.Version) + if errors.Is(err, pkgdef.ErrVersionNotFound) { + return nil, usererr.WithUserMessage(err, "Package %s not found", raw) + } + if err != nil { + return nil, err + } + + pkg := &Package{ + Resolved: ref.WithVersion(resolved.Version).String(), + Version: resolved.Version, + Source: pkgdef.Source, + DefinitionHash: loaded.Hash, + Systems: make(map[string]*SystemInfo, len(resolved.Artifacts)), + } + for system, artifact := range resolved.Artifacts { + info := &SystemInfo{URL: artifact.URL, Sha256: artifact.Sha256} + // A definition without a checksum source gets its digest recorded + // by the first install (trust on first use). Keep that digest when + // re-resolving to the same artifact so `devbox update` doesn't + // throw it away. + if info.Sha256 == "" && existing != nil && existing.Version == resolved.Version { + if old := existing.Systems[system]; old != nil && old.URL == artifact.URL { + info.Sha256 = old.Sha256 + } + } + pkg.Systems[system] = info + } + return pkg, nil +} + +// SetArtifactForPackage records the URL and sha256 of the artifact that was +// installed for system. It is used to fill in digests for definitions that +// have no checksum source. +func (f *File) SetArtifactForPackage(pkg, system, url, sha256 string) error { + locked, err := f.Resolve(pkg) + if err != nil { + return err + } + if locked.Systems == nil { + locked.Systems = map[string]*SystemInfo{} + } + if locked.Systems[system] == nil { + locked.Systems[system] = &SystemInfo{} + } + locked.Systems[system].URL = url + locked.Systems[system].Sha256 = sha256 + return f.Save() +} diff --git a/internal/lock/resolve.go b/internal/lock/resolve.go index 2183d7c6024..5e46dc70d07 100644 --- a/internal/lock/resolve.go +++ b/internal/lock/resolve.go @@ -33,6 +33,9 @@ import ( // `--refresh`. This should only be set on the `devbox update` path; other // callers (Add, install, outdated checks) prefer the cache. func (f *File) FetchResolvedPackage(pkg string, refresh bool) (*Package, error) { + if pkgtype.IsPkgDef(pkg) { + return f.resolvePkgDef(context.TODO(), pkg, refresh) + } if pkgtype.IsFlake(pkg) { installable, err := flake.ParseInstallable(pkg) if err != nil { diff --git a/internal/pkgdef/checksum.go b/internal/pkgdef/checksum.go index 9542d68f636..0a592817dcb 100644 --- a/internal/pkgdef/checksum.go +++ b/internal/pkgdef/checksum.go @@ -13,7 +13,7 @@ import ( // fetchChecksum returns the lowercase hex sha256 for the artifact described // by vars, or "" when the checksum type is "none". -func (c *Client) fetchChecksum(ctx context.Context, src *Checksum, vars Vars) (string, error) { +func (c *Client) fetchChecksum(ctx context.Context, def *Definition, src *Checksum, vars Vars) (string, error) { if src == nil || src.Type == ChecksumNone { return "", nil } @@ -21,6 +21,7 @@ func (c *Client) fetchChecksum(ctx context.Context, src *Checksum, vars Vars) (s if err != nil { return "", err } + rawURL = def.ResolveURL(rawURL) switch src.Type { case ChecksumHTTPJSON: return c.fetchChecksumHTTPJSON(ctx, src, rawURL, vars) diff --git a/internal/pkgdef/client_test.go b/internal/pkgdef/client_test.go index fb6b7dae0d5..12c4cedc3d8 100644 --- a/internal/pkgdef/client_test.go +++ b/internal/pkgdef/client_test.go @@ -556,3 +556,75 @@ func TestInstallHookFailureIncludesOutput(t *testing.T) { require.NoError(t, err) assert.Equal(t, os.FileMode(0o755), info.Mode().Perm(), "install dir is world readable") } + +func TestRelativeFileURLsResolveAgainstDefinitionDir(t *testing.T) { + projectDir := t.TempDir() + defDir := filepath.Join(projectDir, "defs") + require.NoError(t, os.MkdirAll(filepath.Join(defDir, "vendor"), 0o755)) + binary := []byte("#!/bin/sh\necho rel\n") + require.NoError(t, os.WriteFile(filepath.Join(defDir, "vendor", "tool-1.0.0"), binary, 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(defDir, "vendor", "SHA256SUMS"), []byte(sha256Hex(binary)+" tool-1.0.0\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(defDir, "vendor", "versions.json"), []byte(`{"versions": ["1.0.0"]}`), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(defDir, "tool.pkg.json"), []byte(`{ + "schema": 1, "name": "tool", "platforms": ["x86_64-linux"], + "versions": {"type": "http_json", "url": "file://vendor/versions.json", "select": ".versions[]"}, + "install": {"type": "binary", "url": "file://vendor/tool-{{.Version}}", + "checksum": {"type": "sha256_url", "url": "file://vendor/SHA256SUMS"}} + }`), 0o644)) + + cli := NewClient() + cli.CacheDir = t.TempDir() + loaded, err := cli.Load(t.Context(), Ref{Source: "./defs/tool.pkg.json", Version: "latest"}, projectDir, "") + require.NoError(t, err) + + resolved, err := cli.Resolve(t.Context(), loaded.Definition, "latest") + require.NoError(t, err) + assert.Equal(t, "1.0.0", resolved.Version) + // The lock keeps the relative URL so devbox.lock stays portable. + want := "file://vendor/tool-1.0.0" + assert.Equal(t, Artifact{URL: want, Sha256: sha256Hex(binary)}, resolved.Artifacts[testSystem]) + + installed, err := cli.Install(t.Context(), loaded.Definition, "1.0.0", testSystem, resolved.Artifacts[testSystem]) + require.NoError(t, err) + assert.Equal(t, want, installed.URL) + content, err := os.ReadFile(filepath.Join(installed.Dir, "bin", "tool")) + require.NoError(t, err) + assert.Equal(t, binary, content) + + // Absolute file URLs and remote definitions are left alone. + assert.Equal(t, "file:///abs/x", loaded.Definition.ResolveURL("file:///abs/x")) + assert.Equal(t, "https://example.com/x", loaded.Definition.ResolveURL("https://example.com/x")) + assert.Equal(t, "file://rel/x", (&Definition{}).ResolveURL("file://rel/x")) +} + +func TestInstallRedoneWhenRecipeChanges(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("sh is required") + } + vendor := newFakeVendor(t) + cli := vendor.client(t) + def := vendor.archiveDef(t) + art := Artifact{Sha256: vendor.tarSHA} + + first, err := cli.Install(t.Context(), def, "1.2.0", testSystem, art) + require.NoError(t, err) + assert.True(t, cli.IsInstalled(def, "1.2.0", testSystem)) + + // Activation-only edits do not invalidate the install. + def.Env = map[string]string{"X": "y"} + assert.True(t, cli.IsInstalled(def, "1.2.0", testSystem)) + assert.Equal(t, def.InstallFingerprint(testSystem), def.InstallFingerprint(testSystem)) + + // Changing the install recipe does: the hook now writes a different file. + def.Hooks.PostInstall = []string{"echo redone > $DEVBOX_PKG_INSTALL_DIR/redone.txt"} + assert.False(t, cli.IsInstalled(def, "1.2.0", testSystem)) + second, err := cli.Install(t.Context(), def, "1.2.0", testSystem, art) + require.NoError(t, err) + assert.Equal(t, first.Dir, second.Dir) + _, err = os.Stat(filepath.Join(second.Dir, "redone.txt")) + require.NoError(t, err) + _, err = os.Stat(filepath.Join(second.Dir, "hook.txt")) + assert.True(t, os.IsNotExist(err), "the old install was replaced, not layered on") + assert.True(t, cli.IsInstalled(def, "1.2.0", testSystem)) + assert.Equal(t, 2, vendor.hitCount("/dl/hello-1.2.0-linux-amd64.tar.gz")) +} diff --git a/internal/pkgdef/definition.go b/internal/pkgdef/definition.go index 30d4cb28a8c..92c68ccf12b 100644 --- a/internal/pkgdef/definition.go +++ b/internal/pkgdef/definition.go @@ -11,8 +11,11 @@ package pkgdef import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" + "path/filepath" "regexp" "slices" "strings" @@ -84,6 +87,21 @@ type Definition struct { CreateFiles map[string]string `json:"create_files,omitempty"` Hooks Hooks `json:"hooks,omitempty"` + + // baseDir is the directory of the definition file for local + // definitions. Relative file:// URLs are resolved against it. + baseDir string +} + +// ResolveURL makes a relative file:// URL ("file://artifacts/x.tar.gz") +// absolute by resolving it against the directory of a local definition. +// Every other URL is returned unchanged. +func (d *Definition) ResolveURL(rawURL string) string { + p, ok := strings.CutPrefix(rawURL, "file://") + if !ok || filepath.IsAbs(p) || d.baseDir == "" { + return rawURL + } + return "file://" + filepath.Join(d.baseDir, p) } // VersionSource describes how to list the available versions of a package. @@ -390,6 +408,27 @@ func (in *Install) ChecksumType() string { return in.Checksum.Type } +// InstallFingerprint identifies everything about the definition that +// affects what an install of a version on system contains: the install +// strategy, the bin layout, hooks and platform maps. Activation fields +// (env, shell, create_files) are excluded because they are applied at +// shell time. A changed fingerprint makes Install redo an existing install. +func (d *Definition) InstallFingerprint(system string) string { + install := d.InstallFor(system) + payload, err := json.Marshal(struct { + Install Install `json:"install"` + Bin []string `json:"bin"` + Hooks Hooks `json:"hooks"` + OSMap map[string]string `json:"os_map"` + ArchMap map[string]string `json:"arch_map"` + }{install, d.Bin, d.Hooks, d.OSMap, d.ArchMap}) + if err != nil { + return "" + } + sum := sha256.Sum256(payload) + return hex.EncodeToString(sum[:8]) +} + // UsesScripts reports whether the definition runs shell that the user // should be told about: script-type sources or strategies, or post-install // hooks. diff --git a/internal/pkgdef/install.go b/internal/pkgdef/install.go index a1b6c50f4cd..d648335927d 100644 --- a/internal/pkgdef/install.go +++ b/internal/pkgdef/install.go @@ -39,6 +39,10 @@ type marker struct { System string `json:"system"` URL string `json:"url,omitempty"` Sha256 string `json:"sha256,omitempty"` + + // InstallHash is Definition.InstallFingerprint at install time. When + // the definition's install recipe changes, the install is redone. + InstallHash string `json:"install_hash,omitempty"` } // Installed describes a completed install. @@ -46,6 +50,9 @@ type Installed struct { // Dir is the install root. Dir string + // URL is the artifact the install came from. + URL string + // Sha256 is the digest of the artifact that was installed. When the // lockfile had no digest (checksum type "none"), callers should record // this one. @@ -56,12 +63,15 @@ type Installed struct { } // Install makes version of def available on system and returns where it -// landed. It is idempotent: a completed install is returned as is without -// any network access. Concurrent installs of the same package (from other -// processes too) are serialized with a file lock. +// landed. It is idempotent: a completed install from the same recipe is +// returned as is without any network access; an install made from a +// definition whose install recipe has since changed is redone. Concurrent +// installs of the same package (from other processes too) are serialized +// with a file lock. // // artifact.URL may be empty, in which case the URL is rendered from the -// definition. When artifact.Sha256 is set the download must match it. +// definition. Relative file:// URLs are resolved against the definition's +// directory. When artifact.Sha256 is set the download must match it. func (c *Client) Install(ctx context.Context, def *Definition, version, system string, artifact Artifact) (*Installed, error) { if !def.SupportsSystem(system) { return nil, fmt.Errorf("%s@%s: %w (%s; supported: %s)", @@ -69,8 +79,10 @@ func (c *Client) Install(ctx context.Context, def *Definition, version, system s } dir := c.InstallDir(def.Name, version, system) + fingerprint := def.InstallFingerprint(system) + // Fast path: no lock needed to read a finished install. - if m, ok := readMarker(dir); ok { + if m, ok := readMarker(dir); ok && m.InstallHash == fingerprint { return c.installedFromMarker(def, version, dir, m, artifact) } @@ -81,10 +93,10 @@ func (c *Client) Install(ctx context.Context, def *Definition, version, system s defer unlock() // Another process may have finished while we waited for the lock. - if m, ok := readMarker(dir); ok { + if m, ok := readMarker(dir); ok && m.InstallHash == fingerprint { return c.installedFromMarker(def, version, dir, m, artifact) } - return c.install(ctx, def, version, system, dir, artifact) + return c.install(ctx, def, version, system, dir, fingerprint, artifact) } func (c *Client) installedFromMarker(def *Definition, version, dir string, m marker, artifact Artifact) (*Installed, error) { @@ -92,10 +104,10 @@ func (c *Client) installedFromMarker(def *Definition, version, dir string, m mar return nil, fmt.Errorf("%s@%s: %w: %s was installed from an artifact with sha256 %s but the lockfile expects %s; remove the directory to reinstall", def.Name, version, ErrChecksumMismatch, dir, m.Sha256, artifact.Sha256) } - return c.installed(def, dir, m.Sha256), nil + return c.installed(def, dir, m.URL, m.Sha256), nil } -func (c *Client) install(ctx context.Context, def *Definition, version, system, dir string, artifact Artifact) (*Installed, error) { +func (c *Client) install(ctx context.Context, def *Definition, version, system, dir, fingerprint string, artifact Artifact) (*Installed, error) { install := def.InstallFor(system) vars, err := def.TemplateVars(system, version) if err != nil { @@ -119,19 +131,20 @@ func (c *Client) install(ctx context.Context, def *Definition, version, system, } } vars = vars.withURL(artifact.URL) + fetchURL := def.ResolveURL(artifact.URL) // Download next to the install dir so that a rename into place stays on // one filesystem. downloadDir := filepath.Join(c.CacheDir, ".downloads", def.Name, version, system) archivePath := filepath.Join(downloadDir, vars.Filename) - sha, err := c.download(ctx, artifact.URL, archivePath) + sha, err := c.download(ctx, fetchURL, archivePath) if err != nil { return nil, fmt.Errorf("%s@%s: %w", def.Name, version, err) } if artifact.Sha256 != "" && !strings.EqualFold(sha, artifact.Sha256) { _ = os.Remove(archivePath) return nil, fmt.Errorf("%s@%s: %w: %s has sha256 %s, lockfile expects %s", - def.Name, version, ErrChecksumMismatch, artifact.URL, sha, artifact.Sha256) + def.Name, version, ErrChecksumMismatch, fetchURL, sha, artifact.Sha256) } // Unpack into a staging directory and rename it into place so that a @@ -178,25 +191,26 @@ func (c *Client) install(ctx context.Context, def *Definition, version, system, return nil, fmt.Errorf("%s@%s: %w", def.Name, version, err) } - m := marker{Name: def.Name, Version: version, System: system, URL: artifact.URL, Sha256: sha} + m := marker{Name: def.Name, Version: version, System: system, URL: artifact.URL, Sha256: sha, InstallHash: fingerprint} if err := writeMarker(dir, m); err != nil { return nil, err } - return c.installed(def, dir, sha), nil + return c.installed(def, dir, artifact.URL, sha), nil } -// IsInstalled reports whether version of def is fully installed on system. +// IsInstalled reports whether version of def is fully installed on system +// from the definition's current install recipe. func (c *Client) IsInstalled(def *Definition, version, system string) bool { - _, ok := readMarker(c.InstallDir(def.Name, version, system)) - return ok + m, ok := readMarker(c.InstallDir(def.Name, version, system)) + return ok && m.InstallHash == def.InstallFingerprint(system) } -func (c *Client) installed(def *Definition, dir, sha string) *Installed { +func (c *Client) installed(def *Definition, dir, url, sha string) *Installed { bins := make([]string, 0, len(def.BinDirs())) for _, b := range def.BinDirs() { bins = append(bins, filepath.Join(dir, b)) } - return &Installed{Dir: dir, Sha256: sha, BinDirs: bins} + return &Installed{Dir: dir, URL: url, Sha256: sha, BinDirs: bins} } // lockDir takes an exclusive advisory lock on .lock so that only one diff --git a/internal/pkgdef/load.go b/internal/pkgdef/load.go index ed3d4b88960..c475419deb2 100644 --- a/internal/pkgdef/load.go +++ b/internal/pkgdef/load.go @@ -77,6 +77,7 @@ func loadLocal(ref Ref, projectDir string) (*Loaded, error) { if err != nil { return nil, fmt.Errorf("%s: %w", path, err) } + def.baseDir = filepath.Dir(path) return &Loaded{ Ref: ref, Definition: def, diff --git a/internal/pkgdef/lock.go b/internal/pkgdef/lock.go index 9b89149a65d..aacc09de926 100644 --- a/internal/pkgdef/lock.go +++ b/internal/pkgdef/lock.go @@ -69,13 +69,16 @@ func (c *Client) lockSystem(ctx context.Context, def *Definition, version, syste if err != nil { return Artifact{}, err } + // The URL is recorded as rendered (a relative file:// URL stays + // relative) so that devbox.lock is portable; it is resolved against the + // definition's directory when fetching. rawURL, err := Render(install.URL, vars) if err != nil { return Artifact{}, err } vars = vars.withURL(rawURL) - sha, err := c.fetchChecksum(ctx, install.Checksum, vars) + sha, err := c.fetchChecksum(ctx, def, install.Checksum, vars) if err != nil { return Artifact{}, err } diff --git a/internal/pkgdef/versions.go b/internal/pkgdef/versions.go index d24ca4f4282..f320491dc60 100644 --- a/internal/pkgdef/versions.go +++ b/internal/pkgdef/versions.go @@ -19,7 +19,7 @@ func (c *Client) ListVersions(ctx context.Context, def *Definition) ([]string, e case VersionsStatic: return slices.Clone(src.Versions), nil case VersionsHTTPJSON: - return c.listVersionsHTTPJSON(ctx, src) + return c.listVersionsHTTPJSON(ctx, def.ResolveURL(src.URL), src.Select) case VersionsGitHubReleases: return c.listVersionsGitHub(ctx, src) case VersionsScript: @@ -43,20 +43,20 @@ func (c *Client) ResolveVersion(ctx context.Context, def *Definition, requested return version, nil } -func (c *Client) listVersionsHTTPJSON(ctx context.Context, src VersionSource) ([]string, error) { - code, err := compileJQ(src.Select) +func (c *Client) listVersionsHTTPJSON(ctx context.Context, rawURL, selectExpr string) ([]string, error) { + code, err := compileJQ(selectExpr) if err != nil { return nil, err } - body, err := c.get(ctx, src.URL) + body, err := c.get(ctx, rawURL) if err != nil { return nil, err } var input any if err := json.Unmarshal(body, &input); err != nil { - return nil, fmt.Errorf("%s: response is not JSON: %w", src.URL, err) + return nil, fmt.Errorf("%s: response is not JSON: %w", rawURL, err) } - return runJQ(code, src.Select, input) + return runJQ(code, selectExpr, input) } // githubRelease is the subset of the GitHub releases API response we need. diff --git a/internal/plugin/files.go b/internal/plugin/files.go index 1a0d2269a83..4373a688d7d 100644 --- a/internal/plugin/files.go +++ b/internal/plugin/files.go @@ -17,6 +17,9 @@ import ( func getConfigIfAny(inc Includable, projectDir string) (*Config, error) { switch includable := inc.(type) { case *devpkg.Package: + if includable.IsPkgDef() { + return getPkgDefConfig(includable, projectDir) + } return getBuiltinPluginConfigIfExists(includable, projectDir) case *githubPlugin: content, err := includable.Fetch() diff --git a/internal/plugin/pkgdef.go b/internal/plugin/pkgdef.go new file mode 100644 index 00000000000..70e7502d971 --- /dev/null +++ b/internal/plugin/pkgdef.go @@ -0,0 +1,147 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package plugin + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "os" + "path/filepath" + + "go.jetify.com/devbox/internal/devconfig/configfile" + "go.jetify.com/devbox/internal/devpkg" + "go.jetify.com/devbox/internal/lock" + "go.jetify.com/devbox/internal/nix" + "go.jetify.com/devbox/internal/pkgdef" +) + +// pkgDefPlugin is the Includable behind a package definition's activation +// block (env, shell, create_files). Devbox turns every "pkg:" package into +// one of these so the plugin manager applies the definition's environment +// without any special casing. +type pkgDefPlugin struct { + pkg *devpkg.Package + loaded *pkgdef.Loaded + + // vars are the definition's extra template variables (InstallDir, + // Version, ...), applied to plugin.json and to create_files content. + vars map[string]string +} + +// CanonicalName namespaces the definition's name so that its virtenv and +// devbox.d directories never collide with a built-in plugin or nix package +// of the same name (a project may have both nix "go" and "pkg:./go.pkg.json"). +func (p *pkgDefPlugin) CanonicalName() string { + return "pkgdef-" + p.loaded.Definition.Name +} + +// FileContent reads a create_files source relative to the definition file. +// Remote definitions have no directory to read from. +func (p *pkgDefPlugin) FileContent(subpath string) ([]byte, error) { + if p.loaded.Dir == "" { + return nil, errors.New("create_files is only supported for local package definitions") + } + return os.ReadFile(filepath.Join(p.loaded.Dir, subpath)) +} + +func (p *pkgDefPlugin) Hash() string { + return p.loaded.Hash +} + +func (p *pkgDefPlugin) LockfileKey() string { + return p.pkg.Raw +} + +// GetPkgDefConfigsForPackages returns a plugin config for every package +// definition package in packages that has been resolved in the lockfile. +func GetPkgDefConfigsForPackages( + packages []configfile.Package, + lockfile *lock.File, +) ([]*Config, error) { + configs := []*Config{} + for _, pkg := range devpkg.PackagesFromConfig(packages, lockfile) { + if !pkg.IsPkgDef() || !pkg.IsInstallable() { + continue + } + config, err := getPkgDefConfig(pkg, lockfile.ProjectDir()) + if err != nil { + // Loading the config must not depend on the definition being + // reachable: `devbox update` and `devbox rm` are the commands + // that fix a missing or changed definition, and they need the + // config to load. The install step reports the error. + slog.Debug("skipping package definition plugin", "pkg", pkg.Raw, "err", err) + continue + } + if config != nil { + configs = append(configs, config) + } + } + return configs, nil +} + +// pkgDefPluginContent is the subset of a package definition that maps onto +// plugin.json. +type pkgDefPluginContent struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Env map[string]string `json:"env,omitempty"` + Shell map[string]json.RawMessage `json:"shell,omitempty"` + CreateFiles map[string]string `json:"create_files,omitempty"` +} + +// getPkgDefConfig builds the plugin config for a package definition. It +// returns nil when the package has no lockfile entry yet (for example while +// `devbox add` is still resolving it). +func getPkgDefConfig(pkg *devpkg.Package, projectDir string) (*Config, error) { + locked := pkg.Locked() + if locked == nil { + return nil, nil + } + ref, err := pkgdef.ParseRef(pkg.Raw) + if err != nil { + return nil, err + } + + client := pkgdef.NewClient() + loaded, err := client.Load(context.TODO(), ref, projectDir, locked.DefinitionHash) + if err != nil { + return nil, err + } + def := loaded.Definition + + system := nix.System() + vars, err := def.TemplateVars(system, locked.Version) + if err != nil { + return nil, err + } + vars.InstallDir = client.InstallDir(def.Name, locked.Version, system) + vars.CacheDir = client.CacheDir + + content, err := json.Marshal(pkgDefPluginContent{ + Name: def.Name, + Description: def.Description, + Env: def.Env, + Shell: def.Shell, + CreateFiles: def.CreateFiles, + }) + if err != nil { + return nil, err + } + + includable := &pkgDefPlugin{ + pkg: pkg, + loaded: loaded, + vars: map[string]string{ + "InstallDir": vars.InstallDir, + "CacheDir": vars.CacheDir, + "Version": vars.Version, + "System": vars.System, + "OS": vars.OS, + "Arch": vars.Arch, + }, + } + return buildConfigWithVars(includable, projectDir, string(content), includable.vars) +} diff --git a/internal/plugin/plugin.go b/internal/plugin/plugin.go index ca2450d1746..5d2916bcccc 100644 --- a/internal/plugin/plugin.go +++ b/internal/plugin/plugin.go @@ -214,8 +214,7 @@ func (m *Manager) createFile( urlForInput = pkg.URLForFlakeInput() } - var buf bytes.Buffer - if err = tmpl.Execute(&buf, map[string]any{ + data := map[string]any{ "DevboxProjectDir": m.ProjectDir(), "DevboxDir": filepath.Join(m.ProjectDir(), devboxDirName, name), "DevboxDirRoot": filepath.Join(m.ProjectDir(), devboxDirName), @@ -225,7 +224,14 @@ func (m *Manager) createFile( "System": nix.System(), "URLForInput": urlForInput, "Virtenv": filepath.Join(virtenvPath, name), - }); err != nil { + } + if pkgDef, ok := pkg.(*pkgDefPlugin); ok { + for k, v := range pkgDef.vars { + data[k] = v + } + } + var buf bytes.Buffer + if err = tmpl.Execute(&buf, data); err != nil { return errors.WithStack(err) } var fileMode fs.FileMode = 0o644 @@ -246,20 +252,30 @@ func (m *Manager) createFile( // buildConfig returns a plugin.Config func buildConfig(pkg Includable, projectDir, content string) (*Config, error) { + return buildConfigWithVars(pkg, projectDir, content, nil) +} + +// buildConfigWithVars is buildConfig with additional template variables, +// which package definitions use to expose their install location. +func buildConfigWithVars(pkg Includable, projectDir, content string, extraVars map[string]string) (*Config, error) { cfg := &Config{PluginOnlyData: PluginOnlyData{Source: pkg}} name := pkg.CanonicalName() - t, err := template.New(name + "-template").Parse(content) + tmpl, err := template.New(name + "-template").Parse(content) if err != nil { return nil, errors.WithStack(err) } - var buf bytes.Buffer - if err = t.Execute(&buf, map[string]string{ + vars := map[string]string{ "DevboxProjectDir": projectDir, "DevboxDir": filepath.Join(projectDir, devboxDirName, name), "DevboxDirRoot": filepath.Join(projectDir, devboxDirName), "DevboxProfileDefault": filepath.Join(projectDir, nix.ProfilePath), "Virtenv": filepath.Join(projectDir, VirtenvPath, name), - }); err != nil { + } + for k, v := range extraVars { + vars[k] = v + } + var buf bytes.Buffer + if err = tmpl.Execute(&buf, vars); err != nil { return nil, errors.WithStack(err) } diff --git a/specs/package-definitions.md b/specs/package-definitions.md index ee30b76b6fb..a7f28d8a9bd 100644 --- a/specs/package-definitions.md +++ b/specs/package-definitions.md @@ -45,6 +45,12 @@ paths keep working. `` is one of: | absolute path | `/opt/defs/go.pkg.json` | | | `https://` URL | `https://example.com/go.pkg.json` | Content is cached by hash; the lockfile pins the hash. | | `file://` URL | `file:///tmp/go.pkg.json` | Mainly for tests. | + +Inside a definition, a relative `file://` URL (`file://vendor/x.tar.gz`) is +resolved against the directory of a local definition file when fetching. +The lockfile records the URL as rendered, so a project that ships artifacts +next to its definition stays portable. Remote definitions must use absolute +URLs. | `github:` | `github:jetify-com/devbox/pkgs/go/go.pkg.json` | Planned. Sugar for the raw.githubusercontent.com URL. | If no `@version` is given, `devbox add` appends `@latest`. @@ -242,7 +248,14 @@ follow the sprig order with the string last, so pipelines read naturally: is the definition, so init hooks and created files work through the existing plugin manager with no new machinery. `create_files` sources are resolved relative to the definition file; they are only supported for local -definitions. +definitions. The plugin's `{{ .Virtenv }}` and `{{ .DevboxDir }}` are +namespaced as `pkgdef-` so they never collide with a built-in plugin +or nix package of the same name. + +A definition that cannot be loaded (deleted file, unreachable URL, changed +remote content) does not prevent `devbox.json` from loading, so +`devbox update` and `devbox rm` can always run; the install step reports +the error. No deactivate hook exists on purpose: Devbox never mutates the parent shell. Every `devbox shell` and `devbox run` recomputes `PATH` and env from the @@ -279,6 +292,17 @@ shared across projects. Nothing is copied per project; `PATH` points at the cache. Garbage collection of unused versions is a follow-up (`devbox cache gc`). +Each install directory carries a marker with the *install fingerprint* of +the definition: a hash of the install strategy, `bin`, `hooks`, `os_map` +and `arch_map`. When a definition's install recipe changes for a version +that is already installed, the next install redoes it. Activation-only +edits (`env`, `shell`, `create_files`) do not trigger a reinstall; they are +applied at shell time. + +Trust-on-first-use digests are written to `devbox.lock` only by the install +step (`devbox add`, `devbox install`, `devbox update`), never while +computing the environment for `devbox shell` or `devbox shellenv`. + ## Security The trust model equals plugins today: a definition is remote JSON that can @@ -313,7 +337,8 @@ A leaf package with no dependency on `devpkg`, `lock` or `plugin`. ### PR 3: wiring, lockfile, env, plugins, testscripts -Behind the `DEVBOX_FEATURE_PKG_DEFS` flag (on in dev builds). +Behind the `DEVBOX_FEATURE_PKG_DEFS` flag (on in dev builds). While the +flag is off, `pkg:` strings are not recognized at all. - `pkgtype.IsPkgDef`; excluded from flake detection like runx. - `devpkg.Package`: `IsPkgDef`, `IsNix` false so shellgen skips it, @@ -323,11 +348,21 @@ Behind the `DEVBOX_FEATURE_PKG_DEFS` flag (on in dev builds). - `Devbox.installPackages` installs package definitions next to runx; `computeEnv` adds their `bin` directories to `PATH`. - `devconfig.loadRecursive` creates a plugin config per package definition so - `env`, `shell.init_hook` and `create_files` apply. + `env`, `shell.init_hook` and `create_files` apply, with the definition's + template variables available in both. The plugin's virtenv directory is + `pkgdef-`. Definitions excluded on the current platform get no + plugin config. - Local definition file hashes feed the config hash so edits trigger reinstall. -- `devbox update` re-resolves through the same lockfile branch. -- Testscripts under `testscripts/pkgdef/` using `file://` artifacts. +- `devbox update` re-resolves through the same lockfile branch and records + a changed `definition_hash` even when the version and artifacts are the + same. +- Testscripts under `testscripts/pkgdef/` using relative `file://` + artifacts next to the definition. +- `devbox add` fails with the definition's own error (unknown version, + unsupported system, bad checksum) instead of falling back to nixpkgs. +- A definition whose `platforms` exclude the current system fails at + install time with a hint to use `--exclude-platform`. ### Later @@ -360,6 +395,9 @@ Behind the `DEVBOX_FEATURE_PKG_DEFS` flag (on in dev builds). | Spec and Go example | in review (PR 1) | | `internal/pkgdef` library | in review (PR 2) | | Wiring behind feature flag | in review (PR 3) | +| Lockfile fields `definition_hash`, `systems.*.url`, `systems.*.sha256` | in review (PR 3) | +| Activation through the plugin manager (`env`, `shell`, `create_files`) | in review (PR 3) | +| Trust-on-first-use checksum recording | in review (PR 3) | | `manager` strategy | not started | | `script` strategy | not started | | `github:` refs | not started | diff --git a/testscripts/pkgdef/pkgdef.test.txt b/testscripts/pkgdef/pkgdef.test.txt new file mode 100644 index 00000000000..3971835bf39 --- /dev/null +++ b/testscripts/pkgdef/pkgdef.test.txt @@ -0,0 +1,97 @@ +# Package definitions: install a package from a vendor-style artifact, switch +# versions, and remove it. The "vendor" is a set of local files served through +# relative file:// URLs, which resolve against the definition's directory. + +env DEVBOX_FEATURE_PKG_DEFS=1 + +# Add with a version prefix. It resolves to the newest matching version. +exec devbox add pkg:./hello.pkg.json@1 +devboxjson.packages.contains devbox.json 'pkg:./hello.pkg.json@1' +devboxlock.packages.contains devbox.lock 'pkg:./hello.pkg.json@1' +grep '"version": "1.2.0"' devbox.lock +grep '"source": "pkg-def"' devbox.lock +grep '"definition_hash": "' devbox.lock +grep '"sha256": "066b305a08797c899572e0ad5ade421e2959d48d7858d6294f6ae81d03650590"' devbox.lock + +# The binary is on PATH and the definition's env and init hook apply. +exec devbox run -- hello +stdout 'hello 1.2.0' +exec devbox run -- sh -c 'echo "HELLO_HOME=$HELLO_HOME"; cat "$HELLO_HOME/bin/hello" | head -1' +stdout 'HELLO_HOME=.*/devbox/pkgs/hello/1.2.0/' +stdout '#!/bin/sh' +exec devbox run -- sh -c 'echo "HELLO_VERSION=$HELLO_VERSION"' +stdout 'HELLO_VERSION=1.2.0' +exec devbox run -- sh -c 'cat "$HELLO_INIT_FILE"' +stdout 'init hook ran for 1.2.0' +stdout 'virtenv/pkgdef-hello/init.txt' + +# create_files content is templated with the definition's variables. +exec devbox run -- sh -c 'cat .devbox/virtenv/pkgdef-hello/where.txt' +stdout 'install dir: .*/devbox/pkgs/hello/1.2.0/' +stdout 'version: 1.2.0' + +# The lockfile keeps relative file URLs relative, so it is portable. +grep '"url": "file://vendor/hello-1.2.0.sh"' devbox.lock + +# Switching the version in devbox.json switches what is on PATH. +exec devbox add pkg:./hello.pkg.json@2 +! devboxjson.packages.contains devbox.json 'pkg:./hello.pkg.json@1' +devboxjson.packages.contains devbox.json 'pkg:./hello.pkg.json@2' +grep '"version": "2.0.0"' devbox.lock +exec devbox run -- hello +stdout 'hello 2.0.0' + +# A version that does not exist is a clear error. +! exec devbox add pkg:./hello.pkg.json@9 +stderr 'not found' + +# Removing restores a plain project. +exec devbox rm pkg:./hello.pkg.json@2 +! devboxjson.packages.contains devbox.json 'pkg:./hello.pkg.json@2' +! devboxlock.packages.contains devbox.lock 'pkg:./hello.pkg.json@2' + +-- devbox.json -- +{ + "packages": [] +} + +-- hello.pkg.json -- +// A tiny package definition used by the testscript. +{ + "schema": 1, + "name": "hello", + "description": "Says hello", + "platforms": ["aarch64-darwin", "x86_64-darwin", "aarch64-linux", "x86_64-linux"], + "versions": {"type": "static", "versions": ["1.2.0", "2.0.0", "2.1.0-rc1"]}, + "install": { + "type": "binary", + "url": "file://vendor/hello-{{.Version}}.sh", + "checksum": {"type": "sha256_url", "url": "file://vendor/SHA256SUMS"} + }, + "binaries": ["hello"], + "env": { + "HELLO_HOME": "{{ .InstallDir }}", + "HELLO_VERSION": "{{ .Version }}", + "HELLO_INIT_FILE": "{{ .Virtenv }}/init.txt" + }, + "shell": { + "init_hook": ["echo \"init hook ran for {{ .Version }} ($HELLO_INIT_FILE)\" > \"$HELLO_INIT_FILE\""] + }, + "create_files": { + "{{ .Virtenv }}/where.txt": "where.txt.tmpl" + } +} + +-- where.txt.tmpl -- +install dir: {{ .InstallDir }} +version: {{ .Version }} + +-- vendor/hello-1.2.0.sh -- +#!/bin/sh +echo hello 1.2.0 +-- vendor/hello-2.0.0.sh -- +#!/bin/sh +echo hello 2.0.0 +-- vendor/SHA256SUMS -- +066b305a08797c899572e0ad5ade421e2959d48d7858d6294f6ae81d03650590 hello-1.2.0.sh +7c208c07608e0a2d2351a92a27924ddbc3b1c3d9d923638e1999e2d08d68ff11 hello-2.0.0.sh