feat(pkgdef): wire package definitions into add, lock, install and env

internal/boxcli/featureflag/pkgdefs.go

@@ -0,0 +1,6 @@
+package featureflag
+
+// PkgDefs enables package definitions: installing packages from the
+// vendor's binary distribution via "pkg:<ref>@<version>" package strings.
+// See specs/package-definitions.md.
+var PkgDefs = disable("PKG_DEFS").EnableOnDev()

internal/boxcli/multi/sync.go

@@ -49,6 +49,7 @@ func SyncLockfiles(pkgs []string) error {
 					lockFile.Packages[key].Source = latestPkg.Source
 					lockFile.Packages[key].Version = latestPkg.Version
 					lockFile.Packages[key].Systems = latestPkg.Systems
+					lockFile.Packages[key].DefinitionHash = latestPkg.DefinitionHash
 					changed = true
 				}
 			}

internal/devbox/devbox.go

@@ -835,6 +835,12 @@ func (d *Devbox) computeEnv(
 	}
 	devboxEnvPath = envpath.JoinPathLists(devboxEnvPath, runXPaths)
 
+	pkgDefPaths, err := d.pkgDefBinPaths(ctx, false /*recordDigest*/)
+	if err != nil {
+		return nil, err
+	}
+	devboxEnvPath = envpath.JoinPathLists(append([]string{devboxEnvPath}, pkgDefPaths...)...)
+
 	pathStack := envpath.Stack(env, originalEnv)
 	pathStack.Push(env, d.ProjectDirHash(), devboxEnvPath, envOpts.PreservePathStack)
 	env["PATH"] = pathStack.Path(env)

internal/devbox/packages.go

@@ -133,7 +133,7 @@ func (d *Devbox) Add(ctx context.Context, pkgsNames []string, opts devopt.AddOpt
 			// about not building on the current system, since user's can continue
 			// via --exclude-platform flag.
 			packageNameForConfig = pkg.Versioned()
-		} else if !versionedPkg.IsDevboxPackage {
+		} else if !versionedPkg.IsDevboxPackage || versionedPkg.IsPkgDef() {
 			// This means it didn't validate and we don't want to fallback to legacy
 			// Just propagate the error.
 			return err
@@ -482,7 +482,10 @@ func (d *Devbox) installPackages(ctx context.Context, mode installMode) error {
 		return err
 	}
 
-	return d.InstallRunXPackages(ctx)
+	if err := d.InstallRunXPackages(ctx); err != nil {
+		return err
+	}
+	return d.installPkgDefPackages(ctx)
 }
 
 func (d *Devbox) InstallRunXPackages(ctx context.Context) error {
@@ -667,7 +670,7 @@ func (d *Devbox) moveAllowInsecureFromLockfile(writer io.Writer, lockfile *lock.
 func (d *Devbox) FixMissingStorePaths(ctx context.Context) error {
 	packages := d.InstallablePackages()
 	for _, pkg := range packages {
-		if !pkg.IsDevboxPackage || pkg.IsRunX() {
+		if !pkg.IsDevboxPackage || !pkg.IsNix() {
 			continue
 		}
 		existingStorePaths, err := pkg.GetResolvedStorePaths()

internal/devbox/pkgdef.go

@@ -0,0 +1,100 @@
+// Copyright 2024 Jetify Inc. and contributors. All rights reserved.
+// Use of this source code is governed by the license in the LICENSE file.
+
+package devbox
+
+import (
+	"context"
+	"strings"
+
+	"github.com/samber/lo"
+	"go.jetify.com/devbox/internal/boxcli/usererr"
+	"go.jetify.com/devbox/internal/devpkg"
+	"go.jetify.com/devbox/internal/nix"
+	"go.jetify.com/devbox/internal/pkgdef"
+	"go.jetify.com/devbox/internal/ux"
+)
+
+// installPkgDefPackages installs every package-definition package
+// ("pkg:<ref>@<version>") that is not installed yet.
+func (d *Devbox) installPkgDefPackages(ctx context.Context) error {
+	_, err := d.pkgDefBinPaths(ctx, true /*recordDigest*/)
+	return err
+}
+
+// pkgDefBinPaths returns the bin directories of every package-definition
+// package, installing those that are missing. Installs are idempotent, so
+// this is cheap once everything is in place and safe to call whenever the
+// environment is computed.
+//
+// recordDigest controls whether a digest observed for a definition without
+// a checksum source is written to devbox.lock. Only the install path does
+// that; computing the environment must never modify the lockfile.
+func (d *Devbox) pkgDefBinPaths(ctx context.Context, recordDigest bool) ([]string, error) {
+	paths := []string{}
+	for _, pkg := range lo.Filter(d.InstallablePackages(), devpkg.IsPkgDef) {
+		installed, err := d.installPkgDef(ctx, pkg, recordDigest)
+		if err != nil {
+			return nil, err
+		}
+		paths = append(paths, installed.BinDirs...)
+	}
+	return paths, nil
+}
+
+func (d *Devbox) installPkgDef(ctx context.Context, pkg *devpkg.Package, recordDigest bool) (*pkgdef.Installed, error) {
+	locked, err := d.lockfile.Resolve(pkg.Raw)
+	if err != nil {
+		return nil, err
+	}
+	ref, err := pkgdef.ParseRef(pkg.Raw)
+	if err != nil {
+		return nil, err
+	}
+
+	client := pkgdef.NewClient()
+	client.Stderr = d.stderr
+	loaded, err := client.Load(ctx, ref, d.projectDir, locked.DefinitionHash)
+	if err != nil {
+		return nil, err
+	}
+	def := loaded.Definition
+
+	system := nix.System()
+	if !def.SupportsSystem(system) {
+		return nil, usererr.New(
+			"Package %s does not support %s (it supports %s). "+
+				"Run `devbox add %s --exclude-platform %s` to skip it on this machine.",
+			pkg.Raw, system, strings.Join(def.Platforms, ", "), pkg.Raw, system,
+		)
+	}
+
+	artifact := pkgdef.Artifact{}
+	if info := locked.Systems[system]; info != nil {
+		artifact = pkgdef.Artifact{URL: info.URL, Sha256: info.Sha256}
+	}
+
+	if !client.IsInstalled(def, locked.Version, system) {
+		ux.Finfof(d.stderr, "Installing %s@%s from %s\n", def.Name, locked.Version, lo.Ternary(artifact.URL != "", artifact.URL, ref.Source))
+		if def.UsesScripts() {
+			ux.Fwarningf(d.stderr, "Package definition %s runs shell commands while installing.\n", ref.Source)
+		}
+		if artifact.Sha256 == "" {
+			ux.Fwarningf(d.stderr, "Package definition %s has no checksum for %s; the digest of this download will be recorded in devbox.lock.\n", ref.Source, system)
+		}
+	}
+
+	installed, err := client.Install(ctx, def, locked.Version, system, artifact)
+	if err != nil {
+		return nil, err
+	}
+
+	// Trust on first use: record the digest we observed so every other
+	// machine verifies against it.
+	if recordDigest && artifact.Sha256 == "" && installed.Sha256 != "" {
+		if err := d.lockfile.SetArtifactForPackage(pkg.Raw, system, installed.URL, installed.Sha256); err != nil {
+			return nil, err
+		}
+	}
+	return installed, nil
+}

internal/devbox/update.go

@@ -231,6 +231,15 @@ func (d *Devbox) mergeResolvedPackageToLockfile(
 		return nil
 	}
 
+	// A package definition can change without changing the version or the
+	// artifacts (for example a new env var or init hook). The lockfile pins
+	// the definition by hash, so the new hash must be recorded.
+	if existing.DefinitionHash != resolved.DefinitionHash {
+		useResolvedPackageInLockfile(lockfile, pkg, resolved, existing)
+		ux.Finfof(d.stderr, "Updated package definition for %s\n", pkg)
+		return nil
+	}
+
 	ux.Finfof(d.stderr, "Already up-to-date %s %s\n", pkg, existing.Version)
 	return nil
 }

internal/devbox/update_test.go

@@ -384,3 +384,35 @@ func TestDescribeFlakeUpdateFormats(t *testing.T) {
 	newNoDate := &lock.Package{Resolved: "github:numtide/flake-utils/" + newRev + "#pkg"}
 	require.Equal(t, "abc1234 -> f456789", describeFlakeUpdate(oldNoDate, newNoDate))
 }
+
+func TestUpdateRecordsChangedDefinitionHash(t *testing.T) {
+	devbox := devboxForTesting(t)
+
+	raw := "pkg:./hello.pkg.json@1"
+	devPkg := devpkg.PackageFromStringWithDefaults(raw, nil)
+	existing := &lock.Package{
+		Resolved:       "pkg:./hello.pkg.json@1.2.0",
+		Version:        "1.2.0",
+		Source:         "pkg-def",
+		DefinitionHash: "old",
+		Systems: map[string]*lock.SystemInfo{
+			"x86_64-linux": {URL: "file://vendor/hello-1.2.0.sh", Sha256: "abc"},
+		},
+	}
+	resolved := &lock.Package{
+		Resolved:       "pkg:./hello.pkg.json@1.2.0",
+		Version:        "1.2.0",
+		Source:         "pkg-def",
+		DefinitionHash: "new",
+		Systems: map[string]*lock.SystemInfo{
+			"x86_64-linux": {URL: "file://vendor/hello-1.2.0.sh", Sha256: "abc"},
+		},
+	}
+	lockfile := &lock.File{
+		Packages: map[string]*lock.Package{raw: existing},
+	}
+
+	err := devbox.mergeResolvedPackageToLockfile(devPkg, resolved, lockfile)
+	require.NoError(t, err, "update failed")
+	require.Equal(t, "new", lockfile.Packages[raw].DefinitionHash)
+}

internal/devconfig/config.go

@@ -289,6 +289,23 @@ func (c *Config) loadRecursive(
 		included = append(included, includable)
 	}
 
+	// Package definitions ("pkg:" packages) carry their own env, init hook
+	// and create_files. They cannot include anything, so there is nothing
+	// to recurse into.
+	pkgDefs, err := plugin.GetPkgDefConfigsForPackages(
+		c.Root.TopLevelPackages(),
+		lockfile,
+	)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	for _, pkgDef := range pkgDefs {
+		included = append(included, &Config{
+			Root:       pkgDef.ConfigFile,
+			pluginData: &pkgDef.PluginOnlyData,
+		})
+	}
+
 	c.included = included
 	return nil
 }

internal/devpkg/package.go

@@ -25,6 +25,7 @@ import (
 	"go.jetify.com/devbox/internal/devpkg/pkgtype"
 	"go.jetify.com/devbox/internal/lock"
 	"go.jetify.com/devbox/internal/nix"
+	"go.jetify.com/devbox/internal/pkgdef"
 	"go.jetify.com/devbox/internal/ux"
 	"go.jetify.com/devbox/nix/flake"
 	"go.jetify.com/devbox/plugins"
@@ -171,6 +172,11 @@ func resolve(pkg *Package) error {
 	if err != nil {
 		return err
 	}
+	// Package definitions resolve to "pkg:<ref>@<version>", which is not a
+	// flake installable. Locking them is all that's needed here.
+	if pkg.IsPkgDef() {
+		return nil
+	}
 	parsed, err := flake.ParseInstallable(resolved.Resolved)
 	if err != nil {
 		return err
@@ -464,6 +470,14 @@ func (p *Package) Hash() string {
 		// user always gets newest flake.
 		sum, _ = cachehash.File(filepath.Join(p.installable.Ref.Path, "flake.nix"))
 	}
+	if ref, ok := p.pkgDefRef(); ok {
+		// For local package definitions, hash the file so that editing it
+		// changes the project state and re-runs the install step (which
+		// redoes the install if the recipe changed).
+		if path := ref.LocalPath(p.lockfile.ProjectDir()); path != "" {
+			sum, _ = cachehash.File(path)
+		}
+	}
 
 	if sum == "" {
 		sum = cachehash.Bytes([]byte(cmp.Or(p.installable.String(), p.Raw)))
@@ -501,10 +515,23 @@ func (p *Package) CanonicalName() string {
 	if !p.IsDevboxPackage {
 		return ""
 	}
+	if ref, ok := p.pkgDefRef(); ok {
+		return pkgdef.Ref{Source: ref.Source}.String()
+	}
 	name, _, _ := strings.Cut(p.Raw, "@")
 	return name
 }
 
+// pkgDefRef parses the package string as a package definition reference.
+// Definition sources may contain "@", so these split at the last one.
+func (p *Package) pkgDefRef() (pkgdef.Ref, bool) {
+	if !p.IsPkgDef() {
+		return pkgdef.Ref{}, false
+	}
+	ref, err := pkgdef.ParseRef(p.Raw)
+	return ref, err == nil
+}
+
 func (p *Package) Versioned() string {
 	if p.IsDevboxPackage && !p.isVersioned() {
 		return p.Raw + "@latest"
@@ -537,6 +564,9 @@ func EnsureNixpkgsPrefetched(ctx context.Context, w io.Writer, pkgs []*Package)
 // ensureNixpkgsPrefetched should be called via the public EnsureNixpkgsPrefetched.
 // See function comment there.
 func (p *Package) ensureNixpkgsPrefetched(w io.Writer) error {
+	if !p.IsNix() {
+		return nil
+	}
 	inCache, err := p.IsInBinaryCache()
 	if err != nil {
 		return err
@@ -560,11 +590,17 @@ func (p *Package) version() string {
 	if !p.IsDevboxPackage {
 		return ""
 	}
+	if ref, ok := p.pkgDefRef(); ok {
+		return ref.Version
+	}
 	_, version, _ := strings.Cut(p.Raw, "@")
 	return version
 }
 
 func (p *Package) isVersioned() bool {
+	if ref, ok := p.pkgDefRef(); ok {
+		return ref.Version != ""
+	}
 	return p.IsDevboxPackage && strings.Contains(p.Raw, "@")
 }
 
@@ -659,6 +695,18 @@ func (p *Package) IsRunX() bool {
 	return pkgtype.IsRunX(p.Raw)
 }
 
+// IsPkgDef reports whether the package is installed from a package
+// definition ("pkg:<ref>@<version>", see specs/package-definitions.md).
+func (p *Package) IsPkgDef() bool {
+	return pkgtype.IsPkgDef(p.Raw)
+}
+
+// Locked returns the package's lockfile entry, or nil if it has not been
+// resolved yet.
+func (p *Package) Locked() *lock.Package {
+	return p.lockfile.Get(p.LockfileKey())
+}
+
 func (p *Package) IsNix() bool {
 	return IsNix(p, 0)
 }
@@ -682,13 +730,17 @@ func (p *Package) LockfileKey() string {
 }
 
 func IsNix(p *Package, _ int) bool {
-	return !p.IsRunX()
+	return !p.IsRunX() && !p.IsPkgDef()
 }
 
 func IsRunX(p *Package, _ int) bool {
 	return p.IsRunX()
 }
 
+func IsPkgDef(p *Package, _ int) bool {
+	return p.IsPkgDef()
+}
+
 func (p *Package) DocsURL() string {
 	if p.IsRunX() {
 		path, _, _ := strings.Cut(p.RunXPath(), "@")
@@ -703,7 +755,7 @@ func (p *Package) DocsURL() string {
 // GetOutputNames returns the names of the nix package outputs. Outputs can be
 // specified in devbox.json package fields or as part of the flake reference.
 func (p *Package) GetOutputNames() ([]string, error) {
-	if p.IsRunX() {
+	if !p.IsNix() {
 		return []string{}, nil
 	}
 

internal/devpkg/package_test.go

@@ -197,6 +197,9 @@ func TestCanonicalName(t *testing.T) {
 		{"runx:golangci/golangci-lint@latest", "runx:golangci/golangci-lint"},
 		{"runx:golangci/golangci-lint@v0.0.2", "runx:golangci/golangci-lint"},
 		{"runx:golangci/golangci-lint", "runx:golangci/golangci-lint"},
+		{"pkg:./go.pkg.json@1.22", "pkg:./go.pkg.json"},
+		{"pkg:./go.pkg.json", "pkg:./go.pkg.json"},
+		{"pkg:https://cdn.example.com/@acme/tool.pkg.json@latest", "pkg:https://cdn.example.com/@acme/tool.pkg.json"},
 		{"github:NixOS/nixpkgs/12345", ""},
 		{"path:/to/my/file", ""},
 	}

internal/devpkg/pkgtype/flake.go

@@ -7,7 +7,7 @@ import (
 )
 
 func IsFlake(s string) bool {
-	if IsRunX(s) {
+	if IsRunX(s) || IsPkgDef(s) {
 		return false
 	}
 	parsed, err := flake.ParseInstallable(s)