ENG-1634: Add --build to state publish: build, encrypt, wrap, upload

cmd/state/internal/cmdtree/publish.go

@@ -111,6 +111,11 @@ func newPublish(prime *primer.Values) *captain.Command {
 				Description: locale.Tl("author_upload_metafile_description", "A yaml file expressing the ingredient meta information. Use --editor to review the file format."),
 				Value:       &params.MetaFilepath,
 			},
+			{
+				Name:        "build",
+				Description: locale.Tl("author_upload_build_description", "Build, encrypt, and publish a private ingredient from a local source directory. The ingredient name, version, and namespace remain visible to the platform; the source contents do not."),
+				Value:       &params.Build,
+			},
 		},
 		[]*captain.Argument{
 			{

internal/artifactcrypto/artifactcrypto.go

@@ -39,6 +39,8 @@ var (
 	ErrWrongKey = errs.New("key does not match payload fingerprint")
 	// ErrInvalidKeySize indicates the supplied key is not a 32-byte AES-256 key.
 	ErrInvalidKeySize = errs.New("key must be 32 bytes (AES-256)")
+	// ErrHeaderTooLarge indicates the serialized header (driven by the key id length) exceeds the readable maximum.
+	ErrHeaderTooLarge = errs.New("encrypted payload header exceeds the maximum size")
 )
 
 const (

internal/artifactcrypto/artifactcrypto_test.go

@@ -276,3 +276,11 @@ type failingReader struct{}
 func (failingReader) Read([]byte) (int, error) {
 	return 0, errors.New("body should not have been read")
 }
+
+func TestEncryptRejectsOversizeHeader(t *testing.T) {
+	oversizeKeyID := string(bytes.Repeat([]byte("k"), maxHeaderLen+1))
+	err := Encrypt(failingReader{}, io.Discard, testKey, oversizeKeyID)
+	if !errors.Is(err, ErrHeaderTooLarge) {
+		t.Fatalf("error = %v, want ErrHeaderTooLarge", err)
+	}
+}

internal/artifactcrypto/encrypt.go

@@ -20,6 +20,9 @@ func Encrypt(src io.Reader, dst io.Writer, key []byte, keyID string) error {
 	}
 
 	raw := serializeHeader(keyID, Fingerprint(key), uint32(encChunkSize))
+	if len(raw) > maxHeaderLen {
+		return ErrHeaderTooLarge
+	}
 	var lenBuf [4]byte
 	binary.BigEndian.PutUint32(lenBuf[:], uint32(len(raw)))
 	if _, err := dst.Write(lenBuf[:]); err != nil {

internal/python/wheel/distinfo.go

@@ -18,7 +18,7 @@ type record struct {
 }
 
 // buildMetadata returns the dist-info METADATA contents.
-func buildMetadata(meta resolvedMetadata) []byte {
+func buildMetadata(meta Metadata) []byte {
 	var b bytes.Buffer
 	b.WriteString("Metadata-Version: 2.1\n")
 	b.WriteString("Name: " + meta.Name + "\n")

internal/python/wheel/metadata.go

@@ -10,31 +10,24 @@ import (
 	"github.com/BurntSushi/toml"
 )
 
-// resolvedMetadata is the metadata after merging caller overrides with
-// pyproject.toml, with name and version guaranteed non-empty.
-type resolvedMetadata struct {
-	Name    string
-	Version string
-	Summary string
-}
-
-// resolveMetadata fills empty fields of override from srcDir's pyproject.toml and
-// returns the result, erroring if name or version is set by neither source.
-func resolveMetadata(srcDir string, override Metadata) (resolvedMetadata, error) {
+// ResolveMetadata reads srcDir's pyproject.toml [project] table and applies the
+// non-empty fields of override on top, producing the metadata to pack with. It
+// errors when neither source supplies a name or version.
+func ResolveMetadata(srcDir string, override Metadata) (*Metadata, error) {
 	proj, err := readPyProject(filepath.Join(srcDir, "pyproject.toml"))
 	if err != nil {
-		return resolvedMetadata{}, err
+		return nil, err
 	}
 
-	res := resolvedMetadata{
+	meta := Metadata{
 		Name:    firstNonEmpty(override.Name, proj.Name),
 		Version: firstNonEmpty(override.Version, proj.Version),
 		Summary: firstNonEmpty(override.Summary, proj.Description),
 	}
-	if res.Name == "" || res.Version == "" {
-		return resolvedMetadata{}, ErrMissingMetadata
+	if meta.Name == "" || meta.Version == "" {
+		return nil, ErrMissingMetadata
 	}
-	return res, nil
+	return &meta, nil
 }
 
 type pyProject struct {

internal/python/wheel/metadata_test.go

@@ -18,7 +18,7 @@ func TestResolveMetadata(t *testing.T) {
 	t.Run("pyproject fills empty fields", func(t *testing.T) {
 		dir := t.TempDir()
 		writePyproject(t, dir, "[project]\nname = \"proj\"\nversion = \"3.1\"\ndescription = \"from toml\"\n")
-		res, err := resolveMetadata(dir, Metadata{})
+		res, err := ResolveMetadata(dir, Metadata{})
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -30,7 +30,7 @@ func TestResolveMetadata(t *testing.T) {
 	t.Run("caller overrides pyproject", func(t *testing.T) {
 		dir := t.TempDir()
 		writePyproject(t, dir, "[project]\nname = \"proj\"\nversion = \"3.1\"\n")
-		res, err := resolveMetadata(dir, Metadata{Name: "override", Version: "9.9"})
+		res, err := ResolveMetadata(dir, Metadata{Name: "override", Version: "9.9"})
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -41,7 +41,7 @@ func TestResolveMetadata(t *testing.T) {
 
 	t.Run("missing name and version errors", func(t *testing.T) {
 		dir := t.TempDir() // no pyproject.toml
-		if _, err := resolveMetadata(dir, Metadata{Name: "only-name"}); !errors.Is(err, ErrMissingMetadata) {
+		if _, err := ResolveMetadata(dir, Metadata{Name: "only-name"}); !errors.Is(err, ErrMissingMetadata) {
 			t.Errorf("error = %v, want ErrMissingMetadata", err)
 		}
 	})

internal/python/wheel/wheel.go

@@ -49,31 +49,29 @@ type sourceFile struct {
 	abs string
 }
 
-// Pack builds a pure-Python wheel from srcDir, writes it into outDir as
+// Pack builds a pure-Python wheel from the already-resolved meta (use
+// ResolveMetadata to derive it from pyproject.toml), writes it into outDir as
 // {normalized_name}-{version}-py3-none-any.whl, and returns the wheel path.
 //
 // The wheel root mirrors srcDir: the caller points srcDir at the directory whose
-// children are the importable packages. The top-level pyproject.toml (read for
-// metadata), __pycache__ directories, *.pyc/*.pyo files, and version-control
-// directories are not packed. Compiled files (.so/.pyd/.dylib) are rejected.
-// Values in meta override those read from pyproject.toml; the name and version
-// must resolve from one source or the other.
+// children are the importable packages. The top-level pyproject.toml, __pycache__
+// directories, *.pyc/*.pyo files, and version-control directories are not packed.
+// Compiled files (.so/.pyd/.dll/.dylib) are rejected.
 //
 // Output is byte-reproducible: identical input trees produce identical wheels
 // regardless of file timestamps. On any failure no wheel is left at the path.
-func Pack(srcDir string, meta Metadata, outDir string) (_ string, rerr error) {
-	resolved, err := resolveMetadata(srcDir, meta)
-	if err != nil {
-		return "", errs.Wrap(err, "could not resolve package metadata")
+func Pack(srcDir string, meta Metadata, outDir string) (string, error) {
+	if meta.Name == "" || meta.Version == "" {
+		return "", ErrMissingMetadata
 	}
 
 	files, err := collectFiles(srcDir)
 	if err != nil {
 		return "", errs.Wrap(err, "could not scan source tree")
 	}
 
-	outPath := filepath.Join(outDir, wheelFilename(resolved.Name, resolved.Version))
-	if err := writeWheel(files, resolved, outPath); err != nil {
+	outPath := filepath.Join(outDir, wheelFilename(meta.Name, meta.Version))
+	if err := writeWheel(files, meta, outPath); err != nil {
 		return "", errs.Wrap(err, "could not write wheel")
 	}
 	return outPath, nil
@@ -157,7 +155,7 @@ func isNativeFile(rel string) bool {
 
 // writeWheel writes the wheel to a sibling temp file and renames it onto outPath
 // only after the whole archive is written, so a failure leaves outPath untouched.
-func writeWheel(files []sourceFile, meta resolvedMetadata, outPath string) (rerr error) {
+func writeWheel(files []sourceFile, meta Metadata, outPath string) (rerr error) {
 	tmp, err := os.CreateTemp(filepath.Dir(outPath), filepath.Base(outPath)+".tmp-*")
 	if err != nil {
 		return errs.Wrap(err, "could not create temp wheel")

internal/python/wheel/wheel_test.go

@@ -93,10 +93,10 @@ func TestPackProducesValidWheel(t *testing.T) {
 	}
 
 	// METADATA / WHEEL contents.
-	meta := string(entries[di+"/METADATA"])
+	metaFile := string(entries[di+"/METADATA"])
 	for _, want := range []string{"Metadata-Version: 2.1", "Name: My.Pkg-Name", "Version: 1.0", "Summary: a pkg"} {
-		if !bytes.Contains([]byte(meta), []byte(want)) {
-			t.Errorf("METADATA missing %q; got:\n%s", want, meta)
+		if !bytes.Contains([]byte(metaFile), []byte(want)) {
+			t.Errorf("METADATA missing %q; got:\n%s", want, metaFile)
 		}
 	}
 	wheelFile := string(entries[di+"/WHEEL"])

internal/runners/publish/build.go

@@ -0,0 +1,203 @@
+package publish
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/ActiveState/cli/internal/archiver"
+	"github.com/ActiveState/cli/internal/artifactcrypto"
+	"github.com/ActiveState/cli/internal/errs"
+	"github.com/ActiveState/cli/internal/fileutils"
+	"github.com/ActiveState/cli/internal/locale"
+	"github.com/ActiveState/cli/internal/logging"
+	"github.com/ActiveState/cli/internal/python/wheel"
+	"github.com/ActiveState/cli/internal/runbits/orgkey"
+	"github.com/ActiveState/cli/pkg/platform/model"
+)
+
+// generateEncryptedArtifact validates the --build inputs, fetches and validates
+// the org key, builds the wrapped, encrypted artifact, and points the publish
+// flow at it. It fails closed before producing any artifact if the key is
+// unavailable, and returns a cleanup function the caller must defer.
+func (r *Runner) generateEncryptedArtifact(params *Params) (cleanup func(), rerr error) {
+	if params.Filepath != "" {
+		return nil, locale.NewInputError("err_publish_build_and_file", "The '[ACTIONABLE]--build[/RESET]' flag cannot be combined with a source archive filepath.")
+	}
+	if r.project == nil {
+		return nil, locale.NewInputError("err_publish_build_no_project", "The '[ACTIONABLE]--build[/RESET]' flag requires a project so the organization can be determined.")
+	}
+
+	meta, err := wheel.ResolveMetadata(params.Build, wheel.Metadata{Name: params.Name, Version: params.Version})
+	if err != nil {
+		return nil, locale.WrapInputError(err, "err_publish_build_metadata", "Could not determine the ingredient name and version: {{.V0}}", errs.JoinMessage(err))
+	}
+
+	// Fetch and validate the org key before building anything: a private publish
+	// is encrypted-required, so fail closed before any byte could be uploaded.
+	provider := orgkey.New(r.cfg, r.project.Owner())
+	if !provider.Configured() {
+		return nil, locale.NewInputError("err_publish_orgkey_unconfigured", "No organization key service is configured, so this private ingredient cannot be encrypted.")
+	}
+	defer provider.Close()
+	key, keyID, err := provider.Key(context.Background())
+	if err != nil {
+		return nil, locale.WrapInputError(err, "err_publish_orgkey_unavailable", "Could not obtain the organization key, so nothing was uploaded: {{.V0}}", errs.JoinMessage(err))
+	}
+
+	archivePath, cleanup, err := buildWrappedArtifact(params.Build, *meta, key, keyID)
+	if err != nil {
+		return nil, errs.Wrap(err, "Could not build encrypted artifact")
+	}
+
+	// TODO(ENG-1641): once the platform supports the genesis/timeless publish
+	// flag, set it on the publish mutation so a private publish never advances
+	// any commit's at_time. Until then --build cannot be genesis-stamped.
+
+	params.Filepath = archivePath
+	if params.Name == "" {
+		params.Name = meta.Name
+	}
+	if params.Version == "" {
+		params.Version = meta.Version
+	}
+	return cleanup, nil
+}
+
+// requireOrgNamespace ensures ns belongs to the project owner's private org, so
+// an artifact encrypted under that org's key is published under that same org
+// and stays decryptable by its consumers.
+func requireOrgNamespace(ns, owner string) error {
+	org := model.NewNamespaceOrg(owner, "").String()
+	if ns == org || strings.HasPrefix(ns, org+"/") {
+		return nil
+	}
+	return locale.NewInputError("err_publish_build_namespace",
+		"The '[ACTIONABLE]--build[/RESET]' flag requires a namespace under '[ACTIONABLE]{{.V0}}[/RESET]'.", org)
+}
+
+// payloadInstallDir is the directory inside the wrapped artifact that holds the
+// deployable payload; the cleartext runtime.json points the consume side at it.
+const payloadInstallDir = "install"
+
+// buildWrappedArtifact packs srcDir into a wheel under the given metadata,
+// encrypts it under the org key, and wraps the ciphertext together with a
+// cleartext runtime.json into a tar.gz ready for upload. It returns the wrapped
+// archive path and a cleanup function the caller must invoke once the upload is
+// done.
+//
+// Only ciphertext plus the cleartext envdef ever reaches the wrapped archive:
+// the plaintext wheel and payload are removed before the function returns, so no
+// plaintext outlives the build.
+func buildWrappedArtifact(srcDir string, meta wheel.Metadata, key []byte, keyID string) (archivePath string, cleanup func(), rerr error) {
+	tmpDir, err := os.MkdirTemp("", "state-publish-build-")
+	if err != nil {
+		return "", nil, errs.Wrap(err, "Could not create temp dir")
+	}
+	cleanup = func() { _ = os.RemoveAll(tmpDir) }
+	defer func() {
+		if rerr != nil {
+			cleanup()
+		}
+	}()
+
+	wheelPath, err := wheel.Pack(srcDir, meta, tmpDir)
+	if err != nil {
+		return "", nil, errs.Wrap(err, "Could not build a wheel from %s", srcDir)
+	}
+
+	// Assemble the tar.gz that becomes the encrypted payload, placing the wheel
+	// under the install dir the consume side deploys.
+	plaintextPayload := filepath.Join(tmpDir, "payload.tar.gz")
+	if err := archiver.CreateTgz(plaintextPayload, tmpDir, []archiver.FileMap{
+		{Source: wheelPath, Target: path.Join(payloadInstallDir, filepath.Base(wheelPath))},
+	}); err != nil {
+		return "", nil, errs.Wrap(err, "Could not assemble payload")
+	}
+
+	ciphertextPath := filepath.Join(tmpDir, "payload.enc")
+	if err := encryptFile(plaintextPayload, ciphertextPath, key, keyID); err != nil {
+		return "", nil, errs.Wrap(err, "Could not encrypt payload")
+	}
+
+	// Drop the plaintext now that only ciphertext is needed; nothing plaintext
+	// survives into the wrapped artifact or beyond this point.
+	for _, p := range []string{wheelPath, plaintextPayload} {
+		if err := os.Remove(p); err != nil {
+			return "", nil, errs.Wrap(err, "Could not remove plaintext")
+		}
+	}
+
+	runtimeJSONPath := filepath.Join(tmpDir, "runtime.json")
+	if err := writeRuntimeJSON(runtimeJSONPath); err != nil {
+		return "", nil, errs.Wrap(err, "Could not write runtime.json")
+	}
+
+	archivePath = filepath.Join(tmpDir, "ingredient.tar.gz")
+	if err := archiver.CreateTgz(archivePath, tmpDir, []archiver.FileMap{
+		{Source: ciphertextPath, Target: "payload.enc"},
+		{Source: runtimeJSONPath, Target: "runtime.json"},
+	}); err != nil {
+		return "", nil, errs.Wrap(err, "Could not wrap artifact")
+	}
+
+	if sha, err := fileutils.Sha256Hash(archivePath); err == nil {
+		logging.Debug("Built private ingredient artifact %s (sha256=%s)", filepath.Base(archivePath), sha)
+	}
+
+	return archivePath, cleanup, nil
+}
+
+// encryptFile streams srcPath through the content-encryption package into a new
+// file at dstPath under the given key.
+func encryptFile(srcPath, dstPath string, key []byte, keyID string) (rerr error) {
+	src, err := os.Open(srcPath)
+	if err != nil {
+		return errs.Wrap(err, "Could not open payload")
+	}
+	defer func() {
+		if cerr := src.Close(); cerr != nil {
+			rerr = errs.Pack(rerr, errs.Wrap(cerr, "Could not close payload"))
+		}
+	}()
+
+	dst, err := os.Create(dstPath)
+	if err != nil {
+		return errs.Wrap(err, "Could not create ciphertext")
+	}
+	defer func() {
+		if cerr := dst.Close(); cerr != nil {
+			rerr = errs.Pack(rerr, errs.Wrap(cerr, "Could not close ciphertext"))
+		}
+	}()
+
+	if err := artifactcrypto.Encrypt(src, dst, key, keyID); err != nil {
+		return errs.Wrap(err, "Could not encrypt")
+	}
+	return nil
+}
+
+// writeRuntimeJSON writes the minimal cleartext envdef the consume side reads to
+// deploy the decrypted payload.
+func writeRuntimeJSON(destPath string) error {
+	def := struct {
+		Env        []json.RawMessage `json:"env"`
+		Transforms []json.RawMessage `json:"file_transforms"`
+		InstallDir string            `json:"installdir"`
+	}{
+		Env:        []json.RawMessage{},
+		Transforms: []json.RawMessage{},
+		InstallDir: payloadInstallDir,
+	}
+	b, err := json.Marshal(def)
+	if err != nil {
+		return errs.Wrap(err, "Could not marshal runtime.json")
+	}
+	if err := os.WriteFile(destPath, b, 0644); err != nil {
+		return errs.Wrap(err, "Could not write runtime.json")
+	}
+	return nil
+}