From 2f47bf79979e75208f4519b317ac11bf995a67cb Mon Sep 17 00:00:00 2001 From: mitchell Date: Thu, 25 Jun 2026 15:06:08 -0400 Subject: [PATCH 1/7] Add e2e integration test for encrypted publish and install (ENG-1643) TestPublishBuildEncrypted drives the full private-ingredient round trip: `state publish --build` packs a pure-Python source tree into a wheel, encrypts it under the org key, and publishes it under a unique random name; `state install --ts=now` then resolves, downloads, and decrypts it. The test proves decryption by reading the decrypted wheel back out of the depot and finding a sentinel string that only the plaintext source contains. To avoid standing up an HTTPS key service, the org key is supplied to both the publish (encrypt) and install (decrypt) sides through a new environment variable carrying the key-service contract JSON. The orgkey provider reads it only after Configured() passes and validates it through the same path as a fetched contract, so org-binding, fingerprint, and key-length checks all still apply. The contract is stripped from child process environments. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/constants/constants.go | 7 ++ internal/runbits/orgkey/env.go | 1 + internal/runbits/orgkey/https.go | 17 +++- test/integration/publish_int_test.go | 144 +++++++++++++++++++++++++++ 4 files changed, 168 insertions(+), 1 deletion(-) diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 846df9b818..8b78e3892c 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -557,3 +557,10 @@ const PrivateIngredientBearerTokenFileConfig = "privateingredient.bearer_token_f // PrivateIngredientCacheKeyConfig is the config key that opts into caching the // fetched org key on disk (0600) for headless/offline/CI reuse. const PrivateIngredientCacheKeyConfig = "privateingredient.cache_key_on_disk" + +// PrivateIngredientKeyContractEnvVarName is the name of an environment variable +// that may carry an org-key contract (the same JSON document the HTTPS key +// service serves). When set, the org key is read from it and validated exactly +// like a fetched contract, bypassing the network. It exists for development and +// integration testing where standing up a TLS key service is impractical. +const PrivateIngredientKeyContractEnvVarName = "ACTIVESTATE_CLI_PRIVATE_INGREDIENT_KEY_CONTRACT" diff --git a/internal/runbits/orgkey/env.go b/internal/runbits/orgkey/env.go index b57e193002..3f1c7245d5 100644 --- a/internal/runbits/orgkey/env.go +++ b/internal/runbits/orgkey/env.go @@ -13,4 +13,5 @@ func SanitizeChildEnv(cfg stringConfigReader, env map[string]string) { if tokenEnv := cfg.GetString(constants.PrivateIngredientBearerTokenEnvConfig); tokenEnv != "" { delete(env, tokenEnv) } + delete(env, constants.PrivateIngredientKeyContractEnvVarName) } diff --git a/internal/runbits/orgkey/https.go b/internal/runbits/orgkey/https.go index b24913d80a..b168b12169 100644 --- a/internal/runbits/orgkey/https.go +++ b/internal/runbits/orgkey/https.go @@ -43,7 +43,18 @@ func New(cfg configurable, owner string) Provider { } func (p *provider) Configured() bool { - return p.cfg.GetString(constants.PrivateIngredientKeyServiceURLConfig) != "" + return p.cfg.GetString(constants.PrivateIngredientKeyServiceURLConfig) != "" || len(envContract()) > 0 +} + +// envContract returns an org-key contract supplied directly via the environment, +// or nil if none is set. This bypasses the HTTPS key service for development and +// integration testing; the contract is still validated like one fetched from the +// service (see load). +func envContract() []byte { + if v := os.Getenv(constants.PrivateIngredientKeyContractEnvVarName); v != "" { + return []byte(v) + } + return nil } // Key fetches and validates the org key on first call and returns the cached @@ -73,6 +84,10 @@ func (p *provider) load(ctx context.Context) (key []byte, keyID string, err erro return nil, "", ErrNotConfigured } + if raw := envContract(); len(raw) > 0 { + return validateContract(raw, p.owner) + } + if p.diskCacheEnabled() { if raw, ok := p.readDiskCache(); ok { if key, keyID, err := validateContract(raw, p.owner); err == nil { diff --git a/test/integration/publish_int_test.go b/test/integration/publish_int_test.go index 6c2a0ea79f..01ffd6782d 100644 --- a/test/integration/publish_int_test.go +++ b/test/integration/publish_int_test.go @@ -1,15 +1,22 @@ package integration import ( + "archive/zip" + "encoding/base64" + "encoding/json" "fmt" + "io" "os" + "path/filepath" "regexp" + "strings" "testing" "time" "github.com/google/uuid" "gopkg.in/yaml.v2" + "github.com/ActiveState/cli/internal/artifactcrypto" "github.com/ActiveState/cli/internal/constants" "github.com/ActiveState/cli/internal/fileutils" "github.com/ActiveState/cli/internal/rtutils/ptr" @@ -472,6 +479,143 @@ authors: ts.IgnoreLogErrors() // ignore intentional failures like omitted filename, cannot edit description, etc. } +// TestPublishBuildEncrypted exercises the full encrypted private-ingredient round +// trip: `state publish --build` packs a pure-Python source tree into a wheel, +// encrypts it under the org key, and publishes it; then `state install` resolves, +// downloads, and decrypts it. We prove decryption succeeded by reading the +// decrypted wheel back out of the depot and finding a sentinel string that only +// our plaintext source contains. +// +// The org key is supplied to both the publish (encrypt) and install (decrypt) +// sides through the environment, so the test needs no HTTPS key service. +// +// The ingredient is published under a unique, random name. Published private +// ingredients cannot be deleted, so the name must never collide across runs. +func (suite *PublishIntegrationTestSuite) TestPublishBuildEncrypted() { + suite.OnlyRunForTags(tagsuite.Publish) + + ts := e2e.New(suite.T(), false) + defer ts.Close() + + if apiHost := os.Getenv(constants.APIHostEnvVarName); apiHost != "" { + ts.Env = append(ts.Env, constants.APIHostEnvVarName+"="+apiHost) + } + + ts.LoginAsPersistentUser() + + // Supply the org key to publish (encrypt) and install (decrypt) via the + // environment, avoiding an HTTPS key service. The contract is validated just + // like one fetched from a real service, including its binding to this org. + key := make([]byte, artifactcrypto.KeySize) + for i := range key { + key[i] = byte(i + 1) + } + ts.Env = append(ts.Env, + constants.PrivateIngredientKeyContractEnvVarName+"="+orgKeyContract(suite, key, e2e.PersistentUsername)) + + // A pure-Python source tree carrying a unique sentinel. After install we read + // the decrypted wheel back out of the depot and look for the sentinel — + // ciphertext could never yield a valid wheel containing it. + sentinel := "private-ingredient-sentinel-" + strutils.UUID().String() + srcDir := filepath.Join(ts.Dirs.Work, "ingredient-src") + suite.Require().NoError(os.MkdirAll(filepath.Join(srcDir, "greeting"), 0755)) + suite.Require().NoError(fileutils.WriteFile( + filepath.Join(srcDir, "greeting", "__init__.py"), + []byte(fmt.Sprintf("print(%q)\n", sentinel)), + )) + + // Create a fresh project under the testing org. `state publish --build` + // requires a project (to determine the org its key encrypts under), and the + // publish namespace must live under that same org. + projectName := strutils.UUID() + projectNamespace := fmt.Sprintf("%s/%s", e2e.PersistentUsername, projectName) + cp := ts.SpawnWithOpts(e2e.OptArgs("init", "--language", "python", projectNamespace, ts.Dirs.Work)) + cp.Expect("Initializing Project") + cp.Expect("has been successfully initialized", e2e.RuntimeSourcingTimeoutOpt) + cp.ExpectExitCode(0) + ts.NotifyProjectCreated(e2e.PersistentUsername, projectName.String()) + + // Build, encrypt, and publish the private ingredient under a unique name. + ingredientName := strutils.UUID().String() + ingredientNamespace := "private/" + e2e.PersistentUsername + "/language/python" + cp = ts.SpawnWithOpts(e2e.OptArgs( + "publish", "--non-interactive", + "--build", srcDir, + "--namespace", ingredientNamespace, + "--name", ingredientName, + "--version", "0.0.1", + )) + cp.Expect("Successfully published") + cp.ExpectExitCode(0) + + // Install the freshly published ingredient, forcing resolution at the current + // timestamp so the new revision is picked up rather than a cached solve. + cp = ts.SpawnWithOpts(e2e.OptArgs( + "install", ingredientNamespace+":"+ingredientName, "--ts=now", + )) + cp.Expect("All dependencies have been installed and verified", e2e.RuntimeBuildSourcingTimeoutOpt) + cp.ExpectExitCode(0) + + // Decryption proof: the decrypted wheel must be present in the depot and + // contain our sentinel. A failed decrypt would skip the artifact, leaving no + // wheel behind. + suite.assertDecryptedWheelContains(ts, sentinel) +} + +// orgKeyContract builds the org-key contract JSON the key service would serve for +// the given key and organization, for injection via the environment. +func orgKeyContract(suite *PublishIntegrationTestSuite, key []byte, org string) string { + contract := map[string]string{ + "schema": "activestate.pim.orgkey/v1", + "org": org, + "key_id": "integration-test-key", + "algorithm": "AES-256-GCM", + "encoding": "base64", + "key": base64.StdEncoding.EncodeToString(key), + "fingerprint": artifactcrypto.Fingerprint(key), + } + b, err := json.Marshal(contract) + suite.Require().NoError(err) + return string(b) +} + +// assertDecryptedWheelContains finds the decrypted private-ingredient wheel(s) in +// the depot and asserts that one is a valid wheel containing sentinel — proof the +// consume side decrypted the artifact rather than skipping it. +func (suite *PublishIntegrationTestSuite) assertDecryptedWheelContains(ts *e2e.Session, sentinel string) { + matches, err := filepath.Glob(filepath.Join(ts.Dirs.Cache, "depot", "*", "install", "*.whl")) + suite.Require().NoError(err) + suite.Require().NotEmpty(matches, "no decrypted wheel found in the depot; the artifact was likely not decrypted") + + for _, wheelPath := range matches { + if wheelContains(suite, wheelPath, sentinel) { + return + } + } + suite.Fail(fmt.Sprintf("sentinel %q not found in any decrypted wheel under the depot", sentinel)) +} + +// wheelContains reports whether any file inside the wheel (a zip) contains +// sentinel. It fails the test if the wheel is not a readable zip, since a wheel +// that did not decrypt would not be a valid archive. +func wheelContains(suite *PublishIntegrationTestSuite, wheelPath, sentinel string) bool { + zr, err := zip.OpenReader(wheelPath) + suite.Require().NoError(err, "decrypted wheel is not a valid zip: %s", wheelPath) + defer zr.Close() + + for _, f := range zr.File { + rc, err := f.Open() + suite.Require().NoError(err) + content, err := io.ReadAll(rc) + rc.Close() + suite.Require().NoError(err) + if strings.Contains(string(content), sentinel) { + return true + } + } + return false +} + func TestPublishIntegrationTestSuite(t *testing.T) { suite.Run(t, new(PublishIntegrationTestSuite)) } From 0551730306fd53beee7dac3ce39ae88087dee3b8 Mon Sep 17 00:00:00 2001 From: mitchell Date: Thu, 25 Jun 2026 15:34:14 -0400 Subject: [PATCH 2/7] Inline the decryption check and broaden the depot search (ENG-1643) The CI failure surfaced no diagnostics because the assertion lived in an inner helper. Inline the check into the test body so a failure dumps the spawned command output and state logs. Also search the whole depot (any wheel or file) for the sentinel rather than a fixed install/*.whl glob, and log the depot's top two levels, since the on-disk layout depends on how the artifact is packaged on install. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/integration/publish_int_test.go | 117 +++++++++++++++++---------- 1 file changed, 76 insertions(+), 41 deletions(-) diff --git a/test/integration/publish_int_test.go b/test/integration/publish_int_test.go index 01ffd6782d..bb1a8b5730 100644 --- a/test/integration/publish_int_test.go +++ b/test/integration/publish_int_test.go @@ -556,10 +556,82 @@ func (suite *PublishIntegrationTestSuite) TestPublishBuildEncrypted() { cp.Expect("All dependencies have been installed and verified", e2e.RuntimeBuildSourcingTimeoutOpt) cp.ExpectExitCode(0) - // Decryption proof: the decrypted wheel must be present in the depot and - // contain our sentinel. A failed decrypt would skip the artifact, leaving no - // wheel behind. - suite.assertDecryptedWheelContains(ts, sentinel) + // Decryption proof: the decrypted content must be present in the depot and + // contain our sentinel. A failed decrypt would skip the artifact, leaving the + // sentinel absent. This is inlined (rather than in a helper) so a failure + // surfaces the spawned-command output and state logs the e2e harness dumps. + // The search is deliberately broad — any wheel or file anywhere under the + // depot — because the exact on-disk path depends on how the artifact is + // packaged on install. + depot := filepath.Join(ts.Dirs.Cache, "depot") + + // Log the depot's top two levels so a failure shows how the artifact landed + // (e.g. an "install" dir with a wheel vs. a leftover "payload.enc"). + if entries, err := os.ReadDir(depot); err == nil { + for _, e := range entries { + suite.T().Logf("depot/%s", e.Name()) + if e.IsDir() { + if sub, err := os.ReadDir(filepath.Join(depot, e.Name())); err == nil { + for _, s := range sub { + suite.T().Logf("depot/%s/%s", e.Name(), s.Name()) + } + } + } + } + } else { + suite.T().Logf("could not read depot %s: %v", depot, err) + } + + var wheels []string + fileCount := 0 + found := false + walkErr := filepath.WalkDir(depot, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + fileCount++ + + // A decrypted wheel is a zip; scan its entries for the sentinel. + if strings.HasSuffix(d.Name(), ".whl") { + wheels = append(wheels, path) + zr, err := zip.OpenReader(path) + if err != nil { + suite.T().Logf("could not open wheel %s as zip: %v", path, err) + return nil + } + defer zr.Close() + for _, f := range zr.File { + rc, err := f.Open() + if err != nil { + continue + } + content, _ := io.ReadAll(rc) + rc.Close() + if strings.Contains(string(content), sentinel) { + found = true + } + } + return nil + } + + // Otherwise scan the raw file, in case the payload was delivered unpacked + // rather than as a wheel. Skip large files (the sentinel lives in a tiny + // Python source file). + if info, err := d.Info(); err != nil || info.Size() > 5<<20 { + return nil + } + content, err := os.ReadFile(path) + if err == nil && strings.Contains(string(content), sentinel) { + found = true + } + return nil + }) + suite.Require().NoError(walkErr, "could not walk depot %s", depot) + suite.T().Logf("searched %d files under the depot; wheels found: %v", fileCount, wheels) + suite.Require().True(found, "sentinel %q not found in the depot; the artifact was likely not decrypted", sentinel) } // orgKeyContract builds the org-key contract JSON the key service would serve for @@ -579,43 +651,6 @@ func orgKeyContract(suite *PublishIntegrationTestSuite, key []byte, org string) return string(b) } -// assertDecryptedWheelContains finds the decrypted private-ingredient wheel(s) in -// the depot and asserts that one is a valid wheel containing sentinel — proof the -// consume side decrypted the artifact rather than skipping it. -func (suite *PublishIntegrationTestSuite) assertDecryptedWheelContains(ts *e2e.Session, sentinel string) { - matches, err := filepath.Glob(filepath.Join(ts.Dirs.Cache, "depot", "*", "install", "*.whl")) - suite.Require().NoError(err) - suite.Require().NotEmpty(matches, "no decrypted wheel found in the depot; the artifact was likely not decrypted") - - for _, wheelPath := range matches { - if wheelContains(suite, wheelPath, sentinel) { - return - } - } - suite.Fail(fmt.Sprintf("sentinel %q not found in any decrypted wheel under the depot", sentinel)) -} - -// wheelContains reports whether any file inside the wheel (a zip) contains -// sentinel. It fails the test if the wheel is not a readable zip, since a wheel -// that did not decrypt would not be a valid archive. -func wheelContains(suite *PublishIntegrationTestSuite, wheelPath, sentinel string) bool { - zr, err := zip.OpenReader(wheelPath) - suite.Require().NoError(err, "decrypted wheel is not a valid zip: %s", wheelPath) - defer zr.Close() - - for _, f := range zr.File { - rc, err := f.Open() - suite.Require().NoError(err) - content, err := io.ReadAll(rc) - rc.Close() - suite.Require().NoError(err) - if strings.Contains(string(content), sentinel) { - return true - } - } - return false -} - func TestPublishIntegrationTestSuite(t *testing.T) { suite.Run(t, new(PublishIntegrationTestSuite)) } From 0a8e9bc98498dd629d6ce64cd276a6319033594d Mon Sep 17 00:00:00 2001 From: mitchell Date: Thu, 25 Jun 2026 15:49:15 -0400 Subject: [PATCH 3/7] Temporarily induce a failure to try and get terminal output and logs. --- test/integration/publish_int_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/publish_int_test.go b/test/integration/publish_int_test.go index bb1a8b5730..9571eaf85f 100644 --- a/test/integration/publish_int_test.go +++ b/test/integration/publish_int_test.go @@ -554,7 +554,7 @@ func (suite *PublishIntegrationTestSuite) TestPublishBuildEncrypted() { "install", ingredientNamespace+":"+ingredientName, "--ts=now", )) cp.Expect("All dependencies have been installed and verified", e2e.RuntimeBuildSourcingTimeoutOpt) - cp.ExpectExitCode(0) + cp.ExpectExitCode(1) // Decryption proof: the decrypted content must be present in the depot and // contain our sentinel. A failed decrypt would skip the artifact, leaving the From 5b5f35daf5ab204e0de00dd9c6549554241eb49f Mon Sep 17 00:00:00 2001 From: mitchell Date: Thu, 25 Jun 2026 16:52:48 -0400 Subject: [PATCH 4/7] Build private ingredients with the private-builder (ENG-1643) A --build publish now declares a dependency on builder/private-builder. Without a builder dependency the platform falls back to the noop-builder, which ingests the upload but produces an empty artifact, leaving nothing installable on the consume side. The wrapped tar.gz now carries only the encrypted payload; the private-builder generates the runtime.json on ingest, so we no longer write or wrap one. The build unit test is updated to match. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/runners/publish/build.go | 56 ++++++-------------------- internal/runners/publish/build_test.go | 22 +++------- internal/runners/publish/publish.go | 13 ++++++ 3 files changed, 31 insertions(+), 60 deletions(-) diff --git a/internal/runners/publish/build.go b/internal/runners/publish/build.go index 8433dabf9b..007128d0c6 100644 --- a/internal/runners/publish/build.go +++ b/internal/runners/publish/build.go @@ -2,9 +2,7 @@ package publish import ( "context" - "encoding/json" "os" - "path" "path/filepath" "strings" @@ -79,19 +77,19 @@ func requireOrgNamespace(ns, owner string) error { "The '[ACTIONABLE]--build[/RESET]' flag requires a namespace under '[ACTIONABLE]{{.V0}}[/RESET]'.", org) } -// payloadInstallDir is the directory inside the wrapped artifact that holds the -// deployable payload; the cleartext runtime.json points the consume side at it. -const payloadInstallDir = "install" +const ( + privateBuilderNamespace = "builder" + privateBuilderName = "private-builder" +) // buildWrappedArtifact packs srcDir into a wheel under the given metadata, -// encrypts it under the org key, and wraps the ciphertext together with a -// cleartext runtime.json into a tar.gz ready for upload. It returns the wrapped -// archive path and a cleanup function the caller must invoke once the upload is -// done. +// encrypts it under the org key, and wraps the ciphertext in a tar.gz ready for +// upload. It returns the wrapped archive path and a cleanup function the caller +// must invoke once the upload is done. // -// Only ciphertext plus the cleartext envdef ever reaches the wrapped archive: -// the plaintext wheel and payload are removed before the function returns, so no -// plaintext outlives the build. +// Only the ciphertext ever reaches the wrapped archive: the plaintext wheel and +// payload are removed before the function returns, so no plaintext outlives the +// build. func buildWrappedArtifact(srcDir string, meta wheel.Metadata, key []byte, keyID string) (archivePath string, cleanup func(), rerr error) { tmpDir, err := os.MkdirTemp("", "state-publish-build-") if err != nil { @@ -109,11 +107,11 @@ func buildWrappedArtifact(srcDir string, meta wheel.Metadata, key []byte, keyID return "", nil, errs.Wrap(err, "Could not build a wheel from %s", srcDir) } - // Assemble the tar.gz that becomes the encrypted payload, placing the wheel - // under the install dir the consume side deploys. + // Assemble the tar.gz that becomes the encrypted payload, with the wheel at + // its root. plaintextPayload := filepath.Join(tmpDir, "payload.tar.gz") if err := archiver.CreateTgz(plaintextPayload, tmpDir, []archiver.FileMap{ - {Source: wheelPath, Target: path.Join(payloadInstallDir, filepath.Base(wheelPath))}, + {Source: wheelPath, Target: filepath.Base(wheelPath)}, }); err != nil { return "", nil, errs.Wrap(err, "Could not assemble payload") } @@ -131,15 +129,9 @@ func buildWrappedArtifact(srcDir string, meta wheel.Metadata, key []byte, keyID } } - runtimeJSONPath := filepath.Join(tmpDir, "runtime.json") - if err := writeRuntimeJSON(runtimeJSONPath); err != nil { - return "", nil, errs.Wrap(err, "Could not write runtime.json") - } - archivePath = filepath.Join(tmpDir, "ingredient.tar.gz") if err := archiver.CreateTgz(archivePath, tmpDir, []archiver.FileMap{ {Source: ciphertextPath, Target: "payload.enc"}, - {Source: runtimeJSONPath, Target: "runtime.json"}, }); err != nil { return "", nil, errs.Wrap(err, "Could not wrap artifact") } @@ -179,25 +171,3 @@ func encryptFile(srcPath, dstPath string, key []byte, keyID string) (rerr error) } return nil } - -// writeRuntimeJSON writes the minimal cleartext envdef the consume side reads to -// deploy the decrypted payload. -func writeRuntimeJSON(destPath string) error { - def := struct { - Env []json.RawMessage `json:"env"` - Transforms []json.RawMessage `json:"file_transforms"` - InstallDir string `json:"installdir"` - }{ - Env: []json.RawMessage{}, - Transforms: []json.RawMessage{}, - InstallDir: payloadInstallDir, - } - b, err := json.Marshal(def) - if err != nil { - return errs.Wrap(err, "Could not marshal runtime.json") - } - if err := os.WriteFile(destPath, b, 0644); err != nil { - return errs.Wrap(err, "Could not write runtime.json") - } - return nil -} diff --git a/internal/runners/publish/build_test.go b/internal/runners/publish/build_test.go index 292ef969ee..94ad3b11ca 100644 --- a/internal/runners/publish/build_test.go +++ b/internal/runners/publish/build_test.go @@ -4,7 +4,6 @@ import ( "archive/tar" "bytes" "compress/gzip" - "encoding/json" "io" "os" "path/filepath" @@ -110,23 +109,12 @@ func TestBuildWrappedArtifact(t *testing.T) { } defer cleanup() - // The wrapped archive contains exactly the ciphertext and the cleartext envdef. + // The wrapped archive contains exactly the ciphertext. entries := readTarGz(t, archivePath) - if got, want := keysOf(entries), []string{"payload.enc", "runtime.json"}; !equalStrings(got, want) { + if got, want := keysOf(entries), []string{"payload.enc"}; !equalStrings(got, want) { t.Fatalf("wrapped entries = %v, want %v", got, want) } - // runtime.json is the minimal envdef pointing at the install dir. - var def struct { - InstallDir string `json:"installdir"` - } - if err := json.Unmarshal(entries["runtime.json"], &def); err != nil { - t.Fatal(err) - } - if def.InstallDir != payloadInstallDir { - t.Errorf("installdir = %q, want %q", def.InstallDir, payloadInstallDir) - } - // The payload is encrypted before it is wrapped. enc, err := artifactcrypto.IsEncrypted(bytes.NewReader(entries["payload.enc"])) if err != nil { @@ -143,19 +131,19 @@ func TestBuildWrappedArtifact(t *testing.T) { } } - // The ciphertext decrypts to a tar.gz holding the wheel under the install dir. + // The ciphertext decrypts to a tar.gz holding the wheel. innerPath := filepath.Join(t.TempDir(), "inner.tar.gz") if err := artifactcrypto.Decrypt(bytes.NewReader(entries["payload.enc"]), innerPath, key); err != nil { t.Fatalf("Decrypt: %v", err) } foundWheel := false for _, name := range keysOf(readTarGz(t, innerPath)) { - if strings.HasPrefix(name, payloadInstallDir+"/") && strings.HasSuffix(name, ".whl") { + if strings.HasSuffix(name, ".whl") { foundWheel = true } } if !foundWheel { - t.Error("decrypted payload does not contain the wheel under the install dir") + t.Error("decrypted payload does not contain the wheel") } // cleanup removes the build temp dir. diff --git a/internal/runners/publish/publish.go b/internal/runners/publish/publish.go index d9a00403c3..a0eaf03353 100644 --- a/internal/runners/publish/publish.go +++ b/internal/runners/publish/publish.go @@ -222,6 +222,19 @@ func (r *Runner) Run(params *Params) error { return errs.Wrap(err, "Could not prepare request from params") } + if params.Build != "" { + // A --build publish must be ingested by the private-builder. Without a + // builder dependency the platform falls back to the noop-builder, which + // produces an empty artifact. + reqVars.Dependencies = append(reqVars.Dependencies, request.PublishVariableDep{ + Dependency: request.Dependency{ + Name: privateBuilderName, + Namespace: privateBuilderNamespace, + VersionRequirements: ">=0", + }, + }) + } + if params.Editor { if !r.out.Config().Interactive { return locale.NewInputError("err_uploadingredient_editor_not_supported", "Opening in editor is not supported in non-interactive mode") From 0b1c200e0c7f9c05c82f1950e4adebbb49c5f4ea Mon Sep 17 00:00:00 2001 From: mitchell Date: Thu, 25 Jun 2026 17:34:29 -0400 Subject: [PATCH 5/7] Revert "Temporarily induce a failure to try and get terminal output and logs." This reverts commit 0a8e9bc98498dd629d6ce64cd276a6319033594d. --- test/integration/publish_int_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/publish_int_test.go b/test/integration/publish_int_test.go index 9571eaf85f..bb1a8b5730 100644 --- a/test/integration/publish_int_test.go +++ b/test/integration/publish_int_test.go @@ -554,7 +554,7 @@ func (suite *PublishIntegrationTestSuite) TestPublishBuildEncrypted() { "install", ingredientNamespace+":"+ingredientName, "--ts=now", )) cp.Expect("All dependencies have been installed and verified", e2e.RuntimeBuildSourcingTimeoutOpt) - cp.ExpectExitCode(1) + cp.ExpectExitCode(0) // Decryption proof: the decrypted content must be present in the depot and // contain our sentinel. A failed decrypt would skip the artifact, leaving the From 3f9da0241172c36339c0ecf55140ae753f76859a Mon Sep 17 00:00:00 2001 From: mitchell Date: Thu, 25 Jun 2026 18:33:28 -0400 Subject: [PATCH 6/7] Decrypt private payloads nested under the install dir (ENG-1643) The private-builder nests the encrypted payload under the artifact's install directory rather than leaving it at the top level, so the consume side never found it: the search only scanned top-level files, and even once found the decrypted archive was extracted to the artifact root instead of where the runtime.json points. The result was an installed-but-never-decrypted artifact. findEncryptedPayload now searches the artifact recursively, locating the payload by its conventional name and confirming the envelope magic, and the decrypted archive is extracted alongside the ciphertext so it lands under the install dir. The payload filename is now a single shared constant (artifactcrypto.PayloadFilename) used by both the publish (produce) and runtime (consume) sides, so it can no longer drift between them. The integration test's decryption check is factored into reusable suite helpers. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/artifactcrypto/artifactcrypto.go | 4 ++ internal/runners/publish/build.go | 4 +- pkg/runtime/decrypt_test.go | 66 ++++++++++++++++++-- pkg/runtime/setup.go | 47 +++++++++------ test/integration/publish_int_test.go | 73 +++++++++++------------ 5 files changed, 130 insertions(+), 64 deletions(-) diff --git a/internal/artifactcrypto/artifactcrypto.go b/internal/artifactcrypto/artifactcrypto.go index dd595550c3..49996b5bda 100644 --- a/internal/artifactcrypto/artifactcrypto.go +++ b/internal/artifactcrypto/artifactcrypto.go @@ -54,6 +54,10 @@ const ( formatVersion = 1 // DefaultChunkSize is the plaintext size of every chunk except the last. DefaultChunkSize = 1 << 20 // 1 MiB + // PayloadFilename is the conventional name of the encrypted payload file + // inside a private-ingredient artifact. The publish (produce) and runtime + // (consume) sides share it so they agree on what to write and look for. + PayloadFilename = "payload.enc" // maxChunkSize is the largest chunk size accepted from a parsed header. maxChunkSize = 64 << 20 // 64 MiB // maxHeaderLen is the largest serialized header accepted from a stream. diff --git a/internal/runners/publish/build.go b/internal/runners/publish/build.go index 007128d0c6..4c2881534c 100644 --- a/internal/runners/publish/build.go +++ b/internal/runners/publish/build.go @@ -116,7 +116,7 @@ func buildWrappedArtifact(srcDir string, meta wheel.Metadata, key []byte, keyID return "", nil, errs.Wrap(err, "Could not assemble payload") } - ciphertextPath := filepath.Join(tmpDir, "payload.enc") + ciphertextPath := filepath.Join(tmpDir, artifactcrypto.PayloadFilename) if err := encryptFile(plaintextPayload, ciphertextPath, key, keyID); err != nil { return "", nil, errs.Wrap(err, "Could not encrypt payload") } @@ -131,7 +131,7 @@ func buildWrappedArtifact(srcDir string, meta wheel.Metadata, key []byte, keyID archivePath = filepath.Join(tmpDir, "ingredient.tar.gz") if err := archiver.CreateTgz(archivePath, tmpDir, []archiver.FileMap{ - {Source: ciphertextPath, Target: "payload.enc"}, + {Source: ciphertextPath, Target: artifactcrypto.PayloadFilename}, }); err != nil { return "", nil, errs.Wrap(err, "Could not wrap artifact") } diff --git a/pkg/runtime/decrypt_test.go b/pkg/runtime/decrypt_test.go index 075726f1c7..14facf07e1 100644 --- a/pkg/runtime/decrypt_test.go +++ b/pkg/runtime/decrypt_test.go @@ -72,6 +72,9 @@ func makeTarGz(t *testing.T, files, symlinks map[string]string) []byte { func writeFile(t *testing.T, path string, data []byte) { t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } if err := os.WriteFile(path, data, 0644); err != nil { t.Fatal(err) } @@ -96,7 +99,7 @@ func TestFindEncryptedPayload(t *testing.T) { t.Run("finds the encrypted file", func(t *testing.T) { dir := t.TempDir() writeFile(t, filepath.Join(dir, "runtime.json"), []byte(`{"installDir":"."}`)) - payload := filepath.Join(dir, "anything.bin") + payload := filepath.Join(dir, artifactcrypto.PayloadFilename) writeFile(t, payload, encryptToBytes(t, []byte("secret"), key)) got, err := findEncryptedPayload(dir) if err != nil { @@ -106,6 +109,35 @@ func TestFindEncryptedPayload(t *testing.T) { t.Errorf("got %q, want %q", got, payload) } }) + + // Installed payload is under top-level "installdir/" directory. + t.Run("finds encrypted payload", func(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "runtime.json"), []byte(`{"installDir":"installdir"}`)) + payload := filepath.Join(dir, "installdir", artifactcrypto.PayloadFilename) + writeFile(t, payload, encryptToBytes(t, []byte("secret"), key)) + got, err := findEncryptedPayload(dir) + if err != nil { + t.Fatal(err) + } + if got != payload { + t.Errorf("got %q, want %q", got, payload) + } + }) + + // A plaintext file that happens to be named payload.enc is not treated as a + // payload. + t.Run("ignores a plaintext payload.enc", func(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, artifactcrypto.PayloadFilename), []byte("not encrypted")) + got, err := findEncryptedPayload(dir) + if err != nil { + t.Fatal(err) + } + if got != "" { + t.Errorf("found a payload for a plaintext file: %q", got) + } + }) } func TestDecryptPayload(t *testing.T) { @@ -123,7 +155,7 @@ func TestDecryptPayload(t *testing.T) { t.Run("happy path", func(t *testing.T) { dir := t.TempDir() writeFile(t, filepath.Join(dir, "runtime.json"), []byte(`{"installDir":"."}`)) - writeFile(t, filepath.Join(dir, "payload"), encryptToBytes(t, payload, key)) + writeFile(t, filepath.Join(dir, artifactcrypto.PayloadFilename), encryptToBytes(t, payload, key)) s := &setup{opts: &Opts{OrgKey: key}} outcome, err := s.decryptPayload("pkg", dir) @@ -134,7 +166,7 @@ func TestDecryptPayload(t *testing.T) { t.Fatalf("outcome = %v, want decryptDone", outcome) } // Ciphertext is removed. - if _, err := os.Stat(filepath.Join(dir, "payload")); !os.IsNotExist(err) { + if _, err := os.Stat(filepath.Join(dir, artifactcrypto.PayloadFilename)); !os.IsNotExist(err) { t.Error("ciphertext was not removed") } // Archive contents extracted in place; the cleartext runtime.json survives. @@ -163,7 +195,7 @@ func TestDecryptPayload(t *testing.T) { t.Run("missing key skips", func(t *testing.T) { dir := t.TempDir() - writeFile(t, filepath.Join(dir, "payload"), encryptToBytes(t, payload, key)) + writeFile(t, filepath.Join(dir, artifactcrypto.PayloadFilename), encryptToBytes(t, payload, key)) s := &setup{opts: &Opts{}} // no OrgKey outcome, err := s.decryptPayload("pkg", dir) @@ -177,7 +209,7 @@ func TestDecryptPayload(t *testing.T) { t.Run("wrong key fails closed", func(t *testing.T) { dir := t.TempDir() - writeFile(t, filepath.Join(dir, "payload"), encryptToBytes(t, payload, key)) + writeFile(t, filepath.Join(dir, artifactcrypto.PayloadFilename), encryptToBytes(t, payload, key)) wrong := make([]byte, artifactcrypto.KeySize) // all zeros s := &setup{opts: &Opts{OrgKey: wrong}} @@ -199,6 +231,30 @@ func TestDecryptPayload(t *testing.T) { t.Fatalf("outcome = %v, want decryptNotEncrypted", outcome) } }) + + // Ensure the decrypted contents land in that same dir so the runtime.json installdir resolves. + t.Run("nested payload extracts in place", func(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "runtime.json"), []byte(`{"installDir":"installdir"}`)) + writeFile(t, filepath.Join(dir, "installdir", artifactcrypto.PayloadFilename), encryptToBytes(t, payload, key)) + + s := &setup{opts: &Opts{OrgKey: key}} + outcome, err := s.decryptPayload("pkg", dir) + if err != nil { + t.Fatalf("decryptPayload: %v", err) + } + if outcome != decryptDone { + t.Fatalf("outcome = %v, want decryptDone", outcome) + } + // Ciphertext is removed and the archive is extracted alongside it, under + // the install dir. + if _, err := os.Stat(filepath.Join(dir, "installdir", artifactcrypto.PayloadFilename)); !os.IsNotExist(err) { + t.Error("ciphertext was not removed") + } + if got, _ := os.ReadFile(filepath.Join(dir, "installdir", "pkg", "__init__.py")); string(got) != "print('private')\n" { + t.Errorf("payload not extracted under the install dir: got %q", got) + } + }) } func TestPrivateArtifactSurvivesEviction(t *testing.T) { diff --git a/pkg/runtime/setup.go b/pkg/runtime/setup.go index c3ad00646b..7feb074a6e 100644 --- a/pkg/runtime/setup.go +++ b/pkg/runtime/setup.go @@ -505,9 +505,10 @@ const ( decryptSkipped // encrypted, but no org key available ) -// decryptPayload finds an encrypted private-ingredient payload among the -// artifact's top-level files (identified by envelope magic, not filename), -// decrypts it, and extracts the inner tar.gz archive in place of the ciphertext. +// decryptPayload finds the encrypted private-ingredient payload within the +// unpacked artifact (located by name and confirmed by envelope magic), +// decrypts it, and extracts the inner tar.gz archive in place of the +// ciphertext so it lands where the runtime.json points. // // A missing key returns decryptSkipped; a wrong key or corrupt payload returns // an error. @@ -572,8 +573,10 @@ func (s *setup) decryptPayload(artifactName, unpackPath string) (outcome decrypt rerr = errs.Pack(rerr, errs.Wrap(err, "could not close decrypted payload")) } }() + // Extract alongside the ciphertext so the decrypted contents land where the + // runtime.json points. archiveUA := unarchiver.NewTarGz(unarchiver.WithUntrustedSource()) - if err := archiveUA.Unarchive(archive, unpackPath); err != nil { + if err := archiveUA.Unarchive(archive, filepath.Dir(payloadPath)); err != nil { return decryptNotEncrypted, errs.Wrap(err, "could not extract decrypted artifact %s", artifactName) } @@ -585,35 +588,41 @@ func (s *setup) decryptPayload(artifactName, unpackPath string) (outcome decrypt return decryptDone, nil } -// findEncryptedPayload returns the path of the single top-level file in dir that -// is an artifactcrypto envelope, or "" if none is. Subdirectories and plaintext -// files are ignored. +// findEncryptedPayload returns the path of the encrypted private payload within +// dir, searched recursively, or "" if none is present. The payload is located by +// its conventional name (artifactcrypto.PayloadFilename) and confirmed by its +// artifactcrypto envelope magic, so a plaintext file that happens to share the +// name is ignored. func findEncryptedPayload(dir string) (string, error) { - entries, err := os.ReadDir(dir) - if err != nil { - return "", errs.Wrap(err, "could not read artifact directory") - } - for _, e := range entries { - if e.IsDir() { - continue + var found string + walkErr := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || d.Name() != artifactcrypto.PayloadFilename { + return nil } - path := filepath.Join(dir, e.Name()) f, err := os.Open(path) if err != nil { - return "", errs.Wrap(err, "could not open artifact file") + return errs.Wrap(err, "could not open artifact file") } encrypted, err := artifactcrypto.IsEncrypted(f) if cerr := f.Close(); cerr != nil { err = errs.Pack(err, errs.Wrap(cerr, "could not close artifact file")) } if err != nil { - return "", errs.Wrap(err, "could not detect encrypted payload") + return errs.Wrap(err, "could not detect encrypted payload") } if encrypted { - return path, nil + found = path + return filepath.SkipAll } + return nil + }) + if walkErr != nil { + return "", errs.Wrap(walkErr, "could not scan artifact directory") } - return "", nil + return found, nil } func readPayloadHeader(path string) (header artifactcrypto.Header, rerr error) { diff --git a/test/integration/publish_int_test.go b/test/integration/publish_int_test.go index bb1a8b5730..c26bd75c4d 100644 --- a/test/integration/publish_int_test.go +++ b/test/integration/publish_int_test.go @@ -558,29 +558,16 @@ func (suite *PublishIntegrationTestSuite) TestPublishBuildEncrypted() { // Decryption proof: the decrypted content must be present in the depot and // contain our sentinel. A failed decrypt would skip the artifact, leaving the - // sentinel absent. This is inlined (rather than in a helper) so a failure - // surfaces the spawned-command output and state logs the e2e harness dumps. - // The search is deliberately broad — any wheel or file anywhere under the - // depot — because the exact on-disk path depends on how the artifact is - // packaged on install. - depot := filepath.Join(ts.Dirs.Cache, "depot") + // sentinel absent. + suite.assertDecryptedPayloadContains(ts, sentinel) +} - // Log the depot's top two levels so a failure shows how the artifact landed - // (e.g. an "install" dir with a wheel vs. a leftover "payload.enc"). - if entries, err := os.ReadDir(depot); err == nil { - for _, e := range entries { - suite.T().Logf("depot/%s", e.Name()) - if e.IsDir() { - if sub, err := os.ReadDir(filepath.Join(depot, e.Name())); err == nil { - for _, s := range sub { - suite.T().Logf("depot/%s/%s", e.Name(), s.Name()) - } - } - } - } - } else { - suite.T().Logf("could not read depot %s: %v", depot, err) - } +// assertDecryptedPayloadContains fails the test unless a decrypted artifact under +// the depot contains sentinel. It scans every wheel (as a zip) and every small +// plaintext file, since the exact on-disk path depends on how the artifact is +// packaged on install. +func (suite *PublishIntegrationTestSuite) assertDecryptedPayloadContains(ts *e2e.Session, sentinel string) { + depot := filepath.Join(ts.Dirs.Cache, "depot") var wheels []string fileCount := 0 @@ -597,22 +584,8 @@ func (suite *PublishIntegrationTestSuite) TestPublishBuildEncrypted() { // A decrypted wheel is a zip; scan its entries for the sentinel. if strings.HasSuffix(d.Name(), ".whl") { wheels = append(wheels, path) - zr, err := zip.OpenReader(path) - if err != nil { - suite.T().Logf("could not open wheel %s as zip: %v", path, err) - return nil - } - defer zr.Close() - for _, f := range zr.File { - rc, err := f.Open() - if err != nil { - continue - } - content, _ := io.ReadAll(rc) - rc.Close() - if strings.Contains(string(content), sentinel) { - found = true - } + if suite.wheelContains(path, sentinel) { + found = true } return nil } @@ -634,6 +607,30 @@ func (suite *PublishIntegrationTestSuite) TestPublishBuildEncrypted() { suite.Require().True(found, "sentinel %q not found in the depot; the artifact was likely not decrypted", sentinel) } +// wheelContains reports whether any file inside the wheel (a zip) contains +// sentinel. A wheel that failed to decrypt would not be a readable zip, so an +// unreadable wheel is logged and treated as not containing the sentinel. +func (suite *PublishIntegrationTestSuite) wheelContains(wheelPath, sentinel string) bool { + zr, err := zip.OpenReader(wheelPath) + if err != nil { + suite.T().Logf("could not open wheel %s as zip: %v", wheelPath, err) + return false + } + defer zr.Close() + for _, f := range zr.File { + rc, err := f.Open() + if err != nil { + continue + } + content, _ := io.ReadAll(rc) + rc.Close() + if strings.Contains(string(content), sentinel) { + return true + } + } + return false +} + // orgKeyContract builds the org-key contract JSON the key service would serve for // the given key and organization, for injection via the environment. func orgKeyContract(suite *PublishIntegrationTestSuite, key []byte, org string) string { From 4b8fdb7f4617503993b95e8c9d35c688fe597a60 Mon Sep 17 00:00:00 2001 From: mitchell Date: Fri, 26 Jun 2026 09:21:04 -0400 Subject: [PATCH 7/7] Address review feedback on the decryption-check helpers (ENG-1643) Stop walking the depot once the sentinel is found rather than scanning every remaining file, and skip a wheel zip entry whose read fails instead of testing partial bytes. Both are confined to the integration test's verification helpers. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/integration/publish_int_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/integration/publish_int_test.go b/test/integration/publish_int_test.go index c26bd75c4d..616374ce2e 100644 --- a/test/integration/publish_int_test.go +++ b/test/integration/publish_int_test.go @@ -576,6 +576,9 @@ func (suite *PublishIntegrationTestSuite) assertDecryptedPayloadContains(ts *e2e if err != nil { return err } + if found { + return filepath.SkipAll // sentinel located; no need to walk the rest of the depot + } if d.IsDir() { return nil } @@ -622,8 +625,11 @@ func (suite *PublishIntegrationTestSuite) wheelContains(wheelPath, sentinel stri if err != nil { continue } - content, _ := io.ReadAll(rc) + content, err := io.ReadAll(rc) rc.Close() + if err != nil { + continue + } if strings.Contains(string(content), sentinel) { return true }