diff --git a/artifactory_test.go b/artifactory_test.go index 9d5bd245b..41525bab8 100644 --- a/artifactory_test.go +++ b/artifactory_test.go @@ -3683,8 +3683,10 @@ func TestArtifactoryDownloadByBuildUsingSimpleDownloadWithProject(t *testing.T) // Upload files with buildName, buildNumber and project flags runRt(t, "upload", "--spec="+specFileB, "--build-name="+tests.RtBuildName1, "--build-number="+buildNumberA, "--project="+tests.ProjectKey) - // Publish buildInfo with project flag - runRt(t, "build-publish", tests.RtBuildName1, buildNumberA, "--project="+tests.ProjectKey) + // Publish buildInfo with project flag — retried automatically if project cache not yet warm + assert.NoError(t, retryOnProjectNotFound(func() error { + return artifactoryCli.Exec("build-publish", tests.RtBuildName1, buildNumberA, "--project="+tests.ProjectKey) + })) // Download by project, b1 should be downloaded runRt(t, "download", tests.RtRepo1+"/data/b1.in", filepath.Join(tests.Out, "download", "simple_by_build")+fileutils.GetFileSeparator(), @@ -3740,8 +3742,10 @@ func TestArtifactoryDownloadWithEnvProject(t *testing.T) { // Upload files with buildName, buildNumber and project flags runRt(t, "upload", "--spec="+specFileB) - // Publish buildInfo with project flag - runRt(t, "build-publish") + // Publish buildInfo with project flag — retried automatically if project cache not yet warm + assert.NoError(t, retryOnProjectNotFound(func() error { + return artifactoryCli.Exec("build-publish") + })) // Download by project, b1 should be downloaded runRt(t, "download", tests.RtRepo1+"/data/b1.in", filepath.Join(tests.Out, "download", "simple_by_build")+fileutils.GetFileSeparator(), diff --git a/lifecycle_test.go b/lifecycle_test.go index 7d58a229b..a47dfac2c 100644 --- a/lifecycle_test.go +++ b/lifecycle_test.go @@ -670,6 +670,10 @@ func uploadBuildsWithProject(t *testing.T) func() { uploadBuildWithArtifactsAndProject(t, tests.UploadDevSpecA, tests.LcBuildName1, number1, tests.ProjectKey) uploadBuildWithArtifactsAndProject(t, tests.UploadDevSpecB, tests.LcBuildName2, number2, tests.ProjectKey) uploadBuildWithDepsAndProject(t, tests.LcBuildName3, number3, tests.ProjectKey) + // Wait for the Lifecycle service to register the project before any release-bundle + // creation attempts. LC has its own cache separate from Artifactory's; it can take + // several minutes to warm in HA deployments running Artifactory draft builds. + waitForLifecycleProjectVisibility(t, tests.ProjectKey) return func() { inttestutils.DeleteBuild(serverDetails.ArtifactoryUrl, tests.LcBuildName1, artHttpDetails) inttestutils.DeleteBuild(serverDetails.ArtifactoryUrl, tests.LcBuildName2, artHttpDetails) @@ -767,7 +771,9 @@ func createRbWithFlags(t *testing.T, specFilePath, sourceOption, buildName, buil argsAndOptions = append(argsAndOptions, getOption(cliutils.Draft, "true")) } - assert.NoError(t, lcCli.Exec(argsAndOptions...)) + assert.NoError(t, retryOnProjectNotFound(func() error { + return lcCli.Exec(argsAndOptions...) + })) } func updateRbWithFlags(t *testing.T, specFilePath, rbName, rbVersion, project, sourceTypeBuilds string, sync bool) { @@ -1424,7 +1430,6 @@ func TestReleaseBundlesSearchVersions(t *testing.T) { } }() } - deleteBuildsWithProject := uploadBuildsWithProject(t) defer deleteBuildsWithProject() @@ -1643,7 +1648,9 @@ func uploadBuildWithArtifactsAndProject(t *testing.T, specFileName, buildName, b assert.NoError(t, err) runRt(t, "upload", "--spec="+specFile, "--build-name="+buildName, "--build-number="+buildNumber, "--project="+projectKey) - runRt(t, "build-publish", buildName, buildNumber, "--project="+projectKey) + assert.NoError(t, retryOnProjectNotFound(func() error { + return artifactoryCli.Exec("build-publish", buildName, buildNumber, "--project="+projectKey) + })) } func uploadBuildWithDepsAndProject(t *testing.T, buildName, buildNumber, projectKey string) { @@ -1656,7 +1663,9 @@ func uploadBuildWithDepsAndProject(t *testing.T, buildName, buildNumber, project runRt(t, "upload", randFile.Name(), tests.RtDevRepo, "--flat", "--project="+projectKey) assert.NoError(t, lcCli.WithoutCredentials().Exec("rt", "bad", buildName, buildNumber, tests.RtDevRepo+"/dep-file", "--from-rt")) - runRt(t, "build-publish", buildName, buildNumber, "--project="+projectKey) + assert.NoError(t, retryOnProjectNotFound(func() error { + return artifactoryCli.Exec("build-publish", buildName, buildNumber, "--project="+projectKey) + })) } func initLifecycleTest(t *testing.T, minVersion string) (cleanCallback func()) { @@ -1765,3 +1774,55 @@ type KeyPairPayload struct { PublicKey string `json:"publicKey,omitempty"` PrivateKey string `json:"privateKey,omitempty"` // #nosec G117 -- test struct, not a real secret } + +// waitForLifecycleProjectVisibility polls the Lifecycle service until the project is visible +// to it. LC has a project cache separate from Artifactory's and can take many minutes to warm +// in HA deployments. Polling a cheap endpoint avoids wasting the retry budget on expensive +// full CLI commands before LC is actually ready. +func waitForLifecycleProjectVisibility(t *testing.T, projectKey string) { + const ( + timeout = 15 * time.Minute + pollInterval = 15 * time.Second + ) + client, err := httpclient.ClientBuilder().Build() + if !assert.NoError(t, err) { + return + } + // GET on a non-existent RB returns 400 when LC doesn't know the project yet, + // and 404 when it does (RB not found, but project is recognised). + probeURL := *tests.JfrogUrl + "lifecycle/api/v2/release_bundle/records/non-existing-rb?project=" + projectKey + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + resp, _, _, probErr := client.SendGet(probeURL, true, artHttpDetails, "") + if probErr == nil && resp.StatusCode != http.StatusBadRequest { + return + } + time.Sleep(pollInterval) + } + t.Logf("waitForLifecycleProjectVisibility: LC still not aware of project %q after %s; proceeding anyway", projectKey, timeout) +} + +// retryOnProjectNotFound retries fn when Artifactory or Lifecycle reports that a project is not +// yet visible — a transient condition caused by the Access→service cache propagation delay in HA +// deployments. Up to maxRetries attempts are made with retryInterval between each attempt. +func retryOnProjectNotFound(fn func() error) error { + const ( + maxRetries = 10 + retryInterval = 30 * time.Second + ) + var err error + for attempt := 0; attempt < maxRetries; attempt++ { + err = fn() + if err == nil { + return nil + } + errMsg := err.Error() + if !strings.Contains(errMsg, "not found") && !strings.Contains(errMsg, "project key") { + return err + } + if attempt < maxRetries-1 { + time.Sleep(retryInterval) + } + } + return err +} diff --git a/pnpm_test.go b/pnpm_test.go index b471a894d..1019f88a9 100644 --- a/pnpm_test.go +++ b/pnpm_test.go @@ -898,23 +898,18 @@ func TestPnpmBuildPublishWithCIVcsProps(t *testing.T) { func TestPnpmInstallAndPublishWithProject(t *testing.T) { initPnpmTest(t) - // Create Access service manager and project before deferring cleanPnpmTest, - // so that t.Skipf doesn't trigger cleanup asserts that override the skip status. accessManager, err := utils.CreateAccessServiceManager(serverDetails, false) if err != nil { t.Skipf("Skipping project test - cannot create access manager: %v", err) } - // Try creating project first to verify access works before deferring any cleanup projectParams := accessServices.ProjectParams{ ProjectDetails: accessServices.Project{ DisplayName: "pnpm-project-test " + tests.ProjectKey, ProjectKey: tests.ProjectKey, }, } - // First delete if exists, ignoring errors since access might not support it - _ = accessManager.DeleteProject(tests.ProjectKey) - if err = accessManager.CreateProject(projectParams); err != nil { + if err = accessManager.CreateProject(projectParams); err != nil && !strings.Contains(err.Error(), "already exists") { t.Skipf("Skipping project test - cannot create project: %v", err) } @@ -925,7 +920,6 @@ func TestPnpmInstallAndPublishWithProject(t *testing.T) { _ = accessManager.DeleteProject(tests.ProjectKey) }() - // Assign npm repos to the project err = accessManager.AssignRepoToProject(tests.NpmRepo, tests.ProjectKey, true) assert.NoError(t, err) err = accessManager.AssignRepoToProject(tests.NpmRemoteRepo, tests.ProjectKey, true) @@ -941,36 +935,30 @@ func TestPnpmInstallAndPublishWithProject(t *testing.T) { buildName := tests.PnpmBuildName + "-project" buildNumber := "800" - // Clean old build inttestutils.DeleteBuild(serverDetails.ArtifactoryUrl, buildName, artHttpDetails) defer inttestutils.DeleteBuild(serverDetails.ArtifactoryUrl, buildName, artHttpDetails) - // Setup pnpm project pnpmProjectPath := createPnpmProject(t, "pnpmproject") projectDir := filepath.Dir(pnpmProjectPath) prepareArtifactoryForPnpmBuild(t, projectDir) - clientTestUtils.ChangeDirAndAssert(t, projectDir) - // Run pnpm install with --project flag runJfrogCli(t, "pnpm", "install", "--store-dir="+tempCacheDirPath, "--build-name="+buildName, "--build-number="+buildNumber, "--project="+tests.ProjectKey) - // Run pnpm publish with --project flag cleanupAuth := setupPnpmPublishAuth(t, tests.NpmRepo) defer cleanupAuth() runJfrogCli(t, "pnpm", "publish", "--no-git-checks", "--build-name="+buildName, "--build-number="+buildNumber, "--project="+tests.ProjectKey) - // Publish build info with --project flag - assert.NoError(t, artifactoryCli.Exec("bp", buildName, buildNumber, "--project="+tests.ProjectKey)) + assert.NoError(t, retryOnProjectNotFound(func() error { + return artifactoryCli.Exec("bp", buildName, buildNumber, "--project="+tests.ProjectKey) + })) - // Restore working directory clientTestUtils.ChangeDirAndAssert(t, wd) - // Get the published build info with project key servicesManager, err := utils.CreateServiceManager(serverDetails, -1, 0, false) assert.NoError(t, err) params := artServices.NewBuildInfoParams() @@ -982,7 +970,6 @@ func TestPnpmInstallAndPublishWithProject(t *testing.T) { assert.True(t, found, "Build info was not found for project %s", tests.ProjectKey) bi := publishedBuildInfo.BuildInfo - // pnpm install + publish on the same build should produce 1 module with both deps and artifacts if assert.NotEmpty(t, bi.Modules, "Build info should contain modules") { assert.NotEmpty(t, bi.Modules[0].Dependencies, "Module should have dependencies from pnpm install") assert.NotEmpty(t, bi.Modules[0].Artifacts, "Module should have artifacts from pnpm publish") diff --git a/transfer_test.go b/transfer_test.go index c972442b8..8032b06f4 100644 --- a/transfer_test.go +++ b/transfer_test.go @@ -540,10 +540,13 @@ func validateCsvConflicts(t *testing.T, csvPath string, projectsSupported bool) func createTestProject(t *testing.T) func() error { accessManager, err := rtUtils.CreateAccessServiceManager(serverDetails, false) assert.NoError(t, err) - // Delete the project if already exists - deleteProjectIfExists(t, accessManager, tests.ProjectKey) - // Create new project + // Do NOT delete the project before creating it. Project keys are unique per test-suite + // run (timestamp-based), so pre-emptive deletion only occurs on gotestsum re-runs when + // the previous attempt left the project behind. Deleting and immediately recreating the + // same key poisons Artifactory's internal project cache with a "not found" entry that + // takes minutes to expire on some HA nodes, causing all subsequent project-scoped + // operations to fail with 400 "Project was not found" even after retries. adminPrivileges := accessServices.AdminPrivileges{ ManageMembers: utils.Pointer(false), ManageResources: utils.Pointer(false), @@ -558,12 +561,14 @@ func createTestProject(t *testing.T) func() error { ProjectKey: tests.ProjectKey, } - if assert.NoError(t, accessManager.CreateProject(accessServices.ProjectParams{ProjectDetails: projectDetails})) { - return func() error { - return accessManager.DeleteProject(tests.ProjectKey) - } + createErr := accessManager.CreateProject(accessServices.ProjectParams{ProjectDetails: projectDetails}) + if createErr != nil && !strings.Contains(createErr.Error(), "already exists") { + assert.NoError(t, createErr) + return nil + } + return func() error { + return accessManager.DeleteProject(tests.ProjectKey) } - return nil } func updateProjectParams(t *testing.T, projectParams *accessServices.Project, targetAccessManager *access.AccessServicesManager) { diff --git a/utils/tests/utils.go b/utils/tests/utils.go index 569243a2d..900eaf012 100644 --- a/utils/tests/utils.go +++ b/utils/tests/utils.go @@ -88,6 +88,28 @@ var ( timestampAdded bool ) +// nonProjectKeyCharsRegex matches any character that isn't allowed in an Artifactory +// project key (project keys allow only lowercase alphanumeric characters and hyphens). +// We use this to sanitize the --ci.runId value before splicing it into resource names +// whose format is constrained (project keys, GPG keypair names, etc.). Project-key +// charset is a strict subset of the GPG keypair charset, so a single sanitization is +// safe for both. +var nonProjectKeyCharsRegex = regexp.MustCompile(`[^a-z0-9-]+`) + +// SanitizedCiRunId returns the --ci.runId flag value lowercased with any characters +// outside [a-z0-9-] collapsed to a single hyphen and surrounding hyphens trimmed. +// Returns "" if the flag wasn't set. Callers that need a per-runId suffix on +// resources whose name format is constrained (e.g. Artifactory project keys, GPG +// keypair names) should use this so concurrent runs against a shared JPD don't +// clobber each other. +func SanitizedCiRunId() string { + if ciRunId == nil || *ciRunId == "" { + return "" + } + sanitized := nonProjectKeyCharsRegex.ReplaceAllString(strings.ToLower(*ciRunId), "-") + return strings.Trim(sanitized, "-") +} + func init() { JfrogUrl = flag.String("jfrog.url", "http://localhost:8081/", "JFrog platform url") JfrogUser = flag.String("jfrog.user", "admin", "JFrog platform username") @@ -646,7 +668,25 @@ func AddTimestampToGlobalVars() { Password2 += uniqueSuffix + strconv.FormatFloat(randomSequence.Float64(), 'f', 2, 32) // Projects - ProjectKey += timestamp[len(timestamp)-7:] + // Artifactory project keys must be 2-32 lowercase alphanumeric or hyphen + // characters and must start with a letter. We always include the sanitized + // --ci.runId (when set) so that concurrent runs against a shared JPD don't + // clobber each other's project — createTestProject calls + // deleteProjectIfExists(tests.ProjectKey) unconditionally, which means a + // colliding key from another concurrent suite will silently delete the + // project (and every release bundle inside it) out from under us. + projectSuffix := timestamp + if sanitizedRunId := SanitizedCiRunId(); sanitizedRunId != "" { + projectSuffix = sanitizedRunId + "-" + projectSuffix + } + // ProjectKey starts as "prj" (3 chars), and the total must be <= 32. Trim + // from the front so the trailing timestamp (used for visual debuggability) + // is preserved and we don't end up with a key that starts with a hyphen. + const maxProjectKeyLen = 32 + if maxSuffixLen := maxProjectKeyLen - len(ProjectKey); len(projectSuffix) > maxSuffixLen { + projectSuffix = strings.TrimLeft(projectSuffix[len(projectSuffix)-maxSuffixLen:], "-") + } + ProjectKey += projectSuffix timestampAdded = true }