Include --ci.runId in test ProjectKey to prevent cross-suite collisions

artifactory_test.go

@@ -3683,8 +3683,10 @@ func TestArtifactoryDownloadByBuildUsingSimpleDownloadWithProject(t *testing.T)
 	// Upload files with buildName, buildNumber and project flags
 	runRt(t, "upload", "--spec="+specFileB, "--build-name="+tests.RtBuildName1, "--build-number="+buildNumberA, "--project="+tests.ProjectKey)
 
-	// Publish buildInfo with project flag
-	runRt(t, "build-publish", tests.RtBuildName1, buildNumberA, "--project="+tests.ProjectKey)
+	// Publish buildInfo with project flag — retried automatically if project cache not yet warm
+	assert.NoError(t, retryOnProjectNotFound(func() error {
+		return artifactoryCli.Exec("build-publish", tests.RtBuildName1, buildNumberA, "--project="+tests.ProjectKey)
+	}))
 
 	// Download by project, b1 should be downloaded
 	runRt(t, "download", tests.RtRepo1+"/data/b1.in", filepath.Join(tests.Out, "download", "simple_by_build")+fileutils.GetFileSeparator(),
@@ -3740,8 +3742,10 @@ func TestArtifactoryDownloadWithEnvProject(t *testing.T) {
 	// Upload files with buildName, buildNumber and project flags
 	runRt(t, "upload", "--spec="+specFileB)
 
-	// Publish buildInfo with project flag
-	runRt(t, "build-publish")
+	// Publish buildInfo with project flag — retried automatically if project cache not yet warm
+	assert.NoError(t, retryOnProjectNotFound(func() error {
+		return artifactoryCli.Exec("build-publish")
+	}))
 
 	// Download by project, b1 should be downloaded
 	runRt(t, "download", tests.RtRepo1+"/data/b1.in", filepath.Join(tests.Out, "download", "simple_by_build")+fileutils.GetFileSeparator(),

lifecycle_test.go

@@ -670,6 +670,10 @@ func uploadBuildsWithProject(t *testing.T) func() {
 	uploadBuildWithArtifactsAndProject(t, tests.UploadDevSpecA, tests.LcBuildName1, number1, tests.ProjectKey)
 	uploadBuildWithArtifactsAndProject(t, tests.UploadDevSpecB, tests.LcBuildName2, number2, tests.ProjectKey)
 	uploadBuildWithDepsAndProject(t, tests.LcBuildName3, number3, tests.ProjectKey)
+	// Wait for the Lifecycle service to register the project before any release-bundle
+	// creation attempts. LC has its own cache separate from Artifactory's; it can take
+	// several minutes to warm in HA deployments running Artifactory draft builds.
+	waitForLifecycleProjectVisibility(t, tests.ProjectKey)
 	return func() {
 		inttestutils.DeleteBuild(serverDetails.ArtifactoryUrl, tests.LcBuildName1, artHttpDetails)
 		inttestutils.DeleteBuild(serverDetails.ArtifactoryUrl, tests.LcBuildName2, artHttpDetails)
@@ -767,7 +771,9 @@ func createRbWithFlags(t *testing.T, specFilePath, sourceOption, buildName, buil
 		argsAndOptions = append(argsAndOptions, getOption(cliutils.Draft, "true"))
 	}
 
-	assert.NoError(t, lcCli.Exec(argsAndOptions...))
+	assert.NoError(t, retryOnProjectNotFound(func() error {
+		return lcCli.Exec(argsAndOptions...)
+	}))
 }
 
 func updateRbWithFlags(t *testing.T, specFilePath, rbName, rbVersion, project, sourceTypeBuilds string, sync bool) {
@@ -1424,7 +1430,6 @@ func TestReleaseBundlesSearchVersions(t *testing.T) {
 			}
 		}()
 	}
-
 	deleteBuildsWithProject := uploadBuildsWithProject(t)
 	defer deleteBuildsWithProject()
 
@@ -1643,7 +1648,9 @@ func uploadBuildWithArtifactsAndProject(t *testing.T, specFileName, buildName, b
 	assert.NoError(t, err)
 
 	runRt(t, "upload", "--spec="+specFile, "--build-name="+buildName, "--build-number="+buildNumber, "--project="+projectKey)
-	runRt(t, "build-publish", buildName, buildNumber, "--project="+projectKey)
+	assert.NoError(t, retryOnProjectNotFound(func() error {
+		return artifactoryCli.Exec("build-publish", buildName, buildNumber, "--project="+projectKey)
+	}))
 }
 
 func uploadBuildWithDepsAndProject(t *testing.T, buildName, buildNumber, projectKey string) {
@@ -1656,7 +1663,9 @@ func uploadBuildWithDepsAndProject(t *testing.T, buildName, buildNumber, project
 	runRt(t, "upload", randFile.Name(), tests.RtDevRepo, "--flat", "--project="+projectKey)
 	assert.NoError(t, lcCli.WithoutCredentials().Exec("rt", "bad", buildName, buildNumber, tests.RtDevRepo+"/dep-file", "--from-rt"))
 
-	runRt(t, "build-publish", buildName, buildNumber, "--project="+projectKey)
+	assert.NoError(t, retryOnProjectNotFound(func() error {
+		return artifactoryCli.Exec("build-publish", buildName, buildNumber, "--project="+projectKey)
+	}))
 }
 
 func initLifecycleTest(t *testing.T, minVersion string) (cleanCallback func()) {
@@ -1765,3 +1774,55 @@ type KeyPairPayload struct {
 	PublicKey  string `json:"publicKey,omitempty"`
 	PrivateKey string `json:"privateKey,omitempty"` // #nosec G117 -- test struct, not a real secret
 }
+
+// waitForLifecycleProjectVisibility polls the Lifecycle service until the project is visible
+// to it. LC has a project cache separate from Artifactory's and can take many minutes to warm
+// in HA deployments. Polling a cheap endpoint avoids wasting the retry budget on expensive
+// full CLI commands before LC is actually ready.
+func waitForLifecycleProjectVisibility(t *testing.T, projectKey string) {
+	const (
+		timeout      = 15 * time.Minute
+		pollInterval = 15 * time.Second
+	)
+	client, err := httpclient.ClientBuilder().Build()
+	if !assert.NoError(t, err) {
+		return
+	}
+	// GET on a non-existent RB returns 400 when LC doesn't know the project yet,
+	// and 404 when it does (RB not found, but project is recognised).
+	probeURL := *tests.JfrogUrl + "lifecycle/api/v2/release_bundle/records/non-existing-rb?project=" + projectKey
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		resp, _, _, probErr := client.SendGet(probeURL, true, artHttpDetails, "")
+		if probErr == nil && resp.StatusCode != http.StatusBadRequest {
+			return
+		}
+		time.Sleep(pollInterval)
+	}
+	t.Logf("waitForLifecycleProjectVisibility: LC still not aware of project %q after %s; proceeding anyway", projectKey, timeout)
+}
+
+// retryOnProjectNotFound retries fn when Artifactory or Lifecycle reports that a project is not
+// yet visible — a transient condition caused by the Access→service cache propagation delay in HA
+// deployments. Up to maxRetries attempts are made with retryInterval between each attempt.
+func retryOnProjectNotFound(fn func() error) error {
+	const (
+		maxRetries    = 10
+		retryInterval = 30 * time.Second
+	)
+	var err error
+	for attempt := 0; attempt < maxRetries; attempt++ {
+		err = fn()
+		if err == nil {
+			return nil
+		}
+		errMsg := err.Error()
+		if !strings.Contains(errMsg, "not found") && !strings.Contains(errMsg, "project key") {
+			return err
+		}
+		if attempt < maxRetries-1 {
+			time.Sleep(retryInterval)
+		}
+	}
+	return err
+}

pnpm_test.go

@@ -898,23 +898,18 @@ func TestPnpmBuildPublishWithCIVcsProps(t *testing.T) {
 func TestPnpmInstallAndPublishWithProject(t *testing.T) {
 	initPnpmTest(t)
 
-	// Create Access service manager and project before deferring cleanPnpmTest,
-	// so that t.Skipf doesn't trigger cleanup asserts that override the skip status.
 	accessManager, err := utils.CreateAccessServiceManager(serverDetails, false)
 	if err != nil {
 		t.Skipf("Skipping project test - cannot create access manager: %v", err)
 	}
 
-	// Try creating project first to verify access works before deferring any cleanup
 	projectParams := accessServices.ProjectParams{
 		ProjectDetails: accessServices.Project{
 			DisplayName: "pnpm-project-test " + tests.ProjectKey,
 			ProjectKey:  tests.ProjectKey,
 		},
 	}
-	// First delete if exists, ignoring errors since access might not support it
-	_ = accessManager.DeleteProject(tests.ProjectKey)
-	if err = accessManager.CreateProject(projectParams); err != nil {
+	if err = accessManager.CreateProject(projectParams); err != nil && !strings.Contains(err.Error(), "already exists") {
 		t.Skipf("Skipping project test - cannot create project: %v", err)
 	}
 
@@ -925,7 +920,6 @@ func TestPnpmInstallAndPublishWithProject(t *testing.T) {
 		_ = accessManager.DeleteProject(tests.ProjectKey)
 	}()
 
-	// Assign npm repos to the project
 	err = accessManager.AssignRepoToProject(tests.NpmRepo, tests.ProjectKey, true)
 	assert.NoError(t, err)
 	err = accessManager.AssignRepoToProject(tests.NpmRemoteRepo, tests.ProjectKey, true)
@@ -941,36 +935,30 @@ func TestPnpmInstallAndPublishWithProject(t *testing.T) {
 	buildName := tests.PnpmBuildName + "-project"
 	buildNumber := "800"
 
-	// Clean old build
 	inttestutils.DeleteBuild(serverDetails.ArtifactoryUrl, buildName, artHttpDetails)
 	defer inttestutils.DeleteBuild(serverDetails.ArtifactoryUrl, buildName, artHttpDetails)
 
-	// Setup pnpm project
 	pnpmProjectPath := createPnpmProject(t, "pnpmproject")
 	projectDir := filepath.Dir(pnpmProjectPath)
 	prepareArtifactoryForPnpmBuild(t, projectDir)
-
 	clientTestUtils.ChangeDirAndAssert(t, projectDir)
 
-	// Run pnpm install with --project flag
 	runJfrogCli(t, "pnpm", "install", "--store-dir="+tempCacheDirPath,
 		"--build-name="+buildName, "--build-number="+buildNumber,
 		"--project="+tests.ProjectKey)
 
-	// Run pnpm publish with --project flag
 	cleanupAuth := setupPnpmPublishAuth(t, tests.NpmRepo)
 	defer cleanupAuth()
 	runJfrogCli(t, "pnpm", "publish", "--no-git-checks",
 		"--build-name="+buildName, "--build-number="+buildNumber,
 		"--project="+tests.ProjectKey)
 
-	// Publish build info with --project flag
-	assert.NoError(t, artifactoryCli.Exec("bp", buildName, buildNumber, "--project="+tests.ProjectKey))
+	assert.NoError(t, retryOnProjectNotFound(func() error {
+		return artifactoryCli.Exec("bp", buildName, buildNumber, "--project="+tests.ProjectKey)
+	}))
 
-	// Restore working directory
 	clientTestUtils.ChangeDirAndAssert(t, wd)
 
-	// Get the published build info with project key
 	servicesManager, err := utils.CreateServiceManager(serverDetails, -1, 0, false)
 	assert.NoError(t, err)
 	params := artServices.NewBuildInfoParams()
@@ -982,7 +970,6 @@ func TestPnpmInstallAndPublishWithProject(t *testing.T) {
 	assert.True(t, found, "Build info was not found for project %s", tests.ProjectKey)
 
 	bi := publishedBuildInfo.BuildInfo
-	// pnpm install + publish on the same build should produce 1 module with both deps and artifacts
 	if assert.NotEmpty(t, bi.Modules, "Build info should contain modules") {
 		assert.NotEmpty(t, bi.Modules[0].Dependencies, "Module should have dependencies from pnpm install")
 		assert.NotEmpty(t, bi.Modules[0].Artifacts, "Module should have artifacts from pnpm publish")

transfer_test.go

@@ -540,10 +540,13 @@ func validateCsvConflicts(t *testing.T, csvPath string, projectsSupported bool)
 func createTestProject(t *testing.T) func() error {
 	accessManager, err := rtUtils.CreateAccessServiceManager(serverDetails, false)
 	assert.NoError(t, err)
-	// Delete the project if already exists
-	deleteProjectIfExists(t, accessManager, tests.ProjectKey)
 
-	// Create new project
+	// Do NOT delete the project before creating it. Project keys are unique per test-suite
+	// run (timestamp-based), so pre-emptive deletion only occurs on gotestsum re-runs when
+	// the previous attempt left the project behind. Deleting and immediately recreating the
+	// same key poisons Artifactory's internal project cache with a "not found" entry that
+	// takes minutes to expire on some HA nodes, causing all subsequent project-scoped
+	// operations to fail with 400 "Project was not found" even after retries.
 	adminPrivileges := accessServices.AdminPrivileges{
 		ManageMembers:   utils.Pointer(false),
 		ManageResources: utils.Pointer(false),
@@ -558,12 +561,14 @@ func createTestProject(t *testing.T) func() error {
 		ProjectKey:        tests.ProjectKey,
 	}
 
-	if assert.NoError(t, accessManager.CreateProject(accessServices.ProjectParams{ProjectDetails: projectDetails})) {
-		return func() error {
-			return accessManager.DeleteProject(tests.ProjectKey)
-		}
+	createErr := accessManager.CreateProject(accessServices.ProjectParams{ProjectDetails: projectDetails})
+	if createErr != nil && !strings.Contains(createErr.Error(), "already exists") {
+		assert.NoError(t, createErr)
+		return nil
+	}
+	return func() error {
+		return accessManager.DeleteProject(tests.ProjectKey)
 	}
-	return nil
 }
 
 func updateProjectParams(t *testing.T, projectParams *accessServices.Project, targetAccessManager *access.AccessServicesManager) {

utils/tests/utils.go

@@ -88,6 +88,28 @@ var (
 	timestampAdded            bool
 )
 
+// nonProjectKeyCharsRegex matches any character that isn't allowed in an Artifactory
+// project key (project keys allow only lowercase alphanumeric characters and hyphens).
+// We use this to sanitize the --ci.runId value before splicing it into resource names
+// whose format is constrained (project keys, GPG keypair names, etc.). Project-key
+// charset is a strict subset of the GPG keypair charset, so a single sanitization is
+// safe for both.
+var nonProjectKeyCharsRegex = regexp.MustCompile(`[^a-z0-9-]+`)
+
+// SanitizedCiRunId returns the --ci.runId flag value lowercased with any characters
+// outside [a-z0-9-] collapsed to a single hyphen and surrounding hyphens trimmed.
+// Returns "" if the flag wasn't set. Callers that need a per-runId suffix on
+// resources whose name format is constrained (e.g. Artifactory project keys, GPG
+// keypair names) should use this so concurrent runs against a shared JPD don't
+// clobber each other.
+func SanitizedCiRunId() string {
+	if ciRunId == nil || *ciRunId == "" {
+		return ""
+	}
+	sanitized := nonProjectKeyCharsRegex.ReplaceAllString(strings.ToLower(*ciRunId), "-")
+	return strings.Trim(sanitized, "-")
+}
+
 func init() {
 	JfrogUrl = flag.String("jfrog.url", "http://localhost:8081/", "JFrog platform url")
 	JfrogUser = flag.String("jfrog.user", "admin", "JFrog platform  username")
@@ -646,7 +668,25 @@ func AddTimestampToGlobalVars() {
 	Password2 += uniqueSuffix + strconv.FormatFloat(randomSequence.Float64(), 'f', 2, 32)
 
 	// Projects
-	ProjectKey += timestamp[len(timestamp)-7:]
+	// Artifactory project keys must be 2-32 lowercase alphanumeric or hyphen
+	// characters and must start with a letter. We always include the sanitized
+	// --ci.runId (when set) so that concurrent runs against a shared JPD don't
+	// clobber each other's project — createTestProject calls
+	// deleteProjectIfExists(tests.ProjectKey) unconditionally, which means a
+	// colliding key from another concurrent suite will silently delete the
+	// project (and every release bundle inside it) out from under us.
+	projectSuffix := timestamp
+	if sanitizedRunId := SanitizedCiRunId(); sanitizedRunId != "" {
+		projectSuffix = sanitizedRunId + "-" + projectSuffix
+	}
+	// ProjectKey starts as "prj" (3 chars), and the total must be <= 32. Trim
+	// from the front so the trailing timestamp (used for visual debuggability)
+	// is preserved and we don't end up with a key that starts with a hyphen.
+	const maxProjectKeyLen = 32
+	if maxSuffixLen := maxProjectKeyLen - len(ProjectKey); len(projectSuffix) > maxSuffixLen {
+		projectSuffix = strings.TrimLeft(projectSuffix[len(projectSuffix)-maxSuffixLen:], "-")
+	}
+	ProjectKey += projectSuffix
 
 	timestampAdded = true
 }