fix(mutate): run each test process in a private copy of the module

internal/mutate/cover.go

@@ -2,6 +2,7 @@ package mutate
 
 import (
 	"bufio"
+	"cmp"
 	"fmt"
 	"os"
 	"os/exec"
@@ -55,12 +56,12 @@ func inAny(spans []span, line int) bool {
 }
 
 // Cover measures the package in dir: its test binary built once with coverage, then each of its
-// tests run alone into a profile of its own, each holding one of slots while it runs. The slots
+// tests run alone into a profile of its own, each in one of slots' trees while it runs. The slots
 // are the caller's whole budget of test processes, shared with the mutations it runs: a budget
 // of its own per package multiplied with theirs, and a database package's tests start a
 // PostgreSQL per process. Nil where the package has no tests, or they would not build - Run then
 // asks every test, as it would without this.
-func Cover(dir, timeout string, slots chan struct{}) (*Coverage, error) {
+func Cover(dir, timeout string, slots chan *Tree) (*Coverage, error) {
 	work, err := os.MkdirTemp("", "mutate-cover-")
 	if err != nil {
 		return nil, err
@@ -69,9 +70,9 @@ func Cover(dir, timeout string, slots chan struct{}) (*Coverage, error) {
 	bin := filepath.Join(work, "pkg.test")
 	build := exec.Command("go", "test", "-c", "-cover", "-covermode=set", "-o", bin, ".")
 	build.Dir = dir
-	slots <- struct{}{}
+	tree := <-slots
 	out, err := combined("cover: build the package's tests", build)
-	<-slots
+	slots <- tree
 	if err != nil {
 		return nil, fmt.Errorf("mutate: building %s's tests with coverage: %w: %s", dir, err, out)
 	}
@@ -100,21 +101,34 @@ func Cover(dir, timeout string, slots chan struct{}) (*Coverage, error) {
 	c := &Coverage{reached: map[string]map[string][]span{}, blocks: map[string][]span{}, failing: map[string]bool{}}
 	var mu sync.Mutex
 	var wg sync.WaitGroup
+	var treeErr error
 	for i, test := range tests {
 		wg.Add(1)
-		slots <- struct{}{}
+		tree := <-slots
 		go func() {
 			defer wg.Done()
-			defer func() { <-slots }()
+			defer func() { slots <- tree }()
 			profile := filepath.Join(work, strconv.Itoa(i)+".cov")
 			run := exec.Command(bin, "-test.run", "^"+test+"$", "-test.count=1", //nolint:gosec // the binary this built
 				"-test.timeout", timeout, "-test.coverprofile", profile)
-			run.Dir = dir
 			run.Env = testEnv()
 			// A failure here is the package's own tests' to report, not this gate's; what this
 			// gate does with it is ask the test about no edit, since it fails whatever the edit -
 			// and say so, since an edit only it reaches then reads as one nothing holds.
-			said, exceeded, runErr := bounded("cover: run one test", run)
+			var (
+				said     []byte
+				exceeded bool
+				runErr   error
+			)
+			if err := tree.in(dir, func(at, tmp string) {
+				run.Dir, run.Env = at, append(run.Env, "TMPDIR="+tmp)
+				said, exceeded, runErr = bounded("cover: run one test", run)
+			}); err != nil {
+				mu.Lock()
+				treeErr = cmp.Or(treeErr, err)
+				mu.Unlock()
+				return
+			}
 			if exceeded {
 				runErr = fmt.Errorf("held over the %s a test process may, and stopped", bound())
 			}
@@ -147,6 +161,9 @@ func Cover(dir, timeout string, slots chan struct{}) (*Coverage, error) {
 		}()
 	}
 	wg.Wait()
+	if treeErr != nil {
+		return nil, treeErr
+	}
 	return c, nil
 }
 

internal/mutate/helpers_internal_test.go

@@ -0,0 +1,23 @@
+package mutate
+
+import "testing"
+
+// runTests is Run, failing t where the tests could not be asked at all.
+func runTests(t *testing.T, m Mutation, a Asked) (Outcome, string) {
+	t.Helper()
+	outcome, out, err := Run(m, a)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return outcome, out
+}
+
+// askTests is Ask in a tree of its own, failing t where the tests could not be asked at all.
+func askTests(t *testing.T, bin, dir string, id int, timeout string, only []string) (Outcome, string) {
+	t.Helper()
+	outcome, out, err := Ask(NewTree(t.TempDir()), bin, dir, id, timeout, only)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return outcome, out
+}

internal/mutate/helpers_test.go

@@ -0,0 +1,27 @@
+package mutate_test
+
+import (
+	"testing"
+
+	"github.com/spin-stack/go-tools/internal/mutate"
+)
+
+// runTests is mutate.Run, failing t where the tests could not be asked at all.
+func runTests(t *testing.T, m mutate.Mutation, a mutate.Asked) (mutate.Outcome, string) {
+	t.Helper()
+	outcome, out, err := mutate.Run(m, a)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return outcome, out
+}
+
+// askTests is mutate.Ask in a tree of its own, failing t where the tests could not be asked at all.
+func askTests(t *testing.T, bin, dir string, id int, timeout string, only []string) (mutate.Outcome, string) {
+	t.Helper()
+	outcome, out, err := mutate.Ask(mutate.NewTree(t.TempDir()), bin, dir, id, timeout, only)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return outcome, out
+}

internal/mutate/mutate.go

@@ -668,9 +668,6 @@ const (
 	Exceeded
 )
 
-// buildFailed is go test's word that a package it was to test did not build.
-var buildFailed = regexp.MustCompile(`(?m)^FAIL\s+\S+\s+\[build failed\]$`)
-
 // targetedTimeout is how long the tests that reach a mutated line may take.
 const targetedTimeout = "1m"
 
@@ -686,35 +683,50 @@ type Asked struct {
 	Reaching []string
 	// Importers are the packages asked when the package's own tests let the edit through.
 	Importers []string
+	// Tree is where the tests run, the caller's to hold for the call; nil is one of the call's
+	// own. Never the package's own directory (Tree says why).
+	Tree *Tree
 }
 
-// Run builds and runs the tests of the package the mutated file belongs to, stopping at the
-// first that fails.
+// Run builds the tests of the package the mutated file belongs to where its sources are, and runs
+// them in the tree, stopping at the first that fails.
 //
 // An edit its own package's tests let through is asked of the packages that import that one
 // directly, when importers names any: a package's behaviour is often held by its caller's tests -
 // the QMP client's file passing by the chain manager's - and counting those as survivors made the
 // gate report tests that exist as missing. Only a survivor pays for the second run.
-func Run(m Mutation, a Asked) (Outcome, string) {
+//
+// The binaries are built with go test -c and run here rather than by go test, which runs each in
+// its package's directory, where every edit asked at once would read what the others wrote.
+func Run(m Mutation, a Asked) (Outcome, string, error) {
 	dir := filepath.Dir(m.File)
 	tests := hasTests(dir)
 	if !tests && len(a.Importers) == 0 {
-		return Untested, ""
+		return Untested, "", nil
+	}
+	tree := a.Tree
+	if tree == nil {
+		work, err := os.MkdirTemp("", "mutate-tree-")
+		if err != nil {
+			return 0, "", err
+		}
+		defer func() { _ = os.RemoveAll(work) }()
+		tree = NewTree(work)
 	}
-	// The package's own tests are run in its directory; its importers, named from the module's
+	// The package's own tests are named from its directory; its importers, from the module's
 	// root, from where this runs.
 	type testRun struct {
 		dir  string
 		pkgs []string
 		only []string
-		what string // what its process is counted under
+		what string // what its processes are counted under
 	}
 	var runs []testRun
 	if tests && (a.Reaching == nil || len(a.Reaching) > 0) {
-		runs = append(runs, testRun{dir: dir, pkgs: []string{"."}, only: a.Reaching, what: "alone: build and run the package's tests"})
+		runs = append(runs, testRun{dir: dir, pkgs: []string{"."}, only: a.Reaching, what: "the package's tests"})
 	}
 	if len(a.Importers) > 0 {
-		runs = append(runs, testRun{pkgs: a.Importers, what: "alone: build and run the importers' tests"})
+		runs = append(runs, testRun{pkgs: a.Importers, what: "the importers' tests"})
 	}
 	var said strings.Builder
 	for _, set := range runs {
@@ -727,35 +739,75 @@ func Run(m Mutation, a Asked) (Outcome, string) {
 			// package's whole allowance.
 			timeout = targetedTimeout
 		}
-		// -p 1: this run is one of the caller's budget of test processes, and go test over several
-		// packages would otherwise run a binary per core, each with a PostgreSQL of its own.
-		args := append(append([]string{"test"}, set.pkgs...), "-p", "1", "-count=1", "-failfast", "-v", "-timeout", timeout)
-		if a.Overlay != "" {
-			args = append(args, "-overlay", a.Overlay)
-		}
-		if len(set.only) > 0 {
-			args = append(args, "-run", "^("+strings.Join(set.only, "|")+")$")
-		}
-		cmd := exec.Command("go", args...) //nolint:gosec // go test over this repository's packages
-		cmd.Dir = set.dir
-		cmd.Env = testEnv()
-		out, exceeded, err := bounded(set.what, cmd)
-		countTests("", out)
-		said.Write(out)
-		if exceeded {
-			return Exceeded, said.String()
-		}
-		if err == nil {
-			continue
+		list := exec.Command("go", append([]string{"list", "-f", "{{.ImportPath}} {{.Dir}}"}, set.pkgs...)...) //nolint:gosec // go list over this repository's packages
+		list.Dir = set.dir
+		listed, err := output("listing", list)
+		if err != nil {
+			return 0, said.String(), fmt.Errorf("mutate: listing %s: %w", strings.Join(set.pkgs, " "), err)
 		}
-		// go test's own line, not the words anywhere: a test that fails may print them - this
-		// package's do, about the modules they build - and that is an edit refused.
-		if buildFailed.Match(out) || strings.Contains(string(out), "build constraints exclude") {
-			return Unbuildable, said.String()
+		for _, line := range strings.Split(strings.TrimSpace(string(listed)), "\n") {
+			path, pkgDir, _ := strings.Cut(line, " ")
+			outcome, err := runPackage(tree, path, pkgDir, set.dir, set.what, set.only, timeout, a.Overlay, &said)
+			if err != nil || outcome != Survived {
+				return outcome, said.String(), err
+			}
 		}
-		return Killed, said.String()
 	}
-	return Survived, said.String()
+	return Survived, said.String(), nil
+}
+
+// runPackage builds the tests of the package at path, from dir, and runs them in the tree's copy
+// of pkgDir, writing what they said to said. Survived where it has no test this build would run.
+func runPackage(tree *Tree, path, pkgDir, dir, what string, only []string, timeout, overlay string, said *strings.Builder) (Outcome, error) {
+	bin, err := tree.bin()
+	if err != nil {
+		return 0, err
+	}
+	args := []string{"test", "-c", "-vet=off", "-o", bin}
+	if overlay != "" {
+		args = append(args, "-overlay", overlay)
+	}
+	build := exec.Command("go", append(args, path)...) //nolint:gosec // go test over this repository's packages
+	build.Dir = dir
+	out, exceeded, err := bounded("alone: build "+what, build)
+	said.Write(out)
+	if exceeded {
+		return Exceeded, nil
+	}
+	if err != nil {
+		return Unbuildable, nil //nolint:nilerr // a build that fails is the edit's answer: it asks nothing
+	}
+	if _, err := os.Stat(bin); err != nil {
+		return Survived, nil //nolint:nilerr // no test binary: the package has no tests
+	}
+	args = []string{"-test.count=1", "-test.failfast", "-test.v", "-test.timeout", timeout}
+	if len(only) > 0 {
+		args = append(args, "-test.run", "^("+strings.Join(only, "|")+")$")
+	}
+	run := exec.Command(bin, args...) //nolint:gosec // a test binary this run built
+	run.Env = testEnv()
+	var runErr error
+	if err := tree.in(pkgDir, func(at, tmp string) {
+		run.Dir, run.Env = at, append(run.Env, "TMPDIR="+tmp)
+		out, exceeded, runErr = bounded("alone: run "+what, run)
+	}); err != nil {
+		return 0, err
+	}
+	countTests(path, out)
+	said.Write(out)
+	// go test's own line for the package, so what is said names who answered.
+	verdict := "ok  "
+	if runErr != nil {
+		verdict = "FAIL"
+	}
+	fmt.Fprintf(said, "%s\t%s\n", verdict, path)
+	switch {
+	case exceeded:
+		return Exceeded, nil
+	case runErr != nil:
+		return Killed, nil //nolint:nilerr // a test that fails is the edit refused, not this run failing
+	}
+	return Survived, nil
 }
 
 // hasTests is whether the package in dir has a test go would build: one whose tests are all of

internal/mutate/mutate_test.go

@@ -369,25 +369,25 @@ func TestTheLinesAChangeTouchesAreTheOnesItChanged(t *testing.T) {
 func TestAPackageWithNoTestsIsSaidSoAndNotCountedAsASurvivor(t *testing.T) {
 	// Not this package: running its own tests from inside them is a test that runs forever. A
 	// command with no tests of its own: cmd/mutate has the site's example now.
-	outcome, _ := mutate.Run(mutate.Mutation{File: filepath.Join("..", "..", "cmd", "ctxlife", "main.go")}, mutate.Asked{Timeout: "1m"})
+	outcome, _ := runTests(t, mutate.Mutation{File: filepath.Join("..", "..", "cmd", "ctxlife", "main.go")}, mutate.Asked{Timeout: "1m"})
 	assert.Equal(t, mutate.Untested, outcome, "a package with no test files was asked anyway")
 
 	// And one that has tests is run: unmutated, they pass, which is what a survivor looks like.
-	outcome, _ = mutate.Run(mutate.Mutation{File: filepath.Join("..", "fetch", "fetch.go")}, mutate.Asked{Timeout: "2m"})
+	outcome, _ = runTests(t, mutate.Mutation{File: filepath.Join("..", "fetch", "fetch.go")}, mutate.Asked{Timeout: "2m"})
 	assert.Equal(t, mutate.Survived, outcome, "a package whose tests all pass was not run, or was misread")
 
 	// A package with no tests of its own whose importer has some is asked through the importer,
 	// not reported as untested: that is where its behaviour is held.
 	importersRan := func() int {
 		for _, c := range mutate.Costs() {
-			if c.What == "alone: build and run the importers' tests" {
+			if c.What == "alone: run the importers' tests" {
 				return c.N
 			}
 		}
 		return 0
 	}
 	before := importersRan()
-	outcome, out := mutate.Run(mutate.Mutation{File: filepath.Join("..", "..", "cmd", "ctxlife", "main.go")},
+	outcome, out := runTests(t, mutate.Mutation{File: filepath.Join("..", "..", "cmd", "ctxlife", "main.go")},
 		mutate.Asked{Timeout: "2m", Importers: []string{"../fetch"}})
 	assert.Equal(t, mutate.Survived, outcome, "the importer's tests were not run: %s", out)
 	assert.Contains(t, out, "internal/fetch", "the run does not say which package answered")
@@ -402,7 +402,7 @@ func TestAnEditATestRefusesIsCountedRefused(t *testing.T) {
 	m := doubling(t, dir)
 	overlay, err := mutate.Overlay(m, t.TempDir())
 	require.NoError(t, err)
-	outcome, out := mutate.Run(m, mutate.Asked{Overlay: overlay, Timeout: "2m"})
+	outcome, out := runTests(t, m, mutate.Asked{Overlay: overlay, Timeout: "2m"})
 	assert.Equal(t, mutate.Killed, outcome, "a package whose test fails was not counted as refusing: %s", out)
 	assert.Contains(t, out, "not doubled", "what refused it was not the package's own test")
 }
@@ -418,7 +418,7 @@ func TestAPackageWhoseTestsAreOfAnotherBuildIsUntested(t *testing.T) {
 	m := doubling(t, dir)
 	overlay, err := mutate.Overlay(m, t.TempDir())
 	require.NoError(t, err)
-	outcome, out := mutate.Run(m, mutate.Asked{Overlay: overlay, Timeout: "2m"})
+	outcome, out := runTests(t, m, mutate.Asked{Overlay: overlay, Timeout: "2m"})
 	assert.Equal(t, mutate.Untested, outcome, "tests of another build were counted as asked: %s", out)
 }
 
@@ -559,7 +559,7 @@ func G() int {
 // says which, per test.
 func TestAnEditIsPutToTheTestsThatReachIt(t *testing.T) {
 	dir := refuses(t)
-	c, err := mutate.Cover(dir, "2m", make(chan struct{}, 2))
+	c, err := mutate.Cover(dir, "2m", mutate.Slots(t.TempDir(), 2))
 	require.NoError(t, err)
 	require.NotNil(t, c)
 	assert.Equal(t, []string{"TestDouble"}, c.Reaching(filepath.Join(dir, "code.go"), 4))
@@ -571,19 +571,19 @@ func TestAnEditIsPutToTheTestsThatReachIt(t *testing.T) {
 	require.NoError(t, err)
 	// Asked of the one test that reaches it, it is refused; of a test that does not, it survives:
 	// only the named test ran.
-	outcome, out := mutate.Run(m, mutate.Asked{Overlay: overlay, Timeout: "2m", Reaching: []string{"TestDouble"}})
+	outcome, out := runTests(t, m, mutate.Asked{Overlay: overlay, Timeout: "2m", Reaching: []string{"TestDouble"}})
 	assert.Equal(t, mutate.Killed, outcome, "the one test that reaches the edit was not run: %s", out)
-	outcome, out = mutate.Run(m, mutate.Asked{Overlay: overlay, Timeout: "2m", Reaching: []string{"TestHalf"}})
+	outcome, out = runTests(t, m, mutate.Asked{Overlay: overlay, Timeout: "2m", Reaching: []string{"TestHalf"}})
 	assert.Equal(t, mutate.Survived, outcome, "a test that was not named ran: %s", out)
 	// An edit that does not compile asks nothing.
 	broken := m
 	broken.With = "@"
 	overlay, err = mutate.Overlay(broken, t.TempDir())
 	require.NoError(t, err)
-	outcome, out = mutate.Run(broken, mutate.Asked{Overlay: overlay, Timeout: "2m"})
+	outcome, out = runTests(t, broken, mutate.Asked{Overlay: overlay, Timeout: "2m"})
 	assert.Equal(t, mutate.Unbuildable, outcome, "an edit that does not build was counted: %s", out)
 	// And reached by none, it is not run at all.
-	outcome, out = mutate.Run(m, mutate.Asked{Overlay: overlay, Timeout: "2m", Reaching: []string{}})
+	outcome, out = runTests(t, m, mutate.Asked{Overlay: overlay, Timeout: "2m", Reaching: []string{}})
 	assert.Equal(t, mutate.Survived, outcome)
 	assert.Empty(t, out, "the package's tests ran for a line none of them reaches")
 }