test(parity): fail CI on a stale ts-oracle golden

testdata:ts-oracle:check ran in no CI job, so a reference upgrade or a
hand-edited golden went unseen. TestTSOracleGoldensMatchTheReference
bundles the oracle and compares every golden with what the reference's
functions return; task parity:contract, which CI runs after task
reference, now runs it, and testdata:ts-oracle:check is that test.
tsOracle keeps only the bundle and the node process, the lane test
holding it; refreshTSOracleFiles reads, compares and writes, tested
with a fake oracle. build.mjs resolves a symlinked reference/, since
esbuild reports real paths.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: aledbf

Taskfile.yml

@@ -258,8 +258,7 @@ tasks:
     desc: Fail when a testdata/ts-oracle golden differs from what the reference TypeScript functions return
     deps: [reference:compile]
     cmds:
-      - go run ./cmd/devtool ts-oracle
-      - git diff --exit-code -- '*/testdata/ts-oracle/*.json'
+      - go test ./internal/devtools -run '^TestTSOracleGoldensMatchTheReference$' -count=1 -v
 
   reference:
     desc: Install and compile the TypeScript parity oracle
@@ -280,6 +279,8 @@ tasks:
       # Oracle flag-surface gate: every flag the TS oracle declares per command
       # must be in the inventory (needs reference/ checked out, so it lives here).
       - go test ./internal/cli -run TestOracleFlagCoverage -count=1
+      # The ts-oracle goldens must be what the reference's functions return.
+      - go test ./internal/devtools -run '^TestTSOracleGoldensMatchTheReference$' -count=1
       - PARITY_REPORT_FILE={{.ROOT_DIR}}/artifacts/parity-contract.json PARITY_STRICT=true PARITY_LANE=contract PARITY_SKIP_DOCKER=true PARITY_SKIP_NETWORK=true go test ./internal/cli -run TestParityMatrix -count=1 -timeout 10m -v {{.CLI_ARGS}}
 
   parity:semantic:

internal/devtools/command.go

@@ -2,6 +2,7 @@ package devtools
 
 import (
 	"bufio"
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -48,7 +49,7 @@ func Main(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 	case "metrics":
 		err = t.metrics(rest)
 	case "ts-oracle":
-		err = t.tsOracle()
+		err = t.tsOracle(".", false)
 	default:
 		fmt.Fprintf(stderr, "devtool: unknown command %q\n", cmd)
 		return 2
@@ -550,50 +551,68 @@ func (t tool) metrics(args []string) error {
 	return nil
 }
 
-// tsOracle rewrites every internal/**/testdata/ts-oracle/*.json with the
-// answers the reference's own functions give to the golden's requests.
-func (t tool) tsOracle() error {
+// tsOracle rewrites every internal/**/testdata/ts-oracle/*.json under root
+// with the answers the reference's own functions give to the golden's
+// requests; with check it writes nothing and fails on a stale golden.
+//
+// mutate-lane: TestTSOracleGoldensMatchTheReference
+func (t tool) tsOracle(root string, check bool) error {
 	dir, err := os.MkdirTemp("", "ts-oracle-")
 	if err != nil {
 		return err
 	}
 	defer os.RemoveAll(dir)
 	bundle := filepath.Join(dir, "oracle.js")
-	if err := t.run("node", "tools/tsoracle/build.mjs", "reference", bundle); err != nil {
+	if err := t.run("node", filepath.Join(root, "tools/tsoracle/build.mjs"), filepath.Join(root, "reference"), bundle); err != nil {
 		return fmt.Errorf("bundle tools/tsoracle/oracle.ts: %w", err)
 	}
-	paths, err := TSOracleGoldens(os.DirFS("."))
+	return refreshTSOracleFiles(root, func(request []byte) ([]byte, error) {
+		cmd := exec.Command("node", bundle)
+		cmd.Stdin = bytes.NewReader(request)
+		cmd.Stderr = t.errw
+		out, err := cmd.Output()
+		if err != nil {
+			return nil, fmt.Errorf("run the oracle: %w", err)
+		}
+		return out, nil
+	}, check, t.errw)
+}
+
+// refreshTSOracleFiles asks oracle for every golden under root and rewrites
+// the ones whose answers changed, naming each on errw; with check it writes
+// nothing and fails naming the stale ones.
+func refreshTSOracleFiles(root string, oracle func(request []byte) ([]byte, error), check bool, errw io.Writer) error {
+	paths, err := TSOracleGoldens(os.DirFS(root))
 	if err != nil {
 		return err
 	}
 	goldens := map[string][]byte{}
 	for _, p := range paths {
-		if goldens[p], err = os.ReadFile(p); err != nil {
+		if goldens[p], err = os.ReadFile(filepath.Join(root, p)); err != nil {
 			return err
 		}
 	}
-	refreshed, err := RefreshTSOracleGoldens(goldens, func(request []byte) ([]byte, error) {
-		cmd := exec.Command("node", bundle)
-		cmd.Stdin = strings.NewReader(string(request))
-		cmd.Stderr = t.errw
-		out, err := cmd.Output()
-		if err != nil {
-			return nil, fmt.Errorf("run the oracle: %w", err)
-		}
-		return out, nil
-	})
+	refreshed, err := RefreshTSOracleGoldens(goldens, oracle)
 	if err != nil {
 		return err
 	}
+	var stale []string
 	for _, p := range paths {
-		if string(refreshed[p]) == string(goldens[p]) {
+		if bytes.Equal(refreshed[p], goldens[p]) {
 			continue
 		}
-		if err := os.WriteFile(p, refreshed[p], 0o644); err != nil {
+		stale = append(stale, p)
+		if check {
+			continue
+		}
+		if err := os.WriteFile(filepath.Join(root, p), refreshed[p], 0o644); err != nil {
 			return err
 		}
-		fmt.Fprintln(t.errw, "updated", p)
+		fmt.Fprintln(errw, "updated", p)
+	}
+	if check && len(stale) > 0 {
+		return fmt.Errorf("stale ts-oracle goldens, run `go tool task testdata:ts-oracle`: %s", strings.Join(stale, ", "))
 	}
-	fmt.Fprintf(t.errw, "%d ts-oracle goldens checked\n", len(paths))
+	fmt.Fprintf(errw, "%d ts-oracle goldens checked\n", len(paths))
 	return nil
 }

internal/devtools/tsoracle_test.go

@@ -3,6 +3,8 @@ package devtools
 import (
 	"encoding/json"
 	"errors"
+	"os"
+	"path/filepath"
 	"slices"
 	"strings"
 	"testing"
@@ -168,3 +170,63 @@ func TestRefreshTSOracleGoldensAsksWithAnEmptyCasesArray(t *testing.T) {
 		t.Errorf("request = %s, want an empty cases array", asked)
 	}
 }
+
+// refreshTSOracleFiles rewrites only the goldens whose answers changed and
+// names them; with check it writes nothing and fails naming the stale ones.
+func TestRefreshTSOracleFiles(t *testing.T) {
+	root := t.TempDir()
+	write := func(p, content string) {
+		t.Helper()
+		if err := os.MkdirAll(filepath.Dir(filepath.Join(root, p)), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filepath.Join(root, p), []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	const fresh = "{\n  \"cases\": [\n    {\n      \"fn\": \"len\",\n      \"args\": [\n        \"ab\"\n      ],\n      \"result\": 2\n    }\n  ]\n}\n"
+	const stale = `{"cases":[{"fn":"len","args":["abc"]}]}`
+	write("internal/a/testdata/ts-oracle/fresh.json", fresh)
+	write("internal/b/testdata/ts-oracle/stale.json", stale)
+	var sent []byte
+
+	var errw strings.Builder
+	err := refreshTSOracleFiles(root, fakeOracle(&sent), true, &errw)
+	if err == nil || !strings.Contains(err.Error(), "internal/b/testdata/ts-oracle/stale.json") || strings.Contains(err.Error(), "fresh.json") {
+		t.Fatalf("check = %v, want the stale golden named alone", err)
+	}
+	if got, _ := os.ReadFile(filepath.Join(root, "internal/b/testdata/ts-oracle/stale.json")); string(got) != stale {
+		t.Errorf("check rewrote the stale golden: %s", got)
+	}
+
+	errw.Reset()
+	if err := refreshTSOracleFiles(root, fakeOracle(&sent), false, &errw); err != nil {
+		t.Fatal(err)
+	}
+	if want := "updated internal/b/testdata/ts-oracle/stale.json\n2 ts-oracle goldens checked\n"; errw.String() != want {
+		t.Errorf("stderr = %q, want %q", errw.String(), want)
+	}
+	if got, _ := os.ReadFile(filepath.Join(root, "internal/b/testdata/ts-oracle/stale.json")); !strings.Contains(string(got), `"result": 3`) {
+		t.Errorf("stale golden not refreshed: %s", got)
+	}
+	if err := refreshTSOracleFiles(root, fakeOracle(&sent), true, &errw); err != nil {
+		t.Errorf("check after a refresh = %v, want nil", err)
+	}
+	if err := refreshTSOracleFiles(root, func([]byte) ([]byte, error) { return nil, errors.New("node failed") }, true, &errw); err == nil || !strings.Contains(err.Error(), "node failed") {
+		t.Errorf("oracle failure = %v", err)
+	}
+}
+
+// The recorded goldens are what the reference's functions return today. It
+// runs where `task reference` has installed the reference (the parity CI job,
+// through task parity:contract).
+func TestTSOracleGoldensMatchTheReference(t *testing.T) {
+	const root = "../.."
+	if _, err := os.Stat(filepath.Join(root, "reference/node_modules/esbuild")); err != nil {
+		t.Skipf("reference not installed (%v); run `task reference`", err)
+	}
+	var out strings.Builder
+	if err := (tool{out: &out, errw: &out}).tsOracle(root, true); err != nil {
+		t.Fatalf("%v\n%s", err, out.String())
+	}
+}

tools/tsoracle/build.mjs

@@ -3,15 +3,18 @@
 // The plugin exports the module-private reference functions the oracle calls,
 // so they are tested as they are, not copied.
 import { createRequire } from 'node:module';
+import { realpathSync } from 'node:fs';
 import { readFile } from 'node:fs/promises';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 
-const [refDir, outfile] = process.argv.slice(2).map(p => path.resolve(p));
-if (!refDir || !outfile) {
+const [refArg, outfile] = process.argv.slice(2).map(p => path.resolve(p));
+if (!refArg || !outfile) {
 	console.error('usage: node build.mjs <reference dir> <outfile>');
 	process.exit(2);
 }
+// esbuild reports real paths, so a symlinked reference/ must be resolved too.
+const refDir = realpathSync(refArg);
 const esbuild = createRequire(path.join(refDir, 'package.json'))('esbuild');
 
 const privateExports = {