refactor: gitsheets 2.11 — sync afterMove, fetch, single-transaction snapshots

apps/api/package.json

@@ -36,7 +36,7 @@
     "bcryptjs": "^3.0.3",
     "better-sqlite3": "^12.10.0",
     "fastify": "^5.8.5",
-    "gitsheets": "^2.9.0",
+    "gitsheets": "^2.11.0",
     "jose": "^6.2.3",
     "postmark": "^5.1.0",
     "samlify": "^2.13.0",

apps/api/scripts/check-sheet-configs.ts

@@ -22,6 +22,8 @@ import { dirname, join, relative } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { promisify } from 'node:util';
 
+import { openRepo, type Repository } from 'gitsheets';
+
 const exec = promisify(execFile);
 
 // The data repo is private: SSH with a key that can read it (CI: a deploy key).
@@ -40,6 +42,17 @@ async function readTomlTree(dir: string): Promise<Map<string, string>> {
   return out;
 }
 
+/** Every file path under `dir` at `ref`, recursively. */
+async function listFilesUnder(repo: Repository, ref: string, dir: string): Promise<string[]> {
+  const out: string[] = [];
+  for (const entry of (await repo.listTree(ref, dir)) ?? []) {
+    const path = `${dir}/${entry.name}`;
+    if (entry.type === 'tree') out.push(...(await listFilesUnder(repo, ref, path)));
+    else if (entry.type === 'blob') out.push(path);
+  }
+  return out;
+}
+
 function arg(name: string, fallback: string): string {
   const prefix = `--${name}=`;
   return process.argv.find((a) => a.startsWith(prefix))?.slice(prefix.length) ?? fallback;
@@ -51,15 +64,18 @@ async function main(): Promise<number> {
 
   const gitDir = await mkdtemp(join(tmpdir(), 'cfp-sheet-configs-'));
   try {
-    const git = (...args: string[]) => exec('git', ['--git-dir', gitDir, ...args]);
+    // Raw git: gitsheets has no init, and fetches only from a named remote.
     await exec('git', ['init', '--bare', '-q', gitDir]);
-    await git('fetch', '-q', '--depth=1', remote, ref);
+    await exec('git', ['--git-dir', gitDir, 'remote', 'add', 'origin', remote]);
+    const repo = await openRepo({ gitDir });
+    const fetched = await repo.fetch({ remote: 'origin', branch: ref });
+    if (!fetched.ok) throw new Error(`fetching ${remote}#${ref} failed: ${fetched.error}`);
 
-    const { stdout: listing } = await git('ls-tree', '-r', '--name-only', 'FETCH_HEAD', '.gitsheets/');
     const upstream = new Map<string, string>();
-    for (const path of listing.split('\n').filter((p) => p.endsWith('.toml'))) {
-      const { stdout } = await git('show', `FETCH_HEAD:${path}`);
-      upstream.set(path.slice('.gitsheets/'.length), stdout);
+    for (const path of await listFilesUnder(repo, fetched.remoteCommit, '.gitsheets')) {
+      if (!path.endsWith('.toml')) continue;
+      const blob = await repo.readBlob(fetched.remoteCommit, path);
+      upstream.set(path.slice('.gitsheets/'.length), (await blob.read()).toString('utf8'));
     }
 
     const vendored = await readTomlTree(VENDORED_DIR);

apps/api/scripts/scrub-data.ts

@@ -27,15 +27,15 @@
  * would land on another record's path — fails the run instead of being
  * silently collapsed.
  *
- * The source's `.gitsheets/` files (configs + vendored contracts) are copied
- * with `tx.writeFile` into a detached gitsheets orphan commit that the record
- * transaction builds on; the staged tree is then re-committed as the
- * parentless snapshot on `snapshot-<seed>-scrubbed`, HEAD pointed at it and a
- * same-named tag added (`.github/workflows/snapshot.yml` pushes HEAD's
- * branch).
+ * The whole snapshot is one parentless gitsheets transaction on
+ * `snapshot-<seed>-scrubbed`: it stages the source's `.gitsheets/` files
+ * (configs + vendored contracts) with `tx.writeFile`, then upserts the
+ * records, whose sheets read those staged configs. HEAD is pointed at the
+ * branch and a same-named tag added (`.github/workflows/snapshot.yml` pushes
+ * HEAD's branch).
  *
- * If verification fails the script exits non-zero and no target ref moves
- * (the staged objects are left unreferenced).
+ * If verification fails the script exits non-zero and nothing is committed:
+ * the checks run inside the transaction, so a failure discards it.
  */
 
 import { createHash } from 'node:crypto';
@@ -421,39 +421,18 @@ async function initOrOpenTargetRepo(targetPath: string): Promise<GitRunner> {
 }
 
 /**
- * Write a parentless commit holding only the source's `.gitsheets/` files
- * (configs plus vendored contracts), for the record transaction to build on:
- * an orphan `transact({ parent: null })` staging each file with
- * `tx.writeFile`. No branch is named, so the commit is detached and no ref
- * moves.
- *
- * The configs can't go in the record transaction itself: a transaction's
- * sheets load their configs from its parent commit (from HEAD for an orphan),
- * never from files staged with `tx.writeFile` in the same transaction.
+ * Every file path under `dir` at `ref`, recursively (gitsheets `listTree`
+ * lists one level).
  */
-async function writeConfigRootCommit(
-  repo: Repository,
-  configs: ReadonlyArray<{ path: string; content: Buffer }>,
-): Promise<string> {
-  const { commitHash } = await repo.transact(
-    { parent: null, message: 'scrub-data: sheet configs', author: SNAPSHOT_AUTHOR },
-    async (tx) => {
-      for (const { path, content } of configs) tx.writeFile(path, content);
-    },
-  );
-  if (commitHash === null) throw new Error('scrub-data: internal: config root transaction produced no commit');
-  return commitHash;
-}
-
-function gitIdentityEnv(timestamp: string): Record<string, string> {
-  return {
-    GIT_AUTHOR_NAME: SNAPSHOT_AUTHOR.name,
-    GIT_AUTHOR_EMAIL: SNAPSHOT_AUTHOR.email,
-    GIT_COMMITTER_NAME: SNAPSHOT_AUTHOR.name,
-    GIT_COMMITTER_EMAIL: SNAPSHOT_AUTHOR.email,
-    GIT_AUTHOR_DATE: timestamp,
-    GIT_COMMITTER_DATE: timestamp,
-  };
+async function listFilesUnder(repo: Repository, ref: string, dir: string): Promise<string[]> {
+  const entries = (await repo.listTree(ref, dir)) ?? [];
+  const out: string[] = [];
+  for (const entry of entries) {
+    const path = `${dir}/${entry.name}`;
+    if (entry.type === 'tree') out.push(...(await listFilesUnder(repo, ref, path)));
+    else if (entry.type === 'blob') out.push(path);
+  }
+  return out;
 }
 
 /**
@@ -545,11 +524,12 @@ export async function scrubRepo(opts: ScrubOptions): Promise<ScrubResult> {
   }
 
   // -------------------------------------------------------------------------
-  // 6. Stage the snapshot tree through gitsheets: a transaction on top of a
-  //    config-only root commit upserts every scrubbed record (canonical
+  // 6. Write the snapshot through gitsheets: one parentless transaction
+  //    stages the `.gitsheets/` files first (its sheets read configs from the
+  //    transaction's own tree), then upserts every scrubbed record (canonical
   //    serialization, path templates, content-typed formats) and copies
-  //    attachments. The transaction commits on a bare hash parent, so no ref
-  //    moves; its tree is re-committed as the orphan snapshot in step 9.
+  //    attachments. Count parity and verification (steps 7 and 8) run inside
+  //    the handler, so a failure throws before anything commits.
   // -------------------------------------------------------------------------
   const git = await initOrOpenTargetRepo(target);
   // Raw git: resolves a non-bare target's `.git` for openRepo.
@@ -559,26 +539,42 @@ export async function scrubRepo(opts: ScrubOptions): Promise<ScrubResult> {
   // Everything under the source's `.gitsheets/`: sheet configs plus the
   // vendored schema contracts (`contracts/…`) that configs may declare.
   const sourceRef = sourceHeadHash ?? 'HEAD';
-  // Raw git: gitsheets reads a blob at a path but has no subtree listing.
-  const configPaths = (
-    await exec('git', ['ls-tree', '-r', '--name-only', sourceRef, '.gitsheets/'], { cwd: source })
-  ).stdout
-    .split('\n')
-    .filter((p) => p.length > 0);
+  const configPaths = await listFilesUnder(sourceRepo, sourceRef, '.gitsheets');
   const configs = await Promise.all(
     configPaths.map(async (path) => ({ path, content: await (await sourceRepo.readBlob(sourceRef, path)).read() })),
   );
-  const configRoot = await writeConfigRootCommit(targetRepo, configs);
+
+  const timestamp = new Date().toISOString();
+  const branchName = `snapshot-${seed.replace(/[^a-z0-9]/gi, '-')}-scrubbed`;
+  const commitMessage = `snapshot: anonymized data export from ${timestamp}
+
+Generated by apps/api/scripts/scrub-data.ts.
+Source revision: ${sourceHeadHash ?? 'unknown'}
+Seed: ${seed}
+
+This snapshot is intended for contributor onboarding. It contains no
+real names, no emails, no GitHub identities. All personal data has
+been pseudonymized via the documented scrubbing rules.
+
+See apps/api/scripts/scrub-data.ts and specs/behaviors/storage.md.`;
+
+  // An orphan transaction creates its branch and refuses an existing one.
+  // A same-seed re-run (the branch exists) commits detached instead and
+  // moves the branch below only if the tree changed.
+  const existing = await targetRepo.resolveRef(`refs/heads/${branchName}`);
 
   const writtenFiles: SnapshotFile[] = [];
   const scrubCounts: Record<string, number> = {};
-  const staged = await targetRepo.transact(
+  const snapshot = await targetRepo.transact(
     {
-      parent: configRoot,
-      message: 'scrub-data: staged snapshot tree (re-committed as an orphan)',
+      parent: null,
+      ...(existing === null ? { branch: branchName } : {}),
+      message: commitMessage,
       author: SNAPSHOT_AUTHOR,
     },
     async (tx) => {
+      for (const { path, content } of configs) tx.writeFile(path, content);
+
       const pathsBySheet = new Map<string, Map<string, unknown>>();
       for (const { sheetName, record, attachments } of scrubbedRecords) {
         const sheet = tx.sheet(sheetName);
@@ -613,82 +609,55 @@ export async function scrubRepo(opts: ScrubOptions): Promise<ScrubResult> {
           await sheet.setAttachments(record, bytes);
         }
       }
+
+      // ---------------------------------------------------------------------
+      // 7. Count validation — a record that collapsed onto another's path,
+      //    or failed to write, shows up here.
+      // ---------------------------------------------------------------------
+      const countMismatches: string[] = [];
+      for (const [sheetName, sourceCount] of Object.entries(sourceCounts)) {
+        const scrubCount = scrubCounts[sheetName] ?? 0;
+        if (sourceCount !== scrubCount) {
+          countMismatches.push(`Sheet "${sheetName}": source=${sourceCount} scrubbed=${scrubCount}`);
+        }
+      }
+      if (countMismatches.length > 0) {
+        throw new Error(`Record count mismatch:\n${countMismatches.join('\n')}`);
+      }
+
+      // ---------------------------------------------------------------------
+      // 8. Verification pass (before anything commits)
+      // ---------------------------------------------------------------------
+      const verification = verifySnapshotFiles(writtenFiles, new Set(allPersonSlugs));
+      if (!verification.passed) {
+        throw new Error(`Verification failed:\n${verification.errors.join('\n')}`);
+      }
     },
   );
-  const treeHash = staged.treeHash;
-  if (treeHash === null) throw new Error('scrub-data: internal: staging transaction produced no tree');
+  const commitHash = snapshot.commitHash;
+  if (commitHash === null) throw new Error('scrub-data: internal: snapshot transaction produced no commit');
 
   // -------------------------------------------------------------------------
-  // 7. Count validation — a record that collapsed onto another's path, or
-  //    failed to write, shows up here.
+  // 9. Point the branch, HEAD and tag at the snapshot.
   // -------------------------------------------------------------------------
-  const countMismatches: string[] = [];
-  for (const [sheetName, sourceCount] of Object.entries(sourceCounts)) {
-    const scrubCount = scrubCounts[sheetName] ?? 0;
-    if (sourceCount !== scrubCount) {
-      countMismatches.push(`Sheet "${sheetName}": source=${sourceCount} scrubbed=${scrubCount}`);
+  if (existing !== null) {
+    // Same-seed re-run: an unchanged tree is a no-op (the detached commit is
+    // left unreferenced). Equal root listings mean equal trees.
+    const [before, after] = await Promise.all([
+      targetRepo.listTree(existing),
+      targetRepo.listTree(commitHash),
+    ]);
+    if (JSON.stringify(before) === JSON.stringify(after)) {
+      return { sourceCounts, commitHash: null, branchName };
     }
+    // Raw git: gitsheets can't repoint an existing branch at an unrelated
+    // commit (an orphan transact refuses an existing branch).
+    await git('update-ref', `refs/heads/${branchName}`, commitHash);
   }
-  if (countMismatches.length > 0) {
-    throw new Error(`Record count mismatch:\n${countMismatches.join('\n')}`);
-  }
-
-  // -------------------------------------------------------------------------
-  // 8. Verification pass (before any ref moves)
-  // -------------------------------------------------------------------------
-  const verification = verifySnapshotFiles(writtenFiles, new Set(allPersonSlugs));
-  if (!verification.passed) {
-    throw new Error(`Verification failed:\n${verification.errors.join('\n')}`);
-  }
-
-  // -------------------------------------------------------------------------
-  // 9. Commit: orphan squashed commit of the staged tree
-  //
-  //    Stays on raw git, each step for want of a gitsheets API:
-  //    - commit-tree: re-committing the staged tree as a parentless commit.
-  //      gitsheets 2.9's orphan transact starts from an empty tree, and its
-  //      sheets can't see configs staged in the same transaction (see
-  //      writeConfigRootCommit), so the records can't be written straight
-  //      into an orphan.
-  //    - update-ref: gitsheets can't point a branch at an existing commit
-  //      (only commit onto one), and a same-seed re-run overwrites the branch.
-  //    - symbolic-ref HEAD: gitsheets never moves HEAD. The snapshot workflow
-  //      pushes the branch HEAD is on, so HEAD must point at the new branch.
-  //    - tag: gitsheets has no tag API.
-  // -------------------------------------------------------------------------
-  const timestamp = new Date().toISOString();
-  const branchName = `snapshot-${seed.replace(/[^a-z0-9]/gi, '-')}-scrubbed`;
-
-  // Raw git: HEAD's tree hash, which gitsheets doesn't expose.
-  const currentTree = await git('rev-parse', '--verify', '--quiet', 'HEAD^{tree}')
-    .then((r) => r.stdout.trim())
-    .catch(() => null);
-  if (currentTree === treeHash) {
-    return { sourceCounts, commitHash: null, branchName };
-  }
-
-  const commitMessage = `snapshot: anonymized data export from ${timestamp}
-
-Generated by apps/api/scripts/scrub-data.ts.
-Source revision: ${sourceHeadHash ?? 'unknown'}
-Seed: ${seed}
-
-This snapshot is intended for contributor onboarding. It contains no
-real names, no emails, no GitHub identities. All personal data has
-been pseudonymized via the documented scrubbing rules.
-
-See apps/api/scripts/scrub-data.ts and specs/behaviors/storage.md.`;
-
-  const commitEnv = { ...process.env, ...gitIdentityEnv(timestamp) };
-  const { stdout: commitHashRaw } = await exec('git', ['commit-tree', treeHash, '-m', commitMessage], {
-    cwd: target,
-    env: commitEnv,
-  });
-  const commitHash = commitHashRaw.trim();
-
-  await git('update-ref', `refs/heads/${branchName}`, commitHash);
+  // Raw git: gitsheets never moves HEAD. The snapshot workflow pushes the
+  // branch HEAD is on, so HEAD must point at the new branch.
   await git('symbolic-ref', 'HEAD', `refs/heads/${branchName}`);
-  await exec('git', ['tag', '-f', branchName, commitHash], { cwd: target, env: commitEnv });
+  await targetRepo.tag(branchName, commitHash, { force: true });
 
   // Bring a non-bare target's working tree in line with the new HEAD. Raw
   // git: gitsheets works bare-style and never touches a working tree.