From b50613f5fc03aaf37f082ea69ba0549cc8bdd477 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 14:48:12 +0100 Subject: [PATCH 01/31] feat(run-store): accept a caller-supplied execution-snapshot id The decorator that dual-writes snapshots to Redis has to own the snapshot id, or the same snapshot carries a different id in each store and the comparator chases a difference that is not real. Four of the six snapshot input types had no id field, so four write sites could not carry one. Add it to CompletionSnapshotInput, ExpireSnapshotInput, RescheduleSnapshotInput and CreateExecutionSnapshotInput, and thread it through every nested create. createCancelledRun built its create inline and dropped the id its input already carried; it now passes it too. The field is optional everywhere, so an absent id still falls through to Prisma's @default(cuid()) and no existing caller changes. --- .../src/PostgresRunStore.snapshotId.test.ts | 150 ++++++++++++++++++ .../run-store/src/PostgresRunStore.ts | 7 + .../src/testFixtures/snapshotIdFixture.ts | 135 ++++++++++++++++ internal-packages/run-store/src/types.ts | 12 ++ 4 files changed, 304 insertions(+) create mode 100644 internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts create mode 100644 internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts new file mode 100644 index 00000000000..07948670b28 --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts @@ -0,0 +1,150 @@ +// A caller-supplied snapshot id must survive into Postgres, so the decorator can own the id and both +// stores hold the same one under dual-write. Absent, Prisma's @default(cuid()) still supplies it. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { setupSnapshotIdFixture } from "./testFixtures/snapshotIdFixture.js"; + +describe("PostgresRunStore caller-supplied snapshot id", () => { + postgresTest("completeAttemptSuccess writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + id, + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const snapshot = await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } }); + expect(snapshot).not.toBeNull(); + expect(snapshot!.runId).toBe(run.id); + }); + + postgresTest("expireRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + id, + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("expireParkedRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + const id = generateInternalId(); + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { + id, + engine: "V2", + executionStatus: "FINISHED", + description: "Parked run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(result.count).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("rescheduleRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot: { + id, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("createExecutionSnapshot writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + const created = await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(created.id).toBe(id); + }); + + postgresTest("an absent id still gets a generated one", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const created = await store.createExecutionSnapshot({ + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(created.id).toMatch(/^c[a-z0-9]{24}$/); + }); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index b7d5086431b..e860bd6c67a 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -818,6 +818,7 @@ export class PostgresRunStore implements RunStore { ...params.data, executionSnapshots: { create: { + id: params.snapshot.id, engine: params.snapshot.engine, executionStatus: params.snapshot.executionStatus, description: params.snapshot.description, @@ -925,6 +926,7 @@ export class PostgresRunStore implements RunStore { costInCents: data.costInCents, executionSnapshots: { create: { + id: data.snapshot.id, executionStatus: data.snapshot.executionStatus, description: data.snapshot.description, runStatus: data.snapshot.runStatus, @@ -1131,6 +1133,7 @@ export class PostgresRunStore implements RunStore { error: data.error as Prisma.InputJsonValue, executionSnapshots: { create: { + id: data.snapshot.id, engine: data.snapshot.engine, executionStatus: data.snapshot.executionStatus, description: data.snapshot.description, @@ -1365,6 +1368,7 @@ export class PostgresRunStore implements RunStore { error: data.error as Prisma.InputJsonValue, executionSnapshots: { create: { + id: data.snapshot.id, engine: data.snapshot.engine, executionStatus: data.snapshot.executionStatus, description: data.snapshot.description, @@ -1438,6 +1442,7 @@ export class PostgresRunStore implements RunStore { ...(data.snapshot && { executionSnapshots: { create: { + id: data.snapshot.id, engine: "V2", executionStatus: data.snapshot.executionStatus ?? "DELAYED", description: @@ -1969,6 +1974,7 @@ export class PostgresRunStore implements RunStore { prisma: PrismaClientOrTransaction ): Promise> { const { + id, run, snapshot, previousSnapshotId, @@ -1988,6 +1994,7 @@ export class PostgresRunStore implements RunStore { const newSnapshot = await prisma.taskRunExecutionSnapshot.create({ data: { + id, engine: "V2", executionStatus: snapshot.executionStatus, description: snapshot.description, diff --git a/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts b/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts new file mode 100644 index 00000000000..17c0248312b --- /dev/null +++ b/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts @@ -0,0 +1,135 @@ +// Shared setup for the snapshot-id, snapshot-writes and entry-parity suites. Modelled on the +// seedEnvironment/buildCreateRunInput pair in PostgresRunStore.test.ts; the slugs are suffixed so +// several fixtures can coexist in one database. +import type { PrismaClient, TaskRunStatus } from "@trigger.dev/database"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import type { CreateRunData } from "../types.js"; + +export type SnapshotFixtureEnv = { + id: string; + type: "DEVELOPMENT"; + projectId: string; + organizationId: string; +}; + +export type SnapshotIdFixture = { + run: { id: string }; + env: SnapshotFixtureEnv; +}; + +export async function seedSnapshotEnvironment(prisma: PrismaClient): Promise { + const suffix = generateInternalId().slice(-12); + + const organization = await prisma.organization.create({ + data: { title: `Snapshot Org ${suffix}`, slug: `snapshot-org-${suffix}` }, + }); + + const project = await prisma.project.create({ + data: { + name: `Snapshot Project ${suffix}`, + slug: `snapshot-project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: `dev-${suffix}`, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + + return { + id: environment.id, + type: "DEVELOPMENT", + projectId: project.id, + organizationId: organization.id, + }; +} + +export function buildCreateRunData(runId: string, env: SnapshotFixtureEnv): CreateRunData { + return { + id: runId, + engine: "V2", + status: "PENDING", + friendlyId: `run_${runId.slice(-16)}`, + runtimeEnvironmentId: env.id, + environmentType: env.type, + organizationId: env.organizationId, + projectId: env.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${runId.slice(-8)}`, + spanId: `span_${runId.slice(-8)}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +export type SnapshotWorkerFixture = { workerId: string; taskId: string }; + +/** + * Seeds a BackgroundWorker and one of its tasks. The snapshot's `workerId` and the run's + * `lockedById` are both foreign keys, so a made-up id fails the constraint rather than the + * assertion, and the test reports a fixture fault as if it were a parity fault. + */ +export async function seedSnapshotWorker( + prisma: PrismaClient, + env: SnapshotFixtureEnv +): Promise { + const suffix = generateInternalId().slice(-12); + + const worker = await prisma.backgroundWorker.create({ + data: { + friendlyId: `worker_${suffix}`, + engine: "V2", + contentHash: `hash_${suffix}`, + projectId: env.projectId, + runtimeEnvironmentId: env.id, + version: "20260824.1", + metadata: {}, + }, + }); + + const task = await prisma.backgroundWorkerTask.create({ + data: { + slug: "my-task", + friendlyId: `task_${suffix}`, + filePath: "src/trigger/my-task.ts", + exportName: "myTask", + workerId: worker.id, + projectId: env.projectId, + runtimeEnvironmentId: env.id, + }, + }); + + return { workerId: worker.id, taskId: task.id }; +} + +/** + * Seeds an environment plus one run in `status`, with no execution snapshot. The suites that use it + * assert on the snapshot rows a store method writes, so the run must start with none. + */ +export async function setupSnapshotIdFixture( + prisma: PrismaClient, + opts?: { status?: TaskRunStatus } +): Promise { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: opts?.status ?? "PENDING" }, + }); + + return { run: { id: runId }, env }; +} diff --git a/internal-packages/run-store/src/types.ts b/internal-packages/run-store/src/types.ts index 7c7f9566893..5bde5950459 100644 --- a/internal-packages/run-store/src/types.ts +++ b/internal-packages/run-store/src/types.ts @@ -43,6 +43,9 @@ export type CreateRunSnapshotInput = { }; export type CompletionSnapshotInput = { + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; executionStatus: "FINISHED"; description: string; runStatus: TaskRunStatus; @@ -64,6 +67,9 @@ export type PromotePendingVersionArgs = { }; export type ExpireSnapshotInput = { + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; engine: "V2"; executionStatus: "FINISHED"; description: string; @@ -75,6 +81,9 @@ export type ExpireSnapshotInput = { }; export type RescheduleSnapshotInput = { + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; environmentId: string; environmentType: RuntimeEnvironmentType; projectId: string; @@ -292,6 +301,9 @@ export type TaskRunWithWaitpoint = TaskRun & { associatedWaitpoint: Waitpoint | * input — callers pass the high-level shape, not a raw Prisma `data`/`include`. */ export type CreateExecutionSnapshotInput = { + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; run: { id: string; status: TaskRunStatus; attemptNumber?: number | null }; snapshot: { executionStatus: TaskRunExecutionStatus; From db8390cd0cd2f96a23dd09dd122fe2d65193c937 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 14:48:12 +0100 Subject: [PATCH 02/31] feat(run-store): add a generated pass-through RunStore base for decorators RunStore has 71 members. A decorator that intercepts a dozen of them should not restate the other 59 forwarders alongside its real logic, and hand-writing them invites a typo no test would catch. Generate the base from the interface instead. The generator also emits the member-name lists, so the suite can assert that the class and the interface hold exactly the same members: a method added to RunStore and not to the base fails a test rather than becoming a silent hole in the decorator. The one data property on the interface becomes a getter over the delegate, read live rather than captured, so a delegate whose client changes is not cached. --- .../scripts/generateDelegatingRunStore.ts | 190 +++++++++++ .../run-store/src/delegatingRunStore.test.ts | 72 +++++ .../run-store/src/delegatingRunStore.ts | 300 ++++++++++++++++++ .../run-store/src/runStoreMethodNames.ts | 83 +++++ 4 files changed, 645 insertions(+) create mode 100644 internal-packages/run-store/scripts/generateDelegatingRunStore.ts create mode 100644 internal-packages/run-store/src/delegatingRunStore.test.ts create mode 100644 internal-packages/run-store/src/delegatingRunStore.ts create mode 100644 internal-packages/run-store/src/runStoreMethodNames.ts diff --git a/internal-packages/run-store/scripts/generateDelegatingRunStore.ts b/internal-packages/run-store/scripts/generateDelegatingRunStore.ts new file mode 100644 index 00000000000..df901f3a2f7 --- /dev/null +++ b/internal-packages/run-store/scripts/generateDelegatingRunStore.ts @@ -0,0 +1,190 @@ +// One-off generator for the RunStore pass-through base. +// +// The base class is mechanical: 80-odd near-identical forwarders. Generating it removes the chance +// of a hand-typo that no test would catch, and turns "did we miss a method" into a diff rather than +// a review. Re-run after any change to the RunStore interface: +// +// pnpm exec tsx scripts/generateDelegatingRunStore.ts +// +// The interface is scanned directly rather than through the TypeScript compiler API, because +// `require("typescript")` resolves to a stub in this workspace. A parsing miss cannot pass silently: +// delegatingRunStore.test.ts asserts the class and the interface hold exactly the same member set, +// and `implements RunStore` fails typecheck if a method is absent. +import { readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +const source = readFileSync(join(root, "src/types.ts"), "utf8"); + +/** Replaces every comment and string body with spaces, so a brace inside one cannot move the depth. */ +function blankCommentsAndStrings(text: string): string { + let out = ""; + let i = 0; + while (i < text.length) { + const two = text.slice(i, i + 2); + if (two === "//") { + const end = text.indexOf("\n", i); + const stop = end === -1 ? text.length : end; + out += " ".repeat(stop - i); + i = stop; + } else if (two === "/*") { + const end = text.indexOf("*/", i + 2); + const stop = end === -1 ? text.length : end + 2; + out += text.slice(i, stop).replace(/[^\n]/g, " "); + i = stop; + } else if (text[i] === '"' || text[i] === "'" || text[i] === "`") { + const quote = text[i]; + let j = i + 1; + while (j < text.length && text[j] !== quote) { + j += text[j] === "\\" ? 2 : 1; + } + out += quote + " ".repeat(Math.max(0, j - i - 1)) + (text[j] ?? ""); + i = j + 1; + } else { + out += text[i]; + i += 1; + } + } + return out; +} + +const blanked = blankCommentsAndStrings(source); + +const declaration = "export interface RunStore {"; +const start = blanked.indexOf(declaration); +if (start === -1) { + throw new Error("export interface RunStore not found in src/types.ts"); +} + +const bodyStart = start + declaration.length; +let depth = 1; +let bodyEnd = bodyStart; +while (bodyEnd < blanked.length && depth > 0) { + const ch = blanked[bodyEnd]; + if (ch === "{") depth += 1; + else if (ch === "}") depth -= 1; + if (depth > 0) bodyEnd += 1; +} +if (depth !== 0) { + throw new Error("unbalanced braces while reading the RunStore interface body"); +} + +const body = blanked.slice(bodyStart, bodyEnd); + +// Members are separated by `;` at nesting depth 0. Only `{}`, `()` and `[]` count towards depth: +// angle brackets cannot, because `=>` in a callback parameter type carries an unmatched `>`. +function splitMembers(text: string): { offset: number; length: number }[] { + const spans: { offset: number; length: number }[] = []; + let level = 0; + let from = 0; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (ch === "{" || ch === "(" || ch === "[") level += 1; + else if (ch === "}" || ch === ")" || ch === "]") level -= 1; + else if (ch === ";" && level === 0) { + spans.push({ offset: from, length: i - from }); + from = i + 1; + } + } + if (text.slice(from).trim().length > 0) { + spans.push({ offset: from, length: text.length - from }); + } + return spans; +} + +const methods: string[] = []; +const readonlyProperties: { name: string; type: string }[] = []; +const mutableProperties: string[] = []; + +for (const span of splitMembers(body)) { + const blankedMember = body.slice(span.offset, span.offset + span.length).trim(); + if (blankedMember.length === 0) continue; + + // A method: an optional name then `(` or `<`. A property: a name then `:`. + const asMethod = /^([A-Za-z_$][\w$]*)\s*\??\s*[(<]/.exec(blankedMember); + if (asMethod) { + methods.push(asMethod[1]); + continue; + } + + const asProperty = /^(readonly\s+)?([A-Za-z_$][\w$]*)\s*(\??)\s*:([\s\S]*)$/.exec(blankedMember); + if (asProperty) { + const [, isReadonly, name, , type] = asProperty; + if (isReadonly) { + readonlyProperties.push({ name, type: type.trim() }); + } else { + mutableProperties.push(name); + } + continue; + } + + throw new Error(`could not classify a RunStore member: ${blankedMember.slice(0, 80)}`); +} + +if (mutableProperties.length > 0) { + // A writable data property cannot be forwarded by a getter alone, so the base would silently hold + // its own copy instead of the delegate's. Handle it by hand before regenerating. + throw new Error( + `RunStore declares writable data properties the generator cannot forward: ${mutableProperties.join(", ")}` + ); +} + +const unique = [...new Set(methods)]; +if (unique.length === 0) { + throw new Error("RunStore declares no methods, which cannot be right"); +} + +const memberNames = [...unique, ...readonlyProperties.map((p) => p.name)]; + +const header = `// GENERATED by scripts/generateDelegatingRunStore.ts. Do not edit by hand. +// Regenerate after any change to the RunStore interface: +// pnpm exec tsx scripts/generateDelegatingRunStore.ts +`; + +writeFileSync( + join(root, "src/runStoreMethodNames.ts"), + `${header} +// Every method the RunStore interface declares. The decorator suites enumerate this, so a method +// added to the interface and not to the base fails a test instead of becoming a silent hole. +export const RUN_STORE_METHOD_NAMES = [ +${unique.map((n) => ` "${n}",`).join("\n")} +] as const; + +// Data properties the base exposes as getters over the delegate, not as forwarders. +export const RUN_STORE_PROPERTY_NAMES = [ +${readonlyProperties.map((p) => ` "${p.name}",`).join("\n")} +] as const; +` +); + +writeFileSync( + join(root, "src/delegatingRunStore.ts"), + `${header} +// A pass-through over another RunStore. It exists so a decorator can override the handful of methods +// it cares about and inherit the rest, instead of restating 80-odd forwarders alongside real logic. +// +// Arguments and return values are forwarded untouched. The \`any\` signatures carry each method's +// whole overload set through one forwarder, which is the single thing a generated base cannot +// preserve; a subclass that overrides a method restates the real signature there. +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { RunStore } from "./types.js"; + +export class DelegatingRunStore implements RunStore { + constructor(protected readonly delegate: RunStore) {} + +${readonlyProperties + // Indexed access rather than the written type, so the getter needs no import of its own and + // follows the interface if that type is ever changed. + .map((p) => ` get ${p.name}(): RunStore["${p.name}"] {\n return this.delegate.${p.name};\n }`) + .join("\n\n")}${readonlyProperties.length > 0 ? "\n\n" : ""}${unique + .map((n) => ` ${n}(...args: any[]): any {\n return (this.delegate as any).${n}(...args);\n }`) + .join("\n\n")} +} +` +); + +console.log( + `generated ${unique.length} forwarders and ${readonlyProperties.length} getters ` + + `(${memberNames.length} members total)` +); diff --git a/internal-packages/run-store/src/delegatingRunStore.test.ts b/internal-packages/run-store/src/delegatingRunStore.test.ts new file mode 100644 index 00000000000..7065f801d71 --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.test.ts @@ -0,0 +1,72 @@ +// The base must forward EVERY RunStore member. A method added to the interface and not to the base +// is a silent hole in the decorator built on top of it, so this suite enumerates the generated name +// lists rather than restating them by hand. Regenerate the base and both lists together: +// pnpm exec tsx scripts/generateDelegatingRunStore.ts +import { describe, expect, it } from "vitest"; +import { DelegatingRunStore } from "./delegatingRunStore.js"; +import { RUN_STORE_METHOD_NAMES, RUN_STORE_PROPERTY_NAMES } from "./runStoreMethodNames.js"; +import type { RunStore } from "./types.js"; + +function recordingDelegate(): { store: RunStore; calls: { name: string; args: unknown[] }[] } { + const calls: { name: string; args: unknown[] }[] = []; + const store: Record = {}; + + for (const name of RUN_STORE_METHOD_NAMES) { + store[name] = (...args: unknown[]) => { + calls.push({ name, args }); + return `result:${name}`; + }; + } + for (const name of RUN_STORE_PROPERTY_NAMES) { + store[name] = `property:${name}`; + } + + return { store: store as unknown as RunStore, calls }; +} + +describe("DelegatingRunStore", () => { + it("forwards every RunStore method to the delegate, arguments untouched", () => { + const { store, calls } = recordingDelegate(); + const base = new DelegatingRunStore(store) as unknown as Record< + string, + (...args: unknown[]) => unknown + >; + + for (const name of RUN_STORE_METHOD_NAMES) { + expect(base[name]("arg-one", "arg-two")).toBe(`result:${name}`); + } + + expect(calls.map((c) => c.name)).toEqual([...RUN_STORE_METHOD_NAMES]); + for (const call of calls) { + expect(call.args).toEqual(["arg-one", "arg-two"]); + } + }); + + it("reads every RunStore data property from the delegate", () => { + const { store } = recordingDelegate(); + const base = new DelegatingRunStore(store) as unknown as Record; + + for (const name of RUN_STORE_PROPERTY_NAMES) { + expect(base[name]).toBe(`property:${name}`); + } + }); + + it("reads a data property live, so a delegate that changes is not cached", () => { + const store = { primaryReadClient: "first" } as unknown as RunStore; + const base = new DelegatingRunStore(store); + + expect(base.primaryReadClient).toBe("first" as unknown); + (store as unknown as Record).primaryReadClient = "second"; + expect(base.primaryReadClient).toBe("second" as unknown); + }); + + it("declares exactly the members the interface declares, and no others", () => { + const own = Object.getOwnPropertyNames(DelegatingRunStore.prototype) + .filter((name) => name !== "constructor") + .sort(); + + const expected = [...RUN_STORE_METHOD_NAMES, ...RUN_STORE_PROPERTY_NAMES].sort(); + + expect(own).toEqual(expected); + }); +}); diff --git a/internal-packages/run-store/src/delegatingRunStore.ts b/internal-packages/run-store/src/delegatingRunStore.ts new file mode 100644 index 00000000000..6266ac5abd1 --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.ts @@ -0,0 +1,300 @@ +// GENERATED by scripts/generateDelegatingRunStore.ts. Do not edit by hand. +// Regenerate after any change to the RunStore interface: +// pnpm exec tsx scripts/generateDelegatingRunStore.ts + +// A pass-through over another RunStore. It exists so a decorator can override the handful of methods +// it cares about and inherit the rest, instead of restating 80-odd forwarders alongside real logic. +// +// Arguments and return values are forwarded untouched. The `any` signatures carry each method's +// whole overload set through one forwarder, which is the single thing a generated base cannot +// preserve; a subclass that overrides a method restates the real signature there. +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { RunStore } from "./types.js"; + +export class DelegatingRunStore implements RunStore { + constructor(protected readonly delegate: RunStore) {} + + get primaryReadClient(): RunStore["primaryReadClient"] { + return this.delegate.primaryReadClient; + } + + runInTransaction(...args: any[]): any { + return (this.delegate as any).runInTransaction(...args); + } + + createRun(...args: any[]): any { + return (this.delegate as any).createRun(...args); + } + + createCancelledRun(...args: any[]): any { + return (this.delegate as any).createCancelledRun(...args); + } + + createFailedRun(...args: any[]): any { + return (this.delegate as any).createFailedRun(...args); + } + + startAttempt(...args: any[]): any { + return (this.delegate as any).startAttempt(...args); + } + + completeAttemptSuccess(...args: any[]): any { + return (this.delegate as any).completeAttemptSuccess(...args); + } + + recordRetryOutcome(...args: any[]): any { + return (this.delegate as any).recordRetryOutcome(...args); + } + + requeueRun(...args: any[]): any { + return (this.delegate as any).requeueRun(...args); + } + + recordBulkActionMembership(...args: any[]): any { + return (this.delegate as any).recordBulkActionMembership(...args); + } + + cancelRun(...args: any[]): any { + return (this.delegate as any).cancelRun(...args); + } + + failRunPermanently(...args: any[]): any { + return (this.delegate as any).failRunPermanently(...args); + } + + finalizeRun(...args: any[]): any { + return (this.delegate as any).finalizeRun(...args); + } + + expireRun(...args: any[]): any { + return (this.delegate as any).expireRun(...args); + } + + expireRunsBatch(...args: any[]): any { + return (this.delegate as any).expireRunsBatch(...args); + } + + lockRunToWorker(...args: any[]): any { + return (this.delegate as any).lockRunToWorker(...args); + } + + parkPendingVersion(...args: any[]): any { + return (this.delegate as any).parkPendingVersion(...args); + } + + promotePendingVersionRuns(...args: any[]): any { + return (this.delegate as any).promotePendingVersionRuns(...args); + } + + expireParkedRun(...args: any[]): any { + return (this.delegate as any).expireParkedRun(...args); + } + + suspendForCheckpoint(...args: any[]): any { + return (this.delegate as any).suspendForCheckpoint(...args); + } + + resumeFromCheckpoint(...args: any[]): any { + return (this.delegate as any).resumeFromCheckpoint(...args); + } + + rescheduleRun(...args: any[]): any { + return (this.delegate as any).rescheduleRun(...args); + } + + enqueueDelayedRun(...args: any[]): any { + return (this.delegate as any).enqueueDelayedRun(...args); + } + + rewriteDebouncedRun(...args: any[]): any { + return (this.delegate as any).rewriteDebouncedRun(...args); + } + + updateMetadata(...args: any[]): any { + return (this.delegate as any).updateMetadata(...args); + } + + clearIdempotencyKey(...args: any[]): any { + return (this.delegate as any).clearIdempotencyKey(...args); + } + + pushTags(...args: any[]): any { + return (this.delegate as any).pushTags(...args); + } + + pushRealtimeStream(...args: any[]): any { + return (this.delegate as any).pushRealtimeStream(...args); + } + + findRun(...args: any[]): any { + return (this.delegate as any).findRun(...args); + } + + findRunOrThrow(...args: any[]): any { + return (this.delegate as any).findRunOrThrow(...args); + } + + findRunOnPrimary(...args: any[]): any { + return (this.delegate as any).findRunOnPrimary(...args); + } + + findRunOrThrowOnPrimary(...args: any[]): any { + return (this.delegate as any).findRunOrThrowOnPrimary(...args); + } + + findRuns(...args: any[]): any { + return (this.delegate as any).findRuns(...args); + } + + findRunsByIds(...args: any[]): any { + return (this.delegate as any).findRunsByIds(...args); + } + + findRunsByIdempotencyKeys(...args: any[]): any { + return (this.delegate as any).findRunsByIdempotencyKeys(...args); + } + + createBatchTaskRunItem(...args: any[]): any { + return (this.delegate as any).createBatchTaskRunItem(...args); + } + + findLatestExecutionSnapshot(...args: any[]): any { + return (this.delegate as any).findLatestExecutionSnapshot(...args); + } + + findExecutionSnapshot(...args: any[]): any { + return (this.delegate as any).findExecutionSnapshot(...args); + } + + findManyExecutionSnapshots(...args: any[]): any { + return (this.delegate as any).findManyExecutionSnapshots(...args); + } + + createExecutionSnapshot(...args: any[]): any { + return (this.delegate as any).createExecutionSnapshot(...args); + } + + findSnapshotCompletedWaitpointIds(...args: any[]): any { + return (this.delegate as any).findSnapshotCompletedWaitpointIds(...args); + } + + findSnapshotCompletedWaitpointIdsWithPresence(...args: any[]): any { + return (this.delegate as any).findSnapshotCompletedWaitpointIdsWithPresence(...args); + } + + findWaitpointConnectedRunIds(...args: any[]): any { + return (this.delegate as any).findWaitpointConnectedRunIds(...args); + } + + findWaitpointCompletedSnapshotIds(...args: any[]): any { + return (this.delegate as any).findWaitpointCompletedSnapshotIds(...args); + } + + blockRunWithWaitpointEdges(...args: any[]): any { + return (this.delegate as any).blockRunWithWaitpointEdges(...args); + } + + countPendingWaitpoints(...args: any[]): any { + return (this.delegate as any).countPendingWaitpoints(...args); + } + + countPendingWaitpointsWithPresence(...args: any[]): any { + return (this.delegate as any).countPendingWaitpointsWithPresence(...args); + } + + createWaitpoint(...args: any[]): any { + return (this.delegate as any).createWaitpoint(...args); + } + + upsertWaitpoint(...args: any[]): any { + return (this.delegate as any).upsertWaitpoint(...args); + } + + findWaitpoint(...args: any[]): any { + return (this.delegate as any).findWaitpoint(...args); + } + + findWaitpointOnPrimary(...args: any[]): any { + return (this.delegate as any).findWaitpointOnPrimary(...args); + } + + findManyWaitpoints(...args: any[]): any { + return (this.delegate as any).findManyWaitpoints(...args); + } + + updateWaitpoint(...args: any[]): any { + return (this.delegate as any).updateWaitpoint(...args); + } + + updateManyWaitpoints(...args: any[]): any { + return (this.delegate as any).updateManyWaitpoints(...args); + } + + forWaitpointCompletion(...args: any[]): any { + return (this.delegate as any).forWaitpointCompletion(...args); + } + + findManyTaskRunWaitpoints(...args: any[]): any { + return (this.delegate as any).findManyTaskRunWaitpoints(...args); + } + + deleteManyTaskRunWaitpoints(...args: any[]): any { + return (this.delegate as any).deleteManyTaskRunWaitpoints(...args); + } + + findTaskRunAttempt(...args: any[]): any { + return (this.delegate as any).findTaskRunAttempt(...args); + } + + createTaskRunCheckpoint(...args: any[]): any { + return (this.delegate as any).createTaskRunCheckpoint(...args); + } + + createBatchTaskRun(...args: any[]): any { + return (this.delegate as any).createBatchTaskRun(...args); + } + + updateBatchTaskRun(...args: any[]): any { + return (this.delegate as any).updateBatchTaskRun(...args); + } + + findBatchTaskRunById(...args: any[]): any { + return (this.delegate as any).findBatchTaskRunById(...args); + } + + findBatchTaskRunByFriendlyId(...args: any[]): any { + return (this.delegate as any).findBatchTaskRunByFriendlyId(...args); + } + + findBatchTaskRunByIdempotencyKey(...args: any[]): any { + return (this.delegate as any).findBatchTaskRunByIdempotencyKey(...args); + } + + updateManyBatchTaskRun(...args: any[]): any { + return (this.delegate as any).updateManyBatchTaskRun(...args); + } + + countBatchTaskRunItems(...args: any[]): any { + return (this.delegate as any).countBatchTaskRunItems(...args); + } + + updateManyBatchTaskRunItems(...args: any[]): any { + return (this.delegate as any).updateManyBatchTaskRunItems(...args); + } + + findManyBatchTaskRunItems(...args: any[]): any { + return (this.delegate as any).findManyBatchTaskRunItems(...args); + } + + findBatchTaskRunItem(...args: any[]): any { + return (this.delegate as any).findBatchTaskRunItem(...args); + } + + upsertWaitpointTag(...args: any[]): any { + return (this.delegate as any).upsertWaitpointTag(...args); + } + + findManyWaitpointTags(...args: any[]): any { + return (this.delegate as any).findManyWaitpointTags(...args); + } +} diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts new file mode 100644 index 00000000000..989c7dc37bb --- /dev/null +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -0,0 +1,83 @@ +// GENERATED by scripts/generateDelegatingRunStore.ts. Do not edit by hand. +// Regenerate after any change to the RunStore interface: +// pnpm exec tsx scripts/generateDelegatingRunStore.ts + +// Every method the RunStore interface declares. The decorator suites enumerate this, so a method +// added to the interface and not to the base fails a test instead of becoming a silent hole. +export const RUN_STORE_METHOD_NAMES = [ + "runInTransaction", + "createRun", + "createCancelledRun", + "createFailedRun", + "startAttempt", + "completeAttemptSuccess", + "recordRetryOutcome", + "requeueRun", + "recordBulkActionMembership", + "cancelRun", + "failRunPermanently", + "finalizeRun", + "expireRun", + "expireRunsBatch", + "lockRunToWorker", + "parkPendingVersion", + "promotePendingVersionRuns", + "expireParkedRun", + "suspendForCheckpoint", + "resumeFromCheckpoint", + "rescheduleRun", + "enqueueDelayedRun", + "rewriteDebouncedRun", + "updateMetadata", + "clearIdempotencyKey", + "pushTags", + "pushRealtimeStream", + "findRun", + "findRunOrThrow", + "findRunOnPrimary", + "findRunOrThrowOnPrimary", + "findRuns", + "findRunsByIds", + "findRunsByIdempotencyKeys", + "createBatchTaskRunItem", + "findLatestExecutionSnapshot", + "findExecutionSnapshot", + "findManyExecutionSnapshots", + "createExecutionSnapshot", + "findSnapshotCompletedWaitpointIds", + "findSnapshotCompletedWaitpointIdsWithPresence", + "findWaitpointConnectedRunIds", + "findWaitpointCompletedSnapshotIds", + "blockRunWithWaitpointEdges", + "countPendingWaitpoints", + "countPendingWaitpointsWithPresence", + "createWaitpoint", + "upsertWaitpoint", + "findWaitpoint", + "findWaitpointOnPrimary", + "findManyWaitpoints", + "updateWaitpoint", + "updateManyWaitpoints", + "forWaitpointCompletion", + "findManyTaskRunWaitpoints", + "deleteManyTaskRunWaitpoints", + "findTaskRunAttempt", + "createTaskRunCheckpoint", + "createBatchTaskRun", + "updateBatchTaskRun", + "findBatchTaskRunById", + "findBatchTaskRunByFriendlyId", + "findBatchTaskRunByIdempotencyKey", + "updateManyBatchTaskRun", + "countBatchTaskRunItems", + "updateManyBatchTaskRunItems", + "findManyBatchTaskRunItems", + "findBatchTaskRunItem", + "upsertWaitpointTag", + "findManyWaitpointTags", +] as const; + +// Data properties the base exposes as getters over the delegate, not as forwarders. +export const RUN_STORE_PROPERTY_NAMES = [ + "primaryReadClient", +] as const; From e8ac9b3ad4e9fdd32f58ae3d099144c6248edb00 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 14:48:12 +0100 Subject: [PATCH 03/31] feat(run-store): build snapshot entries from write-site inputs, with parity tests No nested write site returns the snapshot it created: createRun returns the run, expireParkedRun returns a count, and the rest return a selected TaskRun. So the Redis entry is built from each site's own input plus the caller-minted id. That means every value Postgres derives rather than receives has to be reproduced: the DEQUEUED-to-PENDING rewrite, the four values lockRunToWorker hard-codes, the three rescheduleRun defaults, and the engine column default a completion leaves unset. The parity suite covers all ten physical write sites, comparing the built entry against the row Postgres actually wrote. It caught the dropped id in createCancelledRun. --- .../src/snapshotEntry.parity.test.ts | 397 ++++++++++++++++++ .../run-store/src/snapshotEntry.test.ts | 185 ++++++++ .../run-store/src/snapshotEntry.ts | 168 ++++++++ 3 files changed, 750 insertions(+) create mode 100644 internal-packages/run-store/src/snapshotEntry.parity.test.ts create mode 100644 internal-packages/run-store/src/snapshotEntry.test.ts create mode 100644 internal-packages/run-store/src/snapshotEntry.ts diff --git a/internal-packages/run-store/src/snapshotEntry.parity.test.ts b/internal-packages/run-store/src/snapshotEntry.parity.test.ts new file mode 100644 index 00000000000..8c176178e0a --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.parity.test.ts @@ -0,0 +1,397 @@ +// The entry is built from a write site's input while Postgres builds the row from the same input by +// a different code path. This suite is the only thing that keeps those two paths equal, so it covers +// every one of the ten physical snapshot-create sites in PostgresRunStore. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, +} from "./snapshotEntry.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +/** + * Compares only what the entry claims. The Redis model carries no `updatedAt` and no join rows, and + * it holds `createdAt` as an ISO string, so those are checked separately or not at all. + */ +function assertParity(entry: SnapshotEntryInput, row: Record) { + expect(row.id).toBe(entry.id); + expect(row.runId).toBe(entry.runId); + expect(row.engine).toBe(entry.engine); + expect(row.executionStatus).toBe(entry.executionStatus); + expect(row.description).toBe(entry.description); + expect(row.runStatus).toBe(entry.runStatus); + expect(row.environmentId).toBe(entry.environmentId); + expect(row.environmentType).toBe(entry.environmentType); + expect(row.projectId).toBe(entry.projectId); + expect(row.organizationId).toBe(entry.organizationId); + expect(row.attemptNumber ?? undefined).toBe(entry.attemptNumber ?? undefined); + expect(row.previousSnapshotId ?? undefined).toBe(entry.previousSnapshotId ?? undefined); + expect(row.batchId ?? undefined).toBe(entry.batchId ?? undefined); + expect(row.checkpointId ?? undefined).toBe(entry.checkpointId ?? undefined); + expect(row.workerId ?? undefined).toBe(entry.workerId ?? undefined); + expect(row.runnerId ?? undefined).toBe(entry.runnerId ?? undefined); + expect(row.isValid).toBe(entry.error === undefined); + expect((row.createdAt as Date).toISOString()).toBe(entry.createdAt); +} + +function birthSnapshot(id: string, env: SnapshotFixtureEnv) { + return { + id, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("entry to Postgres row parity", () => { + postgresTest("createRun, legacy schema", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = birthSnapshot(id, env); + + await store.createRun({ data: buildCreateRunData(runId, env), snapshot }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("createRun with an associated waitpoint, legacy schema", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = birthSnapshot(id, env); + + await store.createRun({ + data: buildCreateRunData(runId, env), + snapshot, + associatedWaitpoint: { + id: generateInternalId(), + friendlyId: `waitpoint_${runId.slice(-12)}`, + type: "RUN", + status: "PENDING", + idempotencyKey: generateInternalId(), + userProvidedIdempotencyKey: false, + projectId: env.projectId, + environmentId: env.id, + }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("createRun carries the worker and runner ids", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const { workerId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = { ...birthSnapshot(id, env), workerId, runnerId: "runner_1" }; + + await store.createRun({ data: buildCreateRunData(runId, env), snapshot }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("createCancelledRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = { + ...birthSnapshot(id, env), + executionStatus: "FINISHED" as const, + description: "Run was cancelled", + runStatus: "CANCELED" as const, + }; + + await store.createCancelledRun({ + data: { + ...buildCreateRunData(runId, env), + status: "CANCELED", + error: { type: "STRING_ERROR", raw: "cancelled" }, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0, + }, + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("completeAttemptSuccess", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const snapshot = { + id, + executionStatus: "FINISHED" as const, + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY" as const, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCompletion({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("expireRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const snapshot = { + id, + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromExpire({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("expireParkedRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + const id = generateInternalId(); + const snapshot = { + id, + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Parked run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot, + }); + + expect(result.count).toBe(1); + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromExpire({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("rescheduleRun with every default applied", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + const snapshot = { + id, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromReschedule({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("rescheduleRun with every value supplied", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + const snapshot = { + id, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + executionStatus: "QUEUED" as const, + runStatus: "PENDING" as const, + description: "custom reschedule", + }; + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromReschedule({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("lockRunToWorker", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const previous = await store.createExecutionSnapshot({ + run: { id: run.id, status: "PENDING", attemptNumber: null }, + snapshot: { executionStatus: "QUEUED", description: "Run was queued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const id = generateInternalId(); + const snapshot = { + id, + previousSnapshotId: previous.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }; + + await store.lockRunToWorker(run.id, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromLock({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("createExecutionSnapshot, the standalone site", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + run: { id: run.id, status: "EXECUTING" as const, attemptNumber: 2 }, + snapshot: { executionStatus: "EXECUTING" as const, description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + const created = await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(created.id).toBe(id); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: row.createdAt }, input), + row + ); + }); + + postgresTest("createExecutionSnapshot rewrites a DEQUEUED run status", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + run: { id: run.id, status: "DEQUEUED" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING" as const, description: "Run was dequeued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.runStatus).toBe("PENDING"); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: row.createdAt }, input), + row + ); + }); + + postgresTest("createExecutionSnapshot with an error is invalid in both", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + run: { id: run.id, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description: "Stale write" }, + error: "snapshot is not the latest", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.isValid).toBe(false); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: row.createdAt }, input), + row + ); + }); +}); diff --git a/internal-packages/run-store/src/snapshotEntry.test.ts b/internal-packages/run-store/src/snapshotEntry.test.ts new file mode 100644 index 00000000000..cc624d690a9 --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.test.ts @@ -0,0 +1,185 @@ +// These mappings are values Postgres derives rather than receives. If either side changes and the +// other does not, dual-write silently stores two different documents for one snapshot. The parity +// suite next to this file checks the same thing against a real Postgres row; this one pins the +// rules on their own, so a failure says which rule broke. +import { describe, expect, it } from "vitest"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, + isTerminalEntry, +} from "./snapshotEntry.js"; + +const ctx = { id: "snap_1", runId: "run_1", createdAt: new Date("2026-08-24T00:00:00.000Z") }; +const scope = { + environmentId: "env_1", + environmentType: "DEVELOPMENT" as const, + projectId: "proj_1", + organizationId: "org_1", +}; + +describe("snapshotEntry derived values", () => { + it("rewrites a DEQUEUED run status to PENDING", () => { + const entry = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "DEQUEUED", attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING", description: "d" }, + ...scope, + }); + + expect(entry.runStatus).toBe("PENDING"); + }); + + it("keeps every other run status unchanged", () => { + const entry = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + ...scope, + }); + + expect(entry.runStatus).toBe("EXECUTING"); + }); + + it("applies the lock site's hard-coded values", () => { + const entry = entryFromLock(ctx, { + id: "snap_1", + previousSnapshotId: "snap_0", + attemptNumber: 2, + completedWaitpointIds: [], + completedWaitpointOrder: [], + ...scope, + }); + + expect(entry.executionStatus).toBe("PENDING_EXECUTING"); + expect(entry.description).toBe("Run was dequeued for execution"); + expect(entry.runStatus).toBe("PENDING"); + expect(entry.engine).toBe("V2"); + expect(entry.previousSnapshotId).toBe("snap_0"); + expect(entry.attemptNumber).toBe(2); + }); + + it("applies the reschedule defaults", () => { + const entry = entryFromReschedule(ctx, { ...scope }); + + expect(entry.executionStatus).toBe("DELAYED"); + expect(entry.runStatus).toBe("DELAYED"); + expect(entry.description).toBe("Delayed run was rescheduled to a future date"); + }); + + it("prefers a supplied reschedule value over the default", () => { + const entry = entryFromReschedule(ctx, { + ...scope, + executionStatus: "QUEUED", + runStatus: "PENDING", + description: "custom", + }); + + expect(entry.executionStatus).toBe("QUEUED"); + expect(entry.runStatus).toBe("PENDING"); + expect(entry.description).toBe("custom"); + }); + + it("sets engine V2 on a completion, which Postgres leaves to the column default", () => { + const entry = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + ...scope, + }); + + expect(entry.engine).toBe("V2"); + }); + + it("carries a null completion attemptNumber through as null", () => { + const entry = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: null, + ...scope, + }); + + expect(entry.attemptNumber).toBeNull(); + }); + + it("omits an absent optional rather than writing undefined into the document", () => { + const entry = entryFromExpire(ctx, { + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + ...scope, + }); + + expect(Object.keys(entry)).not.toContain("workerId"); + expect(Object.keys(entry)).not.toContain("attemptNumber"); + expect(JSON.parse(JSON.stringify(entry))).toEqual(entry); + }); + + it("reports a FINISHED entry as terminal and any other as not", () => { + const finished = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + ...scope, + }); + const running = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + ...scope, + }); + + expect(isTerminalEntry(finished)).toBe(true); + expect(isTerminalEntry(running)).toBe(false); + }); + + it("serialises createdAt as an ISO string", () => { + const entry = entryFromReschedule(ctx, { ...scope }); + + expect(entry.createdAt).toBe("2026-08-24T00:00:00.000Z"); + }); + + it("carries the birth site's worker and runner ids", () => { + const entry = entryFromCreateRun(ctx, { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + workerId: "worker_1", + runnerId: "runner_1", + ...scope, + }); + + expect(entry.workerId).toBe("worker_1"); + expect(entry.runnerId).toBe("runner_1"); + expect(entry.executionStatus).toBe("RUN_CREATED"); + }); + + it("never sets the reserved completedWaitpoints field", () => { + const built = [ + entryFromReschedule(ctx, { ...scope }), + entryFromLock(ctx, { + id: "snap_1", + previousSnapshotId: "snap_0", + completedWaitpointIds: ["w_1"], + completedWaitpointOrder: ["w_1"], + ...scope, + }), + entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + completedWaitpoints: [{ id: "w_1", index: 0 }], + ...scope, + }), + ]; + + // The append script mints the pointer as a sidecar field, and rejects an entry that carries one. + for (const entry of built) { + expect(entry.completedWaitpoints).toBeUndefined(); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotEntry.ts b/internal-packages/run-store/src/snapshotEntry.ts new file mode 100644 index 00000000000..748a95f6d1a --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.ts @@ -0,0 +1,168 @@ +// Builds the Redis entry for each execution-snapshot write site, from that site's own INPUT. +// +// Not from the delegate's return value: no nested write site includes the snapshot in what it +// returns. `createRun` returns the run, `expireParkedRun` returns a count, and the rest return a +// selected `TaskRun`. That means every value Postgres derives rather than receives has to be +// reproduced here, and snapshotEntry.parity.test.ts is what keeps the two sides from drifting. +import type { TaskRunStatus } from "@trigger.dev/database"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; +import type { + CompletionSnapshotInput, + CreateExecutionSnapshotInput, + CreateRunSnapshotInput, + ExpireSnapshotInput, + LockSnapshotInput, + RescheduleSnapshotInput, +} from "./types.js"; + +export type EntryBuildContext = { id: string; runId: string; createdAt: Date }; + +/** + * PostgresRunStore.#createExecutionSnapshot rewrites DEQUEUED to PENDING, because older runners + * reject DEQUEUED on a snapshot. Every site that can carry that status must rewrite it identically. + */ +function snapshotRunStatus(status: TaskRunStatus): string { + return status === "DEQUEUED" ? "PENDING" : status; +} + +function base(ctx: EntryBuildContext) { + return { + id: ctx.id, + runId: ctx.runId, + createdAt: ctx.createdAt.toISOString(), + engine: "V2" as const, + }; +} + +export function entryFromCreateRun( + ctx: EntryBuildContext, + snapshot: CreateRunSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +/** + * `completeAttemptSuccess` writes no `engine` column, so Postgres applies the schema default of + * `V2`. The entry states it, because SnapshotEntryInput requires the field. + */ +export function entryFromCompletion( + ctx: EntryBuildContext, + snapshot: CompletionSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + attemptNumber: snapshot.attemptNumber, + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +/** Serves both `expireRun` and `expireParkedRun`; the two write identical snapshot columns. */ +export function entryFromExpire( + ctx: EntryBuildContext, + snapshot: ExpireSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + }; +} + +/** PostgresRunStore.rescheduleRun supplies these three defaults inline, so the entry repeats them. */ +export function entryFromReschedule( + ctx: EntryBuildContext, + snapshot: RescheduleSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus ?? "DELAYED", + description: snapshot.description ?? "Delayed run was rescheduled to a future date", + runStatus: snapshotRunStatus(snapshot.runStatus ?? "DELAYED"), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + }; +} + +/** PostgresRunStore.#lockRunToWorker hard-codes the status, description and run status. */ +export function entryFromLock( + ctx: EntryBuildContext, + snapshot: LockSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: "PENDING_EXECUTING", + description: "Run was dequeued for execution", + runStatus: "PENDING", + previousSnapshotId: snapshot.previousSnapshotId, + ...(snapshot.attemptNumber !== undefined && { attemptNumber: snapshot.attemptNumber }), + ...(snapshot.batchId !== undefined && { batchId: snapshot.batchId }), + ...(snapshot.checkpointId !== undefined && { checkpointId: snapshot.checkpointId }), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +export function entryFromCreateExecutionSnapshot( + ctx: EntryBuildContext, + input: CreateExecutionSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: input.snapshot.executionStatus, + description: input.snapshot.description, + runStatus: snapshotRunStatus(input.run.status), + ...(input.run.attemptNumber !== undefined && + input.run.attemptNumber !== null && { attemptNumber: input.run.attemptNumber }), + ...(input.previousSnapshotId !== undefined && { previousSnapshotId: input.previousSnapshotId }), + ...(input.batchId !== undefined && { batchId: input.batchId }), + environmentId: input.environmentId, + environmentType: input.environmentType, + projectId: input.projectId, + organizationId: input.organizationId, + ...(input.checkpointId !== undefined && { checkpointId: input.checkpointId }), + ...(input.workerId !== undefined && { workerId: input.workerId }), + ...(input.runnerId !== undefined && { runnerId: input.runnerId }), + ...(input.snapshot.metadata !== undefined && + input.snapshot.metadata !== null && { metadata: input.snapshot.metadata }), + ...(input.error !== undefined && { error: input.error }), + }; +} + +/** + * A terminal entry is what makes the append script apply the completion TTL. FINISHED is the only + * terminal execution status; the run-level status is not consulted, because a run reaches its + * terminal state through a FINISHED snapshot in every path. + */ +export function isTerminalEntry(entry: SnapshotEntryInput): boolean { + return entry.executionStatus === "FINISHED"; +} From 95dd2a6676f98eb4bf5ffff0d7b6a352b993503f Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 14:52:25 +0100 Subject: [PATCH 04/31] feat(run-store): add a snapshotWrites flag that omits every snapshot write The last dial position makes the Redis store the sole snapshot writer, so Postgres has to stop writing snapshot rows without changing anything else it does. One constructor flag does that across all ten write sites. With it off, the nine nested creates are omitted and the run mutation still lands; createExecutionSnapshot echoes its input in the shape callers expect rather than inserting; and the completed-waitpoint join inserts are skipped, since they would otherwise link to a row that no longer exists. Defaults to true, so every existing caller and test is unaffected. --- .../PostgresRunStore.snapshotWrites.test.ts | 308 ++++++++++++++++++ .../run-store/src/PostgresRunStore.ts | 279 +++++++++------- 2 files changed, 473 insertions(+), 114 deletions(-) create mode 100644 internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts new file mode 100644 index 00000000000..81d3d3e658e --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts @@ -0,0 +1,308 @@ +// snapshotWrites: false is the redis-only dial position. Every run mutation still lands; no snapshot +// row is written and no completed-waitpoint join row is inserted. The default stays true, so nothing +// changes for any existing caller. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, +} from "./testFixtures/snapshotIdFixture.js"; + +describe("PostgresRunStore snapshotWrites flag", () => { + postgresTest("defaults to writing snapshots", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + }); + + postgresTest("writes the run mutation but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const updated = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(updated.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("createRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await store.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + }); + + postgresTest("createCancelledRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await store.createCancelledRun({ + data: { + ...buildCreateRunData(runId, env), + status: "CANCELED", + error: { type: "STRING_ERROR", raw: "cancelled" }, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0, + }, + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + }); + + postgresTest("expireRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe("EXPIRED"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("expireParkedRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { + engine: "V2", + executionStatus: "FINISHED", + description: "Parked run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(result.count).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("rescheduleRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const delayUntil = new Date(Date.now() + 60_000); + + await store.rescheduleRun(run.id, { + delayUntil, + snapshot: { + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const updated = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(updated.delayUntil?.toISOString()).toBe(delayUntil.toISOString()); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("lockRunToWorker writes the lock but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + + await store.lockRunToWorker(run.id, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: generateInternalId(), + previousSnapshotId: generateInternalId(), + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe("DEQUEUED"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("createExecutionSnapshot echoes the input when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + const echoed = await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 2 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.id).toBe(id); + expect(echoed.runId).toBe(run.id); + expect(echoed.executionStatus).toBe("EXECUTING"); + expect(echoed.attemptNumber).toBe(2); + expect(echoed.isValid).toBe(true); + expect(echoed.checkpoint).toBeNull(); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("the echoed row rewrites a DEQUEUED run status", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const echoed = await store.createExecutionSnapshot({ + id: generateInternalId(), + run: { id: run.id, status: "DEQUEUED", attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING", description: "Run was dequeued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.runStatus).toBe("PENDING"); + }); + + postgresTest("the echoed row reports an errored snapshot as invalid", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const echoed = await store.createExecutionSnapshot({ + id: generateInternalId(), + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Stale write" }, + error: "snapshot is not the latest", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.isValid).toBe(false); + expect(echoed.error).toBe("snapshot is not the latest"); + }); + + postgresTest("createExecutionSnapshot needs an id when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await expect( + store.createExecutionSnapshot({ + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }) + ).rejects.toThrow(/snapshotWrites is off/); + }); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index e860bd6c67a..c88c7464508 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -118,6 +118,13 @@ export type PostgresRunStoreOptions = { maxWait?: number; /** Env-driven P2028-at-acquisition retry config, threaded from the app boundary (IoC). */ transactionStartRetry?: TransactionStartRetryConfig; + /** + * When false the store writes no execution-snapshot rows: every nested `executionSnapshots.create` + * is omitted and `createExecutionSnapshot` echoes its input instead of inserting. Only the + * redis-only dial position sets this, once the Redis store is the sole snapshot writer. + * Defaults to true, so the store behaves exactly as it always has. + */ + snapshotWrites?: boolean; }; // A caller sub-select for a relation: `{ select?, include? }` or `true` for a bare `key: true`. @@ -638,6 +645,7 @@ export class PostgresRunStore implements RunStore { private readonly prisma: RunOpsCapableClient; private readonly readOnlyPrisma: RunOpsCapableClient; private readonly schemaVariant: RunStoreSchemaVariant; + private readonly snapshotWrites: boolean; private readonly maxWait?: number; private readonly transactionStartRetry?: TransactionStartRetryConfig; @@ -650,6 +658,16 @@ export class PostgresRunStore implements RunStore { this.schemaVariant = options.schemaVariant ?? "legacy"; this.maxWait = options.maxWait; this.transactionStartRetry = options.transactionStartRetry; + this.snapshotWrites = options.snapshotWrites ?? true; + } + + /** + * Wraps a nested snapshot create so a single flag removes it everywhere. Prisma treats an absent + * key and `undefined` alike, so spreading an empty object drops the nested write entirely rather + * than sending an empty one. + */ + #nestedSnapshot(create: T): { executionSnapshots: { create: T } } | Record { + return this.snapshotWrites ? { executionSnapshots: { create } } : {}; } // The writer handle in read-client form, so the routing layer can honor a caller-passed client @@ -743,7 +761,7 @@ export class PostgresRunStore implements RunStore { const run = (await this.#writeClientWithoutTransaction(tx).taskRun.create({ data: { ...params.data, - executionSnapshots: { create: snapshotCreate }, + ...this.#nestedSnapshot(snapshotCreate), }, })) as TaskRun; return { ...run, associatedWaitpoint: null }; @@ -755,7 +773,7 @@ export class PostgresRunStore implements RunStore { const run = (await c.taskRun.create({ data: { ...params.data, - executionSnapshots: { create: snapshotCreate }, + ...this.#nestedSnapshot(snapshotCreate), }, })) as TaskRun; @@ -772,9 +790,7 @@ export class PostgresRunStore implements RunStore { }, data: { ...params.data, - executionSnapshots: { - create: snapshotCreate, - }, + ...this.#nestedSnapshot(snapshotCreate), associatedWaitpoint: params.associatedWaitpoint ? { create: params.associatedWaitpoint, @@ -813,24 +829,24 @@ export class PostgresRunStore implements RunStore { ): Promise { const client = tx ?? this.prisma; + const snapshotCreate = { + id: params.snapshot.id, + engine: params.snapshot.engine, + executionStatus: params.snapshot.executionStatus, + description: params.snapshot.description, + runStatus: params.snapshot.runStatus, + environmentId: params.snapshot.environmentId, + environmentType: params.snapshot.environmentType, + projectId: params.snapshot.projectId, + organizationId: params.snapshot.organizationId, + workerId: params.snapshot.workerId, + runnerId: params.snapshot.runnerId, + }; + return client.taskRun.create({ data: { ...params.data, - executionSnapshots: { - create: { - id: params.snapshot.id, - engine: params.snapshot.engine, - executionStatus: params.snapshot.executionStatus, - description: params.snapshot.description, - runStatus: params.snapshot.runStatus, - environmentId: params.snapshot.environmentId, - environmentType: params.snapshot.environmentType, - projectId: params.snapshot.projectId, - organizationId: params.snapshot.organizationId, - workerId: params.snapshot.workerId, - runnerId: params.snapshot.runnerId, - }, - }, + ...this.#nestedSnapshot(snapshotCreate), }, }); } @@ -924,21 +940,19 @@ export class PostgresRunStore implements RunStore { outputType: data.outputType, usageDurationMs: data.usageDurationMs, costInCents: data.costInCents, - executionSnapshots: { - create: { - id: data.snapshot.id, - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - attemptNumber: data.snapshot.attemptNumber, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - workerId: data.snapshot.workerId, - runnerId: data.snapshot.runnerId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + attemptNumber: data.snapshot.attemptNumber, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + workerId: data.snapshot.workerId, + runnerId: data.snapshot.runnerId, + }), }, { select: args.select } ) as Promise>; @@ -1131,19 +1145,17 @@ export class PostgresRunStore implements RunStore { completedAt: data.completedAt, expiredAt: data.expiredAt, error: data.error as Prisma.InputJsonValue, - executionSnapshots: { - create: { - id: data.snapshot.id, - engine: data.snapshot.engine, - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + engine: data.snapshot.engine, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + }), }, { select: args.select } ) as Promise>; @@ -1263,42 +1275,44 @@ export class PostgresRunStore implements RunStore { cliVersion: data.cliVersion ?? undefined, maxDurationInSeconds: data.maxDurationInSeconds ?? undefined, maxAttempts: data.maxAttempts ?? undefined, - executionSnapshots: { - create: { - id: data.snapshot.id, - engine: "V2", - executionStatus: "PENDING_EXECUTING", - description: "Run was dequeued for execution", - runStatus: "PENDING", - attemptNumber: data.snapshot.attemptNumber ?? undefined, - previousSnapshotId: data.snapshot.previousSnapshotId, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - checkpointId: data.snapshot.checkpointId ?? undefined, - batchId: data.snapshot.batchId ?? undefined, - // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas. - completedWaitpointOrder: data.snapshot.completedWaitpointOrder, - workerId: data.snapshot.workerId ?? undefined, - runnerId: data.snapshot.runnerId ?? undefined, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + engine: "V2", + executionStatus: "PENDING_EXECUTING", + description: "Run was dequeued for execution", + runStatus: "PENDING", + attemptNumber: data.snapshot.attemptNumber ?? undefined, + previousSnapshotId: data.snapshot.previousSnapshotId, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + checkpointId: data.snapshot.checkpointId ?? undefined, + batchId: data.snapshot.batchId ?? undefined, + // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas. + completedWaitpointOrder: data.snapshot.completedWaitpointOrder, + workerId: data.snapshot.workerId ?? undefined, + runnerId: data.snapshot.runnerId ?? undefined, + }), }, }); - if (dedicated) { - await this.#connectCompletedWaitpoints( - prisma, - data.snapshot.id, - data.snapshot.completedWaitpointIds - ); - } else { - await this.#connectCompletedWaitpointsLegacy( - prisma, - data.snapshot.id, - data.snapshot.completedWaitpointIds - ); + // The join rows link to the snapshot row above. With snapshot writes off there is no such row, + // so inserting them would leave dangling links for a snapshot that only the Redis store holds. + if (this.snapshotWrites) { + if (dedicated) { + await this.#connectCompletedWaitpoints( + prisma, + data.snapshot.id, + data.snapshot.completedWaitpointIds + ); + } else { + await this.#connectCompletedWaitpointsLegacy( + prisma, + data.snapshot.id, + data.snapshot.completedWaitpointIds + ); + } } return result; @@ -1366,19 +1380,17 @@ export class PostgresRunStore implements RunStore { completedAt: data.completedAt, expiredAt: data.expiredAt, error: data.error as Prisma.InputJsonValue, - executionSnapshots: { - create: { - id: data.snapshot.id, - engine: data.snapshot.engine, - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + engine: data.snapshot.engine, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + }), }, }); } catch (error) { @@ -1439,22 +1451,19 @@ export class PostgresRunStore implements RunStore { data: { delayUntil: data.delayUntil, ...(data.queueTimestamp !== undefined && { queueTimestamp: data.queueTimestamp }), - ...(data.snapshot && { - executionSnapshots: { - create: { - id: data.snapshot.id, - engine: "V2", - executionStatus: data.snapshot.executionStatus ?? "DELAYED", - description: - data.snapshot.description ?? "Delayed run was rescheduled to a future date", - runStatus: data.snapshot.runStatus ?? "DELAYED", - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, - }), + ...(data.snapshot && + this.#nestedSnapshot({ + id: data.snapshot.id, + engine: "V2", + executionStatus: data.snapshot.executionStatus ?? "DELAYED", + description: + data.snapshot.description ?? "Delayed run was rescheduled to a future date", + runStatus: data.snapshot.runStatus ?? "DELAYED", + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + })), }, }); } @@ -1990,6 +1999,51 @@ export class PostgresRunStore implements RunStore { error, } = input; + const completedWaitpointOrder = + completedWaitpoints + ?.filter((c) => c.index !== undefined) + .sort((a, b) => a.index! - b.index!) + .map((w) => w.id) ?? []; + + // Redis-only: no row is written and the decorator owns the document. Echo the input in the shape + // the caller expects, so every caller of this method keeps working while Postgres holds nothing. + if (!this.snapshotWrites) { + if (!id) { + throw new Error( + "PostgresRunStore.createExecutionSnapshot: snapshotWrites is off, so the caller must supply the snapshot id" + ); + } + + const now = new Date(); + return { + id, + engine: "V2", + executionStatus: snapshot.executionStatus, + description: snapshot.description, + previousSnapshotId: previousSnapshotId ?? null, + runId: run.id, + runStatus: run.status === "DEQUEUED" ? "PENDING" : run.status, + attemptNumber: run.attemptNumber ?? null, + batchId: batchId ?? null, + environmentId, + environmentType, + projectId, + organizationId, + checkpointId: checkpointId ?? null, + workerId: workerId ?? null, + runnerId: runnerId ?? null, + metadata: snapshot.metadata ?? null, + completedWaitpointOrder, + isValid: !error, + error: error ?? null, + createdAt: now, + updatedAt: now, + checkpoint: null, + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { checkpoint: true }; + }>; + } + const dedicated = this.schemaVariant === "dedicated"; const newSnapshot = await prisma.taskRunExecutionSnapshot.create({ @@ -2014,10 +2068,7 @@ export class PostgresRunStore implements RunStore { metadata: snapshot.metadata ?? undefined, // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas, so a // cross-DB (NEW-resident) token can be recorded without a Prisma `connect` existence check. - completedWaitpointOrder: completedWaitpoints - ?.filter((c) => c.index !== undefined) - .sort((a, b) => a.index! - b.index!) - .map((w) => w.id), + completedWaitpointOrder, isValid: !error, error, }, From 6cb49c8cedffa9e1f663c291e5b4b33001c985bd Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 15:28:26 +0100 Subject: [PATCH 05/31] feat(run-store): dual-write execution snapshots in a crash-safe order A decorator over any RunStore that also writes execution snapshots to Redis. It overrides only the methods that touch a snapshot and inherits the rest. Write order is the correctness property, and the two orders differ on purpose. A transition writes Postgres first: a crash in the gap leaves a stale latest snapshot, which the heartbeat stall watchdog already heals. A birth writes Redis first: a crash there leaves an unreachable key for a run that does not exist, where Postgres-first would leave a run with no snapshot at all and no way to read one. Each order is chosen so the crash state is the harmless one. A failed transition append retries three times, then hands the run to the repair job. It never rethrows, because Postgres has already committed and a throw would turn a healable gap into a caller-visible error. A failed birth append is survivable before redis-only, where Postgres still holds the snapshot, and refuses at redis-only, where it would otherwise create a run with no snapshot anywhere; refusing works only because the birth append comes first. None of the four non-failure append outcomes enqueues a repair: an absent keyspace is every pre-cutover run's transitions, a fork means another writer advanced the head, a duplicate is a retry that landed, and a cycle mismatch is the store refusing an untrustworthy pointer on purpose. At mode off the decorator makes no Redis call and builds no entry. --- .../run-store/src/snapshotFaultInjection.ts | 39 ++ ...skRunExecutionSnapshotStore.births.test.ts | 285 ++++++++++ .../taskRunExecutionSnapshotStore.off.test.ts | 94 +++ ...ExecutionSnapshotStore.transitions.test.ts | 453 +++++++++++++++ .../src/taskRunExecutionSnapshotStore.ts | 534 ++++++++++++++++++ 5 files changed, 1405 insertions(+) create mode 100644 internal-packages/run-store/src/snapshotFaultInjection.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts diff --git a/internal-packages/run-store/src/snapshotFaultInjection.ts b/internal-packages/run-store/src/snapshotFaultInjection.ts new file mode 100644 index 00000000000..50029b9518d --- /dev/null +++ b/internal-packages/run-store/src/snapshotFaultInjection.ts @@ -0,0 +1,39 @@ +// Test-only seam for the execution-snapshot write protocol. +// +// The protocol's correctness claim is about crashes: whatever the write order leaves behind at each +// boundary must be a state the existing stall-and-repair machinery heals. Proving that needs a crash +// at an exact point, which is what an injector gives. Production never sets one, so each boundary +// costs one optional call. + +/** The three points a crash can land between the two stores' writes. */ +export type SnapshotFaultBoundary = + /** A transition: Postgres has committed and the Redis append has not started. */ + | "afterPgBeforeRedis" + /** A birth: the Redis append has landed and the Postgres insert has not started. */ + | "afterRedisBirthBeforePg" + /** Inside the append retry loop, after at least one attempt has failed. */ + | "midFlushRetry"; + +export type SnapshotFaultInjector = ( + boundary: SnapshotFaultBoundary, + context: { runId: string; snapshotId: string } +) => void; + +/** + * Thrown by a test injector. The write path tells this apart from a real append failure: an injected + * fault models a process that died, so it is rethrown rather than retried, while a real failure is + * retried and then handed to the repair job. + */ +export class InjectedSnapshotFault extends Error { + readonly boundary: SnapshotFaultBoundary; + + constructor(boundary: SnapshotFaultBoundary) { + super(`injected snapshot fault at ${boundary}`); + this.name = "InjectedSnapshotFault"; + this.boundary = boundary; + } +} + +export function isInjectedFault(error: unknown): error is InjectedSnapshotFault { + return error instanceof InjectedSnapshotFault; +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts new file mode 100644 index 00000000000..e94e41f94b6 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts @@ -0,0 +1,285 @@ +// A birth writes Redis FIRST. The order is proved by crashing between the two writes and observing +// which side survived: an orphaned key with no run row is the harmless state, and a run with no +// snapshot at all is the one the order exists to prevent. +import { describe, expect } from "vitest"; +import { postgresAndRedisTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { InjectedSnapshotFault } from "./snapshotFaultInjection.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + opts?: { + mode?: SnapshotStoreMode; + faults?: ConstructorParameters[1]["faults"]; + unreachableRedis?: boolean; + } +) { + // An unreachable port makes every append throw for real, which is the failure the retry loop and + // the mode-dependent refusal are about. A fault injector cannot stand in: an injected fault means + // "the process died", and the two are handled differently on purpose. + const redis = new RedisSnapshotStore({ + redisOptions: opts?.unreachableRedis + ? ({ ...(redisOptions as object), port: 1, retryStrategy: () => null } as never) + : redisOptions, + completedTtlMs: COMPLETED_TTL_MS, + }); + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "dual-write", + ...(opts?.faults && { faults: opts.faults }), + } + ); + + return { decorated, redis }; +} + +function birthSnapshot(id: string, env: SnapshotFixtureEnv) { + return { + id, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +function cancelledData(runId: string, env: SnapshotFixtureEnv) { + return { + ...buildCreateRunData(runId, env), + status: "CANCELED" as const, + error: { type: "STRING_ERROR", raw: "cancelled" } as never, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0 as const, + }; +} + +describe("birth write ordering", () => { + postgresAndRedisTest("writes Redis then Postgres", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(snapshotId, env), + }); + + const read = await redis.getLatest(runId); + expect(read).not.toBeNull(); + expect(read!.entry.id).toBe(snapshotId); + expect(read!.entry.executionStatus).toBe("RUN_CREATED"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest("mints an id when the caller supplies none", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const { id: _omitted, ...withoutId } = birthSnapshot(generateInternalId(), env); + + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot: withoutId }); + + const read = await redis.getLatest(runId); + expect(read).not.toBeNull(); + // The same minted id must reach both stores, or the comparator chases a difference that is + // not real. + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { runId } }); + expect(read!.entry.id).toBe(row.id); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "a crash after the Redis append leaves an orphan key and no run", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + faults: (boundary) => { + if (boundary === "afterRedisBirthBeforePg") throw new InjectedSnapshotFault(boundary); + }, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await expect( + decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }) + ).rejects.toBeInstanceOf(InjectedSnapshotFault); + + // The harmless state: a keyspace nothing can reach, and no run that lacks a snapshot. + expect(await redis.getLatest(runId)).not.toBeNull(); + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "creates the run anyway when the birth append fails before redis-only", + { timeout: 60_000 }, + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + mode: "dual-write", + unreachableRedis: true, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + // Postgres is authoritative in every position before redis-only, so a Redis outage must not + // stop runs being created. + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(snapshotId, env), + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "refuses to create the run when the birth append fails at redis-only", + { timeout: 60_000 }, + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + mode: "redis-only", + unreachableRedis: true, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + // At redis-only Postgres writes no snapshot, so a run created without its Redis birth would + // have no snapshot anywhere. Failing before the run row exists lets the caller retry clean. + await expect( + decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }) + ).rejects.toThrow(); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("createCancelledRun writes Redis first", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + await decorated.createCancelledRun({ + data: cancelledData(runId, env), + snapshot: { + ...birthSnapshot(snapshotId, env), + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + }, + }); + + const read = await redis.getLatest(runId); + expect(read!.entry.id).toBe(snapshotId); + expect(read!.entry.executionStatus).toBe("FINISHED"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "a born-terminal run gets the completion expiry immediately", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await decorated.createCancelledRun({ + data: cancelledData(runId, env), + snapshot: { + ...birthSnapshot(generateInternalId(), env), + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + }, + }); + + // A born-terminal run never transitions again, so the completion TTL has to be applied by + // the birth itself or the keyspace never expires. + const nonTerminal = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(nonTerminal, env), + snapshot: birthSnapshot(generateInternalId(), env), + }); + + const terminal = await redis.getLatest(runId); + const alive = await redis.getLatest(nonTerminal); + expect(terminal).not.toBeNull(); + expect(alive).not.toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { mode: "off" }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await redis.getLatest(runId)).toBeNull(); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts new file mode 100644 index 00000000000..975c68c371a --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts @@ -0,0 +1,94 @@ +// Mode off is the merge-test position: the decorator must be indistinguishable from its delegate and +// must not touch Redis at all. A Redis store whose every member throws proves the second half, and +// enumerating the generated name list proves the first for every method rather than a chosen few. +import { describe, expect, it } from "vitest"; +import { RUN_STORE_METHOD_NAMES } from "./runStoreMethodNames.js"; +import type { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +function explodingRedisStore(): RedisSnapshotStore { + return new Proxy({} as RedisSnapshotStore, { + get(_target, prop) { + return () => { + throw new Error(`the Redis store must not be called at mode off, but ${String(prop)} was`); + }; + }, + }); +} + +function recordingDelegate(): { store: RunStore; calls: string[] } { + const calls: string[] = []; + const store: Record = {}; + + for (const name of RUN_STORE_METHOD_NAMES) { + store[name] = (...args: unknown[]) => { + calls.push(name); + return `result:${name}`; + }; + } + + return { store: store as unknown as RunStore, calls }; +} + +describe("TaskRunExecutionSnapshotStore at mode off", () => { + it("defaults to mode off", () => { + const { store } = recordingDelegate(); + + const decorated = new TaskRunExecutionSnapshotStore(store, { store: explodingRedisStore() }); + + expect(decorated.mode).toBe("off"); + }); + + it("forwards every method to the delegate and never calls Redis", async () => { + const { store, calls } = recordingDelegate(); + const decorated = new TaskRunExecutionSnapshotStore(store, { + store: explodingRedisStore(), + mode: "off", + }) as unknown as Record unknown>; + + for (const name of RUN_STORE_METHOD_NAMES) { + if (name === "runInTransaction") continue; + expect(await decorated[name]("arg-one", "arg-two")).toBe(`result:${name}`); + } + + expect(calls).toEqual(RUN_STORE_METHOD_NAMES.filter((n) => n !== "runInTransaction")); + }); + + it("hands the delegate's own store to a transaction callback", async () => { + const inner = recordingDelegate().store; + let seen: unknown; + const delegate = { + runInTransaction: async ( + _runId: string | undefined, + fn: (store: RunStore, tx: unknown) => Promise + ) => { + await fn(inner, "tx"); + }, + } as unknown as RunStore; + + const decorated = new TaskRunExecutionSnapshotStore(delegate, { + store: explodingRedisStore(), + mode: "off", + }); + + await decorated.runInTransaction("run_1", async (store) => { + seen = store; + }); + + expect(seen).toBe(inner); + }); + + it("reports every other dial position as one that writes Redis", () => { + const { store } = recordingDelegate(); + const modes = ["dual-write", "compare", "redis-read", "redis-only"] as const; + + for (const mode of modes) { + const decorated = new TaskRunExecutionSnapshotStore(store, { + store: explodingRedisStore(), + mode, + }); + expect(decorated.mode).toBe(mode); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts new file mode 100644 index 00000000000..786e3a74453 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts @@ -0,0 +1,453 @@ +// A transition writes Postgres first and Redis second. The order is proved by observation, not by +// reading the code: with the Redis half made to fail, the Postgres row is still there and the caller +// sees no error, which is only possible if Postgres went first. +import { describe, expect } from "vitest"; +import { postgresAndRedisTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { InjectedSnapshotFault } from "./snapshotFaultInjection.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +type Harness = { + decorated: TaskRunExecutionSnapshotStore; + redis: RedisSnapshotStore; + repairs: { runId: string; snapshotId: string; executionStatus: string }[]; + writes: { site: string; outcome: string }[]; +}; + +function harness( + prisma: never, + redisOptions: never, + opts?: { + mode?: SnapshotStoreMode; + faults?: ConstructorParameters[1]["faults"]; + } +): Harness { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const repairs: Harness["repairs"] = []; + const writes: Harness["writes"] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "dual-write", + ...(opts?.faults && { faults: opts.faults }), + onAppendFailure: async (args) => { + repairs.push(args); + }, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: () => {}, + }, + } + ); + + return { decorated, redis, repairs, writes }; +} + +/** + * Creates the run and its keyspace, so a following transition is not skippedNoKeyspace. + * + * The birth is appended through the raw store rather than the decorator, because the decorator's + * own birth path is a separate concern with its own suite. Keeping it out here means a failure in + * this file is a failure of the transition path and nothing else. + */ +async function seedBirth( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + runId: string, + env: SnapshotFixtureEnv +): Promise { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); +} + +function completionInput(env: SnapshotFixtureEnv) { + return { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED" as const, + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY" as const, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }; +} + +function expireInput(env: SnapshotFixtureEnv) { + return { + error: { type: "STRING_ERROR" as const, raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }; +} + +describe("transition write ordering", () => { + postgresAndRedisTest("writes Postgres then Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.completeAttemptSuccess(runId, completionInput(env), { select: { id: true } }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + const read = await redis.getById(runId, row.id); + + expect(read).not.toBeNull(); + expect(read!.entry.id).toBe(row.id); + expect(read!.entry.executionStatus).toBe("FINISHED"); + expect(writes).toContainEqual({ site: "completeAttemptSuccess", outcome: "written" }); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "keeps the Postgres write and enqueues one repair when the append fails", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs } = harness(prisma as never, redisOptions as never, { + faults: (boundary) => { + if (boundary === "afterPgBeforeRedis") throw new InjectedSnapshotFault(boundary); + }, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + // The caller must NOT see an error: the Postgres mutation already committed, and the stall + // watchdog is the designed compensator. + await decorated.completeAttemptSuccess(runId, completionInput(env), { + select: { id: true }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + + expect(await redis.getById(runId, row.id)).toBeNull(); + expect(repairs).toEqual([ + { runId, snapshotId: row.id, executionStatus: "FINISHED" }, + ]); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "treats a transition on a run with no keyspace as skipped, not failed", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); + try { + // No birth: this is every pre-cutover run's first transition after the dial moves. + const { run, env } = await setupSnapshotIdFixture(prisma); + + await decorated.expireRun(run.id, expireInput(env), { select: { id: true } }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + expect(await redis.getLatest(run.id)).toBeNull(); + expect(repairs).toEqual([]); + expect(writes).toEqual([{ site: "expireRun", outcome: "skippedNoKeyspace" }]); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("appends for expireRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.expireRun(runId, expireInput(env), { select: { id: true } }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + const read = await redis.getById(runId, row.id); + expect(read?.entry.description).toBe("Run expired"); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest("appends for expireParkedRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + await prisma.taskRun.update({ where: { id: runId }, data: { status: "PENDING_VERSION" } }); + + const result = await decorated.expireParkedRun(runId, { + ...expireInput(env), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { ...expireInput(env).snapshot, description: "Parked run expired" }, + }); + + expect(result.count).toBe(1); + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + expect((await redis.getById(runId, row.id))?.entry.description).toBe("Parked run expired"); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "appends nothing when expireParkedRun matches no run", + async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + // The run is PENDING, so the delegate's `status: PENDING_VERSION` guard matches nothing. + + const result = await decorated.expireParkedRun(runId, { + ...expireInput(env), + statusReason: "VERSION_NEVER_ARRIVED", + }); + + expect(result.count).toBe(0); + expect(writes.filter((w) => w.site === "expireParkedRun")).toEqual([]); + const latest = await redis.getLatest(runId); + expect(latest?.entry.executionStatus).toBe("RUN_CREATED"); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("appends for rescheduleRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.rescheduleRun(runId, { + delayUntil: new Date(Date.now() + 60_000), + snapshot: { + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "DELAYED" }, + }); + expect((await redis.getById(runId, row.id))?.entry.description).toBe( + "Delayed run was rescheduled to a future date" + ); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "appends nothing when rescheduleRun carries no snapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.rescheduleRun(runId, { delayUntil: new Date(Date.now() + 60_000) }); + + expect(writes.filter((w) => w.site === "rescheduleRun")).toEqual([]); + expect((await redis.getLatest(runId))?.entry.executionStatus).toBe("RUN_CREATED"); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("appends for lockRunToWorker under a CAS", async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + const read = await redis.getById(runId, snapshotId); + expect(read?.entry.executionStatus).toBe("PENDING_EXECUTING"); + expect(read?.entry.previousSnapshotId).toBe(head!.id); + expect(writes).toContainEqual({ site: "lockRunToWorker", outcome: "written" }); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "reports a forked append without enqueuing a repair", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + // A stale previousSnapshotId: another writer advanced the head. A repair cannot help, so the + // outcome is counted and dropped. + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: generateInternalId(), + previousSnapshotId: generateInternalId(), + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + expect(writes).toContainEqual({ site: "lockRunToWorker", outcome: "forked" }); + expect(repairs).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("appends for the standalone createExecutionSnapshot", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + const created = await decorated.createExecutionSnapshot({ + run: { id: runId, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const read = await redis.getById(runId, created.id); + expect(read).not.toBeNull(); + expect(read!.entry.executionStatus).toBe("EXECUTING"); + // The standalone path is the one whose delegate returns the row, so both stores agree exactly. + expect(read!.entry.createdAt).toBe(created.createdAt.toISOString()); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never, { mode: "off" }); + try { + const { run, env } = await setupSnapshotIdFixture(prisma); + + await decorated.completeAttemptSuccess(run.id, completionInput(env), { + select: { id: true }, + }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + expect(await redis.getLatest(run.id)).toBeNull(); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts new file mode 100644 index 00000000000..69658ecaad1 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -0,0 +1,534 @@ +// Decorates any RunStore so execution snapshots also land in Redis. It overrides only the methods +// that touch a snapshot and inherits the rest from the generated pass-through base. +// +// Write ORDER is the correctness property, and the two orders are deliberately different: +// +// transition Postgres first, Redis second. A crash in the gap leaves a run whose latest snapshot +// is stale, which is exactly the state the heartbeat stall watchdog already heals. +// birth Redis first, Postgres second. A crash in the gap leaves an unreachable key for a run +// that does not exist. Postgres-first would leave a run with no snapshot at all, and +// getLatestExecutionSnapshot treats that as a hard error. +// +// Each order is chosen so the crash state is the harmless one. A lost cross-store write is never +// recovered by a transaction or an outbox: recovery is always the existing stall-and-repair job. +import { Logger } from "@trigger.dev/core/logger"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { DelegatingRunStore } from "./delegatingRunStore.js"; +import type { RedisSnapshotStore, SnapshotEntryInput } from "./redisSnapshotStore.js"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, + isTerminalEntry, +} from "./snapshotEntry.js"; +import { isInjectedFault, type SnapshotFaultInjector } from "./snapshotFaultInjection.js"; +import type { + CompletionSnapshotInput, + CreateCancelledRunInput, + CreateExecutionSnapshotInput, + CreateRunInput, + ExpireSnapshotInput, + LockRunData, + RescheduleSnapshotInput, + RunStore, + TaskRunWithWaitpoint, +} from "./types.js"; +import type { Prisma, PrismaClientOrTransaction, TaskRun } from "@trigger.dev/database"; + +/** One initial attempt plus three retries, per the write protocol. */ +const APPEND_ATTEMPTS = 4; + +/** + * The rollout dial. Postgres stays fully written and authoritative in every position before + * `redis-only`, so every earlier position rolls back losslessly by turning the dial down. + * + * `compare` writes exactly as `dual-write` does. Its sampled dual-read and diff are a later ticket; + * the position is named here so the dial does not have to widen once that lands. + */ +export type SnapshotStoreMode = "off" | "dual-write" | "compare" | "redis-read" | "redis-only"; + +/** + * Enqueues the existing `repairSnapshot` job for a run whose append was lost. The decorator lives in + * run-store and cannot reach the engine's worker, so the binding is injected. That binding must + * reuse the stall watchdog's job id for the run, or the watchdog and this path can start two + * concurrent repairs on one run. + */ +export type SnapshotRepairEnqueuer = (args: { + runId: string; + snapshotId: string; + executionStatus: string; +}) => Promise; + +export type DecoratorMetrics = { + recordWrite(site: string, outcome: string): void; + recordAppendFailed(site: string): void; + recordRead(method: string, source: "redis" | "postgres"): void; +}; + +export type TaskRunExecutionSnapshotStoreOptions = { + store: RedisSnapshotStore; + /** Defaults to `off`, which is a pure pass-through that never touches Redis. */ + mode?: SnapshotStoreMode; + /** Percentage of runs whose reads come from Redis at `redis-read` and `redis-only`. Defaults to 0. */ + readPercent?: number; + onAppendFailure?: SnapshotRepairEnqueuer; + faults?: SnapshotFaultInjector; + metrics?: DecoratorMetrics; + logger?: Logger; + /** + * Internal. Set only by the staging facade this class builds for `runInTransaction`. When present, + * an intercepted write does its Postgres half and pushes its entry here instead of appending, and + * the outer instance flushes the buffer after the transaction commits. + */ + staging?: SnapshotEntryInput[]; +}; + +export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { + readonly mode: SnapshotStoreMode; + protected readonly redis: RedisSnapshotStore; + protected readonly readPercent: number; + protected readonly onAppendFailure?: SnapshotRepairEnqueuer; + protected readonly faults?: SnapshotFaultInjector; + protected readonly metrics?: DecoratorMetrics; + protected readonly logger: Logger; + protected readonly staging?: SnapshotEntryInput[]; + + constructor(delegate: RunStore, options: TaskRunExecutionSnapshotStoreOptions) { + super(delegate); + this.redis = options.store; + this.mode = options.mode ?? "off"; + this.readPercent = options.readPercent ?? 0; + this.onAppendFailure = options.onAppendFailure; + this.faults = options.faults; + this.metrics = options.metrics; + this.logger = options.logger ?? new Logger("TaskRunExecutionSnapshotStore", "debug"); + this.staging = options.staging; + } + + /** True in every position that appends to Redis. */ + protected get writesRedis(): boolean { + return this.mode !== "off"; + } + + /** + * The staging facade. Two writes share one Postgres transaction here, and the Redis half of each + * cannot run until that transaction commits: a rollback would otherwise leave Redis holding a + * transition that never happened. + * + * The callback gets a second decorator over the transaction-bound store, carrying a staging + * buffer. An intercepted write does its Postgres half through that store and pushes its entry + * onto the buffer. After the transaction resolves, this instance flushes the buffer in order + * through the same retry-and-repair path a lone transition uses. If the callback throws, the + * delegate rejects, the flush never runs, and the buffer goes away with the stack — so the + * Postgres rollback and the Redis silence agree. + */ + override async runInTransaction( + runId: string | undefined, + fn: (store: RunStore, tx: PrismaClientOrTransaction) => Promise + ): Promise { + if (!this.writesRedis) { + // At `off` the callback must receive the delegate's own store, untouched, so a transaction + // behaves exactly as it does without the decorator in the chain. + return this.delegate.runInTransaction(runId, fn); + } + + const staged: SnapshotEntryInput[] = []; + + const result = await this.delegate.runInTransaction(runId, (store, tx) => + fn(this.#wrap(store, staged), tx) + ); + + // The transaction committed. Only now can a snapshot claim its partner is durable. + for (const entry of staged) { + await this.#appendTransition("runInTransaction", entry); + } + + return result; + } + + /** + * `forWaitpointCompletion` hands the caller a store to apply a completion on. No snapshot write + * goes through that handle today, so wrapping it changes nothing now; leaving it unwrapped is the + * one hole that would let a future snapshot write bypass the decorator with no signal at all. + */ + override async forWaitpointCompletion( + waitpointId: string, + context: Parameters[1] + ): Promise { + const store = await this.delegate.forWaitpointCompletion(waitpointId, context); + + if (!this.writesRedis) { + return store; + } + + return this.#wrap(store); + } + + /** + * A second decorator over another store, sharing this one's options. One class in both roles keeps + * the write-ordering logic in exactly one place. Passing no buffer gives a plain decorator that + * appends immediately; passing one makes it stage instead. + */ + #wrap(store: RunStore, staging?: SnapshotEntryInput[]): TaskRunExecutionSnapshotStore { + return new TaskRunExecutionSnapshotStore(store, { + store: this.redis, + mode: this.mode, + readPercent: this.readPercent, + logger: this.logger, + ...(this.onAppendFailure && { onAppendFailure: this.onAppendFailure }), + ...(this.faults && { faults: this.faults }), + ...(this.metrics && { metrics: this.metrics }), + ...(staging && { staging }), + }); + } + + // --------------------------------------------------------------------------------------------- + // Births: Redis first, Postgres second. + // --------------------------------------------------------------------------------------------- + + override async createRun( + params: CreateRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + if (!this.writesRedis) { + return this.delegate.createRun(params, tx); + } + + const ctx = this.#context(params.data.id, params.snapshot.id); + const snapshot = { ...params.snapshot, id: ctx.id }; + + await this.#appendBirth("createRun", entryFromCreateRun(ctx, snapshot)); + + return this.delegate.createRun({ ...params, snapshot }, tx); + } + + override async createCancelledRun( + params: CreateCancelledRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + if (!this.writesRedis) { + return this.delegate.createCancelledRun(params, tx); + } + + const ctx = this.#context(params.data.id, params.snapshot.id); + const snapshot = { ...params.snapshot, id: ctx.id }; + + await this.#appendBirth("createCancelledRun", entryFromCreateRun(ctx, snapshot)); + + return this.delegate.createCancelledRun({ ...params, snapshot }, tx); + } + + // --------------------------------------------------------------------------------------------- + // Transitions: Postgres first, Redis second. + // --------------------------------------------------------------------------------------------- + + override async completeAttemptSuccess( + runId: string, + data: { + completedAt: Date; + output?: string; + outputType: string; + usageDurationMs: number; + costInCents: number; + snapshot: CompletionSnapshotInput; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.completeAttemptSuccess(runId, data, args, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { ...data, snapshot: { ...data.snapshot, id: ctx.id } }; + + const result = await this.delegate.completeAttemptSuccess(runId, withId, args, tx); + + await this.#appendTransition( + "completeAttemptSuccess", + entryFromCompletion(ctx, withId.snapshot) + ); + return result; + } + + override async expireRun( + runId: string, + data: { error: unknown; completedAt: Date; expiredAt: Date; snapshot: ExpireSnapshotInput }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.expireRun(runId, data as never, args, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { ...data, snapshot: { ...data.snapshot, id: ctx.id } }; + + const result = await this.delegate.expireRun(runId, withId as never, args, tx); + + await this.#appendTransition("expireRun", entryFromExpire(ctx, withId.snapshot)); + return result; + } + + override async expireParkedRun( + runId: string, + data: { + error: unknown; + completedAt: Date; + expiredAt: Date; + statusReason: string; + snapshot: ExpireSnapshotInput; + }, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + if (!this.writesRedis) { + return this.delegate.expireParkedRun(runId, data as never, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { ...data, snapshot: { ...data.snapshot, id: ctx.id } }; + + const result = await this.delegate.expireParkedRun(runId, withId as never, tx); + + // The delegate writes nothing when the run is no longer PENDING_VERSION, so neither does Redis. + if (result.count > 0) { + await this.#appendTransition("expireParkedRun", entryFromExpire(ctx, withId.snapshot)); + } + return result; + } + + override async rescheduleRun( + runId: string, + data: { delayUntil: Date; queueTimestamp?: Date; snapshot?: RescheduleSnapshotInput }, + tx?: PrismaClientOrTransaction + ): Promise { + // The delegate writes a snapshot only when one is supplied, so an absent snapshot is a plain run + // update with nothing for Redis to mirror. + if (!this.writesRedis || !data.snapshot) { + return this.delegate.rescheduleRun(runId, data, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { ...data, snapshot: { ...data.snapshot, id: ctx.id } }; + + const result = await this.delegate.rescheduleRun(runId, withId, tx); + + await this.#appendTransition("rescheduleRun", entryFromReschedule(ctx, withId.snapshot)); + return result; + } + + override async lockRunToWorker( + runId: string, + data: LockRunData, + tx?: PrismaClientOrTransaction + ): Promise>> { + if (!this.writesRedis) { + return this.delegate.lockRunToWorker(runId, data, tx); + } + + // This is the one transition whose input already carries both an id and the previous snapshot + // id, so it is also the one that can append under a compare-and-set on the current head. + const ctx = { id: data.snapshot.id, runId, createdAt: new Date() }; + + const result = await this.delegate.lockRunToWorker(runId, data, tx); + + await this.#appendTransition( + "lockRunToWorker", + entryFromLock(ctx, data.snapshot), + data.snapshot.previousSnapshotId + ); + return result; + } + + override async createExecutionSnapshot( + input: CreateExecutionSnapshotInput, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.createExecutionSnapshot(input, tx); + } + + const ctx = this.#context(input.run.id, input.id); + const created = await this.delegate.createExecutionSnapshot({ ...input, id: ctx.id }, tx); + + // The standalone path is the only one whose delegate returns the row, so its entry can take the + // exact createdAt Postgres recorded rather than the decorator's own clock. + await this.#appendTransition( + "createExecutionSnapshot", + entryFromCreateExecutionSnapshot({ ...ctx, createdAt: created.createdAt }, input), + input.previousSnapshotId + ); + return created; + } + + // --------------------------------------------------------------------------------------------- + // The append protocol. + // --------------------------------------------------------------------------------------------- + + /** Mints the id when the caller did not, and stamps one clock for both stores. */ + #context(runId: string, suppliedId?: string) { + return { id: suppliedId ?? generateInternalId(), runId, createdAt: new Date() }; + } + + /** + * Births invert the order. Postgres-first would leave a run with no snapshot at all, and + * `getLatestExecutionSnapshot` treats that as a hard error, so the run would be stuck. Redis-first + * leaves an orphaned keyspace for a run that does not exist, which nothing can reach and the + * sweep's second rule reaps. + * + * Being first is also what lets this path refuse. Before `redis-only` a failed birth append is + * survivable, because Postgres is authoritative and holds the snapshot; at `redis-only` Postgres + * writes no snapshot, so a run created without its Redis birth would have no snapshot anywhere. + * Throwing here happens before the run row exists, so the caller retries a clean creation. + */ + async #appendBirth(site: string, entry: SnapshotEntryInput): Promise { + if (this.staging) { + // A birth inside a transaction cannot be staged: staging flushes after the commit, which is + // the opposite of what a birth needs. No caller does this today, so say so and append now. + this.logger.error("a run birth inside a transaction cannot be staged", { + runId: entry.runId, + site, + }); + } + + for (let attempt = 0; attempt < APPEND_ATTEMPTS; attempt++) { + try { + const result = await this.redis.append({ + entry, + kind: "birth", + isTerminal: isTerminalEntry(entry), + }); + this.#recordOutcome(site, entry, result); + + // Modelled AFTER the successful append: the crash this boundary represents is a process that + // died between the two stores, not an append that failed. + this.faults?.("afterRedisBirthBeforePg", { runId: entry.runId, snapshotId: entry.id }); + return; + } catch (error) { + if (isInjectedFault(error)) { + throw error; + } + + if (attempt === APPEND_ATTEMPTS - 1) { + this.metrics?.recordAppendFailed(site); + this.logger.error("snapshot birth append failed after retries", { + runId: entry.runId, + snapshotId: entry.id, + site, + mode: this.mode, + error, + }); + + if (this.mode === "redis-only") { + throw error; + } + return; + } + + await new Promise((resolve) => setTimeout(resolve, 10 * 2 ** attempt)); + } + } + } + + /** + * Postgres has already committed by the time this runs. A throw here would turn a gap the stall + * watchdog heals into a caller-visible failure, so it never rethrows: it retries, then hands the + * run to the repair job and returns. + */ + async #appendTransition( + site: string, + entry: SnapshotEntryInput, + expectedCur?: string + ): Promise { + if (this.staging) { + // Inside a transaction the append cannot run until the Postgres side commits, or a rollback + // leaves Redis holding a transition that never happened. + this.staging.push(entry); + return; + } + + for (let attempt = 0; attempt < APPEND_ATTEMPTS; attempt++) { + try { + this.faults?.(attempt === 0 ? "afterPgBeforeRedis" : "midFlushRetry", { + runId: entry.runId, + snapshotId: entry.id, + }); + + const result = await this.redis.append({ + entry, + kind: "transition", + isTerminal: isTerminalEntry(entry), + ...(expectedCur !== undefined && { expectedCur }), + }); + + this.#recordOutcome(site, entry, result); + return; + } catch (error) { + // An injected fault models a dead process, not a retryable append failure. + if (isInjectedFault(error)) { + this.metrics?.recordAppendFailed(site); + await this.#enqueueRepair(entry); + return; + } + + if (attempt === APPEND_ATTEMPTS - 1) { + this.metrics?.recordAppendFailed(site); + this.logger.error("snapshot append failed after retries", { + runId: entry.runId, + snapshotId: entry.id, + site, + error, + }); + await this.#enqueueRepair(entry); + return; + } + + await new Promise((resolve) => setTimeout(resolve, 10 * 2 ** attempt)); + } + } + } + + /** + * None of the four append outcomes is a failure, and none of them enqueues a repair. + * + * `skippedNoKeyspace` is every pre-cutover run's transitions. `forked` means another writer + * advanced the head, which a repair cannot help. `duplicate` is a retry that already landed. + * `cycleMismatch` means the store refused an untrustworthy waitpoint pointer on purpose. + */ + #recordOutcome( + site: string, + entry: SnapshotEntryInput, + result: Awaited> + ): void { + this.metrics?.recordWrite(site, result.outcome); + + if (result.outcome === "forked") { + this.logger.warn("snapshot append forked", { + runId: entry.runId, + snapshotId: entry.id, + site, + actualCur: result.actualCur, + }); + } + } + + async #enqueueRepair(entry: SnapshotEntryInput): Promise { + if (!this.onAppendFailure) { + return; + } + + try { + await this.onAppendFailure({ + runId: entry.runId, + snapshotId: entry.id, + executionStatus: entry.executionStatus, + }); + } catch (error) { + // The repair enqueue is itself best-effort. Failing it must not fail the caller's write. + this.logger.error("snapshot repair enqueue failed", { runId: entry.runId, error }); + } + } +} From fbe91ac3e11e5cb08982def9e2659612a6324b59 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 15:33:10 +0100 Subject: [PATCH 06/31] test(run-store): cover the snapshot staging facade and the wrapped store handles Proves the deferral from inside the transaction callback rather than assuming it: a staged append is absent from Redis while the transaction is open and present once it commits, and a rollback leaves both stores agreeing the transition never happened. --- ...kRunExecutionSnapshotStore.staging.test.ts | 231 ++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts new file mode 100644 index 00000000000..ac0a877c968 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts @@ -0,0 +1,231 @@ +// Inside a transaction the Redis append cannot run until the Postgres side commits, or a rollback +// leaves Redis holding a transition that never happened. These tests observe the buffer from inside +// the callback, so the deferral is proved rather than assumed. +import { describe, expect } from "vitest"; +import { postgresAndRedisTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build(prisma: never, redisOptions: never, mode: "off" | "dual-write" = "dual-write") { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { store: redis, mode } + ); + return { decorated, redis }; +} + +async function seedBirth( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + runId: string, + env: SnapshotFixtureEnv +): Promise { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); +} + +function snapshotInput(runId: string, env: SnapshotFixtureEnv, id: string, description: string) { + return { + id, + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("the staging facade", () => { + postgresAndRedisTest("flushes the append after the commit", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, id, "Run started"), tx); + + // Still inside the transaction: nothing has reached Redis yet. + expect(await redis.getById(runId, id)).toBeNull(); + }); + + expect(await redis.getById(runId, id)).not.toBeNull(); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "writes nothing to Redis when the transaction rolls back", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + await expect( + decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, id, "Run started"), tx); + throw new Error("rolled back"); + }) + ).rejects.toThrow("rolled back"); + + // Both sides agree that the transition never happened. + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id } })).toBe(0); + expect(await redis.getById(runId, id)).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("flushes several staged appends in order", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const first = generateInternalId(); + const second = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, first, "First"), tx); + await store.createExecutionSnapshot(snapshotInput(runId, env, second, "Second"), tx); + }); + + const firstRead = await redis.getById(runId, first); + const secondRead = await redis.getById(runId, second); + expect(firstRead).not.toBeNull(); + expect(secondRead).not.toBeNull(); + // Order matters: the log is append-only and its seq is what orders a read. + expect(firstRead!.seq).toBeLessThan(secondRead!.seq); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "hands the transaction callback a decorated store", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + let seen: unknown; + + await decorated.runInTransaction(runId, async (store) => { + seen = store; + }); + + expect(seen).toBeInstanceOf(TaskRunExecutionSnapshotStore); + expect((seen as TaskRunExecutionSnapshotStore).mode).toBe("dual-write"); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "hands the transaction callback the plain delegate at mode off", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + let seen: unknown; + + await decorated.runInTransaction(runId, async (store) => { + seen = store; + }); + + expect(seen).not.toBeInstanceOf(TaskRunExecutionSnapshotStore); + expect(await redis.getLatest(runId)).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "wraps the store handle from forWaitpointCompletion", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const handle = await decorated.forWaitpointCompletion(generateInternalId(), { + routeKind: "MANUAL", + } as never); + + // No snapshot write goes through this handle today. Wrapping it is what stops a future one + // from bypassing the decorator with no signal. + expect(handle).toBeInstanceOf(TaskRunExecutionSnapshotStore); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "returns the plain handle from forWaitpointCompletion at mode off", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); + try { + const handle = await decorated.forWaitpointCompletion(generateInternalId(), { + routeKind: "MANUAL", + } as never); + + expect(handle).not.toBeInstanceOf(TaskRunExecutionSnapshotStore); + } finally { + await redis.quit(); + } + } + ); +}); From ac7b37b7e6ab64714f13de41d954f66bd9d5e440 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 15:35:51 +0100 Subject: [PATCH 07/31] feat(run-store): read a snapshot window by createdAt cursor The engine resolves its since-cursor to a createdAt before it asks for the window, so the snapshot id is gone by then and the id-addressed read cannot serve it. Adding a cursor-addressed read is the alternative to changing the engine's read path, which stays untouched. The cursor is exclusive and keeps the same-millisecond blind spot the Postgres read has. Matching it is the requirement, not an oversight: a Redis read that is more correct than the Postgres read shows up as divergence during compare mode, which exists to surface real defects. Closing the blind spot needs seq ordering on both sides and belongs after the cutover. The walk goes newest-first and stops at the first entry at or before the cursor, so its length is the length of the answer rather than the run's history. This adds a read operation. It does not touch the append script, the keyspace, or the write-ordering protocol. --- .../redisSnapshotStore.sinceCreatedAt.test.ts | 191 ++++++++++++++++++ .../run-store/src/redisSnapshotStore.ts | 127 ++++++++++++ 2 files changed, 318 insertions(+) create mode 100644 internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts diff --git a/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts b/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts new file mode 100644 index 00000000000..7e61e619b78 --- /dev/null +++ b/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts @@ -0,0 +1,191 @@ +// getExecutionSnapshotsSince resolves its cursor to a createdAt before it asks for the window, so +// the snapshot id is gone by then and getSince cannot serve it. This read takes the cursor instead, +// and has to agree with the Postgres read it stands in for — same-millisecond blind spot included. +import { describe, expect } from "vitest"; +import { redisTest } from "@internal/testcontainers"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function entry(runId: string, id: string, createdAt: string): SnapshotEntryInput { + return { + id, + engine: "V2", + executionStatus: "EXECUTING", + description: "d", + runId, + runStatus: "EXECUTING", + createdAt, + environmentId: "env_1", + environmentType: "DEVELOPMENT", + projectId: "proj_1", + organizationId: "org_1", + }; +} + +const at = (seconds: number) => + new Date(Date.UTC(2026, 0, 1, 0, 0, seconds)).toISOString(); + +async function seed( + store: RedisSnapshotStore, + runId: string, + stamps: { id: string; createdAt: string }[] +): Promise { + for (const [index, stamp] of stamps.entries()) { + await store.append({ + entry: entry(runId, stamp.id, stamp.createdAt), + kind: index === 0 ? "birth" : "transition", + isTerminal: false, + }); + } +} + +describe("getSinceCreatedAt", () => { + redisTest("returns only entries newer than the cursor, oldest first", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_window"; + await seed( + store, + runId, + [0, 1, 2, 3, 4].map((n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const result = await store.getSinceCreatedAt(runId, at(1)); + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + // Ascending, matching what the engine hands its caller after its own reverse(). + expect(result.entries.map((e) => e.id)).toEqual(["snap_2", "snap_3", "snap_4"]); + } finally { + await store.quit(); + } + }); + + redisTest("misses when the run has no keyspace", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + // A miss is the coexistence path: the caller falls back to Postgres for a pre-cutover run. + expect((await store.getSinceCreatedAt("run_absent", at(0))).kind).toBe("miss"); + } finally { + await store.quit(); + } + }); + + redisTest("returns an empty hit when nothing is newer", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_nothing_newer"; + await seed(store, runId, [{ id: "snap_0", createdAt: at(0) }]); + + const result = await store.getSinceCreatedAt(runId, at(5)); + + // A hit, not a miss: Redis owns this run, so the caller must not fall back and re-read + // Postgres for a window it already answered. + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries).toEqual([]); + } finally { + await store.quit(); + } + }); + + redisTest("drops a same-millisecond neighbour, as Postgres does", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_same_ms"; + const shared = at(1); + await seed(store, runId, [ + { id: "snap_0", createdAt: at(0) }, + { id: "snap_1a", createdAt: shared }, + { id: "snap_1b", createdAt: shared }, + { id: "snap_2", createdAt: at(2) }, + ]); + + const result = await store.getSinceCreatedAt(runId, shared); + + // Postgres serves this window with `createdAt: { gt: cursor }`, which drops both same-ms + // entries. Returning snap_1b here would be more correct than Postgres and would therefore + // read as divergence in compare mode. + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries.map((e) => e.id)).toEqual(["snap_2"]); + } finally { + await store.quit(); + } + }); + + redisTest("caps the window at the limit, keeping the newest", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_capped"; + await seed( + store, + runId, + Array.from({ length: 60 }, (_, n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const result = await store.getSinceCreatedAt(runId, at(0), { limit: 50 }); + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries).toHaveLength(50); + // The engine takes the NEWEST 50 and reverses, so the window ends at the newest entry. + expect(result.entries[result.entries.length - 1]!.id).toBe("snap_59"); + expect(result.entries[0]!.id).toBe("snap_10"); + } finally { + await store.quit(); + } + }); + + redisTest("scans no further than the answer", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_deep_history"; + await seed( + store, + runId, + Array.from({ length: 400 }, (_, n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const started = Date.now(); + const result = await store.getSinceCreatedAt(runId, at(394), { limit: 50 }); + const elapsed = Date.now() - started; + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries.map((e) => e.id)).toEqual([ + "snap_395", + "snap_396", + "snap_397", + "snap_398", + "snap_399", + ]); + // The walk stops at the cursor rather than reading the run's history. The bound is generous + // on purpose: it fails on a full scan of 400 entries, not on ordinary timing noise. + expect(elapsed).toBeLessThan(1_000); + } finally { + await store.quit(); + } + }); + + redisTest("scopes the window to an environment", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_env_scoped"; + await seed(store, runId, [ + { id: "snap_0", createdAt: at(0) }, + { id: "snap_1", createdAt: at(1) }, + ]); + + const foreign = await store.getSinceCreatedAt(runId, at(0), { environmentId: "env_other" }); + + expect(foreign.kind).toBe("hit"); + if (foreign.kind !== "hit") return; + expect(foreign.entries).toEqual([]); + } finally { + await store.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index 2964959c4fe..e36179c6041 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -472,6 +472,64 @@ export class RedisSnapshotStore { }); } + /** + * The same window as {@link getSince}, addressed by a createdAt cursor instead of a snapshot id. + * + * `getExecutionSnapshotsSince` resolves its cursor to a createdAt before it asks for the window, + * so the snapshot id is gone by the time this call is made and `getSince` cannot serve it. The + * cursor is exclusive and keeps Postgres's same-millisecond blind spot, so the two reads agree. + */ + async getSinceCreatedAt( + runId: string, + createdAt: Date | string, + opts?: { environmentId?: string; limit?: number } + ): Promise { + return this.#timed("getSinceCreatedAt", async () => { + const k = snapshotKeys(runId); + const limit = opts?.limit ?? this.sinceLimit; + const cursor = typeof createdAt === "string" ? createdAt : createdAt.toISOString(); + + const reply = await this.redis.readSnapshotsSinceCreatedAt( + k.e, + k.idx, + k.cur, + k.seq, + cursor, + String(limit) + ); + if (reply === null) return { kind: "miss" }; + + const headOrder = reply[1] ?? ""; + const rows: SnapshotRead[] = []; + // Tracks whether the Lua-chosen head row (always the first, i === 2) survives the env filter, + // so headOrder is never attributed to a different, surviving row. + let headSurvived = false; + for (let i = 2; i + 3 < reply.length; i += 4) { + const decoded = this.#decode( + [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], + opts?.environmentId, + runId, + false + ); + if (decoded) { + rows.push(decoded); + if (i === 2) headSurvived = true; + } + } + + rows.reverse(); + const head = headSurvived ? rows[rows.length - 1] : undefined; + const headWaitpointIds = decodeWaitpointIds(head !== undefined, head ? headOrder : ""); + if (head) { + head.completedWaitpointIds = headWaitpointIds; + if (head.cycle) { + this.#checkCycleMismatch(runId, head.cycle.count, headWaitpointIds.order.length); + } + } + return { kind: "hit", entries: rows, headWaitpointIds }; + }); + } + #checkCycleMismatch(runId: string, count: number, orderLength: number): void { if (orderLength === count) return; this.metrics?.recordCycleMismatch(); @@ -682,6 +740,66 @@ export class RedisSnapshotStore { `, }); + this.redis.defineCommand("readSnapshotsSinceCreatedAt", { + numberOfKeys: 4, + lua: ` + ${PRELUDE} + local cursor = ARGV[1] + local limit = tonumber(ARGV[2]) + + -- A run with no keyspace is a MISS, so the caller falls back to Postgres. A run that has one + -- and nothing newer is an empty HIT, so it does not fall back for a window it owns. + if redis.call('EXISTS', eKey) == 0 then return nil end + + -- STRICTLY greater than the cursor, and same-millisecond entries are dropped. Postgres + -- serves this window with createdAt > cursor and drops them too; a Redis read that is more + -- correct than the Postgres read shows up as divergence in compare mode. + -- + -- createdAt is always toISOString() output, one fixed-width UTC format, so a lexicographic + -- compare is a chronological compare. Walking newest-first lets the scan stop at the first + -- entry at or before the cursor, which makes its length the length of the ANSWER rather + -- than the length of the run's history. + local out = { '', '' } + local headId = nil + local offset = 0 + local page = limit + local done = false + + while not done do + local ids = redis.call('ZREVRANGE', idxKey, offset, offset + page - 1) + if #ids == 0 then break end + + for i = 1, #ids do + local id = ids[i] + local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c') + if vals[1] then + local createdAt = cjson.decode(vals[1])['createdAt'] + if not createdAt or createdAt <= cursor then + done = true + break + end + if not headId then headId = id end + out[#out + 1] = id + out[#out + 1] = vals[1] + out[#out + 1] = vals[2] or '' + out[#out + 1] = vals[3] or '' + if (#out - 2) / 4 >= limit then + done = true + break + end + end + end + + offset = offset + page + end + + if headId then + out[2] = orderFor(redis.call('HGET', eKey, headId .. '#c')) + end + return out + `, + }); + this.redis.defineCommand("readSnapshotsSince", { numberOfKeys: 4, lua: ` @@ -780,6 +898,15 @@ declare module "@internal/redis" { id: string, callback?: Callback ): Result; + readSnapshotsSinceCreatedAt( + eKey: string, + idxKey: string, + curKey: string, + seqKey: string, + createdAtCursor: string, + limit: string, + callback?: Callback + ): Result; readSnapshotsSince( eKey: string, idxKey: string, From f73d80633f3bc5d9fe6cd9e4f8a8049f2deab017 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 16:05:36 +0100 Subject: [PATCH 08/31] feat(run-store): serve snapshot reads from Redis with a Postgres fallback Two of the five snapshot reads take arbitrary Prisma arguments, and a key-value store cannot answer an arbitrary query. Only three production call sites exist, all in the engine's executionSnapshotSystem, and both generic ones send a single fixed shape, so the decorator recognises exactly those shapes and delegates everything else. Each matcher rejects an argument object carrying a key it does not know, because a query that has drifted must be answered correctly by Postgres rather than approximately from Redis. A miss is the coexistence path, not an error: a pre-cutover run or expired history falls back to Postgres. The entry supplies every scalar column, and the checkpoint and waitpoint rows are read back through the delegate only when the entry says they exist, so the common read of a running run makes no Postgres call at all. Which runs read from Redis is a hash of the run id, so a run does not change store between two reads of one poll, two instances of the same dial agree, and raising the dial only ever adds runs to the cohort. --- .../run-store/src/snapshotReadShapes.test.ts | 151 +++++++ .../run-store/src/snapshotReadShapes.ts | 95 +++++ ...nExecutionSnapshotStore.readCohort.test.ts | 95 +++++ ...askRunExecutionSnapshotStore.reads.test.ts | 400 ++++++++++++++++++ .../src/taskRunExecutionSnapshotStore.ts | 242 ++++++++++- 5 files changed, 982 insertions(+), 1 deletion(-) create mode 100644 internal-packages/run-store/src/snapshotReadShapes.test.ts create mode 100644 internal-packages/run-store/src/snapshotReadShapes.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts diff --git a/internal-packages/run-store/src/snapshotReadShapes.test.ts b/internal-packages/run-store/src/snapshotReadShapes.test.ts new file mode 100644 index 00000000000..afa33dd3da2 --- /dev/null +++ b/internal-packages/run-store/src/snapshotReadShapes.test.ts @@ -0,0 +1,151 @@ +// A matcher that is too loose is the dangerous failure: it answers a query Redis cannot actually +// serve, and the caller gets a wrong answer rather than a slow one. So most of these tests are +// about what must NOT match. +import { describe, expect, it } from "vitest"; +import { matchSinceCursorLookup, matchSinceWindow } from "./snapshotReadShapes.js"; + +const cursorArgs = { + where: { id: "snap_1", runId: "run_1" }, + select: { createdAt: true }, +}; + +const windowArgs = { + where: { runId: "run_1", isValid: true, createdAt: { gt: new Date("2026-08-24T00:00:00Z") } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, +}; + +describe("matchSinceCursorLookup", () => { + it("matches the engine's since-cursor lookup", () => { + expect(matchSinceCursorLookup(cursorArgs)).toEqual({ id: "snap_1", runId: "run_1" }); + }); + + it("carries an environment scope when present", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, environmentId: "env_1" }, + select: { createdAt: true }, + }) + ).toEqual({ id: "snap_1", runId: "run_1", environmentId: "env_1" }); + }); + + it("ignores keys explicitly set to undefined", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, environmentId: undefined }, + select: { createdAt: true }, + }) + ).toEqual({ id: "snap_1", runId: "run_1" }); + }); + + it("refuses a selection of anything but createdAt", () => { + expect( + matchSinceCursorLookup({ where: cursorArgs.where, select: { description: true } }) + ).toBeUndefined(); + expect( + matchSinceCursorLookup({ + where: cursorArgs.where, + select: { createdAt: true, description: true }, + }) + ).toBeUndefined(); + }); + + it("refuses a where with no run id, because there is no keyspace to look in", () => { + expect( + matchSinceCursorLookup({ where: { id: "snap_1" }, select: { createdAt: true } }) + ).toBeUndefined(); + }); + + it("refuses an unknown where key", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, isValid: true }, + select: { createdAt: true }, + }) + ).toBeUndefined(); + }); + + it("refuses an unknown top-level key", () => { + expect(matchSinceCursorLookup({ ...cursorArgs, orderBy: { createdAt: "desc" } })).toBeUndefined(); + }); + + it("refuses anything that is not an argument object", () => { + expect(matchSinceCursorLookup(undefined)).toBeUndefined(); + expect(matchSinceCursorLookup(null)).toBeUndefined(); + expect(matchSinceCursorLookup("where")).toBeUndefined(); + expect(matchSinceCursorLookup([cursorArgs])).toBeUndefined(); + }); +}); + +describe("matchSinceWindow", () => { + it("matches the engine's window query", () => { + expect(matchSinceWindow(windowArgs)).toEqual({ + runId: "run_1", + createdAt: new Date("2026-08-24T00:00:00Z"), + take: 50, + }); + }); + + it("carries an environment scope when present", () => { + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, environmentId: "env_1" }, + }) + ).toMatchObject({ environmentId: "env_1" }); + }); + + it("refuses a query that also wants the completed waitpoints", () => { + // The engine omits them on purpose to avoid an N x M read. An include that asks for them is a + // different query, and answering it from this path would return them empty. + expect( + matchSinceWindow({ + ...windowArgs, + include: { checkpoint: true, completedWaitpoints: true }, + }) + ).toBeUndefined(); + }); + + it("refuses ascending order", () => { + expect( + matchSinceWindow({ ...windowArgs, orderBy: { createdAt: "asc" } }) + ).toBeUndefined(); + }); + + it("refuses a window that does not filter to valid entries", () => { + expect( + matchSinceWindow({ ...windowArgs, where: { ...windowArgs.where, isValid: false } }) + ).toBeUndefined(); + }); + + it("refuses a cursor that is not a strict greater-than on a Date", () => { + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, createdAt: { gte: new Date() } }, + }) + ).toBeUndefined(); + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, createdAt: { gt: "2026-08-24T00:00:00Z" } }, + }) + ).toBeUndefined(); + }); + + it("refuses a missing take", () => { + const { take: _dropped, ...withoutTake } = windowArgs; + expect(matchSinceWindow(withoutTake)).toBeUndefined(); + }); + + it("refuses an unknown where key", () => { + expect( + matchSinceWindow({ ...windowArgs, where: { ...windowArgs.where, batchId: "batch_1" } }) + ).toBeUndefined(); + }); + + it("refuses an unknown top-level key", () => { + expect(matchSinceWindow({ ...windowArgs, skip: 10 })).toBeUndefined(); + }); +}); diff --git a/internal-packages/run-store/src/snapshotReadShapes.ts b/internal-packages/run-store/src/snapshotReadShapes.ts new file mode 100644 index 00000000000..31dda17f7f0 --- /dev/null +++ b/internal-packages/run-store/src/snapshotReadShapes.ts @@ -0,0 +1,95 @@ +// Shape matchers for the two generic Prisma-args snapshot reads. +// +// `findExecutionSnapshot` and `findManyExecutionSnapshots` take arbitrary Prisma arguments, and a +// key-value store cannot answer an arbitrary query. Only three production call sites exist, all in +// the engine's executionSnapshotSystem, and both generic ones send a single fixed shape. So these +// matchers recognise exactly those shapes and return undefined for anything else, which sends the +// call to Postgres. +// +// Each matcher rejects an argument object carrying any key it does not know about. A query that has +// drifted must fall through and be answered correctly by Postgres, never answered approximately +// from Redis. + +type Unknown = Record; + +function isPlainObject(value: unknown): value is Unknown { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** True when `value` has exactly `allowed` keys, ignoring keys explicitly set to undefined. */ +function hasOnlyKeys(value: Unknown, allowed: string[]): boolean { + const present = Object.keys(value).filter((k) => value[k] !== undefined); + return present.every((k) => allowed.includes(k)); +} + +function isString(value: unknown): value is string { + return typeof value === "string"; +} + +export type SinceCursorLookup = { id: string; runId: string; environmentId?: string }; + +/** + * Step 1 of `getExecutionSnapshotsSince`: resolve a known snapshot id to its createdAt. + * + * { where: { id, runId, environmentId? }, select: { createdAt: true } } + */ +export function matchSinceCursorLookup(args: unknown): SinceCursorLookup | undefined { + if (!isPlainObject(args) || !hasOnlyKeys(args, ["where", "select"])) return undefined; + + const { where, select } = args; + if (!isPlainObject(where) || !isPlainObject(select)) return undefined; + if (!hasOnlyKeys(where, ["id", "runId", "environmentId"])) return undefined; + if (!hasOnlyKeys(select, ["createdAt"]) || select.createdAt !== true) return undefined; + if (!isString(where.id) || !isString(where.runId)) return undefined; + if (where.environmentId !== undefined && !isString(where.environmentId)) return undefined; + + return { + id: where.id, + runId: where.runId, + ...(isString(where.environmentId) && { environmentId: where.environmentId }), + }; +} + +export type SinceWindow = { + runId: string; + createdAt: Date; + take: number; + environmentId?: string; +}; + +/** + * Step 2 of `getExecutionSnapshotsSince`: the capped window after a createdAt cursor. + * + * { where: { runId, isValid: true, createdAt: { gt }, environmentId? }, + * include: { checkpoint: true }, orderBy: { createdAt: "desc" }, take: N } + * + * The engine deliberately omits completedWaitpoints from the include to avoid an N x M read, so an + * include asking for them is a different query and is not matched. + */ +export function matchSinceWindow(args: unknown): SinceWindow | undefined { + if (!isPlainObject(args) || !hasOnlyKeys(args, ["where", "include", "orderBy", "take"])) { + return undefined; + } + + const { where, include, orderBy, take } = args; + if (!isPlainObject(where) || !isPlainObject(include) || !isPlainObject(orderBy)) return undefined; + if (typeof take !== "number") return undefined; + + if (!hasOnlyKeys(where, ["runId", "isValid", "createdAt", "environmentId"])) return undefined; + if (!isString(where.runId) || where.isValid !== true) return undefined; + if (where.environmentId !== undefined && !isString(where.environmentId)) return undefined; + + if (!hasOnlyKeys(include, ["checkpoint"]) || include.checkpoint !== true) return undefined; + if (!hasOnlyKeys(orderBy, ["createdAt"]) || orderBy.createdAt !== "desc") return undefined; + + const cursor = where.createdAt; + if (!isPlainObject(cursor) || !hasOnlyKeys(cursor, ["gt"])) return undefined; + if (!(cursor.gt instanceof Date)) return undefined; + + return { + runId: where.runId, + createdAt: cursor.gt, + take, + ...(isString(where.environmentId) && { environmentId: where.environmentId }), + }; +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts new file mode 100644 index 00000000000..12ad2e5fc54 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts @@ -0,0 +1,95 @@ +// The read cohort is pure arithmetic on the run id, so it needs no containers. Keeping it out of the +// container-backed suite also keeps that suite small enough to run reliably. +import { describe, expect, it } from "vitest"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +type CohortProbe = { readsFromRedis(runId: string): boolean }; + +function probe(mode: SnapshotStoreMode, readPercent: number): CohortProbe { + // lazyConnect keeps the client from dialling anything: no read in this suite reaches the store. + const redis = new RedisSnapshotStore({ + redisOptions: { host: "127.0.0.1", port: 1, lazyConnect: true, retryStrategy: () => null }, + completedTtlMs: 1, + }); + + return new TaskRunExecutionSnapshotStore({} as RunStore, { + store: redis, + mode, + readPercent, + }) as unknown as CohortProbe; +} + +const ids = Array.from({ length: 500 }, (_, n) => `run_cohort_${n}_${n * 7919}`); + +describe("the read cohort", () => { + it("reads nothing from Redis before the read positions", () => { + for (const mode of ["off", "dual-write", "compare"] as const) { + const store = probe(mode, 100); + expect(ids.every((id) => !store.readsFromRedis(id))).toBe(true); + } + }); + + it("reads everything from Redis at 100 percent", () => { + for (const mode of ["redis-read", "redis-only"] as const) { + const store = probe(mode, 100); + expect(ids.every((id) => store.readsFromRedis(id))).toBe(true); + } + }); + + it("reads nothing from Redis at 0 percent", () => { + const store = probe("redis-read", 0); + expect(ids.every((id) => !store.readsFromRedis(id))).toBe(true); + }); + + it("gives one run the same answer every time", () => { + // A run that changed store between two reads of one poll could show the log going backwards. + const store = probe("redis-read", 50); + + for (const id of ids.slice(0, 50)) { + const first = store.readsFromRedis(id); + for (let i = 0; i < 5; i++) { + expect(store.readsFromRedis(id)).toBe(first); + } + } + }); + + it("gives two instances of the same dial the same answer", () => { + // The cohort must not depend on process state, or a redeploy reshuffles every in-flight run. + const first = probe("redis-read", 50); + const second = probe("redis-read", 50); + + for (const id of ids.slice(0, 50)) { + expect(second.readsFromRedis(id)).toBe(first.readsFromRedis(id)); + } + }); + + it("spreads a population across the dial", () => { + const store = probe("redis-read", 50); + const enabled = ids.filter((id) => store.readsFromRedis(id)).length; + + // A wide band: this asserts the hash spreads at all, not that it is uniform. + expect(enabled).toBeGreaterThan(150); + expect(enabled).toBeLessThan(350); + }); + + it("grows the cohort monotonically as the dial rises", () => { + const at = (percent: number) => { + const store = probe("redis-read", percent); + return new Set(ids.filter((id) => store.readsFromRedis(id))); + }; + + const ten = at(10); + const fifty = at(50); + const ninety = at(90); + + // Raising the dial must only ever add runs. A run that fell out on the way up would flip back to + // Postgres mid-flight, which is the thing the stable hash exists to prevent. + expect([...ten].every((id) => fifty.has(id))).toBe(true); + expect([...fifty].every((id) => ninety.has(id))).toBe(true); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts new file mode 100644 index 00000000000..ffdd91a92f9 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts @@ -0,0 +1,400 @@ +// Reads served from Redis must be indistinguishable from the Postgres reads they replace: the same +// payload shape, the same tenant boundary, the same fallback when Redis does not hold the answer. +import { describe, expect } from "vitest"; +import { postgresAndRedisTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + opts?: { mode?: SnapshotStoreMode; readPercent?: number } +) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const reads: { method: string; source: string }[] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "redis-read", + readPercent: opts?.readPercent ?? 100, + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { decorated, redis, reads }; +} + +async function seedRun( + decorated: TaskRunExecutionSnapshotStore, + env: SnapshotFixtureEnv +): Promise { + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + return runId; +} + +function snapshotInput(runId: string, env: SnapshotFixtureEnv, description: string) { + return { + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("snapshot reads", () => { + postgresAndRedisTest("serves the latest snapshot from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(latest!.id).toBe(created.id); + expect(latest!.executionStatus).toBe("EXECUTING"); + expect(latest!.description).toBe("Run started"); + expect(latest!.runId).toBe(runId); + expect(latest!.checkpoint).toBeNull(); + expect(latest!.completedWaitpoints).toEqual([]); + expect(reads).toContainEqual({ method: "findLatestExecutionSnapshot", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "returns the same payload Postgres would", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const fromRedis = await decorated.findLatestExecutionSnapshot(runId); + const fromPostgres = await postgresOnly.findLatestExecutionSnapshot(runId); + + expect(fromRedis!.id).toBe(fromPostgres!.id); + expect(fromRedis!.executionStatus).toBe(fromPostgres!.executionStatus); + expect(fromRedis!.description).toBe(fromPostgres!.description); + expect(fromRedis!.runStatus).toBe(fromPostgres!.runStatus); + expect(fromRedis!.attemptNumber).toBe(fromPostgres!.attemptNumber); + expect(fromRedis!.isValid).toBe(fromPostgres!.isValid); + expect(fromRedis!.environmentId).toBe(fromPostgres!.environmentId); + expect(fromRedis!.createdAt.toISOString()).toBe(fromPostgres!.createdAt.toISOString()); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "reads a foreign environment as not found, so the caller's 404 still fires", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const foreign = await decorated.findLatestExecutionSnapshot(runId, undefined, "env_other"); + + expect(foreign).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "falls back to Postgres for a run with no keyspace", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A pre-cutover run: it exists in Postgres and Redis has never seen it. + await postgresOnly.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(latest!.executionStatus).toBe("RUN_CREATED"); + expect(reads).toContainEqual({ method: "findLatestExecutionSnapshot", source: "postgres" }); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("reads from Postgres at readPercent 0", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + readPercent: 0, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest("reads from Postgres at mode dual-write", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + mode: "dual-write", + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest("serves the since-cursor lookup from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const cursor = await decorated.findExecutionSnapshot({ + where: { id: created.id, runId }, + select: { createdAt: true }, + }); + + expect(cursor).not.toBeNull(); + expect((cursor as { createdAt: Date }).createdAt.toISOString()).toBe( + created.createdAt.toISOString() + ); + expect(reads).toContainEqual({ method: "findExecutionSnapshot", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "delegates a snapshot lookup it does not recognise", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + // A different selection: Redis must not answer it approximately. + const row = await decorated.findExecutionSnapshot({ + where: { id: created.id }, + select: { description: true }, + }); + + expect(row).toEqual({ description: "Run started" }); + expect(reads.filter((r) => r.method === "findExecutionSnapshot")).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("serves the since window from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const first = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "First")); + await new Promise((resolve) => setTimeout(resolve, 5)); + const second = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Second")); + await new Promise((resolve) => setTimeout(resolve, 5)); + const third = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Third")); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + // Descending, exactly as the engine asked; it reverses app-side. + expect(window.map((s) => s.id)).toEqual([third.id, second.id]); + expect(reads).toContainEqual({ method: "findManyExecutionSnapshots", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "delegates a window query it does not recognise", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "First")); + + const rows = await decorated.findManyExecutionSnapshots({ + where: { runId }, + orderBy: { createdAt: "asc" }, + }); + + expect(rows.length).toBeGreaterThan(0); + expect(reads.filter((r) => r.method === "findManyExecutionSnapshots")).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "serves the waitpoint id projections from Redis", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const ids = await decorated.findSnapshotCompletedWaitpointIds(created.id, undefined, runId); + const withPresence = await decorated.findSnapshotCompletedWaitpointIdsWithPresence( + created.id, + undefined, + runId + ); + + expect(ids).toEqual([]); + // present distinguishes "no waitpoints" from "this reader cannot see the snapshot", which is + // what the engine's read-repair keys off. + expect(withPresence).toEqual({ present: true, ids: [] }); + expect(reads).toContainEqual({ + method: "findSnapshotCompletedWaitpointIds", + source: "redis", + }); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "delegates a waitpoint id projection with no run id", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + // Without a run id there is no keyspace to look in. + const ids = await decorated.findSnapshotCompletedWaitpointIds(created.id); + + expect(ids).toEqual([]); + expect(reads.filter((r) => r.method.startsWith("findSnapshot"))).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("never touches Redis for reads at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + mode: "off", + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(await redis.getLatest(runId)).toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index 69658ecaad1..30655d727db 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -14,7 +14,7 @@ import { Logger } from "@trigger.dev/core/logger"; import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; import { DelegatingRunStore } from "./delegatingRunStore.js"; -import type { RedisSnapshotStore, SnapshotEntryInput } from "./redisSnapshotStore.js"; +import type { RedisSnapshotStore, SnapshotEntryInput, SnapshotRead } from "./redisSnapshotStore.js"; import { entryFromCompletion, entryFromCreateExecutionSnapshot, @@ -26,6 +26,7 @@ import { } from "./snapshotEntry.js"; import { isInjectedFault, type SnapshotFaultInjector } from "./snapshotFaultInjection.js"; import type { + ReadClient, CompletionSnapshotInput, CreateCancelledRunInput, CreateExecutionSnapshotInput, @@ -36,6 +37,10 @@ import type { RunStore, TaskRunWithWaitpoint, } from "./types.js"; +import { + matchSinceCursorLookup, + matchSinceWindow, +} from "./snapshotReadShapes.js"; import type { Prisma, PrismaClientOrTransaction, TaskRun } from "@trigger.dev/database"; /** One initial attempt plus three retries, per the write protocol. */ @@ -515,6 +520,241 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } } + // --------------------------------------------------------------------------------------------- + // Reads. + // + // Only three production call sites exist, all in the engine's executionSnapshotSystem, all with + // fixed argument shapes. The two generic Prisma-args methods therefore recognise exactly the + // shapes the engine sends and delegate everything else: an unrecognised shape must go to Postgres, + // never get an approximate answer from Redis. + // --------------------------------------------------------------------------------------------- + + /** + * Whether this run's reads come from Redis. Hashed on the run id so a run does not change store + * between two reads of the same poll, which would let a caller see the log go backwards. + */ + protected readsFromRedis(runId: string): boolean { + if (this.mode !== "redis-read" && this.mode !== "redis-only") return false; + if (this.readPercent >= 100) return true; + if (this.readPercent <= 0) return false; + + let hash = 0; + for (let i = 0; i < runId.length; i++) { + hash = (hash * 31 + runId.charCodeAt(i)) >>> 0; + } + return hash % 100 < this.readPercent; + } + + override async findLatestExecutionSnapshot( + runId: string, + client?: ReadClient, + environmentId?: string + ): Promise | null> { + if (!this.readsFromRedis(runId)) { + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + const read = await this.redis.getLatest(runId, { ...(environmentId && { environmentId }) }); + if (!read) { + // A miss is the coexistence path: a pre-cutover run, or expired history. It is not an error. + this.metrics?.recordRead("findLatestExecutionSnapshot", "postgres"); + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + this.metrics?.recordRead("findLatestExecutionSnapshot", "redis"); + return this.#hydrate(read, runId, client); + } + + override async findExecutionSnapshot( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise | null> { + const shape = matchSinceCursorLookup(args); + if (!shape || !this.readsFromRedis(shape.runId)) { + return this.delegate.findExecutionSnapshot(args, client); + } + + const found = await this.redis.getById(shape.runId, shape.id, { + ...(shape.environmentId && { environmentId: shape.environmentId }), + }); + + if (!found) { + this.metrics?.recordRead("findExecutionSnapshot", "postgres"); + return this.delegate.findExecutionSnapshot(args, client); + } + + this.metrics?.recordRead("findExecutionSnapshot", "redis"); + // The engine selects createdAt only, so the answer is the cursor and nothing else. + return { + createdAt: new Date(found.entry.createdAt as string), + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload; + } + + override async findManyExecutionSnapshots< + T extends Prisma.TaskRunExecutionSnapshotFindManyArgs, + >( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise[]> { + const shape = matchSinceWindow(args); + if (!shape || !this.readsFromRedis(shape.runId)) { + return this.delegate.findManyExecutionSnapshots(args, client); + } + + const result = await this.redis.getSinceCreatedAt(shape.runId, shape.createdAt, { + limit: shape.take, + ...(shape.environmentId && { environmentId: shape.environmentId }), + }); + + if (result.kind === "miss") { + this.metrics?.recordRead("findManyExecutionSnapshots", "postgres"); + return this.delegate.findManyExecutionSnapshots(args, client); + } + + this.metrics?.recordRead("findManyExecutionSnapshots", "redis"); + + // The engine asks for createdAt DESC and reverses app-side; the store returns ascending. + const descending = [...result.entries].reverse(); + const hydrated = await Promise.all( + descending.map((entry) => this.#hydrate(entry, shape.runId, client, { waitpoints: false })) + ); + return hydrated as unknown as Prisma.TaskRunExecutionSnapshotGetPayload[]; + } + + override async findSnapshotCompletedWaitpointIds( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise { + // Without a run id there is no keyspace to look in, so the router's fan-out is the only answer. + if (!runId || !this.readsFromRedis(runId)) { + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + const ids = await this.redis.getSnapshotWaitpointIds(runId, snapshotId); + if (!ids.present) { + this.metrics?.recordRead("findSnapshotCompletedWaitpointIds", "postgres"); + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + this.metrics?.recordRead("findSnapshotCompletedWaitpointIds", "redis"); + return ids.distinctIds; + } + + override async findSnapshotCompletedWaitpointIdsWithPresence( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise<{ present: boolean; ids: string[] }> { + if (!runId || !this.readsFromRedis(runId)) { + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + const ids = await this.redis.getSnapshotWaitpointIds(runId, snapshotId); + if (!ids.present) { + // present=false means this reader cannot see the snapshot, so its empty list is not + // authoritative and the engine's read-repair needs the Postgres answer. + this.metrics?.recordRead("findSnapshotCompletedWaitpointIdsWithPresence", "postgres"); + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + this.metrics?.recordRead("findSnapshotCompletedWaitpointIdsWithPresence", "redis"); + return { present: true, ids: ids.distinctIds }; + } + + /** + * Turns a store entry into the Prisma payload the interface promises. + * + * The entry supplies every scalar column. `checkpoint` and the full waitpoint rows still live in + * Postgres, so they are read back through the delegate — but only when the entry says they exist, + * which keeps the common read (a running run with neither) free of any Postgres call at all. + */ + async #hydrate( + read: SnapshotRead, + runId: string, + client?: ReadClient, + opts?: { waitpoints?: boolean } + ): Promise< + Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { completedWaitpoints: true; checkpoint: true }; + }> + > { + const entry = read.entry as Record; + + const checkpoint = entry.checkpointId + ? await this.#hydrateCheckpoint(runId, read.id, client) + : null; + + let completedWaitpoints: unknown[] = []; + let completedWaitpointOrder: string[] = []; + + if (opts?.waitpoints !== false) { + const ids = + read.completedWaitpointIds ?? (await this.redis.getSnapshotWaitpointIds(runId, read.id)); + completedWaitpointOrder = ids.order; + + if (ids.distinctIds.length > 0) { + completedWaitpoints = await this.delegate.findManyWaitpoints( + { where: { id: { in: ids.distinctIds } } }, + client, + runId + ); + } + } + + return { + id: read.id, + engine: entry.engine ?? "V2", + executionStatus: entry.executionStatus, + description: entry.description, + previousSnapshotId: entry.previousSnapshotId ?? null, + runId: entry.runId, + runStatus: entry.runStatus, + attemptNumber: entry.attemptNumber ?? null, + batchId: entry.batchId ?? null, + environmentId: entry.environmentId, + environmentType: entry.environmentType, + projectId: entry.projectId, + organizationId: entry.organizationId, + checkpointId: entry.checkpointId ?? null, + workerId: entry.workerId ?? null, + runnerId: entry.runnerId ?? null, + metadata: entry.metadata ?? null, + completedWaitpointOrder, + isValid: read.isValid, + error: entry.error ?? null, + createdAt: new Date(entry.createdAt as string), + updatedAt: new Date(entry.createdAt as string), + checkpoint, + completedWaitpoints, + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { completedWaitpoints: true; checkpoint: true }; + }>; + } + + /** + * Reads the checkpoint row through the snapshot the delegate still holds, so the read stays + * residency-aware: the run id in the where is what routes it to the owning database, and the + * decorator sits above the router and has no client of its own. + * + * At `redis-only` the Postgres snapshot row is gone, so this returns null. The checkpoint row + * itself stays in Postgres, but the interface has no residency-aware way to read one directly. + * Closing that needs a narrow lookup on the interface, which the plan freezes for this ticket. + */ + async #hydrateCheckpoint( + runId: string, + snapshotId: string, + client?: ReadClient + ): Promise { + const row = await this.delegate.findExecutionSnapshot( + { where: { id: snapshotId, runId }, include: { checkpoint: true } }, + client + ); + return (row as { checkpoint?: unknown } | null)?.checkpoint ?? null; + } + async #enqueueRepair(entry: SnapshotEntryInput): Promise { if (!this.onAppendFailure) { return; From 910039be4870c1e38ad5bd48e37c87779d31df63 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 16:14:21 +0100 Subject: [PATCH 09/31] feat(run-store): reap orphaned snapshot keyspaces under both sweep rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two rules, because neither can see what the other leaves behind. A terminal run whose keyspace never received the completion expiry gets one applied, so it reaps on the schedule a healthy terminal append would have set. A keyspace with no run row at all, past an age threshold, is deleted outright — that is a crashed birth, which is non-terminal so it carries no expiry and has no run row, so the first rule can never match it. It never reaps on an unknown answer: a live run is left alone however old its keyspace, a young orphan is left for the birth that may still be in flight, and a batch whose run lookup failed is skipped rather than treated as absent. Run rows are resolved through the run store rather than a raw client, because under the run-ops split a run can live on either database and a raw lookup would report a live run as an orphan. Nothing schedules this. The engine's worker has to run it, and run-store cannot reach the engine. Also moves the decorator suites onto the worker-scoped container fixture. The per-test one boots a Postgres and a Redis container for every test, which is what the replication tests need and these do not; the sweeper suite alone went from repeated two-minute timeouts to ten seconds. --- .../src/snapshotOrphanSweeper.test.ts | 325 ++++++++++++++++++ .../run-store/src/snapshotOrphanSweeper.ts | 259 ++++++++++++++ ...skRunExecutionSnapshotStore.births.test.ts | 18 +- ...askRunExecutionSnapshotStore.reads.test.ts | 28 +- ...kRunExecutionSnapshotStore.staging.test.ts | 16 +- ...ExecutionSnapshotStore.transitions.test.ts | 26 +- 6 files changed, 628 insertions(+), 44 deletions(-) create mode 100644 internal-packages/run-store/src/snapshotOrphanSweeper.test.ts create mode 100644 internal-packages/run-store/src/snapshotOrphanSweeper.ts diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts new file mode 100644 index 00000000000..68a9d99cb6d --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts @@ -0,0 +1,325 @@ +// The sweep deletes whole keyspaces, so most of these tests are about what it must NOT touch: a live +// run, a young orphan, and any batch whose Postgres lookup did not come back. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore, snapshotKeys } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { SnapshotOrphanSweeper } from "./snapshotOrphanSweeper.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; +const ORPHAN_AGE_MS = 60 * 60 * 1000; + +function birthEntry(runId: string, env: SnapshotFixtureEnv, createdAt: Date, terminal = false) { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: terminal ? ("FINISHED" as const) : ("RUN_CREATED" as const), + description: "Run was created", + runStatus: terminal ? ("CANCELED" as const) : ("PENDING" as const), + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + return entryFromCreateRun({ id: snapshot.id, runId, createdAt }, snapshot); +} + +describe("SnapshotOrphanSweeper", () => { + containerTest( + "rule 1 expires a terminal run whose keyspace never got one", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // Non-terminal append, so no expiry is ever set — the lost-TTL-set case. + await store.append({ entry: birthEntry(runId, env, new Date()), kind: "birth", isTerminal: false }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const keys = snapshotKeys(runId); + expect(await probe.pttl(keys.e)).toBe(-1); + + const result = await sweeper.sweep(); + + expect(result.expired).toBe(1); + expect(result.deleted).toBe(0); + for (const key of [keys.e, keys.idx, keys.cur, keys.seq]) { + const ttl = await probe.pttl(key); + expect(ttl).toBeGreaterThan(0); + expect(ttl).toBeLessThanOrEqual(COMPLETED_TTL_MS); + } + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "rule 1 leaves a keyspace that already has an expiry alone", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A healthy terminal append sets the completion TTL itself. + await store.append({ + entry: birthEntry(runId, env, new Date(), true), + kind: "birth", + isTerminal: true, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "CANCELED" }, + }); + + const before = await probe.pttl(snapshotKeys(runId).e); + const result = await sweeper.sweep(); + + expect(result.expired).toBe(0); + expect(result.skipped).toBe(1); + const after = await probe.pttl(snapshotKeys(runId).e); + // Not extended: the sweep must not keep resetting a countdown that is already running. + expect(after).toBeLessThanOrEqual(before); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "rule 2 deletes a keyspace with no run row, cycle keys included", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + // The crashed birth: an entry, no Postgres run, non-terminal so no expiry. + await store.append({ entry: birthEntry(runId, env, old), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(runId, env, old), + kind: "transition", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_1", index: 0 }] }, + }); + + const cyclesBefore = await probe.keys(`snap:{${runId}}:wp:*`); + expect(cyclesBefore.length).toBeGreaterThan(0); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(1); + const keys = snapshotKeys(runId); + for (const key of [keys.e, keys.idx, keys.cur, keys.seq]) { + expect(await probe.exists(key)).toBe(0); + } + expect(await probe.keys(`snap:{${runId}}:wp:*`)).toEqual([]); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("rule 2 spares a young orphan", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // Written just now: the Postgres insert of a healthy birth may still be in flight. + await store.append({ entry: birthEntry(runId, env, new Date()), kind: "birth", isTerminal: false }); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.skipped).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "never touches a live run, however old its keyspace", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const ancient = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000); + + // A run waiting on an untimed token can sit non-terminal for weeks. Reaping it would drop + // live state, which is the failure this rule exists to avoid. + await store.append({ + entry: birthEntry(runId, env, ancient), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "WAITING_TO_RESUME" }, + }); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.expired).toBe(0); + expect(result.skipped).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + expect(await probe.pttl(snapshotKeys(runId).e)).toBe(-1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("a dry run reports but changes nothing", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const orphan = generateInternalId(); + const terminal = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + await store.append({ entry: birthEntry(orphan, env, old), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(terminal, env, new Date()), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(terminal, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const result = await sweeper.sweep({ dryRun: true }); + + expect(result.deleted).toBe(1); + expect(result.expired).toBe(1); + expect(await probe.exists(snapshotKeys(orphan).e)).toBe(1); + expect(await probe.pttl(snapshotKeys(terminal).e)).toBe(-1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "skips a batch whose run lookup failed, and deletes nothing", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const failing = { + findRunsByIds: async () => { + throw new Error("run lookup unavailable"); + }, + } as unknown as RunStore; + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: failing, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + await store.append({ entry: birthEntry(runId, env, old), kind: "birth", isTerminal: false }); + + // A failed lookup says nothing about whether the run exists, and rule 2 deletes a whole + // keyspace. The sweep must resolve rather than throw, and must reap nothing. + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.skipped).toBeGreaterThan(0); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("processes every keyspace across batches", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + const orphans = Array.from({ length: 5 }, () => generateInternalId()); + for (const runId of orphans) { + await store.append({ entry: birthEntry(runId, env, old), kind: "birth", isTerminal: false }); + } + + const result = await sweeper.sweep({ batchSize: 2 }); + + expect(result.deleted).toBe(5); + for (const runId of orphans) { + expect(await probe.exists(snapshotKeys(runId).e)).toBe(0); + } + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.ts new file mode 100644 index 00000000000..f94876da37c --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.ts @@ -0,0 +1,259 @@ +// Reaps snapshot keyspaces that no healthy path will ever clean up. +// +// Two rules, because neither can see what the other leaves behind: +// +// 1. The run is terminal in Postgres but its keyspace never got the completion expiry — a +// terminal append whose TTL-set was lost. Applying the expiry now reaps it on the same +// schedule a healthy terminal append would have. +// 2. The keyspace has no Postgres run row at all, and is older than a threshold — a crashed +// birth. It is non-terminal so it carries no expiry, and it has no run row, so rule 1 can +// never match it. Without this rule that leak has no bound. +// +// Nothing schedules this. The engine's worker is what has to run it, and run-store cannot reach the +// engine, so the wiring belongs to the ticket that owns production construction. +import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis"; +import { Logger } from "@trigger.dev/core/logger"; +import type { TaskRunStatus } from "@trigger.dev/database"; +import { snapshotKeys } from "./redisSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +/** + * Mirrors the engine's `finalStatuses`. run-store cannot import from run-engine — the dependency + * runs the other way — so the list is duplicated and a parity test in run-engine asserts the copy + * stays equal to the original. + */ +export const FINAL_RUN_STATUSES: readonly TaskRunStatus[] = [ + "CANCELED", + "INTERRUPTED", + "COMPLETED_SUCCESSFULLY", + "COMPLETED_WITH_ERRORS", + "SYSTEM_FAILURE", + "CRASHED", + "EXPIRED", + "TIMED_OUT", +]; + +const FINAL = new Set(FINAL_RUN_STATUSES); + +/** Comfortably above run-creation latency, so a birth in flight is never mistaken for an orphan. */ +const DEFAULT_ORPHAN_AGE_MS = 24 * 60 * 60 * 1000; +const DEFAULT_BATCH_SIZE = 1000; + +export type SweepResult = { + /** Keyspaces examined. */ + scanned: number; + /** Rule 1: terminal runs whose keyspace was given the completion expiry. */ + expired: number; + /** Rule 2: keyspaces with no run row, deleted. */ + deleted: number; + /** Left alone: a live run, a young orphan, or a batch whose Postgres lookup failed. */ + skipped: number; +}; + +export type SnapshotOrphanSweeperOptions = { + /** + * The sweep opens its own connection rather than borrowing the store's, so a long scan can never + * stall a hot-path client. + */ + redisOptions: RedisOptions; + /** + * Resolved through the run store, not a raw client. Under the run-ops split a run row can live on + * either database, and only the store knows which — a raw lookup would report a live run as an + * orphan and delete its keyspace. + */ + runStore: RunStore; + completedTtlMs: number; + orphanAgeMs?: number; + keyPrefix?: string; + logger?: Logger; +}; + +export class SnapshotOrphanSweeper { + readonly #redis: Redis; + readonly #runStore: RunStore; + readonly #completedTtlMs: number; + readonly #orphanAgeMs: number; + readonly #prefix: string; + readonly #logger: Logger; + #quit?: Promise; + + constructor(options: SnapshotOrphanSweeperOptions) { + this.#logger = options.logger ?? new Logger("SnapshotOrphanSweeper", "debug"); + this.#runStore = options.runStore; + this.#completedTtlMs = options.completedTtlMs; + this.#orphanAgeMs = options.orphanAgeMs ?? DEFAULT_ORPHAN_AGE_MS; + this.#prefix = options.keyPrefix ?? "snap:"; + this.#redis = createRedisClient(options.redisOptions, { + onError: (error) => this.#logger.error("SnapshotOrphanSweeper redis client error", { error }), + }); + } + + async quit(): Promise { + if (!this.#quit) { + this.#quit = this.#redis.quit().then( + () => undefined, + () => undefined + ); + } + await this.#quit; + } + + /** + * One full pass over the keyspace. `dryRun` reports what it would do and changes nothing. + */ + async sweep(opts?: { batchSize?: number; dryRun?: boolean }): Promise { + const batchSize = opts?.batchSize ?? DEFAULT_BATCH_SIZE; + const dryRun = opts?.dryRun ?? false; + const result: SweepResult = { scanned: 0, expired: 0, deleted: 0, skipped: 0 }; + + let cursor = "0"; + do { + const [next, keys] = await this.#redis.scan( + cursor, + "MATCH", + `${this.#prefix}{*}:cur`, + "COUNT", + batchSize + ); + cursor = next; + + const runIds = [...new Set(keys.map((key) => this.#runIdFrom(key)).filter(isString))]; + if (runIds.length === 0) continue; + + await this.#sweepBatch(runIds, dryRun, result); + } while (cursor !== "0"); + + this.#logger.log("SnapshotOrphanSweeper pass complete", { ...result, dryRun }); + return result; + } + + async #sweepBatch(runIds: string[], dryRun: boolean, result: SweepResult): Promise { + result.scanned += runIds.length; + + let rows: Map; + try { + rows = (await this.#runStore.findRunsByIds(runIds, { + select: { id: true, status: true }, + })) as unknown as Map; + } catch (error) { + // Never reap on an unknown answer. A lookup that failed says nothing about whether the run + // exists, and rule 2 deletes a whole keyspace. + this.#logger.error("SnapshotOrphanSweeper skipped a batch after a failed run lookup", { + count: runIds.length, + error, + }); + result.skipped += runIds.length; + return; + } + + for (const runId of runIds) { + const run = rows.get(runId); + + if (!run) { + await this.#applyRuleTwo(runId, dryRun, result); + continue; + } + + if (!FINAL.has(run.status)) { + // A live run. A SUSPENDED run can legitimately wait for weeks, so this is never touched. + result.skipped += 1; + continue; + } + + await this.#applyRuleOne(runId, dryRun, result); + } + } + + /** Rule 1: a terminal run whose keyspace never received the completion expiry. */ + async #applyRuleOne(runId: string, dryRun: boolean, result: SweepResult): Promise { + const keys = await this.#allKeys(runId); + if (keys.length === 0) { + result.skipped += 1; + return; + } + + const ttls = await Promise.all(keys.map((key) => this.#redis.pttl(key))); + // -1 is "exists, no expiry". Anything already counting down was set by a healthy append. + if (!ttls.some((ttl) => ttl === -1)) { + result.skipped += 1; + return; + } + + if (!dryRun) { + const pipeline = this.#redis.pipeline(); + for (const key of keys) { + pipeline.pexpire(key, this.#completedTtlMs); + } + await pipeline.exec(); + } + + result.expired += 1; + } + + /** Rule 2: a keyspace with no run row at all, past the age threshold. */ + async #applyRuleTwo(runId: string, dryRun: boolean, result: SweepResult): Promise { + const keys = await this.#allKeys(runId); + if (keys.length === 0) { + result.skipped += 1; + return; + } + + const age = await this.#newestEntryAgeMs(runId); + if (age === undefined || age < this.#orphanAgeMs) { + // Either the keyspace carries no readable timestamp, or a birth may still be in flight. + result.skipped += 1; + return; + } + + if (!dryRun) { + await this.#redis.del(...keys); + } + + result.deleted += 1; + } + + /** Every key for one run: the four core keys plus each wait-cycle key. */ + async #allKeys(runId: string): Promise { + const core = snapshotKeys(runId); + // Scoped to one hash tag, so this is a lookup inside a single slot rather than a keyspace scan. + const cycles = await this.#redis.keys(`${this.#prefix}{${runId}}:wp:*`); + const candidates = [core.e, core.idx, core.cur, core.seq, ...cycles]; + + const exists = await Promise.all(candidates.map((key) => this.#redis.exists(key))); + return candidates.filter((_key, index) => exists[index] === 1); + } + + /** + * Age of the newest entry, so a keyspace still being written to is never treated as an orphan. + * The newest is the right end: an old first entry says nothing about whether the run is dead. + */ + async #newestEntryAgeMs(runId: string): Promise { + const core = snapshotKeys(runId); + const newest = await this.#redis.zrevrange(core.idx, 0, 0); + const id = newest[0]; + if (!id) return undefined; + + const raw = await this.#redis.hget(core.e, id); + if (!raw) return undefined; + + try { + const createdAt = (JSON.parse(raw) as { createdAt?: string }).createdAt; + if (!createdAt) return undefined; + const parsed = Date.parse(createdAt); + return Number.isNaN(parsed) ? undefined : Date.now() - parsed; + } catch { + return undefined; + } + } + + #runIdFrom(key: string): string | undefined { + const open = key.indexOf("{"); + const close = key.indexOf("}", open + 1); + if (open === -1 || close === -1 || close === open + 1) return undefined; + return key.slice(open + 1, close); + } +} + +function isString(value: string | undefined): value is string { + return typeof value === "string"; +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts index e94e41f94b6..12ab76a0f26 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts @@ -2,7 +2,7 @@ // which side survived: an orphaned key with no run row is the harmless state, and a run with no // snapshot at all is the one the order exists to prevent. import { describe, expect } from "vitest"; -import { postgresAndRedisTest } from "@internal/testcontainers"; +import { containerTest } from "@internal/testcontainers"; import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; import { PostgresRunStore } from "./PostgresRunStore.js"; import { RedisSnapshotStore } from "./redisSnapshotStore.js"; @@ -77,7 +77,7 @@ function cancelledData(runId: string, env: SnapshotFixtureEnv) { } describe("birth write ordering", () => { - postgresAndRedisTest("writes Redis then Postgres", async ({ prisma, redisOptions }) => { + containerTest("writes Redis then Postgres", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -99,7 +99,7 @@ describe("birth write ordering", () => { } }); - postgresAndRedisTest("mints an id when the caller supplies none", async ({ prisma, redisOptions }) => { + containerTest("mints an id when the caller supplies none", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -119,7 +119,7 @@ describe("birth write ordering", () => { } }); - postgresAndRedisTest( + containerTest( "a crash after the Redis append leaves an orphan key and no run", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never, { @@ -147,7 +147,7 @@ describe("birth write ordering", () => { } ); - postgresAndRedisTest( + containerTest( "creates the run anyway when the birth append fails before redis-only", { timeout: 60_000 }, async ({ prisma, redisOptions }) => { @@ -175,7 +175,7 @@ describe("birth write ordering", () => { } ); - postgresAndRedisTest( + containerTest( "refuses to create the run when the birth append fails at redis-only", { timeout: 60_000 }, async ({ prisma, redisOptions }) => { @@ -203,7 +203,7 @@ describe("birth write ordering", () => { } ); - postgresAndRedisTest("createCancelledRun writes Redis first", async ({ prisma, redisOptions }) => { + containerTest("createCancelledRun writes Redis first", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -229,7 +229,7 @@ describe("birth write ordering", () => { } }); - postgresAndRedisTest( + containerTest( "a born-terminal run gets the completion expiry immediately", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); @@ -265,7 +265,7 @@ describe("birth write ordering", () => { } ); - postgresAndRedisTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { + containerTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never, { mode: "off" }); try { const env = await seedSnapshotEnvironment(prisma); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts index ffdd91a92f9..1510773e1ce 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts @@ -1,7 +1,7 @@ // Reads served from Redis must be indistinguishable from the Postgres reads they replace: the same // payload shape, the same tenant boundary, the same fallback when Redis does not hold the answer. import { describe, expect } from "vitest"; -import { postgresAndRedisTest } from "@internal/testcontainers"; +import { containerTest } from "@internal/testcontainers"; import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; import { PostgresRunStore } from "./PostgresRunStore.js"; import { RedisSnapshotStore } from "./redisSnapshotStore.js"; @@ -77,7 +77,7 @@ function snapshotInput(runId: string, env: SnapshotFixtureEnv, description: stri } describe("snapshot reads", () => { - postgresAndRedisTest("serves the latest snapshot from Redis", async ({ prisma, redisOptions }) => { + containerTest("serves the latest snapshot from Redis", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -101,7 +101,7 @@ describe("snapshot reads", () => { } }); - postgresAndRedisTest( + containerTest( "returns the same payload Postgres would", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); @@ -128,7 +128,7 @@ describe("snapshot reads", () => { } ); - postgresAndRedisTest( + containerTest( "reads a foreign environment as not found, so the caller's 404 still fires", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); @@ -146,7 +146,7 @@ describe("snapshot reads", () => { } ); - postgresAndRedisTest( + containerTest( "falls back to Postgres for a run with no keyspace", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never); @@ -181,7 +181,7 @@ describe("snapshot reads", () => { } ); - postgresAndRedisTest("reads from Postgres at readPercent 0", async ({ prisma, redisOptions }) => { + containerTest("reads from Postgres at readPercent 0", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { readPercent: 0, }); @@ -198,7 +198,7 @@ describe("snapshot reads", () => { } }); - postgresAndRedisTest("reads from Postgres at mode dual-write", async ({ prisma, redisOptions }) => { + containerTest("reads from Postgres at mode dual-write", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { mode: "dual-write", }); @@ -215,7 +215,7 @@ describe("snapshot reads", () => { } }); - postgresAndRedisTest("serves the since-cursor lookup from Redis", async ({ prisma, redisOptions }) => { + containerTest("serves the since-cursor lookup from Redis", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -239,7 +239,7 @@ describe("snapshot reads", () => { } }); - postgresAndRedisTest( + containerTest( "delegates a snapshot lookup it does not recognise", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never); @@ -264,7 +264,7 @@ describe("snapshot reads", () => { } ); - postgresAndRedisTest("serves the since window from Redis", async ({ prisma, redisOptions }) => { + containerTest("serves the since window from Redis", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -290,7 +290,7 @@ describe("snapshot reads", () => { } }); - postgresAndRedisTest( + containerTest( "delegates a window query it does not recognise", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never); @@ -312,7 +312,7 @@ describe("snapshot reads", () => { } ); - postgresAndRedisTest( + containerTest( "serves the waitpoint id projections from Redis", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never); @@ -344,7 +344,7 @@ describe("snapshot reads", () => { } ); - postgresAndRedisTest( + containerTest( "delegates a waitpoint id projection with no run id", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never); @@ -366,7 +366,7 @@ describe("snapshot reads", () => { } ); - postgresAndRedisTest("never touches Redis for reads at mode off", async ({ prisma, redisOptions }) => { + containerTest("never touches Redis for reads at mode off", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { mode: "off", }); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts index ac0a877c968..3b75ca4ca05 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts @@ -2,7 +2,7 @@ // leaves Redis holding a transition that never happened. These tests observe the buffer from inside // the callback, so the deferral is proved rather than assumed. import { describe, expect } from "vitest"; -import { postgresAndRedisTest } from "@internal/testcontainers"; +import { containerTest } from "@internal/testcontainers"; import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; import { PostgresRunStore } from "./PostgresRunStore.js"; import { RedisSnapshotStore } from "./redisSnapshotStore.js"; @@ -65,7 +65,7 @@ function snapshotInput(runId: string, env: SnapshotFixtureEnv, id: string, descr } describe("the staging facade", () => { - postgresAndRedisTest("flushes the append after the commit", async ({ prisma, redisOptions }) => { + containerTest("flushes the append after the commit", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -87,7 +87,7 @@ describe("the staging facade", () => { } }); - postgresAndRedisTest( + containerTest( "writes nothing to Redis when the transaction rolls back", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); @@ -113,7 +113,7 @@ describe("the staging facade", () => { } ); - postgresAndRedisTest("flushes several staged appends in order", async ({ prisma, redisOptions }) => { + containerTest("flushes several staged appends in order", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -138,7 +138,7 @@ describe("the staging facade", () => { } }); - postgresAndRedisTest( + containerTest( "hands the transaction callback a decorated store", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); @@ -160,7 +160,7 @@ describe("the staging facade", () => { } ); - postgresAndRedisTest( + containerTest( "hands the transaction callback the plain delegate at mode off", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); @@ -195,7 +195,7 @@ describe("the staging facade", () => { } ); - postgresAndRedisTest( + containerTest( "wraps the store handle from forWaitpointCompletion", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); @@ -213,7 +213,7 @@ describe("the staging facade", () => { } ); - postgresAndRedisTest( + containerTest( "returns the plain handle from forWaitpointCompletion at mode off", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts index 786e3a74453..311b46cc780 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts @@ -2,7 +2,7 @@ // reading the code: with the Redis half made to fail, the Postgres row is still there and the caller // sees no error, which is only possible if Postgres went first. import { describe, expect } from "vitest"; -import { postgresAndRedisTest } from "@internal/testcontainers"; +import { containerTest } from "@internal/testcontainers"; import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; import { PostgresRunStore } from "./PostgresRunStore.js"; import { RedisSnapshotStore } from "./redisSnapshotStore.js"; @@ -134,7 +134,7 @@ function expireInput(env: SnapshotFixtureEnv) { } describe("transition write ordering", () => { - postgresAndRedisTest("writes Postgres then Redis", async ({ prisma, redisOptions }) => { + containerTest("writes Postgres then Redis", async ({ prisma, redisOptions }) => { const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -157,7 +157,7 @@ describe("transition write ordering", () => { } }); - postgresAndRedisTest( + containerTest( "keeps the Postgres write and enqueues one repair when the append fails", async ({ prisma, redisOptions }) => { const { decorated, redis, repairs } = harness(prisma as never, redisOptions as never, { @@ -190,7 +190,7 @@ describe("transition write ordering", () => { } ); - postgresAndRedisTest( + containerTest( "treats a transition on a run with no keyspace as skipped, not failed", async ({ prisma, redisOptions }) => { const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); @@ -210,7 +210,7 @@ describe("transition write ordering", () => { } ); - postgresAndRedisTest("appends for expireRun", async ({ prisma, redisOptions }) => { + containerTest("appends for expireRun", async ({ prisma, redisOptions }) => { const { decorated, redis } = harness(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -229,7 +229,7 @@ describe("transition write ordering", () => { } }); - postgresAndRedisTest("appends for expireParkedRun", async ({ prisma, redisOptions }) => { + containerTest("appends for expireParkedRun", async ({ prisma, redisOptions }) => { const { decorated, redis } = harness(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -253,7 +253,7 @@ describe("transition write ordering", () => { } }); - postgresAndRedisTest( + containerTest( "appends nothing when expireParkedRun matches no run", async ({ prisma, redisOptions }) => { const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); @@ -278,7 +278,7 @@ describe("transition write ordering", () => { } ); - postgresAndRedisTest("appends for rescheduleRun", async ({ prisma, redisOptions }) => { + containerTest("appends for rescheduleRun", async ({ prisma, redisOptions }) => { const { decorated, redis } = harness(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -306,7 +306,7 @@ describe("transition write ordering", () => { } }); - postgresAndRedisTest( + containerTest( "appends nothing when rescheduleRun carries no snapshot", async ({ prisma, redisOptions }) => { const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); @@ -325,7 +325,7 @@ describe("transition write ordering", () => { } ); - postgresAndRedisTest("appends for lockRunToWorker under a CAS", async ({ prisma, redisOptions }) => { + containerTest("appends for lockRunToWorker under a CAS", async ({ prisma, redisOptions }) => { const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -367,7 +367,7 @@ describe("transition write ordering", () => { } }); - postgresAndRedisTest( + containerTest( "reports a forked append without enqueuing a repair", async ({ prisma, redisOptions }) => { const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); @@ -409,7 +409,7 @@ describe("transition write ordering", () => { } ); - postgresAndRedisTest("appends for the standalone createExecutionSnapshot", async ({ prisma, redisOptions }) => { + containerTest("appends for the standalone createExecutionSnapshot", async ({ prisma, redisOptions }) => { const { decorated, redis } = harness(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -435,7 +435,7 @@ describe("transition write ordering", () => { } }); - postgresAndRedisTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { + containerTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { const { decorated, redis } = harness(prisma as never, redisOptions as never, { mode: "off" }); try { const { run, env } = await setupSnapshotIdFixture(prisma); From 81194668a91f30b72a82478d443e88f4911219bb Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 16:23:28 +0100 Subject: [PATCH 10/31] test(run-engine): run the snapshot flows against the decorator with reads on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The engine's own flows, driven against the decorator with every snapshot read served from Redis, injected through the store seam that runStoreInjectability already proves. Same flows, same expectations, different store underneath — the point is that nothing in the engine has to know, so no existing suite changes. Covers a run driven to completion, the execution data at each step, a since-window wider than the fifty cap, and a pre-cutover run with no keyspace falling back to Postgres. The environment-boundary test asserts parity rather than a fixed shape: whatever Postgres answers for a foreign environment, Redis has to answer the same, or the tenant boundary behaves differently once reads move over. --- .../engine/tests/helpers/decoratedStore.ts | 75 +++++ .../tests/snapshotStoreReadGate.test.ts | 298 ++++++++++++++++++ internal-packages/run-store/src/index.ts | 5 + 3 files changed, 378 insertions(+) create mode 100644 internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts create mode 100644 internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts diff --git a/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts b/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts new file mode 100644 index 00000000000..c74aa86db92 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts @@ -0,0 +1,75 @@ +// Builds the snapshot-store decorator over a real PostgresRunStore, for injection through the +// engine's `store` option — the seam runStoreInjectability.test.ts already proves. +// +// The point of injecting it is that the engine suites keep their own assertions: the same flows, +// the same expectations, a different store underneath. +import { + PostgresRunStore, + RedisSnapshotStore, + TaskRunExecutionSnapshotStore, + type SnapshotFaultInjector, + type SnapshotRepairEnqueuer, + type SnapshotStoreMode, +} from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RedisOptions } from "@internal/redis"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +export type DecoratedStoreHarness = { + store: TaskRunExecutionSnapshotStore; + redis: RedisSnapshotStore; + /** Every read the decorator served, and which store answered it. */ + reads: { method: string; source: "redis" | "postgres" }[]; + /** Every append outcome, keyed by the write site that produced it. */ + writes: { site: string; outcome: string }[]; + /** Runs handed to the repair job because their append was lost. */ + repairs: { runId: string; snapshotId: string; executionStatus: string }[]; + quit(): Promise; +}; + +export function buildDecoratedStore(opts: { + prisma: PrismaClient; + redisOptions: RedisOptions; + mode: SnapshotStoreMode; + readPercent?: number; + faults?: SnapshotFaultInjector; + onAppendFailure?: SnapshotRepairEnqueuer; +}): DecoratedStoreHarness { + const redis = new RedisSnapshotStore({ + redisOptions: opts.redisOptions, + completedTtlMs: COMPLETED_TTL_MS, + }); + + const reads: DecoratedStoreHarness["reads"] = []; + const writes: DecoratedStoreHarness["writes"] = []; + const repairs: DecoratedStoreHarness["repairs"] = []; + + const store = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma: opts.prisma as never, readOnlyPrisma: opts.prisma as never }), + { + store: redis, + mode: opts.mode, + readPercent: opts.readPercent ?? 100, + ...(opts.faults && { faults: opts.faults }), + onAppendFailure: async (args) => { + repairs.push(args); + await opts.onAppendFailure?.(args); + }, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { + store, + redis, + reads, + writes, + repairs, + quit: () => redis.quit(), + }; +} diff --git a/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts new file mode 100644 index 00000000000..097f92a5cd0 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts @@ -0,0 +1,298 @@ +// The read gate: the engine's own snapshot flows, run against the decorator with reads served from +// Redis. Same flows, same expectations, different store underneath — the point is that nothing in +// the engine has to know, so no existing suite is modified to make this pass. +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { setTimeout } from "timers/promises"; +import { generateInternalId, RunId } from "@trigger.dev/core/v3/isomorphic"; +import { RunEngine } from "../index.js"; +import { buildDecoratedStore, type DecoratedStoreHarness } from "./helpers/decoratedStore.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; + +vi.setConfig({ testTimeout: 60_000 }); + +function engineOptions(prisma: any, redisOptions: any, harness: DecoratedStoreHarness) { + return { + prisma, + store: harness.store, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} + +const triggerArgs = (taskIdentifier: string, environment: any, n: number) => ({ + number: n, + // A real minted friendly id: the engine converts it back with RunId.fromFriendlyId, which + // rejects anything that is not the prefix plus a cuid body. + friendlyId: RunId.generate().friendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `t_gate_${n}`, + spanId: `s_gate_${n}`, + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], +}); + +describe("snapshot store read gate", () => { + containerTest( + "drives a run to completion with every snapshot read served from Redis", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 1), prisma); + await setTimeout(500); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "gate_consumer", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + expect(attempt.run.status).toBe("EXECUTING"); + + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { ok: true, id: run.id, output: `{"done":true}`, outputType: "application/json" }, + }); + + const finished = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finished.status).toBe("COMPLETED_SUCCESSFULLY"); + + // The gate: the engine read its snapshots, and Redis is what answered. + const fromRedis = harness.reads.filter((r) => r.source === "redis"); + expect(fromRedis.length).toBeGreaterThan(0); + expect(harness.reads.filter((r) => r.source === "postgres")).toEqual([]); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "serves getRunExecutionData from Redis at every step", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 2), prisma); + await setTimeout(500); + + const queued = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(queued); + expect(queued.snapshot.executionStatus).toBe("QUEUED"); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "gate_consumer", + workerQueue: "main", + }); + const pending = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(pending); + expect(pending.snapshot.executionStatus).toBe("PENDING_EXECUTING"); + + await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + const executing = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executing); + expect(executing.snapshot.executionStatus).toBe("EXECUTING"); + expect(executing.run.attemptNumber).toBe(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "keeps the environment boundary on a snapshot read", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 3), prisma); + await setTimeout(500); + + // Scoped to its own environment the run reads normally. + const own = await engine.getRunExecutionData({ + runId: run.id, + environmentId: environment.id, + }); + assertNonNullable(own); + + // Scoped to any other environment the run must not leak across the tenant boundary. The + // assertion is parity rather than a fixed shape: whatever Postgres answers for this call, + // Redis has to answer the same, or the boundary behaves differently once reads move over. + const foreignEnvironmentId = generateInternalId(); + + const viaRedis = await engine + .getRunExecutionData({ runId: run.id, environmentId: foreignEnvironmentId }) + .catch((error: unknown) => ({ threw: (error as Error).constructor.name })); + + const postgresOnly = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); + const engineOff = new RunEngine( + engineOptions(prisma, redisOptions, postgresOnly) as never + ); + let viaPostgres: unknown; + try { + viaPostgres = await engineOff + .getRunExecutionData({ runId: run.id, environmentId: foreignEnvironmentId }) + .catch((error: unknown) => ({ threw: (error as Error).constructor.name })); + } finally { + await engineOff.quit(); + await postgresOnly.quit(); + } + + expect(viaRedis).toEqual(viaPostgres); + // And whatever that shape is, it must not be the run's data. + expect(viaRedis).not.toMatchObject({ run: { id: run.id } }); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "serves a since-window wider than the cap from Redis", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 4), prisma); + await setTimeout(500); + + const first = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(first); + + // More transitions than the 50-cap, so the window is exercised at its boundary. + for (let i = 0; i < 60; i++) { + await harness.store.createExecutionSnapshot({ + run: { id: run.id, status: "PENDING", attemptNumber: null }, + snapshot: { executionStatus: "QUEUED", description: `filler ${i}` }, + environmentId: environment.id, + environmentType: environment.type, + projectId: environment.project.id, + organizationId: environment.organization.id, + }); + } + + const since = await engine.getSnapshotsSince({ + runId: run.id, + snapshotId: first.snapshot.id, + }); + assertNonNullable(since); + + // The newest 50, ascending — the same window Postgres would have produced. + expect(since.length).toBe(50); + expect(since[since.length - 1]!.snapshot.description).toBe("filler 59"); + expect(harness.reads.some((r) => r.source === "redis")).toBe(true); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest("falls back to Postgres for a pre-cutover run", async ({ prisma, redisOptions }) => { + // A run created while the dial was off has no keyspace. Turning reads on must not lose it. + const off = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); + const engineOff = new RunEngine(engineOptions(prisma, redisOptions, off) as never); + + let runId: string; + let environment: any; + try { + environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engineOff, environment, "gate-task"); + const run = await engineOff.trigger(triggerArgs("gate-task", environment, 5), prisma); + runId = run.id; + await setTimeout(500); + } finally { + await engineOff.quit(); + await off.quit(); + } + + const on = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engineOn = new RunEngine(engineOptions(prisma, redisOptions, on) as never); + try { + const data = await engineOn.getRunExecutionData({ runId }); + assertNonNullable(data); + expect(data.snapshot.executionStatus).toBe("QUEUED"); + expect(on.reads.some((r) => r.source === "postgres")).toBe(true); + } finally { + await engineOn.quit(); + await on.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/index.ts b/internal-packages/run-store/src/index.ts index 3717dc01527..16fa696ee43 100644 --- a/internal-packages/run-store/src/index.ts +++ b/internal-packages/run-store/src/index.ts @@ -3,3 +3,8 @@ export * from "./PostgresRunStore.js"; export * from "./runOpsStore.js"; export * from "./readReplicaClient.js"; export * from "./redisSnapshotStore.js"; +export * from "./delegatingRunStore.js"; +export * from "./taskRunExecutionSnapshotStore.js"; +export * from "./snapshotEntry.js"; +export * from "./snapshotFaultInjection.js"; +export * from "./snapshotOrphanSweeper.js"; From aee3f0759d4c2f1d31404436713f0cfe99925a2c Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 17:28:38 +0100 Subject: [PATCH 11/31] fix(run-store): give a snapshot one identity and one instant across both stores Three defects, all of which passed the existing suites because no test drove a snapshot that actually carried waitpoints, and because the parity suite compared createdAt against a value it had just read back from the row. The decorator never passed a cycle to the append, so no wp: key was written for any snapshot and the completed-waitpoint side of Redis was permanently empty. It now mints a cycle when the id set differs from the current head and carries the previous cycleSeq forward when it does not, so a resume writes the record set once and the copy-forwards that follow write no key at all. The since-window hydration returned an empty completedWaitpointOrder. That column is not the join: the engine reads it off the head row as the oracle that gives each completed waitpoint its position in a batch, so an empty order resumed every batched triggerAndWait with an undefined index. Seven of the eight write sites stamped the entry from the app clock while Postgres stamped its own column default, so the two stores held different instants for one snapshot. The decorator now supplies createdAt, and an equal updatedAt, at every site, and the standalone path supplies it too rather than reading the row back. Beyond making the field comparable, this aligns the since-window: the cursor is resolved from one store and applied in the other, and two different instants misfilter that window. The parity suite gains an independent clock-provenance guard, and a case proving an absent instant still takes the database default, which is what keeps the store's behaviour unchanged while the decorator is off. --- .../engine/tests/snapshotStoreChaos.test.ts | 400 ++++++++++++++++++ .../tests/snapshotStoreReadGate.test.ts | 80 ++-- .../scripts/generateDelegatingRunStore.ts | 10 +- .../PostgresRunStore.snapshotWrites.test.ts | 8 +- .../run-store/src/PostgresRunStore.ts | 19 +- .../redisSnapshotStore.sinceCreatedAt.test.ts | 44 +- .../run-store/src/runStoreMethodNames.ts | 4 +- .../src/snapshotEntry.parity.test.ts | 109 ++++- .../src/snapshotOrphanSweeper.test.ts | 30 +- .../run-store/src/snapshotReadShapes.test.ts | 8 +- ...askRunExecutionSnapshotStore.reads.test.ts | 47 +- ...ExecutionSnapshotStore.transitions.test.ts | 51 +-- .../src/taskRunExecutionSnapshotStore.ts | 209 +++++++-- ...utionSnapshotStore.waitpointCycles.test.ts | 363 ++++++++++++++++ .../src/testFixtures/snapshotIdFixture.ts | 32 ++ internal-packages/run-store/src/types.ts | 30 ++ 16 files changed, 1269 insertions(+), 175 deletions(-) create mode 100644 internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts diff --git a/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts new file mode 100644 index 00000000000..a727c9bcc1f --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts @@ -0,0 +1,400 @@ +// The correctness spine: kill the process at each write boundary and prove the run still converges. +// +// The write protocol's whole claim is that whatever a crash leaves behind is a state the existing +// stall-and-repair machinery heals. That claim is not checkable by reading the code, so each test +// here injects a fault at one named boundary and then asserts three things: the run converges, it +// does not hang, and it burns at most one attempt number per crash. +// +// The bound is PER CRASH, not a flat one. The plan records that TLC refuted a flat bound of one in +// seven states, and that the property which holds is pgAttempt - maxLoggedAttempt <= crashCount. +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { generateInternalId, RunId } from "@trigger.dev/core/v3/isomorphic"; +import { + InjectedSnapshotFault, + type SnapshotFaultBoundary, + type SnapshotFaultInjector, +} from "@internal/run-store"; +import { setTimeout } from "timers/promises"; +import { RunEngine } from "../index.js"; +import { buildDecoratedStore, type DecoratedStoreHarness } from "./helpers/decoratedStore.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; + +vi.setConfig({ testTimeout: 60_000 }); + +/** + * A local stand-in for the shared fault harness being built alongside this ticket. Its surface is + * the agreed one — arm, disarm, hook, fired — so swapping the import in costs no test-body change. + * + * `fired` is the guard against a silent pass. A boundary can be armed and never reached, in which + * case the test would go green having proved nothing, so every test asserts its boundary fired. + */ +function createFaultInjector(opts: { error: (boundary: SnapshotFaultBoundary) => Error }) { + const armed = new Map(); + const counts = new Map(); + + return { + arm(boundary: SnapshotFaultBoundary, opts?: { times?: number; runId?: string }) { + armed.set(boundary, { times: opts?.times ?? 1, ...(opts?.runId && { runId: opts.runId }) }); + }, + disarm(boundary: SnapshotFaultBoundary) { + armed.delete(boundary); + }, + fired(boundary: SnapshotFaultBoundary): number { + return counts.get(boundary) ?? 0; + }, + hook: ((boundary, context) => { + const entry = armed.get(boundary); + if (!entry) return; + if (entry.runId && context.runId !== entry.runId) return; + + counts.set(boundary, (counts.get(boundary) ?? 0) + 1); + entry.times -= 1; + if (entry.times <= 0) armed.delete(boundary); + + throw opts.error(boundary); + }) satisfies SnapshotFaultInjector, + }; +} + +function engineOptions( + prisma: any, + redisOptions: any, + harness: DecoratedStoreHarness, + heartbeatMs: number +) { + return { + prisma, + store: harness.store, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + retryOptions: { maxTimeoutInMs: 50 }, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + heartbeatTimeoutsMs: { PENDING_EXECUTING: heartbeatMs }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} + +const triggerArgs = (taskIdentifier: string, environment: any) => ({ + number: 1, + friendlyId: RunId.generate().friendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `t_${generateInternalId().slice(-12)}`, + spanId: `s_${generateInternalId().slice(-12)}`, + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], +}); + +describe("snapshot store crash boundaries", () => { + containerTest( + "afterPgBeforeRedis: the run converges and burns at most one attempt", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + // Crash between the Postgres commit and the Redis append of ONE transition. + faults.arm("afterPgBeforeRedis", { times: 1, runId: run.id }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + faults.disarm("afterPgBeforeRedis"); + + // The boundary was actually reached. Without this the test could pass having proved nothing. + expect(faults.fired("afterPgBeforeRedis")).toBe(1); + + // Postgres committed the attempt bump; the run is not stuck and not lost. + expect(attempt.run.attemptNumber).toBe(1); + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.attemptNumber).toBe(1); + + // The gap handed the run to the repair job rather than failing the caller. + expect(harness.repairs).toHaveLength(1); + expect(harness.repairs[0]!.runId).toBe(run.id); + + // Reads still resolve: the run's state machine is readable, so nothing hangs. + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + + // The bound: one crash costs at most one attempt number. + expect(pgRun.attemptNumber! - 1).toBeLessThanOrEqual(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "afterRedisBirthBeforePg: no run is created, and the next trigger succeeds", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + + faults.arm("afterRedisBirthBeforePg", { times: 1 }); + await expect( + engine.trigger(triggerArgs("chaos-task", environment), prisma) + ).rejects.toBeInstanceOf(InjectedSnapshotFault); + expect(faults.fired("afterRedisBirthBeforePg")).toBe(1); + + // The harmless state: no run row, so nothing can ever read a run that has no snapshot. + const runsAfterCrash = await prisma.taskRun.count(); + expect(runsAfterCrash).toBe(0); + + // A crashed birth must not poison the path: the next trigger runs to completion. + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, + }); + + const finished = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finished.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(finished.attemptNumber).toBe(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "midFlushRetry: a crash during a retry still converges through the repair job", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + // A dead port makes attempt 0 fail FOR REAL, which is the only way the retry boundary is + // reachable: an injected fault at attempt 0 is treated as a dead process and skips the + // retries entirely. Arming midFlushRetry alone would fire nothing and pass for the wrong + // reason, which is what the fired() assertion below catches. + const harness = buildDecoratedStore({ + prisma, + redisOptions: { ...(redisOptions as object), port: 1, retryStrategy: () => null } as never, + mode: "dual-write", + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + + faults.arm("midFlushRetry", { times: 1 }); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + // The birth append failed for real and, before redis-only, that is survivable: Postgres is + // authoritative and the run exists. + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.id).toBe(run.id); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + + // The retry boundary was genuinely reached, not merely armed. + expect(faults.fired("midFlushRetry")).toBeGreaterThanOrEqual(1); + + // The run converges regardless: Postgres holds every snapshot at this dial position. + expect(attempt.run.attemptNumber).toBe(1); + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + expect(data.snapshot.executionStatus).toBe("EXECUTING"); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "a crash-stalled run rejects a stale snapshot rather than hanging", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + faults.arm("afterPgBeforeRedis", { times: 1, runId: run.id }); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + faults.disarm("afterPgBeforeRedis"); + expect(faults.fired("afterPgBeforeRedis")).toBe(1); + + // Postgres advanced; the Redis head did not. Reads come from Redis, so the caller now holds + // a snapshot id that no longer matches what the read store reports as latest. + // + // The contract is that this SURFACES rather than corrupts: the next operation to validate + // against latest rejects with a stale-snapshot error, which is the same answer a caller gets + // from an ordinary lost race. It does not hang, and it does not silently execute against the + // wrong state. + await expect( + engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }) + ).rejects.toThrow(/Snapshot changed/); + + // The run is still readable and still has a coherent state machine. + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + + // The gap was handed to the repair job, which is the compensator the protocol names. + expect(harness.repairs.length).toBeGreaterThanOrEqual(1); + expect(harness.repairs.some((r) => r.runId === run.id)).toBe(true); + + // And no attempt was burned by the rejection itself: the bound is per crash, and the + // rejected call never reached the attempt bump. + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.attemptNumber ?? 0).toBeLessThanOrEqual(faults.fired("afterPgBeforeRedis")); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "two crashes cost at most two attempts, and the divergence does not amplify", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + // dual-write, so reads still come from Postgres and the run can be driven forward through the + // normal API. That isolates the property under test — how many attempts two crashes cost — + // from the stale-read rejection the previous test covers. + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "dual-write", + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + faults.arm("afterPgBeforeRedis", { times: 2, runId: run.id }); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, + }); + + const crashes = faults.fired("afterPgBeforeRedis"); + expect(crashes).toBeGreaterThanOrEqual(1); + + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + + // pgAttempt - maxLoggedAttempt <= crashCount. Each crash costs at most one attempt, and the + // divergence does not amplify: two crashes never cost three. A flat bound of one was + // refuted by the model check, so the assertion is against the crash count, not a constant. + expect((pgRun.attemptNumber ?? 0) - 1).toBeLessThanOrEqual(crashes); + + // Postgres holds every snapshot at this dial position, so the run still converges. + expect(pgRun.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(harness.repairs.length).toBe(crashes); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts index 097f92a5cd0..17c186d074d 100644 --- a/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts +++ b/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts @@ -87,7 +87,12 @@ describe("snapshot store read gate", () => { await engine.completeRunAttempt({ runId: run.id, snapshotId: attempt.snapshot.id, - completion: { ok: true, id: run.id, output: `{"done":true}`, outputType: "application/json" }, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, }); const finished = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); @@ -186,9 +191,7 @@ describe("snapshot store read gate", () => { .catch((error: unknown) => ({ threw: (error as Error).constructor.name })); const postgresOnly = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); - const engineOff = new RunEngine( - engineOptions(prisma, redisOptions, postgresOnly) as never - ); + const engineOff = new RunEngine(engineOptions(prisma, redisOptions, postgresOnly) as never); let viaPostgres: unknown; try { viaPostgres = await engineOff @@ -260,39 +263,42 @@ describe("snapshot store read gate", () => { } ); - containerTest("falls back to Postgres for a pre-cutover run", async ({ prisma, redisOptions }) => { - // A run created while the dial was off has no keyspace. Turning reads on must not lose it. - const off = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); - const engineOff = new RunEngine(engineOptions(prisma, redisOptions, off) as never); - - let runId: string; - let environment: any; - try { - environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - await setupBackgroundWorker(engineOff, environment, "gate-task"); - const run = await engineOff.trigger(triggerArgs("gate-task", environment, 5), prisma); - runId = run.id; - await setTimeout(500); - } finally { - await engineOff.quit(); - await off.quit(); - } + containerTest( + "falls back to Postgres for a pre-cutover run", + async ({ prisma, redisOptions }) => { + // A run created while the dial was off has no keyspace. Turning reads on must not lose it. + const off = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); + const engineOff = new RunEngine(engineOptions(prisma, redisOptions, off) as never); - const on = buildDecoratedStore({ - prisma, - redisOptions, - mode: "redis-read", - readPercent: 100, - }); - const engineOn = new RunEngine(engineOptions(prisma, redisOptions, on) as never); - try { - const data = await engineOn.getRunExecutionData({ runId }); - assertNonNullable(data); - expect(data.snapshot.executionStatus).toBe("QUEUED"); - expect(on.reads.some((r) => r.source === "postgres")).toBe(true); - } finally { - await engineOn.quit(); - await on.quit(); + let runId: string; + let environment: any; + try { + environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engineOff, environment, "gate-task"); + const run = await engineOff.trigger(triggerArgs("gate-task", environment, 5), prisma); + runId = run.id; + await setTimeout(500); + } finally { + await engineOff.quit(); + await off.quit(); + } + + const on = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engineOn = new RunEngine(engineOptions(prisma, redisOptions, on) as never); + try { + const data = await engineOn.getRunExecutionData({ runId }); + assertNonNullable(data); + expect(data.snapshot.executionStatus).toBe("QUEUED"); + expect(on.reads.some((r) => r.source === "postgres")).toBe(true); + } finally { + await engineOn.quit(); + await on.quit(); + } } - }); + ); }); diff --git a/internal-packages/run-store/scripts/generateDelegatingRunStore.ts b/internal-packages/run-store/scripts/generateDelegatingRunStore.ts index df901f3a2f7..c827e10c753 100644 --- a/internal-packages/run-store/scripts/generateDelegatingRunStore.ts +++ b/internal-packages/run-store/scripts/generateDelegatingRunStore.ts @@ -176,10 +176,14 @@ export class DelegatingRunStore implements RunStore { ${readonlyProperties // Indexed access rather than the written type, so the getter needs no import of its own and // follows the interface if that type is ever changed. - .map((p) => ` get ${p.name}(): RunStore["${p.name}"] {\n return this.delegate.${p.name};\n }`) + .map( + (p) => ` get ${p.name}(): RunStore["${p.name}"] {\n return this.delegate.${p.name};\n }` + ) .join("\n\n")}${readonlyProperties.length > 0 ? "\n\n" : ""}${unique - .map((n) => ` ${n}(...args: any[]): any {\n return (this.delegate as any).${n}(...args);\n }`) - .join("\n\n")} + .map( + (n) => ` ${n}(...args: any[]): any {\n return (this.delegate as any).${n}(...args);\n }` + ) + .join("\n\n")} } ` ); diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts index 81d3d3e658e..dda0f483bf5 100644 --- a/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts @@ -150,7 +150,9 @@ describe("PostgresRunStore snapshotWrites flag", () => { { select: { id: true } } ); - expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe("EXPIRED"); + expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe( + "EXPIRED" + ); expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); }); @@ -226,7 +228,9 @@ describe("PostgresRunStore snapshotWrites flag", () => { }, }); - expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe("DEQUEUED"); + expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe( + "DEQUEUED" + ); expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); }); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index c88c7464508..0f00f5848fe 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -744,6 +744,8 @@ export class PostgresRunStore implements RunStore { const snapshotCreate = { id: params.snapshot.id, + createdAt: params.snapshot.createdAt, + updatedAt: params.snapshot.createdAt, engine: params.snapshot.engine, executionStatus: params.snapshot.executionStatus, description: params.snapshot.description, @@ -831,6 +833,8 @@ export class PostgresRunStore implements RunStore { const snapshotCreate = { id: params.snapshot.id, + createdAt: params.snapshot.createdAt, + updatedAt: params.snapshot.createdAt, engine: params.snapshot.engine, executionStatus: params.snapshot.executionStatus, description: params.snapshot.description, @@ -942,6 +946,8 @@ export class PostgresRunStore implements RunStore { costInCents: data.costInCents, ...this.#nestedSnapshot({ id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, executionStatus: data.snapshot.executionStatus, description: data.snapshot.description, runStatus: data.snapshot.runStatus, @@ -1147,6 +1153,8 @@ export class PostgresRunStore implements RunStore { error: data.error as Prisma.InputJsonValue, ...this.#nestedSnapshot({ id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, engine: data.snapshot.engine, executionStatus: data.snapshot.executionStatus, description: data.snapshot.description, @@ -1277,6 +1285,8 @@ export class PostgresRunStore implements RunStore { maxAttempts: data.maxAttempts ?? undefined, ...this.#nestedSnapshot({ id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, engine: "V2", executionStatus: "PENDING_EXECUTING", description: "Run was dequeued for execution", @@ -1382,6 +1392,8 @@ export class PostgresRunStore implements RunStore { error: data.error as Prisma.InputJsonValue, ...this.#nestedSnapshot({ id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, engine: data.snapshot.engine, executionStatus: data.snapshot.executionStatus, description: data.snapshot.description, @@ -1454,6 +1466,8 @@ export class PostgresRunStore implements RunStore { ...(data.snapshot && this.#nestedSnapshot({ id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, engine: "V2", executionStatus: data.snapshot.executionStatus ?? "DELAYED", description: @@ -1984,6 +1998,7 @@ export class PostgresRunStore implements RunStore { ): Promise> { const { id, + createdAt, run, snapshot, previousSnapshotId, @@ -2014,7 +2029,7 @@ export class PostgresRunStore implements RunStore { ); } - const now = new Date(); + const now = createdAt ?? new Date(); return { id, engine: "V2", @@ -2049,6 +2064,8 @@ export class PostgresRunStore implements RunStore { const newSnapshot = await prisma.taskRunExecutionSnapshot.create({ data: { id, + createdAt, + updatedAt: createdAt, engine: "V2", executionStatus: snapshot.executionStatus, description: snapshot.description, diff --git a/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts b/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts index 7e61e619b78..f1394be597e 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts @@ -24,8 +24,7 @@ function entry(runId: string, id: string, createdAt: string): SnapshotEntryInput }; } -const at = (seconds: number) => - new Date(Date.UTC(2026, 0, 1, 0, 0, seconds)).toISOString(); +const at = (seconds: number) => new Date(Date.UTC(2026, 0, 1, 0, 0, seconds)).toISOString(); async function seed( store: RedisSnapshotStore, @@ -42,26 +41,29 @@ async function seed( } describe("getSinceCreatedAt", () => { - redisTest("returns only entries newer than the cursor, oldest first", async ({ redisOptions }) => { - const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); - try { - const runId = "run_window"; - await seed( - store, - runId, - [0, 1, 2, 3, 4].map((n) => ({ id: `snap_${n}`, createdAt: at(n) })) - ); - - const result = await store.getSinceCreatedAt(runId, at(1)); - - expect(result.kind).toBe("hit"); - if (result.kind !== "hit") return; - // Ascending, matching what the engine hands its caller after its own reverse(). - expect(result.entries.map((e) => e.id)).toEqual(["snap_2", "snap_3", "snap_4"]); - } finally { - await store.quit(); + redisTest( + "returns only entries newer than the cursor, oldest first", + async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_window"; + await seed( + store, + runId, + [0, 1, 2, 3, 4].map((n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const result = await store.getSinceCreatedAt(runId, at(1)); + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + // Ascending, matching what the engine hands its caller after its own reverse(). + expect(result.entries.map((e) => e.id)).toEqual(["snap_2", "snap_3", "snap_4"]); + } finally { + await store.quit(); + } } - }); + ); redisTest("misses when the run has no keyspace", async ({ redisOptions }) => { const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts index 989c7dc37bb..d3dd0868676 100644 --- a/internal-packages/run-store/src/runStoreMethodNames.ts +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -78,6 +78,4 @@ export const RUN_STORE_METHOD_NAMES = [ ] as const; // Data properties the base exposes as getters over the delegate, not as forwarders. -export const RUN_STORE_PROPERTY_NAMES = [ - "primaryReadClient", -] as const; +export const RUN_STORE_PROPERTY_NAMES = ["primaryReadClient"] as const; diff --git a/internal-packages/run-store/src/snapshotEntry.parity.test.ts b/internal-packages/run-store/src/snapshotEntry.parity.test.ts index 8c176178e0a..8e0a11fafbe 100644 --- a/internal-packages/run-store/src/snapshotEntry.parity.test.ts +++ b/internal-packages/run-store/src/snapshotEntry.parity.test.ts @@ -23,6 +23,13 @@ import { } from "./testFixtures/snapshotIdFixture.js"; /** + * NOTE ON createdAt. An earlier version of this suite built the expected entry with + * `createdAt: row.createdAt` and then asserted the two matched, which is tautological and hid a + * real divergence: seven of the eight write sites stamped the entry from the app clock while + * Postgres stamped its own column default, so the stores held different instants. The builders are + * now given an INDEPENDENT instant, and the row must carry that same value because the decorator + * passes it through to Postgres. + * * Compares only what the entry claims. The Redis model carries no `updatedAt` and no join rows, and * it holds `createdAt` as an ISO string, so those are checked separately or not at all. */ @@ -45,6 +52,8 @@ function assertParity(entry: SnapshotEntryInput, row: Record) { expect(row.runnerId ?? undefined).toBe(entry.runnerId ?? undefined); expect(row.isValid).toBe(entry.error === undefined); expect((row.createdAt as Date).toISOString()).toBe(entry.createdAt); + // Write-once rows: both columns hold the one instant, so a Prisma-stamped updatedAt would drift. + expect((row.updatedAt as Date).toISOString()).toBe(entry.createdAt); } function birthSnapshot(id: string, env: SnapshotFixtureEnv) { @@ -172,7 +181,10 @@ describe("entry to Postgres row parity", () => { ); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromCompletion({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + assertParity( + entryFromCompletion({ id, runId: run.id, createdAt: row.createdAt }, snapshot), + row + ); }); postgresTest("expireRun", async ({ prisma }) => { @@ -253,7 +265,10 @@ describe("entry to Postgres row parity", () => { }); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromReschedule({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + assertParity( + entryFromReschedule({ id, runId: run.id, createdAt: row.createdAt }, snapshot), + row + ); }); postgresTest("rescheduleRun with every value supplied", async ({ prisma }) => { @@ -277,7 +292,10 @@ describe("entry to Postgres row parity", () => { }); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromReschedule({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + assertParity( + entryFromReschedule({ id, runId: run.id, createdAt: row.createdAt }, snapshot), + row + ); }); postgresTest("lockRunToWorker", async ({ prisma }) => { @@ -395,3 +413,88 @@ describe("entry to Postgres row parity", () => { ); }); }); + +// The clock-provenance guard. Independent of the builders above: it asserts that what Postgres +// stores is the instant the CALLER supplied, not one the database chose. Without this, a snapshot +// has two different creation times depending on which store answers, the compared field can never +// reach zero divergence, and the since-window cursor resolved from one store misfilters the window +// walked in the other. +describe("createdAt provenance", () => { + postgresTest("Postgres stores the caller's instant, not its own", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + // Far enough from now that a database default could never coincide with it. + const stamp = new Date(Date.now() - 5 * 60 * 1000); + + await store.createExecutionSnapshot({ + id, + createdAt: stamp, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.toISOString()).toBe(stamp.toISOString()); + expect(row.updatedAt.toISOString()).toBe(stamp.toISOString()); + }); + + postgresTest("an absent instant still takes the database default", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const before = new Date(Date.now() - 1000); + + // Mode off supplies nothing, so Postgres must behave exactly as it always has. This is what + // keeps the merge test true. + await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.getTime()).toBeGreaterThan(before.getTime()); + }); + + postgresTest("a nested write site stores the caller's instant too", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const stamp = new Date(Date.now() - 5 * 60 * 1000); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + id, + createdAt: stamp, + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.toISOString()).toBe(stamp.toISOString()); + expect(row.updatedAt.toISOString()).toBe(stamp.toISOString()); + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts index 68a9d99cb6d..81e4ea729ef 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts @@ -50,7 +50,11 @@ describe("SnapshotOrphanSweeper", () => { const env = await seedSnapshotEnvironment(prisma); const runId = generateInternalId(); // Non-terminal append, so no expiry is ever set — the lost-TTL-set case. - await store.append({ entry: birthEntry(runId, env, new Date()), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(runId, env, new Date()), + kind: "birth", + isTerminal: false, + }); await prisma.taskRun.create({ data: { ...buildCreateRunData(runId, env), status: "COMPLETED_SUCCESSFULLY" }, }); @@ -130,7 +134,11 @@ describe("SnapshotOrphanSweeper", () => { const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); // The crashed birth: an entry, no Postgres run, non-terminal so no expiry. - await store.append({ entry: birthEntry(runId, env, old), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); await store.append({ entry: birthEntry(runId, env, old), kind: "transition", @@ -169,7 +177,11 @@ describe("SnapshotOrphanSweeper", () => { const env = await seedSnapshotEnvironment(prisma); const runId = generateInternalId(); // Written just now: the Postgres insert of a healthy birth may still be in flight. - await store.append({ entry: birthEntry(runId, env, new Date()), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(runId, env, new Date()), + kind: "birth", + isTerminal: false, + }); const result = await sweeper.sweep(); @@ -279,7 +291,11 @@ describe("SnapshotOrphanSweeper", () => { const env = await seedSnapshotEnvironment(prisma); const runId = generateInternalId(); const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); - await store.append({ entry: birthEntry(runId, env, old), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); // A failed lookup says nothing about whether the run exists, and rule 2 deletes a whole // keyspace. The sweep must resolve rather than throw, and must reap nothing. @@ -309,7 +325,11 @@ describe("SnapshotOrphanSweeper", () => { const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); const orphans = Array.from({ length: 5 }, () => generateInternalId()); for (const runId of orphans) { - await store.append({ entry: birthEntry(runId, env, old), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); } const result = await sweeper.sweep({ batchSize: 2 }); diff --git a/internal-packages/run-store/src/snapshotReadShapes.test.ts b/internal-packages/run-store/src/snapshotReadShapes.test.ts index afa33dd3da2..258731a60b5 100644 --- a/internal-packages/run-store/src/snapshotReadShapes.test.ts +++ b/internal-packages/run-store/src/snapshotReadShapes.test.ts @@ -67,7 +67,9 @@ describe("matchSinceCursorLookup", () => { }); it("refuses an unknown top-level key", () => { - expect(matchSinceCursorLookup({ ...cursorArgs, orderBy: { createdAt: "desc" } })).toBeUndefined(); + expect( + matchSinceCursorLookup({ ...cursorArgs, orderBy: { createdAt: "desc" } }) + ).toBeUndefined(); }); it("refuses anything that is not an argument object", () => { @@ -108,9 +110,7 @@ describe("matchSinceWindow", () => { }); it("refuses ascending order", () => { - expect( - matchSinceWindow({ ...windowArgs, orderBy: { createdAt: "asc" } }) - ).toBeUndefined(); + expect(matchSinceWindow({ ...windowArgs, orderBy: { createdAt: "asc" } })).toBeUndefined(); }); it("refuses a window that does not filter to valid entries", () => { diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts index 1510773e1ce..42924e34c34 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts @@ -101,32 +101,29 @@ describe("snapshot reads", () => { } }); - containerTest( - "returns the same payload Postgres would", - async ({ prisma, redisOptions }) => { - const { decorated, redis } = build(prisma as never, redisOptions as never); - const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); - try { - const env = await seedSnapshotEnvironment(prisma); - const runId = await seedRun(decorated, env); - await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); - - const fromRedis = await decorated.findLatestExecutionSnapshot(runId); - const fromPostgres = await postgresOnly.findLatestExecutionSnapshot(runId); - - expect(fromRedis!.id).toBe(fromPostgres!.id); - expect(fromRedis!.executionStatus).toBe(fromPostgres!.executionStatus); - expect(fromRedis!.description).toBe(fromPostgres!.description); - expect(fromRedis!.runStatus).toBe(fromPostgres!.runStatus); - expect(fromRedis!.attemptNumber).toBe(fromPostgres!.attemptNumber); - expect(fromRedis!.isValid).toBe(fromPostgres!.isValid); - expect(fromRedis!.environmentId).toBe(fromPostgres!.environmentId); - expect(fromRedis!.createdAt.toISOString()).toBe(fromPostgres!.createdAt.toISOString()); - } finally { - await redis.quit(); - } + containerTest("returns the same payload Postgres would", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const fromRedis = await decorated.findLatestExecutionSnapshot(runId); + const fromPostgres = await postgresOnly.findLatestExecutionSnapshot(runId); + + expect(fromRedis!.id).toBe(fromPostgres!.id); + expect(fromRedis!.executionStatus).toBe(fromPostgres!.executionStatus); + expect(fromRedis!.description).toBe(fromPostgres!.description); + expect(fromRedis!.runStatus).toBe(fromPostgres!.runStatus); + expect(fromRedis!.attemptNumber).toBe(fromPostgres!.attemptNumber); + expect(fromRedis!.isValid).toBe(fromPostgres!.isValid); + expect(fromRedis!.environmentId).toBe(fromPostgres!.environmentId); + expect(fromRedis!.createdAt.toISOString()).toBe(fromPostgres!.createdAt.toISOString()); + } finally { + await redis.quit(); } - ); + }); containerTest( "reads a foreign environment as not found, so the caller's 404 still fires", diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts index 311b46cc780..e6468c4f09f 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts @@ -181,9 +181,7 @@ describe("transition write ordering", () => { }); expect(await redis.getById(runId, row.id)).toBeNull(); - expect(repairs).toEqual([ - { runId, snapshotId: row.id, executionStatus: "FINISHED" }, - ]); + expect(repairs).toEqual([{ runId, snapshotId: row.id, executionStatus: "FINISHED" }]); } finally { await redis.quit(); } @@ -409,31 +407,34 @@ describe("transition write ordering", () => { } ); - containerTest("appends for the standalone createExecutionSnapshot", async ({ prisma, redisOptions }) => { - const { decorated, redis } = harness(prisma as never, redisOptions as never); - try { - const env = await seedSnapshotEnvironment(prisma); - const runId = generateInternalId(); - await seedBirth(decorated, redis, runId, env); + containerTest( + "appends for the standalone createExecutionSnapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); - const created = await decorated.createExecutionSnapshot({ - run: { id: runId, status: "EXECUTING", attemptNumber: 1 }, - snapshot: { executionStatus: "EXECUTING", description: "Run started" }, - environmentId: env.id, - environmentType: env.type, - projectId: env.projectId, - organizationId: env.organizationId, - }); + const created = await decorated.createExecutionSnapshot({ + run: { id: runId, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); - const read = await redis.getById(runId, created.id); - expect(read).not.toBeNull(); - expect(read!.entry.executionStatus).toBe("EXECUTING"); - // The standalone path is the one whose delegate returns the row, so both stores agree exactly. - expect(read!.entry.createdAt).toBe(created.createdAt.toISOString()); - } finally { - await redis.quit(); + const read = await redis.getById(runId, created.id); + expect(read).not.toBeNull(); + expect(read!.entry.executionStatus).toBe("EXECUTING"); + // The standalone path is the one whose delegate returns the row, so both stores agree exactly. + expect(read!.entry.createdAt).toBe(created.createdAt.toISOString()); + } finally { + await redis.quit(); + } } - }); + ); containerTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { const { decorated, redis } = harness(prisma as never, redisOptions as never, { mode: "off" }); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index 30655d727db..46a57c6b3bf 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -14,7 +14,13 @@ import { Logger } from "@trigger.dev/core/logger"; import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; import { DelegatingRunStore } from "./delegatingRunStore.js"; -import type { RedisSnapshotStore, SnapshotEntryInput, SnapshotRead } from "./redisSnapshotStore.js"; +import type { + CompletedWaitpointRef, + RedisSnapshotStore, + SnapshotEntryInput, + SnapshotRead, +} from "./redisSnapshotStore.js"; +import { deriveOrder } from "./redisSnapshotStore.js"; import { entryFromCompletion, entryFromCreateExecutionSnapshot, @@ -37,15 +43,19 @@ import type { RunStore, TaskRunWithWaitpoint, } from "./types.js"; -import { - matchSinceCursorLookup, - matchSinceWindow, -} from "./snapshotReadShapes.js"; +import { matchSinceCursorLookup, matchSinceWindow } from "./snapshotReadShapes.js"; +import { boundedIn } from "@trigger.dev/database"; import type { Prisma, PrismaClientOrTransaction, TaskRun } from "@trigger.dev/database"; /** One initial attempt plus three retries, per the write protocol. */ const APPEND_ATTEMPTS = 4; +/** + * Matches the engine's own chunked waitpoint fetch. A batch can complete a thousand waitpoints at + * once, and an unbounded `in:` makes each distinct list length its own prepared statement. + */ +const WAITPOINT_CHUNK_SIZE = 100; + /** * The rollout dial. Postgres stays fully written and authoritative in every position before * `redis-only`, so every earlier position rolls back losslessly by turning the dial down. @@ -88,7 +98,13 @@ export type TaskRunExecutionSnapshotStoreOptions = { * an intercepted write does its Postgres half and pushes its entry here instead of appending, and * the outer instance flushes the buffer after the transaction commits. */ - staging?: SnapshotEntryInput[]; + staging?: StagedAppend[]; +}; + +/** One deferred append: the entry, plus the wait cycle it carries, if any. */ +export type StagedAppend = { + entry: SnapshotEntryInput; + completedWaitpoints?: CompletedWaitpointRef[]; }; export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { @@ -99,7 +115,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { protected readonly faults?: SnapshotFaultInjector; protected readonly metrics?: DecoratorMetrics; protected readonly logger: Logger; - protected readonly staging?: SnapshotEntryInput[]; + protected readonly staging?: StagedAppend[]; constructor(delegate: RunStore, options: TaskRunExecutionSnapshotStoreOptions) { super(delegate); @@ -140,15 +156,20 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { return this.delegate.runInTransaction(runId, fn); } - const staged: SnapshotEntryInput[] = []; + const staged: StagedAppend[] = []; const result = await this.delegate.runInTransaction(runId, (store, tx) => fn(this.#wrap(store, staged), tx) ); // The transaction committed. Only now can a snapshot claim its partner is durable. - for (const entry of staged) { - await this.#appendTransition("runInTransaction", entry); + for (const item of staged) { + await this.#appendTransition( + "runInTransaction", + item.entry, + undefined, + item.completedWaitpoints + ); } return result; @@ -177,7 +198,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { * the write-ordering logic in exactly one place. Passing no buffer gives a plain decorator that * appends immediately; passing one makes it stage instead. */ - #wrap(store: RunStore, staging?: SnapshotEntryInput[]): TaskRunExecutionSnapshotStore { + #wrap(store: RunStore, staging?: StagedAppend[]): TaskRunExecutionSnapshotStore { return new TaskRunExecutionSnapshotStore(store, { store: this.redis, mode: this.mode, @@ -203,7 +224,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } const ctx = this.#context(params.data.id, params.snapshot.id); - const snapshot = { ...params.snapshot, id: ctx.id }; + const snapshot = { ...params.snapshot, id: ctx.id, createdAt: ctx.createdAt }; await this.#appendBirth("createRun", entryFromCreateRun(ctx, snapshot)); @@ -219,7 +240,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } const ctx = this.#context(params.data.id, params.snapshot.id); - const snapshot = { ...params.snapshot, id: ctx.id }; + const snapshot = { ...params.snapshot, id: ctx.id, createdAt: ctx.createdAt }; await this.#appendBirth("createCancelledRun", entryFromCreateRun(ctx, snapshot)); @@ -248,7 +269,10 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } const ctx = this.#context(runId, data.snapshot.id); - const withId = { ...data, snapshot: { ...data.snapshot, id: ctx.id } }; + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; const result = await this.delegate.completeAttemptSuccess(runId, withId, args, tx); @@ -270,7 +294,10 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } const ctx = this.#context(runId, data.snapshot.id); - const withId = { ...data, snapshot: { ...data.snapshot, id: ctx.id } }; + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; const result = await this.delegate.expireRun(runId, withId as never, args, tx); @@ -294,7 +321,10 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } const ctx = this.#context(runId, data.snapshot.id); - const withId = { ...data, snapshot: { ...data.snapshot, id: ctx.id } }; + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; const result = await this.delegate.expireParkedRun(runId, withId as never, tx); @@ -317,7 +347,10 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } const ctx = this.#context(runId, data.snapshot.id); - const withId = { ...data, snapshot: { ...data.snapshot, id: ctx.id } }; + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; const result = await this.delegate.rescheduleRun(runId, withId, tx); @@ -337,13 +370,17 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { // This is the one transition whose input already carries both an id and the previous snapshot // id, so it is also the one that can append under a compare-and-set on the current head. const ctx = { id: data.snapshot.id, runId, createdAt: new Date() }; + const withStamp = { ...data, snapshot: { ...data.snapshot, createdAt: ctx.createdAt } }; - const result = await this.delegate.lockRunToWorker(runId, data, tx); + const result = await this.delegate.lockRunToWorker(runId, withStamp, tx); await this.#appendTransition( "lockRunToWorker", - entryFromLock(ctx, data.snapshot), - data.snapshot.previousSnapshotId + entryFromLock(ctx, withStamp.snapshot), + withStamp.snapshot.previousSnapshotId, + // The lock site already carries the resolved order, so the refs are rebuilt from it rather + // than re-derived: its index IS the position in that list. + withStamp.snapshot.completedWaitpointOrder.map((id, index) => ({ id, index })) ); return result; } @@ -357,14 +394,18 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } const ctx = this.#context(input.run.id, input.id); - const created = await this.delegate.createExecutionSnapshot({ ...input, id: ctx.id }, tx); + const created = await this.delegate.createExecutionSnapshot( + { ...input, id: ctx.id, createdAt: ctx.createdAt }, + tx + ); // The standalone path is the only one whose delegate returns the row, so its entry can take the // exact createdAt Postgres recorded rather than the decorator's own clock. await this.#appendTransition( "createExecutionSnapshot", - entryFromCreateExecutionSnapshot({ ...ctx, createdAt: created.createdAt }, input), - input.previousSnapshotId + entryFromCreateExecutionSnapshot(ctx, input), + input.previousSnapshotId, + input.completedWaitpoints ); return created; } @@ -446,12 +487,13 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { async #appendTransition( site: string, entry: SnapshotEntryInput, - expectedCur?: string + expectedCur?: string, + completedWaitpoints?: CompletedWaitpointRef[] ): Promise { if (this.staging) { // Inside a transaction the append cannot run until the Postgres side commits, or a rollback // leaves Redis holding a transition that never happened. - this.staging.push(entry); + this.staging.push({ entry, ...(completedWaitpoints && { completedWaitpoints }) }); return; } @@ -462,11 +504,14 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { snapshotId: entry.id, }); + const cycle = await this.#resolveCycle(entry.runId, completedWaitpoints); + const result = await this.redis.append({ entry, kind: "transition", isTerminal: isTerminalEntry(entry), ...(expectedCur !== undefined && { expectedCur }), + ...(cycle && { cycle }), }); this.#recordOutcome(site, entry, result); @@ -496,6 +541,52 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } } + /** + * Decides whether this append mints a new wait cycle or points at the one already there. + * + * A resume append carries a newly-differing id set, so it mints a cycle and the record set is + * written once. Every copy-forward append that follows re-passes the SAME list, and re-minting on + * each would rewrite the record set once per entry in the resume chain — the write amplification + * the pointer model exists to remove. So an unchanged id set carries the previous cycleSeq + * forward and writes no key. + * + * The extra read only happens for an append that actually carries waitpoints, which is the resume + * path rather than the hot path. + * + * `records` is deliberately left unset. The record envelope belongs to the waitpoint lane and + * ships empty in this build, so dual-write never re-versions the entry when it arrives. + */ + async #resolveCycle( + runId: string, + completedWaitpoints?: CompletedWaitpointRef[] + ): Promise< + | { kind: "new"; completedWaitpoints: CompletedWaitpointRef[] } + | { kind: "carryForward"; cycleSeq: number } + | undefined + > { + if (!completedWaitpoints || completedWaitpoints.length === 0) { + return undefined; + } + + const order = deriveOrder(completedWaitpoints); + + try { + const head = await this.redis.getLatest(runId); + const previous = head?.completedWaitpointIds?.order; + + if (head?.cycle && previous && sameOrder(previous, order)) { + return { kind: "carryForward", cycleSeq: head.cycle.cycleSeq }; + } + } catch (error) { + // A failed probe must not lose the waitpoints. Minting a fresh cycle is the safe direction: + // it costs one duplicated record set, where a wrong carryForward would point at another + // cycle's ids. + this.logger.warn("snapshot cycle probe failed, minting a new cycle", { runId, error }); + } + + return { kind: "new", completedWaitpoints }; + } + /** * None of the four append outcomes is a failure, and none of them enqueues a repair. * @@ -564,7 +655,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } this.metrics?.recordRead("findLatestExecutionSnapshot", "redis"); - return this.#hydrate(read, runId, client); + return this.#hydrate(read, runId, client, { hydrateWaitpointRows: true }); } override async findExecutionSnapshot( @@ -592,9 +683,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload; } - override async findManyExecutionSnapshots< - T extends Prisma.TaskRunExecutionSnapshotFindManyArgs, - >( + override async findManyExecutionSnapshots( args: Prisma.SelectSubset, client?: ReadClient ): Promise[]> { @@ -617,8 +706,10 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { // The engine asks for createdAt DESC and reverses app-side; the store returns ascending. const descending = [...result.entries].reverse(); + // Rows are hydrated for no entry here: the engine fetches the head's waitpoints itself, from + // the ids this call's head row reports. Each row still carries its own order. const hydrated = await Promise.all( - descending.map((entry) => this.#hydrate(entry, shape.runId, client, { waitpoints: false })) + descending.map((entry) => this.#hydrate(entry, shape.runId, client)) ); return hydrated as unknown as Prisma.TaskRunExecutionSnapshotGetPayload[]; } @@ -675,7 +766,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { read: SnapshotRead, runId: string, client?: ReadClient, - opts?: { waitpoints?: boolean } + opts?: { hydrateWaitpointRows?: boolean } ): Promise< Prisma.TaskRunExecutionSnapshotGetPayload<{ include: { completedWaitpoints: true; checkpoint: true }; @@ -687,22 +778,18 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { ? await this.#hydrateCheckpoint(runId, read.id, client) : null; - let completedWaitpoints: unknown[] = []; - let completedWaitpointOrder: string[] = []; + // `completedWaitpointOrder` is a scalar column, NOT the join. The engine reads it off the head + // row as the index oracle that gives each completed waitpoint its position in a batch, so it + // must be populated even when the waitpoint ROWS are not fetched. Returning an empty order here + // resumes every batched triggerAndWait with `index: undefined`. + const ids = + read.completedWaitpointIds ?? (await this.redis.getSnapshotWaitpointIds(runId, read.id)); + const completedWaitpointOrder = ids.order; - if (opts?.waitpoints !== false) { - const ids = - read.completedWaitpointIds ?? (await this.redis.getSnapshotWaitpointIds(runId, read.id)); - completedWaitpointOrder = ids.order; - - if (ids.distinctIds.length > 0) { - completedWaitpoints = await this.delegate.findManyWaitpoints( - { where: { id: { in: ids.distinctIds } } }, - client, - runId - ); - } - } + // The rows themselves are head-only, mirroring the engine's own N x M avoidance. + const completedWaitpoints = opts?.hydrateWaitpointRows + ? await this.#fetchWaitpointsInChunks(ids.distinctIds, runId, client) + : []; return { id: read.id, @@ -726,6 +813,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { isValid: read.isValid, error: entry.error ?? null, createdAt: new Date(entry.createdAt as string), + // A snapshot row is write-once, so both columns hold the one instant the decorator minted. updatedAt: new Date(entry.createdAt as string), checkpoint, completedWaitpoints, @@ -734,6 +822,30 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { }>; } + /** + * Chunked, and bounded within each chunk, mirroring the engine's own waitpoint fetch. The run id + * routes each chunk to the owning store rather than fanning every one across both databases. + */ + async #fetchWaitpointsInChunks( + waitpointIds: string[], + runId: string, + client?: ReadClient + ): Promise { + if (waitpointIds.length === 0) return []; + + const all: unknown[] = []; + for (let i = 0; i < waitpointIds.length; i += WAITPOINT_CHUNK_SIZE) { + const chunk = waitpointIds.slice(i, i + WAITPOINT_CHUNK_SIZE); + const rows = await this.delegate.findManyWaitpoints( + { where: { id: { in: boundedIn(chunk) } } }, + client, + runId + ); + all.push(...rows); + } + return all; + } + /** * Reads the checkpoint row through the snapshot the delegate still holds, so the read stays * residency-aware: the run id in the where is what routes it to the owning database, and the @@ -772,3 +884,8 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } } } + +/** Position-sensitive: the same ids in a different order are a different wait cycle. */ +function sameOrder(a: string[], b: string[]): boolean { + return a.length === b.length && a.every((id, index) => id === b[index]); +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts new file mode 100644 index 00000000000..1879fe1246b --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts @@ -0,0 +1,363 @@ +// The completed-waitpoint path, which every other suite here was blind to. +// +// Two defects hid behind that blindness. The decorator passed no cycle to `append`, so no +// wp: key was ever written and the Redis waitpoint side was permanently empty. And the +// since-window hydration returned an empty `completedWaitpointOrder`, which is the index oracle the +// engine uses to give each completed waitpoint its position in a batch — an empty order resumes +// every batched triggerAndWait with `index: undefined`. +// +// So these tests all use a snapshot that ACTUALLY carries waitpoints. A test that does not cannot +// tell a working cycle from a missing one. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWaitpoints, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + mode: "dual-write" | "redis-read" = "redis-read" +) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const writes: { site: string; outcome: string }[] = []; + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode, + readPercent: 100, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: () => {}, + }, + } + ); + return { decorated, redis, writes }; +} + +async function seedRun( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + env: SnapshotFixtureEnv +): Promise { + const runId = generateInternalId(); + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); + return runId; +} + +function resumeInput( + runId: string, + env: SnapshotFixtureEnv, + completedWaitpoints: { id: string; index?: number }[], + description = "Run resumed" +) { + return { + id: generateInternalId(), + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + completedWaitpoints, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("completed-waitpoint cycles", () => { + containerTest("a resume append mints a cycle key", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + // The key exists at all — before the fix, none was ever written. + const cycleKeys = await probe.keys(`snap:{${runId}}:wp:*`); + expect(cycleKeys.length).toBe(1); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.present).toBe(true); + expect(ids.order).toEqual([wpA, wpB]); + expect(ids.distinctIds).toEqual([wpA, wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "a copy-forward reuses the cycle and writes no second key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const waitpoints = [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]; + + await decorated.createExecutionSnapshot(resumeInput(runId, env, waitpoints, "resume")); + // The same id set again: this is the copy-forward every dequeue and checkpoint site does. + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, waitpoints, "carry one") + ); + const third = await decorated.createExecutionSnapshot( + resumeInput(runId, env, waitpoints, "carry two") + ); + + // Still ONE key. Re-minting per entry is the write amplification the pointer model removes. + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(1); + + // And every entry still resolves the same order. + for (const id of [second.id, third.id]) { + expect((await redis.getSnapshotWaitpointIds(runId, id)).order).toEqual([wpA, wpB]); + } + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a newly-differing id set mints a second cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA, index: 0 }], "first wait") + ); + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpB, index: 0 }], "second wait") + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + expect((await redis.getSnapshotWaitpointIds(runId, second.id)).order).toEqual([wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the same ids in a different order are a new cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + // Order IS the index oracle, so a reordering is a different cycle, not a carry-forward. + const reordered = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpB, index: 0 }, + { id: wpA, index: 1 }, + ]) + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + expect((await redis.getSnapshotWaitpointIds(runId, reordered.id)).order).toEqual([ + wpB, + wpA, + ]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("a repeated id keeps both of its positions", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpX] = await seedSnapshotWaitpoints(prisma, env, 1); + + // One run batched twice under a single idempotency key: the id repeats, and each position + // must survive, because the runner matches results to positions. + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpX, index: 0 }, + { id: wpX, index: 1 }, + ]) + ); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.order).toEqual([wpX, wpX]); + expect(ids.distinctIds).toEqual([wpX]); + } finally { + await redis.quit(); + } + }); + + containerTest( + "findLatestExecutionSnapshot returns the index oracle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + // completedWaitpointOrder is a scalar column, not the join. Empty here means every batched + // waitpoint resumes with index undefined. + expect(latest!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "the since-window head carries the index oracle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const first = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [], "before the wait") + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + // The head is first in a descending window. This is the row the engine reads the oracle off. + expect(window[0]!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "lockRunToWorker carries its resolved order into the cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: undefined, + lockedToVersionId: undefined, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [wpA, wpB], + completedWaitpointOrder: [wpA, wpB], + }, + } as never); + + const ids = await redis.getSnapshotWaitpointIds(runId, snapshotId); + expect(ids.order).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "an append with no waitpoints writes no cycle key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + + await decorated.createExecutionSnapshot(resumeInput(runId, env, [], "no waitpoints")); + + expect(await probe.keys(`snap:{${runId}}:wp:*`)).toEqual([]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); diff --git a/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts b/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts index 17c0248312b..e0b8a1a2107 100644 --- a/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts +++ b/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts @@ -116,6 +116,38 @@ export async function seedSnapshotWorker( return { workerId: worker.id, taskId: task.id }; } +/** + * Seeds real Waitpoint rows and returns their ids. The legacy completed-waitpoint join carries a + * real foreign key, so an invented id fails the constraint rather than the assertion — the test + * then reports a fixture fault as if it were a defect in the code under test. + */ +export async function seedSnapshotWaitpoints( + prisma: PrismaClient, + env: SnapshotFixtureEnv, + count: number +): Promise { + const ids: string[] = []; + + for (let i = 0; i < count; i++) { + const suffix = generateInternalId().slice(-12); + const waitpoint = await prisma.waitpoint.create({ + data: { + friendlyId: `waitpoint_${suffix}`, + type: "MANUAL", + status: "COMPLETED", + completedAt: new Date(), + idempotencyKey: `idem_${suffix}`, + userProvidedIdempotencyKey: false, + projectId: env.projectId, + environmentId: env.id, + }, + }); + ids.push(waitpoint.id); + } + + return ids; +} + /** * Seeds an environment plus one run in `status`, with no execution snapshot. The suites that use it * assert on the snapshot rows a store method writes, so the run must start with none. diff --git a/internal-packages/run-store/src/types.ts b/internal-packages/run-store/src/types.ts index 5bde5950459..93fffe0ee88 100644 --- a/internal-packages/run-store/src/types.ts +++ b/internal-packages/run-store/src/types.ts @@ -29,6 +29,11 @@ export type IdempotencyKeyRunMatch = { }; export type CreateRunSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; id?: string; engine: "V2"; executionStatus: TaskRunExecutionStatus; @@ -43,6 +48,11 @@ export type CreateRunSnapshotInput = { }; export type CompletionSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ id?: string; @@ -67,6 +77,11 @@ export type PromotePendingVersionArgs = { }; export type ExpireSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ id?: string; @@ -81,6 +96,11 @@ export type ExpireSnapshotInput = { }; export type RescheduleSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ id?: string; @@ -94,6 +114,11 @@ export type RescheduleSnapshotInput = { }; export type LockSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; id: string; previousSnapshotId: string; attemptNumber?: number; @@ -301,6 +326,11 @@ export type TaskRunWithWaitpoint = TaskRun & { associatedWaitpoint: Waitpoint | * input — callers pass the high-level shape, not a raw Prisma `data`/`include`. */ export type CreateExecutionSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ id?: string; From ea0e17bdc3697271760ad6180009394900d37a3e Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 17:48:20 +0100 Subject: [PATCH 12/31] chore(run-store): treat the run-store scripts directory as an entry point The generator that emits the pass-through store base is a runnable script, not dead code, and the same glob covers any script added there later. --- knip.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/knip.json b/knip.json index 84456756ca1..8c32f5e72c3 100644 --- a/knip.json +++ b/knip.json @@ -37,6 +37,9 @@ "internal-packages/observability-map": { "entry": ["src/index.ts", "fixtures/**/*.{js,mjs,cjs,ts,mts,cts,tsx}"] }, + "internal-packages/run-store": { + "entry": ["scripts/**/*.{js,mjs,cjs,ts,mts,cts}"] + }, "internal-packages/otlp-importer": { "ignoreDependencies": ["ts-proto"] }, From 1d4eb7ba467e9dacad3b35e6bf42d1ea956c93f4 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 18:11:44 +0100 Subject: [PATCH 13/31] fix(run-store): address review on the sweep, and make the timestamp parity real The sweep discovered keyspaces by their cur key, which the append script writes only when an entry is valid. A keyspace whose entries all carry an error has no cur and no index, so neither sweep rule could ever see it and it leaked with no expiry, which is the same unbounded leak the second rule exists to close. It now scans on the entry hash, which every append writes, and the age probe falls back to the newest instant in that hash when the index is empty. Enumerating a run's cycle keys used KEYS. That command iterates the whole database and blocks while it does, and a hash tag routes a key without scoping the scan, so a sweep pass would have issued one full keyspace scan per run. It now reads the dense cycle high-water counter the append script maintains, which is the same source the store's own terminal-expiry loop uses, and pipelines the existence checks into one round trip. The timestamp parity assertion was still tautological. The previous commit added a note saying the builders receive an independent instant and did not change the builder calls, which kept reading the value off the row under test. Every case now mints one instant, passes it to the store, and gives the builder the same value, so a write site that stops forwarding the caller's instant fails here. Also documents what an injected fault actually does at each write path, since only the birth path rethrows, and scopes a run count in the chaos suite to the environment under test. --- .../engine/tests/snapshotStoreChaos.test.ts | 4 +- .../src/snapshotEntry.parity.test.ts | 58 +++++++++----- .../run-store/src/snapshotFaultInjection.ts | 12 ++- .../src/snapshotOrphanSweeper.test.ts | 41 ++++++++++ .../run-store/src/snapshotOrphanSweeper.ts | 79 +++++++++++++++++-- 5 files changed, 164 insertions(+), 30 deletions(-) diff --git a/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts index a727c9bcc1f..ea3447acc56 100644 --- a/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts +++ b/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts @@ -186,7 +186,9 @@ describe("snapshot store crash boundaries", () => { expect(faults.fired("afterRedisBirthBeforePg")).toBe(1); // The harmless state: no run row, so nothing can ever read a run that has no snapshot. - const runsAfterCrash = await prisma.taskRun.count(); + const runsAfterCrash = await prisma.taskRun.count({ + where: { runtimeEnvironmentId: environment.id }, + }); expect(runsAfterCrash).toBe(0); // A crashed birth must not poison the path: the next trigger runs to completion. diff --git a/internal-packages/run-store/src/snapshotEntry.parity.test.ts b/internal-packages/run-store/src/snapshotEntry.parity.test.ts index 8e0a11fafbe..635160fd831 100644 --- a/internal-packages/run-store/src/snapshotEntry.parity.test.ts +++ b/internal-packages/run-store/src/snapshotEntry.parity.test.ts @@ -24,11 +24,14 @@ import { /** * NOTE ON createdAt. An earlier version of this suite built the expected entry with - * `createdAt: row.createdAt` and then asserted the two matched, which is tautological and hid a - * real divergence: seven of the eight write sites stamped the entry from the app clock while - * Postgres stamped its own column default, so the stores held different instants. The builders are - * now given an INDEPENDENT instant, and the row must carry that same value because the decorator - * passes it through to Postgres. + * `createdAt: row.createdAt`, reading the value off the row it was checking and then asserting the + * two matched. That can never fail, and it hid a real divergence: seven of the eight write sites + * stamped the entry from the app clock while Postgres stamped its own column default, so the two + * stores held different instants for one snapshot. + * + * Every case now mints ONE instant, passes it to the store call, and gives the builder the same + * value. The row must carry it because the write site forwards it. A write site that stops + * forwarding the caller's instant fails here. * * Compares only what the entry claims. The Redis model carries no `updatedAt` and no join rows, and * it holds `createdAt` as an ISO string, so those are checked separately or not at all. @@ -56,9 +59,13 @@ function assertParity(entry: SnapshotEntryInput, row: Record) { expect((row.updatedAt as Date).toISOString()).toBe(entry.createdAt); } +/** Five minutes in the past, so a database default could never coincide with it. */ +const independentStamp = new Date(Date.now() - 5 * 60 * 1000); + function birthSnapshot(id: string, env: SnapshotFixtureEnv) { return { id, + createdAt: independentStamp, engine: "V2" as const, executionStatus: "RUN_CREATED" as const, description: "Run was created", @@ -81,7 +88,7 @@ describe("entry to Postgres row parity", () => { await store.createRun({ data: buildCreateRunData(runId, env), snapshot }); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromCreateRun({ id, runId, createdAt: row.createdAt }, snapshot), row); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); }); postgresTest("createRun with an associated waitpoint, legacy schema", async ({ prisma }) => { @@ -107,7 +114,7 @@ describe("entry to Postgres row parity", () => { }); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromCreateRun({ id, runId, createdAt: row.createdAt }, snapshot), row); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); }); postgresTest("createRun carries the worker and runner ids", async ({ prisma }) => { @@ -121,7 +128,7 @@ describe("entry to Postgres row parity", () => { await store.createRun({ data: buildCreateRunData(runId, env), snapshot }); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromCreateRun({ id, runId, createdAt: row.createdAt }, snapshot), row); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); }); postgresTest("createCancelledRun", async ({ prisma }) => { @@ -149,7 +156,7 @@ describe("entry to Postgres row parity", () => { }); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromCreateRun({ id, runId, createdAt: row.createdAt }, snapshot), row); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); }); postgresTest("completeAttemptSuccess", async ({ prisma }) => { @@ -158,6 +165,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const snapshot = { id, + createdAt: independentStamp, executionStatus: "FINISHED" as const, description: "Run completed", runStatus: "COMPLETED_SUCCESSFULLY" as const, @@ -182,7 +190,7 @@ describe("entry to Postgres row parity", () => { const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); assertParity( - entryFromCompletion({ id, runId: run.id, createdAt: row.createdAt }, snapshot), + entryFromCompletion({ id, runId: run.id, createdAt: independentStamp }, snapshot), row ); }); @@ -193,6 +201,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const snapshot = { id, + createdAt: independentStamp, engine: "V2" as const, executionStatus: "FINISHED" as const, description: "Run expired", @@ -215,7 +224,10 @@ describe("entry to Postgres row parity", () => { ); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromExpire({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + assertParity( + entryFromExpire({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); }); postgresTest("expireParkedRun", async ({ prisma }) => { @@ -224,6 +236,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const snapshot = { id, + createdAt: independentStamp, engine: "V2" as const, executionStatus: "FINISHED" as const, description: "Parked run expired", @@ -244,7 +257,10 @@ describe("entry to Postgres row parity", () => { expect(result.count).toBe(1); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromExpire({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + assertParity( + entryFromExpire({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); }); postgresTest("rescheduleRun with every default applied", async ({ prisma }) => { @@ -253,6 +269,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const snapshot = { id, + createdAt: independentStamp, environmentId: env.id, environmentType: env.type, projectId: env.projectId, @@ -266,7 +283,7 @@ describe("entry to Postgres row parity", () => { const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); assertParity( - entryFromReschedule({ id, runId: run.id, createdAt: row.createdAt }, snapshot), + entryFromReschedule({ id, runId: run.id, createdAt: independentStamp }, snapshot), row ); }); @@ -277,6 +294,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const snapshot = { id, + createdAt: independentStamp, environmentId: env.id, environmentType: env.type, projectId: env.projectId, @@ -293,7 +311,7 @@ describe("entry to Postgres row parity", () => { const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); assertParity( - entryFromReschedule({ id, runId: run.id, createdAt: row.createdAt }, snapshot), + entryFromReschedule({ id, runId: run.id, createdAt: independentStamp }, snapshot), row ); }); @@ -314,6 +332,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const snapshot = { id, + createdAt: independentStamp, previousSnapshotId: previous.id, attemptNumber: 1, environmentId: env.id, @@ -337,7 +356,7 @@ describe("entry to Postgres row parity", () => { }); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromLock({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + assertParity(entryFromLock({ id, runId: run.id, createdAt: independentStamp }, snapshot), row); }); postgresTest("createExecutionSnapshot, the standalone site", async ({ prisma }) => { @@ -346,6 +365,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const input = { id, + createdAt: independentStamp, run: { id: run.id, status: "EXECUTING" as const, attemptNumber: 2 }, snapshot: { executionStatus: "EXECUTING" as const, description: "Run started" }, environmentId: env.id, @@ -359,7 +379,7 @@ describe("entry to Postgres row parity", () => { const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); expect(created.id).toBe(id); assertParity( - entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: row.createdAt }, input), + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), row ); }); @@ -370,6 +390,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const input = { id, + createdAt: independentStamp, run: { id: run.id, status: "DEQUEUED" as const, attemptNumber: 1 }, snapshot: { executionStatus: "PENDING_EXECUTING" as const, description: "Run was dequeued" }, environmentId: env.id, @@ -383,7 +404,7 @@ describe("entry to Postgres row parity", () => { const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); expect(row.runStatus).toBe("PENDING"); assertParity( - entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: row.createdAt }, input), + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), row ); }); @@ -394,6 +415,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const input = { id, + createdAt: independentStamp, run: { id: run.id, status: "EXECUTING" as const, attemptNumber: 1 }, snapshot: { executionStatus: "EXECUTING" as const, description: "Stale write" }, error: "snapshot is not the latest", @@ -408,7 +430,7 @@ describe("entry to Postgres row parity", () => { const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); expect(row.isValid).toBe(false); assertParity( - entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: row.createdAt }, input), + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), row ); }); diff --git a/internal-packages/run-store/src/snapshotFaultInjection.ts b/internal-packages/run-store/src/snapshotFaultInjection.ts index 50029b9518d..83abf43b179 100644 --- a/internal-packages/run-store/src/snapshotFaultInjection.ts +++ b/internal-packages/run-store/src/snapshotFaultInjection.ts @@ -20,9 +20,15 @@ export type SnapshotFaultInjector = ( ) => void; /** - * Thrown by a test injector. The write path tells this apart from a real append failure: an injected - * fault models a process that died, so it is rethrown rather than retried, while a real failure is - * retried and then handed to the repair job. + * Thrown by a test injector. The write path tells this apart from a real append failure, because an + * injected fault models a process that died rather than a call that failed. The two write paths then + * do different things with it, and both differ from a real failure: + * + * - A transition skips its remaining retries, hands the run to the repair job, and does NOT rethrow. + * Postgres has already committed, so the caller must not see an error. + * - A birth rethrows, so the Postgres insert never runs and the crash leaves an orphaned keyspace + * with no run row, which is the harmless state that ordering exists to produce. + * - A real append failure is retried, and only then handed to the repair job. */ export class InjectedSnapshotFault extends Error { readonly boundary: SnapshotFaultBoundary; diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts index 81e4ea729ef..fc3bdcca39f 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts @@ -310,6 +310,47 @@ describe("SnapshotOrphanSweeper", () => { } ); + containerTest( + "discovers and reaps a keyspace whose entries are all invalid", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + // The append script writes `cur` and indexes the entry only when it is valid, so a keyspace + // whose entries all carry an error has neither. A sweep that discovers keyspaces by their + // `cur` key would never see this one, and neither rule would ever apply to it. + await store.append({ + entry: { ...birthEntry(runId, env, old), error: "stale write" }, + kind: "birth", + isTerminal: false, + }); + + const keys = snapshotKeys(runId); + expect(await probe.exists(keys.e)).toBe(1); + expect(await probe.exists(keys.cur)).toBe(0); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(1); + expect(await probe.exists(keys.e)).toBe(0); + expect(await probe.exists(keys.seq)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + containerTest("processes every keyspace across batches", async ({ prisma, redisOptions }) => { const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.ts index f94876da37c..6ce5c574c2f 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.ts @@ -108,10 +108,14 @@ export class SnapshotOrphanSweeper { let cursor = "0"; do { + // Match on the entry hash, not on `cur`. The append script writes `cur` only when the entry + // is valid, so a keyspace whose entries are all invalid would never be discovered and would + // leak with no expiry, which is the same unbounded leak rule 2 exists to close. `e` is + // written by every append. const [next, keys] = await this.#redis.scan( cursor, "MATCH", - `${this.#prefix}{*}:cur`, + `${this.#prefix}{*}:e`, "COUNT", batchSize ); @@ -212,15 +216,41 @@ export class SnapshotOrphanSweeper { result.deleted += 1; } - /** Every key for one run: the four core keys plus each wait-cycle key. */ + /** + * Every key for one run: the four core keys plus each wait-cycle key. + * + * The cycle keys are enumerated from the `c` high-water field on the seq hash, which the append + * script mints densely with HINCRBY, so 1..high covers every wp key that was ever written. This + * is the same source the store's own terminal-expiry loop uses. + * + * It deliberately does NOT use `KEYS`. That command iterates the whole database and blocks while + * it does, and a hash tag routes a key without scoping the scan, so one sweep pass over a batch + * would issue a full keyspace scan per run. + * + * The trade-off: if the seq hash is evicted while a wp key survives, `high` reads 0 and that + * orphaned cycle key is left behind. That is the right way to be wrong here. Leaving one small + * key costs bytes, where scanning the keyspace to find it costs every hot-path client latency on + * every pass. + */ async #allKeys(runId: string): Promise { const core = snapshotKeys(runId); - // Scoped to one hash tag, so this is a lookup inside a single slot rather than a keyspace scan. - const cycles = await this.#redis.keys(`${this.#prefix}{${runId}}:wp:*`); + + const high = Number((await this.#redis.hget(core.seq, "c")) ?? "0"); + const cycles: string[] = []; + for (let n = 1; n <= high; n++) { + cycles.push(`${this.#prefix}{${runId}}:wp:${n}`); + } + const candidates = [core.e, core.idx, core.cur, core.seq, ...cycles]; - const exists = await Promise.all(candidates.map((key) => this.#redis.exists(key))); - return candidates.filter((_key, index) => exists[index] === 1); + // One round trip for the whole set, rather than one per candidate. + const pipeline = this.#redis.pipeline(); + for (const key of candidates) { + pipeline.exists(key); + } + const replies = await pipeline.exec(); + + return candidates.filter((_key, index) => replies?.[index]?.[1] === 1); } /** @@ -229,11 +259,17 @@ export class SnapshotOrphanSweeper { */ async #newestEntryAgeMs(runId: string): Promise { const core = snapshotKeys(runId); + const newest = await this.#redis.zrevrange(core.idx, 0, 0); const id = newest[0]; - if (!id) return undefined; - const raw = await this.#redis.hget(core.e, id); + const raw = id + ? await this.#redis.hget(core.e, id) + : // The index holds valid entries only, so an all-invalid keyspace has an empty index. Fall + // back to the newest instant in the entry hash, or that keyspace is never old enough to + // reap and the leak survives the scan fix above. + await this.#newestRawFromEntries(core.e); + if (!raw) return undefined; try { @@ -246,6 +282,33 @@ export class SnapshotOrphanSweeper { } } + /** + * The newest entry document in the hash, by its own createdAt. Only reached for a keyspace with + * no index, which is rare, so the whole-hash read is acceptable where it would not be on the + * indexed path. + */ + async #newestRawFromEntries(eKey: string): Promise { + const all = await this.#redis.hgetall(eKey); + let newestRaw: string | undefined; + let newestAt = -Infinity; + + for (const [field, raw] of Object.entries(all)) { + // Sidecar fields hang off the entry ids as `#s` and `#c`; skip them. + if (field.includes("#")) continue; + try { + const at = Date.parse((JSON.parse(raw) as { createdAt?: string }).createdAt ?? ""); + if (!Number.isNaN(at) && at > newestAt) { + newestAt = at; + newestRaw = raw; + } + } catch { + continue; + } + } + + return newestRaw; + } + #runIdFrom(key: string): string | undefined { const open = key.indexOf("{"); const close = key.indexOf("}", open + 1); From 94b9c68325d63c0e72807e658e92b60dc26c61a3 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 09:26:17 +0100 Subject: [PATCH 14/31] test(run-store): prove interface parity in the compiler, not in a fake Review asked why the pass-through base is tested against a hand-built delegate rather than a real store. Checking what the compiler already guarantees showed the test's own stated reason was wrong, and that one of its cases could not fail. implements RunStore already rejects a missing member with TS2420, so the claim that a method added later would become a silent hole was not true. The case comparing the class against the generated name list could not detect a parse miss either, because both the class and the list come from one parse of the interface, so a miss drops the member from both sides. The generator's comment asserting otherwise was false. Parity now lives where it can actually fail: assertions tying the name lists to keyof RunStore in both directions, and one rejecting a public member the class declares and the interface does not. They sit in src rather than in a test, because the build config excludes test files, so a type assertion written in a test is never checked. Each was verified by making it fail. What the compiler cannot see is inside the forwarder bodies, since every one is typed (...args: any[]): any. A forwarder wired to the wrong member, or dropping an argument, typechecks cleanly. The remaining probe covers exactly that, using a per-member sentinel so a misrouted body returns the wrong value rather than merely returning something. Verified by rewiring a forwarder: typecheck passes, the probe fails and names the member. Renames the double to forwardingProbe across both suites and says at the top why a container cannot replace it: no database is involved in whether a pass-through passes through. --- .../scripts/generateDelegatingRunStore.ts | 63 +++++++++++- .../run-store/src/delegatingRunStore.test.ts | 95 +++++++++++-------- .../run-store/src/delegatingRunStore.ts | 14 +++ .../run-store/src/runStoreMethodNames.ts | 37 +++++++- .../taskRunExecutionSnapshotStore.off.test.ts | 30 +++--- 5 files changed, 182 insertions(+), 57 deletions(-) diff --git a/internal-packages/run-store/scripts/generateDelegatingRunStore.ts b/internal-packages/run-store/scripts/generateDelegatingRunStore.ts index c827e10c753..84a6ade0ce7 100644 --- a/internal-packages/run-store/scripts/generateDelegatingRunStore.ts +++ b/internal-packages/run-store/scripts/generateDelegatingRunStore.ts @@ -7,9 +7,18 @@ // pnpm exec tsx scripts/generateDelegatingRunStore.ts // // The interface is scanned directly rather than through the TypeScript compiler API, because -// `require("typescript")` resolves to a stub in this workspace. A parsing miss cannot pass silently: -// delegatingRunStore.test.ts asserts the class and the interface hold exactly the same member set, -// and `implements RunStore` fails typecheck if a method is absent. +// `require("typescript")` resolves to a stub in this workspace. +// +// A parsing miss cannot pass silently, but NOT because of the runtime suite: that compares the class +// against the name list, and both come from this one parse, so a miss drops the member from both +// sides and the comparison still holds. Two compile-time checks catch it instead: +// +// - `implements RunStore` on the generated class fails with TS2420 when a member is absent. +// - The assertions emitted into runStoreMethodNames.ts tie the name lists to `keyof RunStore`, +// in both directions, so a dropped or invented name fails typecheck. +// +// Those live in src rather than in a test because tsconfig.build.json excludes test files, so a +// type-level assertion written in a test is never checked by CI. import { readFileSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -145,8 +154,11 @@ const header = `// GENERATED by scripts/generateDelegatingRunStore.ts. Do not ed writeFileSync( join(root, "src/runStoreMethodNames.ts"), `${header} -// Every method the RunStore interface declares. The decorator suites enumerate this, so a method -// added to the interface and not to the base fails a test instead of becoming a silent hole. +import type { RunStore } from "./types.js"; + +// Every method the RunStore interface declares. The forwarding probe enumerates this to drive one +// call per member; member PRESENCE is proved by the compiler, in the assertions at the foot of this +// file and by \`implements RunStore\` on the generated class. export const RUN_STORE_METHOD_NAMES = [ ${unique.map((n) => ` "${n}",`).join("\n")} ] as const; @@ -155,6 +167,33 @@ ${unique.map((n) => ` "${n}",`).join("\n")} export const RUN_STORE_PROPERTY_NAMES = [ ${readonlyProperties.map((p) => ` "${p.name}",`).join("\n")} ] as const; + +// --------------------------------------------------------------------------- +// Parity with the interface, checked by the compiler. +// +// The lists above are produced by parsing types.ts. These assertions compare them +// against \`keyof RunStore\`, which the compiler derives from the interface itself, +// so a name this generator failed to parse, or invented, is a build failure rather +// than a silent gap. Both directions are checked: a missing name and an extra one. +// --------------------------------------------------------------------------- + +type RunStoreMemberName = + | (typeof RUN_STORE_METHOD_NAMES)[number] + | (typeof RUN_STORE_PROPERTY_NAMES)[number]; + +/** Fails when the interface declares a member the generator did not emit. */ +type _EveryInterfaceMemberIsListed = [Exclude] extends [never] + ? true + : never; +const _everyInterfaceMemberIsListed: _EveryInterfaceMemberIsListed = true; +void _everyInterfaceMemberIsListed; + +/** Fails when the generator emitted a name the interface does not declare. */ +type _EveryListedNameIsOnTheInterface = [Exclude] extends [never] + ? true + : never; +const _everyListedNameIsOnTheInterface: _EveryListedNameIsOnTheInterface = true; +void _everyListedNameIsOnTheInterface; ` ); @@ -185,6 +224,20 @@ ${readonlyProperties ) .join("\n\n")} } + +// \`implements\` above fails when a member of the interface is MISSING here. It says nothing about a +// member that should not exist, so the reverse direction is asserted too: a public member this class +// declares and the interface does not is a build failure. +// +// \`protected delegate\` is correctly absent from \`keyof\`, so the constructor parameter does not +// trip this. +type _ClassDeclaresNoExtraMembers = [ + Exclude, +] extends [never] + ? true + : never; +const _classParity: _ClassDeclaresNoExtraMembers = true; +void _classParity; ` ); diff --git a/internal-packages/run-store/src/delegatingRunStore.test.ts b/internal-packages/run-store/src/delegatingRunStore.test.ts index 7065f801d71..74d6a9e9afb 100644 --- a/internal-packages/run-store/src/delegatingRunStore.test.ts +++ b/internal-packages/run-store/src/delegatingRunStore.test.ts @@ -1,57 +1,88 @@ -// The base must forward EVERY RunStore member. A method added to the interface and not to the base -// is a silent hole in the decorator built on top of it, so this suite enumerates the generated name -// lists rather than restating them by hand. Regenerate the base and both lists together: -// pnpm exec tsx scripts/generateDelegatingRunStore.ts +// What this suite covers, and why it does not use a container. +// +// `DelegatingRunStore` is generated and holds no logic: every member forwards to a delegate. Three +// properties of it are already proved by the compiler and are NOT retested here: +// +// - a member of the interface missing from the class -> `implements RunStore`, TS2420 +// - a public member the interface does not declare -> the parity assertion in the base +// - a name list out of step with the interface -> the assertions in runStoreMethodNames +// +// What the compiler cannot see is inside the forwarder bodies, because each one is typed +// `(...args: any[]): any`. A forwarder wired to the wrong delegate method, dropping an argument, or +// reading a property once at construction instead of on each access, all typecheck cleanly. Those +// are template-correctness properties of the generator's output, and they are what is tested below. +// +// A Testcontainers-backed store cannot demonstrate them. It would mean calling all 70 methods with +// valid arguments and valid foreign-key state, and a real return value cannot show that arguments +// arrived untouched the way a per-member sentinel can. The probe below is not a stand-in for a +// database: no database is involved in whether a pass-through passes through. Behaviour against a +// real store is covered by the container suites for the decorator built on this base. import { describe, expect, it } from "vitest"; import { DelegatingRunStore } from "./delegatingRunStore.js"; import { RUN_STORE_METHOD_NAMES, RUN_STORE_PROPERTY_NAMES } from "./runStoreMethodNames.js"; import type { RunStore } from "./types.js"; -function recordingDelegate(): { store: RunStore; calls: { name: string; args: unknown[] }[] } { - const calls: { name: string; args: unknown[] }[] = []; - const store: Record = {}; +type ProbedCall = { name: string; args: unknown[] }; - for (const name of RUN_STORE_METHOD_NAMES) { - store[name] = (...args: unknown[]) => { - calls.push({ name, args }); - return `result:${name}`; - }; - } - for (const name of RUN_STORE_PROPERTY_NAMES) { - store[name] = `property:${name}`; - } +/** + * A delegate that records what was called on it and answers with a per-member sentinel, so a + * forwarder wired to the wrong member returns the wrong sentinel and fails loudly. + */ +function forwardingProbe(): { store: RunStore; calls: ProbedCall[] } { + const calls: ProbedCall[] = []; + + const store = new Proxy({} as Record, { + get(_target, prop: string) { + if ((RUN_STORE_PROPERTY_NAMES as readonly string[]).includes(prop)) { + return `property:${prop}`; + } + return (...args: unknown[]) => { + calls.push({ name: prop, args }); + return `result:${prop}`; + }; + }, + }); return { store: store as unknown as RunStore, calls }; } describe("DelegatingRunStore", () => { - it("forwards every RunStore method to the delegate, arguments untouched", () => { - const { store, calls } = recordingDelegate(); + it("forwards every method to the member of the same name", () => { + const { store, calls } = forwardingProbe(); const base = new DelegatingRunStore(store) as unknown as Record< string, (...args: unknown[]) => unknown >; for (const name of RUN_STORE_METHOD_NAMES) { - expect(base[name]("arg-one", "arg-two")).toBe(`result:${name}`); + // The sentinel is per member, so a body forwarding to a different method fails here rather + // than passing because both happened to return something. + expect(base[name]()).toBe(`result:${name}`); } expect(calls.map((c) => c.name)).toEqual([...RUN_STORE_METHOD_NAMES]); - for (const call of calls) { - expect(call.args).toEqual(["arg-one", "arg-two"]); - } }); - it("reads every RunStore data property from the delegate", () => { - const { store } = recordingDelegate(); - const base = new DelegatingRunStore(store) as unknown as Record; + it("forwards arguments untouched", () => { + const { store, calls } = forwardingProbe(); + const base = new DelegatingRunStore(store) as unknown as Record< + string, + (...args: unknown[]) => unknown + >; + const args = ["first", { second: true }, undefined, 4]; + + for (const name of RUN_STORE_METHOD_NAMES) { + base[name](...args); + } - for (const name of RUN_STORE_PROPERTY_NAMES) { - expect(base[name]).toBe(`property:${name}`); + for (const call of calls) { + expect(call.args).toEqual(args); } }); it("reads a data property live, so a delegate that changes is not cached", () => { + // A getter is the only correct shape here. Capturing the value in the constructor would + // typecheck and would then serve a stale client for the life of the decorator. const store = { primaryReadClient: "first" } as unknown as RunStore; const base = new DelegatingRunStore(store); @@ -59,14 +90,4 @@ describe("DelegatingRunStore", () => { (store as unknown as Record).primaryReadClient = "second"; expect(base.primaryReadClient).toBe("second" as unknown); }); - - it("declares exactly the members the interface declares, and no others", () => { - const own = Object.getOwnPropertyNames(DelegatingRunStore.prototype) - .filter((name) => name !== "constructor") - .sort(); - - const expected = [...RUN_STORE_METHOD_NAMES, ...RUN_STORE_PROPERTY_NAMES].sort(); - - expect(own).toEqual(expected); - }); }); diff --git a/internal-packages/run-store/src/delegatingRunStore.ts b/internal-packages/run-store/src/delegatingRunStore.ts index 6266ac5abd1..621242f75cf 100644 --- a/internal-packages/run-store/src/delegatingRunStore.ts +++ b/internal-packages/run-store/src/delegatingRunStore.ts @@ -298,3 +298,17 @@ export class DelegatingRunStore implements RunStore { return (this.delegate as any).findManyWaitpointTags(...args); } } + +// `implements` above fails when a member of the interface is MISSING here. It says nothing about a +// member that should not exist, so the reverse direction is asserted too: a public member this class +// declares and the interface does not is a build failure. +// +// `protected delegate` is correctly absent from `keyof`, so the constructor parameter does not +// trip this. +type _ClassDeclaresNoExtraMembers = [Exclude] extends [ + never, +] + ? true + : never; +const _classParity: _ClassDeclaresNoExtraMembers = true; +void _classParity; diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts index d3dd0868676..eff7589fcfe 100644 --- a/internal-packages/run-store/src/runStoreMethodNames.ts +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -2,8 +2,11 @@ // Regenerate after any change to the RunStore interface: // pnpm exec tsx scripts/generateDelegatingRunStore.ts -// Every method the RunStore interface declares. The decorator suites enumerate this, so a method -// added to the interface and not to the base fails a test instead of becoming a silent hole. +import type { RunStore } from "./types.js"; + +// Every method the RunStore interface declares. The forwarding probe enumerates this to drive one +// call per member; member PRESENCE is proved by the compiler, in the assertions at the foot of this +// file and by `implements RunStore` on the generated class. export const RUN_STORE_METHOD_NAMES = [ "runInTransaction", "createRun", @@ -22,7 +25,6 @@ export const RUN_STORE_METHOD_NAMES = [ "lockRunToWorker", "parkPendingVersion", "promotePendingVersionRuns", - "expireParkedRun", "suspendForCheckpoint", "resumeFromCheckpoint", "rescheduleRun", @@ -79,3 +81,32 @@ export const RUN_STORE_METHOD_NAMES = [ // Data properties the base exposes as getters over the delegate, not as forwarders. export const RUN_STORE_PROPERTY_NAMES = ["primaryReadClient"] as const; + +// --------------------------------------------------------------------------- +// Parity with the interface, checked by the compiler. +// +// The lists above are produced by parsing types.ts. These assertions compare them +// against `keyof RunStore`, which the compiler derives from the interface itself, +// so a name this generator failed to parse, or invented, is a build failure rather +// than a silent gap. Both directions are checked: a missing name and an extra one. +// --------------------------------------------------------------------------- + +type RunStoreMemberName = + | (typeof RUN_STORE_METHOD_NAMES)[number] + | (typeof RUN_STORE_PROPERTY_NAMES)[number]; + +/** Fails when the interface declares a member the generator did not emit. */ +type _EveryInterfaceMemberIsListed = [Exclude] extends [never] + ? true + : never; +const _everyInterfaceMemberIsListed: _EveryInterfaceMemberIsListed = true; +void _everyInterfaceMemberIsListed; + +/** Fails when the generator emitted a name the interface does not declare. */ +type _EveryListedNameIsOnTheInterface = [Exclude] extends [ + never, +] + ? true + : never; +const _everyListedNameIsOnTheInterface: _EveryListedNameIsOnTheInterface = true; +void _everyListedNameIsOnTheInterface; diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts index 975c68c371a..4f788b5ec27 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts @@ -17,23 +17,29 @@ function explodingRedisStore(): RedisSnapshotStore { }); } -function recordingDelegate(): { store: RunStore; calls: string[] } { +/** + * Records what the decorator forwarded, and answers with a per-member sentinel. No database is + * involved in whether mode off is a pass-through, so none is started; the behavioural suites for + * every other mode run against a real Postgres and a real Redis. + */ +function forwardingProbe(): { store: RunStore; calls: string[] } { const calls: string[] = []; - const store: Record = {}; - for (const name of RUN_STORE_METHOD_NAMES) { - store[name] = (...args: unknown[]) => { - calls.push(name); - return `result:${name}`; - }; - } + const store = new Proxy({} as Record, { + get(_target, prop: string) { + return (...args: unknown[]) => { + calls.push(prop); + return `result:${prop}`; + }; + }, + }); return { store: store as unknown as RunStore, calls }; } describe("TaskRunExecutionSnapshotStore at mode off", () => { it("defaults to mode off", () => { - const { store } = recordingDelegate(); + const { store } = forwardingProbe(); const decorated = new TaskRunExecutionSnapshotStore(store, { store: explodingRedisStore() }); @@ -41,7 +47,7 @@ describe("TaskRunExecutionSnapshotStore at mode off", () => { }); it("forwards every method to the delegate and never calls Redis", async () => { - const { store, calls } = recordingDelegate(); + const { store, calls } = forwardingProbe(); const decorated = new TaskRunExecutionSnapshotStore(store, { store: explodingRedisStore(), mode: "off", @@ -56,7 +62,7 @@ describe("TaskRunExecutionSnapshotStore at mode off", () => { }); it("hands the delegate's own store to a transaction callback", async () => { - const inner = recordingDelegate().store; + const inner = forwardingProbe().store; let seen: unknown; const delegate = { runInTransaction: async ( @@ -80,7 +86,7 @@ describe("TaskRunExecutionSnapshotStore at mode off", () => { }); it("reports every other dial position as one that writes Redis", () => { - const { store } = recordingDelegate(); + const { store } = forwardingProbe(); const modes = ["dual-write", "compare", "redis-read", "redis-only"] as const; for (const mode of modes) { From f01d299636fe3d81483a6e314bbeaf4ad72a44f6 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 09:27:20 +0100 Subject: [PATCH 15/31] fix(run-store): restore a name dropped from the generated list A member was removed from the generated list while verifying that the new parity assertion fails when one goes missing, and the restore did not run, so the verification state was committed. Regenerated from the interface. The assertion did its job: typecheck rejects the list, naming the missing member. --- internal-packages/run-store/src/runStoreMethodNames.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts index eff7589fcfe..ffb07a7de07 100644 --- a/internal-packages/run-store/src/runStoreMethodNames.ts +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -25,6 +25,7 @@ export const RUN_STORE_METHOD_NAMES = [ "lockRunToWorker", "parkPendingVersion", "promotePendingVersionRuns", + "expireParkedRun", "suspendForCheckpoint", "resumeFromCheckpoint", "rescheduleRun", From b02b42683f2815f5c309e774f93642a46ea9b3c6 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 11:34:31 +0100 Subject: [PATCH 16/31] fix(run-store): keep the fork guard on a staged append, and survive a client key prefix Two defects from review, both silent. An append staged inside a transaction dropped its expected-head argument, and the post-commit flush passed undefined in its place. That disabled the compare-and-set for every snapshot written inside a transaction, which is the path both engine transaction writers use, so a stale append that the store would have refused as forked was written instead and became the head. The expectation now travels with the staged entry. The sweep built its scan pattern without the client key prefix. ioredis prepends that prefix to keys for ordinary commands but not to a SCAN MATCH pattern, and returns matched keys with it still attached, so a prefixed client made the sweep match nothing and report a clean pass. The engine sets a prefix on every other Redis client it builds, so this would have surfaced at wiring time as a reaper that silently protected nothing. Also removes a keyPrefix option on the sweep that could never work: the keyspace prefix belongs to snapshotKeys in the store, which writes snap: keys unconditionally, so there was no other keyspace to point it at. Both fixes have a test verified by reintroducing the defect: the staged stale append is written without the guard, and the prefixed sweep scans nothing. --- .../src/snapshotOrphanSweeper.test.ts | 44 +++++++++++++++ .../run-store/src/snapshotOrphanSweeper.ts | 26 +++++++-- ...kRunExecutionSnapshotStore.staging.test.ts | 55 +++++++++++++++++++ .../src/taskRunExecutionSnapshotStore.ts | 14 ++++- 4 files changed, 132 insertions(+), 7 deletions(-) diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts index fc3bdcca39f..b0d39d27e56 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts @@ -351,6 +351,50 @@ describe("SnapshotOrphanSweeper", () => { } ); + containerTest( + "still finds keyspaces when the client carries a key prefix", + async ({ prisma, redisOptions }) => { + // ioredis prepends its keyPrefix to keys for ordinary commands, but NOT to a SCAN MATCH + // pattern, and it returns matched keys with the prefix still on them. The engine sets a + // prefix on every other Redis client it builds, so a sweep that ignored this would match + // nothing and report a clean pass: a safety net that silently protects nothing. + const prefixed = { ...(redisOptions as object), keyPrefix: "engine:" } as never; + const store = new RedisSnapshotStore({ + redisOptions: prefixed, + completedTtlMs: COMPLETED_TTL_MS, + }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions: prefixed, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(prefixed, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_1", index: 0 }] }, + }); + + const result = await sweeper.sweep(); + + expect(result.scanned).toBe(1); + expect(result.deleted).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(0); + expect(await probe.exists(`snap:{${runId}}:wp:1`)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + containerTest("processes every keyspace across batches", async ({ prisma, redisOptions }) => { const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.ts index 6ce5c574c2f..4d3eed674b4 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.ts @@ -37,6 +37,13 @@ const FINAL = new Set(FINAL_RUN_STATUSES); /** Comfortably above run-creation latency, so a birth in flight is never mistaken for an orphan. */ const DEFAULT_ORPHAN_AGE_MS = 24 * 60 * 60 * 1000; + +/** + * The keyspace prefix, owned by `snapshotKeys` in the store rather than configurable here. A sweep + * that could be pointed at a different prefix would be a fiction: the store writes `snap:` keys + * unconditionally, so there is no other keyspace to point it at. + */ +const SNAPSHOT_KEYSPACE_PREFIX = "snap:"; const DEFAULT_BATCH_SIZE = 1000; export type SweepResult = { @@ -64,7 +71,6 @@ export type SnapshotOrphanSweeperOptions = { runStore: RunStore; completedTtlMs: number; orphanAgeMs?: number; - keyPrefix?: string; logger?: Logger; }; @@ -73,7 +79,13 @@ export class SnapshotOrphanSweeper { readonly #runStore: RunStore; readonly #completedTtlMs: number; readonly #orphanAgeMs: number; - readonly #prefix: string; + /** + * The ioredis client-level prefix, which is NOT the keyspace prefix. ioredis prepends it to keys + * for ordinary commands, but it does not prepend it to a SCAN MATCH pattern, and it does return + * matched keys with it still attached. Unhandled, a prefixed client makes the sweep match nothing + * and report a clean pass, which is the worst outcome for a safety net. + */ + readonly #clientPrefix: string; readonly #logger: Logger; #quit?: Promise; @@ -82,7 +94,7 @@ export class SnapshotOrphanSweeper { this.#runStore = options.runStore; this.#completedTtlMs = options.completedTtlMs; this.#orphanAgeMs = options.orphanAgeMs ?? DEFAULT_ORPHAN_AGE_MS; - this.#prefix = options.keyPrefix ?? "snap:"; + this.#clientPrefix = (options.redisOptions.keyPrefix as string | undefined) ?? ""; this.#redis = createRedisClient(options.redisOptions, { onError: (error) => this.#logger.error("SnapshotOrphanSweeper redis client error", { error }), }); @@ -115,7 +127,7 @@ export class SnapshotOrphanSweeper { const [next, keys] = await this.#redis.scan( cursor, "MATCH", - `${this.#prefix}{*}:e`, + `${this.#clientPrefix}${SNAPSHOT_KEYSPACE_PREFIX}{*}:e`, "COUNT", batchSize ); @@ -238,7 +250,7 @@ export class SnapshotOrphanSweeper { const high = Number((await this.#redis.hget(core.seq, "c")) ?? "0"); const cycles: string[] = []; for (let n = 1; n <= high; n++) { - cycles.push(`${this.#prefix}{${runId}}:wp:${n}`); + cycles.push(`${SNAPSHOT_KEYSPACE_PREFIX}{${runId}}:wp:${n}`); } const candidates = [core.e, core.idx, core.cur, core.seq, ...cycles]; @@ -309,6 +321,10 @@ export class SnapshotOrphanSweeper { return newestRaw; } + /** + * The run id is whatever sits inside the hash tag, so a client prefix on the returned key does not + * need stripping: `engine:snap:{run_x}:e` and `snap:{run_x}:e` both yield `run_x`. + */ #runIdFrom(key: string): string | undefined { const open = key.indexOf("{"); const close = key.indexOf("}", open + 1); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts index 3b75ca4ca05..9952f569e2b 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts @@ -138,6 +138,61 @@ describe("the staging facade", () => { } }); + containerTest( + "keeps the fork guard on an append staged inside a transaction", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + // A stale expectation: this names a head that was never current. Outside a transaction the + // append is rejected as forked and never written. Staging must not weaken that, or a write + // the store would have refused becomes the head purely because it ran inside a transaction. + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot( + { ...snapshotInput(runId, env, id, "stale"), previousSnapshotId: generateInternalId() }, + tx + ); + }); + + expect(await redis.getById(runId, id)).toBeNull(); + + const head = await redis.getLatest(runId); + expect(head?.id).not.toBe(id); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "honours a correct expectation on an append staged inside a transaction", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const head = await redis.getLatest(runId); + const id = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot( + { ...snapshotInput(runId, env, id, "expected"), previousSnapshotId: head!.id }, + tx + ); + }); + + expect((await redis.getById(runId, id))?.entry.description).toBe("expected"); + } finally { + await redis.quit(); + } + } + ); + containerTest( "hands the transaction callback a decorated store", async ({ prisma, redisOptions }) => { diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index 46a57c6b3bf..4b466eca268 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -104,6 +104,12 @@ export type TaskRunExecutionSnapshotStoreOptions = { /** One deferred append: the entry, plus the wait cycle it carries, if any. */ export type StagedAppend = { entry: SnapshotEntryInput; + /** + * The head this append expects, carried through staging so the compare-and-set survives the + * deferral. Dropping it would silently disable the fork guard for every snapshot written inside a + * transaction, and a stale append that should be rejected would instead become the head. + */ + expectedCur?: string; completedWaitpoints?: CompletedWaitpointRef[]; }; @@ -167,7 +173,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { await this.#appendTransition( "runInTransaction", item.entry, - undefined, + item.expectedCur, item.completedWaitpoints ); } @@ -493,7 +499,11 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { if (this.staging) { // Inside a transaction the append cannot run until the Postgres side commits, or a rollback // leaves Redis holding a transition that never happened. - this.staging.push({ entry, ...(completedWaitpoints && { completedWaitpoints }) }); + this.staging.push({ + entry, + ...(expectedCur !== undefined && { expectedCur }), + ...(completedWaitpoints && { completedWaitpoints }), + }); return; } From 7bba6a895766cc4c49fa013c412312810fab3149 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 12:16:48 +0100 Subject: [PATCH 17/31] chore(run-store): drop the pass-through generator, keep its output as source The generator was scaffolding for a one-off job: writing 70 near-identical forwarders. Keeping it meant carrying a hand-rolled scanner over the interface body, because the TypeScript compiler API is not resolvable in this workspace, which is more machinery than a file that changes only when the interface does. The two files it produced are now maintained by hand, and their headers say so. Nothing is lost, because the generator was never what guaranteed they were right. That is the compiler: implements RunStore rejects a missing member, and the parity assertions tie both name lists to keyof RunStore in each direction and reject a public member the interface does not declare. Each was re-verified by making it fail after the generator was removed. Also drops the knip entry that existed only to treat that script as an entry point. --- .../scripts/generateDelegatingRunStore.ts | 247 ------------------ .../run-store/src/delegatingRunStore.ts | 8 +- .../run-store/src/runStoreMethodNames.ts | 9 +- knip.json | 3 - 4 files changed, 11 insertions(+), 256 deletions(-) delete mode 100644 internal-packages/run-store/scripts/generateDelegatingRunStore.ts diff --git a/internal-packages/run-store/scripts/generateDelegatingRunStore.ts b/internal-packages/run-store/scripts/generateDelegatingRunStore.ts deleted file mode 100644 index 84a6ade0ce7..00000000000 --- a/internal-packages/run-store/scripts/generateDelegatingRunStore.ts +++ /dev/null @@ -1,247 +0,0 @@ -// One-off generator for the RunStore pass-through base. -// -// The base class is mechanical: 80-odd near-identical forwarders. Generating it removes the chance -// of a hand-typo that no test would catch, and turns "did we miss a method" into a diff rather than -// a review. Re-run after any change to the RunStore interface: -// -// pnpm exec tsx scripts/generateDelegatingRunStore.ts -// -// The interface is scanned directly rather than through the TypeScript compiler API, because -// `require("typescript")` resolves to a stub in this workspace. -// -// A parsing miss cannot pass silently, but NOT because of the runtime suite: that compares the class -// against the name list, and both come from this one parse, so a miss drops the member from both -// sides and the comparison still holds. Two compile-time checks catch it instead: -// -// - `implements RunStore` on the generated class fails with TS2420 when a member is absent. -// - The assertions emitted into runStoreMethodNames.ts tie the name lists to `keyof RunStore`, -// in both directions, so a dropped or invented name fails typecheck. -// -// Those live in src rather than in a test because tsconfig.build.json excludes test files, so a -// type-level assertion written in a test is never checked by CI. -import { readFileSync, writeFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; - -const root = join(dirname(fileURLToPath(import.meta.url)), ".."); -const source = readFileSync(join(root, "src/types.ts"), "utf8"); - -/** Replaces every comment and string body with spaces, so a brace inside one cannot move the depth. */ -function blankCommentsAndStrings(text: string): string { - let out = ""; - let i = 0; - while (i < text.length) { - const two = text.slice(i, i + 2); - if (two === "//") { - const end = text.indexOf("\n", i); - const stop = end === -1 ? text.length : end; - out += " ".repeat(stop - i); - i = stop; - } else if (two === "/*") { - const end = text.indexOf("*/", i + 2); - const stop = end === -1 ? text.length : end + 2; - out += text.slice(i, stop).replace(/[^\n]/g, " "); - i = stop; - } else if (text[i] === '"' || text[i] === "'" || text[i] === "`") { - const quote = text[i]; - let j = i + 1; - while (j < text.length && text[j] !== quote) { - j += text[j] === "\\" ? 2 : 1; - } - out += quote + " ".repeat(Math.max(0, j - i - 1)) + (text[j] ?? ""); - i = j + 1; - } else { - out += text[i]; - i += 1; - } - } - return out; -} - -const blanked = blankCommentsAndStrings(source); - -const declaration = "export interface RunStore {"; -const start = blanked.indexOf(declaration); -if (start === -1) { - throw new Error("export interface RunStore not found in src/types.ts"); -} - -const bodyStart = start + declaration.length; -let depth = 1; -let bodyEnd = bodyStart; -while (bodyEnd < blanked.length && depth > 0) { - const ch = blanked[bodyEnd]; - if (ch === "{") depth += 1; - else if (ch === "}") depth -= 1; - if (depth > 0) bodyEnd += 1; -} -if (depth !== 0) { - throw new Error("unbalanced braces while reading the RunStore interface body"); -} - -const body = blanked.slice(bodyStart, bodyEnd); - -// Members are separated by `;` at nesting depth 0. Only `{}`, `()` and `[]` count towards depth: -// angle brackets cannot, because `=>` in a callback parameter type carries an unmatched `>`. -function splitMembers(text: string): { offset: number; length: number }[] { - const spans: { offset: number; length: number }[] = []; - let level = 0; - let from = 0; - for (let i = 0; i < text.length; i++) { - const ch = text[i]; - if (ch === "{" || ch === "(" || ch === "[") level += 1; - else if (ch === "}" || ch === ")" || ch === "]") level -= 1; - else if (ch === ";" && level === 0) { - spans.push({ offset: from, length: i - from }); - from = i + 1; - } - } - if (text.slice(from).trim().length > 0) { - spans.push({ offset: from, length: text.length - from }); - } - return spans; -} - -const methods: string[] = []; -const readonlyProperties: { name: string; type: string }[] = []; -const mutableProperties: string[] = []; - -for (const span of splitMembers(body)) { - const blankedMember = body.slice(span.offset, span.offset + span.length).trim(); - if (blankedMember.length === 0) continue; - - // A method: an optional name then `(` or `<`. A property: a name then `:`. - const asMethod = /^([A-Za-z_$][\w$]*)\s*\??\s*[(<]/.exec(blankedMember); - if (asMethod) { - methods.push(asMethod[1]); - continue; - } - - const asProperty = /^(readonly\s+)?([A-Za-z_$][\w$]*)\s*(\??)\s*:([\s\S]*)$/.exec(blankedMember); - if (asProperty) { - const [, isReadonly, name, , type] = asProperty; - if (isReadonly) { - readonlyProperties.push({ name, type: type.trim() }); - } else { - mutableProperties.push(name); - } - continue; - } - - throw new Error(`could not classify a RunStore member: ${blankedMember.slice(0, 80)}`); -} - -if (mutableProperties.length > 0) { - // A writable data property cannot be forwarded by a getter alone, so the base would silently hold - // its own copy instead of the delegate's. Handle it by hand before regenerating. - throw new Error( - `RunStore declares writable data properties the generator cannot forward: ${mutableProperties.join(", ")}` - ); -} - -const unique = [...new Set(methods)]; -if (unique.length === 0) { - throw new Error("RunStore declares no methods, which cannot be right"); -} - -const memberNames = [...unique, ...readonlyProperties.map((p) => p.name)]; - -const header = `// GENERATED by scripts/generateDelegatingRunStore.ts. Do not edit by hand. -// Regenerate after any change to the RunStore interface: -// pnpm exec tsx scripts/generateDelegatingRunStore.ts -`; - -writeFileSync( - join(root, "src/runStoreMethodNames.ts"), - `${header} -import type { RunStore } from "./types.js"; - -// Every method the RunStore interface declares. The forwarding probe enumerates this to drive one -// call per member; member PRESENCE is proved by the compiler, in the assertions at the foot of this -// file and by \`implements RunStore\` on the generated class. -export const RUN_STORE_METHOD_NAMES = [ -${unique.map((n) => ` "${n}",`).join("\n")} -] as const; - -// Data properties the base exposes as getters over the delegate, not as forwarders. -export const RUN_STORE_PROPERTY_NAMES = [ -${readonlyProperties.map((p) => ` "${p.name}",`).join("\n")} -] as const; - -// --------------------------------------------------------------------------- -// Parity with the interface, checked by the compiler. -// -// The lists above are produced by parsing types.ts. These assertions compare them -// against \`keyof RunStore\`, which the compiler derives from the interface itself, -// so a name this generator failed to parse, or invented, is a build failure rather -// than a silent gap. Both directions are checked: a missing name and an extra one. -// --------------------------------------------------------------------------- - -type RunStoreMemberName = - | (typeof RUN_STORE_METHOD_NAMES)[number] - | (typeof RUN_STORE_PROPERTY_NAMES)[number]; - -/** Fails when the interface declares a member the generator did not emit. */ -type _EveryInterfaceMemberIsListed = [Exclude] extends [never] - ? true - : never; -const _everyInterfaceMemberIsListed: _EveryInterfaceMemberIsListed = true; -void _everyInterfaceMemberIsListed; - -/** Fails when the generator emitted a name the interface does not declare. */ -type _EveryListedNameIsOnTheInterface = [Exclude] extends [never] - ? true - : never; -const _everyListedNameIsOnTheInterface: _EveryListedNameIsOnTheInterface = true; -void _everyListedNameIsOnTheInterface; -` -); - -writeFileSync( - join(root, "src/delegatingRunStore.ts"), - `${header} -// A pass-through over another RunStore. It exists so a decorator can override the handful of methods -// it cares about and inherit the rest, instead of restating 80-odd forwarders alongside real logic. -// -// Arguments and return values are forwarded untouched. The \`any\` signatures carry each method's -// whole overload set through one forwarder, which is the single thing a generated base cannot -// preserve; a subclass that overrides a method restates the real signature there. -/* eslint-disable @typescript-eslint/no-explicit-any */ -import type { RunStore } from "./types.js"; - -export class DelegatingRunStore implements RunStore { - constructor(protected readonly delegate: RunStore) {} - -${readonlyProperties - // Indexed access rather than the written type, so the getter needs no import of its own and - // follows the interface if that type is ever changed. - .map( - (p) => ` get ${p.name}(): RunStore["${p.name}"] {\n return this.delegate.${p.name};\n }` - ) - .join("\n\n")}${readonlyProperties.length > 0 ? "\n\n" : ""}${unique - .map( - (n) => ` ${n}(...args: any[]): any {\n return (this.delegate as any).${n}(...args);\n }` - ) - .join("\n\n")} -} - -// \`implements\` above fails when a member of the interface is MISSING here. It says nothing about a -// member that should not exist, so the reverse direction is asserted too: a public member this class -// declares and the interface does not is a build failure. -// -// \`protected delegate\` is correctly absent from \`keyof\`, so the constructor parameter does not -// trip this. -type _ClassDeclaresNoExtraMembers = [ - Exclude, -] extends [never] - ? true - : never; -const _classParity: _ClassDeclaresNoExtraMembers = true; -void _classParity; -` -); - -console.log( - `generated ${unique.length} forwarders and ${readonlyProperties.length} getters ` + - `(${memberNames.length} members total)` -); diff --git a/internal-packages/run-store/src/delegatingRunStore.ts b/internal-packages/run-store/src/delegatingRunStore.ts index 621242f75cf..e158ada0037 100644 --- a/internal-packages/run-store/src/delegatingRunStore.ts +++ b/internal-packages/run-store/src/delegatingRunStore.ts @@ -1,6 +1,8 @@ -// GENERATED by scripts/generateDelegatingRunStore.ts. Do not edit by hand. -// Regenerate after any change to the RunStore interface: -// pnpm exec tsx scripts/generateDelegatingRunStore.ts +// Maintained by hand. It was scaffolded once, and the scaffolding is gone. +// +// When RunStore gains or loses a member, add or remove the forwarder here. You do not have to +// remember: `implements RunStore` fails with TS2420 on a member that is missing, and the assertion +// at the foot of this file fails on a public member the interface does not declare. // A pass-through over another RunStore. It exists so a decorator can override the handful of methods // it cares about and inherit the rest, instead of restating 80-odd forwarders alongside real logic. diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts index ffb07a7de07..3d8fbc63c77 100644 --- a/internal-packages/run-store/src/runStoreMethodNames.ts +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -1,6 +1,9 @@ -// GENERATED by scripts/generateDelegatingRunStore.ts. Do not edit by hand. -// Regenerate after any change to the RunStore interface: -// pnpm exec tsx scripts/generateDelegatingRunStore.ts +// Maintained by hand, alongside delegatingRunStore.ts. It was scaffolded once, and the scaffolding +// is gone. +// +// When RunStore gains or loses a member, update the matching list here. You do not have to remember: +// the assertions at the foot of this file tie both lists to `keyof RunStore` in both directions, so +// a missing or invented name is a build failure that names the member. import type { RunStore } from "./types.js"; diff --git a/knip.json b/knip.json index 92760110a87..c6e8aee8977 100644 --- a/knip.json +++ b/knip.json @@ -38,9 +38,6 @@ "internal-packages/observability-map": { "entry": ["src/index.ts", "fixtures/**/*.{js,mjs,cjs,ts,mts,cts,tsx}"] }, - "internal-packages/run-store": { - "entry": ["scripts/**/*.{js,mjs,cjs,ts,mts,cts}"] - }, "internal-packages/otlp-importer": { "ignoreDependencies": ["ts-proto"] }, From a553562ed40833ccd2a828a73a0bdd814824a842 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 12:30:06 +0100 Subject: [PATCH 18/31] test(run-engine): guard the sweeper's copy of the terminal-status list The sweeper needs to know which run statuses are terminal and cannot import the list, because run-engine depends on run-store rather than the other way round. The copy's comment claimed a parity test kept the two equal. No such test existed, so the claim was false and the copy could drift silently. Drift is not symmetric. A status added to the engine and not the copy makes the sweep treat a finished run as live and never apply its completion expiry. A status removed from the engine and not the copy makes it treat a live run as finished, and that reaps state a run is still using. Verified by removing a status and rebuilding: the test fails and reports seven members against eight. --- .../engine/tests/finalRunStatusParity.test.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts diff --git a/internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts b/internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts new file mode 100644 index 00000000000..50eba9d3a04 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts @@ -0,0 +1,20 @@ +// The snapshot sweeper needs to know which run statuses are terminal, and it cannot import that +// list: run-engine depends on run-store, not the other way round. So the list is duplicated, and +// this is the only thing that keeps the copy honest. +// +// Without it, a status added here and not there makes the sweeper treat a finished run as live and +// never apply its completion expiry. A status removed here and not there makes it treat a live run +// as finished. The second one reaps state a run is still using. +import { describe, expect, it } from "vitest"; +import { FINAL_RUN_STATUSES } from "@internal/run-store"; +import { getFinalRunStatuses } from "../statuses.js"; + +describe("terminal run statuses", () => { + it("match between the engine and the snapshot sweeper", () => { + expect([...FINAL_RUN_STATUSES].sort()).toEqual([...getFinalRunStatuses()].sort()); + }); + + it("are not empty, so the comparison cannot pass vacuously", () => { + expect(FINAL_RUN_STATUSES.length).toBeGreaterThan(0); + }); +}); From 0f6c6d1dbdcc2de679b5961db579cf4fe1e15eb6 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 12:54:39 +0100 Subject: [PATCH 19/31] refactor(run-store): type every forwarder on the pass-through base The forwarders were (...args: any[]): any, so the compiler could not see inside them. A body that called the wrong delegate member, or reordered its arguments, typechecked cleanly. That is not a theoretical gap: it is why a runtime probe existed to catch it, and it is the same shape of hole that let three other defects on this branch pass a green suite. Every member now restates its interface signature and forwards its arguments by name, so both mistakes are compile errors. Verified by making them: a forward to the wrong member produces two type errors, and swapping two arguments produces one. Seven members are overloaded. TypeScript cannot express a single body that satisfies an overload set, so their overloads are declared for callers and their one implementation forwards through a cast. That cast is now the only place the compiler is not checking the forward. The probe shrinks to what is left: those seven casts, a dropped OPTIONAL argument (omitting a trailing tx compiles and silently stops forwarding the transaction), and whether the data property is read live or captured once. Its header states which of those the compiler already covers. Headers on both files now describe what they are rather than that they were once scaffolded. --- .../run-store/src/delegatingRunStore.test.ts | 71 +- .../run-store/src/delegatingRunStore.ts | 966 +++++++++++++----- .../run-store/src/runStoreMethodNames.ts | 11 +- 3 files changed, 753 insertions(+), 295 deletions(-) diff --git a/internal-packages/run-store/src/delegatingRunStore.test.ts b/internal-packages/run-store/src/delegatingRunStore.test.ts index 74d6a9e9afb..7fdcd9ac07b 100644 --- a/internal-packages/run-store/src/delegatingRunStore.test.ts +++ b/internal-packages/run-store/src/delegatingRunStore.test.ts @@ -1,32 +1,51 @@ -// What this suite covers, and why it does not use a container. +// What this suite covers, and why it is now small. // -// `DelegatingRunStore` is generated and holds no logic: every member forwards to a delegate. Three -// properties of it are already proved by the compiler and are NOT retested here: +// `DelegatingRunStore` restates every interface signature and forwards its arguments by name, so +// most of what a pass-through can get wrong is a compile error rather than a test failure: // -// - a member of the interface missing from the class -> `implements RunStore`, TS2420 -// - a public member the interface does not declare -> the parity assertion in the base -// - a name list out of step with the interface -> the assertions in runStoreMethodNames +// member of the interface missing -> `implements RunStore`, TS2420 +// public member the interface lacks -> the parity assertion in the base +// forwarded to the wrong delegate member -> argument types do not match, TS2345/TS2322 +// arguments reordered -> same // -// What the compiler cannot see is inside the forwarder bodies, because each one is typed -// `(...args: any[]): any`. A forwarder wired to the wrong delegate method, dropping an argument, or -// reading a property once at construction instead of on each access, all typecheck cleanly. Those -// are template-correctness properties of the generator's output, and they are what is tested below. +// Three things remain invisible to the compiler, and they are what is left here. // -// A Testcontainers-backed store cannot demonstrate them. It would mean calling all 70 methods with -// valid arguments and valid foreign-key state, and a real return value cannot show that arguments -// arrived untouched the way a per-member sentinel can. The probe below is not a stand-in for a -// database: no database is involved in whether a pass-through passes through. Behaviour against a -// real store is covered by the container suites for the decorator built on this base. +// First, the seven overloaded members. TypeScript cannot express one body that satisfies an overload +// set, so their single implementation forwards through a cast, and the cast is exactly where a +// wrong-member forward would stop being a type error. +// +// Second, a dropped OPTIONAL argument. Omitting a trailing `tx` compiles cleanly and silently stops +// forwarding the caller's transaction. +// +// Third, whether a data property is read live or captured once at construction. Both typecheck; only +// one is correct. +// +// No database is involved in whether a pass-through passes through, so none is started. Behaviour +// against a real store is covered by the container suites for the decorator built on this base. import { describe, expect, it } from "vitest"; import { DelegatingRunStore } from "./delegatingRunStore.js"; import { RUN_STORE_METHOD_NAMES, RUN_STORE_PROPERTY_NAMES } from "./runStoreMethodNames.js"; import type { RunStore } from "./types.js"; +/** + * The members whose implementation forwards through a cast, because they are overloaded. These are + * the only methods where the compiler is not already checking the forward. + */ +const OVERLOADED_MEMBERS = [ + "finalizeRun", + "findRun", + "findRunOrThrow", + "findRunOnPrimary", + "findRunOrThrowOnPrimary", + "findRuns", + "findRunsByIds", +] as const; + type ProbedCall = { name: string; args: unknown[] }; /** - * A delegate that records what was called on it and answers with a per-member sentinel, so a - * forwarder wired to the wrong member returns the wrong sentinel and fails loudly. + * Records what was called and answers with a per-member sentinel, so a forward to the wrong member + * returns the wrong value rather than merely returning something. */ function forwardingProbe(): { store: RunStore; calls: ProbedCall[] } { const calls: ProbedCall[] = []; @@ -55,15 +74,20 @@ describe("DelegatingRunStore", () => { >; for (const name of RUN_STORE_METHOD_NAMES) { - // The sentinel is per member, so a body forwarding to a different method fails here rather - // than passing because both happened to return something. expect(base[name]()).toBe(`result:${name}`); } expect(calls.map((c) => c.name)).toEqual([...RUN_STORE_METHOD_NAMES]); }); - it("forwards arguments untouched", () => { + it("covers every overloaded member, so the list cannot rot", () => { + // If a member gains or loses overloads, the cast set changes and this suite should follow. + for (const name of OVERLOADED_MEMBERS) { + expect(RUN_STORE_METHOD_NAMES).toContain(name); + } + }); + + it("forwards arguments untouched through an overloaded member's cast", () => { const { store, calls } = forwardingProbe(); const base = new DelegatingRunStore(store) as unknown as Record< string, @@ -71,18 +95,19 @@ describe("DelegatingRunStore", () => { >; const args = ["first", { second: true }, undefined, 4]; - for (const name of RUN_STORE_METHOD_NAMES) { + for (const name of OVERLOADED_MEMBERS) { base[name](...args); } + // The overloaded implementations apply the whole argument list, so every argument survives, + // including a trailing optional the typed members would legitimately drop. for (const call of calls) { expect(call.args).toEqual(args); } + expect(calls.map((c) => c.name)).toEqual([...OVERLOADED_MEMBERS]); }); it("reads a data property live, so a delegate that changes is not cached", () => { - // A getter is the only correct shape here. Capturing the value in the constructor would - // typecheck and would then serve a stale client for the life of the decorator. const store = { primaryReadClient: "first" } as unknown as RunStore; const base = new DelegatingRunStore(store); diff --git a/internal-packages/run-store/src/delegatingRunStore.ts b/internal-packages/run-store/src/delegatingRunStore.ts index e158ada0037..008ccfce1c5 100644 --- a/internal-packages/run-store/src/delegatingRunStore.ts +++ b/internal-packages/run-store/src/delegatingRunStore.ts @@ -1,312 +1,746 @@ -// Maintained by hand. It was scaffolded once, and the scaffolding is gone. +// A pass-through over another RunStore. // -// When RunStore gains or loses a member, add or remove the forwarder here. You do not have to -// remember: `implements RunStore` fails with TS2420 on a member that is missing, and the assertion -// at the foot of this file fails on a public member the interface does not declare. - -// A pass-through over another RunStore. It exists so a decorator can override the handful of methods -// it cares about and inherit the rest, instead of restating 80-odd forwarders alongside real logic. +// It exists so a decorator can override the handful of methods it cares about and inherit the rest. +// +// Every member restates the interface signature and forwards its arguments BY NAME, so the +// forwarding is itself type-checked: a body that called the wrong delegate method, or dropped an +// argument, does not compile. That is the whole point of the shape. An untyped forwarder would let +// both mistakes through, because a pass-through has no other behaviour to catch them. // -// Arguments and return values are forwarded untouched. The `any` signatures carry each method's -// whole overload set through one forwarder, which is the single thing a generated base cannot -// preserve; a subclass that overrides a method restates the real signature there. -/* eslint-disable @typescript-eslint/no-explicit-any */ -import type { RunStore } from "./types.js"; +// Seven members are overloaded. Their overloads are declared so callers keep the full contract, and +// their single implementation signature is the one place a cast appears: TypeScript cannot express +// one body that satisfies an overload set without it. +// +// Keeping this in step with the interface is not a matter of memory. `implements RunStore` rejects a +// member that is missing, and the assertion at the foot of the file rejects one the interface never +// declared. + +import type { + BatchTaskRun, + BatchTaskRunItemStatus, + Prisma, + PrismaClientOrTransaction, + TaskRun, + TaskRunStatus, + WaitpointTag, +} from "@trigger.dev/database"; +import type { TaskRunError } from "@trigger.dev/core/v3/schemas"; +import type { Residency } from "@trigger.dev/core/v3/isomorphic"; +import type { + ClearIdempotencyKeyInput, + CompletionSnapshotInput, + CreateBatchTaskRunData, + CreateCancelledRunInput, + CreateExecutionSnapshotInput, + CreateFailedRunInput, + CreateRunInput, + ExpireSnapshotInput, + FinalizeRunData, + ForWaitpointCompletionContext, + IdempotencyKeyRunMatch, + LockRunData, + PromotePendingVersionArgs, + ReadClient, + RescheduleSnapshotInput, + RewriteDebouncedRunData, + RunStore, + TaskRunWithWaitpoint, + WaitpointColocationOptions, +} from "./types.js"; export class DelegatingRunStore implements RunStore { constructor(protected readonly delegate: RunStore) {} - get primaryReadClient(): RunStore["primaryReadClient"] { + runInTransaction( + runId: string | undefined, + fn: (store: RunStore, tx: PrismaClientOrTransaction) => Promise + ): Promise { + return this.delegate.runInTransaction(runId, fn); + } + + createRun(params: CreateRunInput, tx?: PrismaClientOrTransaction): Promise { + return this.delegate.createRun(params, tx); + } + + createCancelledRun( + params: CreateCancelledRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createCancelledRun(params, tx); + } + + createFailedRun( + params: CreateFailedRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createFailedRun(params, tx); + } + + startAttempt( + runId: string, + data: { attemptNumber: number; executedAt?: Date; isWarmStart: boolean }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.startAttempt(runId, data, args, tx); + } + + completeAttemptSuccess( + runId: string, + data: { + completedAt: Date; + output?: string; + outputType: string; + usageDurationMs: number; + costInCents: number; + snapshot: CompletionSnapshotInput; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.completeAttemptSuccess(runId, data, args, tx); + } + + recordRetryOutcome( + runId: string, + data: { machinePreset?: string; usageDurationMs: number; costInCents: number }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.recordRetryOutcome(runId, data, args, tx); + } + + requeueRun( + runId: string, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.requeueRun(runId, args, tx); + } + + recordBulkActionMembership( + runId: string, + bulkActionId: string, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.recordBulkActionMembership(runId, bulkActionId, tx); + } + + cancelRun( + runId: string, + data: { + completedAt?: Date; + error: TaskRunError; + bulkActionId?: string; + usageDurationMs?: number; + costInCents?: number; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.cancelRun(runId, data, args, tx); + } + + failRunPermanently( + runId: string, + data: { + status: TaskRunStatus; + completedAt: Date; + error: TaskRunError; + usageDurationMs: number; + costInCents: number; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.failRunPermanently(runId, data, args, tx); + } + + finalizeRun( + runId: string, + data: FinalizeRunData, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise>; + finalizeRun( + runId: string, + data: FinalizeRunData, + args: { include: I }, + tx?: PrismaClientOrTransaction + ): Promise>; + finalizeRun( + runId: string, + data: FinalizeRunData, + tx?: PrismaClientOrTransaction + ): Promise; + finalizeRun(...args: unknown[]): unknown { + return (this.delegate.finalizeRun as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + expireRun( + runId: string, + data: { + error: TaskRunError; + completedAt: Date; + expiredAt: Date; + snapshot: ExpireSnapshotInput; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.expireRun(runId, data, args, tx); + } + + expireRunsBatch( + runIds: string[], + data: { error: TaskRunError; now: Date }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.expireRunsBatch(runIds, data, tx); + } + + lockRunToWorker( + runId: string, + data: LockRunData, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.lockRunToWorker(runId, data, tx); + } + + parkPendingVersion( + runId: string, + data: { statusReason: string }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.parkPendingVersion(runId, data, args, tx); + } + + promotePendingVersionRuns( + runId: string, + args?: PromotePendingVersionArgs, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.promotePendingVersionRuns(runId, args, tx); + } + + expireParkedRun( + runId: string, + data: { + error: TaskRunError; + completedAt: Date; + expiredAt: Date; + statusReason: string; + snapshot: ExpireSnapshotInput; + }, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.expireParkedRun(runId, data, tx); + } + + suspendForCheckpoint( + runId: string, + args: { include: I }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.suspendForCheckpoint(runId, args, tx); + } + + resumeFromCheckpoint( + runId: string, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.resumeFromCheckpoint(runId, args, tx); + } + + rescheduleRun( + runId: string, + data: { delayUntil: Date; queueTimestamp?: Date; snapshot?: RescheduleSnapshotInput }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.rescheduleRun(runId, data, tx); + } + + enqueueDelayedRun( + runId: string, + data: { queuedAt: Date }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.enqueueDelayedRun(runId, data, tx); + } + + rewriteDebouncedRun( + runId: string, + data: RewriteDebouncedRunData, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.rewriteDebouncedRun(runId, data, tx); + } + + updateMetadata( + runId: string, + data: { + metadata: string | null; + metadataType?: string; + metadataVersion: { increment: number }; + updatedAt: Date; + }, + options: { expectedMetadataVersion?: number }, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.updateMetadata(runId, data, options, tx); + } + + clearIdempotencyKey( + params: ClearIdempotencyKeyInput, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.clearIdempotencyKey(params, tx); + } + + pushTags( + runId: string, + tags: string[], + where: { runtimeEnvironmentId: string }, + tx?: PrismaClientOrTransaction + ): Promise<{ updatedAt: Date }> { + return this.delegate.pushTags(runId, tags, where, tx); + } + + pushRealtimeStream( + runId: string, + streamId: string, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.pushRealtimeStream(runId, streamId, tx); + } + + get primaryReadClient(): ReadClient { return this.delegate.primaryReadClient; } - runInTransaction(...args: any[]): any { - return (this.delegate as any).runInTransaction(...args); - } - - createRun(...args: any[]): any { - return (this.delegate as any).createRun(...args); - } - - createCancelledRun(...args: any[]): any { - return (this.delegate as any).createCancelledRun(...args); - } - - createFailedRun(...args: any[]): any { - return (this.delegate as any).createFailedRun(...args); - } - - startAttempt(...args: any[]): any { - return (this.delegate as any).startAttempt(...args); - } - - completeAttemptSuccess(...args: any[]): any { - return (this.delegate as any).completeAttemptSuccess(...args); - } - - recordRetryOutcome(...args: any[]): any { - return (this.delegate as any).recordRetryOutcome(...args); - } - - requeueRun(...args: any[]): any { - return (this.delegate as any).requeueRun(...args); - } - - recordBulkActionMembership(...args: any[]): any { - return (this.delegate as any).recordBulkActionMembership(...args); - } - - cancelRun(...args: any[]): any { - return (this.delegate as any).cancelRun(...args); - } - - failRunPermanently(...args: any[]): any { - return (this.delegate as any).failRunPermanently(...args); - } - - finalizeRun(...args: any[]): any { - return (this.delegate as any).finalizeRun(...args); - } - - expireRun(...args: any[]): any { - return (this.delegate as any).expireRun(...args); - } - - expireRunsBatch(...args: any[]): any { - return (this.delegate as any).expireRunsBatch(...args); - } - - lockRunToWorker(...args: any[]): any { - return (this.delegate as any).lockRunToWorker(...args); - } - - parkPendingVersion(...args: any[]): any { - return (this.delegate as any).parkPendingVersion(...args); - } - - promotePendingVersionRuns(...args: any[]): any { - return (this.delegate as any).promotePendingVersionRuns(...args); - } - - expireParkedRun(...args: any[]): any { - return (this.delegate as any).expireParkedRun(...args); - } - - suspendForCheckpoint(...args: any[]): any { - return (this.delegate as any).suspendForCheckpoint(...args); - } - - resumeFromCheckpoint(...args: any[]): any { - return (this.delegate as any).resumeFromCheckpoint(...args); - } - - rescheduleRun(...args: any[]): any { - return (this.delegate as any).rescheduleRun(...args); - } - - enqueueDelayedRun(...args: any[]): any { - return (this.delegate as any).enqueueDelayedRun(...args); - } - - rewriteDebouncedRun(...args: any[]): any { - return (this.delegate as any).rewriteDebouncedRun(...args); - } - - updateMetadata(...args: any[]): any { - return (this.delegate as any).updateMetadata(...args); - } - - clearIdempotencyKey(...args: any[]): any { - return (this.delegate as any).clearIdempotencyKey(...args); - } - - pushTags(...args: any[]): any { - return (this.delegate as any).pushTags(...args); - } - - pushRealtimeStream(...args: any[]): any { - return (this.delegate as any).pushRealtimeStream(...args); - } - - findRun(...args: any[]): any { - return (this.delegate as any).findRun(...args); - } - - findRunOrThrow(...args: any[]): any { - return (this.delegate as any).findRunOrThrow(...args); - } - - findRunOnPrimary(...args: any[]): any { - return (this.delegate as any).findRunOnPrimary(...args); - } - - findRunOrThrowOnPrimary(...args: any[]): any { - return (this.delegate as any).findRunOrThrowOnPrimary(...args); - } - - findRuns(...args: any[]): any { - return (this.delegate as any).findRuns(...args); - } - - findRunsByIds(...args: any[]): any { - return (this.delegate as any).findRunsByIds(...args); - } - - findRunsByIdempotencyKeys(...args: any[]): any { - return (this.delegate as any).findRunsByIdempotencyKeys(...args); - } - - createBatchTaskRunItem(...args: any[]): any { - return (this.delegate as any).createBatchTaskRunItem(...args); - } - - findLatestExecutionSnapshot(...args: any[]): any { - return (this.delegate as any).findLatestExecutionSnapshot(...args); - } - - findExecutionSnapshot(...args: any[]): any { - return (this.delegate as any).findExecutionSnapshot(...args); - } - - findManyExecutionSnapshots(...args: any[]): any { - return (this.delegate as any).findManyExecutionSnapshots(...args); - } - - createExecutionSnapshot(...args: any[]): any { - return (this.delegate as any).createExecutionSnapshot(...args); - } - - findSnapshotCompletedWaitpointIds(...args: any[]): any { - return (this.delegate as any).findSnapshotCompletedWaitpointIds(...args); - } - - findSnapshotCompletedWaitpointIdsWithPresence(...args: any[]): any { - return (this.delegate as any).findSnapshotCompletedWaitpointIdsWithPresence(...args); - } - - findWaitpointConnectedRunIds(...args: any[]): any { - return (this.delegate as any).findWaitpointConnectedRunIds(...args); - } - - findWaitpointCompletedSnapshotIds(...args: any[]): any { - return (this.delegate as any).findWaitpointCompletedSnapshotIds(...args); - } - - blockRunWithWaitpointEdges(...args: any[]): any { - return (this.delegate as any).blockRunWithWaitpointEdges(...args); - } - - countPendingWaitpoints(...args: any[]): any { - return (this.delegate as any).countPendingWaitpoints(...args); - } - - countPendingWaitpointsWithPresence(...args: any[]): any { - return (this.delegate as any).countPendingWaitpointsWithPresence(...args); - } - - createWaitpoint(...args: any[]): any { - return (this.delegate as any).createWaitpoint(...args); - } - - upsertWaitpoint(...args: any[]): any { - return (this.delegate as any).upsertWaitpoint(...args); - } - - findWaitpoint(...args: any[]): any { - return (this.delegate as any).findWaitpoint(...args); - } - - findWaitpointOnPrimary(...args: any[]): any { - return (this.delegate as any).findWaitpointOnPrimary(...args); - } - - findManyWaitpoints(...args: any[]): any { - return (this.delegate as any).findManyWaitpoints(...args); - } - - updateWaitpoint(...args: any[]): any { - return (this.delegate as any).updateWaitpoint(...args); - } - - updateManyWaitpoints(...args: any[]): any { - return (this.delegate as any).updateManyWaitpoints(...args); - } - - forWaitpointCompletion(...args: any[]): any { - return (this.delegate as any).forWaitpointCompletion(...args); - } - - findManyTaskRunWaitpoints(...args: any[]): any { - return (this.delegate as any).findManyTaskRunWaitpoints(...args); - } - - deleteManyTaskRunWaitpoints(...args: any[]): any { - return (this.delegate as any).deleteManyTaskRunWaitpoints(...args); + findRun( + where: Prisma.TaskRunWhereInput, + args: { select: S }, + client?: ReadClient + ): Promise | null>; + findRun( + where: Prisma.TaskRunWhereInput, + args: { include: I }, + client?: ReadClient + ): Promise | null>; + findRun(where: Prisma.TaskRunWhereInput, client?: ReadClient): Promise; + findRun(...args: unknown[]): unknown { + return (this.delegate.findRun as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + findRunOrThrow( + where: Prisma.TaskRunWhereInput, + args: { select: S }, + client?: ReadClient + ): Promise>; + findRunOrThrow( + where: Prisma.TaskRunWhereInput, + args: { include: I }, + client?: ReadClient + ): Promise>; + findRunOrThrow(where: Prisma.TaskRunWhereInput, client?: ReadClient): Promise; + findRunOrThrow(...args: unknown[]): unknown { + return (this.delegate.findRunOrThrow as (...a: unknown[]) => unknown).apply( + this.delegate, + args + ); + } + + findRunOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { select: S } + ): Promise | null>; + findRunOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { include: I } + ): Promise | null>; + findRunOnPrimary(where: Prisma.TaskRunWhereInput): Promise; + findRunOnPrimary(...args: unknown[]): unknown { + return (this.delegate.findRunOnPrimary as (...a: unknown[]) => unknown).apply( + this.delegate, + args + ); + } + + findRunOrThrowOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { select: S } + ): Promise>; + findRunOrThrowOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { include: I } + ): Promise>; + findRunOrThrowOnPrimary(where: Prisma.TaskRunWhereInput): Promise; + findRunOrThrowOnPrimary(...args: unknown[]): unknown { + return (this.delegate.findRunOrThrowOnPrimary as (...a: unknown[]) => unknown).apply( + this.delegate, + args + ); + } + + findRuns( + args: { + where: Prisma.TaskRunWhereInput; + select: S; + orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[]; + take?: number; + skip?: number; + cursor?: Prisma.TaskRunWhereUniqueInput; + }, + client?: ReadClient + ): Promise[]>; + findRuns( + args: { + where: Prisma.TaskRunWhereInput; + include: I; + orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[]; + take?: number; + skip?: number; + cursor?: Prisma.TaskRunWhereUniqueInput; + }, + client?: ReadClient + ): Promise[]>; + findRuns( + args: { + where: Prisma.TaskRunWhereInput; + orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[]; + take?: number; + skip?: number; + cursor?: Prisma.TaskRunWhereUniqueInput; + }, + client?: ReadClient + ): Promise; + findRuns(...args: unknown[]): unknown { + return (this.delegate.findRuns as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + findRunsByIds( + ids: string[], + args: { select: S }, + client?: ReadClient + ): Promise>>; + findRunsByIds( + ids: string[], + args: { include: I }, + client?: ReadClient + ): Promise>>; + findRunsByIds(ids: string[], client?: ReadClient): Promise>; + findRunsByIds(...args: unknown[]): unknown { + return (this.delegate.findRunsByIds as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + findRunsByIdempotencyKeys( + args: { runtimeEnvironmentId: string; taskIdentifier: string; idempotencyKeys: string[] }, + client?: ReadClient + ): Promise { + return this.delegate.findRunsByIdempotencyKeys(args, client); + } + + createBatchTaskRunItem( + data: { batchTaskRunId: string; taskRunId: string; status: BatchTaskRunItemStatus }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createBatchTaskRunItem(data, tx); + } + + findLatestExecutionSnapshot( + runId: string, + client?: ReadClient, + // When set, scopes the read to this environment (tenant boundary); a run in another env reads as + // not-found. Omit to read regardless of environment (internal callers). + environmentId?: string + ): Promise | null> { + return this.delegate.findLatestExecutionSnapshot(runId, client); + } + + findExecutionSnapshot( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise | null> { + return this.delegate.findExecutionSnapshot(args, client); + } + + findManyExecutionSnapshots( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise[]> { + return this.delegate.findManyExecutionSnapshots(args, client); + } + + createExecutionSnapshot( + input: CreateExecutionSnapshotInput, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.createExecutionSnapshot(input, tx); + } + + findSnapshotCompletedWaitpointIds( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise { + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + findSnapshotCompletedWaitpointIdsWithPresence( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise<{ present: boolean; ids: string[] }> { + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + findWaitpointConnectedRunIds(waitpointId: string, client?: ReadClient): Promise { + return this.delegate.findWaitpointConnectedRunIds(waitpointId, client); + } + + findWaitpointCompletedSnapshotIds(waitpointId: string, client?: ReadClient): Promise { + return this.delegate.findWaitpointCompletedSnapshotIds(waitpointId, client); + } + + blockRunWithWaitpointEdges(params: { + runId: string; + waitpointIds: string[]; + projectId: string; + spanIdToComplete?: string; + batchId?: string; + batchIndex?: number; + tx?: PrismaClientOrTransaction; + }): Promise { + return this.delegate.blockRunWithWaitpointEdges(params); } - findTaskRunAttempt(...args: any[]): any { - return (this.delegate as any).findTaskRunAttempt(...args); + countPendingWaitpoints( + waitpointIds: string[], + client?: ReadClient, + runId?: string + ): Promise { + return this.delegate.countPendingWaitpoints(waitpointIds, client, runId); } - createTaskRunCheckpoint(...args: any[]): any { - return (this.delegate as any).createTaskRunCheckpoint(...args); + countPendingWaitpointsWithPresence( + waitpointIds: string[], + client?: ReadClient + ): Promise<{ pendingIds: string[]; presentIds: string[] }> { + return this.delegate.countPendingWaitpointsWithPresence(waitpointIds, client); } - createBatchTaskRun(...args: any[]): any { - return (this.delegate as any).createBatchTaskRun(...args); + createWaitpoint( + args: Prisma.SelectSubset, + tx?: PrismaClientOrTransaction, + opts?: WaitpointColocationOptions + ): Promise> { + return this.delegate.createWaitpoint(args, tx, opts); } - updateBatchTaskRun(...args: any[]): any { - return (this.delegate as any).updateBatchTaskRun(...args); + upsertWaitpoint( + args: Prisma.SelectSubset, + tx?: PrismaClientOrTransaction, + opts?: WaitpointColocationOptions + ): Promise> { + return this.delegate.upsertWaitpoint(args, tx, opts); } - findBatchTaskRunById(...args: any[]): any { - return (this.delegate as any).findBatchTaskRunById(...args); + findWaitpoint( + args: Prisma.SelectSubset, + client?: ReadClient, + opts?: WaitpointColocationOptions + ): Promise | null> { + return this.delegate.findWaitpoint(args, client, opts); } - findBatchTaskRunByFriendlyId(...args: any[]): any { - return (this.delegate as any).findBatchTaskRunByFriendlyId(...args); + findWaitpointOnPrimary( + args: Prisma.SelectSubset + ): Promise | null> { + return this.delegate.findWaitpointOnPrimary(args); } - findBatchTaskRunByIdempotencyKey(...args: any[]): any { - return (this.delegate as any).findBatchTaskRunByIdempotencyKey(...args); + findManyWaitpoints( + args: Prisma.SelectSubset, + client?: ReadClient, + runId?: string + ): Promise[]> { + return this.delegate.findManyWaitpoints(args, client, runId); } - updateManyBatchTaskRun(...args: any[]): any { - return (this.delegate as any).updateManyBatchTaskRun(...args); + updateWaitpoint( + args: Prisma.SelectSubset, + tx?: PrismaClientOrTransaction, + opts?: WaitpointColocationOptions + ): Promise> { + return this.delegate.updateWaitpoint(args, tx, opts); } - countBatchTaskRunItems(...args: any[]): any { - return (this.delegate as any).countBatchTaskRunItems(...args); + updateManyWaitpoints( + args: Prisma.WaitpointUpdateManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.updateManyWaitpoints(args, tx); } - updateManyBatchTaskRunItems(...args: any[]): any { - return (this.delegate as any).updateManyBatchTaskRunItems(...args); + forWaitpointCompletion( + waitpointId: string, + context: ForWaitpointCompletionContext + ): Promise { + return this.delegate.forWaitpointCompletion(waitpointId, context); } - findManyBatchTaskRunItems(...args: any[]): any { - return (this.delegate as any).findManyBatchTaskRunItems(...args); + findManyTaskRunWaitpoints( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise[]> { + return this.delegate.findManyTaskRunWaitpoints(args, client); } - findBatchTaskRunItem(...args: any[]): any { - return (this.delegate as any).findBatchTaskRunItem(...args); + deleteManyTaskRunWaitpoints( + args: Prisma.TaskRunWaitpointDeleteManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.deleteManyTaskRunWaitpoints(args, tx); + } + + findTaskRunAttempt( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise | null> { + return this.delegate.findTaskRunAttempt(args, client); } - upsertWaitpointTag(...args: any[]): any { - return (this.delegate as any).upsertWaitpointTag(...args); + createTaskRunCheckpoint( + args: Prisma.SelectSubset, + ownerRunId?: string, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.createTaskRunCheckpoint(args, ownerRunId, tx); } - findManyWaitpointTags(...args: any[]): any { - return (this.delegate as any).findManyWaitpointTags(...args); + createBatchTaskRun( + data: CreateBatchTaskRunData, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createBatchTaskRun(data, tx); + } + + updateBatchTaskRun( + args: { + where: Prisma.BatchTaskRunWhereUniqueInput; + data: Prisma.BatchTaskRunUpdateInput; + select: S; + }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.updateBatchTaskRun(args, tx); + } + + findBatchTaskRunById( + id: string, + args?: { include?: T }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunById(id, args, client); + } + + findBatchTaskRunByFriendlyId( + friendlyId: string, + environmentId: string, + args?: { include?: T }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunByFriendlyId(friendlyId, environmentId, args, client); + } + + findBatchTaskRunByIdempotencyKey( + environmentId: string, + idempotencyKey: string, + args?: { include?: T }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunByIdempotencyKey( + environmentId, + idempotencyKey, + args, + client + ); + } + + updateManyBatchTaskRun( + args: Prisma.BatchTaskRunUpdateManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.updateManyBatchTaskRun(args, tx); + } + + countBatchTaskRunItems( + where: { batchTaskRunId: string; status?: BatchTaskRunItemStatus }, + client?: ReadClient + ): Promise { + return this.delegate.countBatchTaskRunItems(where, client); + } + + updateManyBatchTaskRunItems( + args: Prisma.BatchTaskRunItemUpdateManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.updateManyBatchTaskRunItems(args, tx); + } + + findManyBatchTaskRunItems( + where: { taskRunId?: string; batchTaskRunId?: string }, + args?: { include?: I }, + client?: ReadClient + ): Promise[]> { + return this.delegate.findManyBatchTaskRunItems(where, args, client); + } + + findBatchTaskRunItem( + where: { batchTaskRunId: string; taskRunId?: string }, + args?: { include?: I }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunItem(where, args, client); + } + + upsertWaitpointTag( + data: { environmentId: string; name: string; projectId: string; id?: string }, + tx?: PrismaClientOrTransaction, + // A tag has no owning run to co-locate with; when no minted `id` pins it by id-shape, a + // minted-new env's tags read this residency (NEW) so they land with the env's tokens/runs + // instead of defaulting to LEGACY. Single-store impls ignore it. + residency?: Residency + ): Promise { + return this.delegate.upsertWaitpointTag(data, tx); + } + + findManyWaitpointTags( + args: { + where: Prisma.WaitpointTagWhereInput; + orderBy?: + | Prisma.WaitpointTagOrderByWithRelationInput + | Prisma.WaitpointTagOrderByWithRelationInput[]; + take?: number; + skip?: number; + }, + client?: ReadClient + ): Promise { + return this.delegate.findManyWaitpointTags(args, client); } } -// `implements` above fails when a member of the interface is MISSING here. It says nothing about a +// `implements` above rejects a member of the interface that is missing here. It says nothing about a // member that should not exist, so the reverse direction is asserted too: a public member this class // declares and the interface does not is a build failure. // -// `protected delegate` is correctly absent from `keyof`, so the constructor parameter does not -// trip this. +// `protected delegate` is correctly absent from `keyof`, so the constructor parameter does not trip +// this. type _ClassDeclaresNoExtraMembers = [Exclude] extends [ never, ] diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts index 3d8fbc63c77..2b10bd766d0 100644 --- a/internal-packages/run-store/src/runStoreMethodNames.ts +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -1,10 +1,9 @@ -// Maintained by hand, alongside delegatingRunStore.ts. It was scaffolded once, and the scaffolding -// is gone. +// The member names of RunStore, as data. +// +// The decorator suites enumerate this to drive one call per member. Member PRESENCE is not proved +// here: that is the compiler's job, through `implements RunStore` on the pass-through base and the +// assertions at the foot of this file. // -// When RunStore gains or loses a member, update the matching list here. You do not have to remember: -// the assertions at the foot of this file tie both lists to `keyof RunStore` in both directions, so -// a missing or invented name is a build failure that names the member. - import type { RunStore } from "./types.js"; // Every method the RunStore interface declares. The forwarding probe enumerates this to drive one From e03a18566a8897a4e9682cdba0095c6cab306992 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 13:08:09 +0100 Subject: [PATCH 20/31] fix(run-store): forward the parameters two members were dropping Typing the forwarders closed the wrong-member and reordered-argument holes but not this one: omitting a trailing OPTIONAL argument still compiles. Two forwarders did exactly that, because the retyping pass read parameter names with a pattern that a preceding inline comment defeated, and both affected parameters happened to be optional and commented. The effects were silent and not small. findLatestExecutionSnapshot stopped applying its tenant scope, so a direct use of the base could read across the environment boundary. upsertWaitpointTag stopped applying its residency hint, so a tag write for a new-database environment would land on legacy. A source-level guard now asserts that every single-signature member forwards exactly the parameters it declares, in order. It reads the interface and the base and compares them, because that property is invisible to the compiler by definition. It carries a vacuity check, so a parse failure fails the suite instead of quietly matching nothing, and that check earned itself immediately by catching a parser that skipped every generic member. Verified: with a parameter dropped again, typecheck reports zero errors and the guard names the member and the missing argument. --- .../src/delegatingRunStore.forwarding.test.ts | 188 ++++++++++++++++++ .../run-store/src/delegatingRunStore.ts | 4 +- 2 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts diff --git a/internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts b/internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts new file mode 100644 index 00000000000..081e9efe1d4 --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts @@ -0,0 +1,188 @@ +// Every declared parameter must actually reach the delegate. +// +// The compiler cannot check this. A forwarder that omits a trailing OPTIONAL argument compiles +// cleanly, and the effect is silent: `findLatestExecutionSnapshot` would stop applying its tenant +// scope, and `upsertWaitpointTag` would stop applying its residency hint, so a write would land on +// the wrong database. Both of those shipped in this file before this test existed. +// +// So this reads the source of the base against the source of the interface and asserts that each +// forward passes exactly the parameters its signature declares, in order. Source-level, because +// that is the only place the property is visible. +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const dir = join(import.meta.dirname); +const interfaceSource = readFileSync(join(dir, "types.ts"), "utf8"); +const baseSource = readFileSync(join(dir, "delegatingRunStore.ts"), "utf8"); + +/** Replaces comments and string bodies with spaces, so neither can shift a brace depth. */ +function blank(text: string): string { + let out = ""; + let i = 0; + while (i < text.length) { + const two = text.slice(i, i + 2); + if (two === "//") { + const end = text.indexOf("\n", i); + const stop = end === -1 ? text.length : end; + out += " ".repeat(stop - i); + i = stop; + } else if (two === "/*") { + const end = text.indexOf("*/", i + 2); + const stop = end === -1 ? text.length : end + 2; + out += text.slice(i, stop).replace(/[^\n]/g, " "); + i = stop; + } else if (text[i] === '"' || text[i] === "'" || text[i] === "`") { + const quote = text[i]; + let j = i + 1; + while (j < text.length && text[j] !== quote) j += text[j] === "\\" ? 2 : 1; + out += quote + " ".repeat(Math.max(0, j - i - 1)) + (text[j] ?? ""); + i = j + 1; + } else { + out += text[i]; + i += 1; + } + } + return out; +} + +function interfaceBody(source: string): string { + const blanked = blank(source); + const decl = "export interface RunStore {"; + const start = blanked.indexOf(decl) + decl.length; + let depth = 1; + let end = start; + while (depth > 0 && end < blanked.length) { + if (blanked[end] === "{") depth += 1; + else if (blanked[end] === "}") depth -= 1; + if (depth > 0) end += 1; + } + return blanked.slice(start, end); +} + +/** Splits a balanced parameter list on top-level commas. */ +function splitParams(signature: string): string[] { + // A generic member reads `name(...)`, so the parameter list starts after the + // balanced angle block, not at the first parenthesis. + let searchFrom = 0; + const angle = signature.indexOf("<"); + const paren = signature.indexOf("("); + if (angle !== -1 && angle < paren) { + let angleDepth = 0; + for (let i = angle; i < signature.length; i++) { + if (signature[i] === "<") angleDepth += 1; + else if (signature[i] === ">") { + angleDepth -= 1; + if (angleDepth === 0) { + searchFrom = i; + break; + } + } + } + } + + const open = signature.indexOf("(", searchFrom); + let depth = 0; + let close = open; + for (let i = open; i < signature.length; i++) { + if ("([{<".includes(signature[i]!)) depth += 1; + else if (")]}>".includes(signature[i]!)) { + depth -= 1; + if (depth === 0) { + close = i; + break; + } + } + } + const inner = signature.slice(open + 1, close); + const parts: string[] = []; + let level = 0; + let current = ""; + for (const ch of inner) { + if ("([{<".includes(ch)) level += 1; + else if (")]}>".includes(ch)) level -= 1; + if (ch === "," && level === 0) { + parts.push(current); + current = ""; + } else { + current += ch; + } + } + if (current.trim()) parts.push(current); + return parts; +} + +function paramNames(signature: string): string[] { + return splitParams(signature) + .map((p) => /^\s*([A-Za-z_$][\w$]*)\s*\??\s*:/.exec(p)?.[1]) + .filter((n): n is string => Boolean(n)); +} + +/** Member name to its declared parameter names, for members with a single signature. */ +function declaredParams(): Map { + const body = interfaceBody(interfaceSource); + const spans: string[] = []; + let level = 0; + let from = 0; + for (let i = 0; i < body.length; i++) { + const ch = body[i]!; + if ("{([".includes(ch)) level += 1; + else if ("})]".includes(ch)) level -= 1; + else if (ch === ";" && level === 0) { + spans.push(body.slice(from, i)); + from = i + 1; + } + } + + const seen = new Map(); + for (const span of spans) { + const match = /^\s*(?:readonly\s+)?([A-Za-z_$][\w$]*)\s*\??\s*[(<]/.exec(span); + if (!match) continue; + const name = match[1]!; + seen.set(name, [...(seen.get(name) ?? []), paramNames(span)]); + } + + // Overloaded members forward through a cast and apply the whole argument list, so they are not + // subject to this check. + return new Map( + [...seen].filter(([, sigs]) => sigs.length === 1).map(([n, sigs]) => [n, sigs[0]!]) + ); +} + +describe("the pass-through forwards every declared parameter", () => { + const declared = declaredParams(); + + it("parsed the interface, so a parse failure cannot pass this suite", () => { + expect(declared.size).toBeGreaterThan(50); + expect(declared.get("expireParkedRun")).toEqual(["runId", "data", "tx"]); + expect(declared.get("findLatestExecutionSnapshot")).toEqual([ + "runId", + "client", + "environmentId", + ]); + }); + + it("passes exactly the declared parameters, in order, for every single-signature member", () => { + const wrong: string[] = []; + + for (const [name, params] of declared) { + const forward = new RegExp(`return this\\.delegate\\.${name}\\(([^;]*)\\);`).exec(baseSource); + + if (!forward) { + wrong.push(`${name}: no forward found`); + continue; + } + + const passed = forward[1]! + .split(",") + .map((a) => a.trim()) + .filter(Boolean); + + if (passed.join(",") !== params.join(",")) { + wrong.push(`${name}: declares (${params.join(", ")}) but forwards (${passed.join(", ")})`); + } + } + + expect(wrong).toEqual([]); + }); +}); diff --git a/internal-packages/run-store/src/delegatingRunStore.ts b/internal-packages/run-store/src/delegatingRunStore.ts index 008ccfce1c5..c7fe3225c76 100644 --- a/internal-packages/run-store/src/delegatingRunStore.ts +++ b/internal-packages/run-store/src/delegatingRunStore.ts @@ -459,7 +459,7 @@ export class DelegatingRunStore implements RunStore { ): Promise | null> { - return this.delegate.findLatestExecutionSnapshot(runId, client); + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); } findExecutionSnapshot( @@ -717,7 +717,7 @@ export class DelegatingRunStore implements RunStore { // instead of defaulting to LEGACY. Single-store impls ignore it. residency?: Residency ): Promise { - return this.delegate.upsertWaitpointTag(data, tx); + return this.delegate.upsertWaitpointTag(data, tx, residency); } findManyWaitpointTags( From e9ce90989945d09fb5ac95a2a76a737529eaed5b Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 13:27:57 +0100 Subject: [PATCH 21/31] fix(run-store): keep completed waitpoints that have no batch index A completed waitpoint with no batch index was invisible to every Redis read, so a run resumed from the store lost that wait's result while Postgres still returned it. That is every wait.for, every single triggerAndWait and every token: the engine passes index as batchIndex ?? undefined, so only waits inside a batch carry one. The cause was reading the id set out of the ordered list. That list is the index oracle and its positions ARE the indexes, so it can only ever hold indexed ids, and deduping it yields a set missing exactly the index-less ones. Postgres has no such restriction: its completed-waitpoint join records every id. The cycle key now carries the complete distinct set in its own field, written when the cycle is minted and read back beside the order. The order keeps its meaning and stays index-only. Two tests: one asserting an index-less wait survives a round trip with an empty order, and one asserting the set matches the Postgres join for a mix of indexed and index-less waits. Verified by deriving the set from the order again, which makes the waitpoint vanish. The suites missed this because every earlier case gave each waitpoint an index. --- .../run-store/src/redisSnapshotStore.ts | 46 ++++++++++++-- ...utionSnapshotStore.waitpointCycles.test.ts | 63 +++++++++++++++++++ 2 files changed, 103 insertions(+), 6 deletions(-) diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index e36179c6041..88c18d00595 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -29,6 +29,19 @@ export function deriveOrder(completedWaitpoints: CompletedWaitpointRef[]): strin .map((w) => w.id); } +/** + * The COMPLETE distinct set of completed-waitpoint ids, including those with no batch index. + * + * This is deliberately not `deriveOrder` deduped. `order` is the index oracle and carries only + * batch-indexed ids, because its positions ARE the indexes. A wait with no batch index (every + * `wait.for`, every single `triggerAndWait`, every token) has no position and is absent from it, + * while Postgres records it in the completed-waitpoint join like any other. Reading the id set back + * from `order` therefore loses exactly those waits, and a run resumed from Redis loses their results. + */ +export function deriveDistinctIds(completedWaitpoints: CompletedWaitpointRef[]): string[] { + return [...new Set(completedWaitpoints.map((w) => w.id))]; +} + // isValid is derived, never stored, so the entry JSON stays byte-identical to the caller's document. export function isValidFor(entry: { error?: unknown }): boolean { return !entry.error; @@ -270,12 +283,14 @@ export class RedisSnapshotStore { let cycleMode = "none"; let cycleSeqIn = "0"; let orderJson = ""; + let distinctJson = ""; let records = ""; let orderCount = "0"; if (args.cycle?.kind === "new") { const order = deriveOrder(args.cycle.completedWaitpoints); cycleMode = "new"; orderJson = JSON.stringify(order); + distinctJson = JSON.stringify(deriveDistinctIds(args.cycle.completedWaitpoints)); records = args.cycle.records ? JSON.stringify(args.cycle.records) : ""; orderCount = String(order.length); } else if (args.cycle?.kind === "carryForward") { @@ -300,7 +315,8 @@ export class RedisSnapshotStore { records, orderCount, args.expectedCur ?? "", - args.expectedCur !== undefined ? "1" : "0" + args.expectedCur !== undefined ? "1" : "0", + distinctJson )) as string[]; return this.#interpretAppend(reply, raw, orderJson, records, args.entry.runId); @@ -407,7 +423,7 @@ export class RedisSnapshotStore { return this.#timed("getSnapshotWaitpointIds", async () => { const k = snapshotKeys(runId); const reply = await this.redis.readSnapshotWaitpointIds(k.e, k.idx, k.cur, k.seq, snapshotId); - return decodeWaitpointIds(reply[0] === "1", reply[1] ?? ""); + return decodeWaitpointIds(reply[0] === "1", reply[1] ?? "", reply[2] ?? ""); }); } @@ -588,6 +604,13 @@ export class RedisSnapshotStore { if not cs then return '' end return redis.call('HGET', wpKey(cs), 'order') or '' end + -- The complete id set, which is NOT the order deduped: order holds only batch-indexed ids. + local function distinctFor(pointer) + if not pointer then return '' end + local cs = string.match(pointer, '^(%d+):') + if not cs then return '' end + return redis.call('HGET', wpKey(cs), 'distinct') or '' + end `; this.redis.defineCommand("appendSnapshotEntry", { @@ -607,6 +630,9 @@ export class RedisSnapshotStore { local orderCount = ARGV[11] local expectedCur = ARGV[12] local casEnabled = ARGV[13] == '1' + -- The COMPLETE distinct id set. Not the order deduped: order omits every id with no batch + -- index, and those ids still have to come back on a read. + local distinctJson = ARGV[14] -- Liveness is TWO anchors: e and seq. All keys get the same PEXPIRE but expire independently -- (or seq can vanish under maxmemory eviction while e survives), so checking e alone lets a @@ -641,7 +667,7 @@ export class RedisSnapshotStore { -- The STORE mints cycleSeq, so the sequence is dense by construction and the terminal -- PEXPIRE loop from 1..c is correct. cycleSeq = redis.call('HINCRBY', seqKey, 'c', 1) - redis.call('HSET', wpKey(cycleSeq), 'order', orderJson, 'count', orderCount) + redis.call('HSET', wpKey(cycleSeq), 'order', orderJson, 'count', orderCount, 'distinct', distinctJson) if records ~= '' then redis.call('HSET', wpKey(cycleSeq), 'records', records) else @@ -736,7 +762,7 @@ export class RedisSnapshotStore { return { '0', '' } end local pointer = redis.call('HGET', eKey, id .. '#c') - return { '1', orderFor(pointer) } + return { '1', orderFor(pointer), distinctFor(pointer) } `, }); @@ -848,9 +874,16 @@ export class RedisSnapshotStore { } } -export function decodeWaitpointIds(present: boolean, orderJson: string): WaitpointIds { +export function decodeWaitpointIds( + present: boolean, + orderJson: string, + distinctJson = "" +): WaitpointIds { const order: string[] = orderJson === "" ? [] : (JSON.parse(orderJson) as string[]); - return { present, distinctIds: [...new Set(order)], order }; + // The complete set is stored separately, because `order` omits every id with no batch index. + const distinctIds: string[] = + distinctJson === "" ? [...new Set(order)] : (JSON.parse(distinctJson) as string[]); + return { present, distinctIds, order }; } declare module "@internal/redis" { @@ -873,6 +906,7 @@ declare module "@internal/redis" { orderCount: string, expectedCur: string, casEnabled: string, + distinctJson: string, callback?: Callback ): Result; readSnapshotById( diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts index 1879fe1246b..9f7832a16c6 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts @@ -242,6 +242,69 @@ describe("completed-waitpoint cycles", () => { } }); + containerTest( + "keeps a completed waitpoint that has no batch index", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + // Every wait.for, every single triggerAndWait and every token resumes with no batch index: + // the engine passes `index: b.batchIndex ?? undefined`. Postgres records the id in the + // completed-waitpoint join regardless. The ordered list cannot hold it, because its + // positions ARE the indexes, so the complete set has to be stored separately or the wait's + // result vanishes on a Redis read. + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA }], "single wait") + ); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.present).toBe(true); + expect(ids.distinctIds).toEqual([wpA]); + // No position, so it is absent from the oracle. That part is correct. + expect(ids.order).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "matches the Postgres join for a mix of indexed and index-less waits", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB, wpC] = await seedSnapshotWaitpoints(prisma, env, 3); + + const created = await decorated.createExecutionSnapshot( + resumeInput( + runId, + env, + [{ id: wpA, index: 0 }, { id: wpB }, { id: wpC, index: 1 }], + "mixed wait" + ) + ); + + // Parity with what Postgres holds is the actual requirement: the engine iterates the rows + // this set fetches, and uses the order only to assign each one its index. + const fromRedis = await redis.getSnapshotWaitpointIds(runId, created.id); + const fromPostgres = await new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + }).findSnapshotCompletedWaitpointIds(created.id, undefined, runId); + + expect([...fromRedis.distinctIds].sort()).toEqual([...fromPostgres].sort()); + expect(fromRedis.order).toEqual([wpA, wpC]); + } finally { + await redis.quit(); + } + } + ); + containerTest( "findLatestExecutionSnapshot returns the index oracle", async ({ prisma, redisOptions }) => { From 57018fcbf66ec041c26f2180fb3f94dd39b31e58 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 13:43:38 +0100 Subject: [PATCH 22/31] fix(run-store): three more index-less waitpoint losses on the read path The previous fix stored the complete id set but left three places still deriving it from the ordered list, and the ordered list holds only batch-indexed ids. A carry-forward decided on the order alone. Two DIFFERENT single waits both present an empty order, so they compared equal, the second inherited the first's cycle, and a read returned the wrong waitpoint entirely. The comparison now requires the id set to match as well. The dequeue site built its Redis refs from the ordered list while the delegate connects the complete set in Postgres, so an index-less waitpoint reached Postgres and never reached Redis. Refs are now built from the complete set, with the index taken from the ordered list where the id appears in it. The entry decode derived the set from the order too, which meant getLatest and getById returned an incomplete set. That is the hot read: findLatestExecutionSnapshot hydrates the waitpoint rows from it, so a resume would have fetched no row at all for a single wait. The read scripts now return the stored set alongside the order. Four tests, each verified against its own defect: two consecutive single waits keep separate cycles, a repeated one still carries forward, the dequeue snapshot keeps an index-less id, and the hot read hydrates its row. --- .../run-store/src/redisSnapshotStore.ts | 42 ++++--- .../src/taskRunExecutionSnapshotStore.ts | 55 ++++++-- ...utionSnapshotStore.waitpointCycles.test.ts | 118 ++++++++++++++++++ 3 files changed, 194 insertions(+), 21 deletions(-) diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index 88c18d00595..78efdc24706 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -457,11 +457,12 @@ export class RedisSnapshotStore { } const headOrder = reply[1] ?? ""; + const headDistinct = reply[2] ?? ""; const rows: SnapshotRead[] = []; // Tracks whether the Lua-chosen head row (always the first, i === 2) itself survives the // env filter below -- headOrder must never be attributed to a different, surviving row. let headSurvived = false; - for (let i = 2; i + 3 < reply.length; i += 4) { + for (let i = 3; i + 3 < reply.length; i += 4) { // orderKnown is false here: headOrder covers only the head row, resolved separately below. const decoded = this.#decode( [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], @@ -471,13 +472,17 @@ export class RedisSnapshotStore { ); if (decoded) { rows.push(decoded); - if (i === 2) headSurvived = true; + if (i === 3) headSurvived = true; } } rows.reverse(); const head = headSurvived ? rows[rows.length - 1] : undefined; - const headWaitpointIds = decodeWaitpointIds(head !== undefined, head ? headOrder : ""); + const headWaitpointIds = decodeWaitpointIds( + head !== undefined, + head ? headOrder : "", + head ? headDistinct : "" + ); if (head) { head.completedWaitpointIds = headWaitpointIds; if (head.cycle) { @@ -516,11 +521,12 @@ export class RedisSnapshotStore { if (reply === null) return { kind: "miss" }; const headOrder = reply[1] ?? ""; + const headDistinct = reply[2] ?? ""; const rows: SnapshotRead[] = []; // Tracks whether the Lua-chosen head row (always the first, i === 2) survives the env filter, // so headOrder is never attributed to a different, surviving row. let headSurvived = false; - for (let i = 2; i + 3 < reply.length; i += 4) { + for (let i = 3; i + 3 < reply.length; i += 4) { const decoded = this.#decode( [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], opts?.environmentId, @@ -529,13 +535,17 @@ export class RedisSnapshotStore { ); if (decoded) { rows.push(decoded); - if (i === 2) headSurvived = true; + if (i === 3) headSurvived = true; } } rows.reverse(); const head = headSurvived ? rows[rows.length - 1] : undefined; - const headWaitpointIds = decodeWaitpointIds(head !== undefined, head ? headOrder : ""); + const headWaitpointIds = decodeWaitpointIds( + head !== undefined, + head ? headOrder : "", + head ? headDistinct : "" + ); if (head) { head.completedWaitpointIds = headWaitpointIds; if (head.cycle) { @@ -568,7 +578,7 @@ export class RedisSnapshotStore { orderKnown: boolean ): SnapshotRead | null { if (!reply || reply.length === 0) return null; - const [id, raw, seqStr, pointer, orderJson] = reply; + const [id, raw, seqStr, pointer, orderJson, distinctJson] = reply; const entry = JSON.parse(raw) as Record; if (environmentId !== undefined && entry.environmentId !== environmentId) return null; const read: SnapshotRead = { @@ -582,7 +592,7 @@ export class RedisSnapshotStore { const [cs, count] = pointer.split(":"); read.cycle = { cycleSeq: Number(cs), count: Number(count) }; if (orderKnown) { - const ids = decodeWaitpointIds(true, orderJson); + const ids = decodeWaitpointIds(true, orderJson, distinctJson ?? ""); read.completedWaitpointIds = ids; this.#checkCycleMismatch(runId, Number(count), ids.order.length); } @@ -737,7 +747,7 @@ export class RedisSnapshotStore { local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c') if not vals[1] then return nil end -- Coerce every element: a Lua false TRUNCATES the returned array at that position. - return { id, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]) } + return { id, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]) } `, }); @@ -749,7 +759,7 @@ export class RedisSnapshotStore { if not cur then return nil end local vals = redis.call('HMGET', eKey, cur, cur .. '#s', cur .. '#c') if not vals[1] then return nil end - return { cur, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]) } + return { cur, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]) } `, }); @@ -785,7 +795,7 @@ export class RedisSnapshotStore { -- compare is a chronological compare. Walking newest-first lets the scan stop at the first -- entry at or before the cursor, which makes its length the length of the ANSWER rather -- than the length of the run's history. - local out = { '', '' } + local out = { '', '', '' } local headId = nil local offset = 0 local page = limit @@ -820,7 +830,9 @@ export class RedisSnapshotStore { end if headId then - out[2] = orderFor(redis.call('HGET', eKey, headId .. '#c')) + local headPointer = redis.call('HGET', eKey, headId .. '#c') + out[2] = orderFor(headPointer) + out[3] = distinctFor(headPointer) end return out `, @@ -852,7 +864,7 @@ export class RedisSnapshotStore { -- The head is the newest SURVIVING entry, and it is the only one whose cycle key is read. -- Deriving the order after the loop keeps it paired with the row it is attached to: a row -- dropped for a missing body must not donate its cycle data to the next one. - local out = { sinceRaw, '' } + local out = { sinceRaw, '', '' } local headId = nil for i = 1, #ids do local id = ids[i] @@ -866,7 +878,9 @@ export class RedisSnapshotStore { end end if headId then - out[2] = orderFor(redis.call('HGET', eKey, headId .. '#c')) + local headPointer = redis.call('HGET', eKey, headId .. '#c') + out[2] = orderFor(headPointer) + out[3] = distinctFor(headPointer) end return out `, diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index 4b466eca268..82d57e04a2f 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -20,7 +20,7 @@ import type { SnapshotEntryInput, SnapshotRead, } from "./redisSnapshotStore.js"; -import { deriveOrder } from "./redisSnapshotStore.js"; +import { deriveDistinctIds, deriveOrder } from "./redisSnapshotStore.js"; import { entryFromCompletion, entryFromCreateExecutionSnapshot, @@ -384,9 +384,13 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { "lockRunToWorker", entryFromLock(ctx, withStamp.snapshot), withStamp.snapshot.previousSnapshotId, - // The lock site already carries the resolved order, so the refs are rebuilt from it rather - // than re-derived: its index IS the position in that list. - withStamp.snapshot.completedWaitpointOrder.map((id, index) => ({ id, index })) + // Built from the COMPLETE id set, which is what the delegate connects in Postgres, with the + // index taken from the ordered list where the id appears in it. Building from the ordered list + // instead would drop every id with no batch index, exactly the ids Postgres still records. + lockCycleRefs( + withStamp.snapshot.completedWaitpointIds, + withStamp.snapshot.completedWaitpointOrder + ) ); return result; } @@ -579,12 +583,21 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } const order = deriveOrder(completedWaitpoints); + const distinct = deriveDistinctIds(completedWaitpoints); try { const head = await this.redis.getLatest(runId); - const previous = head?.completedWaitpointIds?.order; - - if (head?.cycle && previous && sameOrder(previous, order)) { + const previousIds = head?.completedWaitpointIds; + + // Both halves must match. Comparing the order alone is not enough: it holds only indexed ids, + // so two DIFFERENT single waits both present an empty order and would compare equal, and the + // second would inherit the first's waitpoint set instead of minting its own. + if ( + head?.cycle && + previousIds && + sameOrder(previousIds.order, order) && + sameSet(previousIds.distinctIds, distinct) + ) { return { kind: "carryForward", cycleSeq: head.cycle.cycleSeq }; } } catch (error) { @@ -899,3 +912,31 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { function sameOrder(a: string[], b: string[]): boolean { return a.length === b.length && a.every((id, index) => id === b[index]); } + +/** + * Turns the lock site's two lists into cycle refs. `completedWaitpointIds` is the complete set the + * delegate connects; `completedWaitpointOrder` gives a position only to the ids that have one, and a + * repeated id keeps each of its positions. + */ +function lockCycleRefs(ids: string[], order: string[]): { id: string; index?: number }[] { + const refs: { id: string; index?: number }[] = []; + const indexed = new Set(); + + order.forEach((id, index) => { + refs.push({ id, index }); + indexed.add(id); + }); + + for (const id of ids) { + if (!indexed.has(id)) refs.push({ id }); + } + + return refs; +} + +/** Membership only, for the id set, which has no meaningful order. */ +function sameSet(a: string[], b: string[]): boolean { + if (a.length !== b.length) return false; + const seen = new Set(a); + return b.every((id) => seen.has(id)); +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts index 9f7832a16c6..80d98aeef0e 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts @@ -305,6 +305,124 @@ describe("completed-waitpoint cycles", () => { } ); + containerTest( + "two consecutive index-less waits do not share a cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + // Neither wait has a batch index, so both present an EMPTY order. Deciding carry-forward on + // the order alone makes them compare equal, and the second silently inherits the first's + // waitpoint set: its own result is never stored and a read returns the wrong id. + const first = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA }], "first single wait") + ); + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpB }], "second single wait") + ); + + expect((await redis.getSnapshotWaitpointIds(runId, first.id)).distinctIds).toEqual([wpA]); + expect((await redis.getSnapshotWaitpointIds(runId, second.id)).distinctIds).toEqual([wpB]); + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the same index-less wait repeated does still carry forward", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + // The copy-forward case must survive the stricter comparison: the same id set, still one key. + await decorated.createExecutionSnapshot(resumeInput(runId, env, [{ id: wpA }], "wait")); + const carried = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA }], "carry") + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(1); + expect((await redis.getSnapshotWaitpointIds(runId, carried.id)).distinctIds).toEqual([wpA]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the dequeue snapshot keeps an index-less waitpoint", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + // Postgres connects completedWaitpointIds, the COMPLETE set. Building the Redis refs from + // completedWaitpointOrder instead drops every id that has no position in it. + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: undefined, + lockedToVersionId: undefined, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [wpA, wpB], + completedWaitpointOrder: [wpA], + }, + } as never); + + const ids = await redis.getSnapshotWaitpointIds(runId, snapshotId); + expect([...ids.distinctIds].sort()).toEqual([wpA, wpB].sort()); + expect(ids.order).toEqual([wpA]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "findLatestExecutionSnapshot hydrates an index-less waitpoint row", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + await decorated.createExecutionSnapshot(resumeInput(runId, env, [{ id: wpA }], "single")); + + // The hot read hydrates the rows from the id set, so an incomplete set means the resume + // gets no waitpoint at all. + const latest = await decorated.findLatestExecutionSnapshot(runId); + expect(latest!.completedWaitpoints.map((w) => w.id)).toEqual([wpA]); + } finally { + await redis.quit(); + } + } + ); + containerTest( "findLatestExecutionSnapshot returns the index oracle", async ({ prisma, redisOptions }) => { From cff7fb9535cfb15c271d6031e056627e6dbe2713 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 13:57:04 +0100 Subject: [PATCH 23/31] fix(run-store): hydrate the column the read path was omitting A sweep for values derived where they should be read found one more. The hydrated payload left out lastHeartbeatAt entirely, so a Redis-served read returned undefined for it where Postgres returns null. No code writes that column, so null is not a guess: it is the only value Postgres ever holds. The effect was small but constant, on every read served from Redis, and it is the kind of difference a comparator has to either explain or chase. Guarded by comparing the KEY SET of the two payloads rather than their values, so a column omitted by the hydrator fails as a missing key rather than passing as an absent value. Verified by removing the line again: the test names the column. Also covers the timestamp write on both schema variants. updatedAt is declared @updatedAt, which Prisma manages, so whether an explicit value survives a create is a property of the client rather than of the schema, and the two variants are separately generated clients. Agreeing declarations were not evidence. Both honour the caller's instant. --- ...ostgresRunStore.snapshotTimestamps.test.ts | 132 ++++++++++++++++++ ...askRunExecutionSnapshotStore.reads.test.ts | 23 +++ .../src/taskRunExecutionSnapshotStore.ts | 4 + 3 files changed, 159 insertions(+) create mode 100644 internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts new file mode 100644 index 00000000000..8026f2f99ec --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts @@ -0,0 +1,132 @@ +// The caller's instant must land in BOTH timestamp columns, on BOTH schema variants. +// +// `updatedAt` is declared `@updatedAt`, which Prisma manages itself, so whether an explicit value +// survives a create is a property of the client rather than of the schema. The two variants are +// separately generated clients over separately declared schemas, so agreeing declarations are not +// evidence that they agree in behaviour. This asserts it on each. +// +// It matters because the decorator writes one instant to both stores. If Prisma overrode it here, +// Postgres and Redis would hold different values for a column the comparator checks for equality, +// on every snapshot. +import { heteroPostgresTest, heteroRunOpsPostgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +/** Five minutes in the past, so a column default could never coincide with it. */ +const STAMP = new Date(Date.now() - 5 * 60 * 1000); + +async function writeSnapshot( + prisma: AnyClient, + schemaVariant: "legacy" | "dedicated", + suffix: string +) { + const scope = + schemaVariant === "dedicated" + ? { + environmentId: `env_${suffix}`, + projectId: `proj_${suffix}`, + organizationId: `org_${suffix}`, + } + : await seedLegacyScope(prisma as PrismaClient, suffix); + + const store = new PostgresRunStore({ + prisma: prisma as never, + readOnlyPrisma: prisma as never, + schemaVariant, + }); + + const runId = generateInternalId(); + const id = generateInternalId(); + + await (prisma as PrismaClient).taskRun.create({ + data: { + id: runId, + engine: "V2", + status: "PENDING", + friendlyId: `run_${suffix}`, + runtimeEnvironmentId: scope.environmentId, + environmentType: "DEVELOPMENT", + organizationId: scope.organizationId, + projectId: scope.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${suffix}`, + spanId: `span_${suffix}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + } as never, + }); + + await store.createExecutionSnapshot({ + id, + createdAt: STAMP, + run: { id: runId, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: scope.environmentId, + environmentType: "DEVELOPMENT", + projectId: scope.projectId, + organizationId: scope.organizationId, + }); + + return (prisma as PrismaClient).taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); +} + +async function seedLegacyScope(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: `dev-${suffix}`, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { + environmentId: environment.id, + projectId: project.id, + organizationId: organization.id, + }; +} + +describe("snapshot timestamps are the caller's, on both schema variants", () => { + heteroPostgresTest("legacy client honours the supplied instant", async ({ prisma14 }) => { + const row = await writeSnapshot(prisma14, "legacy", "tsleg"); + + expect(row.createdAt.toISOString()).toBe(STAMP.toISOString()); + // The one Prisma manages. If it overrode the value, the two stores would disagree here on + // every snapshot. + expect(row.updatedAt.toISOString()).toBe(STAMP.toISOString()); + }); + + heteroRunOpsPostgresTest( + "dedicated client honours the supplied instant", + async ({ prisma17 }) => { + const row = await writeSnapshot(prisma17, "dedicated", "tsded"); + + expect(row.createdAt.toISOString()).toBe(STAMP.toISOString()); + expect(row.updatedAt.toISOString()).toBe(STAMP.toISOString()); + } + ); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts index 42924e34c34..a458e9070ed 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts @@ -125,6 +125,29 @@ describe("snapshot reads", () => { } }); + containerTest( + "returns the same field set Postgres does, key for key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const fromRedis = await decorated.findLatestExecutionSnapshot(runId); + const fromPostgres = await postgresOnly.findLatestExecutionSnapshot(runId); + + // Not a value comparison: a column the hydrator forgets is absent rather than wrong, so it + // shows up as a missing KEY. lastHeartbeatAt was omitted this way and read back undefined + // where Postgres returns null, on every Redis-served read. + expect(Object.keys(fromRedis!).sort()).toEqual(Object.keys(fromPostgres!).sort()); + } finally { + await redis.quit(); + } + } + ); + containerTest( "reads a foreign environment as not found, so the caller's 404 still fires", async ({ prisma, redisOptions }) => { diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index 82d57e04a2f..11771132998 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -832,6 +832,10 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { workerId: entry.workerId ?? null, runnerId: entry.runnerId ?? null, metadata: entry.metadata ?? null, + // A column no code writes, so Postgres returns null for it on every row. The entry does not + // carry it, and omitting it here would hand back undefined where Postgres hands back null, + // on every single read served from Redis. + lastHeartbeatAt: null, completedWaitpointOrder, isValid: read.isValid, error: entry.error ?? null, From bde40b3499c8b046200579117e96465434d2df3a Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 14:12:51 +0100 Subject: [PATCH 24/31] fix(run-store): four defects from an adversarial sweep of the read path An independent pass hunting one shape, a value derived where it should be read, found these. None was reachable from a test that existed. The hot read paid a second Redis call in its most common case. An entry with no wait cycle has no waitpoints by construction, and the hydrator asked the store to confirm that rather than concluding it, on every read of a run that is not resuming from a wait. It now distinguishes the three cases and only asks when it genuinely does not know. decodeWaitpointIds still reconstructed the id set from the ordered list when the stored set was absent. That is the sixth instance of the bug fixed five times, surviving as a fallback. It is unreachable today, because both fields are written by one command, but the reconstruction is lossy by nature and the loss is silent. A missing set beside a non-empty order now reports the entry as not present, which sends the caller to Postgres. The window read checked one liveness anchor where the append script deliberately checks two and explains why. An index lost to eviction while the entry hash survived would have reported an empty hit rather than a miss, so the poll would have returned nothing new for the rest of the run's life while Postgres held the transitions. The wrapped store handle dropped the staging buffer, so a handle taken inside a transaction would have appended before the commit. No caller writes a snapshot through it today. Also restores excess-property checking on the nested snapshot writes. Routing them through a generic helper let a typo'd field name compile and fail at runtime; a concrete parameter type brings the check back at the five sites that pass a fresh literal. Verified: a bogus field is now TS2353. --- .../run-store/src/PostgresRunStore.ts | 10 ++++++++- .../run-store/src/redisSnapshotStore.ts | 22 +++++++++++++++---- .../src/taskRunExecutionSnapshotStore.ts | 12 ++++++++-- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index 0f00f5848fe..980bc6b4569 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -666,7 +666,15 @@ export class PostgresRunStore implements RunStore { * key and `undefined` alike, so spreading an empty object drops the nested write entirely rather * than sending an empty one. */ - #nestedSnapshot(create: T): { executionSnapshots: { create: T } } | Record { + #nestedSnapshot( + create: Prisma.TaskRunExecutionSnapshotUncheckedCreateWithoutRunInput + ): + | { + executionSnapshots: { + create: Prisma.TaskRunExecutionSnapshotUncheckedCreateWithoutRunInput; + }; + } + | Record { return this.snapshotWrites ? { executionSnapshots: { create } } : {}; } diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index 78efdc24706..bc447b37d25 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -785,7 +785,11 @@ export class RedisSnapshotStore { -- A run with no keyspace is a MISS, so the caller falls back to Postgres. A run that has one -- and nothing newer is an empty HIT, so it does not fall back for a window it owns. - if redis.call('EXISTS', eKey) == 0 then return nil end + -- + -- Both anchors, for the reason the append script gives: keys expire independently, and an + -- index lost to eviction while the entry hash survives would otherwise report an empty HIT + -- on every poll for the rest of the run's life, with Postgres holding the transitions. + if redis.call('EXISTS', eKey) == 0 or redis.call('EXISTS', idxKey) == 0 then return nil end -- STRICTLY greater than the cursor, and same-millisecond entries are dropped. Postgres -- serves this window with createdAt > cursor and drops them too; a Redis read that is more @@ -894,9 +898,19 @@ export function decodeWaitpointIds( distinctJson = "" ): WaitpointIds { const order: string[] = orderJson === "" ? [] : (JSON.parse(orderJson) as string[]); - // The complete set is stored separately, because `order` omits every id with no batch index. - const distinctIds: string[] = - distinctJson === "" ? [...new Set(order)] : (JSON.parse(distinctJson) as string[]); + + // The complete set is stored separately, because `order` omits every id with no batch index, so + // deduping the order to recover it silently drops every wait that has none. + // + // A cycle key always holds both fields, written by one command, so a missing `distinct` beside a + // NON-EMPTY `order` means the invariant is broken. Reconstructing from the order there would be + // the same lossy shortcut this field exists to remove, and the loss would be silent. Report the + // entry as not present instead, which sends the caller to Postgres. + if (distinctJson === "" && order.length > 0) { + return { present: false, distinctIds: [], order: [] }; + } + + const distinctIds: string[] = distinctJson === "" ? [] : (JSON.parse(distinctJson) as string[]); return { present, distinctIds, order }; } diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index 11771132998..282817b6df5 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -196,7 +196,9 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { return store; } - return this.#wrap(store); + // Carry the staging buffer through. Without it, a handle taken inside a transaction appends + // immediately, which is the exact ordering the facade exists to prevent. + return this.#wrap(store, this.staging); } /** @@ -805,8 +807,14 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { // row as the index oracle that gives each completed waitpoint its position in a batch, so it // must be populated even when the waitpoint ROWS are not fetched. Returning an empty order here // resumes every batched triggerAndWait with `index: undefined`. + // Three cases, and only the last needs a second Redis call. The read already carries the ids + // when the store decoded them. An entry with no wait cycle has no waitpoints by construction, + // which is the common case and used to cost a round trip to rediscover. Anything else asks. const ids = - read.completedWaitpointIds ?? (await this.redis.getSnapshotWaitpointIds(runId, read.id)); + read.completedWaitpointIds ?? + (read.cycle === undefined + ? { present: true, distinctIds: [], order: [] } + : await this.redis.getSnapshotWaitpointIds(runId, read.id)); const completedWaitpointOrder = ids.order; // The rows themselves are head-only, mirroring the engine's own N x M avoidance. From c4469f70d29f600fbf116009bc79eda9ee76911b Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 14:32:13 +0100 Subject: [PATCH 25/31] fix(run-store): a refused wait cycle no longer leaves an unreadable head Two paths reached the same silent hang, and neither had a test. When the store refuses a carried pointer it was still writing the entry, which then became the run's head with no pointer at all. A read of that answers present-with-nothing, and present-with-nothing is precisely the signal that tells the engine's read-repair it does not need to look, so the runner got a waitpoint-less continue and dropped it. Refusing the pointer stays right; the append now mints a fresh cycle from the refs the caller carried, in the same atomic call, so the entry always has a pointer that can be trusted. Refs are optional and only the fallback needs them, so callers that supply none keep the previous behaviour. The second path needs no refusal at all. An entry whose cycle key has gone still carries its pointer, and the read answered empty for it too. That is reachable by eviction and also by the completion expiry, which is applied to every key for a run at one moment but lets them expire independently. Reads now report such an entry as not present, which sends the caller to Postgres, where the join rows still are. The hot read and the window both fall back rather than serve it. Three tests. The refusal is driven at the store, because the decorator cannot reach it on purpose: its probe sees the id set no longer matches and mints a new cycle, so the refusal only happens when the key vanishes between probe and append. Each verified against its own defect. --- .../run-store/src/PostgresRunStore.ts | 4 +- .../run-store/src/redisSnapshotStore.ts | 101 ++++++++++++--- .../src/taskRunExecutionSnapshotStore.ts | 21 +++- ...utionSnapshotStore.waitpointCycles.test.ts | 119 ++++++++++++++++++ 4 files changed, 226 insertions(+), 19 deletions(-) diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index 980bc6b4569..007dc08092c 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -666,9 +666,7 @@ export class PostgresRunStore implements RunStore { * key and `undefined` alike, so spreading an empty object drops the nested write entirely rather * than sending an empty one. */ - #nestedSnapshot( - create: Prisma.TaskRunExecutionSnapshotUncheckedCreateWithoutRunInput - ): + #nestedSnapshot(create: Prisma.TaskRunExecutionSnapshotUncheckedCreateWithoutRunInput): | { executionSnapshots: { create: Prisma.TaskRunExecutionSnapshotUncheckedCreateWithoutRunInput; diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index bc447b37d25..1160bd3ab4f 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -175,6 +175,12 @@ export type SnapshotRead = { raw: string; cycle?: CompletedWaitpointsPointer; completedWaitpointIds?: WaitpointIds; + /** + * The entry points at a cycle key that no longer exists, so its waitpoints are unreachable rather + * than absent. A caller must not treat this as an empty set: it has to fall back to Postgres, + * which still holds the join rows. + */ + danglingCycle?: boolean; }; export type AppendResult = @@ -266,7 +272,19 @@ export class RedisSnapshotStore { completedWaitpoints: CompletedWaitpointRef[]; records?: CompletedWaitpointRecord[]; } - | { kind: "carryForward"; cycleSeq: number }; + | { + kind: "carryForward"; + cycleSeq: number; + /** + * The same refs a `new` cycle would carry. A carry the store refuses falls back to + * minting inside the same call, and it cannot do that without them: with no refs there is + * nothing to mint from, so the entry is written with no pointer, as before. + * + * Every production caller supplies them. Omitting them gives up the fallback. + */ + completedWaitpoints?: CompletedWaitpointRef[]; + records?: CompletedWaitpointRecord[]; + }; }): Promise { if (args.entry.completedWaitpoints !== undefined) { throw new Error( @@ -296,6 +314,16 @@ export class RedisSnapshotStore { } else if (args.cycle?.kind === "carryForward") { cycleMode = "carry"; cycleSeqIn = String(args.cycle.cycleSeq); + + // Carried for the refusal path only. The script uses these solely when it declines the + // pointer and mints a replacement, and can only do that when the caller supplied them. + if (args.cycle.completedWaitpoints) { + const order = deriveOrder(args.cycle.completedWaitpoints); + orderJson = JSON.stringify(order); + records = args.cycle.records ? JSON.stringify(args.cycle.records) : ""; + orderCount = String(order.length); + distinctJson = JSON.stringify(deriveDistinctIds(args.cycle.completedWaitpoints)); + } } const reply = (await this.redis.appendSnapshotEntry( @@ -423,6 +451,16 @@ export class RedisSnapshotStore { return this.#timed("getSnapshotWaitpointIds", async () => { const k = snapshotKeys(runId); const reply = await this.redis.readSnapshotWaitpointIds(k.e, k.idx, k.cur, k.seq, snapshotId); + // A dangling pointer means this entry's waitpoints are unreachable, not absent. Reporting + // `present: false` is what sends the caller to Postgres, which still holds the join rows. + if (reply[3] === "1") { + this.metrics?.recordCycleMismatch(); + this.logger.warn("RedisSnapshotStore snapshot points at a cycle key that is gone", { + runId, + snapshotId, + }); + return { present: false, distinctIds: [], order: [] }; + } return decodeWaitpointIds(reply[0] === "1", reply[1] ?? "", reply[2] ?? ""); }); } @@ -578,7 +616,7 @@ export class RedisSnapshotStore { orderKnown: boolean ): SnapshotRead | null { if (!reply || reply.length === 0) return null; - const [id, raw, seqStr, pointer, orderJson, distinctJson] = reply; + const [id, raw, seqStr, pointer, orderJson, distinctJson, dangling] = reply; const entry = JSON.parse(raw) as Record; if (environmentId !== undefined && entry.environmentId !== environmentId) return null; const read: SnapshotRead = { @@ -591,6 +629,14 @@ export class RedisSnapshotStore { if (pointer) { const [cs, count] = pointer.split(":"); read.cycle = { cycleSeq: Number(cs), count: Number(count) }; + if (dangling === "1") { + read.danglingCycle = true; + this.metrics?.recordCycleMismatch(); + this.logger.warn("RedisSnapshotStore entry points at a cycle key that is gone", { + runId, + snapshotId: id, + }); + } if (orderKnown) { const ids = decodeWaitpointIds(true, orderJson, distinctJson ?? ""); read.completedWaitpointIds = ids; @@ -614,6 +660,17 @@ export class RedisSnapshotStore { if not cs then return '' end return redis.call('HGET', wpKey(cs), 'order') or '' end + -- A pointer whose cycle key is GONE. Not the same as having no pointer: this entry should + -- have waitpoints and cannot produce them, so a read must refuse rather than answer empty. + -- Reachable by eviction, and by the completion TTL, which is applied to every key for a run + -- at the same moment but lets them expire independently. + local function danglingFor(pointer) + if not pointer then return '0' end + local cs = string.match(pointer, '^(%d+):') + if not cs then return '0' end + if redis.call('EXISTS', wpKey(cs)) == 0 then return '1' end + return '0' + end -- The complete id set, which is NOT the order deduped: order holds only batch-indexed ids. local function distinctFor(pointer) if not pointer then return '' end @@ -673,27 +730,43 @@ export class RedisSnapshotStore { local cycleSeq = 0 local mismatch = 0 - if cycleMode == 'new' then - -- The STORE mints cycleSeq, so the sequence is dense by construction and the terminal - -- PEXPIRE loop from 1..c is correct. - cycleSeq = redis.call('HINCRBY', seqKey, 'c', 1) - redis.call('HSET', wpKey(cycleSeq), 'order', orderJson, 'count', orderCount, 'distinct', distinctJson) + + -- The STORE mints cycleSeq, so the sequence is dense by construction and the terminal + -- PEXPIRE loop from 1..c is correct. + local function mintCycle() + local minted = redis.call('HINCRBY', seqKey, 'c', 1) + redis.call('HSET', wpKey(minted), 'order', orderJson, 'count', orderCount, 'distinct', distinctJson) if records ~= '' then - redis.call('HSET', wpKey(cycleSeq), 'records', records) + redis.call('HSET', wpKey(minted), 'records', records) else -- A new cycle owns the whole key: a lost seq counter can re-mint a cycleSeq whose key -- still holds another cycle's records, and order/count stay mutually consistent so the -- mismatch check cannot see it. No-op on a fresh key. - redis.call('HDEL', wpKey(cycleSeq), 'records') + redis.call('HDEL', wpKey(minted), 'records') end + return minted + end + + if cycleMode == 'new' then + cycleSeq = mintCycle() elseif cycleMode == 'carry' then - -- Attach a pointer only if this incarnation actually minted the cycle. seq can be - -- evicted while a wp: key survives, so a bare key-exists check would adopt a dead + -- Attach the CARRIED pointer only if this incarnation actually minted that cycle. seq can + -- be evicted while a wp: key survives, so a bare key-exists check would adopt a dead -- incarnation's order and records under a consistent count, invisibly. local minted = tonumber(redis.call('HGET', seqKey, 'c') or '0') local c = redis.call('HGET', wpKey(cycleSeqIn), 'count') if not c or minted < cycleSeqIn then + -- Refusing the pointer is right. Writing the entry WITHOUT one is not: it becomes the + -- head with no waitpoints, and a read of it answers present-with-nothing, which is the + -- one answer that tells the engine's repair it need not look. Mint a fresh cycle from + -- the refs the caller carried, in this same atomic call, so the entry always has a + -- pointer that can be trusted. The mismatch is still reported, for the metric. mismatch = 1 + -- Only possible when the caller carried the refs. With none there is nothing to mint + -- from, and the entry is written with no pointer, which is the older behaviour. + if distinctJson ~= '' then + cycleSeq = mintCycle() + end else cycleSeq = cycleSeqIn orderCount = c @@ -747,7 +820,7 @@ export class RedisSnapshotStore { local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c') if not vals[1] then return nil end -- Coerce every element: a Lua false TRUNCATES the returned array at that position. - return { id, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]) } + return { id, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]), danglingFor(vals[3]) } `, }); @@ -759,7 +832,7 @@ export class RedisSnapshotStore { if not cur then return nil end local vals = redis.call('HMGET', eKey, cur, cur .. '#s', cur .. '#c') if not vals[1] then return nil end - return { cur, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]) } + return { cur, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]), danglingFor(vals[3]) } `, }); @@ -772,7 +845,7 @@ export class RedisSnapshotStore { return { '0', '' } end local pointer = redis.call('HGET', eKey, id .. '#c') - return { '1', orderFor(pointer), distinctFor(pointer) } + return { '1', orderFor(pointer), distinctFor(pointer), danglingFor(pointer) } `, }); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index 282817b6df5..e96aef76172 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -577,7 +577,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { completedWaitpoints?: CompletedWaitpointRef[] ): Promise< | { kind: "new"; completedWaitpoints: CompletedWaitpointRef[] } - | { kind: "carryForward"; cycleSeq: number } + | { kind: "carryForward"; cycleSeq: number; completedWaitpoints: CompletedWaitpointRef[] } | undefined > { if (!completedWaitpoints || completedWaitpoints.length === 0) { @@ -600,7 +600,11 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { sameOrder(previousIds.order, order) && sameSet(previousIds.distinctIds, distinct) ) { - return { kind: "carryForward", cycleSeq: head.cycle.cycleSeq }; + return { + kind: "carryForward", + cycleSeq: head.cycle.cycleSeq, + completedWaitpoints, + }; } } catch (error) { // A failed probe must not lose the waitpoints. Minting a fresh cycle is the safe direction: @@ -679,6 +683,14 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); } + if (read.danglingCycle) { + // The entry says it has waitpoints and the cycle key holding them is gone. Serving it would + // hand back an empty set that looks authoritative, and the run would resume with no waits. + // Postgres still has the join rows. + this.metrics?.recordRead("findLatestExecutionSnapshot", "postgres"); + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + this.metrics?.recordRead("findLatestExecutionSnapshot", "redis"); return this.#hydrate(read, runId, client, { hydrateWaitpointRows: true }); } @@ -727,6 +739,11 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { return this.delegate.findManyExecutionSnapshots(args, client); } + if (result.entries.some((entry) => entry.danglingCycle)) { + this.metrics?.recordRead("findManyExecutionSnapshots", "postgres"); + return this.delegate.findManyExecutionSnapshots(args, client); + } + this.metrics?.recordRead("findManyExecutionSnapshots", "redis"); // The engine asks for createdAt DESC and reverses app-side; the store returns ascending. diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts index 80d98aeef0e..9f40e54c599 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts @@ -423,6 +423,125 @@ describe("completed-waitpoint cycles", () => { } ); + containerTest( + "a refused carry mints a fresh cycle rather than writing a pointerless head", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + const snapshotId = generateInternalId(); + + // Driven at the store, because the decorator cannot reach this state deliberately: its + // probe reads the head first, sees the id set no longer matches, and mints a new cycle. + // The refusal is only reachable when the key vanishes BETWEEN that probe and the append, + // which is a race. Naming a cycle this incarnation never minted reproduces the same + // refusal deterministically. + const result = await redis.append({ + entry: { + id: snapshotId, + engine: "V2", + executionStatus: "EXECUTING", + description: "carry a cycle that was never minted", + runId, + runStatus: "EXECUTING", + createdAt: new Date().toISOString(), + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + kind: "transition", + isTerminal: false, + cycle: { + kind: "carryForward", + cycleSeq: 9999, + completedWaitpoints: [{ id: wpA, index: 0 }], + }, + }); + + expect(result.outcome).toBe("written"); + if (result.outcome !== "written") return; + + // Refusing the pointer is right. Writing the entry with NO pointer is not: it becomes the + // head, and a read of it answers present-with-nothing, which is the one answer that stops + // the engine's read-repair from looking. + expect(result.cycleMismatch).toBe(true); + expect(result.cycleSeq).toBeGreaterThan(0); + + const ids = await redis.getSnapshotWaitpointIds(runId, snapshotId); + expect(ids.present).toBe(true); + expect(ids.distinctIds).toEqual([wpA]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "a dangling pointer reads as not present, not as an empty set", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA, index: 0 }], "resume") + ); + + // The entry keeps its pointer and the cycle key goes. Reachable by eviction, and by the + // completion TTL, which is set on every key for a run at once but expires them separately. + for (const key of await probe.keys(`snap:{${runId}}:wp:*`)) await probe.del(key); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + // present:false is what sends the caller to Postgres, which still holds the join rows. + // present:true with an empty set would suppress the engine's read-repair. + expect(ids.present).toBe(false); + expect(ids.distinctIds).toEqual([]); + + // And the projections the engine actually calls fall back rather than answering empty. + const withPresence = await decorated.findSnapshotCompletedWaitpointIdsWithPresence( + created.id, + undefined, + runId + ); + expect(withPresence.ids).toEqual([wpA]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the hot read falls back to Postgres when the cycle key is gone", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA, index: 0 }], "resume") + ); + for (const key of await probe.keys(`snap:{${runId}}:wp:*`)) await probe.del(key); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + // Served from Postgres, so the waitpoint is still there and the resume is not silently + // stripped of it. + expect(latest!.completedWaitpoints.map((w) => w.id)).toEqual([wpA]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + containerTest( "findLatestExecutionSnapshot returns the index oracle", async ({ prisma, redisOptions }) => { From b964f867d902e634f61efcd45285913b8bcb32aa Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 14:36:11 +0100 Subject: [PATCH 26/31] fix(run-store): ignore the read cohort at the last dial position At that position Postgres holds no snapshot rows, so a run routed away from Redis by the cohort percentage reads nothing at all. The percentage is only meaningful while both stores hold the data. Fixing it in the dial rather than documenting the constraint makes the combination unreachable, instead of leaving three settings that have to agree by convention. --- .../src/taskRunExecutionSnapshotStore.readCohort.test.ts | 9 +++++++++ .../run-store/src/taskRunExecutionSnapshotStore.ts | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts index 12ad2e5fc54..e33f7a9b253 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts @@ -46,6 +46,15 @@ describe("the read cohort", () => { expect(ids.every((id) => !store.readsFromRedis(id))).toBe(true); }); + it("ignores the dial at redis-only, whatever it is set to", () => { + // Postgres holds no snapshot rows at that position, so a run routed away from Redis reads + // nothing at all. The percentage is meaningful only while both stores hold the data. + for (const percent of [0, 1, 50, 99]) { + const store = probe("redis-only", percent); + expect(ids.every((id) => store.readsFromRedis(id))).toBe(true); + } + }); + it("gives one run the same answer every time", () => { // A run that changed store between two reads of one poll could show the log going backwards. const store = probe("redis-read", 50); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index e96aef76172..f4a46cb0414 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -655,6 +655,12 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { */ protected readsFromRedis(runId: string): boolean { if (this.mode !== "redis-read" && this.mode !== "redis-only") return false; + + // At `redis-only` the cohort dial has no meaning. Postgres holds no snapshot rows at that + // position, so a run routed away from Redis reads nothing at all. Ignoring the percentage here + // makes that misconfiguration unreachable rather than merely documented. + if (this.mode === "redis-only") return true; + if (this.readPercent >= 100) return true; if (this.readPercent <= 0) return false; From ba91f75e62bf03af37c49746e9b74329920de0dc Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 17:45:44 +0100 Subject: [PATCH 27/31] fix(run-store): make the snapshot sweep and store correct against a Redis cluster SCAN carries no key, so a cluster cannot route it: one connection iterates one node's keyspace and then reports a completed cursor. The sweep now fans out over every master, resolved per pass so a failover cannot leave it scanning a stale node list, and reports how many it covered. Rule 2 deletes a whole keyspace when the run lookup returns no row. That lookup partitions ids by residency and reads each store's replica, so an absent row is not proof of absence. Deletion now needs the keyspace to be seen absent in two separate passes, and any run found to exist clears its mark. Both window reads returned the head's waitpoint order without its dangling flag, so a head whose cycle key had expired came back with an empty order rather than falling back to Postgres, losing every position on a batched resume. Also lets both classes take a caller-built client so they can reach a cluster at all, and gives the sweep a deadline and an abort signal so a pass can stop inside its budget instead of being killed mid-cursor. --- internal-packages/redis/src/index.ts | 21 +- .../run-store/src/redisSnapshotStore.ts | 87 ++++- .../src/snapshotOrphanSweeper.cluster.test.ts | 157 ++++++++ .../src/snapshotOrphanSweeper.confirm.test.ts | 357 ++++++++++++++++++ .../src/snapshotOrphanSweeper.test.ts | 41 +- .../run-store/src/snapshotOrphanSweeper.ts | 299 +++++++++++++-- ...utionSnapshotStore.waitpointCycles.test.ts | 43 +++ 7 files changed, 952 insertions(+), 53 deletions(-) create mode 100644 internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts create mode 100644 internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts diff --git a/internal-packages/redis/src/index.ts b/internal-packages/redis/src/index.ts index 6fb30b9a4bf..622efe613fc 100644 --- a/internal-packages/redis/src/index.ts +++ b/internal-packages/redis/src/index.ts @@ -1,7 +1,24 @@ -import { Redis, type RedisOptions } from "ioredis"; +import { type Cluster, Redis, type RedisOptions } from "ioredis"; import { Logger } from "@trigger.dev/core/logger"; -export { Redis, type Callback, type RedisOptions, type Result, type RedisCommander } from "ioredis"; +export { + Redis, + Cluster, + type Callback, + type RedisOptions, + type ClusterNode, + type ClusterOptions, + type Result, + type RedisCommander, +} from "ioredis"; + +/** + * Either endpoint shape. A component that only issues key-addressed commands works against both, so + * it should accept this rather than pin itself to a standalone connection. Commands with no key — + * SCAN above all — do NOT fan out across a cluster, so anything that issues one must iterate + * `cluster.nodes("master")` itself. + */ +export type RedisClient = Redis | Cluster; /** * Reply-error -> reconnect mapping. Without this hook, an ElastiCache diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index 1160bd3ab4f..2339ab0dd5e 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -1,7 +1,7 @@ import { createRedisClient, type Callback, - type Redis, + type RedisClient, type RedisOptions, type Result, } from "@internal/redis"; @@ -205,8 +205,20 @@ export type SnapshotStoreMetrics = { recordLatency(op: string, ms: number): void; }; -export type RedisSnapshotStoreOptions = { - redisOptions: RedisOptions; +/** + * How the store reaches Redis. Exactly one of the two, enforced by the type rather than a runtime + * check: `never` on the opposite member makes both "neither" and "both" a compile error. + * + * `client` exists because production points at a Valkey/Redis CLUSTER, and cluster topology is not + * this package's business. Every command the store issues is key-addressed and every key carries a + * `{runId}` hashtag, so one slot serves a whole run and both endpoint shapes behave identically. + * A caller-supplied client is owned by the caller: `quit()` leaves it open. + */ +export type RedisSnapshotStoreConnection = + | { client: RedisClient; redisOptions?: never } + | { client?: never; redisOptions: RedisOptions }; + +export type RedisSnapshotStoreOptions = RedisSnapshotStoreConnection & { completedTtlMs: number; sinceLimit?: number; highWater?: { entryBytes?: number; cycleKeyBytes?: number; cycleCount?: number }; @@ -214,13 +226,25 @@ export type RedisSnapshotStoreOptions = { logger?: Logger; }; +/** + * Both window scripts return four leading slots before the first row: the id-cursor variant's + * `sinceRaw`, the head's order, the head's distinct set, and the head's dangling flag. Rows follow + * in four-element groups, so the head row is the group at this offset. + * + * Named because the offset drifted out of the comments describing it twice, and the second drift + * arrived in the change that fixed the first. + */ +const WINDOW_HEAD_ROW_INDEX = 4; + const SKIPPED = "skipped"; const FORKED = "forked"; const WRITTEN = "written"; const DUPLICATE = "duplicate"; export class RedisSnapshotStore { - private readonly redis: Redis; + private readonly redis: RedisClient; + /** Only a client this class opened may be closed by it. */ + private readonly ownsClient: boolean; private readonly logger: Logger; private readonly completedTtlMs: number; private readonly sinceLimit: number; @@ -234,15 +258,22 @@ export class RedisSnapshotStore { this.sinceLimit = options.sinceLimit ?? 50; this.metrics = options.metrics; this.highWater = options.highWater ?? {}; - this.redis = createRedisClient(options.redisOptions, { - onError: (error) => this.logger.error("RedisSnapshotStore redis client error", { error }), - }); + this.ownsClient = options.client === undefined; + this.redis = + options.client ?? + createRedisClient(options.redisOptions, { + onError: (error) => this.logger.error("RedisSnapshotStore redis client error", { error }), + }); this.#registerCommands(); } async quit(): Promise { // Idempotent and error-swallowing: every test calls this in a `finally`, and a double quit() // (or one after a failed connect) must never mask the real assertion failure. + // + // An injected client is the caller's. Closing it here would take down a connection shared with + // the sweeper or with another component, so a borrowed client is left open. + if (!this.ownsClient) return; if (!this.#quit) { this.#quit = this.redis.quit().then( () => undefined, @@ -496,11 +527,12 @@ export class RedisSnapshotStore { const headOrder = reply[1] ?? ""; const headDistinct = reply[2] ?? ""; + const headDangling = reply[3] ?? ""; const rows: SnapshotRead[] = []; - // Tracks whether the Lua-chosen head row (always the first, i === 2) itself survives the + // Tracks whether the Lua-chosen head row (always the first, WINDOW_HEAD_ROW_INDEX) survives the // env filter below -- headOrder must never be attributed to a different, surviving row. let headSurvived = false; - for (let i = 3; i + 3 < reply.length; i += 4) { + for (let i = WINDOW_HEAD_ROW_INDEX; i + 3 < reply.length; i += 4) { // orderKnown is false here: headOrder covers only the head row, resolved separately below. const decoded = this.#decode( [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], @@ -510,7 +542,7 @@ export class RedisSnapshotStore { ); if (decoded) { rows.push(decoded); - if (i === 3) headSurvived = true; + if (i === WINDOW_HEAD_ROW_INDEX) headSurvived = true; } } @@ -523,6 +555,13 @@ export class RedisSnapshotStore { ); if (head) { head.completedWaitpointIds = headWaitpointIds; + // A head whose cycle key has expired carries an empty order that means "unknown", not + // "none". The caller cannot distinguish those, so it has to be told, or it resumes a batch + // with every position lost. This is what makes the decorator's Postgres fallback reachable + // on the since-window path as well as the hot read. + if (headDangling === "1") { + head.danglingCycle = true; + } if (head.cycle) { this.#checkCycleMismatch(runId, head.cycle.count, headWaitpointIds.order.length); } @@ -560,11 +599,12 @@ export class RedisSnapshotStore { const headOrder = reply[1] ?? ""; const headDistinct = reply[2] ?? ""; + const headDangling = reply[3] ?? ""; const rows: SnapshotRead[] = []; - // Tracks whether the Lua-chosen head row (always the first, i === 2) survives the env filter, + // Tracks whether the Lua-chosen head row (always the first, WINDOW_HEAD_ROW_INDEX) survives the env filter, // so headOrder is never attributed to a different, surviving row. let headSurvived = false; - for (let i = 3; i + 3 < reply.length; i += 4) { + for (let i = WINDOW_HEAD_ROW_INDEX; i + 3 < reply.length; i += 4) { const decoded = this.#decode( [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], opts?.environmentId, @@ -573,7 +613,7 @@ export class RedisSnapshotStore { ); if (decoded) { rows.push(decoded); - if (i === 3) headSurvived = true; + if (i === WINDOW_HEAD_ROW_INDEX) headSurvived = true; } } @@ -586,6 +626,13 @@ export class RedisSnapshotStore { ); if (head) { head.completedWaitpointIds = headWaitpointIds; + // A head whose cycle key has expired carries an empty order that means "unknown", not + // "none". The caller cannot distinguish those, so it has to be told, or it resumes a batch + // with every position lost. This is what makes the decorator's Postgres fallback reachable + // on the since-window path as well as the hot read. + if (headDangling === "1") { + head.danglingCycle = true; + } if (head.cycle) { this.#checkCycleMismatch(runId, head.cycle.count, headWaitpointIds.order.length); } @@ -872,7 +919,7 @@ export class RedisSnapshotStore { -- compare is a chronological compare. Walking newest-first lets the scan stop at the first -- entry at or before the cursor, which makes its length the length of the ANSWER rather -- than the length of the run's history. - local out = { '', '', '' } + local out = { '', '', '', '' } local headId = nil local offset = 0 local page = limit @@ -910,6 +957,11 @@ export class RedisSnapshotStore { local headPointer = redis.call('HGET', eKey, headId .. '#c') out[2] = orderFor(headPointer) out[3] = distinctFor(headPointer) + -- The head's cycle key can expire while its entry survives: the completion TTL is applied + -- per key. Without this flag the head returns an EMPTY order and the caller cannot tell + -- that from a head that genuinely had no indexed waitpoints, so a batched resume loses + -- every position instead of falling back to Postgres. + out[4] = danglingFor(headPointer) end return out `, @@ -941,7 +993,7 @@ export class RedisSnapshotStore { -- The head is the newest SURVIVING entry, and it is the only one whose cycle key is read. -- Deriving the order after the loop keeps it paired with the row it is attached to: a row -- dropped for a missing body must not donate its cycle data to the next one. - local out = { sinceRaw, '', '' } + local out = { sinceRaw, '', '', '' } local headId = nil for i = 1, #ids do local id = ids[i] @@ -958,6 +1010,11 @@ export class RedisSnapshotStore { local headPointer = redis.call('HGET', eKey, headId .. '#c') out[2] = orderFor(headPointer) out[3] = distinctFor(headPointer) + -- The head's cycle key can expire while its entry survives: the completion TTL is applied + -- per key. Without this flag the head returns an EMPTY order and the caller cannot tell + -- that from a head that genuinely had no indexed waitpoints, so a batched resume loses + -- every position instead of falling back to Postgres. + out[4] = danglingFor(headPointer) end return out `, diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts new file mode 100644 index 00000000000..d8e70d5b3ec --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts @@ -0,0 +1,157 @@ +// Production points at a Valkey/Redis CLUSTER. SCAN carries no key, so a cluster cannot route it: +// one connection iterates ONE node's keyspace and then returns a completed cursor. A single-client +// sweep would therefore report {scanned, expired, deleted, skipped} looking exactly like a clean +// pass, having examined roughly 1/N of the keyspace, and the rest would leak with nothing left to +// revisit it. Both sweep rules close unbounded leaks, and TRI-13453 gates the rollout dial on an +// OBSERVED sweep pass, so a false green here is the worst failure this component has. +// +// Everything the sweep does after the scan is key-addressed and a cluster client routes it without +// help, so the node list is the whole of the exposure. These tests pin that decision directly. +// +// There is no Redis-cluster container fixture in the repo (@internal/testcontainers ships slot +// arithmetic, not a cluster), so the cluster cases drive a real ioredis `Cluster` object that has +// never connected and assert which method the code reaches for. That is a test of our branch, not +// a simulation of Redis. A true multi-node integration test wants a cluster fixture, and the ticket +// building the cluster client is the one placed to add it. +import { describe, expect, it } from "vitest"; +import { Cluster, Redis } from "@internal/redis"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { clientPrefixOf, scanTargetsOf, SnapshotOrphanSweeper } from "./snapshotOrphanSweeper.js"; + +/** An ioredis Cluster that is never connected. `lazyConnect` keeps the constructor from dialling. */ +function offlineCluster(options?: { keyPrefix?: string }): Cluster { + return new Cluster([{ host: "127.0.0.1", port: 7000 }], { + lazyConnect: true, + redisOptions: options?.keyPrefix ? { keyPrefix: options.keyPrefix } : undefined, + }); +} + +describe("scanTargetsOf", () => { + it("returns the one connection for a standalone client", () => { + const client = new Redis({ lazyConnect: true, port: 65000 }); + try { + expect(scanTargetsOf(client)).toEqual([client]); + } finally { + client.disconnect(); + } + }); + + it("returns every master for a cluster, and never a replica", async () => { + const cluster = offlineCluster(); + const masters = [ + new Redis({ lazyConnect: true, port: 65001 }), + new Redis({ lazyConnect: true, port: 65002 }), + new Redis({ lazyConnect: true, port: 65003 }), + ]; + const replica = new Redis({ lazyConnect: true, port: 65004 }); + + const asked: string[] = []; + // The assertion that matters: the sweep asks for "master" specifically. Asking for "all" would + // scan replicas too and act twice on one keyspace. + (cluster as unknown as { nodes: (role: string) => Redis[] }).nodes = (role: string) => { + asked.push(role); + return role === "master" ? masters : [...masters, replica]; + }; + + expect(scanTargetsOf(cluster)).toEqual(masters); + expect(asked).toEqual(["master"]); + expect(scanTargetsOf(cluster)).not.toContain(replica); + + for (const client of [...masters, replica]) client.disconnect(); + cluster.disconnect(); + }); + + it("resolves the node list per call, so a failover is picked up", () => { + const cluster = offlineCluster(); + let generation = 0; + (cluster as unknown as { nodes: () => Redis[] }).nodes = () => { + generation += 1; + return Array.from({ length: generation }, (_v, i) => new Redis({ lazyConnect: true, port: 65100 + i })); + }; + + expect(scanTargetsOf(cluster)).toHaveLength(1); + expect(scanTargetsOf(cluster)).toHaveLength(2); + cluster.disconnect(); + }); +}); + +describe("clientPrefixOf", () => { + it("reads the top-level keyPrefix on a standalone client", () => { + const client = new Redis({ lazyConnect: true, port: 65000, keyPrefix: "engine:" }); + try { + expect(clientPrefixOf(client)).toBe("engine:"); + } finally { + client.disconnect(); + } + }); + + it("reads the nested redisOptions.keyPrefix on a cluster", () => { + // On a Cluster the prefix lives under redisOptions. Reading the top level yields "", every + // SCAN MATCH then misses, and the pass reports a clean sweep of nothing. + const cluster = offlineCluster({ keyPrefix: "engine:" }); + try { + expect(clientPrefixOf(cluster)).toBe("engine:"); + } finally { + cluster.disconnect(); + } + }); + + it("is empty when no prefix is configured, for either shape", () => { + const client = new Redis({ lazyConnect: true, port: 65000 }); + const cluster = offlineCluster(); + try { + expect(clientPrefixOf(client)).toBe(""); + expect(clientPrefixOf(cluster)).toBe(""); + } finally { + client.disconnect(); + cluster.disconnect(); + } + }); +}); + +describe("SweepResult.nodes", () => { + containerTest( + "a standalone pass reports the one connection it covered", + async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore, + completedTtlMs: 72 * 60 * 60 * 1000, + }); + + try { + const result = await sweeper.sweep({ dryRun: true }); + // Without this field a pass that covered one node of six is indistinguishable from a + // complete one, which is exactly the false green the fan-out exists to prevent. + expect(result.nodes).toBe(1); + } finally { + await sweeper.quit(); + } + } + ); +}); + +describe("client ownership", () => { + containerTest("quit() leaves a borrowed client open", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const client = new Redis(redisOptions); + const sweeper = new SnapshotOrphanSweeper({ + client, + runStore, + completedTtlMs: 72 * 60 * 60 * 1000, + }); + + try { + await sweeper.quit(); + // The store and the sweep can share one cluster client. If quit() closed a client it did not + // open, the first component to shut down would take the other one's connection with it. + await client.set(`ownership:${generateInternalId()}`, "1"); + expect(await client.ping()).toBe("PONG"); + } finally { + client.disconnect(); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts new file mode 100644 index 00000000000..d617ca27834 --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts @@ -0,0 +1,357 @@ +// Rule 2 deletes a whole keyspace on the strength of "findRunsByIds returned no row for it". The +// catch in #sweepBatch covers a lookup that THROWS; it cannot see a lookup that succeeds and is +// incomplete, and a row that exists but did not come back reads exactly like a run that never +// existed. `findRunsByIds` partitions ids by residency and asks each store only for its own, and +// with no client passed it reads each store's replica — both sound today, but neither is something +// this delete path can verify. +// +// A false negative leaks keys, which is bounded and recoverable. A false positive destroys a live +// run's execution state. So deletion requires two sightings across the confirm window, and these +// tests pin that: a single pass never deletes, however old the keyspace. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore, snapshotKeys } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { SnapshotOrphanSweeper } from "./snapshotOrphanSweeper.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; +const ORPHAN_AGE_MS = 60 * 60 * 1000; + +function birthEntry(runId: string, env: SnapshotFixtureEnv, createdAt: Date) { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + return entryFromCreateRun({ id: snapshot.id, runId, createdAt }, snapshot); +} + +describe("rule 2 requires a second sighting", () => { + containerTest( + "one pass marks an orphan and deletes nothing, however old the keyspace", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A thousand times the age gate. Age is not what holds the deletion back. + const ancient = new Date(Date.now() - 1000 * ORPHAN_AGE_MS); + await store.append({ + entry: birthEntry(runId, env, ancient), + kind: "birth", + isTerminal: false, + }); + + const first = await sweeper.sweep(); + + expect(first.deleted).toBe(0); + expect(first.pendingDeletion).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + + const second = await sweeper.sweep(); + + expect(second.deleted).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a marked keyspace is not deleted until the confirm window has passed", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // An hour. Passes minutes apart must not convert a candidate. + confirmOrphanAfterMs: 60 * 60 * 1000, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + + await sweeper.sweep(); + const second = await sweeper.sweep(); + const third = await sweeper.sweep(); + + expect(second.deleted).toBe(0); + expect(second.pendingDeletion).toBe(1); + expect(third.deleted).toBe(0); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a transient miss followed by a found run does not delete, and does not leave the keyspace pre-authorised", + async ({ prisma, redisOptions }) => { + // The case the guard exists for. Pass 1 gets an incomplete answer and marks the keyspace. + // Pass 2 sees the run, so it must clear the mark: were the mark to survive, a LATER genuine + // absence would delete on its own first sighting and the two-sighting rule would be gone. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const real = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + let lie = true; + const flaky = { + ...real, + findRunsByIds: (...args: unknown[]) => + lie + ? // Succeeds and is incomplete: exactly what the catch cannot see. + Promise.resolve(new Map()) + : (real.findRunsByIds as (...rest: unknown[]) => Promise>).apply( + real, + args + ), + } as unknown as RunStore; + + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: flaky, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + // The run is alive and terminal in Postgres the whole time. Only the lookup lies. + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const first = await sweeper.sweep(); + expect(first.deleted).toBe(0); + expect(first.pendingDeletion).toBe(1); + + lie = false; + const second = await sweeper.sweep(); + expect(second.deleted).toBe(0); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + + // The run vanishes for real. With the mark cleared this is a first sighting again. + lie = true; + const third = await sweeper.sweep(); + expect(third.deleted).toBe(0); + expect(third.pendingDeletion).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + + const fourth = await sweeper.sweep(); + expect(fourth.deleted).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); + +describe("the marker cannot expire out from under a candidate", () => { + containerTest( + "a marked keyspace still carries its mark after the whole keyspace is re-read", + async ({ prisma, redisOptions }) => { + // The marker used to be a key with its own TTL derived from the confirm window, which could + // be shorter than the interval between passes: the marker written at T was gone by + // T+interval, every pass wrote a fresh one, and rule 2 deleted nothing while reporting clean. + // It is now a field on the run's `seq` hash, so it lives exactly as long as the keyspace and + // there is no lifetime left to misconfigure. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + + expect((await sweeper.sweep()).pendingDeletion).toBe(1); + + // The mark is a field on seq, and it carries no expiry of its own. + expect(await probe.hget(snapshotKeys(runId).seq, "orph")).not.toBeNull(); + expect(await probe.pttl(snapshotKeys(runId).seq)).toBe(-1); + + expect((await sweeper.sweep()).deleted).toBe(1); + // And it went with the keyspace rather than outliving it. + expect(await probe.exists(snapshotKeys(runId).seq)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); + +describe("a run seen alive clears its marker", () => { + containerTest( + "a lie, then a LIVE run, then a lie again does not delete", + async ({ prisma, redisOptions }) => { + // The hole a long marker lifetime opens. Only terminal runs used to clear the marker, so a + // keyspace marked by an incomplete lookup and then seen ALIVE kept its mark. A later genuine + // absence would then find a mature marker and delete on what is really a first sighting. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const real = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + let lie = true; + const flaky = { + ...real, + findRunsByIds: (...args: unknown[]) => + lie + ? Promise.resolve(new Map()) + : (real.findRunsByIds as (...rest: unknown[]) => Promise>).apply( + real, + args + ), + } as unknown as RunStore; + + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: flaky, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + // EXECUTING, not terminal. This run never reaches rule 1, so rule 1 cannot be what clears + // the mark; a SUSPENDED run can legitimately sit here for weeks. + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "EXECUTING" }, + }); + + expect((await sweeper.sweep()).pendingDeletion).toBe(1); + + lie = false; + const seenAlive = await sweeper.sweep(); + expect(seenAlive.skipped).toBeGreaterThan(0); + expect(seenAlive.deleted).toBe(0); + + lie = true; + const afterAlive = await sweeper.sweep(); + // A first sighting again, because being seen alive cleared the mark. + expect(afterAlive.deleted).toBe(0); + expect(afterAlive.pendingDeletion).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); + +describe("a pass can stop inside a budget", () => { + containerTest("an already-passed deadline yields a partial pass", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); + + try { + const result = await sweeper.sweep({ deadline: Date.now() - 1 }); + + // redis-worker redelivers a job that outlives its visibility timeout, and nothing extends it, + // so a pass that cannot stop on its own runs concurrently with itself. + expect(result.partial).toBe(true); + expect(result.scanned).toBe(0); + } finally { + await sweeper.quit(); + } + }); + + containerTest("an aborted signal yields a partial pass", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); + const controller = new AbortController(); + controller.abort(); + + try { + const result = await sweeper.sweep({ signal: controller.signal }); + expect(result.partial).toBe(true); + } finally { + await sweeper.quit(); + } + }); + + containerTest("a pass with budget to spare is not partial", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); + + try { + const result = await sweeper.sweep({ deadline: Date.now() + 60_000 }); + expect(result.partial).toBe(false); + } finally { + await sweeper.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts index b0d39d27e56..7d7eb2aca0d 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts @@ -44,6 +44,10 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -87,6 +91,10 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -126,6 +134,10 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -149,6 +161,7 @@ describe("SnapshotOrphanSweeper", () => { const cyclesBefore = await probe.keys(`snap:{${runId}}:wp:*`); expect(cyclesBefore.length).toBeGreaterThan(0); + await sweeper.sweep(); const result = await sweeper.sweep(); expect(result.deleted).toBe(1); @@ -171,6 +184,7 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -203,6 +217,10 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -242,6 +260,7 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -262,7 +281,11 @@ describe("SnapshotOrphanSweeper", () => { const result = await sweeper.sweep({ dryRun: true }); - expect(result.deleted).toBe(1); + // A dry pass writes no marker, so an unconfirmed rule 2 candidate reports as pending rather + // than as a deletion. That is what a real pass would do at this instant, which is the honest + // answer for a preview: nothing is confirmed yet, so nothing would be deleted yet. + expect(result.deleted).toBe(0); + expect(result.pendingDeletion).toBe(1); expect(result.expired).toBe(1); expect(await probe.exists(snapshotKeys(orphan).e)).toBe(1); expect(await probe.pttl(snapshotKeys(terminal).e)).toBe(-1); @@ -285,6 +308,10 @@ describe("SnapshotOrphanSweeper", () => { runStore: failing, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -320,6 +347,10 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -340,6 +371,7 @@ describe("SnapshotOrphanSweeper", () => { expect(await probe.exists(keys.e)).toBe(1); expect(await probe.exists(keys.cur)).toBe(0); + await sweeper.sweep(); const result = await sweeper.sweep(); expect(result.deleted).toBe(1); @@ -369,6 +401,10 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(prefixed, { onError: () => {} }); try { @@ -383,6 +419,7 @@ describe("SnapshotOrphanSweeper", () => { cycle: { kind: "new", completedWaitpoints: [{ id: "w_1", index: 0 }] }, }); + await sweeper.sweep(); const result = await sweeper.sweep(); expect(result.scanned).toBe(1); @@ -403,6 +440,7 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -417,6 +455,7 @@ describe("SnapshotOrphanSweeper", () => { }); } + await sweeper.sweep({ batchSize: 2 }); const result = await sweeper.sweep({ batchSize: 2 }); expect(result.deleted).toBe(5); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.ts index 4d3eed674b4..422217cbc84 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.ts @@ -11,7 +11,13 @@ // // Nothing schedules this. The engine's worker is what has to run it, and run-store cannot reach the // engine, so the wiring belongs to the ticket that owns production construction. -import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis"; +import { + Cluster, + createRedisClient, + type Redis, + type RedisClient, + type RedisOptions, +} from "@internal/redis"; import { Logger } from "@trigger.dev/core/logger"; import type { TaskRunStatus } from "@trigger.dev/database"; import { snapshotKeys } from "./redisSnapshotStore.js"; @@ -46,6 +52,12 @@ const DEFAULT_ORPHAN_AGE_MS = 24 * 60 * 60 * 1000; const SNAPSHOT_KEYSPACE_PREFIX = "snap:"; const DEFAULT_BATCH_SIZE = 1000; +/** + * How long a rule 2 candidate must have been marked before it may be deleted. It has to exceed the + * interval between passes, or a candidate is never sighted twice and never converts. + */ +const DEFAULT_ORPHAN_CONFIRM_MS = 60 * 60 * 1000; + export type SweepResult = { /** Keyspaces examined. */ scanned: number; @@ -55,14 +67,39 @@ export type SweepResult = { deleted: number; /** Left alone: a live run, a young orphan, or a batch whose Postgres lookup failed. */ skipped: number; -}; - -export type SnapshotOrphanSweeperOptions = { /** - * The sweep opens its own connection rather than borrowing the store's, so a long scan can never - * stall a hot-path client. + * Rule 2 candidates that were marked but not deleted, because deletion needs a second sighting + * in a later pass. A number that never converts to `deleted` means the confirm window is longer + * than the interval between passes, or the marker TTL is shorter than it. */ - redisOptions: RedisOptions; + pendingDeletion: number; + /** + * Connections the pass iterated: every master of a cluster, or 1 standalone. Reported because the + * failure this component cannot tolerate is a false green, and a pass that covered one node of + * six is indistinguishable from a complete one by any other field here. TRI-13453 gates the dial + * on an observed sweep pass, so the observation has to carry its own coverage. + */ + nodes: number; + /** True when the pass stopped early on its deadline or abort signal, so coverage is incomplete. */ + partial: boolean; +}; + +/** + * Exactly one of `redisOptions` or `client`, enforced by the type rather than a runtime check. + * With `redisOptions` the sweep opens its OWN connection, which is the preferred shape: a long + * scan can then never stall a hot-path client. `client` exists for a caller that has already built + * a client and wants the sweep to use it; a borrowed client is left open by `quit()`. + * + * What the sweep needs is a connection of its OWN, not one it built itself. A caller pointing at a + * cluster should build a SECOND, sweep-dedicated cluster client and pass it here: that keeps a long + * scan off the hot path just as well as `redisOptions` does. Handing over the client the snapshot + * store is using is the case to avoid. + */ +export type SnapshotOrphanSweeperConnection = + | { client: RedisClient; redisOptions?: never } + | { client?: never; redisOptions: RedisOptions }; + +export type SnapshotOrphanSweeperOptions = SnapshotOrphanSweeperConnection & { /** * Resolved through the run store, not a raw client. Under the run-ops split a run row can live on * either database, and only the store knows which — a raw lookup would report a live run as an @@ -71,14 +108,32 @@ export type SnapshotOrphanSweeperOptions = { runStore: RunStore; completedTtlMs: number; orphanAgeMs?: number; + /** + * How long a rule 2 candidate must stay marked before the sweep will delete it. Defaults to one + * hour. + * + * Set it at or below the interval between passes, or the second sighting arrives too early to + * count and every candidate needs three passes instead of two. It does NOT need to exceed the + * interval; the constraint people reach for ("longer than the interval") is the wrong one and + * only costs latency. + * + * There is no marker-lifetime constraint to satisfy alongside it. The marker is a field on the + * run's `seq` hash, so it lives exactly as long as the keyspace it describes: it cannot expire + * out from under a candidate that is still waiting for its second sighting, and it cannot outlive + * a keyspace that was deleted. + */ + confirmOrphanAfterMs?: number; logger?: Logger; }; export class SnapshotOrphanSweeper { - readonly #redis: Redis; + readonly #redis: RedisClient; + /** Only a client this class opened may be closed by it. */ + readonly #ownsClient: boolean; readonly #runStore: RunStore; readonly #completedTtlMs: number; readonly #orphanAgeMs: number; + readonly #confirmOrphanAfterMs: number; /** * The ioredis client-level prefix, which is NOT the keyspace prefix. ioredis prepends it to keys * for ordinary commands, but it does not prepend it to a SCAN MATCH pattern, and it does return @@ -94,13 +149,21 @@ export class SnapshotOrphanSweeper { this.#runStore = options.runStore; this.#completedTtlMs = options.completedTtlMs; this.#orphanAgeMs = options.orphanAgeMs ?? DEFAULT_ORPHAN_AGE_MS; - this.#clientPrefix = (options.redisOptions.keyPrefix as string | undefined) ?? ""; - this.#redis = createRedisClient(options.redisOptions, { - onError: (error) => this.#logger.error("SnapshotOrphanSweeper redis client error", { error }), - }); + this.#confirmOrphanAfterMs = options.confirmOrphanAfterMs ?? DEFAULT_ORPHAN_CONFIRM_MS; + this.#ownsClient = options.client === undefined; + this.#redis = + options.client ?? + createRedisClient(options.redisOptions, { + onError: (error) => + this.#logger.error("SnapshotOrphanSweeper redis client error", { error }), + }); + this.#clientPrefix = clientPrefixOf(this.#redis); } async quit(): Promise { + // A borrowed client belongs to the caller; closing it here would take down a connection the + // snapshot store may still be using. + if (!this.#ownsClient) return; if (!this.#quit) { this.#quit = this.#redis.quit().then( () => undefined, @@ -112,37 +175,93 @@ export class SnapshotOrphanSweeper { /** * One full pass over the keyspace. `dryRun` reports what it would do and changes nothing. + * + * `deadline` and `signal` let the caller stop a pass cleanly instead of having it killed + * mid-cursor. The scheduler needs this: redis-worker moves a dequeued item's score to + * `now + visibilityTimeoutMs` and nothing extends it, so a pass that outlives its timeout is + * redelivered and runs concurrently with itself. A pass that stops inside its budget cannot. + * + * Whichever way it stops, `partial` comes back true. Reporting a truncated pass as a full one is + * the same false green as under-scanning a cluster: TRI-13453 gates the dial on an OBSERVED + * sweep pass, so the observation has to say how much of the keyspace it actually reached. */ - async sweep(opts?: { batchSize?: number; dryRun?: boolean }): Promise { + async sweep(opts?: { + batchSize?: number; + dryRun?: boolean; + /** Epoch ms. The pass stops at the next batch boundary once passed. */ + deadline?: number; + signal?: AbortSignal; + }): Promise { const batchSize = opts?.batchSize ?? DEFAULT_BATCH_SIZE; const dryRun = opts?.dryRun ?? false; - const result: SweepResult = { scanned: 0, expired: 0, deleted: 0, skipped: 0 }; - - let cursor = "0"; - do { - // Match on the entry hash, not on `cur`. The append script writes `cur` only when the entry - // is valid, so a keyspace whose entries are all invalid would never be discovered and would - // leak with no expiry, which is the same unbounded leak rule 2 exists to close. `e` is - // written by every append. - const [next, keys] = await this.#redis.scan( - cursor, - "MATCH", - `${this.#clientPrefix}${SNAPSHOT_KEYSPACE_PREFIX}{*}:e`, - "COUNT", - batchSize - ); - cursor = next; + const result: SweepResult = { + scanned: 0, + expired: 0, + deleted: 0, + skipped: 0, + pendingDeletion: 0, + nodes: 0, + partial: false, + }; + + // Checked at batch boundaries only. Stopping mid-batch would leave a run half-acted-on, and a + // batch is bounded work, so the boundary is both the safe and the timely place. + const outOfBudget = () => + opts?.signal?.aborted === true || + (opts?.deadline !== undefined && Date.now() >= opts.deadline); + + // SCAN carries no key, so a cluster cannot route it: one connection iterates ONE node's + // keyspace and then reports a completed cursor. A single-client sweep against a cluster would + // therefore return a clean-looking result having examined roughly 1/N of the keyspace, and the + // rest would leak with nothing to revisit it. Both rules are unbounded leaks when missed, so + // the pass fans out over every master and only reports done when all of them are done. + const nodes = this.#scanTargets(); + result.nodes = nodes.length; + + for (const node of nodes) { + if (outOfBudget()) { + result.partial = true; + break; + } - const runIds = [...new Set(keys.map((key) => this.#runIdFrom(key)).filter(isString))]; - if (runIds.length === 0) continue; + let cursor = "0"; + do { + // Match on the entry hash, not on `cur`. The append script writes `cur` only when the entry + // is valid, so a keyspace whose entries are all invalid would never be discovered and would + // leak with no expiry, which is the same unbounded leak rule 2 exists to close. `e` is + // written by every append. + const [next, keys] = await node.scan( + cursor, + "MATCH", + `${this.#clientPrefix}${SNAPSHOT_KEYSPACE_PREFIX}{*}:e`, + "COUNT", + batchSize + ); + cursor = next; + + const runIds = [...new Set(keys.map((key) => this.#runIdFrom(key)).filter(isString))]; + if (runIds.length === 0) continue; + + await this.#sweepBatch(runIds, dryRun, result); + + if (outOfBudget()) { + // A cursor mid-iteration means this node is not finished, so the pass is not either. + result.partial = true; + break; + } + } while (cursor !== "0"); - await this.#sweepBatch(runIds, dryRun, result); - } while (cursor !== "0"); + if (result.partial) break; + } this.#logger.log("SnapshotOrphanSweeper pass complete", { ...result, dryRun }); return result; } + #scanTargets(): Redis[] { + return scanTargetsOf(this.#redis); + } + async #sweepBatch(runIds: string[], dryRun: boolean, result: SweepResult): Promise { result.scanned += runIds.length; @@ -162,6 +281,17 @@ export class SnapshotOrphanSweeper { return; } + // Every run that EXISTS clears its rule 2 marker, live ones included. It has to be every one, + // not just the terminal ones: a keyspace marked by an incomplete lookup, then seen alive, then + // missed again would otherwise present a mature marker on what is really a first sighting, and + // the two-sighting rule would be gone exactly when it was needed. This costs one DEL per + // existing run per pass, which is the price of the guard being sound rather than nearly sound. + const present = runIds.filter((runId) => rows.has(runId)); + if (!dryRun && present.length > 0) { + // Individual commands, never one pipeline: these keys span runs, so they span cluster slots. + await Promise.all(present.map((runId) => this.#clearOrphanMarker(runId))); + } + for (const runId of runIds) { const run = rows.get(runId); @@ -206,7 +336,23 @@ export class SnapshotOrphanSweeper { result.expired += 1; } - /** Rule 2: a keyspace with no run row at all, past the age threshold. */ + /** + * Rule 2: a keyspace with no run row at all, past the age threshold. + * + * TWO SIGHTINGS ARE REQUIRED. The `catch` in #sweepBatch covers a lookup that THROWS, but it + * cannot see a lookup that succeeds and is incomplete: a row that exists but did not come back + * reads exactly like a run that never existed, and the response to that is deleting a live run's + * execution state. `findRunsByIds` routes through RoutingRunStore.#findRunsByIdSet, which + * partitions ids by residency and asks each store only for its own — and with no client passed it + * reads each store's REPLICA. Both are sound today (id classification is authoritative for runs, + * and replica lag is nowhere near the 24h age gate), but each is an assumption held somewhere + * else in the codebase, not something this delete path can check. + * + * The asymmetry decides it: a false negative leaks keys, which is bounded and recoverable, while + * a false positive destroys live state. So an absent row marks the keyspace and returns; only a + * candidate still absent in a LATER pass is deleted. Any transient incomplete answer, whatever + * its cause, has to occur twice across the confirm window to do damage. + */ async #applyRuleTwo(runId: string, dryRun: boolean, result: SweepResult): Promise { const keys = await this.#allKeys(runId); if (keys.length === 0) { @@ -221,13 +367,54 @@ export class SnapshotOrphanSweeper { return; } + const seqKey = snapshotKeys(runId).seq; + const markedAtRaw = await this.#redis.hget(seqKey, ORPHAN_MARKER_FIELD); + const markedAt = markedAtRaw === null ? undefined : Number(markedAtRaw); + + if (markedAt === undefined || Number.isNaN(markedAt)) { + if (!dryRun) { + // The TTL is a multiple of the confirm window so a candidate gets several chances to be + // sighted again, while a marker left behind by a run that turned out to be alive cannot + // linger long enough to pre-authorise a later deletion. + // The field carries no TTL of its own; it lives and dies with the seq hash, which the + // keyspace's own completion expiry already governs. That removes the marker-lifetime knob + // whose derivation was wrong in the first place. + await this.#redis.hset(seqKey, ORPHAN_MARKER_FIELD, String(Date.now())); + } + result.pendingDeletion += 1; + return; + } + + if (Date.now() - markedAt < this.#confirmOrphanAfterMs) { + result.pendingDeletion += 1; + return; + } + if (!dryRun) { + // One slot: every key here carries the same `{runId}` hash tag. The marker is a field on + // `seq`, which is in `keys`, so it goes with the keyspace rather than needing its own entry. await this.#redis.del(...keys); } result.deleted += 1; } + /** + * Clears a rule 2 marker for a keyspace whose run turned out to exist after all, so a later + * genuine absence still needs its own two sightings rather than inheriting a stale one. + * + * Only called on a path that already found a run row, and only for terminal runs — a live run + * never reaches rule 2, so it can never hold a marker, and charging every live keyspace a round + * trip to prove that would cost more than the case is worth. + */ + async #clearOrphanMarker(runId: string): Promise { + try { + await this.#redis.hdel(snapshotKeys(runId).seq, ORPHAN_MARKER_FIELD); + } catch { + // Best effort. A marker that outlives its usefulness expires on its own TTL. + } + } + /** * Every key for one run: the four core keys plus each wait-cycle key. * @@ -255,7 +442,9 @@ export class SnapshotOrphanSweeper { const candidates = [core.e, core.idx, core.cur, core.seq, ...cycles]; - // One round trip for the whole set, rather than one per candidate. + // One round trip for the whole set, rather than one per candidate. Cluster-safe: every key here + // carries the same `{runId}` hash tag, so the whole pipeline lands in one slot on one node. + // The same holds for the pexpire pipeline and the multi-key DEL above. const pipeline = this.#redis.pipeline(); for (const key of candidates) { pipeline.exists(key); @@ -333,6 +522,46 @@ export class SnapshotOrphanSweeper { } } +/** + * Every connection a pass must iterate to cover the whole keyspace: each master of a cluster, or + * the one standalone connection. Replicas are excluded — they hold the same keys as their master, + * so scanning them would double-count and act on one keyspace twice. + * + * Module-level and exported so the fan-out decision can be pinned on its own. It is the whole of + * the defect this guards against: everything the sweep does AFTER the scan is key-addressed and a + * cluster client routes it correctly without help, so the node list is the only place a cluster + * can silently cost the pass coverage. + * + * Resolved per pass, never cached: cluster topology changes under failover and resharding, and a + * stale node list is the same silent under-scan this exists to prevent. + */ + +/** + * Rule 2's "seen absent once" marker is a FIELD on the run's `seq` hash, not a key of its own. + * + * As a separate key its removal depended on the deleting call site remembering to append it to the + * DEL, which is the kind of contract a later edit breaks with no test noticing: a marker outliving + * its keyspace would let a recreated keyspace be deleted on what is really a first sighting. `seq` + * is already in `#allKeys`, so as a field the marker cannot outlive the keyspace at all. + */ +const ORPHAN_MARKER_FIELD = "orph"; + +export function scanTargetsOf(client: RedisClient): Redis[] { + return client instanceof Cluster ? client.nodes("master") : [client]; +} + +/** + * The ioredis client-level prefix for either endpoint shape. On a Cluster it lives on the nested + * `redisOptions`, not on the top-level options, and reading the wrong one yields "" — which makes + * every SCAN MATCH miss and the pass report a clean sweep of nothing. + */ +export function clientPrefixOf(client: RedisClient): string { + if (client instanceof Cluster) { + return (client.options.redisOptions?.keyPrefix as string | undefined) ?? ""; + } + return (client.options.keyPrefix as string | undefined) ?? ""; +} + function isString(value: string | undefined): value is string { return typeof value === "string"; } diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts index 9f40e54c599..4bc2d0dc271 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts @@ -602,6 +602,49 @@ describe("completed-waitpoint cycles", () => { } ); + containerTest( + "the since-window falls back to Postgres when the head cycle key is gone", + async ({ prisma, redisOptions }) => { + // The hot read has always handled this. The since-window did not: its Lua returned the head's + // order and distinct set but never its dangling flag, so the decorator's fallback guard was + // dead code and an expired cycle key came back as an EMPTY order. Empty means "no indexed + // waitpoints" to the engine, which is how a batched triggerAndWait resumes with every + // position lost rather than falling back to the store that still knows them. + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const first = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [], "before the wait") + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + // The entry hash survives; only the cycle key goes. The completion TTL is applied per key, + // so this is a state the keyspace reaches on its own. + for (const key of await probe.keys(`snap:{${runId}}:wp:*`)) await probe.del(key); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + expect(window[0]!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + containerTest( "lockRunToWorker carries its resolved order into the cycle", async ({ prisma, redisOptions }) => { From f7292ac29b0aaa0dd8f33fcace860121c98cb075 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 09:17:11 +0100 Subject: [PATCH 28/31] fix(run-store): keep an omitted completedWaitpointOrder omitted on the live write Hoisting the value so the redis-only echo could reuse it also gave the live Prisma write a default it never had. Prisma omits an undefined key, and the column is nullable with no default, so the write stored an empty array where it used to store NULL. The echo needs a concrete array because its return type says so; the write does not, and now does not get one. --- .../run-store/src/PostgresRunStore.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index 5d9cb60185c..7af040eb99c 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -2020,11 +2020,18 @@ export class PostgresRunStore implements RunStore { error, } = input; - const completedWaitpointOrder = - completedWaitpoints - ?.filter((c) => c.index !== undefined) - .sort((a, b) => a.index! - b.index!) - .map((w) => w.id) ?? []; + // Left possibly-undefined ON PURPOSE. Prisma omits an undefined key, so the column keeps taking + // whatever it took before this method was touched: the schema declares `completedWaitpointOrder + // String[]` with no default and the column is nullable, so an omitted key stores NULL, not `{}`. + // Defaulting here would send `{}` instead and change what a live write stores. + // + // The redis-only echo below DOES need a concrete array, because it returns the row shape to the + // caller and that field is not nullable in the payload type. That default belongs to the echo, + // not to the write, so the two are kept apart. + const completedWaitpointOrder = completedWaitpoints + ?.filter((c) => c.index !== undefined) + .sort((a, b) => a.index! - b.index!) + .map((w) => w.id); // Redis-only: no row is written and the decorator owns the document. Echo the input in the shape // the caller expects, so every caller of this method keeps working while Postgres holds nothing. @@ -2054,7 +2061,7 @@ export class PostgresRunStore implements RunStore { workerId: workerId ?? null, runnerId: runnerId ?? null, metadata: snapshot.metadata ?? null, - completedWaitpointOrder, + completedWaitpointOrder: completedWaitpointOrder ?? [], isValid: !error, error: error ?? null, createdAt: now, From 07c3398ab40dc8593d475ef764271414a04ddad8 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 11:00:29 +0100 Subject: [PATCH 29/31] chore(run-store): apply oxfmt to the two new sweeper test files --- .../src/snapshotOrphanSweeper.cluster.test.ts | 5 ++- .../src/snapshotOrphanSweeper.confirm.test.ts | 35 ++++++++++--------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts index d8e70d5b3ec..90df7bfcc1d 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts @@ -68,7 +68,10 @@ describe("scanTargetsOf", () => { let generation = 0; (cluster as unknown as { nodes: () => Redis[] }).nodes = () => { generation += 1; - return Array.from({ length: generation }, (_v, i) => new Redis({ lazyConnect: true, port: 65100 + i })); + return Array.from( + { length: generation }, + (_v, i) => new Redis({ lazyConnect: true, port: 65100 + i }) + ); }; expect(scanTargetsOf(cluster)).toHaveLength(1); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts index d617ca27834..40c90bc4774 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts @@ -301,25 +301,28 @@ describe("a run seen alive clears its marker", () => { }); describe("a pass can stop inside a budget", () => { - containerTest("an already-passed deadline yields a partial pass", async ({ prisma, redisOptions }) => { - const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); - const sweeper = new SnapshotOrphanSweeper({ - redisOptions, - runStore: runStore as unknown as RunStore, - completedTtlMs: COMPLETED_TTL_MS, - }); + containerTest( + "an already-passed deadline yields a partial pass", + async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); - try { - const result = await sweeper.sweep({ deadline: Date.now() - 1 }); + try { + const result = await sweeper.sweep({ deadline: Date.now() - 1 }); - // redis-worker redelivers a job that outlives its visibility timeout, and nothing extends it, - // so a pass that cannot stop on its own runs concurrently with itself. - expect(result.partial).toBe(true); - expect(result.scanned).toBe(0); - } finally { - await sweeper.quit(); + // redis-worker redelivers a job that outlives its visibility timeout, and nothing extends it, + // so a pass that cannot stop on its own runs concurrently with itself. + expect(result.partial).toBe(true); + expect(result.scanned).toBe(0); + } finally { + await sweeper.quit(); + } } - }); + ); containerTest("an aborted signal yields a partial pass", async ({ prisma, redisOptions }) => { const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); From 1bba0a02cd1ee4b42bf86fdc0a8816189c49910d Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 11:31:00 +0100 Subject: [PATCH 30/31] docs(run-store): correct two stale comments on the orphan-marker path Both survived earlier revisions of the same change and now describe behaviour the code does not have. One claimed the marker has a TTL derived from the confirm window; it is a hash field with no TTL. The other claimed the marker is cleared only for terminal runs, when it is cleared for every run the lookup returned, which is what stops a live run's stale marker from pre-authorising a later deletion. Both sit on the path that deletes a keyspace, so a reader acting on either could reopen the hole the two-sighting rule closes. --- .../run-store/src/snapshotOrphanSweeper.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.ts index 422217cbc84..86bb084d5c9 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.ts @@ -373,9 +373,6 @@ export class SnapshotOrphanSweeper { if (markedAt === undefined || Number.isNaN(markedAt)) { if (!dryRun) { - // The TTL is a multiple of the confirm window so a candidate gets several chances to be - // sighted again, while a marker left behind by a run that turned out to be alive cannot - // linger long enough to pre-authorise a later deletion. // The field carries no TTL of its own; it lives and dies with the seq hash, which the // keyspace's own completion expiry already governs. That removes the marker-lifetime knob // whose derivation was wrong in the first place. @@ -403,9 +400,12 @@ export class SnapshotOrphanSweeper { * Clears a rule 2 marker for a keyspace whose run turned out to exist after all, so a later * genuine absence still needs its own two sightings rather than inheriting a stale one. * - * Only called on a path that already found a run row, and only for terminal runs — a live run - * never reaches rule 2, so it can never hold a marker, and charging every live keyspace a round - * trip to prove that would cost more than the case is worth. + * Called for EVERY run row the lookup returned, live ones included, and it has to be: a keyspace + * marked by an earlier incomplete lookup can belong to a run that is perfectly alive, and a + * SUSPENDED run can sit that way for weeks. Leaving the marker in place would let a later genuine + * absence delete on what is really a first sighting, which is the hole the two-sighting rule + * exists to close. It costs one DEL per existing run per pass; that is the price of the guard + * being sound rather than nearly sound. */ async #clearOrphanMarker(runId: string): Promise { try { From 6e976d02ce5f61568ffcc74294f8b75735bbc7da Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 11:50:10 +0100 Subject: [PATCH 31/31] feat(run-store): drop the unimplemented compare dial position, cover redis-only `compare` was a name in the dial with no behaviour behind it: it wrote and read exactly as `dual-write` does, so turning it on would have looked like enabling divergence reporting and delivered plain dual-write. A dial value that silently does something other than its name is worse than a missing one. It returns with the ticket that implements the sampled dual-read and diff. `redis-only` was the thinnest tested position and the only one that cannot be rolled back, since the snapshots written while it is on exist nowhere else. It is also the only position that is a PAIR of settings, the decorator's mode and `snapshotWrites: false` on the store beneath it, and the previous single test used a store that still wrote snapshots. Every test in the new suite builds the pair, and covers the run mutation landing without its snapshot row, transitions, completions, the absent waitpoint join rows, and every read being Redis-served. One test characterises rather than endorses: a read shape the decorator does not recognise is delegated, and at this position Postgres holds nothing, so the caller gets an empty result rather than an error. Only the engine calls that method and it issues the recognised shape, so nothing is broken today. It is pinned so the terminal-cutover ticket decides deliberately whether a fall-through here should throw instead of answering empty. --- .../taskRunExecutionSnapshotStore.off.test.ts | 2 +- ...nExecutionSnapshotStore.readCohort.test.ts | 2 +- ...unExecutionSnapshotStore.redisOnly.test.ts | 321 ++++++++++++++++++ .../src/taskRunExecutionSnapshotStore.ts | 9 +- 4 files changed, 329 insertions(+), 5 deletions(-) create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts index 4f788b5ec27..437872106e7 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts @@ -87,7 +87,7 @@ describe("TaskRunExecutionSnapshotStore at mode off", () => { it("reports every other dial position as one that writes Redis", () => { const { store } = forwardingProbe(); - const modes = ["dual-write", "compare", "redis-read", "redis-only"] as const; + const modes = ["dual-write", "redis-read", "redis-only"] as const; for (const mode of modes) { const decorated = new TaskRunExecutionSnapshotStore(store, { diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts index e33f7a9b253..58c80b4df72 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts @@ -28,7 +28,7 @@ const ids = Array.from({ length: 500 }, (_, n) => `run_cohort_${n}_${n * 7919}`) describe("the read cohort", () => { it("reads nothing from Redis before the read positions", () => { - for (const mode of ["off", "dual-write", "compare"] as const) { + for (const mode of ["off", "dual-write"] as const) { const store = probe(mode, 100); expect(ids.every((id) => !store.readsFromRedis(id))).toBe(true); } diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts new file mode 100644 index 00000000000..b308cb20988 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts @@ -0,0 +1,321 @@ +// `redis-only` is the terminal cutover, and it is the only dial position where Postgres stops being +// authoritative: it cannot be rolled back by turning the dial down, because the snapshots written +// while it was on exist nowhere else. It is also the only position that is a PAIR of settings, not +// one — the decorator's mode AND `snapshotWrites: false` on the store underneath it — and the two +// are set by different tickets. Every test here builds the pair, because testing the mode against a +// store that still writes snapshots would exercise a configuration that never ships. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWaitpoints, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +/** The shipping pair: decorator at `redis-only` over a store that writes no snapshot rows. */ +function build(prisma: never, redisOptions: never) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const reads: { method: string; source: string }[] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + snapshotWrites: false, + }) as unknown as RunStore, + { + store: redis, + mode: "redis-only", + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { decorated, redis, reads }; +} + +function birth(env: SnapshotFixtureEnv, id: string) { + return { + id, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +async function seedRun(decorated: TaskRunExecutionSnapshotStore, env: SnapshotFixtureEnv) { + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birth(env, snapshotId), + }); + return { runId, snapshotId }; +} + +function transition(runId: string, env: SnapshotFixtureEnv, description: string) { + return { + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("redis-only: Postgres stops holding snapshots", () => { + containerTest("the run row lands but no snapshot row does", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId, snapshotId } = await seedRun(decorated, env); + + // The run itself is still Postgres-authoritative at this position. Only its snapshots move. + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + + // And the snapshot is genuinely in Redis under the id the caller minted. + const head = await redis.getLatest(runId); + expect(head?.id).toBe(snapshotId); + } finally { + await redis.quit(); + } + }); + + containerTest("transitions write no snapshot row either", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + + await decorated.createExecutionSnapshot(transition(runId, env, "Run started")); + await decorated.createExecutionSnapshot(transition(runId, env, "Run continued")); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + const since = await redis.getSinceCreatedAt(runId, new Date(Date.now() - 60_000), { + limit: 50, + }); + expect(since.kind).toBe("hit"); + expect(since.kind === "hit" ? since.entries.length : 0).toBeGreaterThanOrEqual(2); + } finally { + await redis.quit(); + } + }); + + containerTest( + "a completion still updates the run row while writing no snapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + + await decorated.completeAttemptSuccess( + runId, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + id: generateInternalId(), + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + // The mutation half of a nested write must still land, or the run never finishes. + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(run.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "no completed-waitpoint join rows are written for a snapshot Postgres does not have", + async ({ prisma, redisOptions }) => { + // The join rows point at a snapshot row. With snapshot writes off there is no such row, so + // inserting them would leave links dangling at a snapshot only Redis holds. + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot({ + ...transition(runId, env, "Run resumed"), + completedWaitpoints: [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ], + }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + const joins = await prisma.$queryRawUnsafe<{ n: bigint }[]>( + `SELECT count(*) AS n FROM "_completedWaitpoints" WHERE "B" = ANY($1::text[])`, + [wpA, wpB] + ); + expect(Number(joins[0]!.n)).toBe(0); + } finally { + await redis.quit(); + } + } + ); +}); + +describe("redis-only: every read is served from Redis", () => { + containerTest( + "the hot read, the since window and the waitpoint lookups all come from Redis", + async ({ prisma, redisOptions }) => { + // At every earlier position a Redis miss falls back to Postgres and the caller never notices. + // Here Postgres holds nothing, so a read that fell back would answer empty rather than wrong, + // and a run would silently lose its state. Each read is asserted to be Redis-sourced. + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + const created = await decorated.createExecutionSnapshot({ + ...transition(runId, env, "Run resumed"), + completedWaitpoints: [{ id: wpA, index: 0 }], + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + expect(latest!.id).toBe(created.id); + expect(latest!.completedWaitpointOrder).toEqual([wpA]); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: new Date(Date.now() - 60_000) } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + expect(window.length).toBeGreaterThan(0); + + const withPresence = await decorated.findSnapshotCompletedWaitpointIdsWithPresence( + created.id, + undefined, + runId + ); + expect(withPresence.ids).toEqual([wpA]); + + expect(reads.length).toBeGreaterThan(0); + expect(reads.every((r) => r.source === "redis")).toBe(true); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "an unrecognised read shape falls through to a Postgres that holds nothing", + async ({ prisma, redisOptions }) => { + // CHARACTERISATION, NOT AN ENDORSEMENT. `findManyExecutionSnapshots` serves from Redis only + // for the since-window shape `matchSinceWindow` recognises; anything else delegates. At every + // dial position before this one that is harmless, because Postgres holds the same rows. Here + // it holds none, so the caller gets an EMPTY result rather than an error, and empty is a + // valid answer to this query. The same is true of the `miss` and `danglingCycle` fallbacks in + // that method: all three are safe everywhere except the one position that cannot fall back. + // + // Only the engine's own call shapes reach this method today, and it issues the since-window + // one, so nothing is broken. It is pinned here so the terminal-cutover ticket decides + // deliberately whether a fall-through at `redis-only` should throw instead of answering + // empty, rather than discovering this shape in production. + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(transition(runId, env, "Run started")); + + // No `createdAt` cursor, so the shape does not match and the read is delegated. + const unmatched = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + expect(unmatched).toEqual([]); + + // The same run, asked the shape the engine actually issues, answers in full from Redis. + const matched = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: new Date(Date.now() - 60_000) } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + expect(matched.length).toBeGreaterThan(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "the read cohort dial cannot route a run away from Redis", + async ({ prisma, redisOptions }) => { + // readPercent is a ramp control for `redis-read`. At `redis-only` a run routed to Postgres + // would read a database that holds no snapshots at all, so the dial must be ignored here + // whatever it is set to. + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const reads: string[] = []; + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ + prisma: prisma as never, + readOnlyPrisma: prisma as never, + snapshotWrites: false, + }) as unknown as RunStore, + { + store: redis, + mode: "redis-only", + readPercent: 0, + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (_m, source) => reads.push(source), + }, + } + ); + + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId, snapshotId } = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest!.id).toBe(snapshotId); + expect(reads).not.toContain("postgres"); + } finally { + await redis.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index f4a46cb0414..c6bc145a91d 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -60,10 +60,13 @@ const WAITPOINT_CHUNK_SIZE = 100; * The rollout dial. Postgres stays fully written and authoritative in every position before * `redis-only`, so every earlier position rolls back losslessly by turning the dial down. * - * `compare` writes exactly as `dual-write` does. Its sampled dual-read and diff are a later ticket; - * the position is named here so the dial does not have to widen once that lands. + * A `compare` position was named here before its behaviour existed, and it read from this type as a + * real dial position while behaving in every respect exactly like `dual-write`. A dial value that + * silently does something other than its name is worse than a missing one: turning it on would have + * looked like enabling divergence reporting and delivered plain dual-write. It is added back by the + * ticket that implements the sampled dual-read and diff, at which point the name will be true. */ -export type SnapshotStoreMode = "off" | "dual-write" | "compare" | "redis-read" | "redis-only"; +export type SnapshotStoreMode = "off" | "dual-write" | "redis-read" | "redis-only"; /** * Enqueues the existing `repairSnapshot` job for a run whose append was lost. The decorator lives in