From f4127406985d4a678650f7647f05aa144ea165d9 Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Thu, 5 Nov 2026 19:51:58 +0400 Subject: [PATCH 1/7] fix(workflows): save overlapping join history in order --- packages/workflows/src/context.ts | 4 + packages/workflows/src/storage.ts | 133 +++-- .../tests/history-persistence.test.ts | 501 ++++++++++++++++++ 3 files changed, 605 insertions(+), 33 deletions(-) create mode 100644 packages/workflows/tests/history-persistence.test.ts diff --git a/packages/workflows/src/context.ts b/packages/workflows/src/context.ts index 7d0871e..05572af 100644 --- a/packages/workflows/src/context.ts +++ b/packages/workflows/src/context.ts @@ -919,6 +919,7 @@ export class WorkflowContextImpl implements WorkflowContextInterface { metadata.status = "completed"; metadata.error = undefined; metadata.completedAt = Date.now(); + metadata.dirty = true; // Ephemeral steps don't trigger an immediate flush. This avoids the // synchronous write overhead for transient operations. Note that the @@ -945,6 +946,9 @@ export class WorkflowContextImpl implements WorkflowContextInterface { entry.kind.data.error = String(error); } entry.dirty = true; + // A sibling branch can save this step's running metadata while the + // callback runs, so the failure status must be saved again. + metadata.dirty = true; // Timeout errors are treated as critical by default. Steps opt // into retrying on timeout with retryOnTimeout: true. diff --git a/packages/workflows/src/storage.ts b/packages/workflows/src/storage.ts index 3073c55..8d121a3 100644 --- a/packages/workflows/src/storage.ts +++ b/packages/workflows/src/storage.ts @@ -229,12 +229,64 @@ export interface PendingDeletions { ranges: { start: Uint8Array; end: Uint8Array }[]; } +interface FlushQueue { + /** Settles when every save started or queued so far has finished. */ + tail: Promise; + /** The next save, while it has not started. Other callers can join it. */ + waiting?: Promise; +} + +// Saves of one workflow run one at a time, so an older snapshot never lands +// after a newer one. +const flushQueues = new WeakMap(); + +/** + * Save workflow history in order, preserving mutations made during each write. + * + * A save that has not started reads storage when it starts, so it also saves + * the changes of callers that join it. They share its result. Saves with + * deletions do not share, so pruning runs in order with the writes around it. + */ +export function flush( + storage: Storage, + driver: EngineDriver, + onHistoryUpdated?: () => void, + pendingDeletions?: PendingDeletions, +): Promise { + let queue = flushQueues.get(storage); + if (!queue) { + queue = { tail: Promise.resolve() }; + flushQueues.set(storage, queue); + } + if (queue.waiting && !pendingDeletions) { + return queue.waiting; + } + + const activeQueue = queue; + const next: Promise = activeQueue.tail.then(() => { + if (activeQueue.waiting === next) { + activeQueue.waiting = undefined; + } + return flushSnapshot( + storage, + driver, + onHistoryUpdated, + pendingDeletions, + ); + }); + if (!pendingDeletions) { + activeQueue.waiting = next; + } + activeQueue.tail = next.catch(() => undefined); + return next; +} + /** * Flush all dirty data to the driver. Optionally includes pending * deletions so that history pruning happens alongside the * state write. */ -export async function flush( +async function flushSnapshot( storage: Storage, driver: EngineDriver, onHistoryUpdated?: () => void, @@ -317,46 +369,61 @@ export async function flush( }); } - if (writes.length > 0) { - if (driver.atomicBatch) { - await driver.batch(writes); - } else { - for (const chunk of splitBatchWrites(writes)) { - await driver.batch(chunk); + // Record what this write contains before awaiting it. A concurrent branch + // can change storage during the write, and those changes must stay dirty. + const savedNameCount = storage.nameRegistry.length; + const savedState = storage.state; + const savedOutput = storage.output; + const savedError = storage.error; + for (const entry of dirtyEntries) { + entry.dirty = false; + } + for (const metadata of dirtyMetadata) { + metadata.dirty = false; + } + try { + if (writes.length > 0) { + if (driver.atomicBatch) { + await driver.batch(writes); + } else { + for (const chunk of splitBatchWrites(writes)) { + await driver.batch(chunk); + } } } - } - // Apply pending deletions after the batch write. These are collected - // by collectLoopPruning so pruning happens alongside the state write. - if (pendingDeletions) { - const deleteOps: Promise[] = []; - for (const prefix of pendingDeletions.prefixes) { - deleteOps.push(driver.deletePrefix(prefix)); - } - for (const range of pendingDeletions.ranges) { - deleteOps.push(driver.deleteRange(range.start, range.end)); + // Apply pending deletions after the batch write. These are collected + // by collectLoopPruning so pruning happens alongside the state write. + if (pendingDeletions) { + const deleteOps: Promise[] = []; + for (const prefix of pendingDeletions.prefixes) { + deleteOps.push(driver.deletePrefix(prefix)); + } + for (const range of pendingDeletions.ranges) { + deleteOps.push(driver.deleteRange(range.start, range.end)); + } + for (const key of pendingDeletions.keys) { + deleteOps.push(driver.delete(key)); + } + if (deleteOps.length > 0) { + await Promise.all(deleteOps); + historyUpdated = true; + } } - for (const key of pendingDeletions.keys) { - deleteOps.push(driver.delete(key)); + } catch (error) { + for (const entry of dirtyEntries) { + entry.dirty = true; } - if (deleteOps.length > 0) { - await Promise.all(deleteOps); - historyUpdated = true; + for (const metadata of dirtyMetadata) { + metadata.dirty = true; } + throw error; } - // Update flushed tracking after successful write - for (const entry of dirtyEntries) { - entry.dirty = false; - } - for (const metadata of dirtyMetadata) { - metadata.dirty = false; - } - storage.flushedNameCount = storage.nameRegistry.length; - storage.flushedState = storage.state; - storage.flushedOutput = storage.output; - storage.flushedError = storage.error; + storage.flushedNameCount = savedNameCount; + storage.flushedState = savedState; + storage.flushedOutput = savedOutput; + storage.flushedError = savedError; if (historyUpdated && onHistoryUpdated) { onHistoryUpdated(); diff --git a/packages/workflows/tests/history-persistence.test.ts b/packages/workflows/tests/history-persistence.test.ts new file mode 100644 index 0000000..7e7efe8 --- /dev/null +++ b/packages/workflows/tests/history-persistence.test.ts @@ -0,0 +1,501 @@ +import { describe, expect, it } from "vitest"; +import { + InMemoryDriver, + type KVWrite, + Loop, + loadStorage, + runWorkflow, + type WorkflowContextInterface, +} from "../src/testing.js"; + +// Holds one history write open so a test can change workflow state while +// that write is in flight. +class HeldWriteDriver extends InMemoryDriver { + holdNext = false; + failNext = false; + readonly writeStarted = Promise.withResolvers(); + readonly releaseWrite = Promise.withResolvers(); + readonly heldWriteFinished = Promise.withResolvers(); + + override async batch(writes: KVWrite[]): Promise { + if (this.holdNext) { + this.holdNext = false; + this.writeStarted.resolve(); + await this.releaseWrite.promise; + await super.batch(writes); + this.heldWriteFinished.resolve(); + return; + } + if (this.failNext) { + this.failNext = false; + throw new Error("write failed"); + } + await super.batch(writes); + } +} + +function newDriver(): HeldWriteDriver { + const driver = new HeldWriteDriver(); + driver.latency = 0; + driver.workerPollInterval = 0; + return driver; +} + +async function savedStep(driver: InMemoryDriver, key: string) { + const storage = await loadStorage(driver); + const entry = storage.history.entries.get(key); + if (entry?.kind.type !== "step") throw new Error(`no step at ${key}`); + return { + output: entry.kind.data.output, + error: entry.kind.data.error, + status: storage.entryMetadata.get(entry.id)?.status, + }; +} + +// Wakes a workflow parked on ctx.queue.next("wake") and runs it again. +async function wakeAndRerun( + driver: InMemoryDriver, + workflow: (ctx: WorkflowContextInterface) => Promise, +) { + await driver.messageDriver.addMessage({ + id: "wake", + name: "wake", + data: {}, + sentAt: Date.now(), + }); + return await runWorkflow("wf", workflow, undefined, driver).result; +} + +describe("history saved while join branches overlap", () => { + it("reloads after a branch starts a new step while a sibling's save is in flight", async () => { + const driver = newDriver(); + driver.latency = 50; + + const result = await runWorkflow( + "wf", + async (ctx) => + await ctx.join("parallel", { + first: { + run: async (branch) => { + await branch.step("fetch-first", async () => 1); + return await branch.step("save-first", async () => { + await new Promise((r) => setTimeout(r, 100)); + return 2; + }); + }, + }, + second: { + run: async (branch) => + await branch.step("fetch-second", async () => { + await new Promise((r) => setTimeout(r, 25)); + return 3; + }), + }, + }), + undefined, + driver, + ).result; + + expect(result).toEqual({ + state: "completed", + output: { first: 2, second: 3 }, + }); + await expect(loadStorage(driver)).resolves.toBeDefined(); + }); + + it("replays after a wake without re-running finished steps, then runs the next step", async () => { + const driver = newDriver(); + const runs: string[] = []; + const workflow = async (ctx: WorkflowContextInterface) => { + const joined = await ctx.join("jobs", { + a: { + run: async (branch) => { + await branch.step("first", async () => { + runs.push("first"); + driver.holdNext = true; + return "a"; + }); + return "a"; + }, + }, + b: { + run: async (branch) => { + // Starts after the sibling's save opens, so this step's name + // is registered during that save. + await driver.writeStarted.promise; + return await branch.step("second", async () => { + runs.push("second"); + // Finish the sibling's save before this step saves. + driver.releaseWrite.resolve(); + await driver.heldWriteFinished.promise; + await new Promise((resolve) => + setImmediate(resolve), + ); + return "b"; + }); + }, + }, + }); + await ctx.queue.next("wait", { names: ["wake"] }); + const after = await ctx.step("after-wake", async () => { + runs.push("after-wake"); + return "woke"; + }); + return { joined, after }; + }; + + const first = await runWorkflow("wf", workflow, undefined, driver) + .result; + expect(first.state).toBe("sleeping"); + + const resumed = await wakeAndRerun(driver, workflow); + + expect({ resumed, runs }).toEqual({ + resumed: { + state: "completed", + output: { joined: { a: "a", b: "b" }, after: "woke" }, + }, + runs: ["first", "second", "after-wake"], + }); + }); + + it("does not re-run a finished step when its branch resumes after a wake", async () => { + const driver = newDriver(); + const runs: string[] = []; + const workflow = async (ctx: WorkflowContextInterface) => + await ctx.join("jobs", { + a: { + run: async (branch) => { + const output = await branch.step("slow", async () => { + runs.push("slow"); + await driver.writeStarted.promise; + return "a"; + }); + // Keeps the branch open, so a wake replays its steps. + await branch.queue.next("wait", { names: ["wake"] }); + return output; + }, + }, + b: { + run: (branch) => + branch.step("record", async () => { + runs.push("record"); + driver.holdNext = true; + return "b"; + }), + }, + }); + + const handle = runWorkflow("wf", workflow, undefined, driver); + await driver.writeStarted.promise; + // Let branch a finish its step and save it behind the held write. + await new Promise((resolve) => setImmediate(resolve)); + driver.releaseWrite.resolve(); + expect((await handle.result).state).toBe("sleeping"); + + const resumed = await wakeAndRerun(driver, workflow); + + expect({ resumed, runs }).toEqual({ + resumed: { state: "completed", output: { a: "a", b: "b" } }, + runs: ["slow", "record"], + }); + }); + + it("keeps a completed step completed when an older save finishes later", async () => { + const driver = newDriver(); + const workflow = (ctx: WorkflowContextInterface) => + ctx.join("jobs", { + a: { + run: (branch) => + branch.step("work", async () => { + driver.holdNext = true; + await driver.writeStarted.promise; + return "a"; + }), + }, + b: { + run: (branch) => branch.step("record", async () => "b"), + }, + }); + const result = runWorkflow("wf", workflow, undefined, driver).result; + await driver.writeStarted.promise; + // Let branch a complete and queue its save behind the held one. + await new Promise((resolve) => setImmediate(resolve)); + driver.releaseWrite.resolve(); + + expect(await result).toEqual({ + state: "completed", + output: { a: "a", b: "b" }, + }); + expect(await savedStep(driver, "jobs/a/work")).toMatchObject({ + status: "completed", + }); + }); + + it("saves a step failure that happens after a sibling saved the step as running", async () => { + const driver = newDriver(); + const siblingSaved = Promise.withResolvers(); + const workflow = (ctx: WorkflowContextInterface) => + ctx.join("jobs", { + a: { + run: (branch) => + branch.step({ + name: "work", + maxRetries: 0, + run: async () => { + await siblingSaved.promise; + throw new Error("down"); + }, + }), + }, + b: { + run: async (branch) => { + await branch.step("record", async () => "b"); + siblingSaved.resolve(); + }, + }, + }); + + await expect( + runWorkflow("wf", workflow, undefined, driver).result, + ).rejects.toThrow(); + + expect(await savedStep(driver, "jobs/a/work")).toMatchObject({ + error: expect.stringContaining("down"), + status: "exhausted", + }); + }); + + it("saves a step failure that happens while a sibling's save is in flight", async () => { + const driver = newDriver(); + const workflow = (ctx: WorkflowContextInterface) => + ctx.join("jobs", { + a: { + run: (branch) => + branch.step({ + name: "work", + maxRetries: 0, + run: async () => { + await driver.writeStarted.promise; + throw new Error("down"); + }, + }), + }, + b: { + run: async (branch) => { + await branch.step("record", async () => { + driver.holdNext = true; + return "b"; + }); + }, + }, + }); + + const result = runWorkflow("wf", workflow, undefined, driver).result; + await driver.writeStarted.promise; + // Let branch a fail while branch b's save is still open. + await new Promise((resolve) => setImmediate(resolve)); + driver.releaseWrite.resolve(); + await expect(result).rejects.toThrow(); + + expect(await savedStep(driver, "jobs/a/work")).toMatchObject({ + error: expect.stringContaining("down"), + status: "exhausted", + }); + }); + + it("keeps a finished step's result when the save carrying it fails", async () => { + const driver = newDriver(); + const workflow = async (ctx: WorkflowContextInterface) => { + // Ephemeral, so the next step's save is the one that carries it. + await ctx.step({ + name: "quiet", + ephemeral: true, + run: async () => "kept", + }); + await ctx.step("work", async () => { + driver.failNext = true; + }); + }; + + const result = await runWorkflow("wf", workflow, undefined, driver) + .result; + + expect(result.state).toBe("sleeping"); + expect(await savedStep(driver, "quiet")).toMatchObject({ + output: "kept", + status: "completed", + }); + }); + + it("does not confirm a step until its result is saved", async () => { + const driver = newDriver(); + let savedWhenConfirmed: unknown; + const workflow = (ctx: WorkflowContextInterface) => + ctx.join("jobs", { + a: { + run: (branch) => + branch.step("held", async () => { + driver.holdNext = true; + return "a"; + }), + }, + b: { + run: async (branch) => { + await driver.writeStarted.promise; + const output = await branch.step("late", async () => { + // Finish while the held save is still open, then let it close. + setImmediate(() => driver.releaseWrite.resolve()); + return "b"; + }); + savedWhenConfirmed = ( + await savedStep(driver, "jobs/b/late") + ).output; + return output; + }, + }, + }); + + await runWorkflow("wf", workflow, undefined, driver).result; + + expect(savedWhenConfirmed).toBe("b"); + }); + + it("keeps every branch's result when a save that several branches wait on fails", async () => { + const driver = newDriver(); + const runs: string[] = []; + const waitingStep = (name: string) => ({ + run: async (branch: WorkflowContextInterface) => { + await driver.writeStarted.promise; + return await branch.step(name, async () => { + runs.push(name); + return name; + }); + }, + }); + const workflow = (ctx: WorkflowContextInterface) => + ctx.join("jobs", { + a: { + run: (branch) => + branch.step("held", async () => { + runs.push("held"); + driver.holdNext = true; + return "held"; + }), + }, + b: waitingStep("b"), + c: waitingStep("c"), + d: waitingStep("d"), + }); + + const first = runWorkflow("wf", workflow, undefined, driver).result; + await driver.writeStarted.promise; + // Let the other branches finish and wait behind the held save. + await new Promise((resolve) => setImmediate(resolve)); + driver.failNext = true; + driver.releaseWrite.resolve(); + expect((await first).state).toBe("sleeping"); + + const resumed = await runWorkflow("wf", workflow, undefined, driver) + .result; + + expect({ resumed, runs: runs.sort() }).toEqual({ + resumed: { + state: "completed", + output: { a: "held", b: "b", c: "c", d: "d" }, + }, + runs: ["b", "c", "d", "held"], + }); + }); + + it("removes pruned loop iterations when the prune overlaps a sibling's save", async () => { + const driver = newDriver(); + const siblingsMayStart = Promise.withResolvers(); + const workflow = (ctx: WorkflowContextInterface) => + ctx.join("jobs", { + held: { + run: async (branch) => { + await siblingsMayStart.promise; + await branch.step("held", async () => { + driver.holdNext = true; + }); + }, + }, + sibling: { + run: async (branch) => { + await driver.writeStarted.promise; + await branch.step("sibling", async () => {}); + }, + }, + looping: { + run: (branch) => + branch.loop({ + name: "loop", + state: 0, + historyPruneInterval: 2, + historySize: 1, + run: async (iteration, i) => { + // Saved before the prune, so the prune must delete it. + await iteration.step({ + name: "work", + ephemeral: i > 0, + run: async () => i, + }); + if (i === 1) { + // The prune starts behind the held save and the sibling's save. + siblingsMayStart.resolve(); + await driver.writeStarted.promise; + await new Promise((resolve) => + setImmediate(resolve), + ); + setImmediate(() => + driver.releaseWrite.resolve(), + ); + } + return i === 3 + ? Loop.break("done") + : Loop.continue(i + 1); + }, + }), + }, + }); + + await runWorkflow("wf", workflow, undefined, driver).result; + + const storage = await loadStorage(driver); + const savedIterations = [...storage.history.entries.keys()].filter( + (key) => key.endsWith("/work"), + ); + expect(savedIterations).toEqual(["jobs/looping/loop/~3/work"]); + }); + + it("does not stall a workflow when another workflow's save is in flight", async () => { + const stuck = newDriver(); + const other = newDriver(); + const stuckResult = runWorkflow( + "stuck", + (ctx) => + ctx.step("work", async () => { + stuck.holdNext = true; + return "stuck"; + }), + undefined, + stuck, + ).result; + await stuck.writeStarted.promise; + + const otherResult = await runWorkflow( + "other", + (ctx) => ctx.step("work", async () => "other"), + undefined, + other, + ).result; + expect(otherResult).toEqual({ state: "completed", output: "other" }); + + stuck.releaseWrite.resolve(); + await expect(stuckResult).resolves.toEqual({ + state: "completed", + output: "stuck", + }); + }); +}); From 2b09f96387128e413eef79f6cc8a9f1602d8f98f Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Sat, 7 Nov 2026 16:14:33 +0400 Subject: [PATCH 2/7] chore(workflows): fix format check and remove a dangling docs link --- docs/AGENTS.md | 1 - examples/docs/onboarding/index.ts | 11 ++++------- 2 files changed, 4 insertions(+), 8 deletions(-) delete mode 120000 docs/AGENTS.md diff --git a/docs/AGENTS.md b/docs/AGENTS.md deleted file mode 120000 index 681311e..0000000 --- a/docs/AGENTS.md +++ /dev/null @@ -1 +0,0 @@ -CLAUDE.md \ No newline at end of file diff --git a/examples/docs/onboarding/index.ts b/examples/docs/onboarding/index.ts index 5747c13..f40bf66 100644 --- a/examples/docs/onboarding/index.ts +++ b/examples/docs/onboarding/index.ts @@ -35,18 +35,15 @@ export const order = workflow({ actions: { // Read which steps have finished so far. getProgress: async (c) => { - return (await c.db.execute( - "SELECT step FROM progress ORDER BY id", - )) as { step: string }[]; + return (await c.db.execute("SELECT step FROM progress ORDER BY id")) as { + step: string; + }[]; }, }, }); // Write one progress row to this order's database. -async function record( - step: WorkflowStepContextOf, - name: string, -) { +async function record(step: WorkflowStepContextOf, name: string) { await step.db.execute("INSERT INTO progress (step) VALUES (?)", name); } From d4ca1bfab12cd0f0bb112f23ac9505ffe984f4fc Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Thu, 5 Nov 2026 19:54:24 +0400 Subject: [PATCH 3/7] fix(workflows): mark settled history as visited on replay --- packages/workflows/src/context.ts | 90 +++++- packages/workflows/tests/race.test.ts | 57 ++++ .../workflows/tests/replay-scopes.test.ts | 265 ++++++++++++++++++ 3 files changed, 399 insertions(+), 13 deletions(-) create mode 100644 packages/workflows/tests/replay-scopes.test.ts diff --git a/packages/workflows/src/context.ts b/packages/workflows/src/context.ts index 05572af..7102d8e 100644 --- a/packages/workflows/src/context.ts +++ b/packages/workflows/src/context.ts @@ -498,6 +498,48 @@ export class WorkflowContextImpl implements WorkflowContextInterface { this.visitedKeys.add(key); } + /** + * Mark all retained history under `location` as visited. + * + * Replay does not re-enter a settled scope, but its entries stay in history. + * Without this, an enclosing validateComplete or getVersion treats them as + * leftovers from changed workflow code. + */ + private markSettledHistoryVisited(location: Location): void { + for (const [key, entry] of this.storage.history.entries) { + if (isLocationPrefix(location, entry.location)) { + this.markVisited(key); + } + } + } + + /** + * Mark the retained history of a resumed loop's finished iterations as + * visited, because the loop continues from `resumeIteration` without + * replaying them. + */ + private markFinishedIterationsVisited( + loopLocation: Location, + resumeIteration: number, + ): void { + const loopSegment = loopLocation[loopLocation.length - 1]; + for (const [key, entry] of this.storage.history.entries) { + if (!isLocationPrefix(loopLocation, entry.location)) { + continue; + } + + const iterationSegment = entry.location[loopLocation.length]; + if ( + iterationSegment && + typeof iterationSegment !== "number" && + iterationSegment.loop === loopSegment && + iterationSegment.iteration < resumeIteration + ) { + this.markVisited(key); + } + } + } + /** * Check if a name has already been used at the current location in this execution. * Throws HistoryDivergedError if duplicate detected. @@ -1174,6 +1216,13 @@ export class WorkflowContextImpl implements WorkflowContextInterface { existing.id, ); + if ( + metadata.status === "completed" || + loopData.output !== undefined + ) { + this.markSettledHistoryVisited(location); + } + if (rollbackMode) { if (loopData.output !== undefined) { return loopData.output as T; @@ -1196,6 +1245,7 @@ export class WorkflowContextImpl implements WorkflowContextInterface { entry = existing; state = loopData.state as S; iteration = loopData.iteration; + this.markFinishedIterationsVisited(location, iteration); if (rollbackMode) { rollbackOutput = loopData.output as T | undefined; rollbackIterationRan = rollbackOutput !== undefined; @@ -2089,24 +2139,27 @@ export class WorkflowContextImpl implements WorkflowContextInterface { ); } + const branchLocation = appendName( + this.storage, + location, + branchName, + ); + // Already completed if (branchStatus.status === "completed") { + this.markSettledHistoryVisited(branchLocation); results[branchName] = branchStatus.output; return; } // Already failed if (branchStatus.status === "failed") { + this.markSettledHistoryVisited(branchLocation); errors[branchName] = new Error(branchStatus.error); return; } // Execute branch - const branchLocation = appendName( - this.storage, - location, - branchName, - ); const branchCtx = this.createBranch(branchLocation); branchStatus.status = "running"; @@ -2249,6 +2302,7 @@ export class WorkflowContextImpl implements WorkflowContextInterface { // Check if we already have a winner const raceKind = existing.kind; if (raceKind.data.winner !== null) { + this.markSettledHistoryVisited(location); const winnerStatus = raceKind.data.branches[raceKind.data.winner]; return { @@ -2325,6 +2379,7 @@ export class WorkflowContextImpl implements WorkflowContextInterface { // If we found a replay winner, return immediately if (hasWinner && winnerName !== null) { + this.markSettledHistoryVisited(location); return { winner: winnerName, value: winnerValue as T }; } @@ -2337,19 +2392,21 @@ export class WorkflowContextImpl implements WorkflowContextInterface { ); } + const branchLocation = appendName( + this.storage, + location, + branch.name, + ); + // Skip already completed/cancelled if ( branchStatus.status !== "pending" && branchStatus.status !== "running" ) { + this.markSettledHistoryVisited(branchLocation); continue; } - const branchLocation = appendName( - this.storage, - location, - branch.name, - ); const branchCtx = this.createBranch( branchLocation, raceAbortController, @@ -2359,7 +2416,15 @@ export class WorkflowContextImpl implements WorkflowContextInterface { branchStatus.error = undefined; entry.dirty = true; - const branchPromise = branch.run(branchCtx).then( + // Validate inside the run promise so a divergence reaches the + // rejection handler below instead of being dropped by allSettled. + const runBranch = async () => { + const output = await branch.run(branchCtx); + branchCtx.validateComplete(); + return output; + }; + + const branchPromise = runBranch().then( async (output) => { if (hasWinner) { // This branch completed after a winner was determined @@ -2383,8 +2448,6 @@ export class WorkflowContextImpl implements WorkflowContextInterface { winnerName = branch.name; winnerValue = output; - branchCtx.validateComplete(); - branchStatus.status = "completed"; branchStatus.output = output; branchStatus.error = undefined; @@ -2583,6 +2646,7 @@ export class WorkflowContextImpl implements WorkflowContextInterface { } // If it's not already marked as removed, we just skip it + this.markSettledHistoryVisited(location); return; } diff --git a/packages/workflows/tests/race.test.ts b/packages/workflows/tests/race.test.ts index 76fac99..80cbf4d 100644 --- a/packages/workflows/tests/race.test.ts +++ b/packages/workflows/tests/race.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it } from "vitest"; import { + HistoryDivergedError, InMemoryDriver, RaceError, runWorkflow, @@ -355,3 +356,59 @@ for (const mode of modes) { }); }); } + +describe("Workflow Engine Race (changed code)", { sequential: true }, () => { + let driver: InMemoryDriver; + + beforeEach(() => { + driver = new InMemoryDriver(); + driver.latency = 0; + }); + + it("should reject when the winning branch diverges from history", async () => { + const original = async (ctx: WorkflowContextInterface) => + await ctx.race("first", [ + { + name: "a", + run: async (branchCtx) => { + await branchCtx.step("removed-step", async () => {}); + await branchCtx.queue.next("go", { names: ["go"] }); + return "a"; + }, + }, + ]); + const changed = async (ctx: WorkflowContextInterface) => + await ctx.race("first", [ + { + name: "a", + run: async (branchCtx) => { + await branchCtx.queue.next("go", { names: ["go"] }); + return "a"; + }, + }, + ]); + + const firstRun = await runWorkflow( + "wf-1", + original, + undefined, + driver, + { + mode: "yield", + }, + ).result; + expect(firstRun.state).toBe("sleeping"); + + await driver.messageDriver.addMessage({ + id: "go-1", + name: "go", + data: {}, + sentAt: Date.now(), + }); + + await expect( + runWorkflow("wf-1", changed, undefined, driver, { mode: "yield" }) + .result, + ).rejects.toThrow(HistoryDivergedError); + }); +}); diff --git a/packages/workflows/tests/replay-scopes.test.ts b/packages/workflows/tests/replay-scopes.test.ts new file mode 100644 index 0000000..3fde693 --- /dev/null +++ b/packages/workflows/tests/replay-scopes.test.ts @@ -0,0 +1,265 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { + HistoryDivergedError, + InMemoryDriver, + Loop, + runWorkflow, + type WorkflowContextInterface, +} from "../src/testing.js"; + +type Body = (ctx: WorkflowContextInterface) => Promise; + +const waitForTick = (ctx: WorkflowContextInterface, name = "tick") => + ctx.queue.next(name, { names: ["tick"] }); + +// Runs to completion in one run, so a later suspend replays it as finished. +const finishedLoop: Body = (ctx) => + ctx.loop({ + name: "inner", + state: { count: 0 }, + run: async (loopCtx, state) => { + if (state.count === 2) { + return Loop.break(state.count); + } + await loopCtx.step("work", async () => state.count); + return Loop.continue({ count: state.count + 1 }); + }, + }); + +// Waits for one tick per iteration, so it resumes past its first iteration. +const resumedLoop: Body = (ctx) => + ctx.loop({ + name: "inner", + state: { count: 0 }, + run: async (loopCtx, state) => { + if (state.count === 3) { + return Loop.break(state.count); + } + await waitForTick(loopCtx); + return Loop.continue({ count: state.count + 1 }); + }, + }); + +const parents: Record Body> = { + loop: (body) => (ctx) => + ctx.loop("outer", async (outerCtx) => { + await body(outerCtx); + return Loop.break("done"); + }), + try: (body) => async (ctx) => { + await ctx.try("guarded", async (tryCtx) => await body(tryCtx)); + return "done"; + }, + join: (body) => async (ctx) => { + await ctx.join("parent", { + only: { run: (branchCtx) => body(branchCtx) }, + }); + return "done"; + }, + race: (body) => async (ctx) => { + await ctx.race("parent", [ + { name: "only", run: (branchCtx) => body(branchCtx) }, + ]); + return "done"; + }, +}; + +const finishedScopes: Record = { + loop: finishedLoop, + join: (ctx) => ctx.join("scope", { a: { run: finishedLoop } }), + race: (ctx) => ctx.race("scope", [{ name: "a", run: finishedLoop }]), +}; + +describe( + "Workflow Engine replay of settled scopes", + { sequential: true }, + () => { + let driver: InMemoryDriver; + + beforeEach(() => { + driver = new InMemoryDriver(); + driver.latency = 0; + }); + + // Delivers one tick before every run after the first until the run completes. + async function runOneTickPerRun(workflow: Body): Promise { + for (let run = 0; run < 20; run++) { + if (run > 0) { + await driver.messageDriver.addMessage({ + id: `tick-${run}`, + name: "tick", + data: {}, + sentAt: Date.now(), + }); + } + const result = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { + mode: "yield", + }, + ).result; + if (result.state === "completed") { + return result.output; + } + expect(result.state).toBe("sleeping"); + } + throw new Error("workflow did not complete"); + } + + for (const [parentName, parent] of Object.entries(parents)) { + it(`completes a resumed loop inside a ${parentName}`, async () => { + expect(await runOneTickPerRun(parent(resumedLoop))).toBe( + "done", + ); + }); + + for (const [scopeName, scope] of Object.entries(finishedScopes)) { + it(`completes when a finished ${scopeName} inside a ${parentName} replays`, async () => { + const output = await runOneTickPerRun( + parent(async (ctx) => { + await scope(ctx); + await waitForTick(ctx, "after-scope"); + }), + ); + + expect(output).toBe("done"); + }); + } + } + + it("completes when a finished join branch replays beside a waiting branch", async () => { + const output = await runOneTickPerRun( + parents.loop((ctx) => + ctx.join("scope", { + finished: { run: finishedLoop }, + waiting: { run: (branchCtx) => waitForTick(branchCtx) }, + }), + ), + ); + + expect(output).toBe("done"); + }); + + it("completes when a failed join branch replays inside a handled try", async () => { + const output = await runOneTickPerRun( + parents.loop(async (ctx) => { + const result = await ctx.try("handled", (tryCtx) => + tryCtx.join("scope", { + failing: { + run: async (branchCtx) => { + await branchCtx.step( + "work", + async () => "kept", + ); + throw new Error("branch failed"); + }, + }, + }), + ); + expect(result.ok).toBe(false); + await waitForTick(ctx, "after-scope"); + }), + ); + + expect(output).toBe("done"); + }); + + it("resolves getVersion to latest after a resumed loop", async () => { + const output = await runOneTickPerRun(async (ctx) => { + await resumedLoop(ctx); + return await ctx.getVersion("gate", 2); + }); + + expect(output).toBe(2); + }); + + it("resolves getVersion to latest after finished scopes replay", async () => { + const output = await runOneTickPerRun(async (ctx) => { + await finishedLoop(ctx); + await ctx.join("scope", { a: { run: finishedLoop } }); + await waitForTick(ctx, "after-scope"); + return await ctx.getVersion("gate", 2); + }); + + expect(output).toBe(2); + }); + + it("completes when ctx.removed replaces a finished loop inside a loop", async () => { + const original: Body = parents.loop(async (ctx) => { + await finishedLoop(ctx); + await waitForTick(ctx, "after-scope"); + }); + const migrated: Body = parents.loop(async (ctx) => { + await ctx.removed("inner", "loop"); + await waitForTick(ctx, "after-scope"); + }); + + const firstRun = await runWorkflow( + "wf-1", + original, + undefined, + driver, + { + mode: "yield", + }, + ).result; + expect(firstRun.state).toBe("sleeping"); + + await driver.messageDriver.addMessage({ + id: "tick-1", + name: "tick", + data: {}, + sentAt: Date.now(), + }); + + const result = await runWorkflow( + "wf-1", + migrated, + undefined, + driver, + { + mode: "yield", + }, + ).result; + expect(result.state).toBe("completed"); + expect(result.output).toBe("done"); + }); + + it("rejects a removed scope that history still holds", async () => { + const original: Body = parents.loop(async (ctx) => { + await finishedLoop(ctx); + await waitForTick(ctx, "after-scope"); + }); + const changed: Body = parents.loop((ctx) => + waitForTick(ctx, "after-scope"), + ); + + const firstRun = await runWorkflow( + "wf-1", + original, + undefined, + driver, + { + mode: "yield", + }, + ).result; + expect(firstRun.state).toBe("sleeping"); + + await driver.messageDriver.addMessage({ + id: "tick-1", + name: "tick", + data: {}, + sentAt: Date.now(), + }); + + await expect( + runWorkflow("wf-1", changed, undefined, driver, { + mode: "yield", + }).result, + ).rejects.toThrow(HistoryDivergedError); + }); + }, +); From 5ccbff913f8015fa1581a556e033678d193e770c Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Thu, 5 Nov 2026 19:55:01 +0400 Subject: [PATCH 4/7] fix(workflows): reject removed try scopes --- packages/workflows/src/context.ts | 35 ++- .../workflows/tests/scope-validation.test.ts | 200 ++++++++++++++++++ 2 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 packages/workflows/tests/scope-validation.test.ts diff --git a/packages/workflows/src/context.ts b/packages/workflows/src/context.ts index 7102d8e..329da81 100644 --- a/packages/workflows/src/context.ts +++ b/packages/workflows/src/context.ts @@ -607,16 +607,39 @@ export class WorkflowContextImpl implements WorkflowContextInterface { } /** - * Validate that every direct entry in this scope was visited. Nested scopes - * validate their own entries when they execute. + * Validate that every entry this scope owns was visited. That covers direct + * entries and the entries of try() blocks inside this scope. */ validateComplete(): void { + // Persisted containers validate their own children when they run, or + // replay a cached result without revisiting them. try() has no entry, + // so its descendants still belong to the surrounding scope. for (const [key, entry] of this.storage.history.entries) { - const isDirectChild = - entry.location.length === this.currentLocation.length + 1 && - isLocationPrefix(this.currentLocation, entry.location); + if ( + entry.location.length <= this.currentLocation.length || + !isLocationPrefix(this.currentLocation, entry.location) || + this.visitedKeys.has(key) + ) { + continue; + } + + let hasParentEntry = false; + for ( + let depth = this.currentLocation.length + 1; + depth < entry.location.length; + depth++ + ) { + const parentKey = locationToKey( + this.storage, + entry.location.slice(0, depth), + ); + if (this.storage.history.entries.has(parentKey)) { + hasParentEntry = true; + break; + } + } - if (isDirectChild && !this.visitedKeys.has(key)) { + if (!hasParentEntry) { throw new HistoryDivergedError( `Entry "${key}" exists in history but was not visited. ` + "Workflow code may have changed. Use ctx.removed() to handle migrations.", diff --git a/packages/workflows/tests/scope-validation.test.ts b/packages/workflows/tests/scope-validation.test.ts new file mode 100644 index 0000000..6d885bf --- /dev/null +++ b/packages/workflows/tests/scope-validation.test.ts @@ -0,0 +1,200 @@ +import { expect, it } from "vitest"; +import { + HistoryDivergedError, + InMemoryDriver, + Loop, + runWorkflow, + type WorkflowContextInterface, +} from "../src/testing.js"; + +function newDriver(): InMemoryDriver { + const driver = new InMemoryDriver(); + driver.latency = 0; + driver.workerPollInterval = 0; + return driver; +} + +// Runs the workflow until `reached` resolves, evicts it, and waits for it to stop. +async function runUntilEvicted( + driver: InMemoryDriver, + workflow: (ctx: WorkflowContextInterface) => Promise, + reached: Promise, + release: () => void, +): Promise { + const handle = runWorkflow("wf", workflow, undefined, driver); + await reached; + handle.evict(); + release(); + await handle.result.catch(() => undefined); +} + +it("resumes a loop inside a try block at its saved iteration", async () => { + const driver = newDriver(); + const timeline: string[] = []; + const reached = Promise.withResolvers(); + const release = Promise.withResolvers(); + let pauseOnce = true; + const workflow = async (ctx: WorkflowContextInterface) => + ctx.try("guarded", (block) => + block.loop({ + name: "jobs", + state: 0, + run: async (iterationCtx, i) => { + await iterationCtx.step("work", async () => { + timeline.push(`${i}`); + }); + if (i === 1 && pauseOnce) { + pauseOnce = false; + reached.resolve(); + await release.promise; + } + if (i >= 2) return Loop.break("looped"); + return Loop.continue(i + 1); + }, + }), + ); + + await runUntilEvicted(driver, workflow, reached.promise, () => + release.resolve(), + ); + expect(timeline).toEqual(["0", "1"]); + + timeline.length = 0; + const resumed = await runWorkflow("wf", workflow, undefined, driver).result; + expect({ state: resumed.state, output: resumed.output, timeline }).toEqual({ + state: "completed", + output: { ok: true, value: "looped" }, + timeline: ["2"], + }); +}); + +it("resumes nested loops at the saved inner iteration", async () => { + const driver = newDriver(); + const timeline: string[] = []; + const reached = Promise.withResolvers(); + const release = Promise.withResolvers(); + let pauseOnce = true; + const workflow = async (ctx: WorkflowContextInterface) => { + await ctx.loop({ + name: "outer", + state: 0, + run: async (outerCtx, i) => { + await outerCtx.loop({ + name: "inner", + state: 0, + run: async (innerCtx, j) => { + await innerCtx.step("work", async () => { + timeline.push(`${i}.${j}`); + }); + if (i === 1 && j === 1 && pauseOnce) { + pauseOnce = false; + reached.resolve(); + await release.promise; + } + if (j >= 2) return Loop.break(undefined); + return Loop.continue(j + 1); + }, + }); + if (i >= 1) return Loop.break(undefined); + return Loop.continue(i + 1); + }, + }); + return "done"; + }; + + await runUntilEvicted(driver, workflow, reached.promise, () => + release.resolve(), + ); + expect(timeline).toEqual(["0.0", "0.1", "0.2", "1.0", "1.1"]); + + timeline.length = 0; + const resumed = await runWorkflow("wf", workflow, undefined, driver).result; + expect({ state: resumed.state, output: resumed.output, timeline }).toEqual({ + state: "completed", + output: "done", + timeline: ["1.2"], + }); +}); + +it("still rejects the recorded steps of a try scope that the code removed", async () => { + const driver = newDriver(); + const first = await runWorkflow( + "wf", + async (ctx) => { + await ctx.try("container", async (outer) => { + await outer.try("optional", async (block) => { + await block.step("recorded", async () => "receipt"); + }); + }); + return "original"; + }, + undefined, + driver, + ).result; + expect(first).toMatchObject({ state: "completed", output: "original" }); + + await expect( + runWorkflow( + "wf", + async (ctx) => ctx.try("container", async () => "changed"), + undefined, + driver, + ).result, + ).rejects.toBeInstanceOf(HistoryDivergedError); +}); + +it("rejects the recorded steps of a top-level try scope that the code removed", async () => { + const driver = newDriver(); + await runWorkflow( + "wf", + async (ctx) => { + await ctx.try("optional", async (block) => { + await block.step("recorded", async () => "receipt"); + }); + return "original"; + }, + undefined, + driver, + ).result; + + await expect( + runWorkflow("wf", async () => "changed", undefined, driver).result, + ).rejects.toBeInstanceOf(HistoryDivergedError); +}); + +it("replays a finished join inside a try scope that the code kept", async () => { + const driver = newDriver(); + const runs: string[] = []; + const workflow = async (ctx: WorkflowContextInterface) => { + const joined = await ctx.try("guarded", (block) => + block.join("jobs", { + a: { + run: (branch) => + branch.step("work", async () => { + runs.push("work"); + return "a"; + }), + }, + }), + ); + await ctx.queue.next("wait", { names: ["wake"] }); + return joined; + }; + await runWorkflow("wf", workflow, undefined, driver).result; + await driver.messageDriver.addMessage({ + id: "wake", + name: "wake", + data: {}, + sentAt: Date.now(), + }); + + const resumed = await runWorkflow("wf", workflow, undefined, driver).result; + + expect({ resumed, runs }).toEqual({ + resumed: { + state: "completed", + output: { ok: true, value: { a: "a" } }, + }, + runs: ["work"], + }); +}); From 30cc7ee0e3974a4ba11210129b7a4896f34e9faa Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Sat, 7 Nov 2026 15:28:32 +0400 Subject: [PATCH 5/7] fix(workflows): wake race branches at their deadlines while other branches run --- packages/workflows/src/context.ts | 391 +++++++++++---- packages/workflows/tests/race.test.ts | 684 +++++++++++++++++++++++++- 2 files changed, 955 insertions(+), 120 deletions(-) diff --git a/packages/workflows/src/context.ts b/packages/workflows/src/context.ts index 329da81..9e86eed 100644 --- a/packages/workflows/src/context.ts +++ b/packages/workflows/src/context.ts @@ -71,6 +71,7 @@ import type { WorkflowQueueNextBatchOptions, WorkflowQueueNextOptions, } from "./types.js"; +import { setLongTimeout } from "./utils.js"; /** * Default values for step configuration. @@ -295,6 +296,40 @@ function buildSchedulerYieldError( return new MessageWaitError(messageNames); } +/** A race branch waiting on the scheduler, and how to run it again in-process. */ +interface YieldedRaceBranch { + error: SleepError | MessageWaitError | StepFailedError; + start: () => void; +} + +/** A branch waiting only on messages has no deadline. */ +function yieldedDeadline(yielded: YieldedRaceBranch): number | undefined { + const { error } = yielded; + if (error instanceof SleepError) { + return error.deadline; + } + if (error instanceof StepFailedError) { + return error.retryAt; + } + return undefined; +} + +function earliestYieldedDeadline( + yieldedBranches: Map, +): number | undefined { + let earliest: number | undefined; + for (const yielded of yieldedBranches.values()) { + const deadline = yieldedDeadline(yielded); + if ( + deadline !== undefined && + (earliest === undefined || deadline < earliest) + ) { + earliest = deadline; + } + } + return earliest; +} + function controlFlowErrorPriority(error: Error): number { if (error instanceof EvictedError) { return 0; @@ -343,6 +378,8 @@ export interface RollbackAction { export class WorkflowContextImpl implements WorkflowContextInterface { private entryInProgress = false; private abortController: AbortController; + /** The run's signal. Race branches have their own, which eviction skips. */ + private evictionSignal: AbortSignal; private currentLocation: Location; private visitedKeys: Set; private mode: "forward" | "rollback"; @@ -372,6 +409,7 @@ export class WorkflowContextImpl implements WorkflowContextInterface { ) { this.currentLocation = location; this.abortController = abortController ?? new AbortController(); + this.evictionSignal = this.abortController.signal; this.mode = mode; this.rollbackActions = rollbackActions; this.rollbackCheckpointSet = rollbackCheckpointSet; @@ -410,6 +448,11 @@ export class WorkflowContextImpl implements WorkflowContextInterface { } } + /** Whether an enclosing race picked a winner and cancelled this branch. */ + private isRaceCancelled(): boolean { + return this.abortSignal.reason instanceof CancelledError; + } + private async flushStorage(): Promise { await flush(this.storage, this.driver, this.historyNotifier); } @@ -421,7 +464,7 @@ export class WorkflowContextImpl implements WorkflowContextInterface { location: Location, abortController?: AbortController, ): WorkflowContextImpl { - return new WorkflowContextImpl( + const branch = new WorkflowContextImpl( this.workflowId, this.storage, this.driver, @@ -436,6 +479,8 @@ export class WorkflowContextImpl implements WorkflowContextInterface { this.logger, this.visitedKeys, ); + branch.evictionSignal = this.evictionSignal; + return branch; } /** @@ -685,6 +730,34 @@ export class WorkflowContextImpl implements WorkflowContextInterface { } } + /** + * Wait until `deadline`, a running race branch settles, or the workflow is + * evicted. The timer and abort listener are torn down on every outcome. + */ + private async waitForRaceWake( + deadline: number, + runningBranches: Set>, + ): Promise { + let timer: ReturnType | undefined; + let onEvicted: (() => void) | undefined; + const wake = new Promise((resolve) => { + timer = setLongTimeout(resolve, Math.max(0, deadline - Date.now())); + onEvicted = resolve; + this.evictionSignal.addEventListener("abort", onEvicted, { + once: true, + }); + }); + + try { + await Promise.race([wake, ...runningBranches]); + } finally { + timer?.abort(); + if (onEvicted) { + this.evictionSignal.removeEventListener("abort", onEvicted); + } + } + } + // === Step === async step( @@ -937,6 +1010,11 @@ export class WorkflowContextImpl implements WorkflowContextInterface { } } + // A race can cancel this branch while the metadata above loads. + if (this.isRaceCancelled()) { + throw new EvictedError(); + } + // Execute the step const entry = existing ?? createEntry(location, { type: "step", data: {} }); @@ -1007,6 +1085,12 @@ export class WorkflowContextImpl implements WorkflowContextInterface { }); return output; } catch (error) { + // The race that cancelled this step deletes its entry, so a + // cancellation is not a failure to record or retry. + if (this.isRaceCancelled()) { + throw new EvictedError(); + } + if (entry.kind.type === "step") { entry.kind.data.error = String(error); } @@ -1157,23 +1241,34 @@ export class WorkflowContextImpl implements WorkflowContextInterface { timeoutMs: number, stepName: string, ): Promise { - if (timeoutMs <= 0) { - return promise; - } - let timeoutId: ReturnType | undefined; - const timeoutPromise = new Promise((_, reject) => { - timeoutId = setTimeout(() => { - reject(new StepTimeoutError(stepName, timeoutMs)); - }, timeoutMs); + let onAbort: (() => void) | undefined; + const interrupted = new Promise((_, reject) => { + if (timeoutMs > 0) { + timeoutId = setTimeout(() => { + reject(new StepTimeoutError(stepName, timeoutMs)); + }, timeoutMs); + } + + // A step that a race cancels stops being awaited, as with a timeout. + // Eviction does not interrupt a step, so its result is still recorded. + onAbort = () => { + if (this.isRaceCancelled()) { + reject(new EvictedError()); + } + }; + this.abortSignal.addEventListener("abort", onAbort, { once: true }); }); try { - return await Promise.race([promise, timeoutPromise]); + return await Promise.race([promise, interrupted]); } finally { if (timeoutId !== undefined) { clearTimeout(timeoutId); } + if (onAbort) { + this.abortSignal.removeEventListener("abort", onAbort); + } } } @@ -2363,8 +2458,11 @@ export class WorkflowContextImpl implements WorkflowContextInterface { // Create abort controller for cancellation const raceAbortController = new AbortController(); - // Track all branch promises to wait for cleanup - const branchPromises: Promise[] = []; + // Track branches with in-process work to wait for cleanup + const runningBranches = new Set>(); + + // Track the wait of each branch that yielded, and how to run it again + const yieldedBranches = new Map(); // Track winner info let winnerName: string | null = null; @@ -2372,7 +2470,6 @@ export class WorkflowContextImpl implements WorkflowContextInterface { let hasWinner = false; const errors: Record = {}; const lateErrors: Array<{ name: string; error: string }> = []; - let schedulerYieldState: SchedulerYieldState | undefined; let propagatedError: Error | undefined; // Check for replay winners first @@ -2430,122 +2527,199 @@ export class WorkflowContextImpl implements WorkflowContextInterface { continue; } - const branchCtx = this.createBranch( - branchLocation, - raceAbortController, - ); - branchStatus.status = "running"; branchStatus.error = undefined; entry.dirty = true; // Validate inside the run promise so a divergence reaches the - // rejection handler below instead of being dropped by allSettled. + // rejection handler below instead of being dropped. const runBranch = async () => { + const branchCtx = this.createBranch( + branchLocation, + raceAbortController, + ); const output = await branch.run(branchCtx); branchCtx.validateComplete(); return output; }; - const branchPromise = runBranch().then( - async (output) => { - if (hasWinner) { - // This branch completed after a winner was determined - // Still record the completion for observability - branchStatus.status = "completed"; - branchStatus.output = output; - branchStatus.error = undefined; - entry.dirty = true; - return; - } - - if (propagatedError) { - branchStatus.status = "completed"; - branchStatus.output = output; - branchStatus.error = undefined; - entry.dirty = true; - return; - } + const startBranch = () => { + const branchPromise = runBranch() + .then( + async (output) => { + if (hasWinner) { + // This branch completed after a winner was determined + // Still record the completion for observability + branchStatus.status = "completed"; + branchStatus.output = output; + branchStatus.error = undefined; + entry.dirty = true; + return; + } + + if (propagatedError) { + branchStatus.status = "completed"; + branchStatus.output = output; + branchStatus.error = undefined; + entry.dirty = true; + return; + } + + hasWinner = true; + winnerName = branch.name; + winnerValue = output; + + branchStatus.status = "completed"; + branchStatus.output = output; + branchStatus.error = undefined; + raceData.winner = branch.name; + entry.dirty = true; + + // Cancel other branches + raceAbortController.abort(new CancelledError()); + }, + (error) => { + if (hasWinner) { + if ( + error instanceof CancelledError || + error instanceof EvictedError || + error instanceof SleepError || + error instanceof MessageWaitError + ) { + branchStatus.status = "cancelled"; + } else { + lateErrors.push({ + name: branch.name, + error: String(error), + }); + } + entry.dirty = true; + return; + } + + if ( + error instanceof SleepError || + error instanceof MessageWaitError || + error instanceof StepFailedError + ) { + yieldedBranches.set(branch.name, { + error, + start: startBranch, + }); + branchStatus.status = "running"; + branchStatus.error = undefined; + entry.dirty = true; + return; + } + + if ( + error instanceof EvictedError || + error instanceof HistoryDivergedError || + error instanceof EntryInProgressError || + error instanceof RollbackCheckpointError || + error instanceof RollbackStopError + ) { + propagatedError = selectControlFlowError( + propagatedError, + error, + ); + branchStatus.status = "running"; + branchStatus.error = undefined; + entry.dirty = true; + return; + } + + if (error instanceof CancelledError) { + branchStatus.status = "cancelled"; + } else { + branchStatus.status = "failed"; + branchStatus.output = undefined; + branchStatus.error = String(error); + errors[branch.name] = error as Error; + } + entry.dirty = true; + }, + ) + .finally(() => { + runningBranches.delete(branchPromise); + }); + runningBranches.add(branchPromise); + }; - hasWinner = true; - winnerName = branch.name; - winnerValue = output; + startBranch(); + } - branchStatus.status = "completed"; - branchStatus.output = output; - branchStatus.error = undefined; - raceData.winner = branch.name; - entry.dirty = true; + // A cancelled branch also cancels the races nested inside it. + const parentSignal = this.abortSignal; + const forwardCancellation = () => { + if (parentSignal.reason instanceof CancelledError) { + raceAbortController.abort(parentSignal.reason); + } + }; + forwardCancellation(); + parentSignal.addEventListener("abort", forwardCancellation, { + once: true, + }); - // Cancel other branches - raceAbortController.abort(); - }, - (error) => { - if (hasWinner) { - if ( - error instanceof CancelledError || - error instanceof EvictedError - ) { - branchStatus.status = "cancelled"; - } else { - lateErrors.push({ - name: branch.name, - error: String(error), - }); + // Wait for all branches to complete or be cancelled. Nothing else wakes + // a yielded branch while another still has in-process work, because the + // scheduler only takes over once every branch has settled. + let scheduledWakeAt: number | undefined; + try { + while (runningBranches.size > 0) { + const canWake = + !propagatedError && + !raceAbortController.signal.aborted && + !this.evictionSignal.aborted; + + if (canWake) { + const now = Date.now(); + for (const [branchName, yielded] of yieldedBranches) { + if ((yieldedDeadline(yielded) ?? Infinity) <= now) { + yieldedBranches.delete(branchName); + yielded.start(); } - entry.dirty = true; - return; } + } - if ( - error instanceof SleepError || - error instanceof MessageWaitError || - error instanceof StepFailedError - ) { - schedulerYieldState = mergeSchedulerYield( - schedulerYieldState, - error, - ); - branchStatus.status = "running"; - branchStatus.error = undefined; - entry.dirty = true; - return; - } + const wakeAt = canWake + ? earliestYieldedDeadline(yieldedBranches) + : undefined; + if (wakeAt === undefined) { + await Promise.race(runningBranches); + continue; + } - if ( - error instanceof EvictedError || - error instanceof HistoryDivergedError || - error instanceof EntryInProgressError || - error instanceof RollbackCheckpointError || - error instanceof RollbackStopError - ) { + // Keep the deadline durable, because the scheduler only arms it + // if this run reaches a yield. + if (wakeAt !== scheduledWakeAt) { + scheduledWakeAt = wakeAt; + try { + await this.driver.setAlarm(this.workflowId, wakeAt); + } catch (error) { propagatedError = selectControlFlowError( propagatedError, - error, + error as Error, ); - branchStatus.status = "running"; - branchStatus.error = undefined; - entry.dirty = true; - return; } - if (error instanceof CancelledError) { - branchStatus.status = "cancelled"; - } else { - branchStatus.status = "failed"; - branchStatus.output = undefined; - branchStatus.error = String(error); - errors[branch.name] = error as Error; - } - entry.dirty = true; - }, - ); + // A branch can settle during this write, so compute the wake again. + continue; + } - branchPromises.push(branchPromise); + await this.waitForRaceWake(wakeAt, runningBranches); + } + } finally { + parentSignal.removeEventListener("abort", forwardCancellation); } - // Wait for all branches to complete or be cancelled - await Promise.allSettled(branchPromises); + // A cancelled race has no winner to report to its parent branch. + if (!hasWinner && raceAbortController.signal.aborted) { + propagatedError = selectControlFlowError( + propagatedError, + new EvictedError(), + ); + } if (propagatedError) { await this.flushStorage(); @@ -2560,6 +2734,13 @@ export class WorkflowContextImpl implements WorkflowContextInterface { ) ) { await this.flushStorage(); + let schedulerYieldState: SchedulerYieldState | undefined; + for (const { error } of yieldedBranches.values()) { + schedulerYieldState = mergeSchedulerYield( + schedulerYieldState, + error, + ); + } if (!schedulerYieldState) { throw new Error( `Race "${name}" has pending branches without a scheduler yield`, @@ -2572,6 +2753,10 @@ export class WorkflowContextImpl implements WorkflowContextInterface { if (hasWinner && winnerName !== null) { for (const branch of branches) { if (branch.name !== winnerName) { + if (yieldedBranches.has(branch.name)) { + raceData.branches[branch.name].status = "cancelled"; + entry.dirty = true; + } const branchLocation = appendName( this.storage, location, diff --git a/packages/workflows/tests/race.test.ts b/packages/workflows/tests/race.test.ts index 80cbf4d..db8e6be 100644 --- a/packages/workflows/tests/race.test.ts +++ b/packages/workflows/tests/race.test.ts @@ -1,14 +1,53 @@ -import { beforeEach, describe, expect, it } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { + CriticalError, HistoryDivergedError, InMemoryDriver, + Loop, RaceError, runWorkflow, type WorkflowContextInterface, + type WorkflowErrorEvent, + type WorkflowHistorySnapshot, } from "../src/testing.js"; const modes = ["yield", "live"] as const; +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: Error) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +function timeoutBranch(durationMs: number) { + return { + name: "timeout", + run: async (ctx: WorkflowContextInterface) => { + await ctx.sleep("wait", durationMs); + return "timed-out"; + }, + }; +} + +function raceBranchStatuses( + history: WorkflowHistorySnapshot | undefined, +): Record { + for (const entry of history?.entries ?? []) { + if (entry.kind.type === "race") { + return Object.fromEntries( + Object.entries(entry.kind.data.branches).map( + ([name, branch]) => [name, branch.status], + ), + ); + } + } + throw new Error("history has no race entry"); +} + for (const mode of modes) { describe(`Workflow Engine Race (${mode})`, { sequential: true }, () => { let driver: InMemoryDriver; @@ -319,7 +358,10 @@ for (const mode of modes) { ]); }; - const firstResult = await runWorkflow( + // The first run suspends if the retry comes due after every branch + // has yielded. It finishes in-process if the sleep branch is still + // writing, so only the final outcome is fixed. + let result = await runWorkflow( "wf-race-retry", workflow, undefined, @@ -327,36 +369,644 @@ for (const mode of modes) { { mode }, ).result; - if (mode === "yield") { - expect(firstResult.state).toBe("sleeping"); + if (result.state === "sleeping") { await new Promise((resolve) => setTimeout(resolve, 10)); - - const secondResult = await runWorkflow( + result = await runWorkflow( "wf-race-retry", workflow, undefined, driver, { mode }, ).result; + } - expect(secondResult.state).toBe("completed"); - expect(secondResult.output).toEqual({ - winner: "flaky", - value: "winner", + expect(result.state).toBe("completed"); + expect(result.output).toEqual({ + winner: "flaky", + value: "winner", + }); + expect(attempts).toBe(2); + }); + }); +} + +for (const mode of modes) { + describe( + `Workflow Engine Race Deadlines (${mode})`, + { sequential: true }, + () => { + let driver: InMemoryDriver; + + beforeEach(() => { + driver = new InMemoryDriver(); + driver.latency = 0; + }); + + it("should time out a step that is still running and keep the result on replay", async () => { + const work = deferred(); + let workRuns = 0; + let nextStepRuns = 0; + let afterRaceRuns = 0; + + const workflow = async (ctx: WorkflowContextInterface) => { + const result = await ctx.race("timeout-race", [ + { + name: "work", + run: async (branchCtx) => { + const value = await branchCtx.step( + "slow", + async () => { + workRuns += 1; + return await work.promise; + }, + ); + await branchCtx.step("next", async () => { + nextStepRuns += 1; + }); + return value; + }, + }, + timeoutBranch(150), + ]); + await ctx.step("after-race", async () => { + afterRaceRuns += 1; + }); + return result; + }; + const expected = { winner: "timeout", value: "timed-out" }; + + const startedAt = Date.now(); + const updates: WorkflowHistorySnapshot[] = []; + const first = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { + mode, + onHistoryUpdated: (history) => updates.push(history), + }, + ).result; + + expect(first.state).toBe("completed"); + expect(first.output).toEqual(expected); + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(140); + expect(raceBranchStatuses(updates.at(-1))).toEqual({ + work: "cancelled", + timeout: "completed", }); - } else { - expect(firstResult.state).toBe("completed"); - expect(firstResult.output).toEqual({ + + // The late result must not change what the workflow recorded. + work.resolve("late"); + await new Promise((resolve) => setTimeout(resolve, 10)); + + const replayed = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + + expect(replayed.output).toEqual(expected); + expect(workRuns).toBe(1); + expect(nextStepRuns).toBe(0); + expect(afterRaceRuns).toBe(1); + }); + + it("should neither retry nor report a cancelled step that fails late", async () => { + const work = deferred(); + let workRuns = 0; + const events: WorkflowErrorEvent[] = []; + + const workflow = async (ctx: WorkflowContextInterface) => + await ctx.race("timeout-race", [ + { + name: "work", + run: async (branchCtx) => + await branchCtx.step("slow", async () => { + workRuns += 1; + return await work.promise; + }), + }, + timeoutBranch(50), + ]); + + const result = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { + mode, + onError: (event) => { + events.push(event); + }, + }, + ).result; + expect(result.output?.winner).toBe("timeout"); + + work.reject(new Error("failed after the race ended")); + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(workRuns).toBe(1); + expect(events).toEqual([]); + }); + + it("should let a step that finishes first win against a far deadline", async () => { + const updates: WorkflowHistorySnapshot[] = []; + const fortyDaysMs = 40 * 24 * 60 * 60 * 1000; + + const workflow = async (ctx: WorkflowContextInterface) => + await ctx.race("timeout-race", [ + { + name: "work", + run: async (branchCtx) => + await branchCtx.step("slow", async () => { + await new Promise((resolve) => + setTimeout(resolve, 50), + ); + return "done"; + }), + }, + timeoutBranch(fortyDaysMs), + ]); + + const result = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { + mode, + onHistoryUpdated: (history) => updates.push(history), + }, + ).result; + + expect(result.output).toEqual({ + winner: "work", + value: "done", + }); + expect(raceBranchStatuses(updates.at(-1))).toEqual({ + work: "completed", + timeout: "cancelled", + }); + }); + + it("should wake each sleep of a branch while a step is running", async () => { + const work = deferred(); + const startedAt = Date.now(); + + const workflow = async (ctx: WorkflowContextInterface) => + await ctx.race("timeout-race", [ + { + name: "work", + run: async (branchCtx) => + await branchCtx.step( + "slow", + async () => await work.promise, + ), + }, + { + name: "timeout", + run: async (branchCtx) => { + await branchCtx.sleep("first", 120); + await branchCtx.sleep("second", 120); + return "timed-out"; + }, + }, + ]); + + const result = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + + expect(result.output?.winner).toBe("timeout"); + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(230); + work.resolve("late"); + }); + + it("should pick the earliest deadline among sleeping branches", async () => { + const work = deferred(); + + const workflow = async (ctx: WorkflowContextInterface) => + await ctx.race("timeout-race", [ + { + name: "work", + run: async (branchCtx) => + await branchCtx.step( + "slow", + async () => await work.promise, + ), + }, + { + name: "far", + run: async (branchCtx) => { + await branchCtx.sleep("far-wait", 60_000); + return "far"; + }, + }, + { + name: "near", + run: async (branchCtx) => { + await branchCtx.sleep("near-wait", 120); + return "near"; + }, + }, + ]); + + const result = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + + expect(result.output).toEqual({ + winner: "near", + value: "near", + }); + work.resolve("late"); + }); + + it("should retry a failed step on schedule while another branch runs", async () => { + const work = deferred(); + let attempts = 0; + + const workflow = async (ctx: WorkflowContextInterface) => + await ctx.race("retry-race", [ + { + name: "work", + run: async (branchCtx) => + await branchCtx.step( + "slow", + async () => await work.promise, + ), + }, + { + name: "flaky", + run: async (branchCtx) => + await branchCtx.step({ + name: "flaky-step", + maxRetries: 1, + retryBackoffBase: 20, + retryBackoffMax: 20, + run: async () => { + attempts += 1; + if (attempts === 1) { + throw new Error("retry"); + } + return "recovered"; + }, + }), + }, + ]); + + const result = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + + expect(result.output).toEqual({ winner: "flaky", - value: "winner", + value: "recovered", + }); + expect(attempts).toBe(2); + work.resolve("late"); + }); + + it("should end a queue wait at its timeout while a step is running", async () => { + const work = deferred(); + + const workflow = async (ctx: WorkflowContextInterface) => + await ctx.race("queue-race", [ + { + name: "work", + run: async (branchCtx) => + await branchCtx.step( + "slow", + async () => await work.promise, + ), + }, + { + name: "queue", + run: async (branchCtx) => { + const messages = + await branchCtx.queue.nextBatch("wait", { + names: ["never-sent"], + timeout: 120, + }); + return `received-${messages.length}`; + }, + }, + ]); + + const result = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + + expect(result.output).toEqual({ + winner: "queue", + value: "received-0", + }); + work.resolve("late"); + }); + + const loserScopes: Record< + string, + ( + ctx: WorkflowContextInterface, + slow: (ctx: WorkflowContextInterface) => Promise, + ) => Promise + > = { + race: (ctx, slow) => + ctx.race("scope", [ + { name: "inner", run: slow }, + { + name: "inner-timeout", + run: async (innerCtx) => { + await innerCtx.sleep("inner-wait", 60_000); + return "inner-timed-out"; + }, + }, + ]), + join: (ctx, slow) => + ctx.join("scope", { inner: { run: slow } }), + try: (ctx, slow) => ctx.try("scope", slow), + loop: (ctx, slow) => + ctx.loop("scope", async (loopCtx) => { + await slow(loopCtx); + return Loop.break(undefined); + }), + }; + + for (const [scope, runScope] of Object.entries(loserScopes)) { + it(`should cancel a step inside a losing ${scope} and replay the result`, async () => { + const work = deferred(); + const warn = vi + .spyOn(console, "warn") + .mockImplementation(() => {}); + + const workflow = async (ctx: WorkflowContextInterface) => + await ctx.race("outer", [ + { + name: "work", + run: async (branchCtx) => { + await runScope( + branchCtx, + async (innerCtx) => + await innerCtx.step( + "slow", + async () => await work.promise, + ), + ); + return "work-done"; + }, + }, + timeoutBranch(120), + ]); + const expected = { winner: "timeout", value: "timed-out" }; + + try { + const first = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + expect(first.output).toEqual(expected); + expect(warn).not.toHaveBeenCalled(); + } finally { + warn.mockRestore(); + } + + work.resolve("late"); + await new Promise((resolve) => setTimeout(resolve, 10)); + + const replayed = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + expect(replayed.output).toEqual(expected); }); } - expect(attempts).toBe(2); - }); - }); + it("should keep concurrent races from observing each other", async () => { + const slowWork = deferred(); + + const workflow = async (ctx: WorkflowContextInterface) => + await ctx.join("both", { + timesOut: { + run: async (branchCtx) => + await branchCtx.race("times-out", [ + { + name: "work", + run: async (raceCtx) => + await raceCtx.step( + "slow", + async () => + await slowWork.promise, + ), + }, + timeoutBranch(100), + ]), + }, + finishes: { + run: async (branchCtx) => + await branchCtx.race("finishes", [ + { + name: "work", + run: async (raceCtx) => + await raceCtx.step( + "quick", + async () => { + await new Promise( + (resolve) => + setTimeout( + resolve, + 200, + ), + ); + return "done"; + }, + ), + }, + timeoutBranch(60_000), + ]), + }, + }); + + const result = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + + expect(result.output).toEqual({ + timesOut: { winner: "timeout", value: "timed-out" }, + finishes: { winner: "work", value: "done" }, + }); + slowWork.resolve("late"); + }); + + it("should leave a durable wake when evicted and time out on resume", async () => { + const work = deferred(); + + const workflow = async (ctx: WorkflowContextInterface) => + await ctx.race("timeout-race", [ + { + name: "work", + run: async (branchCtx) => + await branchCtx.step( + "slow", + async () => await work.promise, + ), + }, + timeoutBranch(150), + ]); + + const startedAt = Date.now(); + const evicted = runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ); + // The alarm write is the only signal that the race is armed. + await vi.waitFor(() => { + expect(driver.getAlarm("wf-1")).toBeDefined(); + }); + evicted.evict(); + + const wakeAt = driver.getAlarm("wf-1") as number; + expect(wakeAt).toBeGreaterThanOrEqual(startedAt + 150); + + // The evicted run must not start the sleeping branch again. + await new Promise((resolve) => + setTimeout(resolve, wakeAt - Date.now() + 50), + ); + const resumed = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + expect(resumed.output).toEqual({ + winner: "timeout", + value: "timed-out", + }); + expect(driver.getAlarm("wf-1")).toBeUndefined(); + + work.resolve("late"); + await evicted.result; + }); + + it("should suspend at once when every branch is waiting and wake at the deadline", async () => { + const workflow = async (ctx: WorkflowContextInterface) => + await ctx.race("timeout-race", [ + { + name: "message", + run: async (branchCtx) => { + await branchCtx.queue.next("wait", { + names: ["never-sent"], + }); + return "message"; + }, + }, + timeoutBranch(500), + ]); + const expected = { winner: "timeout", value: "timed-out" }; + + const first = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + + if (mode === "live") { + expect(first.output).toEqual(expected); + return; + } + + const sleepUntil = first.sleepUntil as number; + expect(first.state).toBe("sleeping"); + expect(Date.now()).toBeLessThan(sleepUntil); + expect(driver.getAlarm("wf-1")).toBe(sleepUntil); + + await new Promise((resolve) => + setTimeout(resolve, sleepUntil - Date.now() + 20), + ); + const resumed = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + expect(resumed.output).toEqual(expected); + }); + }, + ); } +describe("Workflow Engine Race (replay)", { sequential: true }, () => { + it("should suspend at once on replay when a settled branch is skipped", async () => { + const driver = new InMemoryDriver(); + driver.latency = 0; + + const workflow = async (ctx: WorkflowContextInterface) => + await ctx.race("timeout-race", [ + { + name: "failed", + run: async (): Promise => { + throw new CriticalError("failed for good"); + }, + }, + timeoutBranch(60_000), + ]); + + const first = await runWorkflow("wf-1", workflow, undefined, driver, { + mode: "yield", + }).result; + expect(first.state).toBe("sleeping"); + + // A hung replay fails this test on its timeout. + const replayed = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { + mode: "yield", + }, + ).result; + expect(replayed.state).toBe("sleeping"); + expect(replayed.sleepUntil).toBe(first.sleepUntil); + }); +}); + describe("Workflow Engine Race (changed code)", { sequential: true }, () => { let driver: InMemoryDriver; From 12768edf11d1e0341a35d246c2e2b6346a0ff75c Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Sat, 7 Nov 2026 15:29:24 +0400 Subject: [PATCH 6/7] fix(workflows): save the race winner before deleting losing branches --- packages/workflows/src/context.ts | 16 ++-- packages/workflows/tests/race.test.ts | 127 ++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 6 deletions(-) diff --git a/packages/workflows/src/context.ts b/packages/workflows/src/context.ts index 9e86eed..c061f09 100644 --- a/packages/workflows/src/context.ts +++ b/packages/workflows/src/context.ts @@ -31,8 +31,8 @@ import { registerName, } from "./location.js"; import { + collectDeletionsForPrefix, createEntry, - deleteEntriesWithPrefix, flush, getOrCreateMetadata, loadMetadata, @@ -2749,8 +2749,12 @@ export class WorkflowContextImpl implements WorkflowContextInterface { throw buildSchedulerYieldError(schedulerYieldState); } - // Clean up entries from non-winning branches + // Clean up entries from non-winning branches. The flush saves the + // winner before it deletes them, so a crash in between leaves only + // stale loser entries, which replay skips because the race has a winner. + let loserDeletions: PendingDeletions | undefined; if (hasWinner && winnerName !== null) { + loserDeletions = { prefixes: [], keys: [], ranges: [] }; for (const branch of branches) { if (branch.name !== winnerName) { if (yieldedBranches.has(branch.name)) { @@ -2762,18 +2766,18 @@ export class WorkflowContextImpl implements WorkflowContextInterface { location, branch.name, ); - await deleteEntriesWithPrefix( + const deletions = collectDeletionsForPrefix( this.storage, - this.driver, branchLocation, - this.historyNotifier, ); + loserDeletions.prefixes.push(...deletions.prefixes); + loserDeletions.keys.push(...deletions.keys); } } } // Flush final state - await this.flushStorage(); + await this.flushStorageWithDeletions(loserDeletions); // Log late errors if any (these occurred after a winner was determined) if (lateErrors.length > 0) { diff --git a/packages/workflows/tests/race.test.ts b/packages/workflows/tests/race.test.ts index db8e6be..8244a6f 100644 --- a/packages/workflows/tests/race.test.ts +++ b/packages/workflows/tests/race.test.ts @@ -33,6 +33,65 @@ function timeoutBranch(durationMs: number) { }; } +type CrashPoint = "before-delete" | "after-history-delete"; + +/** + * Stands in for a process that dies during race cleanup. After the crash, + * every write and delete fails, so nothing more reaches storage. `restart` + * keeps the stored data, like a new process opening the same storage. + */ +class CrashingDriver extends InMemoryDriver { + crashPoint: CrashPoint | undefined; + #crashed = false; + + restart() { + this.crashPoint = undefined; + this.#crashed = false; + } + + #checkAlive() { + if (this.#crashed) { + throw new Error("driver crashed"); + } + } + + #crashBeforeDelete() { + this.#checkAlive(); + if (this.crashPoint === "before-delete") { + this.#crashed = true; + throw new Error("driver crashed"); + } + } + + override async set(key: Uint8Array, value: Uint8Array) { + this.#checkAlive(); + await super.set(key, value); + } + + override async batch(writes: Parameters[0]) { + this.#checkAlive(); + await super.batch(writes); + } + + override async delete(key: Uint8Array) { + this.#crashBeforeDelete(); + await super.delete(key); + } + + override async deleteRange(start: Uint8Array, end: Uint8Array) { + this.#crashBeforeDelete(); + await super.deleteRange(start, end); + } + + override async deletePrefix(prefix: Uint8Array) { + this.#crashBeforeDelete(); + await super.deletePrefix(prefix); + if (this.crashPoint === "after-history-delete") { + this.#crashed = true; + } + } +} + function raceBranchStatuses( history: WorkflowHistorySnapshot | undefined, ): Record { @@ -971,6 +1030,74 @@ for (const mode of modes) { ); } +for (const mode of modes) { + describe( + `Workflow Engine Race Crash Recovery (${mode})`, + { sequential: true }, + () => { + const crashPoints: CrashPoint[] = [ + "before-delete", + "after-history-delete", + ]; + + for (const crashPoint of crashPoints) { + it(`should keep the winner and not rerun a finished loser step after a crash ${crashPoint}`, async () => { + const driver = new CrashingDriver(); + driver.latency = 0; + let confirmedRuns = 0; + + const workflow = async (ctx: WorkflowContextInterface) => + await ctx.race("race", [ + { + name: "loser", + run: async (branchCtx) => { + await branchCtx.step( + "confirmed", + async () => { + confirmedRuns += 1; + }, + ); + await branchCtx.sleep("wait", 60_000); + return "loser"; + }, + }, + { + name: "winner", + run: async (branchCtx) => + await branchCtx.step("work", async () => { + await new Promise((resolve) => + setTimeout(resolve, 30), + ); + return "won"; + }), + }, + ]); + + driver.crashPoint = crashPoint; + await runWorkflow("wf-1", workflow, undefined, driver, { + mode, + }).result.catch(() => undefined); + + driver.restart(); + const resumed = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + + expect(resumed.output).toEqual({ + winner: "winner", + value: "won", + }); + expect(confirmedRuns).toBe(1); + }); + } + }, + ); +} + describe("Workflow Engine Race (replay)", { sequential: true }, () => { it("should suspend at once on replay when a settled branch is skipped", async () => { const driver = new InMemoryDriver(); From f83cb9ed0ae2e6645bfd73f3ff4e5947210df06d Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Sun, 8 Nov 2026 18:58:24 +0530 Subject: [PATCH 7/7] fix(workflows): stop long step timeouts from firing immediately --- packages/workflows/src/context.ts | 45 +++-- packages/workflows/src/types.ts | 2 +- packages/workflows/src/utils.ts | 4 + .../workflows/tests/eviction-cancel.test.ts | 48 +++++ .../tests/fixtures/long-timeout-step.ts | 22 +++ packages/workflows/tests/steps.test.ts | 181 +++++++++++++++++- 6 files changed, 277 insertions(+), 25 deletions(-) create mode 100644 packages/workflows/tests/fixtures/long-timeout-step.ts diff --git a/packages/workflows/src/context.ts b/packages/workflows/src/context.ts index c061f09..03b4bbd 100644 --- a/packages/workflows/src/context.ts +++ b/packages/workflows/src/context.ts @@ -71,7 +71,7 @@ import type { WorkflowQueueNextBatchOptions, WorkflowQueueNextOptions, } from "./types.js"; -import { setLongTimeout } from "./utils.js"; +import { type LongTimeoutHandle, setLongTimeout } from "./utils.js"; /** * Default values for step configuration. @@ -1241,34 +1241,33 @@ export class WorkflowContextImpl implements WorkflowContextInterface { timeoutMs: number, stepName: string, ): Promise { - let timeoutId: ReturnType | undefined; - let onAbort: (() => void) | undefined; + let interrupt!: (error: Error) => void; const interrupted = new Promise((_, reject) => { - if (timeoutMs > 0) { - timeoutId = setTimeout(() => { - reject(new StepTimeoutError(stepName, timeoutMs)); - }, timeoutMs); - } - - // A step that a race cancels stops being awaited, as with a timeout. - // Eviction does not interrupt a step, so its result is still recorded. - onAbort = () => { - if (this.isRaceCancelled()) { - reject(new EvictedError()); - } - }; - this.abortSignal.addEventListener("abort", onAbort, { once: true }); + interrupt = reject; }); + // A timeout of 0 or a non-finite timeout such as Infinity disables the timeout. + let timeout: LongTimeoutHandle | undefined; + if (timeoutMs > 0 && Number.isFinite(timeoutMs)) { + timeout = setLongTimeout(() => { + interrupt(new StepTimeoutError(stepName, timeoutMs)); + }, timeoutMs); + } + + // A race that cancels this step stops awaiting it, as a timeout does. + // Eviction does not interrupt a step, so its result is still recorded. + const onAbort = () => { + if (this.isRaceCancelled()) { + interrupt(new EvictedError()); + } + }; + this.abortSignal.addEventListener("abort", onAbort, { once: true }); + try { return await Promise.race([promise, interrupted]); } finally { - if (timeoutId !== undefined) { - clearTimeout(timeoutId); - } - if (onAbort) { - this.abortSignal.removeEventListener("abort", onAbort); - } + timeout?.abort(); + this.abortSignal.removeEventListener("abort", onAbort); } } diff --git a/packages/workflows/src/types.ts b/packages/workflows/src/types.ts index 5b5cb18..3687494 100644 --- a/packages/workflows/src/types.ts +++ b/packages/workflows/src/types.ts @@ -419,7 +419,7 @@ export interface StepConfig { retryBackoffBase?: number; /** Maximum delay in ms for exponential backoff (default: 30000). */ retryBackoffMax?: number; - /** Timeout in ms for step execution (default: 30000). Set to 0 to disable. */ + /** Timeout in ms for step execution (default: 30000). Set to 0 or Infinity to disable. */ timeout?: number; /** If true, step timeouts retry like any other error instead of failing immediately as critical. Default: false. */ retryOnTimeout?: boolean; diff --git a/packages/workflows/src/utils.ts b/packages/workflows/src/utils.ts index 16de15f..31fe9a2 100644 --- a/packages/workflows/src/utils.ts +++ b/packages/workflows/src/utils.ts @@ -9,6 +9,10 @@ const TIMEOUT_MAX = 2147483647; export type LongTimeoutHandle = { abort: () => void }; +/** + * Like setTimeout, but waits out delays over 2^31-1 ms (about 24.8 days). + * setTimeout fires such delays after 1 ms, so this chains shorter timers. + */ export function setLongTimeout( listener: () => void, after: number, diff --git a/packages/workflows/tests/eviction-cancel.test.ts b/packages/workflows/tests/eviction-cancel.test.ts index 5e566db..28f49c8 100644 --- a/packages/workflows/tests/eviction-cancel.test.ts +++ b/packages/workflows/tests/eviction-cancel.test.ts @@ -8,6 +8,14 @@ import { const modes = ["yield", "live"] as const; +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((res) => { + resolve = res; + }); + return { promise, resolve }; +} + for (const mode of modes) { describe( `Workflow Engine Eviction and Cancellation (${mode})`, @@ -83,6 +91,46 @@ for (const mode of modes) { .result, ).rejects.toThrow(EvictedError); }); + + it("should keep the result of a step that finishes after eviction", async () => { + const started = deferred(); + const work = deferred(); + let runs = 0; + const workflow = async (ctx: WorkflowContextInterface) => + await ctx.step({ + name: "slow-step", + timeout: 60_000, + run: async () => { + runs++; + started.resolve(); + return await work.promise; + }, + }); + + const handle = runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ); + await started.promise; + handle.evict(); + work.resolve("finished after eviction"); + await handle.result; + + const resumed = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + + expect(resumed.state).toBe("completed"); + expect(resumed.output).toBe("finished after eviction"); + expect(runs).toBe(1); + }); }, ); } diff --git a/packages/workflows/tests/fixtures/long-timeout-step.ts b/packages/workflows/tests/fixtures/long-timeout-step.ts new file mode 100644 index 0000000..933733d --- /dev/null +++ b/packages/workflows/tests/fixtures/long-timeout-step.ts @@ -0,0 +1,22 @@ +// Runs one step that finishes well before its 30-day timeout, then prints the +// result. The process must exit on its own once the workflow completes. +import { InMemoryDriver, runWorkflow } from "../../src/testing.js"; + +const mode = process.argv[2] === "live" ? "live" : "yield"; +const driver = new InMemoryDriver(); +driver.latency = 0; + +const result = await runWorkflow( + "wf-1", + async (ctx) => + await ctx.step({ + name: "quick-step", + timeout: 30 * 24 * 60 * 60 * 1000, + run: async () => "done", + }), + undefined, + driver, + { mode }, +).result; + +process.stdout.write(JSON.stringify(result)); diff --git a/packages/workflows/tests/steps.test.ts b/packages/workflows/tests/steps.test.ts index 609b726..499ca5f 100644 --- a/packages/workflows/tests/steps.test.ts +++ b/packages/workflows/tests/steps.test.ts @@ -1,4 +1,8 @@ -import { beforeEach, describe, expect, it } from "vitest"; +import { execFile } from "node:child_process"; +import { getEventListeners } from "node:events"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { deserializeEntry } from "../schemas/serde.js"; import { buildHistoryPrefixAll } from "../src/keys.js"; import { @@ -16,6 +20,15 @@ import { const modes = ["yield", "live"] as const; +// Longer than the 2^31-1 ms that a single setTimeout can wait. +const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000; +const JUST_OVER_SET_TIMEOUT_LIMIT_MS = 2 ** 31; + +const execFileAsync = promisify(execFile); +const LONG_TIMEOUT_STEP_FIXTURE = fileURLToPath( + new URL("./fixtures/long-timeout-step.ts", import.meta.url), +); + class CountingDriver extends InMemoryDriver { batchCalls = 0; @@ -542,6 +555,172 @@ for (const mode of modes) { ).rejects.toThrow(CriticalError); }); + for (const [label, timeout] of [ + ["30 days", THIRTY_DAYS_MS], + ["Infinity", Number.POSITIVE_INFINITY], + ["0", 0], + ["-1", -1], + ["NaN", Number.NaN], + ] as const) { + it(`should finish steps with a timeout of ${label}`, async () => { + const workflow = async (ctx: WorkflowContextInterface) => { + return await ctx.step({ + name: "long-timeout-step", + timeout, + run: async () => { + await new Promise((resolve) => setTimeout(resolve, 25)); + return "done"; + }, + }); + }; + + const result = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + + expect(result.state).toBe("completed"); + expect(result.output).toBe("done"); + }); + } + + for (const [label, timeout] of [ + ["2^31 ms", JUST_OVER_SET_TIMEOUT_LIMIT_MS], + ["30 days", THIRTY_DAYS_MS], + ] as const) { + it(`should time out a step when a timeout of ${label} elapses`, async () => { + vi.useFakeTimers(); + try { + const workflow = async (ctx: WorkflowContextInterface) => { + return await ctx.step({ + name: "stuck-step", + timeout, + run: () => new Promise(() => {}), + }); + }; + + const result = runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + let settled = false; + result.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + + await vi.advanceTimersByTimeAsync(timeout - 1000); + expect(settled).toBe(false); + + await vi.advanceTimersByTimeAsync(2000); + await expect(result).rejects.toThrow(CriticalError); + await expect(result).rejects.toThrow( + `Step "stuck-step" timed out after ${timeout}ms`, + ); + } finally { + vi.useRealTimers(); + } + }); + } + + it("should exit the process when a step finishes before a long timeout", async () => { + // A timer left behind keeps the process alive for 24.8 days. + const { stdout } = await execFileAsync( + process.execPath, + ["--import", "tsx", LONG_TIMEOUT_STEP_FIXTURE, mode], + { timeout: 8000 }, + ); + + expect(JSON.parse(stdout)).toEqual({ + state: "completed", + output: "done", + }); + }); + + it("should leave no abort listeners on ctx.abortSignal after many steps", async () => { + // Node never warns about AbortSignal listeners, so count them. + const workflow = async (ctx: WorkflowContextInterface) => { + const listenerCount = () => + getEventListeners(ctx.abortSignal, "abort").length; + const before = listenerCount(); + for (let i = 0; i < 30; i++) { + await ctx.step({ + name: `step-${i}`, + timeout: THIRTY_DAYS_MS, + run: async () => i, + }); + } + const after = listenerCount(); + return { before, after }; + }; + + const result = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + + expect(result.state).toBe("completed"); + expect(result.output?.after).toBe(result.output?.before); + }); + + for (const retryOnTimeout of [false, true]) { + it(`should retry a step that throws when retryOnTimeout is ${retryOnTimeout}`, async () => { + let attempts = 0; + const workflow = async (ctx: WorkflowContextInterface) => { + return await ctx.step({ + name: "flaky-step", + timeout: 1000, + retryOnTimeout, + maxRetries: 3, + retryBackoffBase: 1, + retryBackoffMax: 1, + run: async () => { + attempts++; + if (attempts < 3) { + throw new Error("Transient failure"); + } + return "success"; + }, + }); + }; + + let result = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + while (result.state === "sleeping") { + await new Promise((resolve) => setTimeout(resolve, 5)); + result = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + } + + expect(result.state).toBe("completed"); + expect(result.output).toBe("success"); + expect(attempts).toBe(3); + }); + } + it("should retry steps that exceed timeout when retryOnTimeout is set", async () => { let attempts = 0; const workflow = async (ctx: WorkflowContextInterface) => {