From 0b9a240b437de3ea6939d9d1697b0997b63747c7 Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Sat, 7 Nov 2026 03:42:39 +0400 Subject: [PATCH] fix(workflows): stop long step timeouts from firing immediately --- packages/workflows/src/context.ts | 45 +++-- packages/workflows/src/types.ts | 2 +- packages/workflows/src/utils.ts | 4 + .../workflows/tests/eviction-cancel.test.ts | 48 +++++ .../tests/fixtures/long-timeout-step.ts | 22 +++ packages/workflows/tests/steps.test.ts | 181 +++++++++++++++++- 6 files changed, 277 insertions(+), 25 deletions(-) create mode 100644 packages/workflows/tests/fixtures/long-timeout-step.ts diff --git a/packages/workflows/src/context.ts b/packages/workflows/src/context.ts index c061f09..03b4bbd 100644 --- a/packages/workflows/src/context.ts +++ b/packages/workflows/src/context.ts @@ -71,7 +71,7 @@ import type { WorkflowQueueNextBatchOptions, WorkflowQueueNextOptions, } from "./types.js"; -import { setLongTimeout } from "./utils.js"; +import { type LongTimeoutHandle, setLongTimeout } from "./utils.js"; /** * Default values for step configuration. @@ -1241,34 +1241,33 @@ export class WorkflowContextImpl implements WorkflowContextInterface { timeoutMs: number, stepName: string, ): Promise { - let timeoutId: ReturnType | undefined; - let onAbort: (() => void) | undefined; + let interrupt!: (error: Error) => void; const interrupted = new Promise((_, reject) => { - if (timeoutMs > 0) { - timeoutId = setTimeout(() => { - reject(new StepTimeoutError(stepName, timeoutMs)); - }, timeoutMs); - } - - // A step that a race cancels stops being awaited, as with a timeout. - // Eviction does not interrupt a step, so its result is still recorded. - onAbort = () => { - if (this.isRaceCancelled()) { - reject(new EvictedError()); - } - }; - this.abortSignal.addEventListener("abort", onAbort, { once: true }); + interrupt = reject; }); + // A timeout of 0 or a non-finite timeout such as Infinity disables the timeout. + let timeout: LongTimeoutHandle | undefined; + if (timeoutMs > 0 && Number.isFinite(timeoutMs)) { + timeout = setLongTimeout(() => { + interrupt(new StepTimeoutError(stepName, timeoutMs)); + }, timeoutMs); + } + + // A race that cancels this step stops awaiting it, as a timeout does. + // Eviction does not interrupt a step, so its result is still recorded. + const onAbort = () => { + if (this.isRaceCancelled()) { + interrupt(new EvictedError()); + } + }; + this.abortSignal.addEventListener("abort", onAbort, { once: true }); + try { return await Promise.race([promise, interrupted]); } finally { - if (timeoutId !== undefined) { - clearTimeout(timeoutId); - } - if (onAbort) { - this.abortSignal.removeEventListener("abort", onAbort); - } + timeout?.abort(); + this.abortSignal.removeEventListener("abort", onAbort); } } diff --git a/packages/workflows/src/types.ts b/packages/workflows/src/types.ts index 5b5cb18..3687494 100644 --- a/packages/workflows/src/types.ts +++ b/packages/workflows/src/types.ts @@ -419,7 +419,7 @@ export interface StepConfig { retryBackoffBase?: number; /** Maximum delay in ms for exponential backoff (default: 30000). */ retryBackoffMax?: number; - /** Timeout in ms for step execution (default: 30000). Set to 0 to disable. */ + /** Timeout in ms for step execution (default: 30000). Set to 0 or Infinity to disable. */ timeout?: number; /** If true, step timeouts retry like any other error instead of failing immediately as critical. Default: false. */ retryOnTimeout?: boolean; diff --git a/packages/workflows/src/utils.ts b/packages/workflows/src/utils.ts index 16de15f..31fe9a2 100644 --- a/packages/workflows/src/utils.ts +++ b/packages/workflows/src/utils.ts @@ -9,6 +9,10 @@ const TIMEOUT_MAX = 2147483647; export type LongTimeoutHandle = { abort: () => void }; +/** + * Like setTimeout, but waits out delays over 2^31-1 ms (about 24.8 days). + * setTimeout fires such delays after 1 ms, so this chains shorter timers. + */ export function setLongTimeout( listener: () => void, after: number, diff --git a/packages/workflows/tests/eviction-cancel.test.ts b/packages/workflows/tests/eviction-cancel.test.ts index 5e566db..28f49c8 100644 --- a/packages/workflows/tests/eviction-cancel.test.ts +++ b/packages/workflows/tests/eviction-cancel.test.ts @@ -8,6 +8,14 @@ import { const modes = ["yield", "live"] as const; +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((res) => { + resolve = res; + }); + return { promise, resolve }; +} + for (const mode of modes) { describe( `Workflow Engine Eviction and Cancellation (${mode})`, @@ -83,6 +91,46 @@ for (const mode of modes) { .result, ).rejects.toThrow(EvictedError); }); + + it("should keep the result of a step that finishes after eviction", async () => { + const started = deferred(); + const work = deferred(); + let runs = 0; + const workflow = async (ctx: WorkflowContextInterface) => + await ctx.step({ + name: "slow-step", + timeout: 60_000, + run: async () => { + runs++; + started.resolve(); + return await work.promise; + }, + }); + + const handle = runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ); + await started.promise; + handle.evict(); + work.resolve("finished after eviction"); + await handle.result; + + const resumed = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + + expect(resumed.state).toBe("completed"); + expect(resumed.output).toBe("finished after eviction"); + expect(runs).toBe(1); + }); }, ); } diff --git a/packages/workflows/tests/fixtures/long-timeout-step.ts b/packages/workflows/tests/fixtures/long-timeout-step.ts new file mode 100644 index 0000000..933733d --- /dev/null +++ b/packages/workflows/tests/fixtures/long-timeout-step.ts @@ -0,0 +1,22 @@ +// Runs one step that finishes well before its 30-day timeout, then prints the +// result. The process must exit on its own once the workflow completes. +import { InMemoryDriver, runWorkflow } from "../../src/testing.js"; + +const mode = process.argv[2] === "live" ? "live" : "yield"; +const driver = new InMemoryDriver(); +driver.latency = 0; + +const result = await runWorkflow( + "wf-1", + async (ctx) => + await ctx.step({ + name: "quick-step", + timeout: 30 * 24 * 60 * 60 * 1000, + run: async () => "done", + }), + undefined, + driver, + { mode }, +).result; + +process.stdout.write(JSON.stringify(result)); diff --git a/packages/workflows/tests/steps.test.ts b/packages/workflows/tests/steps.test.ts index 609b726..499ca5f 100644 --- a/packages/workflows/tests/steps.test.ts +++ b/packages/workflows/tests/steps.test.ts @@ -1,4 +1,8 @@ -import { beforeEach, describe, expect, it } from "vitest"; +import { execFile } from "node:child_process"; +import { getEventListeners } from "node:events"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { deserializeEntry } from "../schemas/serde.js"; import { buildHistoryPrefixAll } from "../src/keys.js"; import { @@ -16,6 +20,15 @@ import { const modes = ["yield", "live"] as const; +// Longer than the 2^31-1 ms that a single setTimeout can wait. +const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000; +const JUST_OVER_SET_TIMEOUT_LIMIT_MS = 2 ** 31; + +const execFileAsync = promisify(execFile); +const LONG_TIMEOUT_STEP_FIXTURE = fileURLToPath( + new URL("./fixtures/long-timeout-step.ts", import.meta.url), +); + class CountingDriver extends InMemoryDriver { batchCalls = 0; @@ -542,6 +555,172 @@ for (const mode of modes) { ).rejects.toThrow(CriticalError); }); + for (const [label, timeout] of [ + ["30 days", THIRTY_DAYS_MS], + ["Infinity", Number.POSITIVE_INFINITY], + ["0", 0], + ["-1", -1], + ["NaN", Number.NaN], + ] as const) { + it(`should finish steps with a timeout of ${label}`, async () => { + const workflow = async (ctx: WorkflowContextInterface) => { + return await ctx.step({ + name: "long-timeout-step", + timeout, + run: async () => { + await new Promise((resolve) => setTimeout(resolve, 25)); + return "done"; + }, + }); + }; + + const result = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + + expect(result.state).toBe("completed"); + expect(result.output).toBe("done"); + }); + } + + for (const [label, timeout] of [ + ["2^31 ms", JUST_OVER_SET_TIMEOUT_LIMIT_MS], + ["30 days", THIRTY_DAYS_MS], + ] as const) { + it(`should time out a step when a timeout of ${label} elapses`, async () => { + vi.useFakeTimers(); + try { + const workflow = async (ctx: WorkflowContextInterface) => { + return await ctx.step({ + name: "stuck-step", + timeout, + run: () => new Promise(() => {}), + }); + }; + + const result = runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + let settled = false; + result.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + + await vi.advanceTimersByTimeAsync(timeout - 1000); + expect(settled).toBe(false); + + await vi.advanceTimersByTimeAsync(2000); + await expect(result).rejects.toThrow(CriticalError); + await expect(result).rejects.toThrow( + `Step "stuck-step" timed out after ${timeout}ms`, + ); + } finally { + vi.useRealTimers(); + } + }); + } + + it("should exit the process when a step finishes before a long timeout", async () => { + // A timer left behind keeps the process alive for 24.8 days. + const { stdout } = await execFileAsync( + process.execPath, + ["--import", "tsx", LONG_TIMEOUT_STEP_FIXTURE, mode], + { timeout: 8000 }, + ); + + expect(JSON.parse(stdout)).toEqual({ + state: "completed", + output: "done", + }); + }); + + it("should leave no abort listeners on ctx.abortSignal after many steps", async () => { + // Node never warns about AbortSignal listeners, so count them. + const workflow = async (ctx: WorkflowContextInterface) => { + const listenerCount = () => + getEventListeners(ctx.abortSignal, "abort").length; + const before = listenerCount(); + for (let i = 0; i < 30; i++) { + await ctx.step({ + name: `step-${i}`, + timeout: THIRTY_DAYS_MS, + run: async () => i, + }); + } + const after = listenerCount(); + return { before, after }; + }; + + const result = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + + expect(result.state).toBe("completed"); + expect(result.output?.after).toBe(result.output?.before); + }); + + for (const retryOnTimeout of [false, true]) { + it(`should retry a step that throws when retryOnTimeout is ${retryOnTimeout}`, async () => { + let attempts = 0; + const workflow = async (ctx: WorkflowContextInterface) => { + return await ctx.step({ + name: "flaky-step", + timeout: 1000, + retryOnTimeout, + maxRetries: 3, + retryBackoffBase: 1, + retryBackoffMax: 1, + run: async () => { + attempts++; + if (attempts < 3) { + throw new Error("Transient failure"); + } + return "success"; + }, + }); + }; + + let result = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + while (result.state === "sleeping") { + await new Promise((resolve) => setTimeout(resolve, 5)); + result = await runWorkflow( + "wf-1", + workflow, + undefined, + driver, + { mode }, + ).result; + } + + expect(result.state).toBe("completed"); + expect(result.output).toBe("success"); + expect(attempts).toBe(3); + }); + } + it("should retry steps that exceed timeout when retryOnTimeout is set", async () => { let attempts = 0; const workflow = async (ctx: WorkflowContextInterface) => {