Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 22 additions & 23 deletions packages/workflows/src/context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ import type {
WorkflowQueueNextBatchOptions,
WorkflowQueueNextOptions,
} from "./types.js";
import { setLongTimeout } from "./utils.js";
import { type LongTimeoutHandle, setLongTimeout } from "./utils.js";

/**
* Default values for step configuration.
Expand Down Expand Up @@ -1241,34 +1241,33 @@ export class WorkflowContextImpl implements WorkflowContextInterface {
timeoutMs: number,
stepName: string,
): Promise<T> {
let timeoutId: ReturnType<typeof setTimeout> | undefined;
let onAbort: (() => void) | undefined;
let interrupt!: (error: Error) => void;
const interrupted = new Promise<never>((_, reject) => {
if (timeoutMs > 0) {
timeoutId = setTimeout(() => {
reject(new StepTimeoutError(stepName, timeoutMs));
}, timeoutMs);
}

// A step that a race cancels stops being awaited, as with a timeout.
// Eviction does not interrupt a step, so its result is still recorded.
onAbort = () => {
if (this.isRaceCancelled()) {
reject(new EvictedError());
}
};
this.abortSignal.addEventListener("abort", onAbort, { once: true });
interrupt = reject;
});

// A timeout of 0 or a non-finite timeout such as Infinity disables the timeout.
let timeout: LongTimeoutHandle | undefined;
if (timeoutMs > 0 && Number.isFinite(timeoutMs)) {
timeout = setLongTimeout(() => {
interrupt(new StepTimeoutError(stepName, timeoutMs));
}, timeoutMs);
}

// A race that cancels this step stops awaiting it, as a timeout does.
// Eviction does not interrupt a step, so its result is still recorded.
const onAbort = () => {
if (this.isRaceCancelled()) {
interrupt(new EvictedError());
}
};
this.abortSignal.addEventListener("abort", onAbort, { once: true });

try {
return await Promise.race([promise, interrupted]);
} finally {
if (timeoutId !== undefined) {
clearTimeout(timeoutId);
}
if (onAbort) {
this.abortSignal.removeEventListener("abort", onAbort);
}
timeout?.abort();
this.abortSignal.removeEventListener("abort", onAbort);
}
}

Expand Down
2 changes: 1 addition & 1 deletion packages/workflows/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -419,7 +419,7 @@ export interface StepConfig<T> {
retryBackoffBase?: number;
/** Maximum delay in ms for exponential backoff (default: 30000). */
retryBackoffMax?: number;
/** Timeout in ms for step execution (default: 30000). Set to 0 to disable. */
/** Timeout in ms for step execution (default: 30000). Set to 0 or Infinity to disable. */
timeout?: number;
/** If true, step timeouts retry like any other error instead of failing immediately as critical. Default: false. */
retryOnTimeout?: boolean;
Expand Down
4 changes: 4 additions & 0 deletions packages/workflows/src/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ const TIMEOUT_MAX = 2147483647;

export type LongTimeoutHandle = { abort: () => void };

/**
* Like setTimeout, but waits out delays over 2^31-1 ms (about 24.8 days).
* setTimeout fires such delays after 1 ms, so this chains shorter timers.
*/
export function setLongTimeout(
listener: () => void,
after: number,
Expand Down
48 changes: 48 additions & 0 deletions packages/workflows/tests/eviction-cancel.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,14 @@ import {

const modes = ["yield", "live"] as const;

function deferred<T>() {
let resolve!: (value: T) => void;
const promise = new Promise<T>((res) => {
resolve = res;
});
return { promise, resolve };
}

for (const mode of modes) {
describe(
`Workflow Engine Eviction and Cancellation (${mode})`,
Expand Down Expand Up @@ -83,6 +91,46 @@ for (const mode of modes) {
.result,
).rejects.toThrow(EvictedError);
});

it("should keep the result of a step that finishes after eviction", async () => {
const started = deferred<void>();
const work = deferred<string>();
let runs = 0;
const workflow = async (ctx: WorkflowContextInterface) =>
await ctx.step({
name: "slow-step",
timeout: 60_000,
run: async () => {
runs++;
started.resolve();
return await work.promise;
},
});

const handle = runWorkflow(
"wf-1",
workflow,
undefined,
driver,
{ mode },
);
await started.promise;
handle.evict();
work.resolve("finished after eviction");
await handle.result;

const resumed = await runWorkflow(
"wf-1",
workflow,
undefined,
driver,
{ mode },
).result;

expect(resumed.state).toBe("completed");
expect(resumed.output).toBe("finished after eviction");
expect(runs).toBe(1);
});
},
);
}
22 changes: 22 additions & 0 deletions packages/workflows/tests/fixtures/long-timeout-step.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
// Runs one step that finishes well before its 30-day timeout, then prints the
// result. The process must exit on its own once the workflow completes.
import { InMemoryDriver, runWorkflow } from "../../src/testing.js";

const mode = process.argv[2] === "live" ? "live" : "yield";
const driver = new InMemoryDriver();
driver.latency = 0;

const result = await runWorkflow(
"wf-1",
async (ctx) =>
await ctx.step({
name: "quick-step",
timeout: 30 * 24 * 60 * 60 * 1000,
run: async () => "done",
}),
undefined,
driver,
{ mode },
).result;

process.stdout.write(JSON.stringify(result));
181 changes: 180 additions & 1 deletion packages/workflows/tests/steps.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
import { beforeEach, describe, expect, it } from "vitest";
import { execFile } from "node:child_process";
import { getEventListeners } from "node:events";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { deserializeEntry } from "../schemas/serde.js";
import { buildHistoryPrefixAll } from "../src/keys.js";
import {
Expand All @@ -16,6 +20,15 @@ import {

const modes = ["yield", "live"] as const;

// Longer than the 2^31-1 ms that a single setTimeout can wait.
const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000;
const JUST_OVER_SET_TIMEOUT_LIMIT_MS = 2 ** 31;

const execFileAsync = promisify(execFile);
const LONG_TIMEOUT_STEP_FIXTURE = fileURLToPath(
new URL("./fixtures/long-timeout-step.ts", import.meta.url),
);

class CountingDriver extends InMemoryDriver {
batchCalls = 0;

Expand Down Expand Up @@ -542,6 +555,172 @@ for (const mode of modes) {
).rejects.toThrow(CriticalError);
});

for (const [label, timeout] of [
["30 days", THIRTY_DAYS_MS],
["Infinity", Number.POSITIVE_INFINITY],
["0", 0],
["-1", -1],
["NaN", Number.NaN],
] as const) {
it(`should finish steps with a timeout of ${label}`, async () => {
const workflow = async (ctx: WorkflowContextInterface) => {
return await ctx.step({
name: "long-timeout-step",
timeout,
run: async () => {
await new Promise((resolve) => setTimeout(resolve, 25));
return "done";
},
});
};

const result = await runWorkflow(
"wf-1",
workflow,
undefined,
driver,
{ mode },
).result;

expect(result.state).toBe("completed");
expect(result.output).toBe("done");
});
}

for (const [label, timeout] of [
["2^31 ms", JUST_OVER_SET_TIMEOUT_LIMIT_MS],
["30 days", THIRTY_DAYS_MS],
] as const) {
it(`should time out a step when a timeout of ${label} elapses`, async () => {
vi.useFakeTimers();
try {
const workflow = async (ctx: WorkflowContextInterface) => {
return await ctx.step({
name: "stuck-step",
timeout,
run: () => new Promise<string>(() => {}),
});
};

const result = runWorkflow(
"wf-1",
workflow,
undefined,
driver,
{ mode },
).result;
let settled = false;
result.then(
() => {
settled = true;
},
() => {
settled = true;
},
);

await vi.advanceTimersByTimeAsync(timeout - 1000);
expect(settled).toBe(false);

await vi.advanceTimersByTimeAsync(2000);
await expect(result).rejects.toThrow(CriticalError);
await expect(result).rejects.toThrow(
`Step "stuck-step" timed out after ${timeout}ms`,
);
} finally {
vi.useRealTimers();
}
});
}

it("should exit the process when a step finishes before a long timeout", async () => {
// A timer left behind keeps the process alive for 24.8 days.
const { stdout } = await execFileAsync(
process.execPath,
["--import", "tsx", LONG_TIMEOUT_STEP_FIXTURE, mode],
{ timeout: 8000 },
);

expect(JSON.parse(stdout)).toEqual({
state: "completed",
output: "done",
});
});

it("should leave no abort listeners on ctx.abortSignal after many steps", async () => {
// Node never warns about AbortSignal listeners, so count them.
const workflow = async (ctx: WorkflowContextInterface) => {
const listenerCount = () =>
getEventListeners(ctx.abortSignal, "abort").length;
const before = listenerCount();
for (let i = 0; i < 30; i++) {
await ctx.step({
name: `step-${i}`,
timeout: THIRTY_DAYS_MS,
run: async () => i,
});
}
const after = listenerCount();
return { before, after };
};

const result = await runWorkflow(
"wf-1",
workflow,
undefined,
driver,
{ mode },
).result;

expect(result.state).toBe("completed");
expect(result.output?.after).toBe(result.output?.before);
});

for (const retryOnTimeout of [false, true]) {
it(`should retry a step that throws when retryOnTimeout is ${retryOnTimeout}`, async () => {
let attempts = 0;
const workflow = async (ctx: WorkflowContextInterface) => {
return await ctx.step({
name: "flaky-step",
timeout: 1000,
retryOnTimeout,
maxRetries: 3,
retryBackoffBase: 1,
retryBackoffMax: 1,
run: async () => {
attempts++;
if (attempts < 3) {
throw new Error("Transient failure");
}
return "success";
},
});
};

let result = await runWorkflow(
"wf-1",
workflow,
undefined,
driver,
{ mode },
).result;
while (result.state === "sleeping") {
await new Promise((resolve) => setTimeout(resolve, 5));
result = await runWorkflow(
"wf-1",
workflow,
undefined,
driver,
{ mode },
).result;
}

expect(result.state).toBe("completed");
expect(result.output).toBe("success");
expect(attempts).toBe(3);
});
}

it("should retry steps that exceed timeout when retryOnTimeout is set", async () => {
let attempts = 0;
const workflow = async (ctx: WorkflowContextInterface) => {
Expand Down
Loading