Skip to content

Commit 06d2bb7

Browse files
committed
Fix AI provider request cancellation, refs #845.
1 parent 3f3eb55 commit 06d2bb7

5 files changed

Lines changed: 67 additions & 3 deletions

File tree

‎docs/guide/ai.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,17 @@ through that installation's proxy. AI Gateway cache, timeout and retry headers,
156156
such as `cf-aig-cache-ttl` and `cf-aig-max-attempts`, still apply there; the
157157
platform ignores logging, metadata and cost headers.
158158

159+
Pass an `AbortSignal` to cancel a request or stop reading its response stream:
160+
161+
```ts
162+
import { ai } from 'void/ai';
163+
164+
const response = await ai.provider('openai').fetch('/chat/completions', {
165+
signal: AbortSignal.timeout(30_000),
166+
body: { model: 'gpt-4o', messages: [{ role: 'user', content: 'Hello' }] },
167+
});
168+
```
169+
159170
### OpenAI
160171

161172
```ts

‎packages/void/src/runtime/ai/index.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -376,7 +376,7 @@ function makeProxyBackend(send: ProxySend, runtimeEnv: RuntimeEnv): AiBackend {
376376
async function proxyFetch(
377377
action: string,
378378
payload: Record<string, unknown>,
379-
options: { throwOnError?: boolean } = {},
379+
options: { throwOnError?: boolean; signal?: AbortSignal | null } = {},
380380
): Promise<Response> {
381381
let signal: AbortSignal | undefined;
382382
if (typeof payload.options === 'object' && payload.options !== null) {
@@ -385,7 +385,7 @@ function makeProxyBackend(send: ProxySend, runtimeEnv: RuntimeEnv): AiBackend {
385385
signal = bindingSignal;
386386
payload = { ...payload, options: wireOptions };
387387
}
388-
const res = await send(action, payload, signal);
388+
const res = await send(action, payload, options.signal ?? signal);
389389
const limit = await limitFromResponse(res, 'ai');
390390
if (limit) {
391391
throw limit;
@@ -412,7 +412,7 @@ function makeProxyBackend(send: ProxySend, runtimeEnv: RuntimeEnv): AiBackend {
412412
return proxyFetch(
413413
'provider',
414414
await providerRequestBody(provider, path, init, options, runtimeEnv),
415-
{ throwOnError: false },
415+
{ throwOnError: false, signal: init?.signal },
416416
);
417417
},
418418
async models(params) {
@@ -546,6 +546,7 @@ function makeDirectBackend(
546546
method: init?.method ?? 'POST',
547547
headers,
548548
body,
549+
signal: init?.signal,
549550
});
550551
},
551552
models: directAi.models.bind(directAi),

‎packages/void/src/runtime/ai/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ export type AiProviderFetchInit = {
4848
method?: string;
4949
headers?: HeadersInit;
5050
body?: unknown;
51+
signal?: AbortSignal | null;
5152
};
5253

5354
/** Maps provider prefix to expected env var name for the API key */

‎packages/void/test/unit/ai.test.ts‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,55 @@ function makeDirectAi(
1818
}
1919

2020
describe('runtime ai', () => {
21+
it.each(['direct', 'http', 'service'] as const)(
22+
'cancels provider response streams on the %s transport without serializing the signal',
23+
async (transport) => {
24+
vi.resetModules();
25+
const { envContext } = await import('../../src/runtime/env/raw.ts');
26+
const { ai } = await import('../../src/runtime/ai/index.ts');
27+
const controller = new AbortController();
28+
const request = vi.fn(
29+
async (_url: string, init: RequestInit) =>
30+
new Response(
31+
new ReadableStream({
32+
start(stream) {
33+
init.signal?.addEventListener('abort', () => stream.error(init.signal!.reason));
34+
},
35+
}),
36+
),
37+
);
38+
vi.stubGlobal('fetch', request);
39+
const env =
40+
transport === 'direct'
41+
? { AI: makeDirectAi().AI, __VOID_AI_GATEWAY: 'gateway' }
42+
: transport === 'service'
43+
? {
44+
__VOID_PROXY: { fetch: request },
45+
__PROJECT_ID: 'project',
46+
__VOID_PROXY_TOKEN: 'token',
47+
}
48+
: {
49+
__VOID_TOKEN: 'token',
50+
__VOID_PROJECT_ID: 'project',
51+
__VOID_PROXY_URL: 'https://proxy.test',
52+
};
53+
await envContext.run({ ...env, OPENAI_API_KEY: 'synthetic-provider-key' }, async () => {
54+
const response = await ai.provider('openai').fetch('/chat/completions', {
55+
body: { messages: [] },
56+
signal: controller.signal,
57+
});
58+
const init = request.mock.calls[0]![1];
59+
expect(init.signal).toBe(controller.signal);
60+
const payload = JSON.parse(init.body as string);
61+
expect(payload).not.toHaveProperty('signal');
62+
expect(payload.options).toBeUndefined();
63+
const reading = response.text();
64+
controller.abort(new Error('provider request cancelled'));
65+
await expect(reading).rejects.toThrow('provider request cancelled');
66+
});
67+
},
68+
);
69+
2170
it.each(['http', 'service'] as const)(
2271
'keeps cancellation on the %s transport instead of serializing it as a binding option',
2372
async (transport) => {

‎packages/void/test/unit/ai.typecheck.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,14 @@ expectTypeOf(client.provider('openai').fetch('/chat/completions')).toEqualTypeOf
1212
>();
1313

1414
client.provider('openai').fetch('/chat/completions', {
15+
signal: AbortSignal.timeout(30_000),
1516
body: {
1617
model: 'gpt-4o',
1718
messages: [{ role: 'user', content: 'Hello' }],
1819
},
1920
});
2021
client.provider('custom', { apiKeyEnv: 'CUSTOM_AI_KEY' }).fetch('/v1/respond', {
22+
signal: null,
2123
method: 'POST',
2224
headers: { 'x-test': '1' },
2325
body: new Uint8Array([1, 2]),

0 commit comments

Comments
 (0)