Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions lib/index.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion lib/index.js.map

Large diffs are not rendered by default.

5 changes: 4 additions & 1 deletion src/adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2032,6 +2032,8 @@ interface GenerateConnectDeps {
options: GenerateOptions
connection: CommandCodeConnectionOptions
fetchImpl: typeof fetch
/** Requested ceiling before the raw-history clamp, for compression-aware forwarding. */
requestedMaxTokens: number
/** Observe headers even on rejected attempts before rotation/fallback. */
onResponse: (response: Response) => void
}
Expand Down Expand Up @@ -2106,6 +2108,7 @@ async function connectGenerate(
...IDENTITY_ENCODING_HEADER,
Authorization: `Bearer ${key}`,
Accept: 'text/event-stream',
'x-bili-output-budget': `${deps.requestedMaxTokens}:${body.max_tokens}`,
// Deliberately no x-command-code-version / x-cli-environment:
// this is the documented OpenAI-format surface, not the CLI
// transport — do not "fix" these in. ZDR is the one header both
Expand Down Expand Up @@ -3020,7 +3023,7 @@ export class CommandCodeAdapter<C extends CommandCodeConnectionOptions = Command
let attemptNumber = 0
let responseMetadata: ResponseMetadata = { protocol, headers: {} }
const connectDeps: GenerateConnectDeps = {
options, connection, fetchImpl: this.fetchImpl,
options, connection, fetchImpl: this.fetchImpl, requestedMaxTokens,
onResponse: (response) => {
// Any answer from the gateway ends a header-timeout streak, whatever
// its status: the silence being measured is over.
Expand Down
28 changes: 28 additions & 0 deletions tests/adapter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3579,6 +3579,34 @@ test('stream() clamps a large output reservation against the known model window
}
})

test('stream() hands off the requested output budget without weakening the direct request clamp', async () => {
const model = 'stealth/pixel-canary'
const catalog = JSON.stringify({ object: 'list', data: [{ id: model, name: 'Pixel Canary', context_length: 262_144 }] })
for (const protocol of ['openai', 'cli'] as const) {
const sent: { budget: number; handoff: string | null }[] = []
const adapter = makeAdapter({
options: () => ({ ...OPENAI_OPTIONS(), protocol }),
fetchImpl: (async (input: RequestInfo | URL, init?: RequestInit) => {
if (String(input).endsWith('/models')) return new Response(catalog)
const body = JSON.parse(String(init?.body)) as Record<string, unknown>
const params = protocol === 'cli' ? body.params as Record<string, unknown> : body
sent.push({ budget: params.max_tokens as number, handoff: new Headers(init?.headers).get('x-bili-output-budget') })
return new Response(protocol === 'cli'
? 'data: {"type":"text-delta","text":"ok"}\n\ndata: {"type":"finish","finishReason":"stop"}\n\n'
: 'data: {"choices":[{"delta":{"content":"ok"},"finish_reason":"stop"}]}\n\n')
}) as typeof fetch,
})
await adapter.listModels('commandcode', { unfiltered: true })
const messages = [userMessage('alpha '.repeat(300_000))]
await collect(adapter.stream({ provider: 'commandcode', model, maxTokens: 131_072, messages }))
await collect(adapter.stream({ provider: 'commandcode', model, maxTokens: 1_024, messages }))
assert.deepEqual(sent, [
{ budget: 1_024, handoff: protocol === 'openai' ? '131072:1024' : null },
{ budget: 1_024, handoff: protocol === 'openai' ? '1024:1024' : null },
])
}
})

test('stream() budgets Chinese text against the model window', async () => {
const model = 'stealth/pixel-canary'
const catalog = JSON.stringify({ object: 'list', data: [{ id: model, name: 'Pixel Canary', context_length: 100_000 }] })
Expand Down