Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 72 additions & 24 deletions src/server/providers/adapters/azure.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { afterEach, describe, expect, it } from 'vitest'
import { afterEach, describe, expect, it, vi } from 'vitest'

import { AZURE_MODELS_URL, AZURE_REASONING_URL } from '../azure-models.ts'
import { AZURE_PRICES_URL, AZURE_PRICING_PAGE_URL } from '../azure-pricing.ts'
Expand Down Expand Up @@ -96,8 +96,12 @@ function stubFetch(
urls: Array<string> = [],
pricingPage = PRICING_PAGE,
reasoning = REASONING,
prices: Record<string, unknown> | 'throttle-page-2' = PRICES,
prices:
| Record<string, unknown>
| 'throttle-page-2'
| 'throttle-page-2-once' = PRICES,
) {
let pageTwoCalls = 0
globalThis.fetch = ((input: string, init?: RequestInit) => {
const url = String(input)
urls.push(url)
Expand All @@ -107,15 +111,23 @@ function stubFetch(
new Response(url === AZURE_MODELS_URL ? models : reasoning),
)
}
if (prices === 'throttle-page-2' && url === PAGE_TWO) {
if (url === PAGE_TWO) pageTwoCalls++
if (
url === PAGE_TWO &&
(prices === 'throttle-page-2' ||
(prices === 'throttle-page-2-once' && pageTwoCalls === 1))
) {
return Promise.resolve(
new Response('Too many requests at this point. Please retry later.', {
status: 429,
statusText: 'Too Many Requests',
}),
)
}
const priceDoc = prices === 'throttle-page-2' ? PRICES : prices
const priceDoc =
prices === 'throttle-page-2' || prices === 'throttle-page-2-once'
? PRICES
: prices
if (url in priceDoc) return Promise.resolve(Response.json(priceDoc[url]))
if (url === AZURE_PRICING_PAGE_URL) {
return Promise.resolve(new Response(pricingPage))
Expand Down Expand Up @@ -173,27 +185,63 @@ describe('azure', () => {
expect(docsFailures).toEqual({ failed: 0, skipped: 0, first: [] })
})

it('keeps the poll when a later retail prices page is throttled', async () => {
stubFetch(MODELS, [], PRICING_PAGE, REASONING, 'throttle-page-2')
const { models, docsFailures } = await provider.listModels({})
/** Advance the price-list retry sleeps without waiting on the clock. */
async function flushed<T>(promise: Promise<T>): Promise<T> {
void promise.catch(() => undefined)
await vi.runAllTimersAsync()
return promise
}

expect(models.map((model) => model.rawId)).toEqual(['o3', 'gpt-image-2'])
expect(models[0]).toMatchObject({
contextWindow: 200_000,
reasoning: {
mode: 'effort',
mandatory: null,
efforts: ['low', 'medium', 'high'],
},
absent: { pricing: 'unavailable' },
})
expect(models[0]?.pricing).toBeNull()
expect(docsFailures).toMatchObject({ failed: 1, skipped: 0 })
expect(docsFailures?.first).toHaveLength(1)
expect(docsFailures?.first[0]?.source).toBe(AZURE_PRICES_URL)
expect(docsFailures?.first[0]?.error).toContain(
`${PAGE_TWO} → 429 Too Many Requests`,
)
it('retries a 429 on a later retail prices page and keeps the price', async () => {
const urls: Array<string> = []
stubFetch(MODELS, urls, PRICING_PAGE, REASONING, 'throttle-page-2-once')
vi.useFakeTimers()
try {
const { models, docsFailures } = await flushed(provider.listModels({}))

expect(urls.filter((url) => url === PAGE_TWO)).toHaveLength(2)
expect(docsFailures).toEqual({ failed: 0, skipped: 0, first: [] })
expect(models[0]).toMatchObject({
pricing: {
tables: {
rate: { base: { input_tokens: 2e-6, output_tokens: 8e-6 } },
},
},
})
expect(models[0]?.absent).toBeUndefined()
} finally {
vi.useRealTimers()
}
})

it('keeps the poll when every retry of a later retail prices page is throttled', async () => {
const urls: Array<string> = []
stubFetch(MODELS, urls, PRICING_PAGE, REASONING, 'throttle-page-2')
vi.useFakeTimers()
try {
const { models, docsFailures } = await flushed(provider.listModels({}))

expect(urls.filter((url) => url === PAGE_TWO)).toHaveLength(5)
expect(models.map((model) => model.rawId)).toEqual(['o3', 'gpt-image-2'])
expect(models[0]).toMatchObject({
contextWindow: 200_000,
reasoning: {
mode: 'effort',
mandatory: null,
efforts: ['low', 'medium', 'high'],
},
absent: { pricing: 'unavailable' },
})
expect(models[0]?.pricing).toBeNull()
expect(docsFailures).toMatchObject({ failed: 1, skipped: 0 })
expect(docsFailures?.first).toHaveLength(1)
expect(docsFailures?.first[0]?.source).toBe(AZURE_PRICES_URL)
expect(docsFailures?.first[0]?.error).toContain(
`fetch failed after 5 attempts: ${PAGE_TWO} → 429 Too Many Requests`,
)
} finally {
vi.useRealTimers()
}
})

it('reports a docs failure when the pricing page labels no threshold', async () => {
Expand Down
5 changes: 3 additions & 2 deletions src/server/providers/adapters/azure.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,9 @@ async function listModels(
hash: await sha256Text(markdown),
}
}),
// One throttled page throws. That keeps the stored cards; the next
// poll is the retry. The models article stays outside this wrapper.
// Each price-list page retries 429 and 5xx. When every attempt on a
// page fails the load throws, which keeps the stored cards. The models
// article stays outside this wrapper.
tryDocs(docs, AZURE_PRICES_URL, (cached) =>
cached(kv, AZURE_PRICES_URL, fetchAzurePrices),
),
Expand Down
18 changes: 13 additions & 5 deletions src/server/providers/azure-pricing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
*/
import { compileTokenCard } from '@modelschemas/rate-card'

import { fetchJson, fetchText, sha256Text } from './types.ts'
import { fetchText, fetchWithRetry, sha256Text } from './types.ts'
import type { ModelInfo } from './types.ts'

/**
Expand Down Expand Up @@ -112,6 +112,11 @@ interface RetailItem {
retailPrice?: number
}

interface RetailPage {
Items?: Array<RetailItem>
NextPageLink?: string | null
}

const UNIT_TOKENS: Record<string, number> = { '1K': 1e3, '1M': 1e6 }

export function parseAzureMeters(items: Array<RetailItem>): Array<AzureMeter> {
Expand Down Expand Up @@ -251,10 +256,13 @@ async function fetchMeters(): Promise<Array<AzureMeter>> {
if (page === MAX_PAGES) {
throw new Error(`azure: price list runs past ${String(MAX_PAGES)} pages`)
}
const body = (await fetchJson(url)) as {
Items?: Array<RetailItem>
NextPageLink?: string | null
}
// A 429 on any page used to fail the whole price load (#691). Retry
// that page; the last failure still throws with its URL.
const body: RetailPage = await fetchWithRetry(
url,
() => ({}),
async (response) => (await response.json()) as RetailPage,
)
items.push(...(body.Items ?? []))
url = body.NextPageLink ?? null
}
Expand Down
Loading