Skip to content

Commit 6a73862

Browse files
committed
fix: harden workspace-disabled rotation retry
1 parent 247e22f commit 6a73862

6 files changed

Lines changed: 265 additions & 72 deletions

File tree

index.ts

Lines changed: 42 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1478,7 +1478,7 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => {
14781478
);
14791479
}
14801480

1481-
while (attempted.size < Math.max(1, accountCount)) {
1481+
accountAttemptLoop: while (attempted.size < Math.max(1, accountCount)) {
14821482
let account = null;
14831483
if (
14841484
!usedPreferredSessionAccount &&
@@ -1967,75 +1967,84 @@ while (attempted.size < Math.max(1, accountCount)) {
19671967
blockedModel,
19681968
);
19691969
}
1970-
if (errorResponse.status === 403 && !unsupportedModelInfo.isUnsupported) {
1971-
entitlementCache.markBlocked(
1972-
entitlementAccountKey,
1973-
model ?? modelFamily,
1974-
"plan-entitlement",
1975-
);
1976-
capabilityPolicyStore.recordFailure(
1977-
entitlementAccountKey,
1978-
capabilityModelKey,
1970+
const workspaceErrorCode =
1971+
(errorBody as { error?: { code?: string } } | undefined)?.error?.code ?? "";
1972+
const workspaceErrorMessage =
1973+
(errorBody as { error?: { message?: string } } | undefined)?.error?.message ?? "";
1974+
const isDisabledWorkspaceError =
1975+
errorResponse.status === 403 &&
1976+
isWorkspaceDisabledError(
1977+
errorResponse.status,
1978+
workspaceErrorCode,
1979+
workspaceErrorMessage,
19791980
);
1980-
}
19811981

19821982
// Handle workspace disabled/expired errors by rotating to next workspace within account
1983-
if (errorResponse.status === 403 && errorBody) {
1984-
const errorCode = (errorBody as { error?: { code?: string } })?.error?.code ?? "";
1985-
const errorMessage = (errorBody as { error?: { message?: string } })?.error?.message ?? "";
1986-
1987-
if (isWorkspaceDisabledError(errorResponse.status, errorCode, errorMessage)) {
1983+
if (isDisabledWorkspaceError) {
19881984
runtimeMetrics.failedRequests++;
19891985
runtimeMetrics.lastError = `Workspace disabled for account ${account.index + 1}`;
1990-
1986+
19911987
// Get current workspace info for logging
19921988
const currentWorkspace = accountManager.getCurrentWorkspace(account);
19931989
const workspaceName = currentWorkspace?.name ?? currentWorkspace?.id ?? "unknown";
1994-
1990+
19951991
logWarn(
1996-
`Workspace disabled/expired for account ${account.index + 1} (${account.email ?? "unknown"}) - workspace: ${workspaceName}. Rotating to next workspace.`,
1997-
{ errorCode, errorMessage },
1992+
`Workspace disabled/expired for account ${account.index + 1} - workspace: ${workspaceName}. Rotating to next workspace.`,
1993+
{ errorCode: workspaceErrorCode },
19981994
);
1999-
1995+
20001996
// Disable the current workspace
20011997
accountManager.disableCurrentWorkspace(account);
2002-
1998+
20031999
// Try to rotate to next enabled workspace
20042000
const nextWorkspace = accountManager.rotateToNextWorkspace(account);
2005-
2001+
20062002
if (nextWorkspace) {
2007-
// Found another enabled workspace, retry with it
2003+
// Found another enabled workspace, persist it and restart the
2004+
// outer account loop so accountId and headers are rebuilt.
20082005
accountManager.saveToDiskDebounced();
2009-
2006+
20102007
const newWorkspaceName = nextWorkspace.name ?? nextWorkspace.id;
20112008
await showToast(
20122009
`Workspace ${workspaceName} disabled. Switched to ${newWorkspaceName}.`,
20132010
"warning",
20142011
{ duration: toastDurationMs },
20152012
);
2016-
2013+
20172014
logInfo(`Rotated to workspace ${newWorkspaceName} for account ${account.index + 1}`);
2018-
2019-
// Retry the request with the new workspace (continue the loop)
2020-
continue;
2015+
2016+
// Allow the same account to be selected again with fresh request state.
2017+
attempted.delete(account.index);
2018+
continue accountAttemptLoop;
20212019
} else {
20222020
// No more enabled workspaces, disable the entire account
20232021
logWarn(`All workspaces disabled for account ${account.index + 1}. Disabling account.`);
2024-
2022+
20252023
accountManager.setAccountEnabled(account.index, false);
20262024
accountManager.saveToDiskDebounced();
2027-
2025+
20282026
await showToast(
20292027
`All workspaces disabled for account ${account.index + 1}. Switching to another account.`,
20302028
"warning",
20312029
{ duration: toastDurationMs },
20322030
);
2033-
2031+
20342032
// Forget session affinity and rotate to next account
20352033
sessionAffinityStore?.forgetSession(sessionAffinityKey);
20362034
break;
20372035
}
2038-
}
2036+
}
2037+
2038+
if (errorResponse.status === 403 && !unsupportedModelInfo.isUnsupported && !isDisabledWorkspaceError) {
2039+
entitlementCache.markBlocked(
2040+
entitlementAccountKey,
2041+
model ?? modelFamily,
2042+
"plan-entitlement",
2043+
);
2044+
capabilityPolicyStore.recordFailure(
2045+
entitlementAccountKey,
2046+
capabilityModelKey,
2047+
);
20392048
}
20402049

20412050
if (recoveryHook && errorBody && isRecoverableError(errorBody)) {

lib/accounts.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -849,7 +849,7 @@ export class AccountManager {
849849
const workspace = account.workspaces[idx];
850850
if (!workspace) return false;
851851
workspace.enabled = false;
852-
workspace.disabledAt = Date.now();
852+
workspace.disabledAt = nowMs();
853853
return true;
854854
}
855855

lib/request/fetch-helpers.ts

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -290,22 +290,24 @@ export function isWorkspaceDisabledError(status: number, code: string, bodyText:
290290
}
291291
}
292292

293-
// Check for specific error codes that indicate workspace issues
294-
const workspaceErrorCodes = [
295-
"workspace_disabled",
296-
"workspace_expired",
297-
"workspace_terminated",
298-
"account_disabled",
299-
"account_expired",
300-
"organization_disabled",
301-
"billing_failed",
302-
"payment_required",
303-
];
304-
305-
if (workspaceErrorCodes.some((c) => code.toLowerCase().includes(c))) {
306-
return true;
293+
// Check for specific error codes that indicate workspace issues.
294+
if (status === 403) {
295+
const workspaceErrorCodes = [
296+
"workspace_disabled",
297+
"workspace_expired",
298+
"workspace_terminated",
299+
"account_disabled",
300+
"account_expired",
301+
"organization_disabled",
302+
"billing_failed",
303+
"payment_required",
304+
];
305+
306+
if (workspaceErrorCodes.some((c) => code.toLowerCase().includes(c))) {
307+
return true;
308+
}
307309
}
308-
310+
309311
return false;
310312
}
311313

lib/storage/migrations.ts

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,10 @@
55

66
import { MODEL_FAMILIES, type ModelFamily } from "../prompts/codex.js";
77
import type { AccountIdSource } from "../types.js";
8+
import type { Workspace } from "../accounts.js";
89

910
export type CooldownReason = "auth-failure" | "network-error" | "rate-limit";
1011

11-
export interface Workspace {
12-
id: string;
13-
name?: string;
14-
enabled: boolean;
15-
disabledAt?: number;
16-
isDefault?: boolean;
17-
}
18-
1912
export interface RateLimitStateV3 {
2013
[key: string]: number | undefined;
2114
}

test/fetch-helpers.test.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,8 @@ describe('isWorkspaceDisabledError', () => {
346346
expect(isWorkspaceDisabledError(400, '', 'Your workspace has been disabled')).toBe(false);
347347
expect(isWorkspaceDisabledError(401, '', 'Your workspace has been disabled')).toBe(false);
348348
expect(isWorkspaceDisabledError(500, '', 'Your workspace has been disabled')).toBe(false);
349+
expect(isWorkspaceDisabledError(400, 'workspace_disabled', '')).toBe(false);
350+
expect(isWorkspaceDisabledError(402, 'payment_required', '')).toBe(false);
349351
});
350352

351353
it('returns false for 403 with unrelated messages', () => {

0 commit comments

Comments
 (0)