Skip to content

Commit 4c34aa7

Browse files
vtkovapivtkovapi
committed
feat: add cleanup commands and enhance API request handling
- Introduced `cleanup:dev` and `cleanup:prod` commands in package.json for resource management. - Implemented throttling and retry logic in API requests to handle rate limits and improve reliability. - Enhanced error handling for API responses, including detailed logging for rate limit scenarios. - Updated state management to support new cleanup functionality and ensure proper resource deletion.
1 parent d76fb80 commit 4c34aa7

8 files changed

Lines changed: 360 additions & 26 deletions

File tree

‎package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
"pull:prod": "tsx src/pull.ts prod",
1212
"call:dev": "tsx src/call.ts dev",
1313
"call:prod": "tsx src/call.ts prod",
14+
"cleanup:dev": "tsx src/cleanup.ts dev",
15+
"cleanup:prod": "tsx src/cleanup.ts prod",
1416
"build": "tsc --noEmit"
1517
},
1618
"devDependencies": {

‎src/api.ts‎

Lines changed: 64 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,47 +5,94 @@ import type { VapiResponse } from "./types.ts";
55
// HTTP Client for Vapi API
66
// ─────────────────────────────────────────────────────────────────────────────
77

8+
const MAX_RETRIES = 5;
9+
const INITIAL_DELAY_MS = 2000;
10+
const REQUEST_DELAY_MS = 700; // Delay between requests to avoid rate limits
11+
12+
let lastRequestTime = 0;
13+
14+
async function sleep(ms: number): Promise<void> {
15+
return new Promise((resolve) => setTimeout(resolve, ms));
16+
}
17+
18+
async function throttle(): Promise<void> {
19+
const now = Date.now();
20+
const timeSinceLastRequest = now - lastRequestTime;
21+
if (timeSinceLastRequest < REQUEST_DELAY_MS) {
22+
await sleep(REQUEST_DELAY_MS - timeSinceLastRequest);
23+
}
24+
lastRequestTime = Date.now();
25+
}
26+
827
export async function vapiRequest<T = VapiResponse>(
928
method: "POST" | "PATCH",
1029
endpoint: string,
1130
body: Record<string, unknown>
1231
): Promise<T> {
1332
const url = `${VAPI_BASE_URL}${endpoint}`;
1433

15-
const response = await fetch(url, {
16-
method,
17-
headers: {
18-
"Content-Type": "application/json",
19-
Authorization: `Bearer ${VAPI_TOKEN}`,
20-
},
21-
body: JSON.stringify(body),
22-
});
34+
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
35+
await throttle();
36+
const response = await fetch(url, {
37+
method,
38+
headers: {
39+
"Content-Type": "application/json",
40+
Authorization: `Bearer ${VAPI_TOKEN}`,
41+
},
42+
body: JSON.stringify(body),
43+
});
44+
45+
if (response.ok) {
46+
return response.json() as Promise<T>;
47+
}
48+
49+
// Handle rate limit with retry
50+
if (response.status === 429 && attempt < MAX_RETRIES) {
51+
const delay = INITIAL_DELAY_MS * Math.pow(2, attempt);
52+
console.log(` ⏳ Rate limited, retrying in ${delay / 1000}s (attempt ${attempt + 1}/${MAX_RETRIES})...`);
53+
await sleep(delay);
54+
continue;
55+
}
2356

24-
if (!response.ok) {
2557
const errorText = await response.text();
2658
throw new Error(
2759
`API ${method} ${endpoint} failed (${response.status}): ${errorText}`
2860
);
2961
}
3062

31-
return response.json() as Promise<T>;
63+
throw new Error(`API ${method} ${endpoint} failed: max retries exceeded`);
3264
}
3365

3466
export async function vapiDelete(endpoint: string): Promise<void> {
3567
const url = `${VAPI_BASE_URL}${endpoint}`;
3668

37-
const response = await fetch(url, {
38-
method: "DELETE",
39-
headers: {
40-
Authorization: `Bearer ${VAPI_TOKEN}`,
41-
},
42-
});
69+
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
70+
await throttle();
71+
const response = await fetch(url, {
72+
method: "DELETE",
73+
headers: {
74+
Authorization: `Bearer ${VAPI_TOKEN}`,
75+
},
76+
});
77+
78+
if (response.ok) {
79+
return;
80+
}
81+
82+
// Handle rate limit with retry
83+
if (response.status === 429 && attempt < MAX_RETRIES) {
84+
const delay = INITIAL_DELAY_MS * Math.pow(2, attempt);
85+
console.log(` ⏳ Rate limited, retrying in ${delay / 1000}s (attempt ${attempt + 1}/${MAX_RETRIES})...`);
86+
await sleep(delay);
87+
continue;
88+
}
4389

44-
if (!response.ok) {
4590
const errorText = await response.text();
4691
throw new Error(
4792
`API DELETE ${endpoint} failed (${response.status}): ${errorText}`
4893
);
4994
}
95+
96+
throw new Error(`API DELETE ${endpoint} failed: max retries exceeded`);
5097
}
5198

‎src/apply.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { vapiRequest } from "./api.ts";
2-
import { VAPI_ENV, VAPI_BASE_URL, removeExcludedKeys } from "./config.ts";
2+
import { VAPI_ENV, VAPI_BASE_URL, FORCE_DELETE, removeExcludedKeys } from "./config.ts";
33
import { loadState, saveState } from "./state.ts";
44
import { loadResources } from "./resources.ts";
55
import { resolveReferences, resolveAssistantIds } from "./resolver.ts";
@@ -162,8 +162,8 @@ export async function applyScenario(
162162
const { resourceId, data } = resource;
163163
const existingUuid = state.scenarios[resourceId];
164164

165-
// Scenarios have no external references to resolve
166-
const payload = data as Record<string, unknown>;
165+
// Resolve structuredOutputId references in evaluations
166+
const payload = resolveReferences(data as Record<string, unknown>, state);
167167

168168
if (existingUuid) {
169169
const updatePayload = removeExcludedKeys(payload, "scenarios");
@@ -301,6 +301,7 @@ async function main(): Promise<void> {
301301
console.log("═══════════════════════════════════════════════════════════════");
302302
console.log(`🚀 Vapi GitOps Apply - Environment: ${VAPI_ENV}`);
303303
console.log(` API: ${VAPI_BASE_URL}`);
304+
console.log(` Deletions: ${FORCE_DELETE ? "⚠️ ENABLED (--force)" : "🔒 Disabled (dry-run)"}`);
304305
console.log("═══════════════════════════════════════════════════════════════\n");
305306

306307
// Load current state

‎src/cleanup.ts‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
import { VAPI_ENV, VAPI_BASE_URL, VAPI_TOKEN } from "./config.ts";
2+
import { loadState } from "./state.ts";
3+
4+
// ─────────────────────────────────────────────────────────────────────────────
5+
// Dangerous Sync - Delete everything NOT in state file
6+
// ─────────────────────────────────────────────────────────────────────────────
7+
8+
const REQUEST_DELAY_MS = 700;
9+
10+
async function sleep(ms: number): Promise<void> {
11+
return new Promise((resolve) => setTimeout(resolve, ms));
12+
}
13+
14+
async function vapiGet<T>(endpoint: string, debug = false): Promise<T> {
15+
await sleep(REQUEST_DELAY_MS);
16+
const response = await fetch(`${VAPI_BASE_URL}${endpoint}`, {
17+
headers: { Authorization: `Bearer ${VAPI_TOKEN}` },
18+
});
19+
if (!response.ok) {
20+
throw new Error(`GET ${endpoint} failed: ${response.status}`);
21+
}
22+
const data = await response.json();
23+
24+
if (debug) {
25+
console.log(` DEBUG: Response keys: ${Object.keys(data)}`);
26+
}
27+
28+
// Handle paginated responses - check various wrapper formats
29+
if (data && typeof data === "object" && !Array.isArray(data)) {
30+
// Try common pagination patterns: { data }, { results }, { items }, { structuredOutputs }
31+
const possibleArrayKeys = ["data", "results", "items", "structuredOutputs", "assistants", "tools", "squads"];
32+
for (const key of possibleArrayKeys) {
33+
if (Array.isArray(data[key])) {
34+
return data[key] as T;
35+
}
36+
}
37+
}
38+
39+
return data as T;
40+
}
41+
42+
async function vapiDelete(endpoint: string): Promise<void> {
43+
await sleep(REQUEST_DELAY_MS);
44+
const response = await fetch(`${VAPI_BASE_URL}${endpoint}`, {
45+
method: "DELETE",
46+
headers: { Authorization: `Bearer ${VAPI_TOKEN}` },
47+
});
48+
if (!response.ok && response.status !== 404) {
49+
throw new Error(`DELETE ${endpoint} failed: ${response.status}`);
50+
}
51+
}
52+
53+
interface VapiResource {
54+
id: string;
55+
name?: string;
56+
}
57+
58+
async function main(): Promise<void> {
59+
const dryRun = !process.argv.includes("--force");
60+
61+
console.log("═══════════════════════════════════════════════════════════════");
62+
console.log(`🧹 Vapi Cleanup - Environment: ${VAPI_ENV}`);
63+
console.log(` API: ${VAPI_BASE_URL}`);
64+
console.log(` Mode: ${dryRun ? "🔒 DRY-RUN (use --force to delete)" : "⚠️ DELETING"}`);
65+
console.log("═══════════════════════════════════════════════════════════════\n");
66+
67+
const state = loadState();
68+
const stateIds = new Set([
69+
...Object.values(state.assistants),
70+
...Object.values(state.tools),
71+
...Object.values(state.structuredOutputs),
72+
...Object.values(state.squads),
73+
...Object.values(state.personalities),
74+
...Object.values(state.scenarios),
75+
...Object.values(state.simulations),
76+
...Object.values(state.simulationSuites),
77+
]);
78+
79+
console.log(`📄 State file has ${stateIds.size} resource IDs to keep\n`);
80+
81+
const toDelete: { type: string; id: string; name: string; endpoint: string }[] = [];
82+
83+
// Fetch and compare each resource type
84+
const resourceTypes = [
85+
{ name: "assistants", endpoint: "/assistant", deleteEndpoint: "/assistant" },
86+
{ name: "tools", endpoint: "/tool", deleteEndpoint: "/tool" },
87+
{ name: "structured outputs", endpoint: "/structured-output", deleteEndpoint: "/structured-output" },
88+
{ name: "squads", endpoint: "/squad", deleteEndpoint: "/squad" },
89+
{ name: "personalities", endpoint: "/eval/simulation/personality", deleteEndpoint: "/eval/simulation/personality" },
90+
{ name: "scenarios", endpoint: "/eval/simulation/scenario", deleteEndpoint: "/eval/simulation/scenario" },
91+
{ name: "simulations", endpoint: "/eval/simulation", deleteEndpoint: "/eval/simulation" },
92+
{ name: "simulation suites", endpoint: "/eval/simulation/suite", deleteEndpoint: "/eval/simulation/suite" },
93+
];
94+
95+
for (const { name, endpoint, deleteEndpoint } of resourceTypes) {
96+
console.log(`📥 Fetching ${name}...`);
97+
try {
98+
// Enable debug for structured outputs to see response format
99+
const debug = name === "structured outputs";
100+
const resources = await vapiGet<VapiResource[]>(endpoint, debug);
101+
102+
if (!Array.isArray(resources)) {
103+
console.log(` ⚠️ Unexpected response format for ${name}: ${typeof resources}, keys: ${Object.keys(resources as object)}`);
104+
continue;
105+
}
106+
107+
const orphans = resources.filter((r) => !stateIds.has(r.id));
108+
109+
if (orphans.length > 0) {
110+
console.log(` Found ${orphans.length} orphaned ${name} (${resources.length} total)`);
111+
for (const r of orphans) {
112+
toDelete.push({
113+
type: name,
114+
id: r.id,
115+
name: r.name || "(unnamed)",
116+
endpoint: `${deleteEndpoint}/${r.id}`,
117+
});
118+
}
119+
} else {
120+
console.log(` ✅ All ${resources.length} ${name} are in state`);
121+
}
122+
} catch (error) {
123+
console.log(` ⚠️ Could not fetch ${name}: ${error}`);
124+
}
125+
}
126+
127+
console.log("\n═══════════════════════════════════════════════════════════════");
128+
129+
if (toDelete.length === 0) {
130+
console.log("✅ Nothing to delete - all resources match state file\n");
131+
return;
132+
}
133+
134+
console.log(`\n⚠️ Found ${toDelete.length} resources to delete:\n`);
135+
136+
for (const { type, id, name } of toDelete) {
137+
console.log(` 🗑️ ${type}: ${name} (${id})`);
138+
}
139+
140+
if (dryRun) {
141+
console.log("\n═══════════════════════════════════════════════════════════════");
142+
console.log("🔒 DRY-RUN MODE - No resources were deleted");
143+
console.log(" To actually delete, run:");
144+
console.log(` npm run cleanup:${VAPI_ENV} -- --force`);
145+
console.log("═══════════════════════════════════════════════════════════════\n");
146+
return;
147+
}
148+
149+
console.log("\n🗑️ Deleting...\n");
150+
151+
let deleted = 0;
152+
let failed = 0;
153+
154+
for (const { type, id, name, endpoint } of toDelete) {
155+
try {
156+
await vapiDelete(endpoint);
157+
console.log(` ✅ Deleted ${type}: ${name}`);
158+
deleted++;
159+
} catch (error) {
160+
console.log(` ❌ Failed to delete ${type}: ${name} - ${error}`);
161+
failed++;
162+
}
163+
}
164+
165+
console.log("\n═══════════════════════════════════════════════════════════════");
166+
console.log(`✅ Cleanup complete: ${deleted} deleted, ${failed} failed`);
167+
console.log("═══════════════════════════════════════════════════════════════\n");
168+
}
169+
170+
main().catch((error) => {
171+
console.error("\n❌ Cleanup failed:", error);
172+
process.exit(1);
173+
});

‎src/config.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import type { Environment, ResourceType } from "./types.ts";
55
import { VALID_ENVIRONMENTS } from "./types.ts";
66

77
// ─────────────────────────────────────────────────────────────────────────────
8-
// Environment Parsing
8+
// CLI Argument Parsing
99
// ─────────────────────────────────────────────────────────────────────────────
1010

1111
function parseEnvironment(): Environment {
@@ -14,6 +14,7 @@ function parseEnvironment(): Environment {
1414
if (!envArg) {
1515
console.error("❌ Environment argument is required");
1616
console.error(" Usage: npm run apply:dev | apply:prod");
17+
console.error(" Flags: --force (enable deletions)");
1718
process.exit(1);
1819
}
1920

@@ -26,6 +27,13 @@ function parseEnvironment(): Environment {
2627
return envArg;
2728
}
2829

30+
function parseFlags(): { forceDelete: boolean } {
31+
const args = process.argv.slice(3);
32+
return {
33+
forceDelete: args.includes("--force"),
34+
};
35+
}
36+
2937
// ─────────────────────────────────────────────────────────────────────────────
3038
// Environment File Loading
3139
// ─────────────────────────────────────────────────────────────────────────────
@@ -76,8 +84,9 @@ function loadEnvFile(env: string, baseDir: string): void {
7684
const __dirname = dirname(fileURLToPath(import.meta.url));
7785
export const BASE_DIR = join(__dirname, "..");
7886

79-
// Parse environment and load env files
87+
// Parse environment, flags, and load env files
8088
export const VAPI_ENV = parseEnvironment();
89+
export const { forceDelete: FORCE_DELETE } = parseFlags();
8190
loadEnvFile(VAPI_ENV, BASE_DIR);
8291

8392
// API configuration

0 commit comments

Comments
 (0)