From cbb76b938f109e8e0a99b87164345191b5b858a2 Mon Sep 17 00:00:00 2001 From: "MK (fengmk2)" Date: Fri, 13 Nov 2026 13:27:27 +0800 Subject: [PATCH 1/2] fix(cleanup): release generations whose multipart upload can't be aborted Publication left the upload ID of a completed multipart upload on the generation. When the generation expired, cleanup aborted that upload before it deleted the objects and the row, in one try block. R2 does not promise that a resumed upload still exists, so if the abort rejects, the row stays in 'deleting' with its charged bytes and is retried every 10 minutes forever. Enough such rows fill the shared byte budget and the 16-row cleanup batch for every namespace. Clear the upload ID when publication completes the upload, and treat a failed abort as non-fatal. The bucket lifecycle already aborts uploads left incomplete after one day. The second change also releases rows that were published before this fix. Co-Authored-By: Claude Opus 5.5 --- src/database.ts | 12 +++++++----- test/failures.test.ts | 44 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/src/database.ts b/src/database.ts index 1c6e444..bb7c93f 100644 --- a/src/database.ts +++ b/src/database.ts @@ -197,7 +197,7 @@ export async function publish( db.batch([ db .prepare(`UPDATE generations SET state = 'ready', key = ?, secondary_key = ?, blob_id = ?, - value_size = ?, blob_size = ?, charged_bytes = ?, + value_size = ?, blob_size = ?, charged_bytes = ?, upload_id = NULL, expires_at = unixepoch() + (SELECT retention_seconds FROM scopes WHERE scope_id = generations.scope_id), gc_after = unixepoch() + (SELECT retention_seconds FROM scopes WHERE scope_id = generations.scope_id) WHERE generation_id = ? AND state = 'uploading' AND lease_until > unixepoch() AND token_exp > unixepoch() @@ -253,11 +253,13 @@ export async function cleanup(env: Env): Promise { let deleted = 0; for (const generation of claimed.results) { try { + // A recorded upload may already be complete, aborted, or gone: R2 doesn't check that a + // resumed upload still exists. Failing to abort must not keep the generation and its charge + // forever. The bucket lifecycle aborts any upload left incomplete. if (generation.upload_id) - await env.ARTIFACTS.resumeMultipartUpload( - generation.blob_object, - generation.upload_id, - ).abort(); + await env.ARTIFACTS.resumeMultipartUpload(generation.blob_object, generation.upload_id) + .abort() + .catch(() => {}); await env.ARTIFACTS.delete([generation.value_object, generation.blob_object]); // FK deletion only removes entries still selecting this generation. await env.INDEX.prepare( diff --git a/test/failures.test.ts b/test/failures.test.ts index c0dd91b..680bfde 100644 --- a/test/failures.test.ts +++ b/test/failures.test.ts @@ -212,6 +212,50 @@ test('failed deletion remains charged and a retry removes only the claimed gener } }); +test('cleanup deletes a generation whose recorded multipart upload R2 can no longer abort', async () => { + const h = await harness(); + try { + // Publication clears the ID of the upload it completed. + assert.equal( + (await h.store(bytes('A'), bytes('S'), bytes('v'), new Uint8Array(5 * 1024 * 1024 + 1))) + .status, + 200, + ); + assert.equal( + (await h.db + .prepare('SELECT count(*) AS n FROM generations WHERE upload_id IS NOT NULL') + .first())!.n, + 0, + ); + // Generations published before that change still carry the ID. + await h.db + .prepare("UPDATE generations SET upload_id = 'completed', expires_at = 0, gc_after = 0") + .run(); + const original = await h.mf.getBindings(); + const bucket = new Proxy(original.ARTIFACTS, { + get(target, prop) { + if (prop === 'resumeMultipartUpload') + return () => ({ + abort: async () => { + throw new Error('Injected: no such upload'); + }, + }); + const value = Reflect.get(target, prop); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + await cleanup({ ...original, ARTIFACTS: bucket }); + assert.equal((await h.db.prepare('SELECT count(*) AS n FROM generations').first())!.n, 0); + assert.equal( + (await h.db.prepare('SELECT charged_bytes FROM deployment').first())!.charged_bytes, + 0, + ); + assert.equal((await h.bucket.list()).objects.length, 0); + } finally { + await h.close(); + } +}); + test('unknown-length requests reserve the full limit and cancellation prevents publication', async () => { const h = await harness(); try { From 1b779b08173e009a83cd7416a749293d9e121d9a Mon Sep 17 00:00:00 2001 From: "MK (fengmk2)" Date: Fri, 13 Nov 2026 14:40:24 +0800 Subject: [PATCH 2/2] fix(cleanup): ignore only abort errors that confirm the upload is gone Review feedback: the catch also hid transient R2 failures for uploads that still hold parts. Cleanup then deleted the row and released its charge, so new stores could reuse that quota while the parts still used storage, and the upload ID needed for a retry was lost. Continue only when R2 reports NoSuchUpload (10024). Any other abort error keeps the generation, its charge, and its upload ID for the next cleanup run. Co-Authored-By: Claude Opus 5.5 --- src/database.ts | 14 ++++++-- test/failures.test.ts | 78 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 76 insertions(+), 16 deletions(-) diff --git a/src/database.ts b/src/database.ts index bb7c93f..8a544a8 100644 --- a/src/database.ts +++ b/src/database.ts @@ -234,6 +234,12 @@ export async function abandon(db: D1Database, id: string): Promise { .run(); } +// R2 binding errors read ": ()". Code 10024 (NoSuchUpload) means the +// multipart upload no longer exists. +function uploadGone(error: unknown): boolean { + return error instanceof Error && error.message.endsWith('(10024)'); +} + export async function cleanup(env: Env): Promise { const limit = Number(env.GC_BATCH_SIZE); if (!Number.isInteger(limit) || limit < 1 || limit > 256) @@ -254,12 +260,14 @@ export async function cleanup(env: Env): Promise { for (const generation of claimed.results) { try { // A recorded upload may already be complete, aborted, or gone: R2 doesn't check that a - // resumed upload still exists. Failing to abort must not keep the generation and its charge - // forever. The bucket lifecycle aborts any upload left incomplete. + // resumed upload still exists. Only an error that confirms this lets deletion go on. Any + // other failure keeps the generation, its charge, and its upload ID for the next retry. if (generation.upload_id) await env.ARTIFACTS.resumeMultipartUpload(generation.blob_object, generation.upload_id) .abort() - .catch(() => {}); + .catch((error: unknown) => { + if (!uploadGone(error)) throw error; + }); await env.ARTIFACTS.delete([generation.value_object, generation.blob_object]); // FK deletion only removes entries still selecting this generation. await env.INDEX.prepare( diff --git a/test/failures.test.ts b/test/failures.test.ts index 680bfde..dec65c3 100644 --- a/test/failures.test.ts +++ b/test/failures.test.ts @@ -227,24 +227,19 @@ test('cleanup deletes a generation whose recorded multipart upload R2 can no lon .first())!.n, 0, ); - // Generations published before that change still carry the ID. + // Generations published before that change still carry the ID. R2 reports that the upload no + // longer exists. await h.db .prepare("UPDATE generations SET upload_id = 'completed', expires_at = 0, gc_after = 0") .run(); const original = await h.mf.getBindings(); - const bucket = new Proxy(original.ARTIFACTS, { - get(target, prop) { - if (prop === 'resumeMultipartUpload') - return () => ({ - abort: async () => { - throw new Error('Injected: no such upload'); - }, - }); - const value = Reflect.get(target, prop); - return typeof value === 'function' ? value.bind(target) : value; - }, + await cleanup({ + ...original, + ARTIFACTS: failingAbort( + original.ARTIFACTS, + 'abortMultipartUpload: The specified multipart upload does not exist. (10024)', + ), }); - await cleanup({ ...original, ARTIFACTS: bucket }); assert.equal((await h.db.prepare('SELECT count(*) AS n FROM generations').first())!.n, 0); assert.equal( (await h.db.prepare('SELECT charged_bytes FROM deployment').first())!.charged_bytes, @@ -256,6 +251,63 @@ test('cleanup deletes a generation whose recorded multipart upload R2 can no lon } }); +test('cleanup keeps an unfinished upload and its charge when the abort fails transiently', async () => { + const h = await harness(); + try { + const original = await h.mf.getBindings(); + const part = 5 * 1024 * 1024; + const scope = await getScope(h.db, 'test'); + const generation = await reserve(h.db, scope, Math.floor(Date.now() / 1000) + 300, part); + const upload = await original.ARTIFACTS.createMultipartUpload(generation.blob_object); + const uploaded = await upload.uploadPart(1, new Uint8Array(part)); + await h.db + .prepare('UPDATE generations SET upload_id = ?, gc_after = 0 WHERE generation_id = ?') + .bind(upload.uploadId, generation.generation_id) + .run(); + const remaining = async () => + h.db + .prepare( + 'SELECT (SELECT count(*) FROM generations WHERE upload_id IS NOT NULL) AS n, charged_bytes FROM deployment', + ) + .first(); + await cleanup({ + ...original, + ARTIFACTS: failingAbort( + original.ARTIFACTS, + 'abortMultipartUpload: We encountered an internal error. Please try again. (10001)', + ), + }); + assert.deepEqual(await remaining(), { n: 1, charged_bytes: part }); + // The upload still exists, so a retry can abort it, delete the generation, and release the charge. + await h.db.prepare('UPDATE generations SET gc_after = 0').run(); + await cleanup(original); + assert.deepEqual(await remaining(), { n: 0, charged_bytes: 0 }); + await assert.rejects( + original.ARTIFACTS.resumeMultipartUpload(generation.blob_object, upload.uploadId).complete([ + uploaded, + ]), + ); + } finally { + await h.close(); + } +}); + +// An R2 binding whose multipart aborts reject with the given binding error message. +function failingAbort(bucket: R2Bucket, message: string): R2Bucket { + return new Proxy(bucket, { + get(target, prop) { + if (prop === 'resumeMultipartUpload') + return () => ({ + abort: async () => { + throw new Error(message); + }, + }); + const value = Reflect.get(target, prop); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); +} + test('unknown-length requests reserve the full limit and cancellation prevents publication', async () => { const h = await harness(); try {