diff --git a/src/database.ts b/src/database.ts index 1c6e444..8a544a8 100644 --- a/src/database.ts +++ b/src/database.ts @@ -197,7 +197,7 @@ export async function publish( db.batch([ db .prepare(`UPDATE generations SET state = 'ready', key = ?, secondary_key = ?, blob_id = ?, - value_size = ?, blob_size = ?, charged_bytes = ?, + value_size = ?, blob_size = ?, charged_bytes = ?, upload_id = NULL, expires_at = unixepoch() + (SELECT retention_seconds FROM scopes WHERE scope_id = generations.scope_id), gc_after = unixepoch() + (SELECT retention_seconds FROM scopes WHERE scope_id = generations.scope_id) WHERE generation_id = ? AND state = 'uploading' AND lease_until > unixepoch() AND token_exp > unixepoch() @@ -234,6 +234,12 @@ export async function abandon(db: D1Database, id: string): Promise { .run(); } +// R2 binding errors read ": ()". Code 10024 (NoSuchUpload) means the +// multipart upload no longer exists. +function uploadGone(error: unknown): boolean { + return error instanceof Error && error.message.endsWith('(10024)'); +} + export async function cleanup(env: Env): Promise { const limit = Number(env.GC_BATCH_SIZE); if (!Number.isInteger(limit) || limit < 1 || limit > 256) @@ -253,11 +259,15 @@ export async function cleanup(env: Env): Promise { let deleted = 0; for (const generation of claimed.results) { try { + // A recorded upload may already be complete, aborted, or gone: R2 doesn't check that a + // resumed upload still exists. Only an error that confirms this lets deletion go on. Any + // other failure keeps the generation, its charge, and its upload ID for the next retry. if (generation.upload_id) - await env.ARTIFACTS.resumeMultipartUpload( - generation.blob_object, - generation.upload_id, - ).abort(); + await env.ARTIFACTS.resumeMultipartUpload(generation.blob_object, generation.upload_id) + .abort() + .catch((error: unknown) => { + if (!uploadGone(error)) throw error; + }); await env.ARTIFACTS.delete([generation.value_object, generation.blob_object]); // FK deletion only removes entries still selecting this generation. await env.INDEX.prepare( diff --git a/test/failures.test.ts b/test/failures.test.ts index c0dd91b..dec65c3 100644 --- a/test/failures.test.ts +++ b/test/failures.test.ts @@ -212,6 +212,102 @@ test('failed deletion remains charged and a retry removes only the claimed gener } }); +test('cleanup deletes a generation whose recorded multipart upload R2 can no longer abort', async () => { + const h = await harness(); + try { + // Publication clears the ID of the upload it completed. + assert.equal( + (await h.store(bytes('A'), bytes('S'), bytes('v'), new Uint8Array(5 * 1024 * 1024 + 1))) + .status, + 200, + ); + assert.equal( + (await h.db + .prepare('SELECT count(*) AS n FROM generations WHERE upload_id IS NOT NULL') + .first())!.n, + 0, + ); + // Generations published before that change still carry the ID. R2 reports that the upload no + // longer exists. + await h.db + .prepare("UPDATE generations SET upload_id = 'completed', expires_at = 0, gc_after = 0") + .run(); + const original = await h.mf.getBindings(); + await cleanup({ + ...original, + ARTIFACTS: failingAbort( + original.ARTIFACTS, + 'abortMultipartUpload: The specified multipart upload does not exist. (10024)', + ), + }); + assert.equal((await h.db.prepare('SELECT count(*) AS n FROM generations').first())!.n, 0); + assert.equal( + (await h.db.prepare('SELECT charged_bytes FROM deployment').first())!.charged_bytes, + 0, + ); + assert.equal((await h.bucket.list()).objects.length, 0); + } finally { + await h.close(); + } +}); + +test('cleanup keeps an unfinished upload and its charge when the abort fails transiently', async () => { + const h = await harness(); + try { + const original = await h.mf.getBindings(); + const part = 5 * 1024 * 1024; + const scope = await getScope(h.db, 'test'); + const generation = await reserve(h.db, scope, Math.floor(Date.now() / 1000) + 300, part); + const upload = await original.ARTIFACTS.createMultipartUpload(generation.blob_object); + const uploaded = await upload.uploadPart(1, new Uint8Array(part)); + await h.db + .prepare('UPDATE generations SET upload_id = ?, gc_after = 0 WHERE generation_id = ?') + .bind(upload.uploadId, generation.generation_id) + .run(); + const remaining = async () => + h.db + .prepare( + 'SELECT (SELECT count(*) FROM generations WHERE upload_id IS NOT NULL) AS n, charged_bytes FROM deployment', + ) + .first(); + await cleanup({ + ...original, + ARTIFACTS: failingAbort( + original.ARTIFACTS, + 'abortMultipartUpload: We encountered an internal error. Please try again. (10001)', + ), + }); + assert.deepEqual(await remaining(), { n: 1, charged_bytes: part }); + // The upload still exists, so a retry can abort it, delete the generation, and release the charge. + await h.db.prepare('UPDATE generations SET gc_after = 0').run(); + await cleanup(original); + assert.deepEqual(await remaining(), { n: 0, charged_bytes: 0 }); + await assert.rejects( + original.ARTIFACTS.resumeMultipartUpload(generation.blob_object, upload.uploadId).complete([ + uploaded, + ]), + ); + } finally { + await h.close(); + } +}); + +// An R2 binding whose multipart aborts reject with the given binding error message. +function failingAbort(bucket: R2Bucket, message: string): R2Bucket { + return new Proxy(bucket, { + get(target, prop) { + if (prop === 'resumeMultipartUpload') + return () => ({ + abort: async () => { + throw new Error(message); + }, + }); + const value = Reflect.get(target, prop); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); +} + test('unknown-length requests reserve the full limit and cancellation prevents publication', async () => { const h = await harness(); try {