Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 15 additions & 5 deletions src/database.ts
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ export async function publish(
db.batch([
db
.prepare(`UPDATE generations SET state = 'ready', key = ?, secondary_key = ?, blob_id = ?,
value_size = ?, blob_size = ?, charged_bytes = ?,
value_size = ?, blob_size = ?, charged_bytes = ?, upload_id = NULL,
expires_at = unixepoch() + (SELECT retention_seconds FROM scopes WHERE scope_id = generations.scope_id),
gc_after = unixepoch() + (SELECT retention_seconds FROM scopes WHERE scope_id = generations.scope_id)
WHERE generation_id = ? AND state = 'uploading' AND lease_until > unixepoch() AND token_exp > unixepoch()
Expand Down Expand Up @@ -234,6 +234,12 @@ export async function abandon(db: D1Database, id: string): Promise<void> {
.run();
}

// R2 binding errors read "<operation>: <message> (<code>)". Code 10024 (NoSuchUpload) means the
// multipart upload no longer exists.
function uploadGone(error: unknown): boolean {
return error instanceof Error && error.message.endsWith('(10024)');
}

export async function cleanup(env: Env): Promise<void> {
const limit = Number(env.GC_BATCH_SIZE);
if (!Number.isInteger(limit) || limit < 1 || limit > 256)
Expand All @@ -253,11 +259,15 @@ export async function cleanup(env: Env): Promise<void> {
let deleted = 0;
for (const generation of claimed.results) {
try {
// A recorded upload may already be complete, aborted, or gone: R2 doesn't check that a
// resumed upload still exists. Only an error that confirms this lets deletion go on. Any
// other failure keeps the generation, its charge, and its upload ID for the next retry.
if (generation.upload_id)
await env.ARTIFACTS.resumeMultipartUpload(
generation.blob_object,
generation.upload_id,
).abort();
await env.ARTIFACTS.resumeMultipartUpload(generation.blob_object, generation.upload_id)
.abort()
.catch((error: unknown) => {
if (!uploadGone(error)) throw error;
});
await env.ARTIFACTS.delete([generation.value_object, generation.blob_object]);
// FK deletion only removes entries still selecting this generation.
await env.INDEX.prepare(
Expand Down
96 changes: 96 additions & 0 deletions test/failures.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,102 @@ test('failed deletion remains charged and a retry removes only the claimed gener
}
});

test('cleanup deletes a generation whose recorded multipart upload R2 can no longer abort', async () => {
const h = await harness();
try {
// Publication clears the ID of the upload it completed.
assert.equal(
(await h.store(bytes('A'), bytes('S'), bytes('v'), new Uint8Array(5 * 1024 * 1024 + 1)))
.status,
200,
);
assert.equal(
(await h.db
.prepare('SELECT count(*) AS n FROM generations WHERE upload_id IS NOT NULL')
.first())!.n,
0,
);
// Generations published before that change still carry the ID. R2 reports that the upload no
// longer exists.
await h.db
.prepare("UPDATE generations SET upload_id = 'completed', expires_at = 0, gc_after = 0")
.run();
const original = await h.mf.getBindings<Env>();
await cleanup({
...original,
ARTIFACTS: failingAbort(
original.ARTIFACTS,
'abortMultipartUpload: The specified multipart upload does not exist. (10024)',
),
});
assert.equal((await h.db.prepare('SELECT count(*) AS n FROM generations').first())!.n, 0);
assert.equal(
(await h.db.prepare('SELECT charged_bytes FROM deployment').first())!.charged_bytes,
0,
);
assert.equal((await h.bucket.list()).objects.length, 0);
} finally {
await h.close();
}
});

test('cleanup keeps an unfinished upload and its charge when the abort fails transiently', async () => {
const h = await harness();
try {
const original = await h.mf.getBindings<Env>();
const part = 5 * 1024 * 1024;
const scope = await getScope(h.db, 'test');
const generation = await reserve(h.db, scope, Math.floor(Date.now() / 1000) + 300, part);
const upload = await original.ARTIFACTS.createMultipartUpload(generation.blob_object);
const uploaded = await upload.uploadPart(1, new Uint8Array(part));
await h.db
.prepare('UPDATE generations SET upload_id = ?, gc_after = 0 WHERE generation_id = ?')
.bind(upload.uploadId, generation.generation_id)
.run();
const remaining = async () =>
h.db
.prepare(
'SELECT (SELECT count(*) FROM generations WHERE upload_id IS NOT NULL) AS n, charged_bytes FROM deployment',
)
.first();
await cleanup({
...original,
ARTIFACTS: failingAbort(
original.ARTIFACTS,
'abortMultipartUpload: We encountered an internal error. Please try again. (10001)',
),
});
assert.deepEqual(await remaining(), { n: 1, charged_bytes: part });
// The upload still exists, so a retry can abort it, delete the generation, and release the charge.
await h.db.prepare('UPDATE generations SET gc_after = 0').run();
await cleanup(original);
assert.deepEqual(await remaining(), { n: 0, charged_bytes: 0 });
await assert.rejects(
original.ARTIFACTS.resumeMultipartUpload(generation.blob_object, upload.uploadId).complete([
uploaded,
]),
);
} finally {
await h.close();
}
});

// An R2 binding whose multipart aborts reject with the given binding error message.
function failingAbort(bucket: R2Bucket, message: string): R2Bucket {
return new Proxy(bucket, {
get(target, prop) {
if (prop === 'resumeMultipartUpload')
return () => ({
abort: async () => {
throw new Error(message);
},
});
const value = Reflect.get(target, prop);
return typeof value === 'function' ? value.bind(target) : value;
},
});
}

test('unknown-length requests reserve the full limit and cancellation prevents publication', async () => {
const h = await harness();
try {
Expand Down
Loading