diff --git a/src/app/index.ts b/src/app/index.ts index e436c10..5dd5e39 100644 --- a/src/app/index.ts +++ b/src/app/index.ts @@ -1,3 +1,8 @@ +import { getExtensionsDb } from "../lib/db"; +import { + maintainExtensionResources, + reportExtensionResources +} from "../services/extensions/v2/db/resource-maintenance"; import { Hono } from "hono"; import { HTTPException } from "hono/http-exception"; import centralAlertsV1 from "../services/central-alerts/v1"; @@ -69,6 +74,19 @@ app.all("/*", (c) => { ); }); -export default app; +export default { + fetch: app.fetch, + request: app.request.bind(app), + scheduled: async ( + _event: ScheduledController, + env: CloudflareBindings + ): Promise => { + const db = getExtensionsDb(env.DB_EXTENSIONS); + await maintainExtensionResources(db, { + mode: env.EXTENSIONS_RETENTION_MODE + }); + await reportExtensionResources(db, env.EXTENSIONS_RETENTION_MODE); + } +}; export { PreviewGitHubBudget } from "../lib/adapters/cloudflare/preview-github-budget"; diff --git a/src/services/extensions/v2/README.md b/src/services/extensions/v2/README.md index 84dd545..6210103 100644 --- a/src/services/extensions/v2/README.md +++ b/src/services/extensions/v2/README.md @@ -283,3 +283,130 @@ enforced atomically with the write using indexed, immutable history and survives profile deletion, recreation, and ownership transfer. Audit records remain append-only; this bounds growth per account per day rather than total retention. Apply migration `0025_luxuriant_franklin_richards.sql` before deploying. + +## Extension resource admission and retention + +Content writes (`POST /extensions`, `PUT /extensions/{id}`, and moderator +correction) count the actual request stream before JSON parsing. The raw request +limit is **512 KiB**, including whitespace, escapes, and unknown fields. An +oversized stream is canceled and returns `413 BODY_TOO_LARGE`; `Content-Length` +cannot bypass counting. The separate normalized JSON content limit remains +**256 KiB**, with the existing field/release maxima. New extension slug IDs are at most +200 characters. Previously stored IDs had no length cap and remain supported. + +The `EXTENSION_WRITE_RATE_LIMITER` binding paces IP and authenticated-account +attempts at 60/minute, including validation failures. This edge control is +approximate and must not be used as the durable quota. Missing/unavailable +attempt pacing fails closed with `503 ADMISSION_UNAVAILABLE`. Unavailable durable +write admission returns the same response with `Retry-After: 60`. + +Migration `0026_resource_bounds.sql` enforces the following accepted-write and +retained-resource limits **inside the write transaction**, including moderator +corrections. Its trigger guards and accounting are shared by create, edit, +approval, correction, withdrawal and retention; a failed batch leaves neither +an extension nor an accepted-write charge behind. + +| Resource | Account | Developer | +| ------------------------------------- | ------- | --------- | +| Accepted revisions per rolling minute | 5 | 5 | +| Accepted revisions per rolling day | 50 | 50 | +| Retained content bytes | 25 MiB | 25 MiB | +| Extension records | 100 | 100 | +| Revision metadata records | 1,000 | 1,000 | + +Aggregate storage and record counts are measured for visibility; they do not +impose a whole-system admission limit. + +The pending budget remains ten revisions per submitter and one per extension. +The byte quota includes retained revision JSON **and** published content +columns. It counts UTF-8 bytes, not JavaScript string length. Original creators +are charged for extension records and published projections; revision bodies +and metadata remain charged to their submitters and original developer IDs. +Transferring a profile does not silently shift these charges to the recipient. +The `created_by` attribution is immutable. Legacy unowned publications are +charged to a synthetic `legacy` account bucket. + +Each accepted write also removes up to 50 expired ledger entries, so expiry +cleanup scales with write traffic. The hourly job handles idle cleanup. + +The accepted-write ledger has no deletion-cascading foreign keys: review, +withdrawal, profile replacement and account reactivation do not refund a +rolling allowance. Minute/day exhaustion returns `429` with a domain +code (`WRITE_RATE_MINUTE`, `WRITE_RATE_DAY`) and a conservative +`Retry-After` of 60 or 86400 seconds respectively. Retained quota exhaustion +returns `409 RESOURCE_QUOTA`. Withdrawing an unpublished extension releases its +stored bytes and record counts, but not its accepted-write allowance. + +`resource-limits.ts` mirrors the policy values used by middleware and maintenance. + +### Revision list/detail contract + +`GET /revisions` and `GET /extensions/{id}/revisions` now return +`ExtensionRevisionSummary` pages. They select stored, bounded summary fields +(`name`, `version`, `description`), decision metadata, `content_bytes`, +`content_available`, `content_hash` and `compacted_at`. They do **not** select, +transfer or parse content JSON, including the pagination lookahead row. +Timestamp/ID keyset ordering, limit 1–100 and default 50 are unchanged. + +Fetch `GET /extensions/{id}/revisions/{revisionId}` for full content on demand. +The caller must be the current owner or an active moderator, and authorization +is repeated in the actual data query. A compacted revision returns `content: +null`, `content_available: false`, a SHA-256 hash of the original stored bytes, +and its compaction timestamp. `content_bytes` measures the currently retained +body (the two-byte `{}` placeholder after compaction), not the historical body. + +Oversized legacy content remains stored and counted, but is never fetched or +parsed by list or detail handlers. Its revision detail returns `409 +CONTENT_UNAVAILABLE`. An owner/moderator extension detail with an oversized +published or pending body also returns this error. Oversized published legacy +rows are excluded from public catalogue cards. Historical release collections +are checked for safe count/tag bounds before version sorting; unsafe collections +also return `CONTENT_UNAVAILABLE`; their summaries report `content_available: false`. +The stored readability flag is calculated alongside summary metadata, including +the safe legacy release count and tag bounds (100 Unicode code points). +Anonymous public extension detail reads also return `409 CONTENT_UNAVAILABLE` +for oversized published bodies. Moderator lists still expose +the published card for correction without reading its oversized body. +Cards bound display fields and project license/repository metadata; oversized +legacy URLs are null (or an omitted optional icon), rather than clipped links. +Detail reads preserve the original fields when the body is safe to fetch. +Reject an oversized pending revision, then ask its +owner to resubmit under current bounds; use moderator correction for published +content. Administrative export is required if the original oversized body must +be recovered. The migration backfills byte counts/summaries in SQL without +loading old bodies into a Worker or discarding any rows. Existing over-quota +collections cannot grow until usage is reduced. + +### Maintenance and monitoring + +The Worker runs scheduled maintenance **hourly**, handling at most +**20 bodies** and 500 expired ledger rows per invocation. Retention defaults to +`dry-run`: it reports eligible bodies and reclaimable bytes without fetching or +changing bodies. Only `EXTENSIONS_RETENTION_MODE=compact` enables compaction; +missing or invalid mode values keep retention non-destructive. Expired admission +ledger entries and empty usage counters are cleaned in either mode. Reviewed revision +bodies older than **180 days** are compacted to `{}` while retaining their +bounded summaries, review note, reviewer, dates, status and original content +hash. Pending bodies and the currently published revision are always protected. +Oversized legacy bodies are left for administrative handling. Metadata is +retained under the count quota; reaching that quota needs deliberate export +and administrative cleanup rather than silent deletion of decisions. + +Compaction rechecks age, status, body equality and the published pointer in the +UPDATE, so concurrent review/publication changes cannot discard protected +content. Accounting is updated in the same transaction. Repeated runs are +idempotent. Expired write events and empty usage counters are cleaned in bounded +batches. + +Full resource inventory is logged after each hourly maintenance run. +Structured logs record revision-response bytes/duration, admission reason codes and maintenance/inventory totals: +`retained_bytes`, `extensions`, `revisions`, `pending`, +`oversized_legacy_revisions`, `cleanup_backlog`, `compacted`, and +`expired_events`. They contain no revision bodies or user identifiers. Warning-level signals flag +oversized legacy bodies and an active +compaction backlog of 300 or more bodies. Dry-run eligibility is informational. + +`db/resource-inventory.sql` is a manual, read-only reconciliation diagnostic, +separate from the hourly usage report. It lists at most 100 accounting +discrepancies using stored scalar sizes without returning content bodies, +ordered by descending byte/count drift with stable scope/subject tie-breaks. diff --git a/src/services/extensions/v2/db/errors.ts b/src/services/extensions/v2/db/errors.ts index ee76c70..7ad32e3 100644 --- a/src/services/extensions/v2/db/errors.ts +++ b/src/services/extensions/v2/db/errors.ts @@ -2,14 +2,9 @@ import { DatabaseError } from "../../../../lib/interfaces"; import { ExtensionsDb } from "../../../../lib/db"; import { UsersDatabase } from "./users"; import { DatabaseResult } from "../../../../lib/interfaces"; -import { logError } from "../../../../lib/logger"; +import { logInfo, logError } from "../../../../lib/logger"; -// Drizzle wraps the real D1 driver error in a DrizzleError whose own -// .message is a generic "Failed to run the query ''" - the actual -// SQLite/D1 message (e.g. "UNIQUE constraint failed: ...") lives in -// .cause, not .message. Regex-matching driver error text (see -// the ownership/id conflict classifiers need the whole chain, not just the -// outermost message. +// Drizzle wraps driver errors; constraint classifiers inspect the cause chain. export function errorMessageChain(error: unknown): string { const parts: string[] = []; let current: unknown = error; @@ -55,14 +50,59 @@ export const isOwnershipEpochRollback = (error: unknown) => // pre-flight check instead of exposing it as a generic database error. export const isDeveloperIdConflict = uniqueConstraintMatcher(/developers\.id/); -// Logs the real error server-side and returns a generic message to the -// caller β€” DB exception text can leak schema/backend details otherwise. +// Log safe driver diagnostics; exception messages can contain SQL and content. export function databaseError( context: string, error: unknown ): DatabaseResult { + let cause = error; + while (cause instanceof Error && cause.cause instanceof Error) + cause = cause.cause; + const message = cause instanceof Error ? cause.message : ""; + const policy = [ + [ + "extension_write_rate_minute", + "WRITE_RATE_MINUTE", + "Five proposals per rolling minute allowed" + ], + [ + "extension_write_rate_day", + "WRITE_RATE_DAY", + "Fifty proposals per rolling day allowed" + ], + [ + "extension_resource_quota", + "RESOURCE_QUOTA", + "The retained extension resource quota is exhausted" + ], + [ + "extension_content_size", + "CONFLICT", + "Extension content must not exceed 256 KiB" + ] + ].find(([marker]) => message.includes(marker)); + if (policy) { + logInfo("extensions-v2", "Resource admission rejected", { + reason: policy[1] + }); + return { data: null, error: { code: policy[1], message: policy[2] } }; + } + const driverCode = + cause instanceof Error && "code" in cause ? cause.code : undefined; + const backendCode = + typeof driverCode === "number" + ? driverCode + : typeof driverCode === "string" && + /^[A-Z][A-Z0-9_]{0,63}$/.test(driverCode) + ? driverCode + : message.match(/\b(?:SQLITE|D1)_[A-Z_]+\b/)?.[0]; logError("extensions-v2", context, { - error: error instanceof Error ? errorMessageChain(error) : String(error) + reason: "backend_failure", + error_type: + cause instanceof Error && /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(cause.name) + ? cause.name + : "UnknownError", + backend_code: backendCode }); return { data: null, @@ -70,6 +110,24 @@ export function databaseError( }; } +// Content insertion and its durable budget share one transaction. A backend +// failure means admission is unavailable; policy rejections retain their code. +export function contentAdmissionError( + context: string, + error: unknown +): DatabaseResult { + const result = databaseError(context, error); + return result.error?.code === "DATABASE_ERROR" + ? { + data: null, + error: { + code: "ADMISSION_UNAVAILABLE", + message: "Write admission unavailable" + } + } + : result; +} + // Every guarded write in this service repeats an active-account check inside // its own statement, because requireActiveAuth() can only reject before the // write. When such a statement affects no rows the diagnosis has to ask this diff --git a/src/services/extensions/v2/db/extensions.ts b/src/services/extensions/v2/db/extensions.ts index ecbc5fc..9553532 100644 --- a/src/services/extensions/v2/db/extensions.ts +++ b/src/services/extensions/v2/db/extensions.ts @@ -1,3 +1,4 @@ +import { MAX_CONTENT_BYTES } from "../resource-limits"; import { and, asc, eq, isNotNull, isNull, or, sql } from "drizzle-orm"; import { alias } from "drizzle-orm/sqlite-core"; import { DatabaseError, DatabaseResult } from "../../../../lib/interfaces"; @@ -7,6 +8,7 @@ import { parseJSON } from "../../../../lib/json"; import { extensions, extensionRevisions, developers, users } from "./schema"; import { databaseError, + contentAdmissionError, inactiveActorError, moderatorActorError } from "./errors"; @@ -42,33 +44,100 @@ const DEVELOPER_COLUMNS = { developerOwnerUserId: developers.ownerUserId }; +const publishedBytes = extensions.publishedBytes; +// Cards project small fields independently of the full body's size. In +// particular an oversized legacy README must not turn a published row into +// an apparent draft. JSON fields are projected rather than discarded. const CONTENT_COLUMNS = { - type: extensions.type, - name: extensions.name, - description: extensions.description, + type: sql`substr(${extensions.type}, 1, 100)`, + name: sql`substr(${extensions.name}, 1, 120)`, + description: sql`substr(${extensions.description}, 1, 4000)`, releases: extensions.releases, - website: extensions.website, - license: extensions.license, - iconUrl: extensions.iconUrl, + website: sql< + string | null + >`CASE WHEN length(${extensions.website}) <= 2048 THEN ${extensions.website} ELSE NULL END`, + license: sql< + string | null + >`CASE WHEN length(CAST(${extensions.license} AS BLOB)) <= ${MAX_CONTENT_BYTES} AND json_valid(${extensions.license}) THEN + json_patch(json_object('name', substr(json_extract(${extensions.license}, '$.name'), 1, 100)), + json_patch(CASE WHEN json_type(${extensions.license}, '$.spdx_id') = 'text' + THEN json_object('spdx_id', substr(json_extract(${extensions.license}, '$.spdx_id'), 1, 100)) ELSE '{}' END, + CASE WHEN json_type(${extensions.license}, '$.URL') = 'text' AND length(json_extract(${extensions.license}, '$.URL')) <= 2048 + THEN json_object('URL', json_extract(${extensions.license}, '$.URL')) ELSE '{}' END)) + ELSE '{"name":"Unavailable"}' END`, + iconUrl: sql< + string | null + >`CASE WHEN length(${extensions.iconUrl}) <= 2048 THEN ${extensions.iconUrl} ELSE NULL END`, readme: extensions.readme, - source: extensions.source, - version: extensions.version, - downloadUrl: extensions.downloadUrl + source: sql< + string | null + >`CASE WHEN length(CAST(${extensions.source} AS BLOB)) <= ${MAX_CONTENT_BYTES} AND json_valid(${extensions.source}) THEN + json_object('type', CASE WHEN json_extract(${extensions.source}, '$.type') IN ('github', 'gitlab', 'custom') + THEN json_extract(${extensions.source}, '$.type') ELSE 'custom' END, + 'repo', substr(json_extract(${extensions.source}, '$.repo'), 1, 500)) + ELSE '{"type":"custom","repo":"Unavailable"}' END`, + version: sql`substr(${extensions.version}, 1, 100)`, + downloadUrl: sql< + string | null + >`CASE WHEN length(${extensions.downloadUrl}) <= 2048 THEN ${extensions.downloadUrl} ELSE NULL END` }; -const EXTENSION_COLUMNS = { - id: extensions.id, - ...CONTENT_COLUMNS, - ...DEVELOPER_COLUMNS +// Detail preserves stored fields; the overall size guard prevents oversized reads. +const DETAIL_CONTENT_COLUMNS = { + type: sql< + string | null + >`CASE WHEN ${publishedBytes} <= ${MAX_CONTENT_BYTES} THEN ${extensions.type} ELSE NULL END`, + name: sql< + string | null + >`CASE WHEN ${publishedBytes} <= ${MAX_CONTENT_BYTES} THEN ${extensions.name} ELSE NULL END`, + description: sql< + string | null + >`CASE WHEN ${publishedBytes} <= ${MAX_CONTENT_BYTES} THEN ${extensions.description} ELSE NULL END`, + releases: sql< + string | null + >`CASE WHEN ${publishedBytes} <= ${MAX_CONTENT_BYTES} THEN ${extensions.releases} ELSE NULL END`, + website: sql< + string | null + >`CASE WHEN ${publishedBytes} <= ${MAX_CONTENT_BYTES} THEN ${extensions.website} ELSE NULL END`, + license: sql< + string | null + >`CASE WHEN ${publishedBytes} <= ${MAX_CONTENT_BYTES} THEN ${extensions.license} ELSE NULL END`, + iconUrl: sql< + string | null + >`CASE WHEN ${publishedBytes} <= ${MAX_CONTENT_BYTES} THEN ${extensions.iconUrl} ELSE NULL END`, + readme: sql< + string | null + >`CASE WHEN ${publishedBytes} <= ${MAX_CONTENT_BYTES} THEN ${extensions.readme} ELSE NULL END`, + source: sql< + string | null + >`CASE WHEN ${publishedBytes} <= ${MAX_CONTENT_BYTES} THEN ${extensions.source} ELSE NULL END`, + version: sql< + string | null + >`CASE WHEN ${publishedBytes} <= ${MAX_CONTENT_BYTES} THEN ${extensions.version} ELSE NULL END`, + downloadUrl: sql< + string | null + >`CASE WHEN ${publishedBytes} <= ${MAX_CONTENT_BYTES} THEN ${extensions.downloadUrl} ELSE NULL END` }; -// Derived by subtraction so a column added to CONTENT_COLUMNS cannot be -// forgotten here: catalogue cards omit only the two large fields. const { readme: _readme, releases: _releases, - ...EXTENSION_LIST_COLUMNS -} = EXTENSION_COLUMNS; + ...CARD_CONTENT_COLUMNS +} = CONTENT_COLUMNS; + +const EXTENSION_COLUMNS = { + id: extensions.id, + ...DETAIL_CONTENT_COLUMNS, + publishedBytes, + ...DEVELOPER_COLUMNS +}; + +const EXTENSION_LIST_COLUMNS = { + id: extensions.id, + ...CARD_CONTENT_COLUMNS, + publishedBytes, + ...DEVELOPER_COLUMNS +}; // The owner view joins extension_revisions twice: once for the unreviewed // edit (at most one - idx_extension_revisions_pending), once for the most @@ -104,23 +173,17 @@ const REVIEW_COLUMNS = { pendingCreatedAt: PENDING.createdAt, reviewedId: REVIEWED.id, reviewedStatus: REVIEWED.status, - reviewedNote: REVIEWED.reviewNote, + reviewedNote: sql`substr(${REVIEWED.reviewNote}, 1, 2000)`, reviewedAt: REVIEWED.reviewedAt }; -const { - readme: _ownedReadme, - releases: _ownedReleases, - ...CARD_CONTENT_COLUMNS -} = CONTENT_COLUMNS; - // The owner list drops the same two large published fields the catalogue does, // and the pending revision's stored content (up to 256 KiB per row) with it. const OWNED_LIST_COLUMNS = { id: extensions.id, publishedAt: extensions.publishedAt, delistedAt: extensions.delistedAt, - delistReason: extensions.delistReason, + delistReason: sql`substr(${extensions.delistReason}, 1, 2000)`, createdAt: extensions.createdAt, updatedAt: extensions.updatedAt, ...CARD_CONTENT_COLUMNS, @@ -130,9 +193,12 @@ const OWNED_LIST_COLUMNS = { const OWNED_COLUMNS = { ...OWNED_LIST_COLUMNS, - readme: extensions.readme, - releases: extensions.releases, - pendingContent: PENDING.content + ...DETAIL_CONTENT_COLUMNS, + pendingContent: sql< + string | null + >`CASE WHEN length(CAST(${PENDING.content} AS BLOB)) <= ${MAX_CONTENT_BYTES} THEN ${PENDING.content} ELSE NULL END`, + pendingBytes: sql`COALESCE(length(CAST(${PENDING.content} AS BLOB)),0)`, + publishedBytes }; // Repeated rather than factored out: drizzle's builder types are keyed on the @@ -186,9 +252,13 @@ interface PublishedRow extends DeveloperRow { source: string; version: string; downloadUrl: string; + publishedBytes: number; } -type PublishedListRow = Omit; +type PublishedListRow = Omit< + PublishedRow, + "readme" | "releases" | "website" | "downloadUrl" +> & { website: string | null; downloadUrl: string | null }; export interface ExtensionListFilters { type?: string; @@ -231,6 +301,7 @@ export class ExtensionsDatabase { const limit = filters.limit ?? 50; const conditions = [ isNotNull(extensions.publishedAt), + sql`${extensions.publishedBytes} <= ${MAX_CONTENT_BYTES}`, isNull(extensions.delistedAt) ]; if (filters.type) conditions.push(eq(extensions.type, filters.type)); @@ -261,7 +332,7 @@ export class ExtensionsDatabase { const last = pageRows.at(-1); return { data: { - items: pageRows.map(parseListRow), + items: pageRows.map((row) => parseListRow(row)), hasMore, nextCursor: hasMore && last ? encodeCursor(last.id) : null }, @@ -289,7 +360,13 @@ export class ExtensionsDatabase { const row = rows[0]; if (!row) return notFound(id); - return { data: parseRow(row), error: null }; + if (row.publishedBytes > MAX_CONTENT_BYTES) return oversizedContent(); + try { + return { data: parseRow(row), error: null }; + } catch (error) { + if (error instanceof LegacyContentError) return oversizedContent(); + return databaseError("getById", error); + } } async listOwned(filters: { @@ -501,13 +578,23 @@ export class ExtensionsDatabase { const row = rows[0]; if (!row) return notFound(id); - return { - data: { - extension: parseOwnedRow(row), - ownerUserId: row.developerOwnerUserId - }, - error: null - }; + if ( + row.publishedBytes > MAX_CONTENT_BYTES || + row.pendingBytes > MAX_CONTENT_BYTES + ) + return oversizedContent(); + try { + return { + data: { + extension: parseOwnedRow(row), + ownerUserId: row.developerOwnerUserId + }, + error: null + }; + } catch (error) { + if (error instanceof LegacyContentError) return oversizedContent(); + return databaseError("getOwned", error); + } } // Creates the extension record and its first pending revision as one @@ -523,8 +610,8 @@ export class ExtensionsDatabase { let results; try { const extensionStmt = toD1Statement(this.db.$client, { - sql: `INSERT INTO extensions (id, developer_id, created_at, updated_at) - SELECT ?, d.id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + sql: `INSERT INTO extensions (id, developer_id, created_by, created_at, updated_at) + SELECT ?, d.id, d.owner_user_id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP FROM developers d WHERE d.id = ? AND d.owner_user_id = ? AND d.ownership_epoch = ? AND EXISTS ( @@ -563,14 +650,14 @@ export class ExtensionsDatabase { results = await this.db.$client.batch([extensionStmt, revisionStmt]); } catch (error) { - return databaseError("create", error); + return contentAdmissionError("create", error); } if (!results[0]?.meta?.changes) { try { return { data: null, error: await this.createBlockedError(input) }; } catch (error) { - return databaseError("create", error); + return contentAdmissionError("create", error); } } @@ -939,7 +1026,8 @@ export class ExtensionsDatabase { AND EXISTS ( SELECT 1 FROM users u WHERE u.id = ? AND u.deleted_at IS NULL AND u.is_moderator = 1 - )`, + ) + RETURNING extension_id`, params: [ revisionId, moderatorId, @@ -978,28 +1066,18 @@ export class ExtensionsDatabase { results = await this.db.$client.batch([correctStmt, publishStmt]); } catch (error) { - return databaseError("moderatorCorrect", error); + return contentAdmissionError("moderatorCorrect", error); } if (!results[0]?.meta?.changes) { return this.moderatorCorrectBlockedError(id, moderatorId); } - // Canonical id for the response (the path param may differ in case). - // Inside error handling like every other read here: the correction is - // already committed, so a failure must report a database error rather - // than throw past the route. - let row: { canonicalId: string } | undefined; - try { - [row] = await this.db - .select({ canonicalId: extensions.id }) - .from(extensions) - .where(sql`LOWER(${extensions.id}) = LOWER(${id})`); - } catch (error) { - return databaseError("moderatorCorrect", error); - } return { - data: { id: row?.canonicalId ?? id, revisionId }, + data: { + id: (results[0].results[0] as { extension_id: string }).extension_id, + revisionId + }, error: null }; } @@ -1172,18 +1250,48 @@ function parseDeveloper(row: DeveloperRow): PublicDeveloper { // Shared by both parsers so the catalogue card and the detail view can never // disagree about the embedded developer. -function parseListRow(row: PublishedListRow): ExtensionListItem { +function cardUrl(url: string | null): string | null { + return url && url.length <= 2048 ? url : null; +} + +function cardLicense(stored: string | null): License { + const license = parseJSON(stored ?? "", { name: "Unavailable" }); + license.name = + typeof license.name === "string" && license.name + ? license.name.slice(0, 100) + : "Unavailable"; + if (license.URL && license.URL.length > 2048) delete license.URL; + return license; +} + +function cardSource(stored: string | null): Repository { + const source = parseJSON(stored ?? "", { + type: "custom", + repo: "Unavailable" + }); + source.repo = + typeof source.repo === "string" && source.repo + ? source.repo.slice(0, 500) + : "Unavailable"; + return source; +} + +function parseListRow(row: PublishedListRow, card = true): ExtensionListItem { return { id: row.id, type: row.type as ExtensionListItem["type"], - name: row.name, - description: row.description, - website: row.website, - license: parseJSON(row.license, { name: "" }), - icon_url: row.iconUrl ?? undefined, - source: parseJSON(row.source, { type: "custom", repo: "" }), - version: row.version, - download_url: row.downloadUrl, + name: card ? row.name.slice(0, 120) : row.name, + description: card ? row.description.slice(0, 4000) : row.description, + website: card ? cardUrl(row.website) : row.website, + license: card + ? cardLicense(row.license) + : parseJSON(row.license, { name: "" }), + icon_url: (card ? cardUrl(row.iconUrl) : row.iconUrl) ?? undefined, + source: card + ? cardSource(row.source) + : parseJSON(row.source, { type: "custom", repo: "" }), + version: card ? row.version.slice(0, 100) : row.version, + download_url: card ? cardUrl(row.downloadUrl) : row.downloadUrl, developer: parseDeveloper(row) }; } @@ -1192,30 +1300,41 @@ function parseListRow(row: PublishedListRow): ExtensionListItem { // catalogue query deliberately omits. function parseRow(row: PublishedRow): Extension { return { - ...parseListRow(row), + ...parseListRow(row, false), + website: row.website, + download_url: row.downloadUrl, readme: row.readme, - releases: sortReleasesDescending(parseJSON(row.releases, [])) + releases: boundedReleases(parseJSON(row.releases, [])) }; } // Only ever called for a row whose published_at is set, where // extensions_published_content_check guarantees each of these is present. function publishedContent( - row: OwnedListRow -): Omit { + row: OwnedListRow, + card = true +): NonNullable { return { type: row.type as ExtensionContent["type"], - name: row.name as string, - description: row.description as string, - website: row.website as string, - license: parseJSON(row.license as string, { name: "" }), - icon_url: row.iconUrl ?? undefined, - source: parseJSON(row.source as string, { - type: "custom", - repo: "" - }), - version: row.version as string, - download_url: row.downloadUrl as string + name: card ? (row.name as string).slice(0, 120) : (row.name as string), + description: card + ? (row.description as string).slice(0, 4000) + : (row.description as string), + website: card ? cardUrl(row.website) : row.website, + license: card + ? cardLicense(row.license) + : parseJSON(row.license as string, { name: "" }), + icon_url: (card ? cardUrl(row.iconUrl) : row.iconUrl) ?? undefined, + source: card + ? cardSource(row.source) + : parseJSON(row.source as string, { + type: "custom", + repo: "" + }), + version: card + ? (row.version as string).slice(0, 100) + : (row.version as string), + download_url: card ? cardUrl(row.downloadUrl) : row.downloadUrl }; } @@ -1251,9 +1370,11 @@ function parseOwnedRow(row: OwnedRow): OwnedExtension { ...parseOwnedListRow(row), published: row.publishedAt ? { - ...publishedContent(row), + ...publishedContent(row, false), + website: row.website as string, + download_url: row.downloadUrl as string, readme: row.readme as string, - releases: sortReleasesDescending( + releases: boundedReleases( parseJSON(row.releases as string, []) ) } @@ -1275,6 +1396,39 @@ export function parseContent(stored: string | null): StoredExtensionContent { const content = parseJSON(stored ?? "", {}); return { ...content, - releases: sortReleasesDescending(content.releases ?? []) + releases: boundedReleases(content.releases ?? []) + }; +} + +export class LegacyContentError extends Error {} + +// Legacy bodies bypassed today's schema. Bound the collection and tag work +// before semver sorting, while retaining support for partial historical content. +function boundedReleases(value: unknown): Release[] { + if ( + !Array.isArray(value) || + value.length > 100 || + value.some( + (release) => + !release || + typeof release !== "object" || + typeof release.tag !== "string" || + Array.from(release.tag).length > 100 + ) + ) + throw new LegacyContentError( + "Legacy release collection exceeds safe read bounds" + ); + return sortReleasesDescending(value as Release[]); +} + +export function oversizedContent(): DatabaseResult { + return { + data: null, + error: { + code: "CONTENT_UNAVAILABLE", + message: + "Legacy content exceeds safe read bounds; resubmit or export it administratively" + } }; } diff --git a/src/services/extensions/v2/db/migrations/0026_resource_bounds.sql b/src/services/extensions/v2/db/migrations/0026_resource_bounds.sql new file mode 100644 index 0000000..27e2bd8 --- /dev/null +++ b/src/services/extensions/v2/db/migrations/0026_resource_bounds.sql @@ -0,0 +1,432 @@ +-- Admission and accounting share the content-write transaction. Backfill +-- preserves legacy content; existing over-quota collections cannot grow. +ALTER TABLE extensions ADD COLUMN created_by TEXT; +ALTER TABLE extensions ADD COLUMN published_bytes INTEGER NOT NULL DEFAULT 0; +UPDATE extensions SET published_bytes = COALESCE(length(CAST(extensions.type AS BLOB)),0) + + COALESCE(length(CAST(extensions.name AS BLOB)),0) + + COALESCE(length(CAST(extensions.description AS BLOB)),0) + + COALESCE(length(CAST(extensions.releases AS BLOB)),0) + + COALESCE(length(CAST(extensions.website AS BLOB)),0) + + COALESCE(length(CAST(extensions.license AS BLOB)),0) + + COALESCE(length(CAST(extensions.icon_url AS BLOB)),0) + + COALESCE(length(CAST(extensions.readme AS BLOB)),0) + + COALESCE(length(CAST(extensions.source AS BLOB)),0) + + COALESCE(length(CAST(extensions.version AS BLOB)),0) + + COALESCE(length(CAST(extensions.download_url AS BLOB)),0); +ALTER TABLE extension_revisions ADD COLUMN content_bytes INTEGER NOT NULL DEFAULT 0; +UPDATE extension_revisions SET content_bytes = length(CAST(content AS BLOB)); +ALTER TABLE extension_revisions ADD COLUMN content_hash TEXT; +ALTER TABLE extension_revisions ADD COLUMN compacted_at TEXT; +ALTER TABLE extension_revisions ADD COLUMN summary_name TEXT; +ALTER TABLE extension_revisions ADD COLUMN summary_version TEXT; +ALTER TABLE extension_revisions ADD COLUMN summary_description TEXT; +ALTER TABLE extension_revisions ADD COLUMN content_readable INTEGER NOT NULL DEFAULT 0; +UPDATE extension_revisions SET content_readable = CASE WHEN length(CAST(content AS BLOB)) <= 262144 AND json_valid(content) THEN + CASE WHEN json_type(content) != 'object' THEN 0 + WHEN json_type(content, '$.releases') IS NULL OR json_type(content, '$.releases') = 'null' THEN 1 + WHEN json_type(content, '$.releases') != 'array' THEN 0 + WHEN json_array_length(content, '$.releases') > 100 THEN 0 + ELSE NOT EXISTS (SELECT 1 FROM json_each(content, '$.releases') r + WHERE CASE WHEN r.type = 'object' THEN + json_type(r.value, '$.tag') IS NOT 'text' OR length(json_extract(r.value, '$.tag')) > 100 + ELSE 1 END) + END ELSE 0 END, + summary_name = CASE WHEN content_bytes <= 262144 AND json_valid(content) + THEN CASE WHEN json_type(content,'$.name')='text' THEN substr(json_extract(content,'$.name'),1,120) ELSE NULL END + ELSE NULL END, + summary_version = CASE WHEN content_bytes <= 262144 AND json_valid(content) + THEN CASE WHEN json_type(content,'$.version')='text' THEN substr(json_extract(content,'$.version'),1,100) ELSE NULL END + ELSE NULL END, + summary_description = CASE WHEN content_bytes <= 262144 AND json_valid(content) + THEN CASE WHEN json_type(content,'$.description')='text' THEN substr(json_extract(content,'$.description'),1,4000) ELSE NULL END + ELSE NULL END; + +UPDATE extensions SET created_by = COALESCE( + (SELECT submitted_by FROM extension_revisions r WHERE r.extension_id = extensions.id ORDER BY created_at, id LIMIT 1), + (SELECT owner_user_id FROM developers d WHERE d.id = extensions.developer_id) +); +CREATE TABLE extension_resource_usage ( + scope TEXT NOT NULL, subject TEXT NOT NULL, bytes INTEGER NOT NULL DEFAULT 0, + extensions INTEGER NOT NULL DEFAULT 0, revisions INTEGER NOT NULL DEFAULT 0 +); +CREATE UNIQUE INDEX idx_extension_resource_usage_subject ON extension_resource_usage(scope,subject); +CREATE TABLE extension_write_events ( + id TEXT PRIMARY KEY NOT NULL, account_id TEXT NOT NULL, + developer_id TEXT NOT NULL, occurred_at INTEGER NOT NULL +); +CREATE INDEX idx_extension_write_events_account ON extension_write_events(account_id,occurred_at); +CREATE INDEX idx_extension_write_events_developer ON extension_write_events(developer_id,occurred_at); +CREATE INDEX idx_extension_write_events_time ON extension_write_events(occurred_at); + +--> statement-breakpoint +INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) +SELECT 'global', 'all', COALESCE(SUM(t.published_bytes),0), COUNT(*), 0 +FROM extensions t GROUP BY 'all' +ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes, extensions=extensions+excluded.extensions, revisions=revisions+excluded.revisions; +--> statement-breakpoint +INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) +SELECT 'account', COALESCE(t.created_by,'legacy'), COALESCE(SUM(t.published_bytes),0), COUNT(*), 0 +FROM extensions t GROUP BY COALESCE(t.created_by,'legacy') +ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes, extensions=extensions+excluded.extensions, revisions=revisions+excluded.revisions; +--> statement-breakpoint +INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) +SELECT 'developer', t.developer_id, COALESCE(SUM(t.published_bytes),0), COUNT(*), 0 +FROM extensions t GROUP BY t.developer_id +ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes, extensions=extensions+excluded.extensions, revisions=revisions+excluded.revisions; +--> statement-breakpoint +INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) +SELECT 'global', 'all', COALESCE(SUM(t.content_bytes),0), 0, COUNT(*) +FROM extension_revisions t GROUP BY 'all' +ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes, extensions=extensions+excluded.extensions, revisions=revisions+excluded.revisions; +--> statement-breakpoint +INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) +SELECT 'account', t.submitted_by, COALESCE(SUM(t.content_bytes),0), 0, COUNT(*) +FROM extension_revisions t GROUP BY t.submitted_by +ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes, extensions=extensions+excluded.extensions, revisions=revisions+excluded.revisions; +--> statement-breakpoint +INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) +SELECT 'developer', t.developer_id, COALESCE(SUM(t.content_bytes),0), 0, COUNT(*) +FROM extension_revisions t GROUP BY t.developer_id +ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes, extensions=extensions+excluded.extensions, revisions=revisions+excluded.revisions; +--> statement-breakpoint +INSERT INTO extension_resource_usage(scope,subject) VALUES ('global','all') + ON CONFLICT DO NOTHING; +--> statement-breakpoint +CREATE TRIGGER resource_extensions_insert AFTER INSERT ON extensions BEGIN + INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) + SELECT scope, subject, COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0), 1, 0 + FROM ( + SELECT 'global' AS scope, 'all' AS subject + UNION ALL SELECT 'account', COALESCE(NEW.created_by,'legacy') + UNION ALL SELECT 'developer', NEW.developer_id + ) WHERE true + ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes, + extensions=extensions+excluded.extensions, + revisions=revisions+excluded.revisions; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM extension_resource_usage + WHERE ((scope='account' AND subject=COALESCE(NEW.created_by,'legacy')) + OR (scope='developer' AND subject=NEW.developer_id)) + AND (bytes>26214400 OR extensions>100 OR revisions>1000) + ) THEN RAISE(ABORT,'extension_resource_quota') END; +END; +--> statement-breakpoint +CREATE TRIGGER resource_extensions_delete AFTER DELETE ON extensions BEGIN + UPDATE extension_resource_usage + SET bytes=bytes-(COALESCE(length(CAST(OLD.type AS BLOB)),0) + + COALESCE(length(CAST(OLD.name AS BLOB)),0) + + COALESCE(length(CAST(OLD.description AS BLOB)),0) + + COALESCE(length(CAST(OLD.releases AS BLOB)),0) + + COALESCE(length(CAST(OLD.website AS BLOB)),0) + + COALESCE(length(CAST(OLD.license AS BLOB)),0) + + COALESCE(length(CAST(OLD.icon_url AS BLOB)),0) + + COALESCE(length(CAST(OLD.readme AS BLOB)),0) + + COALESCE(length(CAST(OLD.source AS BLOB)),0) + + COALESCE(length(CAST(OLD.version AS BLOB)),0) + + COALESCE(length(CAST(OLD.download_url AS BLOB)),0)), extensions=extensions-1 + WHERE (scope='global' AND subject='all') + OR (scope='account' AND subject=COALESCE(OLD.created_by,'legacy')) + OR (scope='developer' AND subject=OLD.developer_id); +END; +--> statement-breakpoint +-- Internal byte/summary synchronization must not create usage a second time. +-- Account only updates of the actual charged fields. +CREATE TRIGGER resource_extensions_update AFTER UPDATE OF type,name,description,releases,website,license,icon_url,readme,source,version,download_url ON extensions BEGIN + UPDATE extension_resource_usage + SET bytes = bytes + (COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) - (COALESCE(length(CAST(OLD.type AS BLOB)),0) + + COALESCE(length(CAST(OLD.name AS BLOB)),0) + + COALESCE(length(CAST(OLD.description AS BLOB)),0) + + COALESCE(length(CAST(OLD.releases AS BLOB)),0) + + COALESCE(length(CAST(OLD.website AS BLOB)),0) + + COALESCE(length(CAST(OLD.license AS BLOB)),0) + + COALESCE(length(CAST(OLD.icon_url AS BLOB)),0) + + COALESCE(length(CAST(OLD.readme AS BLOB)),0) + + COALESCE(length(CAST(OLD.source AS BLOB)),0) + + COALESCE(length(CAST(OLD.version AS BLOB)),0) + + COALESCE(length(CAST(OLD.download_url AS BLOB)),0)) + WHERE (scope='global' AND subject='all') + OR (scope='account' AND subject=COALESCE(NEW.created_by,'legacy')) + OR (scope='developer' AND subject=NEW.developer_id); + SELECT CASE WHEN (COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) > (COALESCE(length(CAST(OLD.type AS BLOB)),0) + + COALESCE(length(CAST(OLD.name AS BLOB)),0) + + COALESCE(length(CAST(OLD.description AS BLOB)),0) + + COALESCE(length(CAST(OLD.releases AS BLOB)),0) + + COALESCE(length(CAST(OLD.website AS BLOB)),0) + + COALESCE(length(CAST(OLD.license AS BLOB)),0) + + COALESCE(length(CAST(OLD.icon_url AS BLOB)),0) + + COALESCE(length(CAST(OLD.readme AS BLOB)),0) + + COALESCE(length(CAST(OLD.source AS BLOB)),0) + + COALESCE(length(CAST(OLD.version AS BLOB)),0) + + COALESCE(length(CAST(OLD.download_url AS BLOB)),0)) AND EXISTS ( + SELECT 1 FROM extension_resource_usage + WHERE ((scope='account' AND subject=COALESCE(NEW.created_by,'legacy')) + OR (scope='developer' AND subject=NEW.developer_id)) + AND (bytes>26214400 OR extensions>100 OR revisions>1000) + ) + THEN RAISE(ABORT,'extension_resource_quota') END; +END; +--> statement-breakpoint +CREATE TRIGGER resource_extension_revisions_insert AFTER INSERT ON extension_revisions BEGIN + INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) + SELECT scope, subject, length(CAST(NEW.content AS BLOB)), 0, 1 + FROM ( + SELECT 'global' AS scope, 'all' AS subject + UNION ALL SELECT 'account', NEW.submitted_by + UNION ALL SELECT 'developer', NEW.developer_id + ) WHERE true + ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes, + extensions=extensions+excluded.extensions, + revisions=revisions+excluded.revisions; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM extension_resource_usage + WHERE ((scope='account' AND subject=NEW.submitted_by) + OR (scope='developer' AND subject=NEW.developer_id)) + AND (bytes>26214400 OR extensions>100 OR revisions>1000) + ) THEN RAISE(ABORT,'extension_resource_quota') END; +END; +--> statement-breakpoint +CREATE TRIGGER resource_extension_revisions_delete AFTER DELETE ON extension_revisions BEGIN + UPDATE extension_resource_usage + SET bytes=bytes-(length(CAST(OLD.content AS BLOB))), revisions=revisions-1 + WHERE (scope='global' AND subject='all') + OR (scope='account' AND subject=OLD.submitted_by) + OR (scope='developer' AND subject=OLD.developer_id); +END; +--> statement-breakpoint +CREATE TRIGGER resource_extension_revisions_update AFTER UPDATE OF content ON extension_revisions +WHEN NEW.submitted_by=OLD.submitted_by AND NEW.developer_id=OLD.developer_id BEGIN + UPDATE extension_resource_usage + SET bytes = bytes + length(CAST(NEW.content AS BLOB)) - length(CAST(OLD.content AS BLOB)) + WHERE (scope='global' AND subject='all') + OR (scope='account' AND subject=NEW.submitted_by) + OR (scope='developer' AND subject=NEW.developer_id); + SELECT CASE WHEN length(CAST(NEW.content AS BLOB)) > length(CAST(OLD.content AS BLOB)) AND EXISTS ( + SELECT 1 FROM extension_resource_usage + WHERE ((scope='account' AND subject=NEW.submitted_by) + OR (scope='developer' AND subject=NEW.developer_id)) + AND (bytes>26214400 OR extensions>100 OR revisions>1000) + ) + THEN RAISE(ABORT,'extension_resource_quota') END; +END; +--> statement-breakpoint +CREATE TRIGGER resource_extension_revisions_identity_update AFTER UPDATE OF submitted_by,developer_id ON extension_revisions +WHEN NEW.submitted_by IS NOT OLD.submitted_by OR NEW.developer_id IS NOT OLD.developer_id BEGIN +UPDATE extension_resource_usage SET bytes=bytes+((length(CAST(OLD.content AS BLOB))) * -1), revisions=revisions+(-1) + WHERE scope='global' AND subject='all'; +UPDATE extension_resource_usage SET bytes=bytes+((length(CAST(OLD.content AS BLOB))) * -1), revisions=revisions+(-1) + WHERE scope='account' AND subject=OLD.submitted_by; +UPDATE extension_resource_usage SET bytes=bytes+((length(CAST(OLD.content AS BLOB))) * -1), revisions=revisions+(-1) + WHERE scope='developer' AND subject=OLD.developer_id; +INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) VALUES ('global','all',(length(CAST(NEW.content AS BLOB))),0,1) + ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes,extensions=extensions+excluded.extensions,revisions=revisions+excluded.revisions; +INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) VALUES ('account',NEW.submitted_by,(length(CAST(NEW.content AS BLOB))),0,1) + ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes,extensions=extensions+excluded.extensions,revisions=revisions+excluded.revisions; +SELECT CASE WHEN ((length(CAST(NEW.content AS BLOB))) > (length(CAST(OLD.content AS BLOB)))) AND EXISTS (SELECT 1 FROM extension_resource_usage + WHERE scope='account' AND subject=NEW.submitted_by AND (bytes>26214400 OR extensions>100 OR revisions>1000)) + THEN RAISE(ABORT,'extension_resource_quota') END; +INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) VALUES ('developer',NEW.developer_id,(length(CAST(NEW.content AS BLOB))),0,1) + ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes,extensions=extensions+excluded.extensions,revisions=revisions+excluded.revisions; +SELECT CASE WHEN ((length(CAST(NEW.content AS BLOB))) > (length(CAST(OLD.content AS BLOB)))) AND EXISTS (SELECT 1 FROM extension_resource_usage + WHERE scope='developer' AND subject=NEW.developer_id AND (bytes>26214400 OR extensions>100 OR revisions>1000)) + THEN RAISE(ABORT,'extension_resource_quota') END; +END; +--> statement-breakpoint +-- A rolling-day ledger, separate from revisions: withdrawal and review do +-- not refund allowance. CURRENT_TIMESTAMP is supplied by the server. The +-- event uses created_at so historical fixture/import writes keep their age. +CREATE TRIGGER extension_revision_admission BEFORE INSERT ON extension_revisions BEGIN + -- Cleanup scales with accepted writes; idle cleanup still runs hourly. + DELETE FROM extension_write_events WHERE id IN ( + SELECT id FROM extension_write_events WHERE occurred_at <= unixepoch()-86400 + ORDER BY occurred_at LIMIT 50 + ); + SELECT CASE WHEN length(CAST(NEW.content AS BLOB)) > 262144 + + THEN RAISE(ABORT,'extension_content_size') END; + SELECT CASE WHEN (SELECT COUNT(*) FROM extension_write_events WHERE account_id=NEW.submitted_by AND occurred_at>unixepoch()-60)>=5 + OR (SELECT COUNT(*) FROM extension_write_events WHERE developer_id=NEW.developer_id AND occurred_at>unixepoch()-60)>=5 + + THEN RAISE(ABORT,'extension_write_rate_minute') END; + SELECT CASE WHEN (SELECT COUNT(*) FROM extension_write_events WHERE account_id=NEW.submitted_by AND occurred_at>unixepoch()-86400)>=50 + OR (SELECT COUNT(*) FROM extension_write_events WHERE developer_id=NEW.developer_id AND occurred_at>unixepoch()-86400)>=50 + + THEN RAISE(ABORT,'extension_write_rate_day') END; +END; +CREATE TRIGGER extension_revision_write_event AFTER INSERT ON extension_revisions BEGIN + INSERT INTO extension_write_events(id,account_id,developer_id,occurred_at) + VALUES (NEW.id,NEW.submitted_by,NEW.developer_id,unixepoch(NEW.created_at)); +END; +CREATE TRIGGER extension_revision_content_bound BEFORE UPDATE OF content ON extension_revisions +WHEN length(CAST(NEW.content AS BLOB)) > 262144 AND length(CAST(NEW.content AS BLOB)) > length(CAST(OLD.content AS BLOB)) +BEGIN SELECT RAISE(ABORT,'extension_content_size'); END; +CREATE TRIGGER extension_revision_identity BEFORE UPDATE OF id,extension_id,developer_id,submitted_by ON extension_revisions +WHEN NEW.id IS NOT OLD.id OR NEW.extension_id IS NOT OLD.extension_id + OR NEW.developer_id IS NOT OLD.developer_id OR NEW.submitted_by IS NOT OLD.submitted_by +BEGIN SELECT RAISE(ABORT,'extension_resource_identity'); END; +CREATE TRIGGER extension_resource_identity BEFORE UPDATE OF created_by,developer_id ON extensions +WHEN NEW.created_by IS NOT OLD.created_by OR NEW.developer_id IS NOT OLD.developer_id +BEGIN SELECT RAISE(ABORT,'extension_resource_identity'); END; + +--> statement-breakpoint +CREATE TRIGGER sync_extensions_bytes_insert AFTER INSERT ON extensions +WHEN NEW.published_bytes != (COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) BEGIN + UPDATE extensions SET published_bytes=(COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) + WHERE id=NEW.id; +END; + +--> statement-breakpoint +CREATE TRIGGER sync_extensions_bytes_update AFTER UPDATE OF type,name,description,releases,website,license,icon_url,readme,source,version,download_url,published_bytes ON extensions +WHEN NEW.published_bytes != (COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) BEGIN + UPDATE extensions SET published_bytes=(COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) + WHERE id=NEW.id; +END; + +--> statement-breakpoint +CREATE TRIGGER sync_extension_revisions_bytes_insert AFTER INSERT ON extension_revisions +WHEN NEW.content_bytes != (length(CAST(NEW.content AS BLOB))) BEGIN + UPDATE extension_revisions SET content_bytes=(length(CAST(NEW.content AS BLOB))) + WHERE id=NEW.id; +END; + +--> statement-breakpoint +CREATE TRIGGER sync_extension_revisions_bytes_update AFTER UPDATE OF content,content_bytes ON extension_revisions +WHEN NEW.content_bytes != (length(CAST(NEW.content AS BLOB))) BEGIN + UPDATE extension_revisions SET content_bytes=(length(CAST(NEW.content AS BLOB))) + WHERE id=NEW.id; +END; + +--> statement-breakpoint +CREATE TRIGGER sync_revision_summary_insert AFTER INSERT ON extension_revisions +WHEN NEW.compacted_at IS NULL BEGIN + UPDATE extension_revisions SET content_readable = CASE WHEN length(CAST(NEW.content AS BLOB)) <= 262144 AND json_valid(NEW.content) THEN + CASE WHEN json_type(NEW.content) != 'object' THEN 0 + WHEN json_type(NEW.content, '$.releases') IS NULL OR json_type(NEW.content, '$.releases') = 'null' THEN 1 + WHEN json_type(NEW.content, '$.releases') != 'array' THEN 0 + WHEN json_array_length(NEW.content, '$.releases') > 100 THEN 0 + ELSE NOT EXISTS (SELECT 1 FROM json_each(NEW.content, '$.releases') r + WHERE CASE WHEN r.type = 'object' THEN + json_type(r.value, '$.tag') IS NOT 'text' OR length(json_extract(r.value, '$.tag')) > 100 + ELSE 1 END) + END ELSE 0 END, + summary_name = CASE WHEN length(CAST(NEW.content AS BLOB)) <= 262144 AND json_valid(NEW.content) + THEN CASE WHEN json_type(NEW.content,'$.name')='text' THEN substr(json_extract(NEW.content,'$.name'),1,120) ELSE NULL END + ELSE NULL END, + summary_version = CASE WHEN length(CAST(NEW.content AS BLOB)) <= 262144 AND json_valid(NEW.content) + THEN CASE WHEN json_type(NEW.content,'$.version')='text' THEN substr(json_extract(NEW.content,'$.version'),1,100) ELSE NULL END + ELSE NULL END, + summary_description = CASE WHEN length(CAST(NEW.content AS BLOB)) <= 262144 AND json_valid(NEW.content) + THEN CASE WHEN json_type(NEW.content,'$.description')='text' THEN substr(json_extract(NEW.content,'$.description'),1,4000) ELSE NULL END + ELSE NULL END + WHERE id=NEW.id; +END; + +--> statement-breakpoint +CREATE TRIGGER sync_revision_summary_update AFTER UPDATE OF content ON extension_revisions +WHEN NEW.compacted_at IS NULL BEGIN + UPDATE extension_revisions SET content_readable = CASE WHEN length(CAST(NEW.content AS BLOB)) <= 262144 AND json_valid(NEW.content) THEN + CASE WHEN json_type(NEW.content) != 'object' THEN 0 + WHEN json_type(NEW.content, '$.releases') IS NULL OR json_type(NEW.content, '$.releases') = 'null' THEN 1 + WHEN json_type(NEW.content, '$.releases') != 'array' THEN 0 + WHEN json_array_length(NEW.content, '$.releases') > 100 THEN 0 + ELSE NOT EXISTS (SELECT 1 FROM json_each(NEW.content, '$.releases') r + WHERE CASE WHEN r.type = 'object' THEN + json_type(r.value, '$.tag') IS NOT 'text' OR length(json_extract(r.value, '$.tag')) > 100 + ELSE 1 END) + END ELSE 0 END, + summary_name = CASE WHEN length(CAST(NEW.content AS BLOB)) <= 262144 AND json_valid(NEW.content) + THEN CASE WHEN json_type(NEW.content,'$.name')='text' THEN substr(json_extract(NEW.content,'$.name'),1,120) ELSE NULL END + ELSE NULL END, + summary_version = CASE WHEN length(CAST(NEW.content AS BLOB)) <= 262144 AND json_valid(NEW.content) + THEN CASE WHEN json_type(NEW.content,'$.version')='text' THEN substr(json_extract(NEW.content,'$.version'),1,100) ELSE NULL END + ELSE NULL END, + summary_description = CASE WHEN length(CAST(NEW.content AS BLOB)) <= 262144 AND json_valid(NEW.content) + THEN CASE WHEN json_type(NEW.content,'$.description')='text' THEN substr(json_extract(NEW.content,'$.description'),1,4000) ELSE NULL END + ELSE NULL END + WHERE id=NEW.id; +END; + +--> statement-breakpoint +CREATE INDEX idx_extensions_published_revision ON extensions(published_revision_id); +--> statement-breakpoint +CREATE INDEX idx_extension_revisions_retention ON extension_revisions(compacted_at,reviewed_at,id,status); diff --git a/src/services/extensions/v2/db/migrations/meta/0026_snapshot.json b/src/services/extensions/v2/db/migrations/meta/0026_snapshot.json new file mode 100644 index 0000000..8acb7f3 --- /dev/null +++ b/src/services/extensions/v2/db/migrations/meta/0026_snapshot.json @@ -0,0 +1,1274 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "6580dc1f-aa42-40cb-88c2-cf7971598bb6", + "prevId": "2a49cad7-5b9b-4280-aa04-04c373cfe355", + "tables": { + "claim_verification_budgets": { + "name": "claim_verification_budgets", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_claim_verification_budgets_expiry": { + "name": "idx_claim_verification_budgets_expiry", + "columns": ["expires_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developer_claims": { + "name": "developer_claims", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "claimant_id": { + "name": "claimant_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "review_note": { + "name": "review_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reviewer_id": { + "name": "reviewer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_org_verified": { + "name": "github_org_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verification_note": { + "name": "github_verification_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developer_claims_developer": { + "name": "idx_developer_claims_developer", + "columns": ["developer_id"], + "isUnique": false + }, + "idx_developer_claims_claimant": { + "name": "idx_developer_claims_claimant", + "columns": ["claimant_id"], + "isUnique": false + }, + "idx_developer_claims_pending_unique": { + "name": "idx_developer_claims_pending_unique", + "columns": ["developer_id", "claimant_id"], + "isUnique": true, + "where": "\"developer_claims\".\"status\" = 'pending'" + }, + "idx_developer_claims_pending_queue": { + "name": "idx_developer_claims_pending_queue", + "columns": ["created_at"], + "isUnique": false, + "where": "\"developer_claims\".\"status\" = 'pending'" + } + }, + "foreignKeys": { + "developer_claims_developer_id_developers_id_fk": { + "name": "developer_claims_developer_id_developers_id_fk", + "tableFrom": "developer_claims", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_claims_claimant_id_users_id_fk": { + "name": "developer_claims_claimant_id_users_id_fk", + "tableFrom": "developer_claims", + "tableTo": "users", + "columnsFrom": ["claimant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_claims_reviewer_id_users_id_fk": { + "name": "developer_claims_reviewer_id_users_id_fk", + "tableFrom": "developer_claims", + "tableTo": "users", + "columnsFrom": ["reviewer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "developer_claims_status_check": { + "name": "developer_claims_status_check", + "value": "\"developer_claims\".\"status\" IN ('pending', 'approved', 'rejected')" + }, + "developer_claims_github_org_verified_check": { + "name": "developer_claims_github_org_verified_check", + "value": "\"developer_claims\".\"github_org_verified\" IN (0, 1)" + } + } + }, + "developer_history": { + "name": "developer_history", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "changed_by": { + "name": "changed_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "changed_at": { + "name": "changed_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_developer_history_developer_changed_at": { + "name": "idx_developer_history_developer_changed_at", + "columns": ["developer_id", "changed_at"], + "isUnique": false + }, + "idx_developer_history_account_changed_at": { + "name": "idx_developer_history_account_changed_at", + "columns": ["changed_by", "changed_at"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_history_changed_by_users_id_fk": { + "name": "developer_history_changed_by_users_id_fk", + "tableFrom": "developer_history", + "tableTo": "users", + "columnsFrom": ["changed_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developer_transfers": { + "name": "developer_transfers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_by": { + "name": "accepted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "accepted_at": { + "name": "accepted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developer_transfers_token": { + "name": "idx_developer_transfers_token", + "columns": ["token_hash"], + "isUnique": true + }, + "idx_developer_transfers_pending": { + "name": "idx_developer_transfers_pending", + "columns": ["developer_id"], + "isUnique": true, + "where": "\"developer_transfers\".\"accepted_at\" IS NULL AND \"developer_transfers\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": { + "developer_transfers_developer_id_developers_id_fk": { + "name": "developer_transfers_developer_id_developers_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_transfers_created_by_users_id_fk": { + "name": "developer_transfers_created_by_users_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "users", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_transfers_accepted_by_users_id_fk": { + "name": "developer_transfers_accepted_by_users_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "users", + "columnsFrom": ["accepted_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developers": { + "name": "developers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "approved_at": { + "name": "approved_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01-01T00:00:00.000Z'" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01-01T00:00:00.000Z'" + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "contact_email": { + "name": "contact_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "content_revision": { + "name": "content_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "approved_revision": { + "name": "approved_revision", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "approved_by": { + "name": "approved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_org_verified": { + "name": "github_org_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verification_note": { + "name": "github_verification_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verified_at": { + "name": "github_verified_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_url_verified": { + "name": "github_url_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url_check_cooldown_until": { + "name": "url_check_cooldown_until", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developers_owner_unique": { + "name": "idx_developers_owner_unique", + "columns": ["owner_user_id"], + "isUnique": true + }, + "idx_developers_approved": { + "name": "idx_developers_approved", + "columns": ["approved_at"], + "isUnique": false + } + }, + "foreignKeys": { + "developers_owner_user_id_users_id_fk": { + "name": "developers_owner_user_id_users_id_fk", + "tableFrom": "developers", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "developers_ownership_epoch_check": { + "name": "developers_ownership_epoch_check", + "value": "\"developers\".\"ownership_epoch\" >= 1" + }, + "developers_content_revision_check": { + "name": "developers_content_revision_check", + "value": "\"developers\".\"content_revision\" >= 1" + }, + "developers_github_org_verified_check": { + "name": "developers_github_org_verified_check", + "value": "\"developers\".\"github_org_verified\" IN (0, 1)" + }, + "developers_github_url_verified_check": { + "name": "developers_github_url_verified_check", + "value": "\"developers\".\"github_url_verified\" = 1" + } + } + }, + "extension_resource_usage": { + "name": "extension_resource_usage", + "columns": { + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "extensions": { + "name": "extensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "revisions": { + "name": "revisions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_extension_resource_usage_subject": { + "name": "idx_extension_resource_usage_subject", + "columns": ["scope", "subject"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "extension_revisions": { + "name": "extension_revisions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "submitted_by": { + "name": "submitted_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_bytes": { + "name": "content_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "summary_name": { + "name": "summary_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_version": { + "name": "summary_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_description": { + "name": "summary_description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "compacted_at": { + "name": "compacted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reviewer_id": { + "name": "reviewer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "review_note": { + "name": "review_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "content_readable": { + "name": "content_readable", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_extension_revisions_retention": { + "name": "idx_extension_revisions_retention", + "columns": ["compacted_at", "reviewed_at", "id", "status"], + "isUnique": false + }, + "idx_extension_revisions_submitted_by": { + "name": "idx_extension_revisions_submitted_by", + "columns": ["submitted_by"], + "isUnique": false + }, + "idx_extension_revisions_developer": { + "name": "idx_extension_revisions_developer", + "columns": ["developer_id"], + "isUnique": false + }, + "idx_extension_revisions_pending": { + "name": "idx_extension_revisions_pending", + "columns": ["extension_id"], + "isUnique": true, + "where": "\"extension_revisions\".\"status\" = 'pending'" + }, + "idx_extension_revisions_extension_page": { + "name": "idx_extension_revisions_extension_page", + "columns": ["extension_id", "\"created_at\" desc", "\"id\" desc"], + "isUnique": false + }, + "idx_extension_revisions_submitter_page": { + "name": "idx_extension_revisions_submitter_page", + "columns": ["submitted_by", "\"created_at\" desc", "\"id\" desc"], + "isUnique": false + }, + "idx_extension_revisions_queue_page": { + "name": "idx_extension_revisions_queue_page", + "columns": ["status", "created_at", "id"], + "isUnique": false + }, + "idx_extension_revisions_reviewed": { + "name": "idx_extension_revisions_reviewed", + "columns": ["extension_id", "reviewed_at"], + "isUnique": false, + "where": "\"extension_revisions\".\"status\" IN ('approved', 'rejected')" + }, + "idx_extension_revisions_submitter_pending": { + "name": "idx_extension_revisions_submitter_pending", + "columns": ["submitted_by"], + "isUnique": false, + "where": "\"extension_revisions\".\"status\" = 'pending'" + } + }, + "foreignKeys": { + "extension_revisions_extension_id_extensions_id_fk": { + "name": "extension_revisions_extension_id_extensions_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "extensions", + "columnsFrom": ["extension_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "extension_revisions_submitted_by_users_id_fk": { + "name": "extension_revisions_submitted_by_users_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "users", + "columnsFrom": ["submitted_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "extension_revisions_reviewer_id_users_id_fk": { + "name": "extension_revisions_reviewer_id_users_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "users", + "columnsFrom": ["reviewer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "extension_revisions_status_check": { + "name": "extension_revisions_status_check", + "value": "\"extension_revisions\".\"status\" IN ('pending', 'approved', 'rejected')" + }, + "extension_revisions_ownership_epoch_check": { + "name": "extension_revisions_ownership_epoch_check", + "value": "\"extension_revisions\".\"ownership_epoch\" >= 1" + } + } + }, + "extension_write_events": { + "name": "extension_write_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_extension_write_events_account": { + "name": "idx_extension_write_events_account", + "columns": ["account_id", "occurred_at"], + "isUnique": false + }, + "idx_extension_write_events_developer": { + "name": "idx_extension_write_events_developer", + "columns": ["developer_id", "occurred_at"], + "isUnique": false + }, + "idx_extension_write_events_time": { + "name": "idx_extension_write_events_time", + "columns": ["occurred_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "extensions": { + "name": "extensions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_bytes": { + "name": "published_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "published_at": { + "name": "published_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_revision_id": { + "name": "published_revision_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "releases": { + "name": "releases", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "license": { + "name": "license", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "icon_url": { + "name": "icon_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "readme": { + "name": "readme", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "download_url": { + "name": "download_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "delisted_at": { + "name": "delisted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delist_reason": { + "name": "delist_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_extensions_published_revision": { + "name": "idx_extensions_published_revision", + "columns": ["published_revision_id"], + "isUnique": false + }, + "idx_extensions_id_nocase": { + "name": "idx_extensions_id_nocase", + "columns": ["lower(\"id\")"], + "isUnique": true + }, + "idx_extensions_developer_order": { + "name": "idx_extensions_developer_order", + "columns": ["developer_id", "lower(\"id\")", "id"], + "isUnique": false + }, + "idx_extensions_catalogue_order": { + "name": "idx_extensions_catalogue_order", + "columns": ["lower(\"id\")", "id"], + "isUnique": false, + "where": "\"extensions\".\"published_at\" IS NOT NULL AND \"extensions\".\"delisted_at\" IS NULL" + }, + "idx_extensions_type_catalogue_order": { + "name": "idx_extensions_type_catalogue_order", + "columns": ["type", "lower(\"id\")", "id"], + "isUnique": false, + "where": "\"extensions\".\"published_at\" IS NOT NULL AND \"extensions\".\"delisted_at\" IS NULL" + } + }, + "foreignKeys": { + "extensions_developer_id_developers_id_fk": { + "name": "extensions_developer_id_developers_id_fk", + "tableFrom": "extensions", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "extensions_published_content_check": { + "name": "extensions_published_content_check", + "value": "\"extensions\".\"published_at\" IS NULL OR (\n \"extensions\".\"type\" IS NOT NULL AND \"extensions\".\"name\" IS NOT NULL AND\n \"extensions\".\"description\" IS NOT NULL AND \"extensions\".\"releases\" IS NOT NULL AND\n \"extensions\".\"website\" IS NOT NULL AND \"extensions\".\"license\" IS NOT NULL AND\n \"extensions\".\"readme\" IS NOT NULL AND \"extensions\".\"source\" IS NOT NULL AND\n \"extensions\".\"version\" IS NOT NULL AND \"extensions\".\"download_url\" IS NOT NULL\n )" + } + } + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "picture": { + "name": "picture", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_moderator": { + "name": "is_moderator", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_login": { + "name": "github_login", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_orgs": { + "name": "github_orgs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_orgs_expires_at": { + "name": "github_orgs_expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": { + "idx_extension_revisions_extension_page": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_extension_revisions_submitter_page": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_extensions_id_nocase": { + "columns": { + "lower(\"id\")": { + "isExpression": true + } + } + }, + "idx_extensions_developer_order": { + "columns": { + "lower(\"id\")": { + "isExpression": true + } + } + }, + "idx_extensions_catalogue_order": { + "columns": { + "lower(\"id\")": { + "isExpression": true + } + } + }, + "idx_extensions_type_catalogue_order": { + "columns": { + "lower(\"id\")": { + "isExpression": true + } + } + } + } + } +} diff --git a/src/services/extensions/v2/db/migrations/meta/_journal.json b/src/services/extensions/v2/db/migrations/meta/_journal.json index 1f688a7..8f82e2d 100644 --- a/src/services/extensions/v2/db/migrations/meta/_journal.json +++ b/src/services/extensions/v2/db/migrations/meta/_journal.json @@ -85,6 +85,13 @@ "when": 1791143160901, "tag": "0025_luxuriant_franklin_richards", "breakpoints": true + }, + { + "idx": 26, + "version": "6", + "when": 1791315878730, + "tag": "0026_resource_bounds", + "breakpoints": true } ] -} \ No newline at end of file +} diff --git a/src/services/extensions/v2/db/resource-inventory.sql b/src/services/extensions/v2/db/resource-inventory.sql new file mode 100644 index 0000000..f7c5313 --- /dev/null +++ b/src/services/extensions/v2/db/resource-inventory.sql @@ -0,0 +1,30 @@ +-- Manual read-only reconciliation; scalar stored sizes avoid transferring bodies. +WITH component_usage AS ( +SELECT 'global' scope, 'all' subject, SUM(published_bytes) bytes, COUNT(*) extensions, 0 revisions FROM extensions GROUP BY 'all' +UNION ALL +SELECT 'account' scope, COALESCE(created_by,'legacy') subject, SUM(published_bytes) bytes, COUNT(*) extensions, 0 revisions FROM extensions GROUP BY COALESCE(created_by,'legacy') +UNION ALL +SELECT 'developer' scope, developer_id subject, SUM(published_bytes) bytes, COUNT(*) extensions, 0 revisions FROM extensions GROUP BY developer_id +UNION ALL +SELECT 'global' scope, 'all' subject, SUM(content_bytes) bytes, 0 extensions, COUNT(*) revisions FROM extension_revisions GROUP BY 'all' +UNION ALL +SELECT 'account' scope, submitted_by subject, SUM(content_bytes) bytes, 0 extensions, COUNT(*) revisions FROM extension_revisions GROUP BY submitted_by +UNION ALL +SELECT 'developer' scope, developer_id subject, SUM(content_bytes) bytes, 0 extensions, COUNT(*) revisions FROM extension_revisions GROUP BY developer_id +), expected AS ( + SELECT scope,subject,SUM(bytes) bytes,SUM(extensions) extensions,SUM(revisions) revisions + FROM component_usage GROUP BY scope,subject +), subjects AS ( + SELECT scope,subject FROM expected UNION SELECT scope,subject FROM extension_resource_usage +) +SELECT s.scope,s.subject,COALESCE(e.bytes,0) expected_bytes,COALESCE(u.bytes,0) stored_bytes, + COALESCE(e.extensions,0) expected_extensions,COALESCE(u.extensions,0) stored_extensions, + COALESCE(e.revisions,0) expected_revisions,COALESCE(u.revisions,0) stored_revisions +FROM subjects s LEFT JOIN expected e USING(scope,subject) +LEFT JOIN extension_resource_usage u USING(scope,subject) +WHERE COALESCE(e.bytes,0)!=COALESCE(u.bytes,0) OR COALESCE(e.extensions,0)!=COALESCE(u.extensions,0) + OR COALESCE(e.revisions,0)!=COALESCE(u.revisions,0) +ORDER BY abs(COALESCE(e.bytes,0)-COALESCE(u.bytes,0)) DESC, + abs(COALESCE(e.extensions,0)-COALESCE(u.extensions,0)) DESC, + abs(COALESCE(e.revisions,0)-COALESCE(u.revisions,0)) DESC, s.scope, s.subject +LIMIT 100; diff --git a/src/services/extensions/v2/db/resource-maintenance.ts b/src/services/extensions/v2/db/resource-maintenance.ts new file mode 100644 index 0000000..101effd --- /dev/null +++ b/src/services/extensions/v2/db/resource-maintenance.ts @@ -0,0 +1,136 @@ +import { sql } from "drizzle-orm"; +import { ExtensionsDb } from "../../../../lib/db"; +import { logInfo, logWarn } from "../../../../lib/logger"; +import { + MAX_CONTENT_BYTES, + RETENTION_DAYS, + MAINTENANCE_BATCH_SIZE +} from "../resource-limits"; + +// Pending, published and oversized legacy bodies are never compacted. +export async function maintainExtensionResources( + db: ExtensionsDb, + options: { mode?: string } = {} +): Promise { + const mode = options.mode === "compact" ? "compact" : "dry-run"; + const candidates = + mode === "compact" + ? await db.all<{ id: string }>(sql` + SELECT r.id FROM extension_revisions r + WHERE r.status IN ('approved','rejected') AND r.compacted_at IS NULL + AND r.reviewed_at < datetime('now', ${`-${RETENTION_DAYS} days`}) + AND r.content_bytes <= ${MAX_CONTENT_BYTES} + AND NOT EXISTS (SELECT 1 FROM extensions e WHERE e.published_revision_id=r.id) + ORDER BY r.reviewed_at, r.id LIMIT ${MAINTENANCE_BATCH_SIZE} + `) + : []; + let compacted = 0; + for (const { id } of candidates) { + const [row] = await db.all<{ content: string }>(sql` + SELECT content FROM extension_revisions WHERE id=${id} + AND content_bytes <= ${MAX_CONTENT_BYTES} AND compacted_at IS NULL + `); + if (!row) continue; + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(row.content) + ); + const hash = Array.from(new Uint8Array(digest), (b) => + b.toString(16).padStart(2, "0") + ).join(""); + const result = await db.run(sql` + UPDATE extension_revisions SET content='{}', content_hash=${hash}, compacted_at=CURRENT_TIMESTAMP + WHERE id=${id} AND content=${row.content} AND compacted_at IS NULL + AND status IN ('approved','rejected') + AND reviewed_at < datetime('now', ${`-${RETENTION_DAYS} days`}) + AND NOT EXISTS (SELECT 1 FROM extensions e WHERE e.published_revision_id=extension_revisions.id) + `); + compacted += result.meta?.changes ?? 0; + } + const expired = await db.run(sql` + DELETE FROM extension_write_events WHERE id IN ( + SELECT id FROM extension_write_events WHERE occurred_at <= unixepoch()-86400 + ORDER BY occurred_at LIMIT 500 + ) + `); + await db.run(sql` + DELETE FROM extension_resource_usage WHERE rowid IN ( + SELECT rowid FROM extension_resource_usage WHERE scope!='global' + AND bytes=0 AND extensions=0 AND revisions=0 LIMIT 500 + ) + `); + logInfo("extensions-v2", "Resource maintenance", { + retention_mode: mode, + compacted, + expired_events: expired.meta?.changes ?? 0 + }); +} + +// Read-only inventory, reported after hourly cleanup. +export async function reportExtensionResources( + db: ExtensionsDb, + retentionMode?: string +): Promise { + const mode = retentionMode === "compact" ? "compact" : "dry-run"; + const retention = await inventoryExtensionRetention(db); + const [usage] = await db.all<{ + bytes: number; + extensions: number; + revisions: number; + }>(sql` + SELECT bytes, extensions, revisions FROM extension_resource_usage WHERE scope='global' AND subject='all' + `); + const [backlog] = await db.all<{ + oversized: number; + pending: number; + }>(sql` + SELECT SUM(content_bytes > ${MAX_CONTENT_BYTES}) AS oversized, + SUM(status='pending') AS pending + FROM extension_revisions + `); + logInfo("extensions-v2", "Resource inventory", { + retention_mode: mode, + eligible_bodies: retention.eligible_bodies, + reclaimable_bytes: retention.reclaimable_bytes, + retained_bytes: usage?.bytes ?? 0, + extensions: usage?.extensions ?? 0, + revisions: usage?.revisions ?? 0, + pending: backlog?.pending ?? 0, + oversized_legacy_revisions: backlog?.oversized ?? 0, + cleanup_backlog: retention.eligible_bodies + }); + const thresholds = [ + ["legacy_content_present", backlog?.oversized ?? 0, 1], + [ + "cleanup_backlog_high", + mode === "compact" ? retention.eligible_bodies : 0, + 300 + ] + ] as const; + for (const [reason, value, threshold] of thresholds) { + if (value >= threshold) + logWarn("extensions-v2", "Resource threshold exceeded", { + reason, + value, + threshold + }); + } +} + +// Read-only preview: no full bodies are selected, hashed or mutated. +export async function inventoryExtensionRetention( + db: ExtensionsDb +): Promise<{ eligible_bodies: number; reclaimable_bytes: number }> { + const [row] = await db.all<{ + eligible_bodies: number; + reclaimable_bytes: number; + }>(sql` + SELECT COUNT(*) AS eligible_bodies, COALESCE(SUM(MAX(r.content_bytes - 2, 0)), 0) AS reclaimable_bytes + FROM extension_revisions r + WHERE r.status IN ('approved','rejected') AND r.compacted_at IS NULL + AND r.reviewed_at < datetime('now', ${`-${RETENTION_DAYS} days`}) + AND r.content_bytes <= ${MAX_CONTENT_BYTES} + AND NOT EXISTS (SELECT 1 FROM extensions e WHERE e.published_revision_id=r.id) + `); + return row ?? { eligible_bodies: 0, reclaimable_bytes: 0 }; +} diff --git a/src/services/extensions/v2/db/revisions.ts b/src/services/extensions/v2/db/revisions.ts index 33a3fd2..c03fc95 100644 --- a/src/services/extensions/v2/db/revisions.ts +++ b/src/services/extensions/v2/db/revisions.ts @@ -1,23 +1,34 @@ +import { MAX_CONTENT_BYTES } from "../resource-limits"; import { and, asc, desc, eq, gt, lt, or, sql, SQL } from "drizzle-orm"; import { DatabaseError, DatabaseResult } from "../../../../lib/interfaces"; import { ExtensionsDb } from "../../../../lib/db"; import { extensionRevisions, developers, extensions, users } from "./schema"; import { databaseError, + contentAdmissionError, inactiveActorError, moderatorActorError } from "./errors"; import { toD1Statement } from "./batch"; import { encodeCursor as encode, decodeCursor as decode } from "./cursor"; -import { MAX_PENDING_REVISIONS_PER_USER, parseContent } from "./extensions"; +import { + MAX_PENDING_REVISIONS_PER_USER, + parseContent, + LegacyContentError, + oversizedContent +} from "./extensions"; import { ExtensionContent, ExtensionContentSchema } from "../schemas/extensions"; -import { ExtensionRevision, RevisionStatus } from "../schemas/revisions"; +import { + ExtensionRevision, + ExtensionRevisionSummary, + RevisionStatus +} from "../schemas/revisions"; export interface RevisionPage { - items: ExtensionRevision[]; + items: ExtensionRevisionSummary[]; nextCursor: string | null; hasMore: boolean; } @@ -51,7 +62,14 @@ interface RevisionRow { developerId: string; submittedBy: string; status: string; - content: string; + content: string | null; + contentBytes: number; + contentReadable: number; + name: string | null; + version: string | null; + description: string | null; + contentHash: string | null; + compactedAt: string | null; reviewerId: string | null; reviewNote: string | null; createdAt: string; @@ -61,16 +79,8 @@ interface RevisionRow { function parseRevisionRow(row: RevisionRow): StoredRevision { const revision = { - id: row.id, - extension_id: row.extensionId, - developer_id: row.developerId, - submitted_by: row.submittedBy, - status: row.status as RevisionStatus, - content: parseContent(row.content), - reviewer_id: row.reviewerId, - review_note: row.reviewNote, - created_at: row.createdAt, - reviewed_at: row.reviewedAt + ...parseSummaryRow(row), + content: row.compactedAt ? null : parseContent(row.content) } as StoredRevision; Object.defineProperty(revision, "ownershipEpoch", { value: Number(row.ownershipEpoch ?? 1), @@ -85,14 +95,51 @@ const REVISION_COLUMNS = { developerId: extensionRevisions.developerId, submittedBy: extensionRevisions.submittedBy, status: extensionRevisions.status, - content: extensionRevisions.content, + content: sql< + string | null + >`CASE WHEN length(CAST(${extensionRevisions.content} AS BLOB)) <= ${MAX_CONTENT_BYTES} THEN ${extensionRevisions.content} ELSE NULL END`, + name: extensionRevisions.summaryName, + version: extensionRevisions.summaryVersion, + description: extensionRevisions.summaryDescription, + contentBytes: extensionRevisions.contentBytes, + contentReadable: extensionRevisions.contentReadable, + contentHash: extensionRevisions.contentHash, + compactedAt: extensionRevisions.compactedAt, reviewerId: extensionRevisions.reviewerId, - reviewNote: extensionRevisions.reviewNote, + reviewNote: sql< + string | null + >`substr(${extensionRevisions.reviewNote}, 1, 2000)`, createdAt: extensionRevisions.createdAt, reviewedAt: extensionRevisions.reviewedAt, ownershipEpoch: extensionRevisions.ownershipEpoch }; +const { content: _content, ...SUMMARY_COLUMNS } = REVISION_COLUMNS; +type SummaryRow = Omit; +function parseSummaryRow(row: SummaryRow): ExtensionRevisionSummary { + return { + id: row.id, + extension_id: row.extensionId, + developer_id: row.developerId, + submitted_by: row.submittedBy, + status: row.status as RevisionStatus, + reviewer_id: row.reviewerId, + review_note: row.reviewNote, + created_at: row.createdAt, + reviewed_at: row.reviewedAt, + name: row.name, + version: row.version, + description: row.description, + content_bytes: row.contentBytes, + content_available: + !row.compactedAt && + row.contentBytes <= MAX_CONTENT_BYTES && + Boolean(row.contentReadable), + content_hash: row.contentHash, + compacted_at: row.compactedAt + }; +} + export class ExtensionRevisionsDatabase { constructor(private db: ExtensionsDb) {} @@ -128,14 +175,14 @@ export class ExtensionRevisionsDatabase { ON CONFLICT DO NOTHING `); } catch (error) { - return databaseError("propose", error); + return contentAdmissionError("propose", error); } if (!result.meta?.changes) { try { return { data: null, error: await this.proposeBlockedError(input) }; } catch (error) { - return databaseError("propose", error); + return contentAdmissionError("propose", error); } } @@ -202,7 +249,9 @@ export class ExtensionRevisionsDatabase { baseCondition: SQL, direction: "asc" | "desc", limit: number, - cursor?: string + cursor?: string, + readerId?: string, + extensionId?: string ): Promise> { const decoded = cursor ? decodeCursor(cursor) : null; if (cursor && !decoded) { @@ -215,9 +264,17 @@ export class ExtensionRevisionsDatabase { const [beyond, order] = direction === "desc" ? [lt, desc] : ([gt, asc] as const); - let rows: RevisionRow[]; + let rows: SummaryRow[]; try { const conditions = [baseCondition]; + if (readerId) + conditions.push(sql`EXISTS ( + SELECT 1 FROM users u WHERE u.id=${readerId} AND u.deleted_at IS NULL + AND (u.is_moderator=1 OR (${extensionId ?? null} IS NOT NULL AND EXISTS ( + SELECT 1 FROM extensions e JOIN developers d ON d.id=e.developer_id + WHERE e.id=${extensionId ?? null} AND d.owner_user_id=${readerId} + ))) + )`); if (decoded) { const { createdAt, id: cursorId } = decoded; conditions.push( @@ -231,7 +288,7 @@ export class ExtensionRevisionsDatabase { ); } rows = await this.db - .select(REVISION_COLUMNS) + .select(SUMMARY_COLUMNS) .from(extensionRevisions) .where(and(...conditions)) .orderBy( @@ -244,7 +301,7 @@ export class ExtensionRevisionsDatabase { } const hasMore = rows.length > limit; - const items = rows.slice(0, limit).map(parseRevisionRow); + const items = rows.slice(0, limit).map(parseSummaryRow); const last = items.at(-1); return { data: { @@ -264,6 +321,7 @@ export class ExtensionRevisionsDatabase { // narrows both modes. async listScoped(filters: { extensionId?: string; + readerId?: string; status?: RevisionStatus; sort?: "newest" | "oldest"; limit?: number; @@ -289,7 +347,15 @@ export class ExtensionRevisionsDatabase { conditions.length > 1 ? and(...conditions)! : (conditions[0] ?? sql`1 = 1`); - return this.page("listScoped", base, direction, limit, filters.cursor); + return this.page( + "listScoped", + base, + direction, + limit, + filters.cursor, + filters.readerId, + filters.extensionId + ); } // Queue totals for the admin tabs: one GROUP BY rather than a COUNT per @@ -327,7 +393,8 @@ export class ExtensionRevisionsDatabase { async getById( extensionId: string, - id: string + id: string, + readerId?: string ): Promise> { let row: RevisionRow | undefined; try { @@ -337,7 +404,16 @@ export class ExtensionRevisionsDatabase { .where( and( eq(extensionRevisions.id, id), - sql`LOWER(${extensionRevisions.extensionId}) = LOWER(${extensionId})` + sql`LOWER(${extensionRevisions.extensionId}) = LOWER(${extensionId})`, + readerId + ? sql`EXISTS ( + SELECT 1 FROM users u WHERE u.id=${readerId} AND u.deleted_at IS NULL + AND (u.is_moderator=1 OR EXISTS ( + SELECT 1 FROM extensions e JOIN developers d ON d.id=e.developer_id + WHERE e.id=extension_revisions.extension_id AND d.owner_user_id=${readerId} + )) + )` + : undefined ) ); } catch (error) { @@ -345,7 +421,22 @@ export class ExtensionRevisionsDatabase { } if (!row) return revisionNotFound(id); - return { data: parseRevisionRow(row), error: null }; + if (!row.compactedAt && !row.contentReadable) return oversizedContent(); + if (row.contentBytes > MAX_CONTENT_BYTES) + return { + data: null, + error: { + code: "CONTENT_UNAVAILABLE", + message: + "Legacy revision exceeds the content limit; resubmit or export it administratively" + } + }; + try { + return { data: parseRevisionRow(row), error: null }; + } catch (error) { + if (error instanceof LegacyContentError) return oversizedContent(); + return databaseError("getById", error); + } } // Notes what happened to an id-scoped write that didn't affect any rows: diff --git a/src/services/extensions/v2/db/schema.ts b/src/services/extensions/v2/db/schema.ts index 7077594..ec294f8 100644 --- a/src/services/extensions/v2/db/schema.ts +++ b/src/services/extensions/v2/db/schema.ts @@ -45,6 +45,8 @@ export const extensions = sqliteTable( developerId: text("developer_id") .notNull() .references(() => developers.id), + createdBy: text("created_by"), + publishedBytes: integer("published_bytes").notNull().default(0), publishedAt: text("published_at"), // Which revision produced the current published content. Deliberately not // a FK: extension_revisions.extension_id already points the other way, and @@ -80,6 +82,7 @@ export const extensions = sqliteTable( delistReason: text("delist_reason") }, (table) => [ + index("idx_extensions_published_revision").on(table.publishedRevisionId), // Case-insensitive id uniqueness. The id is a lowercase slug by schema, // but adopted rows predate that, and this is what stops two developers // racing for ids that differ only in case β€” the job migration 0011's @@ -229,6 +232,13 @@ export const extensionRevisions = sqliteTable( .references(() => users.id), status: text("status").notNull().default("pending"), content: text("content").notNull(), + contentBytes: integer("content_bytes").notNull().default(0), + summaryName: text("summary_name"), + summaryVersion: text("summary_version"), + summaryDescription: text("summary_description"), + contentReadable: integer("content_readable").notNull().default(0), + contentHash: text("content_hash"), + compactedAt: text("compacted_at"), reviewerId: text("reviewer_id").references(() => users.id), reviewNote: text("review_note"), createdAt: text("created_at") @@ -238,6 +248,12 @@ export const extensionRevisions = sqliteTable( ownershipEpoch: integer("ownership_epoch").notNull().default(1) }, (table) => [ + index("idx_extension_revisions_retention").on( + table.compactedAt, + table.reviewedAt, + table.id, + table.status + ), index("idx_extension_revisions_submitted_by").on(table.submittedBy), index("idx_extension_revisions_developer").on(table.developerId), // At most one unreviewed revision per extension. This replaces migration @@ -411,3 +427,44 @@ export const claimVerificationBudgets = sqliteTable( index("idx_claim_verification_budgets_expiry").on(table.expiresAt) ] ); + +// Derived usage is maintained by 0026's transactional triggers. Account +// charges stay with the original creator/submitter across ownership changes. +export const extensionResourceUsage = sqliteTable( + "extension_resource_usage", + { + scope: text("scope").notNull(), + subject: text("subject").notNull(), + bytes: integer("bytes").notNull().default(0), + extensions: integer("extensions").notNull().default(0), + revisions: integer("revisions").notNull().default(0) + }, + (table) => [ + uniqueIndex("idx_extension_resource_usage_subject").on( + table.scope, + table.subject + ) + ] +); + +// No foreign keys: deleting content or a profile cannot reset write pacing. +export const extensionWriteEvents = sqliteTable( + "extension_write_events", + { + id: text("id").primaryKey(), + accountId: text("account_id").notNull(), + developerId: text("developer_id").notNull(), + occurredAt: integer("occurred_at").notNull() + }, + (table) => [ + index("idx_extension_write_events_account").on( + table.accountId, + table.occurredAt + ), + index("idx_extension_write_events_developer").on( + table.developerId, + table.occurredAt + ), + index("idx_extension_write_events_time").on(table.occurredAt) + ] +); diff --git a/src/services/extensions/v2/index.ts b/src/services/extensions/v2/index.ts index b351a8b..0783782 100644 --- a/src/services/extensions/v2/index.ts +++ b/src/services/extensions/v2/index.ts @@ -1,3 +1,8 @@ +import { + boundContentRequest, + paceContentIp, + observeResourceResponses +} from "./middleware"; import { OpenAPIHono } from "@hono/zod-openapi"; import { Scalar } from "@scalar/hono-api-reference"; import { cors } from "hono/cors"; @@ -33,6 +38,28 @@ const extensionsV2 = new OpenAPIHono<{ Bindings: CloudflareBindings }>({ // it to schedule their retry. extensionsV2.use("/*", cors({ origin: "*", exposeHeaders: ["Retry-After"] })); extensionsV2.use("/*", trimTrailingSlash()); +extensionsV2.on( + "GET", + [ + "/revisions", + "/extensions/:id/revisions", + "/extensions/:id/revisions/:revisionId" + ], + observeResourceResponses() +); +// Run before authentication and JSON validation, including invalid bodies. +extensionsV2.on( + "POST", + ["/extensions", "/extensions/:id/moderator-correct"], + paceContentIp(), + boundContentRequest() +); +extensionsV2.on( + "PUT", + "/extensions/:id", + paceContentIp(), + boundContentRequest() +); extensionsV2.openAPIRegistry.registerComponent("securitySchemes", "Bearer", { type: "http", scheme: "bearer" diff --git a/src/services/extensions/v2/middleware.ts b/src/services/extensions/v2/middleware.ts index 6ccf2ad..4cc59c7 100644 --- a/src/services/extensions/v2/middleware.ts +++ b/src/services/extensions/v2/middleware.ts @@ -1,3 +1,5 @@ +import { logInfo, logError } from "../../../lib/logger"; +import { MAX_RAW_BODY_BYTES } from "./resource-limits"; import { type Context, type MiddlewareHandler } from "hono"; import { bearerAssertionVerifier, @@ -133,3 +135,133 @@ export function requireModerator(): MiddlewareHandler { ); }); } + +// Registered before JSON validators on the three content-write routes. +export function boundContentRequest(): MiddlewareHandler { + return async (c, next) => { + const reader = c.req.raw.body?.getReader(); + if (reader) { + let body = new Uint8Array(0); + let size = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > MAX_RAW_BODY_BYTES) { + await reader.cancel().catch(() => {}); + logInfo("extensions-v2", "Resource admission rejected", { + reason: "raw_body_size" + }); + return c.json( + { + error: { + message: "Request body must not exceed 512 KiB", + code: "BODY_TOO_LARGE" + } + }, + 413 + ); + } + if (size > body.byteLength) { + const grown = new Uint8Array( + Math.min( + MAX_RAW_BODY_BYTES, + Math.max(size, body.byteLength * 2, 1024) + ) + ); + grown.set(body); + body = grown; + } + body.set(value, size - value.byteLength); + } + } catch { + return c.json( + { + error: { + message: "Unable to read request body", + code: "BAD_REQUEST" + } + }, + 400 + ); + } finally { + reader.releaseLock(); + } + c.req.raw = new Request(c.req.raw, { body: body.subarray(0, size) }); + } + await next(); + }; +} + +function paceContentAttempts(identity: "ip" | "account"): MiddlewareHandler { + return async (c, next) => { + const subject = + identity === "ip" + ? (c.req.header("CF-Connecting-IP") ?? "unknown") + : getAuth(c).userId; + try { + const { success } = await c.env.EXTENSION_WRITE_RATE_LIMITER.limit({ + key: `${identity}:${subject}` + }); + if (!success) { + c.header("Retry-After", "60"); + logInfo("extensions-v2", "Resource admission rejected", { + reason: identity === "ip" ? "attempt_rate" : "account_attempt_rate" + }); + return c.json( + { + error: { + message: "Too many extension write attempts", + code: "RATE_LIMITED" + } + }, + 429 + ); + } + } catch { + logError("extensions-v2", "Attempt limiter unavailable", { identity }); + c.header("Retry-After", "60"); + return c.json( + { + error: { + message: "Write admission unavailable", + code: "ADMISSION_UNAVAILABLE" + } + }, + 503 + ); + } + await next(); + }; +} + +export const paceContentIp = () => paceContentAttempts("ip"); +export const paceContentAccount = () => paceContentAttempts("account"); + +// Count streamed response bytes without buffering or copying response bodies. +export function observeResourceResponses(): MiddlewareHandler { + return async (c, next) => { + const started = performance.now(); + await next(); + if (!c.res.body) return; + const status = c.res.status; + let bytes = 0; + const body = c.res.body.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + bytes += chunk.byteLength; + controller.enqueue(chunk); + }, + flush() { + logInfo("extensions-v2", "Revision response", { + status, + response_bytes: bytes, + duration_ms: Math.round(performance.now() - started) + }); + } + }) + ); + c.res = new Response(body, c.res); + }; +} diff --git a/src/services/extensions/v2/resource-limits.ts b/src/services/extensions/v2/resource-limits.ts new file mode 100644 index 0000000..43cf26d --- /dev/null +++ b/src/services/extensions/v2/resource-limits.ts @@ -0,0 +1,8 @@ +// The content and retained-byte quotas also live in migration 0026; +// change those durable ceilings with a new migration. Request and +// maintenance bounds below are enforced in the Worker. +export const MAX_RAW_BODY_BYTES = 512 * 1024; +export const MAX_CONTENT_BYTES = 256 * 1024; +export const RETENTION_DAYS = 180; +export const MAINTENANCE_BATCH_SIZE = 20; +export const MAX_ACCOUNT_BYTES = 25 * 1024 * 1024; diff --git a/src/services/extensions/v2/routes/errors.ts b/src/services/extensions/v2/routes/errors.ts index ff4b42c..3d6f98b 100644 --- a/src/services/extensions/v2/routes/errors.ts +++ b/src/services/extensions/v2/routes/errors.ts @@ -1,3 +1,4 @@ +import type { Context } from "hono"; import { DatabaseError } from "../../../../lib/interfaces"; // Routes that do not declare a 409 response pass false so unexpected conflict @@ -15,7 +16,13 @@ export function statusFromErrorCode( includeConflict = true ): 404 | 409 | 500 { if (code === "NOT_FOUND") return 404; - if (includeConflict && code === "CONFLICT") return 409; + if ( + includeConflict && + (code === "CONFLICT" || + code === "RESOURCE_QUOTA" || + code === "CONTENT_UNAVAILABLE") + ) + return 409; return 500; } @@ -67,3 +74,29 @@ export function errorBody( } }; } + +export function setContentRetryAfter(c: Context, code?: string): void { + if (code === "ADMISSION_UNAVAILABLE" || code === "WRITE_RATE_MINUTE") + c.header("Retry-After", "60"); + else if (code === "WRITE_RATE_DAY") c.header("Retry-After", "86400"); +} + +export function statusFromContentWriteError( + code?: string +): 403 | 404 | 409 | 429 | 500 | 503 { + if (code === "ADMISSION_UNAVAILABLE") return 503; + if (code?.startsWith("WRITE_RATE_")) return 429; + if (code === "RESOURCE_QUOTA" || code === "CONTENT_UNAVAILABLE") return 409; + return statusFromWriteErrorCode(code); +} +export function statusFromContentReadError(code?: string): 404 | 409 | 500 { + if (code === "CONTENT_UNAVAILABLE") return 409; + return statusFromErrorCode(code, false); +} + +export function statusFromContentCreateError( + code?: string +): 403 | 409 | 429 | 500 | 503 { + const status = statusFromContentWriteError(code); + return status === 404 ? 500 : status; +} diff --git a/src/services/extensions/v2/routes/moderation.ts b/src/services/extensions/v2/routes/moderation.ts index 878cfe8..f1571f0 100644 --- a/src/services/extensions/v2/routes/moderation.ts +++ b/src/services/extensions/v2/routes/moderation.ts @@ -1,9 +1,14 @@ -import { requireModerator } from "../middleware"; +import { paceContentAccount, requireModerator } from "../middleware"; import { getExtensionsDb } from "../../../../lib/db"; import { getPlatform } from "../../../../lib/middleware"; import { getAuth } from "../../../../lib/auth"; import { createRoute, z } from "@hono/zod-openapi"; -import { errorBody, statusFromWriteErrorCode } from "./errors"; +import { + errorBody, + setContentRetryAfter, + statusFromContentWriteError, + statusFromWriteErrorCode +} from "./errors"; import { ActiveAccountRequiredResponse, CursorPaginationQuerySchema, @@ -360,7 +365,7 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { tags: ["Moderation"], summary: "Correct a published extension's live content as a moderator", security: [{ Bearer: [] }], - middleware: [requireModerator()] as const, + middleware: [requireModerator(), paceContentAccount()] as const, request: { params: IdParamSchema, body: { @@ -400,6 +405,12 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { 409: errorResponse( "Extension is unpublished or delisted, or an edit is already awaiting review" ), + 400: errorResponse("Unable to read request body"), + 413: errorResponse("Raw request exceeds 512 KiB"), + 429: errorResponse( + "Extension write allowance exhausted; see Retry-After" + ), + 503: errorResponse("Write admission unavailable"), 422: errorResponse( "Path params, content, or correction_note failed validation" ), @@ -420,7 +431,8 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { correction_note ); if (error || !data) { - const status = statusFromWriteErrorCode(error?.code); + setContentRetryAfter(c, error?.code); + const status = statusFromContentWriteError(error?.code); return c.json(errorBody(error, "Unable to correct extension"), status); } revalidateCatalogue(c); diff --git a/src/services/extensions/v2/routes/owner-extensions.ts b/src/services/extensions/v2/routes/owner-extensions.ts index f0d288c..3ce1026 100644 --- a/src/services/extensions/v2/routes/owner-extensions.ts +++ b/src/services/extensions/v2/routes/owner-extensions.ts @@ -1,9 +1,13 @@ +import type { Context } from "hono"; +import type { DatabaseResult } from "../../../../lib/interfaces"; import { errorBody, - statusFromErrorCode, + setContentRetryAfter, + statusFromContentCreateError, + statusFromContentWriteError, statusFromWriteErrorCode } from "./errors"; -import { requireActiveAuth } from "../middleware"; +import { paceContentAccount, requireActiveAuth } from "../middleware"; import { getExtensionsDb } from "../../../../lib/db"; import { getAuth } from "../../../../lib/auth"; import { createRoute, z } from "@hono/zod-openapi"; @@ -19,6 +23,8 @@ import { } from "../schemas/extensions"; import { ExtensionRevisionSchema, + ExtensionRevisionSummarySchema, + RevisionIdParamSchema, RevisionHistoryQuerySchema } from "../schemas/revisions"; import { DeveloperProfilesDatabase } from "../db/developer-profiles"; @@ -28,6 +34,33 @@ import { UsersDatabase } from "../db/users"; import { revalidateCatalogue } from "../revalidate"; import { ExtensionsV2App } from "./app"; +// Both history and detail authorize against current account/profile state; +// their actual data queries repeat this guard to contain mid-request changes. +async function revisionReadAccess( + c: Context<{ Bindings: CloudflareBindings }>, + id: string +): Promise> { + const db = getExtensionsDb(c.env.DB_EXTENSIONS); + const auth = getAuth(c); + const [ownership, access] = await Promise.all([ + new ExtensionsDatabase(db).getOwnership(id), + new UsersDatabase(db).moderatorAccess(auth.userId) + ]); + if (ownership.error || !ownership.data) return ownership; + if (access.error) return { data: null, error: access.error }; + if (!access.data?.active) + return { + data: null, + error: { code: "ACCOUNT_INACTIVE", message: "Active account required" } + }; + if (ownership.data.ownerUserId !== auth.userId && !access.data.moderator) + return { + data: null, + error: { code: "FORBIDDEN", message: "You do not own this extension" } + }; + return { data: { extensionId: ownership.data.extensionId }, error: null }; +} + const AcceptedRevisionSchema = z.object({ result: z.object({ id: z.string(), @@ -43,7 +76,7 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { tags: ["Extensions"], summary: "Create an extension and submit its first version for review", security: [{ Bearer: [] }], - middleware: [requireActiveAuth()] as const, + middleware: [requireActiveAuth(), paceContentAccount()] as const, request: { body: { content: { "application/json": { schema: ExtensionCreateSchema } } @@ -64,6 +97,11 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { 409: errorResponse( "The id is taken, ownership changed, or the pending-revision limit was reached" ), + 413: errorResponse("Raw request exceeds 512 KiB"), + 429: errorResponse( + "Extension write allowance exhausted; see Retry-After" + ), + 503: errorResponse("Write admission unavailable"), 422: errorResponse("Body failed validation"), 500: errorResponse("Database error") } @@ -101,9 +139,10 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { content }); if (error || !data) { + setContentRetryAfter(c, error?.code); return c.json( errorBody(error, "Unable to create extension"), - statusFromWriteErrorCode(error?.code, false) + statusFromContentCreateError(error?.code) ); } return c.json( @@ -124,7 +163,7 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { tags: ["Extensions"], summary: "Submit an edit to an extension the caller owns", security: [{ Bearer: [] }], - middleware: [requireActiveAuth()] as const, + middleware: [requireActiveAuth(), paceContentAccount()] as const, request: { params: IdParamSchema, body: { @@ -147,6 +186,11 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { 409: errorResponse( "An edit is already awaiting review, or the pending-revision limit was reached" ), + 413: errorResponse("Raw request exceeds 512 KiB"), + 429: errorResponse( + "Extension write allowance exhausted; see Retry-After" + ), + 503: errorResponse("Write admission unavailable"), 422: errorResponse("Body failed validation"), 500: errorResponse("Database error") } @@ -165,9 +209,10 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { content }); if (error || !data) { + setContentRetryAfter(c, error?.code); return c.json( errorBody(error, "Unable to submit edit"), - statusFromWriteErrorCode(error?.code) + statusFromContentWriteError(error?.code) ); } return c.json( @@ -235,7 +280,7 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { content: { "application/json": { schema: z.object({ - result: z.array(ExtensionRevisionSchema), + result: z.array(ExtensionRevisionSummarySchema), pagination: PaginationSchema }) } @@ -259,75 +304,24 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { const auth = getAuth(c); const { id } = c.req.valid("param"); const { limit, cursor } = c.req.valid("query"); - const extensionsDb = new ExtensionsDatabase( - getExtensionsDb(c.env.DB_EXTENSIONS) - ); - // Light probe instead of the full owner view: authorising the caller - // needs only the owner id and the canonical extension id, and the full - // view would ship up to 256 KiB of readme/pendingContent per read. - const ownership = await extensionsDb.getOwnership(id); - if (ownership.error || !ownership.data) { + const access = await revisionReadAccess(c, id); + if (access.error || !access.data) return c.json( - errorBody(ownership.error, "Extension not found"), - statusFromErrorCode(ownership.error?.code, false) + errorBody(access.error, "Unable to read revisions"), + access.error?.code === "NOT_FOUND" + ? 404 + : access.error?.code === "FORBIDDEN" || + access.error?.code === "ACCOUNT_INACTIVE" + ? 403 + : 500 ); - } - - if (ownership.data.ownerUserId !== auth.userId) { - const users = new UsersDatabase(getExtensionsDb(c.env.DB_EXTENSIONS)); - const moderator = await users.moderatorAccess(auth.userId); - if (moderator.error) { - return c.json( - errorBody(moderator.error, "Unable to check moderator access"), - 500 - ); - } - if (!moderator.data?.active) { - return c.json( - { - error: { - message: "Active account required", - code: "ACCOUNT_INACTIVE" - } - }, - 403 - ); - } - if (!moderator.data.moderator) { - return c.json( - { - error: { - message: "You do not own this extension", - code: "FORBIDDEN" - } - }, - 403 - ); - } - } else { - const users = new UsersDatabase(getExtensionsDb(c.env.DB_EXTENSIONS)); - const active = await users.isActive(auth.userId); - if (active.error) { - return c.json(errorBody(active.error, "Unable to check account"), 500); - } - if (!active.data) { - return c.json( - { - error: { - message: "Active account required", - code: "ACCOUNT_INACTIVE" - } - }, - 403 - ); - } - } const db = new ExtensionRevisionsDatabase( getExtensionsDb(c.env.DB_EXTENSIONS) ); const { data, error } = await db.listScoped({ - extensionId: ownership.data.extensionId, + extensionId: access.data.extensionId, + readerId: auth.userId, sort: "newest", limit, cursor @@ -348,4 +342,81 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { res.headers.set("Vary", "Authorization"); return res; }); + const revisionDetailRoute = createRoute({ + method: "get", + path: "/extensions/{id}/revisions/{revisionId}", + tags: ["Extensions"], + summary: "Read one revision; compacted content is null", + security: [{ Bearer: [] }], + middleware: [requireActiveAuth()] as const, + request: { params: RevisionIdParamSchema }, + responses: { + 200: { + content: { + "application/json": { + schema: z.object({ + result: ExtensionRevisionSchema + }) + } + }, + description: + "One revision with its review outcome; content is null when compacted" + }, + 401: errorResponse("Missing or invalid bearer token"), + 403: { + ...ActiveAccountRequiredResponse, + description: + "The account is inactive, or the caller neither owns this extension nor moderates" + }, + 404: errorResponse("No extension or revision with that id"), + 409: errorResponse( + "Oversized legacy content requires administrative export or resubmission" + ), + 422: errorResponse("Path failed validation"), + 500: errorResponse("Database error") + } + }); + + app.openapi(revisionDetailRoute, async (c) => { + const auth = getAuth(c); + const { id, revisionId } = c.req.valid("param"); + const access = await revisionReadAccess(c, id); + if (access.error || !access.data) + return c.json( + errorBody(access.error, "Unable to read revisions"), + access.error?.code === "NOT_FOUND" + ? 404 + : access.error?.code === "FORBIDDEN" || + access.error?.code === "ACCOUNT_INACTIVE" + ? 403 + : 500 + ); + + const db = new ExtensionRevisionsDatabase( + getExtensionsDb(c.env.DB_EXTENSIONS) + ); + const { data, error } = await db.getById( + access.data.extensionId, + revisionId, + auth.userId + ); + if (error || !data) { + return c.json( + errorBody(error, "Unable to load revisions"), + error?.code === "NOT_FOUND" + ? 404 + : error?.code === "CONTENT_UNAVAILABLE" + ? 409 + : 500 + ); + } + const res = c.json( + { + result: data + }, + 200 + ); + res.headers.set("Vary", "Authorization"); + return res; + }); } diff --git a/src/services/extensions/v2/routes/public-extensions.ts b/src/services/extensions/v2/routes/public-extensions.ts index e04e8b6..2d8a856 100644 --- a/src/services/extensions/v2/routes/public-extensions.ts +++ b/src/services/extensions/v2/routes/public-extensions.ts @@ -1,6 +1,6 @@ import { getExtensionsDb } from "../../../../lib/db"; import { createRoute } from "@hono/zod-openapi"; -import { errorBody, statusFromErrorCode } from "./errors"; +import { errorBody, statusFromContentReadError } from "./errors"; import { IdParamSchema, errorResponse } from "../schemas/common"; import { UnifiedExtensionListQuerySchema, @@ -254,6 +254,7 @@ export function registerPublicExtensionsRoutes(app: ExtensionsV2App): void { }, 401: errorResponse("Invalid bearer token"), 404: errorResponse("No extension with that id"), + 409: errorResponse("Oversized legacy content requires resubmission"), 422: errorResponse("id param failed validation"), 500: errorResponse("Database error") } @@ -299,7 +300,7 @@ export function registerPublicExtensionsRoutes(app: ExtensionsV2App): void { owned.data.ownerUserId !== auth.userId ) { if (owned.error && owned.error.code !== "NOT_FOUND") { - const status = statusFromErrorCode(owned.error.code, false); + const status = statusFromContentReadError(owned.error.code); return c.json( errorBody(owned.error, "Extension not found"), status @@ -321,7 +322,7 @@ export function registerPublicExtensionsRoutes(app: ExtensionsV2App): void { if (access.data?.moderator) { const owned = await db.getOwned(id); if (owned.error || !owned.data) { - const status = statusFromErrorCode(owned.error?.code, false); + const status = statusFromContentReadError(owned.error?.code); return c.json( errorBody(owned.error, "Extension not found"), status @@ -339,7 +340,7 @@ export function registerPublicExtensionsRoutes(app: ExtensionsV2App): void { const { data, error } = await db.getById(id); if (error || !data) { - const status = statusFromErrorCode(error?.code, false); + const status = statusFromContentReadError(error?.code); return c.json(errorBody(error, "Extension not found"), status); } const res = c.json({ result: data }, 200); diff --git a/src/services/extensions/v2/routes/revisions.ts b/src/services/extensions/v2/routes/revisions.ts index d22e6dc..26b5d13 100644 --- a/src/services/extensions/v2/routes/revisions.ts +++ b/src/services/extensions/v2/routes/revisions.ts @@ -1,3 +1,4 @@ +import { getAuth } from "../../../../lib/auth"; import { getExtensionsDb } from "../../../../lib/db"; import { createRoute, z } from "@hono/zod-openapi"; import { errorBody } from "./errors"; @@ -7,7 +8,7 @@ import { errorResponse } from "../schemas/common"; import { - ExtensionRevisionSchema, + ExtensionRevisionSummarySchema, RevisionQueueQuerySchema } from "../schemas/revisions"; import { ExtensionRevisionsDatabase } from "../db/revisions"; @@ -28,7 +29,7 @@ export function registerRevisionRoutes(app: ExtensionsV2App): void { content: { "application/json": { schema: z.object({ - result: z.array(ExtensionRevisionSchema), + result: z.array(ExtensionRevisionSummarySchema), pagination: PaginationSchema }) } @@ -55,6 +56,7 @@ export function registerRevisionRoutes(app: ExtensionsV2App): void { ); const { data, error } = await db.listScoped({ status: status ?? "pending", + readerId: getAuth(c).userId, sort: "oldest", limit, cursor diff --git a/src/services/extensions/v2/schemas/extensions.ts b/src/services/extensions/v2/schemas/extensions.ts index 7d2615a..436abfc 100644 --- a/src/services/extensions/v2/schemas/extensions.ts +++ b/src/services/extensions/v2/schemas/extensions.ts @@ -1,3 +1,4 @@ +import { MAX_CONTENT_BYTES } from "../resource-limits"; import { z } from "@hono/zod-openapi"; import SPDX_LICENSE_IDS from "spdx-license-ids/index.json"; import { httpUrl, lowercaseId, PaginationSchema } from "./common"; @@ -101,8 +102,19 @@ export type ExtensionContent = z.infer; // bounds are kept, since those still say something true about the shape. // Nothing is weakened for publication - approve() revalidates against the // strict schema before anything reaches the catalogue. +// SQLite length() counts Unicode code points. Use the same bound for legacy +// reads, with a 200-code-unit ceiling so supplementary characters still fit. +const StoredReleaseSchema = ReleaseSchema.extend({ + tag: z + .string() + .min(1) + .max(200) + .refine((tag) => Array.from(tag).length <= 100) + .describe("At most 100 Unicode code points in historical content") +}); + export const StoredExtensionContentSchema = ExtensionContentSchema.extend({ - releases: z.array(ReleaseSchema).max(100) + releases: z.array(StoredReleaseSchema).max(100) }) .partial() .openapi("StoredExtensionContent"); @@ -111,8 +123,6 @@ export type StoredExtensionContent = z.infer< typeof StoredExtensionContentSchema >; -const MAX_CONTENT_BYTES = 256 * 1024; - // Applied to both the create and the edit body. The stored revision is this // object verbatim, so bounding it here bounds the row. function refineContentSize(content: unknown, ctx: z.RefinementCtx): void { @@ -128,7 +138,7 @@ function refineContentSize(content: unknown, ctx: z.RefinementCtx): void { // POST /extensions. The id is chosen once here and is immutable afterwards. // No developer field: a user owns at most one profile, so the server knows it. export const ExtensionCreateSchema = ExtensionContentSchema.extend({ - id: lowercaseId("extension") + id: lowercaseId("extension").max(200) }) .strict() .superRefine(refineContentSize) @@ -152,18 +162,25 @@ export type Extension = z.infer; // Catalogue cards do not need the potentially large README or every historic // release. Consumers can fetch those fields from GET /extensions/{id} when a // visitor opens an extension's detail page. -export const ExtensionListItemSchema = ExtensionSchema.omit({ +const ExtensionCardContentSchema = ExtensionContentSchema.omit({ readme: true, releases: true +}).extend({ + website: httpUrl() + .nullable() + .describe("Null when a legacy URL exceeds the card length bound"), + download_url: httpUrl() + .nullable() + .describe("Null when a legacy URL exceeds the card length bound") +}); + +export const ExtensionListItemSchema = ExtensionCardContentSchema.extend({ + id: z.string(), + developer: PublicDeveloperSchema }).openapi("ExtensionListItem"); export type ExtensionListItem = z.infer; -const ExtensionCardContentSchema = ExtensionContentSchema.omit({ - readme: true, - releases: true -}); - // The published projection as its *owner* sees it. Identical to the // catalogue's, except releases may be empty: v1 constrained // extensions.releases to NOT NULL and nothing more, so a row adopted by @@ -172,7 +189,7 @@ const ExtensionCardContentSchema = ExtensionContentSchema.omit({ // This can only ever describe a pre-v2 row - approve() requires a release // before anything reaches the catalogue through v2. const PublishedExtensionContentSchema = ExtensionContentSchema.extend({ - releases: z.array(ReleaseSchema).max(100) + releases: z.array(StoredReleaseSchema).max(100) }); // The most recent decision, kept alongside a later pending revision so the diff --git a/src/services/extensions/v2/schemas/revisions.ts b/src/services/extensions/v2/schemas/revisions.ts index c2b384b..96b47db 100644 --- a/src/services/extensions/v2/schemas/revisions.ts +++ b/src/services/extensions/v2/schemas/revisions.ts @@ -14,7 +14,14 @@ export const ExtensionRevisionSchema = z developer_id: z.string(), submitted_by: z.string(), status: RevisionStatusSchema, - content: StoredExtensionContentSchema, + content: StoredExtensionContentSchema.nullable(), + name: z.string().max(120).nullable(), + version: z.string().max(100).nullable(), + description: z.string().max(4000).nullable(), + content_bytes: z.number().int().nonnegative(), + content_available: z.boolean(), + content_hash: z.string().nullable(), + compacted_at: z.string().nullable(), reviewer_id: z.string().nullable(), review_note: z.string().nullable(), created_at: z.string(), @@ -22,6 +29,13 @@ export const ExtensionRevisionSchema = z }) .openapi("ExtensionRevision"); +export const ExtensionRevisionSummarySchema = ExtensionRevisionSchema.omit({ + content: true +}).openapi("ExtensionRevisionSummary"); +export type ExtensionRevisionSummary = z.infer< + typeof ExtensionRevisionSummarySchema +>; + export type ExtensionRevision = z.infer; export const RevisionIdParamSchema = z.object({ diff --git a/test/services/extensions/v2/db-fixtures.ts b/test/services/extensions/v2/db-fixtures.ts index b6ae1b3..0d311fa 100644 --- a/test/services/extensions/v2/db-fixtures.ts +++ b/test/services/extensions/v2/db-fixtures.ts @@ -131,6 +131,7 @@ async function clearDeveloperHistory(db: D1Database): Promise { } export async function resetExtensionsDb(db: D1Database): Promise { + await db.prepare("DELETE FROM extension_write_events").run(); await db.prepare("DELETE FROM claim_verification_budgets").run(); await clearDeveloperHistory(db); for (const table of [ @@ -143,6 +144,9 @@ export async function resetExtensionsDb(db: D1Database): Promise { ]) { await db.prepare(`DELETE FROM ${table}`).run(); } + await db + .prepare("DELETE FROM extension_resource_usage WHERE scope != 'global'") + .run(); } export async function insertUser( diff --git a/test/services/extensions/v2/extension-writes.test.ts b/test/services/extensions/v2/extension-writes.test.ts index 0775d7d..8d5f75e 100644 --- a/test/services/extensions/v2/extension-writes.test.ts +++ b/test/services/extensions/v2/extension-writes.test.ts @@ -344,6 +344,13 @@ describe("Extensions API v2 writes", () => { extensionId: `new-ext-${index}` }); expect(result.status).toBe(201); + // Age the traffic ledger while retaining all pending work. This + // test exercises pending admission independently of write pacing. + await db + .prepare( + "UPDATE extension_write_events SET occurred_at = unixepoch()-120" + ) + .run(); } const overLimit = await createExtension("user-1", { diff --git a/test/services/extensions/v2/harness.ts b/test/services/extensions/v2/harness.ts index 9537010..6f5ffcf 100644 --- a/test/services/extensions/v2/harness.ts +++ b/test/services/extensions/v2/harness.ts @@ -70,6 +70,9 @@ export function setupExtensionsV2Tests(): void { beforeEach(async () => { db = env.DB_EXTENSIONS; await resetExtensionsDb(db); + env.EXTENSION_WRITE_RATE_LIMITER = { + limit: async () => ({ success: true }) + }; env.PROFILE_CREATION_RATE_LIMITER = freshProfileCreationRateLimiter(); vi.clearAllMocks(); mockGithubEntityNotFound(); diff --git a/test/services/extensions/v2/migrations.test.ts b/test/services/extensions/v2/migrations.test.ts index 139b7b1..ada2e58 100644 --- a/test/services/extensions/v2/migrations.test.ts +++ b/test/services/extensions/v2/migrations.test.ts @@ -3,6 +3,7 @@ import { dirname, join, resolve } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; +import { unstable_splitSqlQuery } from "wrangler"; const migrationsDirectory = resolve( dirname(fileURLToPath(import.meta.url)), @@ -731,3 +732,173 @@ describe("Extensions D1 migrations", () => { } }); }); + +describe("Resource-bound migration", () => { + it("ranks manual counter discrepancies before limiting the report", () => { + const db = new DatabaseSync(":memory:"); + try { + for (const name of migrationNames) + for (const statement of unstable_splitSqlQuery(migration(name))) + db.exec(statement); + const insert = db.prepare( + "INSERT INTO extension_resource_usage(scope,subject,bytes) VALUES('account',?,?)" + ); + for (let i = 1; i <= 110; i++) insert.run(`subject-${i}`, i * 10); + const rows = db + .prepare( + readFileSync( + join(migrationsDirectory, "../resource-inventory.sql"), + "utf8" + ) + ) + .all() as Array<{ subject: string }>; + expect(rows).toHaveLength(100); + expect(rows[0].subject).toBe("subject-110"); + expect(rows[99].subject).toBe("subject-11"); + } finally { + db.close(); + } + }); + + it("backfills readability without treating unsafe or malformed legacy bodies as available", () => { + const db = new DatabaseSync(":memory:"); + try { + for (const name of migrationNames.filter( + (candidate) => candidate < "0026" + )) + db.exec(migration(name)); + db.exec( + "INSERT INTO users(id,created_at,updated_at) VALUES('owner',CURRENT_TIMESTAMP,CURRENT_TIMESTAMP); INSERT INTO developers(id,type,name,owner_user_id) VALUES('developer','user','Developer','owner'); INSERT INTO extensions(id,developer_id) VALUES('legacy','developer')" + ); + const records = [ + ["safe", JSON.stringify({ name: "Safe" }), 1], + ["malformed", "not JSON", 0], + ["not-object", "[]", 0], + [ + "unsafe", + JSON.stringify({ + releases: Array.from({ length: 101 }, () => ({ tag: "1.0.0" })) + }), + 0 + ], + [ + "unicode", + JSON.stringify({ releases: [{ tag: "πŸ˜€".repeat(100) }] }), + 1 + ] + ] as const; + for (const [id, content] of records) + db.prepare( + "INSERT INTO extension_revisions(id,extension_id,developer_id,submitted_by,status,content) VALUES(?,'legacy','developer','owner','rejected',?)" + ).run(id, content); + for (const statement of unstable_splitSqlQuery( + migration("0026_resource_bounds.sql") + )) + db.exec(statement); + for (const [id, content, content_readable] of records) + expect( + db + .prepare( + "SELECT content,content_readable FROM extension_revisions WHERE id=?" + ) + .get(id) + ).toEqual({ content, content_readable }); + } finally { + db.close(); + } + }); + + it("preserves oversized legacy content and reconciles accounting using Wrangler's SQL splitter", () => { + const db = new DatabaseSync(":memory:"); + try { + db.exec("PRAGMA foreign_keys=ON"); + for (const name of migrationNames.filter( + (candidate) => candidate < "0026" + )) + db.exec(migration(name)); + db.exec( + "INSERT INTO users(id,created_at,updated_at) VALUES('owner',CURRENT_TIMESTAMP,CURRENT_TIMESTAMP); INSERT INTO developers(id,type,name,owner_user_id) VALUES('developer','user','Developer','owner'); INSERT INTO extensions(id,developer_id) VALUES('legacy','developer')" + ); + const content = JSON.stringify({ + name: "Legacy", + readme: "πŸ˜€".repeat(70000) + }); + db.prepare( + "INSERT INTO extension_revisions(id,extension_id,developer_id,submitted_by,status,content,reviewed_at) VALUES('legacy-revision','legacy','developer','owner','rejected',?,'2000-01-01')" + ).run(content); + db.exec("BEGIN"); + for (const statement of unstable_splitSqlQuery( + migration("0026_resource_bounds.sql") + )) + db.exec(statement); + db.exec("COMMIT"); + expect( + db + .prepare( + "SELECT content,content_bytes,summary_name,compacted_at FROM extension_revisions" + ) + .get() + ).toEqual({ + content, + content_bytes: Buffer.byteLength(content), + summary_name: null, + compacted_at: null + }); + expect(db.prepare("SELECT created_by FROM extensions").get()).toEqual({ + created_by: "owner" + }); + expect( + db + .prepare( + readFileSync( + join(migrationsDirectory, "../resource-inventory.sql"), + "utf8" + ) + ) + .all() + ).toEqual([]); + expect(() => + db + .prepare( + "INSERT INTO extension_revisions(id,extension_id,developer_id,submitted_by,status,content) VALUES('new','legacy','developer','owner','rejected',?)" + ) + .run(content) + ).toThrow(/extension_content_size/); + + const bounded = JSON.stringify({ + readme: "x".repeat( + 262144 - Buffer.byteLength(JSON.stringify({ readme: "" })) + ) + }); + expect(Buffer.byteLength(bounded)).toBe(262144); + db.prepare( + "INSERT INTO extension_revisions(id,extension_id,developer_id,submitted_by,status,content) VALUES('boundary','legacy','developer','owner','rejected',?)" + ).run(bounded); + expect(() => + db + .prepare( + "UPDATE extension_revisions SET content=? WHERE id='boundary'" + ) + .run(bounded.replace("xxx", "xxxx")) + ).toThrow(/extension_content_size/); + // Existing oversized bodies may shrink without a destructive migration. + db.prepare( + "UPDATE extension_revisions SET content=? WHERE id='legacy-revision'" + ).run(JSON.stringify({ name: "Legacy", readme: "πŸ˜€".repeat(69999) })); + db.exec("DELETE FROM extensions WHERE id='legacy'"); + expect( + db + .prepare( + readFileSync( + join(migrationsDirectory, "../resource-inventory.sql"), + "utf8" + ) + ) + .all() + ).toEqual([]); + expect(db.prepare("PRAGMA foreign_key_check").all()).toEqual([]); + } finally { + db.close(); + } + }); +}); diff --git a/test/services/extensions/v2/moderation-correct.test.ts b/test/services/extensions/v2/moderation-correct.test.ts index dd9882f..79f74c8 100644 --- a/test/services/extensions/v2/moderation-correct.test.ts +++ b/test/services/extensions/v2/moderation-correct.test.ts @@ -51,6 +51,64 @@ async function seedPublished(): Promise { } describe("POST /extensions/{id}/moderator-correct (api#251)", () => { + it("returns the canonical id without a fallible post-commit read", async () => { + await seedPublished(); + env.DB_EXTENSIONS = wrapD1WithHook(db, (query) => { + if (/^select "id" from "extensions"/i.test(query)) + throw new Error("canonical read failed"); + }); + const response = await post( + PATH.replace("live-ext", "LIVE-EXT"), + await authHeaders("mod-1"), + correctBody() + ); + expect(response.status).toBe(200); + await expect(response.json()).resolves.toMatchObject({ + result: { id: "live-ext" } + }); + expect(await countRevisions(db)).toBe(1); + }); + + it("charges a first-time moderator exactly once and keeps totals reconciled", async () => { + await seedPublished(); + const response = await post( + PATH, + await authHeaders("mod-1"), + correctBody() + ); + expect(response.status).toBe(200); + const { correction_note: _note, ...content } = correctBody(); + const bytes = new TextEncoder().encode(JSON.stringify(content)).byteLength; + expect( + await db + .prepare( + "SELECT bytes,revisions,extensions FROM extension_resource_usage WHERE scope='account' AND subject='mod-1'" + ) + .first() + ).toEqual({ bytes, revisions: 1, extensions: 0 }); + const expected = await db + .prepare( + "SELECT (SELECT COALESCE(SUM(content_bytes),0) FROM extension_revisions)+(SELECT COALESCE(SUM(published_bytes),0) FROM extensions) AS bytes, (SELECT COUNT(*) FROM extension_revisions) AS revisions, (SELECT COUNT(*) FROM extensions) AS extensions" + ) + .first(); + expect( + await db + .prepare( + "SELECT bytes,revisions,extensions FROM extension_resource_usage WHERE scope='global'" + ) + .first() + ).toEqual(expected); + for (const scope of ["account", "developer"]) { + expect( + await db + .prepare( + "SELECT SUM(bytes) AS bytes, SUM(revisions) AS revisions, SUM(extensions) AS extensions FROM extension_resource_usage WHERE scope=?" + ) + .bind(scope) + .first() + ).toEqual(expected); + } + }); it("requires auth", async () => { const res = await post( PATH, diff --git a/test/services/extensions/v2/moderation.test.ts b/test/services/extensions/v2/moderation.test.ts index 18c8ef7..0d7fe1b 100644 --- a/test/services/extensions/v2/moderation.test.ts +++ b/test/services/extensions/v2/moderation.test.ts @@ -564,27 +564,25 @@ describe("Extensions API v2", () => { // But it is still readable as history, with the empty releases intact. const history = await get( - "/extensions/v2/extensions/legacy-ext/revisions", + "/extensions/v2/extensions/legacy-ext/revisions/legacy-revision", await authHeaders("user-1") ); expect(history.status).toBe(200); const body = (await history.json()) as { - result: Array<{ content: Record }>; + result: { content: Record }; }; // Served as stored, with the fields it never had simply absent. - expect(body.result[0].content).toMatchObject({ + expect(body.result.content).toMatchObject({ type: "mod", name: "Legacy" }); - expect(body.result[0].content.description).toBeUndefined(); + expect(body.result.content.description).toBeUndefined(); // The advertised contract has to accept what is actually served. Hono // does not validate responses at runtime, so nothing else catches a // response schema that disagrees with the data - the generated client // would be the first to find out. - expect(ExtensionRevisionSchema.safeParse(body.result[0]).success).toBe( - true - ); + expect(ExtensionRevisionSchema.safeParse(body.result).success).toBe(true); }); // reviewed_at is only second-granular, so two reviews can share one and diff --git a/test/services/extensions/v2/resource-bounds.test.ts b/test/services/extensions/v2/resource-bounds.test.ts new file mode 100644 index 0000000..9c7b070 --- /dev/null +++ b/test/services/extensions/v2/resource-bounds.test.ts @@ -0,0 +1,1202 @@ +import { describe, expect, it, vi } from "vitest"; +import { + createExecutionContext, + waitOnExecutionContext +} from "cloudflare:test"; +import { env } from "cloudflare:workers"; +import app from "../../../../src/app"; +import { getExtensionsDb } from "../../../../src/lib/db"; +import { + maintainExtensionResources, + reportExtensionResources, + inventoryExtensionRetention +} from "../../../../src/services/extensions/v2/db/resource-maintenance"; +import { + ExtensionListItemSchema, + ExtensionUpdateSchema, + OwnedExtensionListItemSchema +} from "../../../../src/services/extensions/v2/schemas/extensions"; +import { + MAX_ACCOUNT_BYTES, + MAX_CONTENT_BYTES, + MAX_RAW_BODY_BYTES, + MAINTENANCE_BATCH_SIZE +} from "../../../../src/services/extensions/v2/resource-limits"; +import { databaseError } from "../../../../src/services/extensions/v2/db/errors"; +import { ExtensionsDatabase } from "../../../../src/services/extensions/v2/db/extensions"; +import { wrapD1WithHook } from "./db-interceptor"; +import { + setupExtensionsV2Tests, + db, + authHeaders, + seedDeveloper, + sampleCreate, + sampleContent, + post, + put, + get +} from "./harness"; +import { + insertExtension, + insertRevision, + insertUser, + countExtensions, + countRevisions +} from "./db-fixtures"; +vi.mock("@octokit/request", async () => + (await import("../../../mocks/octokit")).octokitRequestMock() +); +setupExtensionsV2Tests(); + +async function withdraw(id: string, user = "owner") { + const ctx = createExecutionContext(); + const result = await app.request( + `/extensions/v2/extensions/${id}`, + { method: "DELETE", headers: await authHeaders(user) }, + env, + ctx + ); + await waitOnExecutionContext(ctx); + return result; +} +async function create(id: string) { + return post("/extensions/v2/extensions", await authHeaders("owner"), { + ...sampleCreate(), + id + }); +} +async function owned() { + await seedDeveloper("developer", "owner"); + await insertExtension(db, { id: "live", developer_id: "developer" }); +} +async function ageEvents(seconds = 120) { + await db + .prepare("UPDATE extension_write_events SET occurred_at=unixepoch()-?") + .bind(seconds) + .run(); +} +async function insertHistory(id: string, status = "rejected") { + await insertRevision(db, { + id, + extension_id: "live", + developer_id: "developer", + submitted_by: "owner", + content: JSON.stringify(sampleContent()), + status, + created_at: "2000-01-01 00:00:00", + reviewed_at: "2000-01-01 00:00:00" + }); +} + +describe("Extension resource admission", () => { + it("retires expired events with accepted writes across unrelated accounts", async () => { + await owned(); + await db.batch( + Array.from({ length: 120 }, (_, i) => + db + .prepare( + "INSERT INTO extension_write_events VALUES (?, ?, ?, unixepoch()-86401)" + ) + .bind(`expired-${i}`, `other-${i}`, `dev-${i}`) + ) + ); + await db + .prepare( + "INSERT INTO extension_write_events VALUES ('live-event','other','other',unixepoch()-120)" + ) + .run(); + for (const id of ["one", "two", "three"]) { + expect((await create(id)).status).toBe(201); + } + expect( + await db + .prepare( + "SELECT COUNT(*) AS n FROM extension_write_events WHERE occurred_at<=unixepoch()-86400" + ) + .first("n") + ).toBe(0); + expect( + await db + .prepare( + "SELECT COUNT(*) AS n FROM extension_write_events WHERE id='live-event'" + ) + .first("n") + ).toBe(1); + }); + it("uses retention ordering without a temporary sort", async () => { + const plan = await db + .prepare( + "EXPLAIN QUERY PLAN SELECT r.id FROM extension_revisions r WHERE r.compacted_at IS NULL AND r.status IN ('approved','rejected') AND r.reviewed_at < datetime('now','-180 days') ORDER BY r.reviewed_at,r.id LIMIT 20" + ) + .all<{ detail: string }>(); + expect( + plan.results.some((row) => + row.detail.includes("idx_extension_revisions_retention") + ) + ).toBe(true); + expect(plan.results.some((row) => row.detail.includes("TEMP B-TREE"))).toBe( + false + ); + }); + it("keeps oversized published extensions visible as published cards", async () => { + await owned(); + await db + .prepare("UPDATE extensions SET readme=? WHERE id='live'") + .bind("x".repeat(MAX_CONTENT_BYTES + 1)) + .run(); + const extensions = new ExtensionsDatabase(getExtensionsDb(db)); + const list = await extensions.listOwned({ developerId: "developer" }); + expect(list.error).toBeNull(); + expect(list.data?.items[0].published).not.toBeNull(); + expect( + OwnedExtensionListItemSchema.safeParse(list.data?.items[0]).success + ).toBe(true); + expect((await extensions.getOwned("live")).error?.code).toBe( + "CONTENT_UNAVAILABLE" + ); + }); + + it("prevents revision attribution changes from moving retained quota charges", async () => { + await owned(); + await insertHistory("fixed-attribution"); + await insertUser(db, { id: "other" }); + await seedDeveloper("other-developer", "other"); + for (const column of [ + "submitted_by", + "developer_id", + "extension_id", + "id" + ]) { + const value = column === "developer_id" ? "other-developer" : "other"; + await expect( + db + .prepare( + `UPDATE extension_revisions SET ${column}=? WHERE id='fixed-attribution'` + ) + .bind(value) + .run() + ).rejects.toThrow(/extension_resource_identity/); + } + expect( + await db + .prepare( + "SELECT submitted_by,developer_id,extension_id,id FROM extension_revisions WHERE id='fixed-attribution'" + ) + .first() + ).toEqual({ + submitted_by: "owner", + developer_id: "developer", + extension_id: "live", + id: "fixed-attribution" + }); + }); + it("reconciles every charged published column against independent UTF-8 byte counts", async () => { + await owned(); + const columns = [ + "type", + "name", + "description", + "releases", + "website", + "license", + "icon_url", + "readme", + "source", + "version", + "download_url" + ]; + async function reconcile() { + const rows = await db + .prepare("SELECT * FROM extensions") + .all>(); + const revisions = await db + .prepare("SELECT content FROM extension_revisions") + .all<{ content: string }>(); + const bytes = + rows.results.reduce( + (sum, row) => + sum + + columns.reduce( + (n, column) => + n + new TextEncoder().encode(row[column] ?? "").byteLength, + 0 + ), + 0 + ) + + revisions.results.reduce( + (sum, row) => sum + new TextEncoder().encode(row.content).byteLength, + 0 + ); + expect( + await db + .prepare( + "SELECT bytes FROM extension_resource_usage WHERE scope='global'" + ) + .first("bytes") + ).toBe(bytes); + for (const scope of ["account", "developer"]) + expect( + await db + .prepare( + "SELECT COALESCE(SUM(bytes),0) AS n FROM extension_resource_usage WHERE scope=?" + ) + .bind(scope) + .first("n") + ).toBe(bytes); + } + await reconcile(); + const content = sampleContent(); + const values = [ + content.type, + "πŸ˜€ Name", + "RΓ©sumΓ© πŸ§ͺ", + JSON.stringify(content.releases), + "https://example.test/Γ©", + JSON.stringify({ name: "LΓ―cence" }), + "https://example.test/πŸ˜€.png", + "θͺ­γ‚“でください", + JSON.stringify({ type: "custom", repo: "cafΓ©/πŸ˜€" }), + "1.0.0", + "https://example.test/Γ©.zip" + ]; + for (let i = 0; i < columns.length; i++) { + await db + .prepare(`UPDATE extensions SET ${columns[i]}=? WHERE id='live'`) + .bind(values[i]) + .run(); + await reconcile(); + } + await insertHistory("charged-revision"); + await reconcile(); + await db + .prepare( + "UPDATE extension_revisions SET content=? WHERE id='charged-revision'" + ) + .bind(JSON.stringify({ ...content, readme: "πŸ˜€" })) + .run(); + await reconcile(); + await db.prepare("DELETE FROM extensions WHERE id='live'").run(); + await reconcile(); + }); + it("preserves overflow status when stream cancellation rejects", async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(MAX_RAW_BODY_BYTES + 1)); + }, + cancel() { + throw new Error("cancel failed"); + } + }); + const response = await app.request( + "/extensions/v2/extensions", + { method: "POST", headers: await authHeaders("owner"), body: stream }, + env + ); + expect(response.status).toBe(413); + await expect(response.json()).resolves.toMatchObject({ + error: { code: "BODY_TOO_LARGE" } + }); + }); + it("accepts a valid JSON body delivered in small chunks", async () => { + await owned(); + const bytes = new TextEncoder().encode( + JSON.stringify({ ...sampleCreate(), id: "chunked" }) + ); + let position = 0; + const stream = new ReadableStream({ + pull(controller) { + if (position === bytes.length) { + controller.close(); + return; + } + controller.enqueue(bytes.subarray(position, position + 73)); + position = Math.min(position + 73, bytes.length); + } + }); + const response = await app.request( + "/extensions/v2/extensions", + { method: "POST", headers: await authHeaders("owner"), body: stream }, + env + ); + expect(response.status).toBe(201); + }); + + it("bounds new extension IDs while preserving reads and edits of longer legacy IDs", async () => { + await owned(); + const id = "e".repeat(201); + expect((await create(id)).status).toBe(422); + await insertExtension(db, { id, developer_id: "developer" }); + expect((await get(`/extensions/v2/extensions/${id}`, {})).status).toBe(200); + expect( + ( + await put( + `/extensions/v2/extensions/${id}`, + await authHeaders("owner"), + sampleContent() + ) + ).status + ).toBe(202); + }); + it("logs safe backend diagnostics without SQL or submitted content", () => { + const error = Object.assign( + new Error("SQLITE_BUSY: SELECT secret_content FROM private_table"), + { code: "SQLITE_BUSY", name: "SqliteError" } + ); + const log = vi.spyOn(console, "error").mockImplementation(() => {}); + try { + databaseError( + "create", + new Error("INSERT submitted payload", { cause: error }) + ); + expect(log).toHaveBeenCalledOnce(); + expect(log.mock.calls[0][0]).toContain('"error_type":"SqliteError"'); + expect(log.mock.calls[0][0]).toContain('"backend_code":"SQLITE_BUSY"'); + expect(log.mock.calls[0][0]).not.toMatch( + /SELECT|INSERT|secret_content|submitted payload/ + ); + } finally { + log.mockRestore(); + } + }); + it("counts actual streamed bytes, cancels at overflow and ignores false length hints", async () => { + let canceled = false; + let pulls = 0; + const headers = await authHeaders("owner"); + headers["Content-Length"] = "1"; + const stream = new ReadableStream({ + pull(controller) { + pulls++; + controller.enqueue(new Uint8Array(MAX_RAW_BODY_BYTES + 1)); + }, + cancel() { + canceled = true; + } + }); + const res = await app.request( + "/extensions/v2/extensions", + { method: "POST", headers, body: stream }, + env + ); + expect(res.status).toBe(413); + expect(canceled).toBe(true); + expect(pulls).toBeLessThanOrEqual(2); + expect(await countExtensions(db)).toBe(0); + }); + it("accepts the exact raw-byte cap and rejects one byte more without Content-Length", async () => { + await seedDeveloper("developer", "owner"); + const body = JSON.stringify({ ...sampleCreate(), id: "exact" }); + const padded = + body + + " ".repeat( + MAX_RAW_BODY_BYTES - new TextEncoder().encode(body).byteLength + ); + const headers = await authHeaders("owner"); + expect( + ( + await app.request( + "/extensions/v2/extensions", + { method: "POST", headers, body: padded }, + env + ) + ).status + ).toBe(201); + expect( + ( + await app.request( + "/extensions/v2/extensions", + { method: "POST", headers, body: padded + " " }, + env + ) + ).status + ).toBe(413); + }); + it("bounds raw moderator corrections and rejected unknown-field padding", async () => { + for (const path of [ + "/extensions/v2/extensions", + "/extensions/v2/extensions/live/moderator-correct" + ]) { + const response = await app.request( + path, + { + method: "POST", + headers: await authHeaders("owner"), + body: JSON.stringify({ padding: "x".repeat(MAX_RAW_BODY_BYTES) }) + }, + env + ); + expect(response.status).toBe(413); + } + }); + it("enforces the exact serialized content bound, including escaping", () => { + const content = { ...sampleContent(), readme: "" }; + const base = new TextEncoder().encode(JSON.stringify(content)).byteLength; + const remaining = MAX_CONTENT_BYTES - base; + content.readme = + "\0".repeat(Math.floor(remaining / 6)) + "x".repeat(remaining % 6); + expect(new TextEncoder().encode(JSON.stringify(content)).byteLength).toBe( + MAX_CONTENT_BYTES + ); + expect(ExtensionUpdateSchema.safeParse(content).success).toBe(true); + expect( + ExtensionUpdateSchema.safeParse({ + ...content, + readme: content.readme + "x" + }).success + ).toBe(false); + }); + it("paces authenticated attempts before validation and fails closed when pacing is unavailable", async () => { + await seedDeveloper("developer", "owner"); + env.EXTENSION_WRITE_RATE_LIMITER = { + limit: async ({ key }) => ({ success: !key.startsWith("account:") }) + }; + const res = await post( + "/extensions/v2/extensions", + await authHeaders("owner"), + { invalid: true } + ); + expect(res.status).toBe(429); + expect(res.headers.get("Retry-After")).toBe("60"); + env.EXTENSION_WRITE_RATE_LIMITER = { + limit: async () => { + throw new Error("unavailable"); + } + }; + expect((await create("unavailable")).status).toBe(503); + expect(await countExtensions(db)).toBe(0); + }); + it.each(["create", "edit", "correct"] as const)( + "fails closed with 503 when durable %s admission is unavailable", + async (operation) => { + await owned(); + await insertUser(db, { id: "mod", is_moderator: 1 }); + env.DB_EXTENSIONS = wrapD1WithHook(db, (query) => { + if ( + /INSERT INTO (?:"?extension_revisions"?|"?extensions"?)/i.test(query) + ) + throw new Error("D1 backend unavailable"); + }); + const res = + operation === "create" + ? await create("unavailable") + : operation === "edit" + ? await put( + "/extensions/v2/extensions/live", + await authHeaders("owner"), + sampleContent() + ) + : await post( + "/extensions/v2/extensions/live/moderator-correct", + await authHeaders("mod"), + { ...sampleContent(), correction_note: "Fix" } + ); + expect(res.status).toBe(503); + expect(res.headers.get("Retry-After")).toBe("60"); + await expect(res.json()).resolves.toMatchObject({ + error: { code: "ADMISSION_UNAVAILABLE" } + }); + expect(await countRevisions(db)).toBe(0); + expect( + await db + .prepare("SELECT COUNT(*) AS n FROM extension_write_events") + .first("n") + ).toBe(0); + expect(await countExtensions(db)).toBe(1); + } + ); + it("atomically shares five accepted writes across concurrent creates and edits", async () => { + await owned(); + const results = await Promise.all([ + ...Array.from({ length: 6 }, (_, i) => create(`concurrent-${i}`)), + put( + "/extensions/v2/extensions/live", + await authHeaders("owner"), + sampleContent() + ) + ]); + expect( + results.filter((r) => r.status === 201 || r.status === 202) + ).toHaveLength(5); + expect(results.filter((r) => r.status === 429)).toHaveLength(2); + expect(await countRevisions(db)).toBe(5); + expect( + await db + .prepare("SELECT COUNT(*) AS n FROM extension_write_events") + .first("n") + ).toBe(5); + }); + it("does not refund accepted allowance on withdrawal or profile/account lifecycle changes", async () => { + await seedDeveloper("developer", "owner"); + for (let i = 0; i < 5; i++) { + expect((await create(`churn-${i}`)).status).toBe(201); + expect((await withdraw(`churn-${i}`)).status).toBe(200); + } + await db.prepare("DELETE FROM developers WHERE id='developer'").run(); + await seedDeveloper("replacement", "owner"); + await db + .prepare("UPDATE users SET deleted_at=CURRENT_TIMESTAMP WHERE id='owner'") + .run(); + await db.prepare("UPDATE users SET deleted_at=NULL WHERE id='owner'").run(); + expect((await create("churn-again")).status).toBe(429); + expect(await countExtensions(db)).toBe(0); + await ageEvents(); + expect((await create("after-window")).status).toBe(201); + }); + it("enforces the rolling day budget without a whole-system write cap", async () => { + await seedDeveloper("developer", "owner"); + await db.batch( + Array.from({ length: 50 }, (_, i) => + db + .prepare( + "INSERT INTO extension_write_events VALUES (?, 'owner', 'developer', unixepoch()-120)" + ) + .bind(`day-${i}`) + ) + ); + const day = await create("day-limit"); + expect(day.status).toBe(429); + expect(day.headers.get("Retry-After")).toBe("86400"); + await db.prepare("DELETE FROM extension_write_events").run(); + await db.batch( + Array.from({ length: 300 }, (_, i) => + db + .prepare( + "INSERT INTO extension_write_events VALUES (?, ?, ?, unixepoch()-120)" + ) + .bind(`global-${i}`, `user-${i}`, `developer-${i}`) + ) + ); + const global = await create("global-limit"); + expect(global.status).toBe(201); + expect(global.headers.has("Retry-After")).toBe(false); + expect(await countExtensions(db)).toBe(1); + }); + it("rolls back extensions and write charges when quota admission fails", async () => { + await seedDeveloper("developer", "owner"); + expect((await create("first")).status).toBe(201); + await ageEvents(); + await db + .prepare( + "UPDATE extension_resource_usage SET bytes=? WHERE scope='account' AND subject='owner'" + ) + .bind(MAX_ACCOUNT_BYTES) + .run(); + const response = await create("over-quota"); + expect(response.status).toBe(409); + await expect(response.json()).resolves.toMatchObject({ + error: { code: "RESOURCE_QUOTA" } + }); + expect(await countExtensions(db)).toBe(1); + expect(await countRevisions(db)).toBe(1); + expect( + await db + .prepare("SELECT COUNT(*) AS n FROM extension_write_events") + .first("n") + ).toBe(1); + }); + it.each([ + ["account", "owner", "extensions", 100], + ["developer", "developer", "extensions", 100], + ["account", "owner", "revisions", 1000], + ["developer", "developer", "revisions", 1000], + ["developer", "developer", "bytes", MAX_ACCOUNT_BYTES] + ] as const)( + "enforces %s %s retained %s quota", + async (scope, subject, column, limit) => { + await seedDeveloper("developer", "owner"); + expect((await create("first")).status).toBe(201); + await ageEvents(); + const original = await db + .prepare( + `SELECT ${column} AS value FROM extension_resource_usage WHERE scope=? AND subject=?` + ) + .bind(scope, subject) + .first("value"); + try { + await db + .prepare( + `UPDATE extension_resource_usage SET ${column}=? WHERE scope=? AND subject=?` + ) + .bind(limit, scope, subject) + .run(); + expect((await create("blocked")).status).toBe(409); + expect(await countExtensions(db)).toBe(1); + expect(await countRevisions(db)).toBe(1); + } finally { + // Restore injected counter state; normal teardown verifies the real + // accounting triggers, including the persistent global bucket. + await db + .prepare( + `UPDATE extension_resource_usage SET ${column}=? WHERE scope=? AND subject=?` + ) + .bind(original, scope, subject) + .run(); + } + } + ); + it("rolls back approval when the published copy would exceed retained quota", async () => { + await seedDeveloper("developer", "owner"); + await insertUser(db, { id: "mod", is_moderator: 1 }); + expect((await create("publication")).status).toBe(201); + const revision = await db + .prepare( + "SELECT id FROM extension_revisions WHERE extension_id='publication'" + ) + .first("id"); + await db + .prepare( + "UPDATE extension_resource_usage SET bytes=? WHERE scope='account' AND subject='owner'" + ) + .bind(MAX_ACCOUNT_BYTES) + .run(); + const response = await post( + `/extensions/v2/extensions/publication/revisions/${revision}/approve?notify=false`, + await authHeaders("mod"), + {} + ); + expect(response.status).toBe(409); + await expect(response.json()).resolves.toMatchObject({ + error: { code: "RESOURCE_QUOTA" } + }); + expect( + await db + .prepare("SELECT status FROM extension_revisions WHERE id=?") + .bind(revision) + .first("status") + ).toBe("pending"); + expect( + await db + .prepare("SELECT published_at FROM extensions WHERE id='publication'") + .first("published_at") + ).toBeNull(); + }); + it("accounts UTF-8 bytes exactly and releases them transactionally without shifting account charges", async () => { + await seedDeveloper("developer", "owner"); + const content = { ...sampleCreate(), id: "multibyte", readme: "πŸ˜€Γ©" }; + const { id: _id, ...stored } = content; + expect( + ( + await post( + "/extensions/v2/extensions", + await authHeaders("owner"), + content + ) + ).status + ).toBe(201); + const bytes = new TextEncoder().encode(JSON.stringify(stored)).byteLength; + expect( + await db + .prepare( + "SELECT bytes FROM extension_resource_usage WHERE scope='account' AND subject='owner'" + ) + .first("bytes") + ).toBe(bytes); + await insertUser(db, { id: "recipient" }); + await db + .prepare( + "UPDATE developers SET owner_user_id='recipient' WHERE id='developer'" + ) + .run(); + expect( + await db + .prepare( + "SELECT bytes FROM extension_resource_usage WHERE scope='account' AND subject='owner'" + ) + .first("bytes") + ).toBe(bytes); + expect((await withdraw("multibyte", "recipient")).status).toBe(200); + expect( + await db + .prepare( + "SELECT bytes FROM extension_resource_usage WHERE scope='global'" + ) + .first("bytes") + ).toBe(0); + }); +}); + +describe("Bounded revision reads and maintenance", () => { + it("pages metadata with equal timestamps, without selecting or parsing content", async () => { + await owned(); + for (const id of ["a", "b", "c"]) await insertHistory(id); + await insertUser(db, { id: "mod", is_moderator: 1 }); + const sql: string[] = []; + env.DB_EXTENSIONS = wrapD1WithHook(db, (query) => { + sql.push(query); + }); + const headers = await authHeaders("mod"); + const first = await get( + "/extensions/v2/revisions?status=rejected&limit=2", + headers + ); + const page = (await first.json()) as { + result: Array<{ id: string; content?: unknown }>; + pagination: { next_cursor: string }; + }; + expect(page.result.map((r) => r.id)).toEqual(["a", "b"]); + expect(page.result.every((r) => !("content" in r))).toBe(true); + const second = await get( + `/extensions/v2/revisions?status=rejected&limit=2&cursor=${encodeURIComponent(page.pagination.next_cursor)}`, + headers + ); + await expect(second.json()).resolves.toMatchObject({ + result: [{ id: "c" }], + pagination: { has_more: false } + }); + const selection = sql.find( + (q) => q.includes('from "extension_revisions"') && q.includes("order by") + ); + expect(selection).toBeDefined(); + expect(selection?.split(" from ")[0]).not.toMatch(/"content"/); + expect( + ( + await get( + "/extensions/v2/extensions/live/revisions/a", + await authHeaders("intruder") + ) + ).status + ).toBe(403); + const detail = await get( + "/extensions/v2/extensions/live/revisions/a", + await authHeaders("owner") + ); + await expect(detail.json()).resolves.toMatchObject({ + result: { + content: { name: sampleContent().name }, + content_available: true + } + }); + }); + it("rejects unsafe legacy release collections before sorting", async () => { + await owned(); + for (const [id, releases] of [ + ["too-many", Array.from({ length: 101 }, () => ({ tag: "1.0.0" }))], + ["bad-shape", [null]], + ["huge-tag", [{ tag: "x".repeat(101) }]], + ["huge-unicode-tag", [{ tag: "πŸ˜€".repeat(101) }]], + ["bad-tag", [{ tag: 1 }]], + ["bad-array", "oops"] + ] as const) { + await insertRevision(db, { + id, + extension_id: "live", + developer_id: "developer", + submitted_by: "owner", + content: JSON.stringify({ ...sampleContent(), releases }), + status: "rejected", + created_at: "2000-01-01" + }); + const detail = await get( + `/extensions/v2/extensions/live/revisions/${id}`, + await authHeaders("owner") + ); + expect(detail.status).toBe(409); + await expect(detail.json()).resolves.toMatchObject({ + error: { code: "CONTENT_UNAVAILABLE" } + }); + const list = await get( + "/extensions/v2/extensions/live/revisions", + await authHeaders("owner") + ); + const body = (await list.json()) as { + result: Array<{ id: string; content_available: boolean }>; + }; + expect( + body.result.find((revision) => revision.id === id)?.content_available + ).toBe(false); + } + }); + it("keeps Unicode legacy tag availability consistent with detail reads", async () => { + await owned(); + await insertRevision(db, { + id: "unicode-tag", + extension_id: "live", + developer_id: "developer", + submitted_by: "owner", + content: JSON.stringify({ + releases: [{ ...sampleContent().releases[0], tag: "πŸ˜€".repeat(100) }] + }), + status: "rejected" + }); + const detail = await get( + "/extensions/v2/extensions/live/revisions/unicode-tag", + await authHeaders("owner") + ); + expect(detail.status).toBe(200); + await expect(detail.json()).resolves.toMatchObject({ + result: { content_available: true } + }); + }); + it("compacts only old reviewed bodies, preserves published/pending records and remains idempotent", async () => { + await owned(); + for (const id of ["old", "published"]) await insertHistory(id, "approved"); + await db + .prepare( + "UPDATE extensions SET published_revision_id='published' WHERE id='live'" + ) + .run(); + await insertRevision(db, { + id: "pending", + extension_id: "live", + developer_id: "developer", + submitted_by: "owner", + content: JSON.stringify(sampleContent()), + created_at: "2000-01-01 00:00:00" + }); + await insertRevision(db, { + id: "recent", + extension_id: "live", + developer_id: "developer", + submitted_by: "owner", + content: JSON.stringify(sampleContent()), + status: "rejected", + created_at: "2000-01-01 00:00:00", + reviewed_at: "2099-01-01" + }); + const before = await db + .prepare( + "SELECT bytes FROM extension_resource_usage WHERE scope='global'" + ) + .first("bytes"); + await maintainExtensionResources(getExtensionsDb(db), { mode: "compact" }); + const old = await db + .prepare( + "SELECT content,content_hash,compacted_at FROM extension_revisions WHERE id='old'" + ) + .first<{ content: string; content_hash: string; compacted_at: string }>(); + expect(old?.content).toBe("{}"); + const expectedHash = Array.from( + new Uint8Array( + await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(JSON.stringify(sampleContent())) + ) + ), + (b) => b.toString(16).padStart(2, "0") + ).join(""); + expect(old?.content_hash).toBe(expectedHash); + expect(old?.compacted_at).toBeTruthy(); + for (const id of ["published", "pending", "recent"]) + expect( + await db + .prepare("SELECT compacted_at FROM extension_revisions WHERE id=?") + .bind(id) + .first("compacted_at") + ).toBeNull(); + const after = await db + .prepare( + "SELECT bytes FROM extension_resource_usage WHERE scope='global'" + ) + .first("bytes"); + expect(after).toBeLessThan(before!); + await maintainExtensionResources(getExtensionsDb(db), { mode: "compact" }); + expect( + await db + .prepare( + "SELECT bytes FROM extension_resource_usage WHERE scope='global'" + ) + .first("bytes") + ).toBe(after); + const detail = await get( + "/extensions/v2/extensions/live/revisions/old", + await authHeaders("owner") + ); + await expect(detail.json()).resolves.toMatchObject({ + result: { + content: null, + content_available: false, + content_hash: expectedHash + } + }); + expect(await countRevisions(db)).toBe(4); + }); + it("keeps aggregate usage observational even above the former global caps", async () => { + await seedDeveloper("developer", "owner"); + expect((await create("first")).status).toBe(201); + await ageEvents(); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + await db + .prepare( + "UPDATE extension_resource_usage SET bytes=524288001, extensions=10001, revisions=100001 WHERE scope='global'" + ) + .run(); + expect((await create("still-allowed")).status).toBe(201); + await reportExtensionResources(getExtensionsDb(db)); + expect(warn).not.toHaveBeenCalled(); + } finally { + await db + .prepare( + "UPDATE extension_resource_usage SET bytes=(SELECT COALESCE(SUM(published_bytes),0) FROM extensions)+(SELECT COALESCE(SUM(content_bytes),0) FROM extension_revisions), extensions=(SELECT COUNT(*) FROM extensions), revisions=(SELECT COUNT(*) FROM extension_revisions) WHERE scope='global'" + ) + .run(); + warn.mockRestore(); + } + }); + it("bounds response amplification for 100 maximum-sized bodies", async () => { + await owned(); + await seedDeveloper("second", "second-owner"); + await insertExtension(db, { id: "second-live", developer_id: "second" }); + await insertUser(db, { id: "mod", is_moderator: 1 }); + const content = { + ...sampleContent(), + description: "\u0001".repeat(4000), + readme: "" + }; + const remaining = + MAX_CONTENT_BYTES - + new TextEncoder().encode(JSON.stringify(content)).byteLength; + content.readme = + "\0".repeat(Math.floor(remaining / 6)) + "x".repeat(remaining % 6); + expect(ExtensionUpdateSchema.safeParse(content).success).toBe(true); + for (let i = 0; i < 100; i++) + await insertRevision(db, { + id: `max-${i}`, + extension_id: i < 50 ? "live" : "second-live", + developer_id: i < 50 ? "developer" : "second", + submitted_by: i < 50 ? "owner" : "second-owner", + content: JSON.stringify(content), + status: "rejected", + created_at: "2000-01-01", + reviewed_at: "2000-01-01" + }); + const headers = await authHeaders("mod"); + const queue = await get( + "/extensions/v2/revisions?status=rejected&limit=100", + headers + ); + const text = await queue.text(); + expect(queue.status).toBe(200); + const responseBytes = new TextEncoder().encode(text).byteLength; + expect(responseBytes).toBeLessThan(3 * 1024 * 1024); + const result = JSON.parse(text).result as Array>; + expect(result).toHaveLength(100); + expect(result.every((row) => !("content" in row))).toBe(true); + expect(result.every((row) => row.description === content.description)).toBe( + true + ); + const detail = await get( + "/extensions/v2/extensions/live/revisions/max-0", + headers + ); + const detailBytes = new TextEncoder().encode( + await detail.text() + ).byteLength; + expect(detail.status).toBe(200); + // Detail repeats the bounded summary; escaped description characters can + // add 24 KiB even when the canonical body is exactly at its limit. + expect(detailBytes).toBeLessThan(MAX_CONTENT_BYTES + 32 * 1024); + }); + it("previews retention and defaults to non-destructive scheduled runs", async () => { + await owned(); + await insertHistory("old"); + const bytes = new TextEncoder().encode( + JSON.stringify(sampleContent()) + ).byteLength; + const queries: string[] = []; + const hooked = wrapD1WithHook(db, (q) => { + queries.push(q); + }); + expect(await inventoryExtensionRetention(getExtensionsDb(hooked))).toEqual({ + eligible_bodies: 1, + reclaimable_bytes: bytes - 2 + }); + expect(queries).toHaveLength(1); + expect(queries[0]).not.toMatch(/SELECT content /i); + await app.scheduled({} as ScheduledController, env); + expect( + await db + .prepare("SELECT compacted_at FROM extension_revisions WHERE id='old'") + .first("compacted_at") + ).toBeNull(); + await maintainExtensionResources(getExtensionsDb(db), { mode: "invalid" }); + expect( + await db + .prepare("SELECT compacted_at FROM extension_revisions WHERE id='old'") + .first("compacted_at") + ).toBeNull(); + await maintainExtensionResources(getExtensionsDb(db), { mode: "compact" }); + expect( + await db + .prepare("SELECT compacted_at FROM extension_revisions WHERE id='old'") + .first("compacted_at") + ).toBeTruthy(); + }); + it("runs bounded cleanup and inventory hourly, keeping inventory read-only", async () => { + await owned(); + await insertHistory("old"); + const queries: string[] = []; + const hooked = wrapD1WithHook(db, (query) => { + queries.push(query); + }); + await maintainExtensionResources(getExtensionsDb(hooked)); + expect(queries.every((query) => !/\b(?:SUM|COUNT)\s*\(/i.test(query))).toBe( + true + ); + queries.length = 0; + env.DB_EXTENSIONS = hooked; + await app.scheduled({ cron: "0 * * * *" } as ScheduledController, env); + expect( + await db + .prepare("SELECT compacted_at FROM extension_revisions WHERE id='old'") + .first("compacted_at") + ).toBeNull(); + expect(queries.some((query) => /SUM\(/i.test(query))).toBe(true); + expect( + queries.some((query) => /DELETE FROM extension_write_events/i.test(query)) + ).toBe(true); + queries.length = 0; + await reportExtensionResources(getExtensionsDb(hooked), "compact"); + expect( + queries.every((query) => !/\b(?:INSERT|UPDATE|DELETE)\b/i.test(query)) + ).toBe(true); + }); + it.each([ + [{ repo: "example/repo" }, "custom"], + [{ type: "bitbucket", repo: "example/repo" }, "custom"], + [{ type: null, repo: "example/repo" }, "custom"], + [{ type: 42, repo: "example/repo" }, "custom"], + [{ type: "github", repo: "example/repo" }, "github"], + [{ type: "gitlab", repo: "example/repo" }, "gitlab"], + [{ type: "custom", repo: "example/repo" }, "custom"] + ])( + "normalizes legacy source %j to %s in public and owner cards", + async (source, type) => { + await owned(); + await db + .prepare("UPDATE extensions SET source=? WHERE id='live'") + .bind(JSON.stringify(source)) + .run(); + const extensions = new ExtensionsDatabase(getExtensionsDb(db)); + const publicCards = await extensions.list({}); + expect(publicCards.error).toBeNull(); + const publicCard = publicCards.data?.items[0]; + expect(publicCard?.source).toEqual({ type, repo: "example/repo" }); + expect(ExtensionListItemSchema.safeParse(publicCard).success).toBe(true); + const ownerCards = await extensions.listOwned({ + developerId: "developer" + }); + expect(ownerCards.error).toBeNull(); + const ownerCard = ownerCards.data?.items[0]; + expect(ownerCard?.published?.source).toEqual({ + type, + repo: "example/repo" + }); + expect(OwnedExtensionListItemSchema.safeParse(ownerCard).success).toBe( + true + ); + expect((await extensions.getById("live")).data?.source).toEqual(source); + } + ); + it("preserves under-limit legacy fields in public and owner details", async () => { + await owned(); + const website = "https://example.test/" + "x".repeat(3000); + const license = { name: "L".repeat(5000) }; + const source = { type: "custom", repo: "R".repeat(5000) }; + await db + .prepare( + "UPDATE extensions SET website=?, license=?, source=? WHERE id='live'" + ) + .bind(website, JSON.stringify(license), JSON.stringify(source)) + .run(); + const extensions = new ExtensionsDatabase(getExtensionsDb(db)); + const cards = await extensions.list({}); + expect(cards.error).toBeNull(); + expect( + ExtensionListItemSchema.safeParse(cards.data?.items[0]).success + ).toBe(true); + expect(cards.data?.items[0]).toMatchObject({ + website: null, + license: { name: "L".repeat(100) }, + source: { type: "custom", repo: "R".repeat(500) } + }); + const publicDetail = await extensions.getById("live"); + expect(publicDetail.error).toBeNull(); + expect(publicDetail.data).toMatchObject({ website, license, source }); + const ownerDetail = await extensions.getOwned("live"); + expect(ownerDetail.error).toBeNull(); + expect(ownerDetail.data?.extension.published).toMatchObject({ + website, + license, + source + }); + }); + it("limits maintenance work per invocation", async () => { + await owned(); + for (let i = 0; i <= MAINTENANCE_BATCH_SIZE; i++) + await insertHistory(`old-${i}`); + await maintainExtensionResources(getExtensionsDb(db), { mode: "compact" }); + expect( + await db + .prepare( + "SELECT COUNT(*) AS n FROM extension_revisions WHERE compacted_at IS NOT NULL" + ) + .first("n") + ).toBe(MAINTENANCE_BATCH_SIZE); + await maintainExtensionResources(getExtensionsDb(db), { mode: "compact" }); + expect( + await db + .prepare( + "SELECT COUNT(*) AS n FROM extension_revisions WHERE compacted_at IS NOT NULL" + ) + .first("n") + ).toBe(MAINTENANCE_BATCH_SIZE + 1); + }); + it("keeps oversized legacy bodies out of queue/detail reads and automatic compaction", async () => { + await owned(); + await insertHistory("legacy"); + // Emulate an imported pre-0026 row transactionally, restoring the guard + // before any request. Admission must never allow this for new content. + const trigger = await db + .prepare( + "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='extension_revision_content_bound'" + ) + .first("sql"); + await db.batch([ + db.prepare("DROP TRIGGER extension_revision_content_bound"), + db + .prepare("UPDATE extension_revisions SET content=? WHERE id='legacy'") + .bind("x".repeat(MAX_CONTENT_BYTES + 1)), + db.prepare(trigger!) + ]); + const list = await get( + "/extensions/v2/extensions/live/revisions", + await authHeaders("owner") + ); + await expect(list.json()).resolves.toMatchObject({ + result: [ + { + id: "legacy", + content_available: false, + content_bytes: MAX_CONTENT_BYTES + 1 + } + ] + }); + const detail = await get( + "/extensions/v2/extensions/live/revisions/legacy", + await authHeaders("owner") + ); + expect(detail.status).toBe(409); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + await maintainExtensionResources(getExtensionsDb(db), { + mode: "compact" + }); + await reportExtensionResources(getExtensionsDb(db), "compact"); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0][0]).toContain( + '"reason":"legacy_content_present"' + ); + expect(warn.mock.calls[0][0]).not.toContain("owner"); + expect(warn.mock.calls[0][0]).not.toContain(sampleContent().name); + } finally { + warn.mockRestore(); + } + expect( + await db + .prepare( + "SELECT compacted_at FROM extension_revisions WHERE id='legacy'" + ) + .first("compacted_at") + ).toBeNull(); + }); +}); diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index ee19e1a..ecceafa 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,5 +1,5 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: 9acc9d39a37fd9c6a402ae4b208ea835) +// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: 0850b9bff7a7ecb229a353368b706ad6) // Runtime types generated with workerd@1.20260911.1 2026-06-24 nodejs_compat interface __BaseEnv_CloudflareBindings { AUTH_KV: KVNamespace; @@ -7,7 +7,9 @@ interface __BaseEnv_CloudflareBindings { DOWNLOAD_BUCKET: R2Bucket; DB_CENTRAL_ALERTS: D1Database; DB_EXTENSIONS: D1Database; + EXTENSION_WRITE_RATE_LIMITER: RateLimit; PROFILE_CREATION_RATE_LIMITER: RateLimit; + EXTENSIONS_RETENTION_MODE: "dry-run"; EXTENSIONS_V2_EMAIL_PROVIDER: "mxroute"; EXTENSIONS_V2_MXROUTE_SERVER: "wednesday.mxrouting.net"; EXTENSIONS_V2_MXROUTE_USERNAME: "extensions@fossbilling.org"; @@ -30,7 +32,7 @@ type StringifyValues> = { [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; }; declare namespace NodeJS { - interface ProcessEnv extends StringifyValues> {} + interface ProcessEnv extends StringifyValues> {} } // Begin runtime types diff --git a/wrangler.jsonc b/wrangler.jsonc index 577e062..1f39a5f 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -19,6 +19,7 @@ "preview_urls": true, "keep_vars": true, "vars": { + "EXTENSIONS_RETENTION_MODE": "dry-run", "EXTENSIONS_V2_EMAIL_PROVIDER": "mxroute", "EXTENSIONS_V2_MXROUTE_SERVER": "wednesday.mxrouting.net", "EXTENSIONS_V2_MXROUTE_USERNAME": "extensions@fossbilling.org" @@ -78,7 +79,13 @@ "new_sqlite_classes": ["PreviewGitHubBudget"] } ], + "triggers": { "crons": ["0 * * * *"] }, "ratelimits": [ + { + "name": "EXTENSION_WRITE_RATE_LIMITER", + "namespace_id": "1002", + "simple": { "limit": 60, "period": 60 } + }, { "name": "PROFILE_CREATION_RATE_LIMITER", "namespace_id": "1001",