diff --git a/AGENTS.md b/AGENTS.md index 17a6dea9..ced1faab 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,7 @@ Crove is the open-source Business OS (AGPL). It must run with `docker compose up - Shared dev PostgreSQL: `migrate.db.test.ts` creates cluster-wide `t100_test_runner_*` roles and drops them afterwards; a run killed before that leaves them behind, and every later migrate on the cluster then fails the T17 check with `role t100_test_runner_..._proxy can create temporary objects`. The next run of that test drops leftovers older than 10 minutes; to clear them at once (with no run of it in progress): `psql "$CROVE_TEST_ADMIN_URL" -c "DO \$\$ DECLARE r text; BEGIN FOR r IN SELECT rolname FROM pg_roles WHERE rolname LIKE 't100\_test\_runner\_%' LOOP EXECUTE format('DROP OWNED BY %I', r); EXECUTE format('DROP ROLE %I', r); END LOOP; END \$\$"` - `pnpm db:migrate` applies `supabase/migrations` with the built-in runner - `pnpm db:types` (needs `CROVE_TEST_ADMIN_URL`) regenerates `packages/db/src/generated/database.ts`, the Kysely types of every table `crove_tenant` may read, with Kanel from a throwaway database the migrations build. Run it after a migration that changes such a table or its grants and commit the file; CI fails when it is out of date. Insert and update types follow `crove_tenant`'s column privileges: a column the role may not write has the write type `NotGranted`, so a forbidden write (an insert into `outbox_event` or the ledger, an update of an append-only table) fails typecheck (through `values()` and `set()`; deletes, raw SQL and insert ... select are refused only at runtime); a refinement cannot lift that. Refine a column (a CHECK-constraint union, a trigger-set column) in `packages/db/src/schema.ts`, never in the generated file. A CHECK-constraint union is a const array there (`TASK_STATUSES` and its type `TaskStatus`); `packages/db/test/check-unions.db.test.ts` compares every fixed-set CHECK constraint with its array, so a migration that adds, widens or narrows one fails until the array follows and is listed in that test. +- `pnpm db:vendor:pgboss` writes the migration for the pg-boss schema version the pinned pg-boss expects (a unit test fails until it exists after a pg-boss upgrade) - `docker compose up --build` runs the full stack on http://localhost:8080 - `pnpm test:e2e` runs Playwright against a stack started with `pnpm --filter @crove/e2e stack:up` (compose project `crove-e2e`, port 8080; `stack:down` removes it with its volumes). Install the browser once with `pnpm --filter @crove/e2e exec playwright install chromium` diff --git a/apps/kernel/package.json b/apps/kernel/package.json index 972fa58b..df8a4d26 100644 --- a/apps/kernel/package.json +++ b/apps/kernel/package.json @@ -10,7 +10,8 @@ "start": "node src/main.ts all", "migrate": "node --env-file-if-exists=../../.env src/main.ts migrate", "auth-grants": "node --env-file-if-exists=../../.env src/main.ts auth-grants", - "typecheck": "tsc --noEmit -p tsconfig.json" + "typecheck": "tsc --noEmit -p tsconfig.json", + "vendor:pgboss": "node src/vendor-pgboss.ts" }, "dependencies": { "@crove/agent": "workspace:*", diff --git a/apps/kernel/src/http.ts b/apps/kernel/src/http.ts index 8bd9fde2..e2929e72 100644 --- a/apps/kernel/src/http.ts +++ b/apps/kernel/src/http.ts @@ -30,6 +30,7 @@ import { publicAuthUrl, type Config } from './config.ts'; import { exportAad, exportSealer } from './export-seal.ts'; import { MAX_EMAIL_BYTES, acceptInboundEmail, secretMatches } from './inbound.ts'; import { createInviteMailSender } from './invite-mail.ts'; +import { pgBossSchemaVersion } from './jobs-schema.ts'; import type { Mailer } from './mail.ts'; import { RateLimiter } from './rate-limit.ts'; import { createRpcHandler, resolveMembership, resolveOrg, type RpcChannels } from './rpc.ts'; @@ -232,6 +233,9 @@ export function createHttpApp(deps: HttpDeps) { try { const missing = await schemaMissing(client, config.MIGRATIONS_DIR); if (missing.length > 0) return c.json({ ready: false, reason: 'schema' }, 503); + // The vendored crove_jobs schema must be the version this pg-boss release expects (External-tenant gate T6). + const jobs = await client.query<{ version: number | null }>('SELECT crove_kernel.jobs_schema_version() AS version'); + if (jobs.rows[0]?.version !== pgBossSchemaVersion()) return c.json({ ready: false, reason: 'jobs_schema' }, 503); // The session check fails closed until it can read Supabase Auth's tables (ADR-0028). if (config.AUTH_SESSION_CHECK === 'db') { const { rows } = await client.query<{ ok: boolean }>('SELECT crove_kernel.auth_grants_ready() AS ok'); diff --git a/apps/kernel/src/jobs-schema.ts b/apps/kernel/src/jobs-schema.ts new file mode 100644 index 00000000..488c4fcd --- /dev/null +++ b/apps/kernel/src/jobs-schema.ts @@ -0,0 +1,169 @@ +// The pg-boss schema (`crove_jobs`) is committed SQL like every other schema (plan principle 4, External-tenant +// gate T6). `pnpm db:vendor:pgboss` renders pg-boss's own plans into a migration; no code path runs pg-boss with +// `migrate: true`. The worker starts pg-boss with `migrate: false`, which refuses any other schema version, and +// /readyz compares the database's version with the one the pinned pg-boss expects. +import { readFileSync } from 'node:fs'; +import { readdir, readFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import { join } from 'node:path'; +import { getConstructionPlans, getMigrationPlans } from 'pg-boss'; + +export const JOB_SCHEMA = 'crove_jobs'; + +// Created by the migration that vendors the schema. A new queue, or a new policy, needs a new migration. +export const QUEUES = { + extract: 'finance-file-extract', + inbound: 'kernel-inbound-email', + signOut: 'kernel-session-signout', + orgExport: 'kernel-org-export', + retention: 'kernel-retention-purge', + agentDispatch: 'kernel-agent-dispatch', + agentWake: 'kernel-agent-wake', + agentSweep: 'kernel-agent-sweep', +} as const; +type QueueName = (typeof QUEUES)[keyof typeof QUEUES]; + +// The pg-boss policy of each queue that is not 'standard'. kernel-agent-wake is 'stately' with the singleton key +// :: at most one queued and one active job per agent, so wake-ups coalesce and an agent runs once at a +// time (ADR-0029 section 5; the database also allows one running ai_run per agent). +const QUEUE_POLICIES: Partial> = { [QUEUES.agentWake]: 'stately' }; + +// What `boss.createQueue(name, { policy })` sends to crove_jobs.create_queue. +export function queueOptions(queue: string): string { + return JSON.stringify({ policy: QUEUE_POLICIES[queue as QueueName] ?? 'standard' }); +} + +// First line of every vendored migration. +const MARKER = /^-- pg-boss schema version: (\d+)$/m; +const PIN = 'SET search_path = pg_catalog, pg_temp'; + +let expected: number | null = null; + +// The schema version the installed pg-boss release expects (its package.json `pgboss.schema`). +export function pgBossSchemaVersion(): number { + if (expected !== null) return expected; + const path = createRequire(import.meta.url).resolve('pg-boss/package.json'); + const pkg = JSON.parse(readFileSync(path, 'utf8')) as { pgboss?: { schema?: number } }; + const version = pkg.pgboss?.schema; + if (!Number.isInteger(version)) throw new Error('pg-boss package.json has no pgboss.schema'); + expected = version as number; + return expected; +} + +// The highest pg-boss schema version any committed migration vendors, or null if none does. +export async function vendoredSchemaVersion(migrationsDir: string): Promise { + let highest: number | null = null; + for (const name of (await readdir(migrationsDir)).filter((n) => n.endsWith('.sql'))) { + const match = MARKER.exec(await readFile(join(migrationsDir, name), 'utf8')); + if (match?.[1]) highest = Math.max(highest ?? 0, Number(match[1])); + } + return highest; +} + +// pg-boss plans run as their own transaction under an advisory lock; the migration runner already applies each +// file in one transaction under its lock, so the wrapper goes. Every function pins search_path (AGENTS.md). +export function adaptPlans(plans: string): string { + let body = plans + .replace(/^\s*BEGIN;\s*$/m, '') + .replace(/^\s*SET LOCAL (?:lock_timeout|idle_in_transaction_session_timeout) = \d+;\s*$/gm, '') + .replace(/^\s*SELECT pg_advisory_xact_lock\(.*\);\s*$/m, '') + .replace(/^\s*COMMIT;\s*$/m, ''); + if (/^\s*(?:BEGIN|COMMIT|ROLLBACK);/m.test(body)) throw new Error('pg-boss plans contain more than one transaction'); + const functions = body.match(/CREATE (?:OR REPLACE )?FUNCTION /g)?.length ?? 0; + let pinned = 0; + body = body.replace(/(\n\s*LANGUAGE (?:sql|plpgsql)\b[^;\n]*);/g, (_, head: string) => { + pinned += 1; + return `${head} ${PIN};`; + }); + if (pinned !== functions) throw new Error(`pinned search_path on ${pinned} of ${functions} pg-boss functions`); + if (body.includes('$pgboss$')) throw new Error('pg-boss plans contain the $pgboss$ quote tag'); + return body.trim(); +} + +const GRANTS = `-- The worker (pg-boss and the outbox relay) uses crove_jobs; nobody else may. +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA ${JOB_SCHEMA} FROM PUBLIC; +GRANT USAGE ON SCHEMA ${JOB_SCHEMA} TO crove_worker; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA ${JOB_SCHEMA} TO crove_worker; +-- The version row decides whether a vendored upgrade runs, so only migrations write it. pg-boss started with +-- migrate: false only stamps its maintenance columns. +REVOKE INSERT, UPDATE, DELETE, TRUNCATE ON ${JOB_SCHEMA}.version FROM crove_worker; +GRANT UPDATE (cron_on, bam_on, flow_on, reindex_on, monitor_backoff_on) ON ${JOB_SCHEMA}.version TO crove_worker; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA ${JOB_SCHEMA} TO crove_worker; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA ${JOB_SCHEMA} TO crove_worker; +ALTER DEFAULT PRIVILEGES IN SCHEMA ${JOB_SCHEMA} GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO crove_worker;`; + +// Renders the migration that brings crove_jobs to `target`: pg-boss's construction plans on a fresh database +// (`from` null), else its migration plans from `from`. Re-applying it is a no-op; any other version stops it. +export function renderJobsMigration(target: number, from: number | null, queues: readonly string[]): string { + const plans = from === null ? getConstructionPlans(JOB_SCHEMA) : getMigrationPlans(JOB_SCHEMA, from); + // The schema holds exactly one version row; anything else stops the migration. + const versions = `(SELECT array_agg(version) FROM ${JOB_SCHEMA}.version)`; + const held = `(SELECT string_agg(version::text, ', ') FROM ${JOB_SCHEMA}.version)`; + const found = + from === null + ? `IF ${versions} = ARRAY[${target}] THEN + RETURN; + END IF; + RAISE EXCEPTION '${JOB_SCHEMA} holds pg-boss schema version %, this migration installs ${target}', ${held};` + : `IF ${versions} = ARRAY[${target}] THEN + RETURN; + END IF; + IF ${versions} IS DISTINCT FROM ARRAY[${from}] THEN + RAISE EXCEPTION '${JOB_SCHEMA} holds pg-boss schema version %, this migration upgrades ${from} to ${target}', ${held}; + END IF; + -- pg-boss's upgrade plans put queue.table_name into SQL as is, and the worker may write that column. + IF EXISTS (SELECT 1 FROM ${JOB_SCHEMA}.queue WHERE table_name !~ '^(job_common|j[0-9a-f]{56})$') THEN + RAISE EXCEPTION '${JOB_SCHEMA}.queue holds a table name pg-boss does not create'; + END IF;`; + const missing = from === null ? `\n ELSE\n EXECUTE $pgboss$\n${adaptPlans(plans)}\n$pgboss$;` : ''; + const upgrade = from === null ? '' : `\n EXECUTE $pgboss$\n${adaptPlans(plans)}\n$pgboss$;`; + // One create_queue statement per set of options, so each queue gets the policy the worker expects. + const byOptions = new Map(); + for (const queue of queues) byOptions.set(queueOptions(queue), [...(byOptions.get(queueOptions(queue)) ?? []), `'${queue.replaceAll("'", "''")}'`]); + const createQueues = [...byOptions] + .map(([options, names]) => `SELECT ${JOB_SCHEMA}.create_queue(q, '${options}'::jsonb) FROM unnest(ARRAY[${names.join(', ')}]) AS q;`) + .join('\n'); + return `-- pg-boss schema version: ${target} +-- Generated by \`pnpm db:vendor:pgboss\` from pg-boss ${from === null ? 'getConstructionPlans' : `getMigrationPlans (from ${from})`}('${JOB_SCHEMA}'). +-- Do not edit: upgrade pg-boss, run the script again and commit the new migration it writes. +-- Changes to pg-boss's SQL: no BEGIN, COMMIT or advisory lock (the runner applies this file in one transaction +-- under its own lock), and every function pins search_path. Before #25 the migrate role let pg-boss create the +-- schema at runtime, so an existing database at version ${target} keeps its tables and only gets the pins. +DO $vendor$ +BEGIN + IF to_regclass('${JOB_SCHEMA}.version') IS NOT NULL THEN + ${found}${missing} + END IF;${upgrade} +END +$vendor$; + +-- A schema pg-boss created at runtime (before #25) has functions without a pinned search_path. The migration role +-- created them then, so it owns them now. +DO $pin$ +DECLARE + fn regprocedure; +BEGIN + FOR fn IN + SELECT p.oid::regprocedure FROM pg_catalog.pg_proc p + WHERE p.pronamespace = '${JOB_SCHEMA}'::regnamespace + AND p.proconfig IS DISTINCT FROM ARRAY['search_path=pg_catalog, pg_temp'] + LOOP + EXECUTE pg_catalog.format('ALTER FUNCTION %s ${PIN}', fn); + END LOOP; +END +$pin$; +${from === null ? ` +-- Queues the worker uses (QUEUES in apps/kernel/src/jobs-schema.ts); create_queue skips existing ones. +${createQueues} + +-- /readyz (the API role) compares this with the version its pg-boss expects. +CREATE OR REPLACE FUNCTION crove_kernel.jobs_schema_version() +RETURNS integer LANGUAGE sql STABLE SECURITY DEFINER ${PIN} AS $$ + SELECT max(v.version) FROM ${JOB_SCHEMA}.version v +$$; +REVOKE ALL ON FUNCTION crove_kernel.jobs_schema_version() FROM PUBLIC; +GRANT EXECUTE ON FUNCTION crove_kernel.jobs_schema_version() TO crove_app; +` : ''} +${GRANTS} +`; +} diff --git a/apps/kernel/src/main.ts b/apps/kernel/src/main.ts index ebc855c8..dc86a24e 100644 --- a/apps/kernel/src/main.ts +++ b/apps/kernel/src/main.ts @@ -1,5 +1,5 @@ // Crove kernel entry point. One image, several roles: -// migrate apply SQL migrations, set role passwords, create the job schema +// migrate apply SQL migrations (the job schema included) and set role passwords // auth-grants after Supabase Auth has started: let the session check read auth.sessions, auth.users and // auth.mfa_factors (user_id, status) // api HTTP API + web UI @@ -18,7 +18,6 @@ import { runPilotCheck } from './pilot-check.ts'; import { runProber } from './prober.ts'; import { describeStorage } from './storage.ts'; import { runUsageReport } from './usage-report.ts'; -import { migrateJobs } from './worker.ts'; type Role = 'migrate' | 'auth-grants' | 'prober' | 'usage-report' | 'pilot-check' | ServerRole; const ROLES: readonly Role[] = ['migrate', 'auth-grants', 'api', 'worker', 'all', 'prober', 'usage-report', 'pilot-check']; @@ -34,7 +33,6 @@ async function runMigrate(config: Config) { crove_reporter: config.CROVE_REPORTER_DB_PASSWORD, supabase_auth_admin: config.AUTH_DB_PASSWORD, }); - await migrateJobs(adminUrl); console.log(`[migrate] done, ${applied.length} new migration(s)`); } diff --git a/apps/kernel/src/vendor-pgboss.ts b/apps/kernel/src/vendor-pgboss.ts new file mode 100644 index 00000000..a9258ab5 --- /dev/null +++ b/apps/kernel/src/vendor-pgboss.ts @@ -0,0 +1,21 @@ +// `pnpm db:vendor:pgboss`: when the pinned pg-boss expects a schema version no committed migration vendors yet, +// writes the migration that installs (first time) or upgrades crove_jobs to it. Commit the file it writes. +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { QUEUES, pgBossSchemaVersion, renderJobsMigration, vendoredSchemaVersion } from './jobs-schema.ts'; + +const MIGRATIONS = process.env.MIGRATIONS_DIR ?? fileURLToPath(new URL('../../../supabase/migrations', import.meta.url)); + +const target = pgBossSchemaVersion(); +const vendored = await vendoredSchemaVersion(MIGRATIONS); +if (vendored === target) { + console.log(`crove_jobs is vendored at pg-boss schema version ${target}; nothing to do`); +} else if (vendored !== null && vendored > target) { + throw new Error(`migrations vendor pg-boss schema version ${vendored}, but the installed pg-boss expects ${target}; upgrade pg-boss`); +} else { + const stamp = new Date().toISOString().replace(/\D/g, '').slice(0, 14); + const file = join(MIGRATIONS, `${stamp}_crove_jobs_pgboss_v${target}.sql`); + await writeFile(file, renderJobsMigration(target, vendored, Object.values(QUEUES))); + console.log(`wrote ${file}${vendored === null ? '' : ` (upgrade from ${vendored})`}`); +} diff --git a/apps/kernel/src/worker.ts b/apps/kernel/src/worker.ts index 7622ab06..3378990c 100644 --- a/apps/kernel/src/worker.ts +++ b/apps/kernel/src/worker.ts @@ -11,25 +11,11 @@ import type { ExtractClient } from '@crove/extract-contract'; import { extractAndPropose, recordExtractionFailure } from '@crove/mod-finance'; import { processInboundEmail, type InboundJob } from './inbound.ts'; import { ExportSealer } from './export-seal.ts'; +import { JOB_SCHEMA, QUEUES } from './jobs-schema.ts'; import { DEFAULT_EXPORT_RETENTION_DAYS, runOrgExport } from './org-export.ts'; import { RETENTION_CRON, RETENTION_TZ, expireExports, runRetention, sweepInboundBlobs, sweepStaging } from './retention.ts'; import type { SessionRevoker, SignOutTarget } from './sessions.ts'; -export const JOB_SCHEMA = 'crove_jobs'; -export const QUEUES = { - extract: 'finance-file-extract', - inbound: 'kernel-inbound-email', - signOut: 'kernel-session-signout', - orgExport: 'kernel-org-export', - retention: 'kernel-retention-purge', - agentDispatch: 'kernel-agent-dispatch', - agentWake: 'kernel-agent-wake', - agentSweep: 'kernel-agent-sweep', -} as const; -// Queue options by queue. kernel-agent-wake is 'stately' with the singleton key :: at most one queued and -// one active job per agent, so wake-ups coalesce and an agent runs once at a time (ADR-0029 section 5; the database -// also allows one running ai_run per agent). -const QUEUE_OPTIONS: Partial> = { [QUEUES.agentWake]: { policy: 'stately' } }; const AGENT_RETRY_LIMIT = 3; // Every 10 minutes the worker wakes agents whose wake-up is still pending and that can run now // (crove_kernel.agent_wakeups_due): a run refused earlier by the daily cap, a kill switch or a budget that has since @@ -397,24 +383,7 @@ export async function startWorker(deps: WorkerDeps): Promise<() => Promise }; } -// Creates the pg-boss schema and queues with an admin connection, then grants the worker role. -export async function migrateJobs(adminUrl: string): Promise { - const boss = new PgBoss({ connectionString: adminUrl, schema: JOB_SCHEMA, migrate: true, supervise: false, schedule: false }); - boss.on('error', (error) => console.error('[pg-boss migrate]', error)); - await boss.start(); - for (const queue of Object.values(QUEUES)) await boss.createQueue(queue, QUEUE_OPTIONS[queue] ?? {}); - await boss.stop({ graceful: false }); - const client = new pg.Client({ connectionString: adminUrl, application_name: 'crove-migrate' }); - await client.connect(); - try { - await client.query(` - GRANT USAGE ON SCHEMA ${JOB_SCHEMA} TO crove_worker; - GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA ${JOB_SCHEMA} TO crove_worker; - GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA ${JOB_SCHEMA} TO crove_worker; - GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA ${JOB_SCHEMA} TO crove_worker; - ALTER DEFAULT PRIVILEGES IN SCHEMA ${JOB_SCHEMA} GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO crove_worker; - `); - } finally { - await client.end(); - } -} +// Kept only because the invariant suite on main still calls it, and protected.yml runs main's copy of that suite +// against this code. The schema now comes from the vendored migration (#25), so this does nothing; delete it +// together with those calls once they are gone from main. +export async function migrateJobs(_adminUrl: string): Promise {} diff --git a/apps/kernel/test/blob-sweep.db.test.ts b/apps/kernel/test/blob-sweep.db.test.ts index d675b933..9e74ba26 100644 --- a/apps/kernel/test/blob-sweep.db.test.ts +++ b/apps/kernel/test/blob-sweep.db.test.ts @@ -12,7 +12,8 @@ import { createDb, createPool, type Db } from '@crove/db'; import { migrate, setRolePasswords } from '@crove/db/migrate'; import { BlobNotFoundError, LocalDiskStore, inboundKey, newInboundBlobId, sha256Hex } from '@crove/kernel-runtime'; import { INBOUND_BLOB_MAX_AGE_DAYS, sweepInboundBlobs } from '../src/retention.ts'; -import { QUEUES, migrateJobs } from '../src/worker.ts'; +import { QUEUES } from '../src/jobs-schema.ts'; +import { migrateJobs } from '../src/worker.ts'; const ADMIN = process.env.CROVE_TEST_ADMIN_URL; const MIGRATIONS = fileURLToPath(new URL('../../../supabase/migrations', import.meta.url)); diff --git a/apps/kernel/test/inbound-rotation.db.test.ts b/apps/kernel/test/inbound-rotation.db.test.ts index f5f5320a..6d1fbc93 100644 --- a/apps/kernel/test/inbound-rotation.db.test.ts +++ b/apps/kernel/test/inbound-rotation.db.test.ts @@ -17,7 +17,7 @@ import { financeCommands, financeQueries } from '@crove/mod-finance'; import type { AuthUser } from '../src/auth.ts'; import { loadConfig } from '../src/config.ts'; import { createHttpApp } from '../src/http.ts'; -import { migrateJobs, startWorker } from '../src/worker.ts'; +import { startWorker } from '../src/worker.ts'; import { startExtractService } from './extract-service.ts'; const ADMIN = process.env.CROVE_TEST_ADMIN_URL; @@ -248,12 +248,7 @@ suite('inbound address rotation on PostgreSQL', () => { } }; // Starts the real worker, runs fn, stops it. The job schema is installed once. - let jobsReady = false; async function withWorker(fn: () => Promise) { - if (!jobsReady) { - await migrateJobs(adminUrl); - jobsReady = true; - } const stop = await startWorker({ connectionString: workerUrl, workerDb, runtime: workerRuntime, store, extractor: null, extract, inboundDomain: 'in.test' }); try { await fn(); diff --git a/apps/kernel/test/jobs-schema.test.ts b/apps/kernel/test/jobs-schema.test.ts new file mode 100644 index 00000000..069d701c --- /dev/null +++ b/apps/kernel/test/jobs-schema.test.ts @@ -0,0 +1,45 @@ +import { fileURLToPath } from 'node:url'; +import { getConstructionPlans } from 'pg-boss'; +import { describe, expect, it } from 'vitest'; +import { QUEUES, adaptPlans, pgBossSchemaVersion, queueOptions, renderJobsMigration, vendoredSchemaVersion } from '../src/jobs-schema.ts'; + +const MIGRATIONS = fileURLToPath(new URL('../../../supabase/migrations', import.meta.url)); + +describe('vendored pg-boss schema (External-tenant gate T6)', () => { + it('matches the schema version the pinned pg-boss expects', async () => { + // A pg-boss upgrade that changes the schema fails here until `pnpm db:vendor:pgboss` adds its migration. + expect(await vendoredSchemaVersion(MIGRATIONS)).toBe(pgBossSchemaVersion()); + }); + + it('drops the transaction wrapper and pins search_path on every function', () => { + const body = adaptPlans(getConstructionPlans('crove_jobs')); + expect(body).not.toMatch(/^\s*(BEGIN|COMMIT);/m); + expect(body).not.toContain('pg_advisory_xact_lock'); + const functions = body.match(/CREATE (?:OR REPLACE )?FUNCTION /g)?.length ?? 0; + expect(functions).toBeGreaterThan(0); + expect(body.match(/SET search_path = pg_catalog, pg_temp;/g)?.length).toBe(functions); + }); + + it('creates every queue with the policy the worker expects', () => { + const sql = renderJobsMigration(pgBossSchemaVersion(), null, Object.values(QUEUES)); + const created = [...sql.matchAll(/create_queue\(q, '(\{[^']*\})'::jsonb\) FROM unnest\(ARRAY\[([^\]]*)\]\)/g)].flatMap(([, options, names]) => + (names ?? '').split(', ').map((name) => [name.slice(1, -1), JSON.parse(options ?? '{}').policy as string]), + ); + expect(Object.fromEntries(created)).toEqual( + Object.fromEntries(Object.values(QUEUES).map((name) => [name, name === QUEUES.agentWake ? 'stately' : 'standard'])), + ); + expect(queueOptions(QUEUES.agentWake)).toBe('{"policy":"stately"}'); + expect(queueOptions(QUEUES.extract)).toBe('{"policy":"standard"}'); + }); + + it('renders an upgrade that refuses any starting version but its own', () => { + const target = pgBossSchemaVersion(); + const sql = renderJobsMigration(target, target - 1, Object.values(QUEUES)); + expect(sql.split('\n')[0]).toBe(`-- pg-boss schema version: ${target}`); + expect(sql).toContain(`IS DISTINCT FROM ARRAY[${target - 1}]`); + expect(sql).toContain(`upgrades ${target - 1} to ${target}`); + // pg-boss's upgrade plans read queue.table_name into SQL; the worker can write it. + expect(sql).toContain("table_name !~ '^(job_common|j[0-9a-f]{56})$'"); + expect(sql).not.toMatch(/^\s*(BEGIN|COMMIT);/m); + }); +}); diff --git a/apps/kernel/test/jobs.db.test.ts b/apps/kernel/test/jobs.db.test.ts new file mode 100644 index 00000000..a87e948d --- /dev/null +++ b/apps/kernel/test/jobs.db.test.ts @@ -0,0 +1,155 @@ +// The crove_jobs schema comes only from committed migrations (External-tenant gate T6). Set CROVE_TEST_ADMIN_URL +// to a superuser URL; each test creates and drops its own database. +import { fileURLToPath } from 'node:url'; +import pg from 'pg'; +import { PgBoss } from 'pg-boss'; +import { afterAll, describe, expect, it } from 'vitest'; +import { migrate, setRolePasswords } from '@crove/db/migrate'; +import { JOB_SCHEMA, QUEUES, pgBossSchemaVersion } from '../src/jobs-schema.ts'; + +const ADMIN = process.env.CROVE_TEST_ADMIN_URL; +const MIGRATIONS = fileURLToPath(new URL('../../../supabase/migrations', import.meta.url)); +const suite = ADMIN ? describe : describe.skip; +const created: string[] = []; + +async function withClient(url: string, fn: (client: pg.Client) => Promise): Promise { + const client = new pg.Client({ connectionString: url }); + await client.connect(); + try { + return await fn(client); + } finally { + await client.end(); + } +} + +async function createDatabase(): Promise { + const name = `crove_test_jobs_${Date.now()}_${Math.floor(Math.random() * 1e6)}`; + await withClient(ADMIN as string, (admin) => admin.query(`CREATE DATABASE ${name}`)); + created.push(name); + const url = new URL(ADMIN as string); + url.pathname = `/${name}`; + return url.toString(); +} + +function loginUrl(adminUrl: string, username: string, password: string): string { + const url = new URL(adminUrl); + url.username = username; + url.password = password; + return url.toString(); +} + +suite('crove_jobs from committed migrations', () => { + afterAll(async () => { + if (!ADMIN) return; + for (const name of created) await withClient(ADMIN, (admin) => admin.query(`DROP DATABASE IF EXISTS ${name} WITH (FORCE)`)); + }); + + it('installs the expected version and queues; only the worker may use the schema', async () => { + const adminUrl = await createDatabase(); + await migrate({ adminUrl, dir: MIGRATIONS }); + await withClient(adminUrl, async (admin) => { + const version = await admin.query<{ version: number }>(`SELECT max(version) AS version FROM ${JOB_SCHEMA}.version`); + expect(version.rows[0]?.version).toBe(pgBossSchemaVersion()); + const queues = await admin.query<{ name: string; policy: string }>(`SELECT name, policy FROM ${JOB_SCHEMA}.queue ORDER BY name`); + expect(queues.rows).toEqual( + Object.values(QUEUES) + .sort() + .map((name) => ({ name, policy: name === QUEUES.agentWake ? 'stately' : 'standard' })), + ); + + const functions = await admin.query<{ fn: string; pinned: boolean; public: boolean; worker: boolean; app: boolean }>(` + SELECT p.oid::regprocedure::text AS fn, + coalesce('search_path=pg_catalog, pg_temp' = ANY (p.proconfig), false) AS pinned, + EXISTS (SELECT 1 FROM aclexplode(p.proacl) a WHERE a.grantee = 0) AS public, + has_function_privilege('crove_worker', p.oid, 'EXECUTE') AS worker, + has_function_privilege('crove_app', p.oid, 'EXECUTE') AS app + FROM pg_proc p WHERE p.pronamespace = '${JOB_SCHEMA}'::regnamespace ORDER BY 1`); + expect(functions.rows.length).toBeGreaterThan(0); + for (const row of functions.rows) expect(row, row.fn).toMatchObject({ pinned: true, public: false, worker: true, app: false }); + + const schema = await admin.query<{ worker: boolean; app: boolean; worker_create: boolean }>(` + SELECT has_schema_privilege('crove_worker', '${JOB_SCHEMA}', 'USAGE') AS worker, + has_schema_privilege('crove_app', '${JOB_SCHEMA}', 'USAGE') AS app, + has_schema_privilege('crove_worker', '${JOB_SCHEMA}', 'CREATE') AS worker_create`); + expect(schema.rows).toEqual([{ worker: true, app: false, worker_create: false }]); + }); + + // The API role sees only the version, for /readyz. + await setRolePasswords(adminUrl, { crove_app: 'test-app-pw', crove_worker: 'test-worker-pw' }); + await withClient(loginUrl(adminUrl, 'crove_app', 'test-app-pw'), async (app) => { + const { rows } = await app.query<{ version: number }>('SELECT crove_kernel.jobs_schema_version() AS version'); + expect(rows[0]?.version).toBe(pgBossSchemaVersion()); + await expect(app.query(`SELECT 1 FROM ${JOB_SCHEMA}.queue`)).rejects.toThrow(/permission denied/); + }); + + // pg-boss as the worker, with migrate: false: starts on the vendored schema, sends and fetches a job. + const boss = new PgBoss({ connectionString: loginUrl(adminUrl, 'crove_worker', 'test-worker-pw'), schema: JOB_SCHEMA, migrate: false, createSchema: false, supervise: false }); + boss.on('error', () => {}); + await boss.start(); + try { + const id = await boss.send(QUEUES.extract, { probe: true }); + const [job] = await boss.fetch<{ probe: boolean }>(QUEUES.extract); + expect(job?.id).toBe(id); + expect(job?.data).toEqual({ probe: true }); + } finally { + await boss.stop({ graceful: false }); + } + + // The version row decides whether a vendored upgrade runs, so only migrations write it; pg-boss with + // migrate: false stamps the maintenance columns (cron, BAM, flow, reindex, monitor back-off). + await withClient(loginUrl(adminUrl, 'crove_worker', 'test-worker-pw'), async (worker) => { + await expect(worker.query(`UPDATE ${JOB_SCHEMA}.version SET version = version + 1`)).rejects.toThrow(/permission denied/); + await expect(worker.query(`INSERT INTO ${JOB_SCHEMA}.version (version) VALUES (0)`)).rejects.toThrow(/permission denied/); + await expect(worker.query(`DELETE FROM ${JOB_SCHEMA}.version`)).rejects.toThrow(/permission denied/); + const { rows } = await worker.query( + `UPDATE ${JOB_SCHEMA}.version SET cron_on = now(), bam_on = now(), flow_on = now(), reindex_on = now(), monitor_backoff_on = now() + WHERE version = ${pgBossSchemaVersion()} RETURNING true AS stamped`, + ); + expect(rows).toEqual([{ stamped: true }]); + }); + }); + + it('leaves a schema pg-boss already created at the same version in place', async () => { + const adminUrl = await createDatabase(); + // Before #25 the migrate role let pg-boss create crove_jobs at runtime. + const boss = new PgBoss({ connectionString: adminUrl, schema: JOB_SCHEMA, migrate: true, supervise: false, schedule: false }); + boss.on('error', () => {}); + await boss.start(); + await boss.createQueue(QUEUES.extract); + await boss.stop({ graceful: false }); + + await withClient(adminUrl, async (admin) => { + const { rows } = await admin.query<{ n: number }>(`SELECT count(*)::int AS n FROM pg_proc WHERE pronamespace = '${JOB_SCHEMA}'::regnamespace AND proconfig IS NULL`); + expect(rows[0]?.n).toBeGreaterThan(0); + }); + + await migrate({ adminUrl, dir: MIGRATIONS }); + await withClient(adminUrl, async (admin) => { + const { rows } = await admin.query<{ n: number }>(`SELECT count(*)::int AS n FROM ${JOB_SCHEMA}.queue`); + expect(rows[0]?.n).toBe(Object.values(QUEUES).length); + // The functions pg-boss created get the pinned search_path, like the vendored ones. + const unpinned = await admin.query<{ fn: string }>( + `SELECT p.oid::regprocedure::text AS fn FROM pg_proc p + WHERE p.pronamespace = '${JOB_SCHEMA}'::regnamespace AND p.proconfig IS DISTINCT FROM ARRAY['search_path=pg_catalog, pg_temp']`, + ); + expect(unpinned.rows).toEqual([]); + }); + }); + + it('stops on a crove_jobs schema with more than one version row', async () => { + const adminUrl = await createDatabase(); + const target = pgBossSchemaVersion(); + await withClient(adminUrl, (admin) => + admin.query(`CREATE SCHEMA ${JOB_SCHEMA}; CREATE TABLE ${JOB_SCHEMA}.version (version int PRIMARY KEY); INSERT INTO ${JOB_SCHEMA}.version VALUES (${target}), (${target + 1})`), + ); + await expect(migrate({ adminUrl, dir: MIGRATIONS })).rejects.toThrow(`holds pg-boss schema version ${target}, ${target + 1}`); + }); + + it('stops on a crove_jobs schema at another version', async () => { + const adminUrl = await createDatabase(); + await withClient(adminUrl, (admin) => + admin.query(`CREATE SCHEMA ${JOB_SCHEMA}; CREATE TABLE ${JOB_SCHEMA}.version (version int PRIMARY KEY); INSERT INTO ${JOB_SCHEMA}.version VALUES (${pgBossSchemaVersion() - 1})`), + ); + await expect(migrate({ adminUrl, dir: MIGRATIONS })).rejects.toThrow(`holds pg-boss schema version ${pgBossSchemaVersion() - 1}`); + }); +}); diff --git a/apps/kernel/test/kernel.db.test.ts b/apps/kernel/test/kernel.db.test.ts index 2776c0ee..d0b6ee0a 100644 --- a/apps/kernel/test/kernel.db.test.ts +++ b/apps/kernel/test/kernel.db.test.ts @@ -17,7 +17,7 @@ import { extractAndPropose, financeCommands, financeQueries, recordExtractionFai import { loadConfig } from '../src/config.ts'; import { createHttpApp } from '../src/http.ts'; import { processInboundEmail } from '../src/inbound.ts'; -import { migrateJobs, startWorker } from '../src/worker.ts'; +import { startWorker } from '../src/worker.ts'; import { startExtractService } from './extract-service.ts'; const ADMIN = process.env.CROVE_TEST_ADMIN_URL; @@ -559,9 +559,8 @@ suite('kernel on PostgreSQL', () => { await withAdmin((admin) => admin.query(`DELETE FROM crove_kernel.inbound_pending WHERE size_bytes = 1`)); expect(await pending()).toBe(1); - await migrateJobs(adminUrl); - // pg-boss functions are SECURITY INVOKER, which is why the hardening guard skips them; a rerun of the - // runner (and its guard) passes with the job schema installed. + // pg-boss functions (from the vendored crove_jobs migration) are SECURITY INVOKER, which is why the hardening + // guard skips them; a rerun of the runner (and its guard) passes with the job schema installed. await withAdmin(async (admin) => { const { rows } = await admin.query<{ total: number; definers: number }>( `SELECT count(*)::int AS total, count(*) FILTER (WHERE prosecdef)::int AS definers FROM pg_proc WHERE pronamespace = 'crove_jobs'::regnamespace`, diff --git a/apps/kernel/test/module-enablement.db.test.ts b/apps/kernel/test/module-enablement.db.test.ts index 5e57fc21..fecc7183 100644 --- a/apps/kernel/test/module-enablement.db.test.ts +++ b/apps/kernel/test/module-enablement.db.test.ts @@ -15,8 +15,8 @@ import { createOrg, upsertIdentity } from '@crove/kernel'; import { DomainError, Runtime, defineCommand, defineQuery, sha256Hex, type BlobStore, type Lookup } from '@crove/kernel-runtime'; import type { AuthUser } from '../src/auth.ts'; import { loadConfig } from '../src/config.ts'; +import { QUEUES } from '../src/jobs-schema.ts'; import { createKernel, defaultAdapters, moduleDefinitions, type IdentityPort } from '../src/kernel.ts'; -import { QUEUES, migrateJobs } from '../src/worker.ts'; import { startExtractService } from './extract-service.ts'; const ADMIN = process.env.CROVE_TEST_ADMIN_URL; @@ -111,7 +111,6 @@ suite('module enablement on PostgreSQL (plan Decision 12, ADR-0031)', () => { adminUrl = url.toString(); await migrate({ adminUrl, dir: MIGRATIONS }); await setRolePasswords(adminUrl, { crove_app: 'test-app-pw', crove_worker: 'test-worker-pw' }); - await migrateJobs(adminUrl); const loginUrl = (username: string, password: string) => { const u = new URL(adminUrl); u.username = username; diff --git a/apps/kernel/test/prober.db.test.ts b/apps/kernel/test/prober.db.test.ts index 8ce18603..c8b1686f 100644 --- a/apps/kernel/test/prober.db.test.ts +++ b/apps/kernel/test/prober.db.test.ts @@ -13,7 +13,8 @@ import { migrate, setRolePasswords } from '@crove/db/migrate'; import { createOrg, kernelCommands, kernelQueries, upsertIdentity } from '@crove/kernel'; import { Runtime } from '@crove/kernel-runtime'; import { EXIT, evaluateHealth, probeOnce, readHealth, runProber, type ProberConfig } from '../src/prober.ts'; -import { JOB_SCHEMA, QUEUES, beat, migrateJobs } from '../src/worker.ts'; +import { JOB_SCHEMA, QUEUES } from '../src/jobs-schema.ts'; +import { beat } from '../src/worker.ts'; const ADMIN = process.env.CROVE_TEST_ADMIN_URL; const MIGRATIONS = fileURLToPath(new URL('../../../supabase/migrations', import.meta.url)); @@ -105,26 +106,40 @@ suite('prober on PostgreSQL', () => { await root.end(); }); - it('reports a missing job schema and a missing heartbeat before the worker ever ran', async () => { + it('reports a missing heartbeat before the worker ever ran; the job schema comes with the migrations', async () => { const health = await readHealth(proberUrl, 3600); expect(health).toMatchObject({ outbox_pending: 0, outbox_oldest_age_seconds: null, inbound_pending: 0, inbound_oldest_age_seconds: null, - jobs_schema_present: false, - jobs_queued: null, + jobs_schema_present: true, + jobs_queued: 0, job_oldest_queued_age_seconds: null, - dead_letter_count: null, + dead_letter_count: 0, worker_heartbeat_age_seconds: null, // pg_notification_queue_usage(), which crove_prober may call like any core function (issue #281). notify_queue_usage: 0, }); - expect(evaluateHealth(health, limits, 3600).map((f) => f.check)).toEqual(['jobs_schema', 'heartbeat']); + expect(evaluateHealth(health, limits, 3600).map((f) => f.check)).toEqual(['heartbeat']); }); - it('is healthy once the job schema exists, the relay is current and the worker beats', async () => { - await migrateJobs(adminUrl); + it('reports a missing job schema', async () => { + // The vendored migration creates crove_jobs (#25); hide it to see what an incomplete migrate looks like. + const admin = new pg.Client({ connectionString: adminUrl }); + await admin.connect(); + try { + await admin.query(`ALTER SCHEMA ${JOB_SCHEMA} RENAME TO ${JOB_SCHEMA}_hidden`); + const health = await readHealth(proberUrl, 3600); + expect(health).toMatchObject({ jobs_schema_present: false, jobs_queued: null, job_oldest_queued_age_seconds: null, dead_letter_count: null }); + expect(evaluateHealth(health, limits, 3600).map((f) => f.check)).toEqual(['jobs_schema', 'heartbeat']); + } finally { + await admin.query(`ALTER SCHEMA ${JOB_SCHEMA}_hidden RENAME TO ${JOB_SCHEMA}`); + await admin.end(); + } + }); + + it('is healthy once the relay is current and the worker beats', async () => { await beat(workerDb, 'worker-a'); const health = await readHealth(proberUrl, 3600); expect(health).toMatchObject({ outbox_pending: 0, outbox_oldest_age_seconds: null, jobs_schema_present: true, jobs_queued: 0, dead_letter_count: 0 }); diff --git a/docs/plan/crove-os-kernel.md b/docs/plan/crove-os-kernel.md index e1114237..947c4cfd 100644 --- a/docs/plan/crove-os-kernel.md +++ b/docs/plan/crove-os-kernel.md @@ -118,7 +118,7 @@ Crove Cloud accepts its first external tenant only when every row below is green | T3 | Ledger immutability: only the posting path (a definer) writes journal entries and lines; a posted entry cannot gain lines; every entry has at least 2 lines and balances at commit | Database tests as the module and login roles: direct INSERT into `journal_entry` or `journal_line` is denied; adding a line to an entry committed earlier fails; an entry with 0 or 1 line, or unbalanced, fails at commit; reversal nets to zero; a fast-check property test of the posting path | Done (#71, #76, #138). In #71, with database tests: `crove_finance.post_journal_entry`, owned by `crove_definer_tenant`, is the only writer (tenant roles lost INSERT on both tables); a `created_xact_id` stamp rejects lines added to an entry from any later transaction; entries with no lines and unbalanced entries are rejected at commit. Done in #76, with database tests (`apps/kernel/test/invariant/ledger-visibility.db.test.ts`, under a test-only RESTRICTIVE policy that hides some lines from every tenant role): the balance and line-count checks (`require_entry_balanced`, called by the commit trigger `check_entry_balanced` and by `assert_entry_balanced`) are definers owned by the migration runner, which owns `journal_line` and is not filtered by its row security (a migration guard fails otherwise), so they count every line of the one entry they check; no runtime role can call them, and `assert_entry_balanced` is callable only by `crove_definer_tenant` and only for the org in the sealed context; an entry that balances only over the visible lines is refused, a balanced entry with hidden lines is accepted, and an entry with exactly 1 line is refused; the refusal reports the difference over every line, which is safe only because a line can join an entry only in the transaction that created it, so the caller supplied every summed line (a future feature that appends lines to an existing entry must drop the amount from the error first); the posted-entry rule also compares `created_xact_start`, so an `xid8` repeated after a logical restore does not reopen an entry (§7.9). Done in #138, with database tests (`apps/kernel/test/invariant/ledger-reversal.db.test.ts`, `ledger-posting-property.db.test.ts`): a reversal is a new entry posted through `post_journal_entry` (an overload that takes `reverses_entry_id`; the original signature calls it), whose lines mirror the original's (same line number, account, party and memo, debit and credit swapped), linked by `reverses_entry_id` (composite FK `(org_id, reverses_entry_id)` and a unique index, so an entry is reversed at most once; of two concurrent reversals the second waits on the index and fails); it never adds lines to an existing entry, so the balance error above stays safe; a reversal cannot itself be reversed (to restore the original effect, post a new entry) and is never dated before the original; the commit trigger on `journal_entry` (once per entry) and `assert_entry_balanced` also run `require_entry_mirrors`, a definer owned by the migration runner that compares every line of both entries and reports no amounts, and each line's trigger checks that line against the original's line with the same number (so running the checks early with `SET CONSTRAINTS ... IMMEDIATE` and adding lines afterwards is refused too); the self-FK is `ON DELETE NO ACTION`, so deleting an original that has a reversal fails; so the rules hold for every writer, superusers included, and nobody can reverse an entry with lines they cannot see; the pair nets to zero per account. A reversal is a posting, so it runs only through an approved proposal under the same rules as a purchase (§4.7: R3, P10, P11, P12, approvers `owner` and `admin`): `kernel.proposal.request` (R1, `owner`, `admin` or `member`) runs `finance.journal.prepare_reversal`, which checks the entry and builds the proposal, and `kernel.proposal.approve` runs `finance.journal.reverse` (R3, proposal only). A fast-check property test (fixed seed, 100 runs) posts random line sets as `crove_app` and commits: a set is accepted exactly when it has at least 2 lines and balances. Since #165: the API route `inbox.request` (an idempotency key) and the screens: "Đảo bút toán" on the journal (date, required reason), a reversal card in the Inbox, the reversal link both ways in the journal, and a plain message when the database refuses a partial mirror; reversing a purchase entry voids Crove's record of its document (Phase 2a, Finance) | #19 (closed by #71), #76, #138, #165 | | T4 | Org creation runs through `execute()` with an audit row | Database test: creating an org writes `tenant`, the owner `tenant_member` and an `audit_event` row in one transaction; login roles cannot execute the raw definer (EXECUTE matrix test); a per-user rate limit | Done in #96 and #107, with database tests (`apps/kernel/test/kernel.db.test.ts`, `apps/kernel/test/org-create-limit.db.test.ts`): `kernel.org.create` is an org-less command run by `Runtime.executeOrgless()`. The definer `create_or_replay_org`, executable by `crove_app` alone, creates the tenant and the owner membership (or returns the org an earlier request with the same idempotency key created, serialized by a per-user advisory lock); the runtime then enters that org as the caller and writes the audit row and the idempotency key in the same transaction, and a replay creates nothing. A deferred constraint trigger refuses to commit a tenant without its `kernel.org.create` audit row written by the tenant's creator (the creator check since #102), even for the database owner; the raw `create_org` definer is dropped; the EXECUTE matrix test covers both functions. Since #107 the definer also refuses a new org, under the same lock and so across API instances, when the caller created 5 in the last 24 hours or 20 in total, closed orgs included (`ORG_CREATE_LIMIT_PER_DAY`, `ORG_CREATE_LIMIT_TOTAL`): 429 `RATE_LIMITED`, nothing written; a replay of an existing key does not count; of concurrent requests at the boundary exactly one wins. A refusal has no org to be audited in, so it writes no audit row until `security_event` exists (§4.4 step 3) | #23, #94 | | T5 | Inbound email processing moved out of the API role's system context: the API verifies the shared secret, limits size and stores the raw message; parsing and `kernel.inbound.receive` run in the worker | EXECUTE matrix test: the API login role cannot call `enter_system_context`; integration test: a forwarded e-invoice still yields a proposal, processed by the worker | Done in #78, with database tests (`apps/kernel/test/kernel.db.test.ts`): the API verifies the secret, limits size, stores the raw MIME as `inbound/` and enqueues it through the narrow definer `inbound_enqueue` (202; 503 past 10000 pending messages); the worker checks the blob's size and hash, parses it, runs `kernel.inbound.receive` and deletes the blob once the message is filed or dropped. Only `crove_worker` may execute `enter_system_context`, which also requires `session_user` to be `crove_worker`; only `crove_app` may execute `enter_context`; startup guards refuse an API role that can enter system context and a worker role that cannot. The EXECUTE matrix test and the forwarded e-invoice test (HTTP app, then the real worker, then a proposal) are the evidence | #24 | -| T6 | The pg-boss schema is vendored into committed migrations; no code path runs pg-boss with `migrate: true` | Committed, version-guarded migration for `crove_jobs` (`pnpm db:vendor:pgboss`); CI check that the pinned pg-boss version matches the vendored schema version; migration replay creates `crove_jobs` with no runtime DDL | The `migrate` role starts pg-boss with `migrate: true` (`migrateJobs` in `apps/kernel/src/worker.ts`) | #25 | +| T6 | The pg-boss schema is vendored into committed migrations; no code path runs pg-boss with `migrate: true` | Committed, version-guarded migration for `crove_jobs` (`pnpm db:vendor:pgboss`); CI check that the pinned pg-boss version matches the vendored schema version; migration replay creates `crove_jobs` with no runtime DDL | Done (#25): `pnpm db:vendor:pgboss` renders pg-boss's own plans into `supabase/migrations/*_crove_jobs_pgboss_v46.sql` (no transaction wrapper, every function pins `search_path`, version-guarded `DO` block that leaves an existing version-46 install alone and stops on any other version), which also creates the queues and grants only `crove_worker`. Nothing runs pg-boss with `migrate: true`; the worker starts with `migrate: false`, which refuses another version. `/readyz` reports `jobs_schema` until `crove_kernel.jobs_schema_version()` equals the version the pinned pg-boss expects; a unit test fails CI when a pg-boss upgrade changes the schema version without its migration; database tests cover the fresh install, the existing install, a wrong version and a job round trip as `crove_worker` | #25 | | T7 | An S3-compatible storage adapter, shared by API and worker | `BlobStore` S3 implementation with integration tests against an S3-compatible server in CI; an end-to-end run with `api` and `worker` as separate containers and no shared volume; keys and references per §7.6 | Partly done in #110. `STORAGE=s3` selects `S3BlobStore` (`packages/kernel-runtime/src/s3-store.ts`, AWS SDK v3), built from config by `createBlobStore` in both the `api` and `worker` roles; `STORAGE=local` (the default) keeps `LocalDiskStore`. Both stores accept only `org//` and `inbound/` keys, refuse an org key that is not the sha256 of its bytes, and raise `BlobNotFoundError` for a missing blob. S3 writes are create-only (`If-None-Match: *`; on GCS `x-goog-if-generation-match: 0`) and carry `x-amz-checksum-sha256`, which the server verifies; the same bytes again dedupe, different bytes raise `BlobConflictError`, and the store reads an object back when a server does not confirm the checksum. Local disk writes are create-only too (#111: a temporary file is fsynced and hard linked to the key, so the same bytes dedupe and different bytes raise `BlobConflictError`; without hard links the store falls back to an exclusive create and warns; stale temporary files are swept at startup). The bucket policy must grant `s3:ListBucket`, or a missing key reads as 403 and `get` raises `BlobStoreAccessError`. Bodies are buffered in memory (at most 25 MB for a file). Since #109 (ADR-0013): every file row keeps the triple (key, `generation`, sha256), where `put` returns the generation (`s3:`, `gcs:`, or `content:` on local disk and unversioned buckets); every read requests that version and checks the sha256 (`readVerified`), so a swapped object is never served or parsed (S25); `S3_VERSIONING=required` (default) refuses writes the bucket returns without a version; file rows never change (guard trigger). With `STORAGE=s3` browsers upload through a single-use presigned POST to a random `staging/` key (5 minutes, 1 byte to 25 MB, grant row in `crove_kernel.file_upload`) and download through a 60-second presigned GET of the recorded version; local disk keeps streaming through the kernel, and so does a GCS endpoint until presigned POST is verified there. Tests for #109: mocked-client and offline-signing unit tests, MinIO with versioning (presigned POST, size, key and expiry refused, overwrite still reads and downloads the original), PostgreSQL plus MinIO through the HTTP API (single use, expiry, other users), the invariant suite (`file-upload.db.test.ts`) and the RLS matrix; the split stack runs on a versioned bucket with presigned URLs off, so its downloads stream through `readVerified` with the version pinned. Tests: unit tests with a mocked client and on the real SDK request; integration tests against MinIO in the CI `check` job (`CROVE_TEST_S3_URL`); files written through the API upload and inbound paths read back from a new store instance in a new API app and a new worker call, on local disk and on MinIO (`apps/kernel/test/storage.test.ts`). End-to-end run with `api` and `worker` as separate containers (#108): `docker-compose.split.yml` runs both roles from the same image with `STORAGE=s3` against MinIO and no files volume (only the worker is on the `extract` network); the CI job `split-stack` brings it up and `tools/e2e/split-stack.mjs` uploads the e-invoice and posts an inbound email through the HTTP API, waits until the worker has read both into proposals, and checks that both files are objects in the bucket with their bytes, the raw message was deleted from it, and nothing is under `/data/files` in either container (job `split-stack` in [ci.yml](../../.github/workflows/ci.yml), [first green run](https://github.com/DOS/Crove/actions/runs/38830847915/job/116749026299)). Open: per-object retention holds set by Crove (bucket defaults apply); GCS and R2 not yet run against | #26, #108, #109 | | T8 | Generated cross-org RLS matrix (MVP exit criterion 2) | The CI `db` job generates cases for every tenant table: cross-org read, insert, update and delete denied or 0 rows; no context gives 0 rows; a new tenant table without cases fails CI | Done in #114, in the invariant suite (`apps/kernel/test/invariant/rls-matrix.db.test.ts`, run by the CI `db` job and, from the base branch, by the protected-path job). The cases come from the catalog: every relation with `org_id` in any non-system schema (19 tables today, `crove_kernel` and `crove_finance`) gets its own block. Structural check: a table (not a view), RLS enabled, no runtime role owns it or acts as its owner, no role holds TRUNCATE, REFERENCES or TRIGGER or has BYPASSRLS, and either exactly the `tenant_isolation` policy of `apply_tenant_rls` or, for the listed definer-only machinery tables (`auth_session_org`, `outbox_pending`), no privilege for any role. Probes, each in its own rolled-back transaction, as `crove_app` in a member's context, `crove_worker` in system context, `crove_definer_tenant` in a context, and both login roles with no context: reading org B (0 rows, own rows visible as the control), inserting an org B row (42501 row-level security, or permission denied where the role has no INSERT; the same factory's own-org row is accepted as the control, so a denial is never a malformed row), updating org B rows or moving own rows to org B, updating and deleting with no WHERE clause (only rows the actor sees may change), deleting org B rows and TRUNCATE. A table with `org_id` and no row factory, RLS off, another permissive policy or an unlisted no-access table fails with a message naming it. The matrix found `outbox_pending` with RLS off (no role held a privilege on it, so not reachable); `20261015160000_crove_kernel_outbox_pending_rls.sql` enables it. Open: cross-legal-entity and forged-context cases wait for legal entities and the seal (T11) | #17 | | T9 | Playwright end-to-end test (MVP exit criterion 4) | A required CI job runs `docker compose up` and Playwright through MVP exit criterion 1 (sign up, create org, upload an e-invoice, approve, lists, export) | Done (#18): the `e2e` workflow builds the image, starts `docker compose up` on fresh volumes and runs `e2e/tests/inbox.spec.ts` on every PR: sign up, create org, upload the sample e-invoice, approve, the vendor, document and entry lists, the Excel export, then the vendor, purchase document, balanced journal entry and audit rows read from the database. Open: mark the job required in branch protection (owner) | #18 | @@ -229,7 +229,7 @@ The architecture and plan TL;DR follows. - Postgres enforces: tenancy (RLS + composite FKs), a ledger that always balances and cannot be edited, gapless document numbers, idempotency, append-only logs. - RBAC, business rules and AI policy live in one TypeScript pipeline. 3. **Trust comes only from sealed context.** Org, actor, legal-entity scope and sensitivity flags are derived by `enter_context` and HMAC-signed. No definer, job or tool accepts these values as trusted parameters. -4. **Committed SQL is the only schema.** Idempotent migrations live in `supabase/migrations` of the Crove repo and are applied by the built-in runner (`migrate` role); prod and beta receive schema only through the pipeline. No ORM push, no runtime DDL, no vendor self-migration (one accepted exception: Supabase Auth migrates its own `auth` schema, ADR-0027), no out-of-band DDL. On `main`, pg-boss still creates `crove_jobs` itself (External-tenant gate T6). +4. **Committed SQL is the only schema.** Idempotent migrations live in `supabase/migrations` of the Crove repo and are applied by the built-in runner (`migrate` role); prod and beta receive schema only through the pipeline. No ORM push, no runtime DDL, no vendor self-migration (one accepted exception: Supabase Auth migrates its own `auth` schema, ADR-0027), no out-of-band DDL. Since #25 `crove_jobs` is a vendored migration too (External-tenant gate T6). 5. **Shared tables in Crove's DB, `org_id` everywhere, default-deny roles.** Login roles are NOINHERIT and own nothing. `service_role` and BYPASSRLS never run user, agent or job code. 6. **Modules are packages with contracts, arranged as a DAG `kernel-runtime < kernel < modules`.** - Each module has one schema and one role. @@ -289,13 +289,13 @@ The architecture and plan TL;DR follows. **Client platforms.** Because the app shell is a static SPA and every write goes through the kernel API, desktop and mobile clients need no new backend. PWA, Mobile and Desktop above are direction only, built when their trigger is met. -**State on `main` (2026-10-14).** The MVP already uses pnpm 12 (`packageManager: pnpm@12.11.2`, `catalogMode: strict`), Node 26, TypeScript 7.0.2, oxlint 1.88, Vitest 5, Hono 4, oRPC 1.15, Kysely 0.29, pg-boss 12, Vite 8, TanStack Router and Query, `@supabase/auth-js`, `jose`, `use-intl` (#206), Tailwind 4, shadcn on `@base-ui/react` 1.8 in `@crove/ui` (#46; `pnpm lint` rejects raw colors in the SPA) and Playwright 1.64 (`e2e/`, the `e2e` workflow). Kanel 4 + `kanel-kysely` generate the Kysely table types from the migrated database (`pnpm db:types`, #48; CI fails when `packages/db/src/generated/database.ts` is out of date): only the tables `crove_tenant` may read, so control-plane tables have no type, with hand-written refinements in `packages/db/src/schema.ts` that typecheck against the generated columns. Not adopted yet: Vite+ (scripts still call `oxlint`, `vitest` and `pnpm -r` directly), a formatter, the vendored pg-boss schema (pg-boss creates `crove_jobs` at migrate time) and an S3 storage adapter (local disk only). Since #15 the landing page (`apps/web`) uses catalog versions (Next.js 16.4, React 19.3, Tailwind 4, TypeScript 7.0.2); its own Dockerfile still starts from `node:22-alpine`. +**State on `main` (2026-10-14).** The MVP already uses pnpm 12 (`packageManager: pnpm@12.11.2`, `catalogMode: strict`), Node 26, TypeScript 7.0.2, oxlint 1.88, Vitest 5, Hono 4, oRPC 1.15, Kysely 0.29, pg-boss 12, Vite 8, TanStack Router and Query, `@supabase/auth-js`, `jose`, `use-intl` (#206), Tailwind 4, shadcn on `@base-ui/react` 1.8 in `@crove/ui` (#46; `pnpm lint` rejects raw colors in the SPA) and Playwright 1.64 (`e2e/`, the `e2e` workflow). Kanel 4 + `kanel-kysely` generate the Kysely table types from the migrated database (`pnpm db:types`, #48; CI fails when `packages/db/src/generated/database.ts` is out of date): only the tables `crove_tenant` may read, so control-plane tables have no type, with hand-written refinements in `packages/db/src/schema.ts` that typecheck against the generated columns. Not adopted yet: Vite+ (scripts still call `oxlint`, `vitest` and `pnpm -r` directly), a formatter and an S3 storage adapter (local disk only). Since #15 the landing page (`apps/web`) uses catalog versions (Next.js 16.4, React 19.3, Tailwind 4, TypeScript 7.0.2); its own Dockerfile still starts from `node:22-alpine`. ## 3. Topology One image, `crove-kernel` (root `Dockerfile`), runs every deployment. Its entrypoint (`apps/kernel/src/main.ts`) takes a role: -- `migrate`: apply the committed migrations with the built-in runner, set login-role passwords from the environment, and (on `main`, until External-tenant gate T6) create the pg-boss schema; +- `migrate`: apply the committed migrations with the built-in runner, and set login-role passwords from the environment (the pg-boss schema is a vendored migration since #25); - `api`: the HTTP API plus the SPA's static files, on one origin; - `worker`: the outbox relay and pg-boss jobs; - `all`: `api` and `worker` in one process, the self-host default; @@ -461,7 +461,7 @@ The login roles below are the decided design. On `main` the API uses `crove_app` **Self-host (implemented):** 1. Pull or build the new image. -2. `migrate` runs first. The built-in runner takes an advisory lock, applies each new file in its own transaction, records version and sha256 in `crove_meta.schema_migrations`, and refuses a file that was edited after it was applied. It then sets role passwords and (until T6) creates the pg-boss schema. Compose starts `auth` and `crove` only after `migrate` completes. +2. `migrate` runs first. The built-in runner takes an advisory lock, applies each new file in its own transaction, records version and sha256 in `crove_meta.schema_migrations`, and refuses a file that was edited after it was applied. It then sets role passwords. Compose starts `auth` and `crove` only after `migrate` completes. 3. Per ADR-0028 (not on `main`): after `auth` starts, the one-shot `auth-grants` role waits until Supabase Auth has created `auth.sessions` and `auth.users`, grants SELECT on them to `crove_auth_reader` and exits; it connects with `ADMIN_DATABASE_URL`, like `migrate`, because only the `auth` table owner or a superuser can grant on them. Compose starts `crove` only after it completes. It reruns on every `docker compose up`. Installs without compose run `auth-grants` after Supabase Auth's first start and after each Supabase Auth upgrade; until then requests get 503 and `/readyz` reports `auth_grants`. 4. `crove` starts; `/readyz` returns 503 until every committed migration is in the ledger. 5. With split roles: `migrate`, Supabase Auth, `auth-grants`, then `worker`, then `api`. diff --git a/docs/plan/crove-os-kernel.vi.html b/docs/plan/crove-os-kernel.vi.html index 07f7f2f4..d01c42df 100644 --- a/docs/plan/crove-os-kernel.vi.html +++ b/docs/plan/crove-os-kernel.vi.html @@ -278,7 +278,7 @@

4. Cổng trước khách ngoài

T3Sổ cái bất biến: chỉ đường post ghi bút toán; bút toán cân và có ít nhất 2 dòng lúc commitĐã xong
#71 đã làm, có test DB. #76: kiểm tra cân và số dòng đọc mọi dòng của bút toán, kể cả dòng người gọi không thấy (test với policy RESTRICTIVE); không role runtime nào gọi được với tham số tùy ý; xid8 sau khi restore logic không mở lại bút toán cũ (§7.9). #138: đảo bút toán (reversal) ghi một bút toán MỚI qua post_journal_entry, các dòng đối xứng với bút toán gốc (đổi Nợ và Có), liên kết qua reverses_entry_id; mỗi bút toán chỉ đảo được một lần, kể cả khi hai yêu cầu chạy đồng thời; không đảo một bút toán đảo (muốn khôi phục thì ghi bút toán mới); không ghi ngày trước bút toán gốc; database so đối xứng trên mọi dòng của cả hai bút toán, kể cả dòng người gọi không thấy. Đảo bút toán cũng là ghi sổ nên phải qua đề xuất và duyệt như hóa đơn mua vào (R3, P10, P11, P12). Có property test bằng fast-check cho đường post. #165: có API route inbox.request (kèm idempotency key) và giao diện: nút "Đảo bút toán" trên sổ bút toán (chọn ngày, bắt buộc nhập lý do), thẻ đề xuất đảo bút toán trong Inbox, liên kết hai chiều giữa bút toán gốc và bút toán đảo, và thông báo rõ ràng khi database từ chối vì người duyệt không thấy hết các dòng. Đảo bút toán của chứng từ mua thì Crove hủy chứng từ đó trong sổ của mình (một dòng hủy append-only trỏ tới bút toán đảo), để ghi sổ lại đúng hóa đơn với tài khoản đúng; trùng hóa đơn chỉ bỏ qua chứng từ đã hủy. Đây chỉ là chỉnh sổ của Crove, hóa đơn điện tử của người bán không đổi; cần kế toán xác nhận cách xử lý này. #204: database bắt mọi bên ghi (kể cả superuser hoặc gọi thẳng post_journal_entry) tuân theo chiều ngược lại: bút toán đảo của chứng từ mua chỉ commit được khi có dòng hủy chứng từ trong cùng transaction (constraint trigger hoãn tới lúc commit); sổ bút toán hiện đề xuất đảo đang chờ duyệt thay vì cho gửi thêm yêu cầu; kernel hỏi module finance qua ctx.query để biết đề xuất nào có chứng từ đã hủy, không đọc thẳng bảng finance. #254: khi bút toán đã có đề xuất đảo đang chờ duyệt thì yêu cầu đảo mới bị từ chối (REVERSAL_PENDING), nên qua kernel.proposal.request mỗi bút toán có tối đa một đề xuất đảo đang mở.#19, #76, #138, #165 T4Tạo org đi qua execute() và có dòng auditĐã xong
PR #96: tạo org là lệnh kernel.org.create chạy qua execute(), ghi tenant, owner, dòng audit và khóa idempotency trong cùng một transaction; gửi lại cùng khóa không tạo thêm gì; tenant không có dòng audit thì không commit được. PR #107: mỗi người dùng tạo tối đa 5 org trong 24 giờ và 20 org tổng cộng, tính cả org đã đóng (đổi qua env ORG_CREATE_LIMIT_PER_DAY, ORG_CREATE_LIMIT_TOTAL); definer đếm dưới advisory lock theo người dùng nên giới hạn đúng trên mọi instance API, hai yêu cầu đồng thời ở ngưỡng chỉ một cái thắng; vượt giới hạn trả 429 RATE_LIMITED và không ghi gì; gửi lại khóa idempotency cũ không bị tính. Lần bị từ chối chưa có dòng audit vì chưa có org để ghi, chờ security_event.#23, #94 T5Xử lý email đến chuyển khỏi system context của role API sang workerĐã xong
PR #78: API chỉ kiểm secret và lưu email thô; worker parse và xử lý; chỉ crove_worker được vào system context.#24 -T6Schema pg-boss vendored vào migration, không chạy migrate: trueChưa làm
Role migrate vẫn khởi động pg-boss với migrate: true.#25 +T6Schema pg-boss vendored vào migration, không chạy migrate: trueĐã xong
#25: pnpm db:vendor:pgboss sinh migration từ chính SQL của pg-boss (có chặn theo phiên bản, ghim search_path, tạo queue, chỉ cấp quyền cho crove_worker). Không còn chỗ nào chạy migrate: true; /readyz kiểm phiên bản schema; CI báo lỗi khi nâng pg-boss mà chưa sinh migration mới.#25 T7Adapter lưu trữ S3-compatible dùng chung cho API và workerMột phần
PR #110: STORAGE=s3 chọn S3BlobStore, cả api và worker dựng store từ config; mặc định vẫn là local disk. Ghi kiểu chỉ tạo mới (If-None-Match: *, trên GCS là x-goog-if-generation-match: 0) kèm sha256 để server kiểm; ghi lại đúng bytes cũ thì bỏ qua, khác bytes thì báo lỗi. Có test với MinIO trong CI và test đọc lại sau khi khởi động lại từ API và worker. Đã chạy end-to-end với api và worker là hai container riêng (#108): docker-compose.split.yml chạy hai role từ cùng một image, STORAGE=s3 trên MinIO, không có volume files (chỉ worker vào mạng extract); job CI split-stack dựng stack, tools/e2e/split-stack.mjs tải hóa đơn điện tử lên và gửi một email đến qua HTTP API, chờ worker đọc cả hai thành đề xuất, rồi kiểm tra hai tệp nằm trong bucket đúng bytes, thư gốc đã bị xóa khỏi bucket và không có gì trong /data/files của cả hai container (job split-stack trong ci.yml, lần chạy xanh đầu tiên). Từ #109 (ADR-0013): mỗi tệp lưu bộ ba (key, generation, sha256); khi đọc luôn xin đúng phiên bản đã ghi và kiểm sha256, nên tệp bị tráo sau khi duyệt không bao giờ được trả về hay đọc lại (S25); mặc định bucket phải bật versioning. Với STORAGE=s3, trình duyệt tải lên qua presigned POST dùng một lần vào khóa staging/ ngẫu nhiên (5 phút, tối đa 25 MB) và tải về qua presigned GET đúng phiên bản (60 giây); local disk vẫn đi qua kernel. Còn mở: retention hold đặt theo từng object; chưa chạy thử với GCS và R2.#26, #108, #109 T8Ma trận RLS chéo org sinh tự động (tiêu chí MVP 2)Đã xong
PR #114: test trong bộ invariant (rls-matrix.db.test.ts) đọc catalog, sinh case cho mọi bảng có org_id (19 bảng hiện nay). Mỗi bảng: kiểm cấu trúc (bật RLS, đúng policy tenant_isolation, không role runtime nào sở hữu bảng, không có quyền TRUNCATE) và thử đọc, insert, update, delete dữ liệu org B với vai crove_app, crove_worker, crove_definer_tenant và khi không có context; mỗi lần thử một transaction riêng, có case đối chứng trên org của chính mình. Bảng mới có org_id mà thiếu RLS, thiếu policy hoặc thiếu row factory thì test fail và nêu tên bảng. Ma trận phát hiện outbox_pending chưa bật RLS (không role nào có quyền nên chưa khai thác được); migration mới đã bật. Còn mở: case chéo pháp nhân và context giả mạo, chờ pháp nhân và seal (T11).#17 T9Test Playwright end-to-end (tiêu chí MVP 4)Đã xong
Workflow e2e dựng stack docker compose trên volume mới và chạy Playwright qua cả luồng (đăng ký, tạo org, tải hóa đơn, duyệt, danh sách, xuất Excel, kiểm tra dòng audit) ở mọi PR. Owner cần đặt job này là bắt buộc trong branch protection.#18 diff --git a/package.json b/package.json index 229c27a6..b246361c 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "docs:check": "node tools/docs-check.mjs", "db:migrate": "pnpm --filter @crove/db migrate", "db:types": "node tools/db-types.mjs", + "db:vendor:pgboss": "pnpm --filter @crove/kernel-server vendor:pgboss", "db:auth-grants": "pnpm --filter @crove/kernel-server auth-grants", "alert:test": "node --env-file-if-exists=.env apps/kernel/src/main.ts prober --test-alert", "usage:report": "node --env-file-if-exists=.env apps/kernel/src/main.ts usage-report", diff --git a/packages/db/src/migrate.ts b/packages/db/src/migrate.ts index 915f1f92..31e10f4d 100644 --- a/packages/db/src/migrate.ts +++ b/packages/db/src/migrate.ts @@ -121,10 +121,12 @@ export const PINNED_SEARCH_PATH = 'search_path=pg_catalog, pg_temp'; * on a table the definer writes, a helper) runs with the definer's privileges, so `SET search_path = ''` * there is the same hole, and a later `ALTER FUNCTION ... SECURITY DEFINER` keeps the old settings. * Excluded: extension members (their scripts own them; extensions live in `extensions` anyway) and the - * pg-boss schema `crove_jobs`, which pg-boss creates and versions itself (and checks for drift); its - * functions are SECURITY INVOKER, so a SECURITY DEFINER function appearing there is still reported. - * - the `crove_jobs` exemption holds only while no role but the runner (and superusers) may CREATE in that - * schema; otherwise such a role could plant an unscanned function there. + * pg-boss schema `crove_jobs`, vendored from pg-boss's own SQL (`pnpm db:vendor:pgboss`, #25; the vendored + * functions pin the same search_path) and version-checked by pg-boss; its functions are SECURITY INVOKER, + * so a SECURITY DEFINER function appearing there is still reported. + * - the `crove_jobs` exemption holds only while no role but the runner (roles that act as it, and superusers) + * may CREATE in that schema; otherwise such a role could plant an unscanned function there. A role that acts + * as the runner already has everything the runner has, as in the TEMPORARY check below. * - every function outside the `crove_*` schemas that a `crove_*` object reaches without naming it through * search_path either pins `PINNED_SEARCH_PATH`, or sets no search_path and is SECURITY INVOKER (it then runs * with its caller's path, which inside a definer is the pinned one). Reached means a function, or an @@ -287,6 +289,7 @@ async function listHardeningViolations(client: pg.ClientBase): Promise `SELECT r.rolname FROM pg_catalog.pg_roles r WHERE pg_catalog.to_regnamespace('crove_jobs') IS NOT NULL AND r.rolname <> current_user AND NOT r.rolsuper + AND NOT ${actsAs('r.oid', 'current_user')} AND pg_catalog.has_schema_privilege(r.oid, 'crove_jobs', 'CREATE') ORDER BY 1`, ); diff --git a/supabase/migrations/20261019034500_crove_jobs_pgboss_v46.sql b/supabase/migrations/20261019034500_crove_jobs_pgboss_v46.sql new file mode 100644 index 00000000..f2bdaeaa --- /dev/null +++ b/supabase/migrations/20261019034500_crove_jobs_pgboss_v46.sql @@ -0,0 +1,545 @@ +-- pg-boss schema version: 46 +-- Generated by `pnpm db:vendor:pgboss` from pg-boss getConstructionPlans('crove_jobs'). +-- Do not edit: upgrade pg-boss, run the script again and commit the new migration it writes. +-- Changes to pg-boss's SQL: no BEGIN, COMMIT or advisory lock (the runner applies this file in one transaction +-- under its own lock), and every function pins search_path. Before #25 the migrate role let pg-boss create the +-- schema at runtime, so an existing database at version 46 keeps its tables and only gets the pins. +DO $vendor$ +BEGIN + IF to_regclass('crove_jobs.version') IS NOT NULL THEN + IF (SELECT array_agg(version) FROM crove_jobs.version) = ARRAY[46] THEN + RETURN; + END IF; + RAISE EXCEPTION 'crove_jobs holds pg-boss schema version %, this migration installs 46', (SELECT string_agg(version::text, ', ') FROM crove_jobs.version); + ELSE + EXECUTE $pgboss$ +CREATE SCHEMA IF NOT EXISTS crove_jobs; + + CREATE TYPE crove_jobs.job_state AS ENUM ( + 'created', + 'retry', + 'active', + 'completed', + 'cancelled', + 'failed' + ) + ; + + CREATE FUNCTION crove_jobs.job_now() + RETURNS timestamp with time zone AS + $$ + SELECT pg_catalog.now(); + $$ + LANGUAGE sql STABLE SET search_path = pg_catalog, pg_temp; + ; + + CREATE TABLE crove_jobs.version ( + version int primary key, + cron_on timestamp with time zone, + bam_on timestamp with time zone, + flow_on timestamp with time zone, + reindex_on timestamp with time zone, + monitor_backoff_on timestamp with time zone + ) + ; + + CREATE TABLE crove_jobs.queue ( + name text NOT NULL, + policy text NOT NULL, + retry_limit int NOT NULL, + retry_delay int NOT NULL, + retry_backoff bool NOT NULL, + retry_delay_max int, + expire_seconds int NOT NULL, + retention_seconds int NOT NULL, + deletion_seconds int NOT NULL, + dead_letter text REFERENCES crove_jobs.queue (name) CHECK (dead_letter IS DISTINCT FROM name), + partition bool NOT NULL, + table_name text NOT NULL, + deferred_count int NOT NULL default 0, + blocked_count int NOT NULL default 0, + queued_count int NOT NULL default 0, + ready_count int NOT NULL default 0, + warning_queued int NOT NULL default 0, + active_count int NOT NULL default 0, + failed_count int NOT NULL default 0, + total_count int NOT NULL default 0, + retained_count int, + retained_on timestamp with time zone, + created_delta int NOT NULL default 0, + completed_delta int NOT NULL default 0, + failed_delta int NOT NULL default 0, + delta_on timestamp with time zone, + delta_seconds int, + wait_bins int[], + run_bins int[], + ready_oldest_seconds int, + ready_history int[] NOT NULL default '{}', + heartbeat_seconds int, + notify bool NOT NULL DEFAULT false, + singletons_active text[], + monitor_claim_on timestamp with time zone, + monitor_on timestamp with time zone, + maintain_on timestamp with time zone, + created_on timestamp with time zone not null default now(), + updated_on timestamp with time zone not null default now(), + PRIMARY KEY (name) + ) + ; + + CREATE TABLE crove_jobs.schedule ( + name text REFERENCES crove_jobs.queue ON DELETE CASCADE, + key text not null DEFAULT '', + kind text not null DEFAULT 'cron' CHECK (kind IN ('cron', 'rrule')), + cron text not null, + timezone text DEFAULT 'UTC', + data jsonb, + options jsonb, + created_on timestamp with time zone not null default now(), + updated_on timestamp with time zone not null default now(), + last_job_id uuid, + PRIMARY KEY (name, key) + ) + ; + + CREATE TABLE crove_jobs.subscription ( + event text not null, + name text not null REFERENCES crove_jobs.queue ON DELETE CASCADE, + created_on timestamp with time zone not null default now(), + updated_on timestamp with time zone not null default now(), + PRIMARY KEY(event, name) + ) + ; + + CREATE TABLE crove_jobs.bam ( + id uuid PRIMARY KEY default gen_random_uuid(), + name text NOT NULL, + version int NOT NULL, + status text NOT NULL DEFAULT 'pending', + queue text, + table_name text NOT NULL, + command text NOT NULL, + error text, + created_on timestamp with time zone NOT NULL DEFAULT clock_timestamp(), + started_on timestamp with time zone, + completed_on timestamp with time zone + ) + ; + + CREATE FUNCTION crove_jobs.job_table_format(command text, table_name text) + RETURNS text AS + $$ + SELECT format( + regexp_replace( + regexp_replace(command, '\.job\y', '.%1$I', 'g'), + '\yjob_i(\d+)', '%1$s_i\1', 'g' + ), + table_name + ); + $$ + LANGUAGE sql IMMUTABLE SET search_path = pg_catalog, pg_temp; + ; + + CREATE FUNCTION crove_jobs.job_table_run(command text, tbl_name text DEFAULT NULL, queue_name text DEFAULT NULL) + RETURNS VOID AS + $$ + DECLARE + tbl RECORD; + BEGIN + IF queue_name IS NOT NULL THEN + SELECT table_name INTO tbl_name FROM crove_jobs.queue WHERE name = queue_name; + END IF; + + IF tbl_name IS NOT NULL THEN + EXECUTE crove_jobs.job_table_format(command, tbl_name); + RETURN; + END IF; + + EXECUTE crove_jobs.job_table_format(command, 'job_common'); + + FOR tbl IN SELECT table_name FROM crove_jobs.queue WHERE partition = true + LOOP + EXECUTE crove_jobs.job_table_format(command, tbl.table_name); + END LOOP; + END; + $$ + LANGUAGE plpgsql SET search_path = pg_catalog, pg_temp; + ; + + CREATE FUNCTION crove_jobs.job_table_run_async(command_name text, version int, command text, tbl_name text DEFAULT NULL, queue_name text DEFAULT NULL) + RETURNS VOID AS + $$ + BEGIN + IF queue_name IS NOT NULL THEN + SELECT table_name INTO tbl_name FROM crove_jobs.queue WHERE name = queue_name; + END IF; + + IF tbl_name IS NOT NULL THEN + INSERT INTO crove_jobs.bam (name, version, status, queue, table_name, command) + VALUES ( + command_name, + version, + 'pending', + queue_name, + tbl_name, + crove_jobs.job_table_format(command, tbl_name) + ); + RETURN; + END IF; + + INSERT INTO crove_jobs.bam (name, version, status, queue, table_name, command) + SELECT + command_name, + version, + 'pending', + NULL, + 'job_common', + crove_jobs.job_table_format(command, 'job_common') + UNION ALL + SELECT + command_name, + version, + 'pending', + queue.name, + queue.table_name, + crove_jobs.job_table_format(command, queue.table_name) + FROM crove_jobs.queue + WHERE partition = true; + END; + $$ + LANGUAGE plpgsql SET search_path = pg_catalog, pg_temp; + ; + + CREATE TABLE crove_jobs.job ( + id uuid not null default gen_random_uuid(), + name text not null, + priority integer not null default(0), + data jsonb, + state crove_jobs.job_state not null default 'created', + retry_limit integer not null default 2, + retry_count integer not null default 0, + retry_delay integer not null default 0, + retry_backoff boolean not null default false, + retry_delay_max integer, + expire_seconds int not null default 900, + deletion_seconds int not null default 604800, + singleton_key text, + singleton_on timestamp without time zone, + group_id text, + group_tier text, + start_after timestamp with time zone not null default now(), + created_on timestamp with time zone not null default now(), + started_on timestamp with time zone, + completed_on timestamp with time zone, + keep_until timestamp with time zone NOT NULL default now() + interval '1209600', + output jsonb, + dead_letter text, + policy text, + heartbeat_on timestamp with time zone, + heartbeat_seconds int, + blocked boolean not null default false, + blocking boolean not null default false, + pending_dependencies int not null default 0, + source_name text, + source_id uuid, + source_created_on timestamp with time zone, + source_retry_count int, + source_output jsonb, + source_root_id uuid, + trace_context jsonb, + upsert_by_key bool + ) PARTITION BY LIST (name) + ; +ALTER TABLE crove_jobs.job ADD PRIMARY KEY (name, id); + + CREATE TABLE crove_jobs.job_common (LIKE crove_jobs.job INCLUDING GENERATED INCLUDING DEFAULTS); + + SELECT crove_jobs.job_table_run($cmd$ALTER TABLE crove_jobs.job ADD PRIMARY KEY (name, id)$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$ALTER TABLE crove_jobs.job ADD CONSTRAINT q_fkey FOREIGN KEY (name) REFERENCES crove_jobs.queue (name) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$ALTER TABLE crove_jobs.job ADD CONSTRAINT dlq_fkey FOREIGN KEY (dead_letter) REFERENCES crove_jobs.queue (name) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$CREATE UNIQUE INDEX job_i1 ON crove_jobs.job (name, COALESCE(singleton_key, '')) WHERE state = 'created' AND policy = 'short'$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$CREATE UNIQUE INDEX job_i2 ON crove_jobs.job (name, COALESCE(singleton_key, '')) WHERE state = 'active' AND policy = 'singleton'$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$CREATE UNIQUE INDEX job_i3 ON crove_jobs.job (name, state, COALESCE(singleton_key, '')) WHERE state <= 'active' AND policy = 'stately'$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$CREATE UNIQUE INDEX job_i6 ON crove_jobs.job (name, COALESCE(singleton_key, '')) WHERE state <= 'active' AND policy = 'exclusive'$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$CREATE UNIQUE INDEX job_i8 ON crove_jobs.job (name, singleton_key) WHERE state IN ('active', 'retry', 'failed') AND policy = 'key_strict_fifo'$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$CREATE INDEX job_i10 ON crove_jobs.job (name, singleton_key, state DESC, created_on, id) INCLUDE (start_after) WHERE state < 'active' AND NOT blocked AND policy = 'key_strict_fifo'$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$ALTER TABLE crove_jobs.job ADD CONSTRAINT job_key_strict_fifo_singleton_key_check CHECK (NOT (policy = 'key_strict_fifo' AND singleton_key IS NULL))$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$CREATE UNIQUE INDEX job_i4 ON crove_jobs.job (name, singleton_on, COALESCE(singleton_key, '')) WHERE state <> 'cancelled' AND singleton_on IS NOT NULL$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$CREATE INDEX job_i11 ON crove_jobs.job (name, priority DESC, created_on, start_after) WHERE state < 'active' AND NOT blocked$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$CREATE INDEX job_i7 ON crove_jobs.job (name, group_id) WHERE state = 'active' AND group_id IS NOT NULL$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$CREATE INDEX job_i9 ON crove_jobs.job (name, id) WHERE blocking AND state = 'completed'$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$CREATE INDEX job_i12 ON crove_jobs.job (source_root_id) WHERE source_root_id IS NOT NULL$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$CREATE UNIQUE INDEX job_i13 ON crove_jobs.job (name, singleton_key) WHERE state = 'created' AND upsert_by_key$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$CREATE INDEX job_i14 ON crove_jobs.job (name, state, blocked, start_after) WHERE state IN ('created', 'retry', 'active', 'failed')$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$CREATE INDEX job_i15 ON crove_jobs.job (name, created_on)$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$CREATE INDEX job_i16 ON crove_jobs.job (name, completed_on) WHERE completed_on IS NOT NULL$cmd$, 'job_common'); + SELECT crove_jobs.job_table_run($cmd$CREATE INDEX job_i17 ON crove_jobs.job (name, singleton_key, created_on) WHERE state < 'active' AND singleton_key IS NOT NULL$cmd$, 'job_common'); + + ALTER TABLE crove_jobs.job ATTACH PARTITION crove_jobs.job_common DEFAULT; + ; + + CREATE TABLE crove_jobs.warning ( + id uuid PRIMARY KEY default gen_random_uuid(), + type text NOT NULL, + message text NOT NULL, + data jsonb, + created_on timestamp with time zone NOT NULL DEFAULT now() + ) + ; +CREATE INDEX warning_i1 ON crove_jobs.warning (created_on DESC); + + CREATE TABLE crove_jobs.queue_stats ( + id uuid NOT NULL DEFAULT gen_random_uuid(), + name text NOT NULL, + deferred_count int NOT NULL DEFAULT 0, + queued_count int NOT NULL DEFAULT 0, + ready_count int NOT NULL DEFAULT 0, + active_count int NOT NULL DEFAULT 0, + failed_count int NOT NULL DEFAULT 0, + total_count int NOT NULL DEFAULT 0, + created_delta int, + completed_delta int, + failed_delta int, + delta_seconds int, + delta_on timestamptz, + wait_bins int[], + run_bins int[], + ready_oldest_seconds int, + captured_on timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (id, captured_on) + ) PARTITION BY RANGE (captured_on) + ; +CREATE INDEX queue_stats_i1 ON crove_jobs.queue_stats (name, captured_on DESC) INCLUDE (deferred_count, queued_count, ready_count, active_count, failed_count, total_count); +; + + DO $$ + DECLARE + d date; + i int; + part_name text; + BEGIN + FOR i IN 0..1 LOOP + d := (crove_jobs.job_now() AT TIME ZONE 'UTC')::date + i; + part_name := 'queue_stats_' || to_char(d, 'YYYYMMDD'); + IF NOT EXISTS ( + SELECT 1 FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'crove_jobs' AND c.relname = part_name + ) THEN + EXECUTE format( + 'CREATE TABLE crove_jobs.%I PARTITION OF crove_jobs.queue_stats FOR VALUES FROM (%L) TO (%L)', + part_name, + to_char(d, 'YYYY-MM-DD') || ' 00:00:00+00', + to_char(d + 1, 'YYYY-MM-DD') || ' 00:00:00+00' + ); + END IF; + END LOOP; + END; + $$ + ; + + CREATE TABLE crove_jobs.job_dependency ( + child_name text NOT NULL, + child_id uuid NOT NULL, + parent_name text NOT NULL, + parent_id uuid NOT NULL, + PRIMARY KEY (child_name, child_id, parent_name, parent_id) + ) + ; +CREATE INDEX IF NOT EXISTS job_dep_parent_idx ON crove_jobs.job_dependency (parent_name, parent_id); + + CREATE TABLE crove_jobs.instance ( + id uuid PRIMARY KEY, + name text, + host text NOT NULL, + pid int NOT NULL, + version text NOT NULL, + node_version text NOT NULL, + application_name text, + heartbeat_seconds int NOT NULL, + supervise bool NOT NULL, + schedule bool NOT NULL, + migrate bool NOT NULL, + persist_queue_stats bool NOT NULL, + persist_warnings bool NOT NULL, + pool_max int, + pool_total int, + pool_idle int, + pool_waiting int, + workers jsonb NOT NULL DEFAULT '[]'::jsonb, + metrics jsonb, + config jsonb NOT NULL DEFAULT '{}'::jsonb, + crash_restarts int NOT NULL DEFAULT 0, + crash_restarts_since timestamptz, + started_on timestamptz NOT NULL DEFAULT now(), + heartbeat_on timestamptz NOT NULL DEFAULT now(), + stopped_on timestamptz + ) + ; + + CREATE FUNCTION crove_jobs.create_queue(queue_name text, options jsonb) + RETURNS VOID AS + $$ + DECLARE + tablename varchar := CASE WHEN options->>'partition' = 'true' + THEN 'j' || encode(sha224(queue_name::bytea), 'hex') + ELSE 'job_common' + END; + queue_created_on timestamptz; + BEGIN + + WITH q as ( + INSERT INTO crove_jobs.queue ( + name, + policy, + retry_limit, + retry_delay, + retry_backoff, + retry_delay_max, + expire_seconds, + retention_seconds, + deletion_seconds, + warning_queued, + dead_letter, + partition, + table_name, + heartbeat_seconds, + notify, + created_on, + updated_on + ) + VALUES ( + queue_name, + options->>'policy', + COALESCE((options->>'retryLimit')::int, 2), + COALESCE((options->>'retryDelay')::int, 0), + COALESCE((options->>'retryBackoff')::bool, false), + (options->>'retryDelayMax')::int, + COALESCE((options->>'expireInSeconds')::int, 900), + COALESCE((options->>'retentionSeconds')::int, 1209600), + COALESCE((options->>'deleteAfterSeconds')::int, 604800), + COALESCE((options->>'warningQueueSize')::int, 0), + options->>'deadLetter', + COALESCE((options->>'partition')::bool, false), + tablename, + (options->>'heartbeatSeconds')::int, + COALESCE((options->>'notify')::bool, false), + crove_jobs.job_now(), + crove_jobs.job_now() + ) + ON CONFLICT DO NOTHING + RETURNING created_on + ) + SELECT created_on into queue_created_on from q; + + IF queue_created_on IS NULL OR options->>'partition' IS DISTINCT FROM 'true' THEN + RETURN; + END IF; + + EXECUTE format('CREATE TABLE crove_jobs.%I (LIKE crove_jobs.job INCLUDING DEFAULTS)', tablename); + + EXECUTE crove_jobs.job_table_format($cmd$ALTER TABLE crove_jobs.job ADD PRIMARY KEY (name, id)$cmd$, tablename); + EXECUTE crove_jobs.job_table_format($cmd$ALTER TABLE crove_jobs.job ADD CONSTRAINT q_fkey FOREIGN KEY (name) REFERENCES crove_jobs.queue (name) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED$cmd$, tablename); + EXECUTE crove_jobs.job_table_format($cmd$ALTER TABLE crove_jobs.job ADD CONSTRAINT dlq_fkey FOREIGN KEY (dead_letter) REFERENCES crove_jobs.queue (name) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED$cmd$, tablename); + + EXECUTE crove_jobs.job_table_format($cmd$CREATE INDEX job_i11 ON crove_jobs.job (name, priority DESC, created_on, start_after) WHERE state < 'active' AND NOT blocked$cmd$, tablename); + EXECUTE crove_jobs.job_table_format($cmd$CREATE UNIQUE INDEX job_i4 ON crove_jobs.job (name, singleton_on, COALESCE(singleton_key, '')) WHERE state <> 'cancelled' AND singleton_on IS NOT NULL$cmd$, tablename); + EXECUTE crove_jobs.job_table_format($cmd$CREATE INDEX job_i7 ON crove_jobs.job (name, group_id) WHERE state = 'active' AND group_id IS NOT NULL$cmd$, tablename); + EXECUTE crove_jobs.job_table_format($cmd$CREATE INDEX job_i9 ON crove_jobs.job (name, id) WHERE blocking AND state = 'completed'$cmd$, tablename); + EXECUTE crove_jobs.job_table_format($cmd$CREATE INDEX job_i12 ON crove_jobs.job (source_root_id) WHERE source_root_id IS NOT NULL$cmd$, tablename); + EXECUTE crove_jobs.job_table_format($cmd$CREATE UNIQUE INDEX job_i13 ON crove_jobs.job (name, singleton_key) WHERE state = 'created' AND upsert_by_key$cmd$, tablename); + EXECUTE crove_jobs.job_table_format($cmd$CREATE INDEX job_i14 ON crove_jobs.job (name, state, blocked, start_after) WHERE state IN ('created', 'retry', 'active', 'failed')$cmd$, tablename); + EXECUTE crove_jobs.job_table_format($cmd$CREATE INDEX job_i15 ON crove_jobs.job (name, created_on)$cmd$, tablename); + EXECUTE crove_jobs.job_table_format($cmd$CREATE INDEX job_i16 ON crove_jobs.job (name, completed_on) WHERE completed_on IS NOT NULL$cmd$, tablename); + EXECUTE crove_jobs.job_table_format($cmd$CREATE INDEX job_i17 ON crove_jobs.job (name, singleton_key, created_on) WHERE state < 'active' AND singleton_key IS NOT NULL$cmd$, tablename); + + IF options->>'policy' = 'short' THEN + EXECUTE crove_jobs.job_table_format($cmd$CREATE UNIQUE INDEX job_i1 ON crove_jobs.job (name, COALESCE(singleton_key, '')) WHERE state = 'created' AND policy = 'short'$cmd$, tablename); + ELSIF options->>'policy' = 'singleton' THEN + EXECUTE crove_jobs.job_table_format($cmd$CREATE UNIQUE INDEX job_i2 ON crove_jobs.job (name, COALESCE(singleton_key, '')) WHERE state = 'active' AND policy = 'singleton'$cmd$, tablename); + ELSIF options->>'policy' = 'stately' THEN + EXECUTE crove_jobs.job_table_format($cmd$CREATE UNIQUE INDEX job_i3 ON crove_jobs.job (name, state, COALESCE(singleton_key, '')) WHERE state <= 'active' AND policy = 'stately'$cmd$, tablename); + ELSIF options->>'policy' = 'exclusive' THEN + EXECUTE crove_jobs.job_table_format($cmd$CREATE UNIQUE INDEX job_i6 ON crove_jobs.job (name, COALESCE(singleton_key, '')) WHERE state <= 'active' AND policy = 'exclusive'$cmd$, tablename); + ELSIF options->>'policy' = 'key_strict_fifo' THEN + EXECUTE crove_jobs.job_table_format($cmd$CREATE UNIQUE INDEX job_i8 ON crove_jobs.job (name, singleton_key) WHERE state IN ('active', 'retry', 'failed') AND policy = 'key_strict_fifo'$cmd$, tablename); + EXECUTE crove_jobs.job_table_format($cmd$CREATE INDEX job_i10 ON crove_jobs.job (name, singleton_key, state DESC, created_on, id) INCLUDE (start_after) WHERE state < 'active' AND NOT blocked AND policy = 'key_strict_fifo'$cmd$, tablename); + EXECUTE crove_jobs.job_table_format($cmd$ALTER TABLE crove_jobs.job ADD CONSTRAINT job_key_strict_fifo_singleton_key_check CHECK (NOT (policy = 'key_strict_fifo' AND singleton_key IS NULL))$cmd$, tablename); + END IF; + + EXECUTE format('ALTER TABLE crove_jobs.%I ADD CONSTRAINT cjc CHECK (name=%L)', tablename, queue_name); + EXECUTE format('ALTER TABLE crove_jobs.job ATTACH PARTITION crove_jobs.%I FOR VALUES IN (%L)', tablename, queue_name); + END; + $$ + LANGUAGE plpgsql SET search_path = pg_catalog, pg_temp; + ; + + CREATE FUNCTION crove_jobs.delete_queue(queue_name text) + RETURNS VOID AS + $$ + DECLARE + v_table varchar; + v_partition bool; + BEGIN + + SELECT table_name, partition + FROM crove_jobs.queue + WHERE name = queue_name + INTO v_table, v_partition; + + IF v_partition THEN + EXECUTE format('DROP TABLE IF EXISTS crove_jobs.%I', v_table); + ELSE + EXECUTE format('DELETE FROM crove_jobs.%I WHERE name = %L', v_table, queue_name); + END IF; + + DELETE FROM crove_jobs.queue WHERE name = queue_name; + END; + $$ + LANGUAGE plpgsql SET search_path = pg_catalog, pg_temp; + ; +INSERT INTO crove_jobs.version(version) VALUES ('46'); +$pgboss$; + END IF; +END +$vendor$; + +-- A schema pg-boss created at runtime (before #25) has functions without a pinned search_path. The migration role +-- created them then, so it owns them now. +DO $pin$ +DECLARE + fn regprocedure; +BEGIN + FOR fn IN + SELECT p.oid::regprocedure FROM pg_catalog.pg_proc p + WHERE p.pronamespace = 'crove_jobs'::regnamespace + AND p.proconfig IS DISTINCT FROM ARRAY['search_path=pg_catalog, pg_temp'] + LOOP + EXECUTE pg_catalog.format('ALTER FUNCTION %s SET search_path = pg_catalog, pg_temp', fn); + END LOOP; +END +$pin$; + +-- Queues the worker uses (QUEUES in apps/kernel/src/jobs-schema.ts); create_queue skips existing ones. +SELECT crove_jobs.create_queue(q, '{"policy":"standard"}'::jsonb) FROM unnest(ARRAY['finance-file-extract', 'kernel-inbound-email', 'kernel-session-signout', 'kernel-org-export', 'kernel-retention-purge', 'kernel-agent-dispatch', 'kernel-agent-sweep']) AS q; +SELECT crove_jobs.create_queue(q, '{"policy":"stately"}'::jsonb) FROM unnest(ARRAY['kernel-agent-wake']) AS q; + +-- /readyz (the API role) compares this with the version its pg-boss expects. +CREATE OR REPLACE FUNCTION crove_kernel.jobs_schema_version() +RETURNS integer LANGUAGE sql STABLE SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ + SELECT max(v.version) FROM crove_jobs.version v +$$; +REVOKE ALL ON FUNCTION crove_kernel.jobs_schema_version() FROM PUBLIC; +GRANT EXECUTE ON FUNCTION crove_kernel.jobs_schema_version() TO crove_app; + +-- The worker (pg-boss and the outbox relay) uses crove_jobs; nobody else may. +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA crove_jobs FROM PUBLIC; +GRANT USAGE ON SCHEMA crove_jobs TO crove_worker; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA crove_jobs TO crove_worker; +-- The version row decides whether a vendored upgrade runs, so only migrations write it. pg-boss started with +-- migrate: false only stamps its maintenance columns. +REVOKE INSERT, UPDATE, DELETE, TRUNCATE ON crove_jobs.version FROM crove_worker; +GRANT UPDATE (cron_on, bam_on, flow_on, reindex_on, monitor_backoff_on) ON crove_jobs.version TO crove_worker; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA crove_jobs TO crove_worker; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA crove_jobs TO crove_worker; +ALTER DEFAULT PRIVILEGES IN SCHEMA crove_jobs GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO crove_worker;