From 1f967d5653d96519de5c938819b49e0676e6a8d9 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sun, 19 Jul 2026 21:47:10 +0000 Subject: [PATCH 1/2] feat(export): explicit excludeCategories filtering in exportMigrations --- pgpm/cli/src/commands/export.ts | 14 +- pgpm/export/__tests__/export-flow.test.ts | 155 ++++++++++++++++++++++ pgpm/export/src/export-migrations.ts | 46 +++++-- 3 files changed, 202 insertions(+), 13 deletions(-) diff --git a/pgpm/cli/src/commands/export.ts b/pgpm/cli/src/commands/export.ts index 5d784c0ac9..6764eeaf17 100644 --- a/pgpm/cli/src/commands/export.ts +++ b/pgpm/cli/src/commands/export.ts @@ -23,10 +23,13 @@ Options: --author Project author (default: from git config) --extensionName Extension name --metaExtensionName Meta extension name (default: svc) + --exclude-categories Comma-separated sql_actions categories to omit + (e.g. security,permissions,auth,memberships). SQL mode only. --cwd Working directory (default: current directory) Examples: pgpm export Export migrations from selected database (SQL mode) + pgpm export --exclude-categories security,permissions,auth,memberships pgpm export --graphql-endpoint 'http://[::1]:3002/graphql' --migrate-endpoint 'http://[::1]:3000/graphql' --migrate-host db_migrate.localhost:3000 `; @@ -53,6 +56,14 @@ export default async ( const migrateHeadersRaw = argv['migrate-headers'] || argv.migrateHeaders; const token = argv.token; + const excludeCategoriesRaw = argv['exclude-categories'] || argv.excludeCategories; + const excludeCategories: string[] | undefined = + typeof excludeCategoriesRaw === 'string' + ? excludeCategoriesRaw.split(',').map((s: string) => s.trim()).filter(Boolean) + : Array.isArray(excludeCategoriesRaw) + ? excludeCategoriesRaw + : undefined; + if (graphqlEndpoint) { // ========================================================================= // GraphQL export mode @@ -305,7 +316,8 @@ export default async ( metaExtensionName, prompter, argv, - username + username, + excludeCategories }); } diff --git a/pgpm/export/__tests__/export-flow.test.ts b/pgpm/export/__tests__/export-flow.test.ts index 1bcc0d50a1..122b62569b 100644 --- a/pgpm/export/__tests__/export-flow.test.ts +++ b/pgpm/export/__tests__/export-flow.test.ts @@ -759,4 +759,159 @@ relocatable = false } }); }); + + describe('Export with excludeCategories', () => { + const DATABASE_ID = 'a1b2c3d4-e5f6-4708-b250-000000000001'; + const OPEN_EXTENSION_NAME = 'pets-open'; + const OPEN_META_EXTENSION_NAME = 'pets-open-svc'; + const POLICY_DEPLOY = 'schemas/pets_public/tables/pets/policies/pets_policy'; + let exportError: Error | null = null; + + const scaffoldModule = (name: string, description: string) => { + const moduleDir = join(exportWorkspaceDir, 'packages', name); + mkdirSync(moduleDir, { recursive: true }); + writeFileSync(join(moduleDir, 'package.json'), JSON.stringify({ + name, + version: '1.0.0', + description + }, null, 2)); + writeFileSync(join(moduleDir, `${name}.control`), `# ${name} extension +comment = '${description}' +default_version = '1.0.0' +relocatable = false +`); + writeFileSync(join(moduleDir, 'pgpm.plan'), `%syntax-version=1.0.0 +%project=${name} +%uri=https://github.com/test/${name} +`); + }; + + beforeAll(async () => { + // Add a category column (the real db_migrate.sql_actions has one) and + // tag a policy action as 'security' + await pg.query(`ALTER TABLE db_migrate.sql_actions ADD COLUMN IF NOT EXISTS category text DEFAULT 'ddl'`); + await pg.query( + `INSERT INTO db_migrate.sql_actions (name, database_id, deploy, deps, content, revert, verify, category) + VALUES ( + 'Create pets policy', + $1, + $2, + ARRAY['schemas/pets_public/tables/pets/table']::text[], + 'CREATE POLICY pets_policy ON pets_public.pets FOR SELECT USING (true);', + 'DROP POLICY pets_policy ON pets_public.pets;', + 'SELECT 1;', + 'security' + ) + ON CONFLICT (database_id, deploy) DO NOTHING`, + [DATABASE_ID, POLICY_DEPLOY] + ); + + const schemaResult = await pg.query( + 'SELECT schema_name FROM metaschema_public.schema WHERE database_id = $1', + [DATABASE_ID] + ); + const schemaNames = schemaResult.rows.map((r: any) => r.schema_name); + + scaffoldModule(OPEN_EXTENSION_NAME, 'Exported pets database schema (security excluded)'); + scaffoldModule(OPEN_META_EXTENSION_NAME, 'Exported pets service metadata (security excluded)'); + + const project = new PgpmPackage(exportWorkspaceDir); + + try { + await exportMigrations({ + project, + options: { + pg: dbConfig + }, + dbInfo: { + dbname: dbConfig.database, + databaseName: 'pets', + database_ids: [DATABASE_ID] + }, + author: 'test ', + outdir: join(exportWorkspaceDir, 'packages'), + schema_names: schemaNames, + extensionName: OPEN_EXTENSION_NAME, + extensionDesc: 'Exported pets database schema (security excluded)', + metaExtensionName: OPEN_META_EXTENSION_NAME, + metaExtensionDesc: 'Exported pets service metadata (security excluded)', + excludeCategories: ['security'] + }); + } catch (err) { + exportError = err as Error; + } + }, 180000); + + it('should have completed export without errors', () => { + if (exportError) { + console.error('Export error:', exportError); + } + expect(exportError).toBeNull(); + }); + + it('should omit security-categorized actions from the plan', () => { + const planPath = join(exportWorkspaceDir, 'packages', OPEN_EXTENSION_NAME, 'pgpm.plan'); + expect(existsSync(planPath)).toBe(true); + + // The replacer renames pets_public -> pets_open_public in the exported module + const planContent = readFileSync(planPath, 'utf-8'); + expect(planContent).not.toContain('/policies/pets_policy'); + // Non-security actions (category 'ddl' via the column default) are kept + expect(planContent).toContain('schemas/pets_open_public/tables/pets/table'); + }); + + it('should not write deploy files for excluded actions', () => { + const policyDeployPath = join( + exportWorkspaceDir, 'packages', OPEN_EXTENSION_NAME, 'deploy', + 'schemas/pets_open_public/tables/pets/policies/pets_policy.sql' + ); + expect(existsSync(policyDeployPath)).toBe(false); + + const tableDeployPath = join( + exportWorkspaceDir, 'packages', OPEN_EXTENSION_NAME, 'deploy', 'schemas/pets_open_public/tables/pets/table.sql' + ); + expect(existsSync(tableDeployPath)).toBe(true); + }); + + it('should include security actions when excludeCategories is not passed', async () => { + const FULL_EXTENSION_NAME = 'pets-full'; + const FULL_META_EXTENSION_NAME = 'pets-full-svc'; + + const schemaResult = await pg.query( + 'SELECT schema_name FROM metaschema_public.schema WHERE database_id = $1', + [DATABASE_ID] + ); + const schemaNames = schemaResult.rows.map((r: any) => r.schema_name); + + scaffoldModule(FULL_EXTENSION_NAME, 'Exported pets database schema (full)'); + scaffoldModule(FULL_META_EXTENSION_NAME, 'Exported pets service metadata (full)'); + + const project = new PgpmPackage(exportWorkspaceDir); + + await exportMigrations({ + project, + options: { + pg: dbConfig + }, + dbInfo: { + dbname: dbConfig.database, + databaseName: 'pets', + database_ids: [DATABASE_ID] + }, + author: 'test ', + outdir: join(exportWorkspaceDir, 'packages'), + schema_names: schemaNames, + extensionName: FULL_EXTENSION_NAME, + extensionDesc: 'Exported pets database schema (full)', + metaExtensionName: FULL_META_EXTENSION_NAME, + metaExtensionDesc: 'Exported pets service metadata (full)' + }); + + const planContent = readFileSync( + join(exportWorkspaceDir, 'packages', FULL_EXTENSION_NAME, 'pgpm.plan'), + 'utf-8' + ); + expect(planContent).toContain('schemas/pets_full_public/tables/pets/policies/pets_policy'); + }, 180000); + }); }); diff --git a/pgpm/export/src/export-migrations.ts b/pgpm/export/src/export-migrations.ts index 9edf0fc62d..e2417570d6 100644 --- a/pgpm/export/src/export-migrations.ts +++ b/pgpm/export/src/export-migrations.ts @@ -44,6 +44,12 @@ interface ExportMigrationsToDiskOptions { * Useful for self-referential introspection where you want to apply policies to real schemas. */ skipSchemaRenaming?: boolean; + /** + * Action categories to exclude from the export (e.g. ['security', 'permissions']). + * Rows in db_migrate.sql_actions whose category matches are omitted from the + * generated pgpm plan and SQL files. Rows with a NULL category are always kept. + */ + excludeCategories?: string[]; } interface ExportOptions { @@ -75,6 +81,12 @@ interface ExportOptions { * Useful for self-referential introspection where you want to apply policies to real schemas. */ skipSchemaRenaming?: boolean; + /** + * Action categories to exclude from the export (e.g. ['security', 'permissions']). + * Rows in db_migrate.sql_actions whose category matches are omitted from the + * generated pgpm plan and SQL files. Rows with a NULL category are always kept. + */ + excludeCategories?: string[]; } const exportMigrationsToDisk = async ({ @@ -95,7 +107,8 @@ const exportMigrationsToDisk = async ({ repoName, username, serviceOutdir, - skipSchemaRenaming = false + skipSchemaRenaming = false, + excludeCategories }: ExportMigrationsToDiskOptions): Promise => { const normalizedOutdir = normalizeOutdir(outdir); // Use serviceOutdir for service module, defaulting to outdir if not provided @@ -140,13 +153,22 @@ const exportMigrationsToDisk = async ({ name }); - // Filter sql_actions by database_id to avoid cross-database pollution - // Previously this query had no WHERE clause, which could export actions - // from unrelated databases in a persistent database environment - const results = await pgPool.query( - `select * from db_migrate.sql_actions where database_id = $1 order by id`, - [databaseId] - ); + // Filter sql_actions by database_id to avoid cross-database pollution. + // Category exclusion is applied here explicitly rather than via the + // export.exclude_categories session GUC, which only affects the session + // it was set on and therefore cannot be relied upon across connections. + const results = excludeCategories && excludeCategories.length > 0 + ? await pgPool.query( + `select * from db_migrate.sql_actions + where database_id = $1 + and (category is null or category != ALL($2::text[])) + order by id`, + [databaseId, excludeCategories] + ) + : await pgPool.query( + `select * from db_migrate.sql_actions where database_id = $1 order by id`, + [databaseId] + ); const opts: SqlWriteOptions = { name, @@ -276,8 +298,6 @@ ${META_COMMON_FOOTER} writePgpmPlan(metaPackage, opts); writePgpmFiles(metaPackage, opts); - - pgPool.end(); }; export const exportMigrations = async ({ @@ -296,7 +316,8 @@ export const exportMigrations = async ({ repoName, username, serviceOutdir, - skipSchemaRenaming + skipSchemaRenaming, + excludeCategories }: ExportOptions): Promise => { for (let v = 0; v < dbInfo.database_ids.length; v++) { const databaseId = dbInfo.database_ids[v]; @@ -318,7 +339,8 @@ export const exportMigrations = async ({ repoName, username, serviceOutdir, - skipSchemaRenaming + skipSchemaRenaming, + excludeCategories }); } }; From 89fcfbd7cf7d51366c0bf1da3ae7c1ffdcb4f373 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sun, 19 Jul 2026 22:52:31 +0000 Subject: [PATCH 2/2] feat(export): excludeCategories filtering in GraphQL export mode Adds category to the migrate-client schema/ORM (SqlAction field, filter, orderBy) and filters sql_actions server-side in exportGraphQL via or: [category isNull, category notIn excludeCategories], keeping null-category rows. CLI now passes --exclude-categories in both modes. --- pgpm/cli/src/commands/export.ts | 5 +++-- pgpm/export/src/export-graphql.ts | 22 ++++++++++++++++--- sdk/migrate-client/schemas/migrate.graphql | 6 +++++ sdk/migrate-client/src/migrate/orm/README.md | 7 +++--- .../src/migrate/orm/input-types.ts | 8 +++++++ 5 files changed, 40 insertions(+), 8 deletions(-) diff --git a/pgpm/cli/src/commands/export.ts b/pgpm/cli/src/commands/export.ts index 6764eeaf17..b40b37896c 100644 --- a/pgpm/cli/src/commands/export.ts +++ b/pgpm/cli/src/commands/export.ts @@ -24,7 +24,7 @@ Options: --extensionName Extension name --metaExtensionName Meta extension name (default: svc) --exclude-categories Comma-separated sql_actions categories to omit - (e.g. security,permissions,auth,memberships). SQL mode only. + (e.g. security,permissions,auth,memberships). Works in SQL and GraphQL modes. --cwd Working directory (default: current directory) Examples: @@ -179,7 +179,8 @@ export default async ( metaExtensionName, prompter, argv, - username + username, + excludeCategories }); } else { // ========================================================================= diff --git a/pgpm/export/src/export-graphql.ts b/pgpm/export/src/export-graphql.ts index ea7de9ca1b..d74353f154 100644 --- a/pgpm/export/src/export-graphql.ts +++ b/pgpm/export/src/export-graphql.ts @@ -70,6 +70,12 @@ export interface ExportGraphQLOptions { username?: string; serviceOutdir?: string; skipSchemaRenaming?: boolean; + /** + * sql_actions categories to exclude from the export (e.g. ['security', + * 'permissions']). Rows whose category matches are omitted; rows with a + * null category are always kept. + */ + excludeCategories?: string[]; } export const exportGraphQL = async ({ @@ -94,7 +100,8 @@ export const exportGraphQL = async ({ repoName, username, serviceOutdir, - skipSchemaRenaming = false + skipSchemaRenaming = false, + excludeCategories }: ExportGraphQLOptions): Promise => { const normalizedOutdir = normalizeOutdir(outdir); const svcOutdir = normalizeOutdir(serviceOutdir || outdir); @@ -147,10 +154,19 @@ export const exportGraphQL = async ({ actionName: true, actionId: true, actorId: true, - payload: true + payload: true, + category: true }, where: { - databaseId: { equalTo: databaseId } + databaseId: { equalTo: databaseId }, + ...(excludeCategories && excludeCategories.length > 0 + ? { + or: [ + { category: { isNull: true } }, + { category: { notIn: excludeCategories } } + ] + } + : {}) }, orderBy: ['ID_ASC'], first: PAGE_SIZE, diff --git a/sdk/migrate-client/schemas/migrate.graphql b/sdk/migrate-client/schemas/migrate.graphql index ad29a0a1a1..2c91b6cc2f 100644 --- a/sdk/migrate-client/schemas/migrate.graphql +++ b/sdk/migrate-client/schemas/migrate.graphql @@ -662,6 +662,7 @@ type SqlAction { actionId: UUID actionName: String actorId: UUID + category: String content: String createdAt: Datetime databaseId: UUID @@ -716,6 +717,9 @@ input SqlActionFilter { """Checks for all expressions in this list.""" and: [SqlActionFilter!] + """Filter by the object’s `category` field.""" + category: StringFilter + """Filter by the object’s `content` field.""" content: StringFilter @@ -758,6 +762,8 @@ enum SqlActionOrderBy { ACTION_NAME_DESC ACTOR_ID_ASC ACTOR_ID_DESC + CATEGORY_ASC + CATEGORY_DESC CONTENT_ASC CONTENT_DESC CREATED_AT_ASC diff --git a/sdk/migrate-client/src/migrate/orm/README.md b/sdk/migrate-client/src/migrate/orm/README.md index ab6cd464ad..2a25653687 100644 --- a/sdk/migrate-client/src/migrate/orm/README.md +++ b/sdk/migrate-client/src/migrate/orm/README.md @@ -78,6 +78,7 @@ CRUD operations for SqlAction records. | `actionId` | UUID | Yes | | `actionName` | String | Yes | | `actorId` | UUID | Yes | +| `category` | String | Yes | | `content` | String | Yes | | `createdAt` | Datetime | No | | `databaseId` | UUID | Yes | @@ -93,13 +94,13 @@ CRUD operations for SqlAction records. ```typescript // List all sqlAction records -const items = await db.sqlAction.findMany({ select: { actionId: true, actionName: true, actorId: true, content: true, createdAt: true, databaseId: true, deploy: true, deps: true, id: true, name: true, payload: true, revert: true, verify: true } }).execute(); +const items = await db.sqlAction.findMany({ select: { actionId: true, actionName: true, actorId: true, category: true, content: true, createdAt: true, databaseId: true, deploy: true, deps: true, id: true, name: true, payload: true, revert: true, verify: true } }).execute(); // Get one by id -const item = await db.sqlAction.findOne({ id: '', select: { actionId: true, actionName: true, actorId: true, content: true, createdAt: true, databaseId: true, deploy: true, deps: true, id: true, name: true, payload: true, revert: true, verify: true } }).execute(); +const item = await db.sqlAction.findOne({ id: '', select: { actionId: true, actionName: true, actorId: true, category: true, content: true, createdAt: true, databaseId: true, deploy: true, deps: true, id: true, name: true, payload: true, revert: true, verify: true } }).execute(); // Create -const created = await db.sqlAction.create({ data: { actionId: '', actionName: '', actorId: '', content: '', databaseId: '', deploy: '', deps: '', name: '', payload: '', revert: '', verify: '' }, select: { id: true } }).execute(); +const created = await db.sqlAction.create({ data: { actionId: '', actionName: '', actorId: '', category: '', content: '', databaseId: '', deploy: '', deps: '', name: '', payload: '', revert: '', verify: '' }, select: { id: true } }).execute(); // Update const updated = await db.sqlAction.update({ where: { id: '' }, data: { actionId: '' }, select: { id: true } }).execute(); diff --git a/sdk/migrate-client/src/migrate/orm/input-types.ts b/sdk/migrate-client/src/migrate/orm/input-types.ts index fa3df05750..848f9ec1b4 100644 --- a/sdk/migrate-client/src/migrate/orm/input-types.ts +++ b/sdk/migrate-client/src/migrate/orm/input-types.ts @@ -250,6 +250,7 @@ export interface SqlAction { actionId?: string | null; actionName?: string | null; actorId?: string | null; + category?: string | null; content?: string | null; createdAt?: string | null; databaseId?: string | null; @@ -299,6 +300,7 @@ export type SqlActionSelect = { actionId?: boolean; actionName?: boolean; actorId?: boolean; + category?: boolean; content?: boolean; createdAt?: boolean; databaseId?: boolean; @@ -354,6 +356,8 @@ export interface SqlActionFilter { actorId?: UUIDFilter; /** Checks for all expressions in this list. */ and?: SqlActionFilter[]; + /** Filter by the object’s `category` field. */ + category?: StringFilter; /** Filter by the object’s `content` field. */ content?: StringFilter; /** Filter by the object’s `createdAt` field. */ @@ -413,6 +417,8 @@ export type SqlActionOrderBy = | 'ACTION_NAME_DESC' | 'ACTOR_ID_ASC' | 'ACTOR_ID_DESC' + | 'CATEGORY_ASC' + | 'CATEGORY_DESC' | 'CONTENT_ASC' | 'CONTENT_DESC' | 'CREATED_AT_ASC' @@ -479,6 +485,7 @@ export interface CreateSqlActionInput { actionId: string; actionName?: string; actorId: string; + category?: string; content?: string; databaseId: string; deploy?: string; @@ -493,6 +500,7 @@ export interface SqlActionPatch { actionId?: string | null; actionName?: string | null; actorId?: string | null; + category?: string | null; content?: string | null; databaseId?: string | null; deploy?: string | null;