Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions pgpm/cli/src/commands/export.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,13 @@ Options:
--author <name> Project author (default: from git config)
--extensionName <name> Extension name
--metaExtensionName <name> Meta extension name (default: svc)
--exclude-categories <list> Comma-separated sql_actions categories to omit
(e.g. security,permissions,auth,memberships). Works in SQL and GraphQL modes.
--cwd <directory> Working directory (default: current directory)

Examples:
pgpm export Export migrations from selected database (SQL mode)
pgpm export --exclude-categories security,permissions,auth,memberships
pgpm export --graphql-endpoint 'http://[::1]:3002/graphql' --migrate-endpoint 'http://[::1]:3000/graphql' --migrate-host db_migrate.localhost:3000
`;

Expand All @@ -53,6 +56,14 @@ export default async (
const migrateHeadersRaw = argv['migrate-headers'] || argv.migrateHeaders;
const token = argv.token;

const excludeCategoriesRaw = argv['exclude-categories'] || argv.excludeCategories;
const excludeCategories: string[] | undefined =
typeof excludeCategoriesRaw === 'string'
? excludeCategoriesRaw.split(',').map((s: string) => s.trim()).filter(Boolean)
: Array.isArray(excludeCategoriesRaw)
? excludeCategoriesRaw
: undefined;

if (graphqlEndpoint) {
// =========================================================================
// GraphQL export mode
Expand Down Expand Up @@ -168,7 +179,8 @@ export default async (
metaExtensionName,
prompter,
argv,
username
username,
excludeCategories
});
} else {
// =========================================================================
Expand Down Expand Up @@ -305,7 +317,8 @@ export default async (
metaExtensionName,
prompter,
argv,
username
username,
excludeCategories
});
}

Expand Down
155 changes: 155 additions & 0 deletions pgpm/export/__tests__/export-flow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -759,4 +759,159 @@ relocatable = false
}
});
});

describe('Export with excludeCategories', () => {
const DATABASE_ID = 'a1b2c3d4-e5f6-4708-b250-000000000001';
const OPEN_EXTENSION_NAME = 'pets-open';
const OPEN_META_EXTENSION_NAME = 'pets-open-svc';
const POLICY_DEPLOY = 'schemas/pets_public/tables/pets/policies/pets_policy';
let exportError: Error | null = null;

const scaffoldModule = (name: string, description: string) => {
const moduleDir = join(exportWorkspaceDir, 'packages', name);
mkdirSync(moduleDir, { recursive: true });
writeFileSync(join(moduleDir, 'package.json'), JSON.stringify({
name,
version: '1.0.0',
description
}, null, 2));
writeFileSync(join(moduleDir, `${name}.control`), `# ${name} extension
comment = '${description}'
default_version = '1.0.0'
relocatable = false
`);
writeFileSync(join(moduleDir, 'pgpm.plan'), `%syntax-version=1.0.0
%project=${name}
%uri=https://github.com/test/${name}
`);
};

beforeAll(async () => {
// Add a category column (the real db_migrate.sql_actions has one) and
// tag a policy action as 'security'
await pg.query(`ALTER TABLE db_migrate.sql_actions ADD COLUMN IF NOT EXISTS category text DEFAULT 'ddl'`);
await pg.query(
`INSERT INTO db_migrate.sql_actions (name, database_id, deploy, deps, content, revert, verify, category)
VALUES (
'Create pets policy',
$1,
$2,
ARRAY['schemas/pets_public/tables/pets/table']::text[],
'CREATE POLICY pets_policy ON pets_public.pets FOR SELECT USING (true);',
'DROP POLICY pets_policy ON pets_public.pets;',
'SELECT 1;',
'security'
)
ON CONFLICT (database_id, deploy) DO NOTHING`,
[DATABASE_ID, POLICY_DEPLOY]
);

const schemaResult = await pg.query(
'SELECT schema_name FROM metaschema_public.schema WHERE database_id = $1',
[DATABASE_ID]
);
const schemaNames = schemaResult.rows.map((r: any) => r.schema_name);

scaffoldModule(OPEN_EXTENSION_NAME, 'Exported pets database schema (security excluded)');
scaffoldModule(OPEN_META_EXTENSION_NAME, 'Exported pets service metadata (security excluded)');

const project = new PgpmPackage(exportWorkspaceDir);

try {
await exportMigrations({
project,
options: {
pg: dbConfig
},
dbInfo: {
dbname: dbConfig.database,
databaseName: 'pets',
database_ids: [DATABASE_ID]
},
author: 'test <test@test.local>',
outdir: join(exportWorkspaceDir, 'packages'),
schema_names: schemaNames,
extensionName: OPEN_EXTENSION_NAME,
extensionDesc: 'Exported pets database schema (security excluded)',
metaExtensionName: OPEN_META_EXTENSION_NAME,
metaExtensionDesc: 'Exported pets service metadata (security excluded)',
excludeCategories: ['security']
});
} catch (err) {
exportError = err as Error;
}
}, 180000);

it('should have completed export without errors', () => {
if (exportError) {
console.error('Export error:', exportError);
}
expect(exportError).toBeNull();
});

it('should omit security-categorized actions from the plan', () => {
const planPath = join(exportWorkspaceDir, 'packages', OPEN_EXTENSION_NAME, 'pgpm.plan');
expect(existsSync(planPath)).toBe(true);

// The replacer renames pets_public -> pets_open_public in the exported module
const planContent = readFileSync(planPath, 'utf-8');
expect(planContent).not.toContain('/policies/pets_policy');
// Non-security actions (category 'ddl' via the column default) are kept
expect(planContent).toContain('schemas/pets_open_public/tables/pets/table');
});

it('should not write deploy files for excluded actions', () => {
const policyDeployPath = join(
exportWorkspaceDir, 'packages', OPEN_EXTENSION_NAME, 'deploy',
'schemas/pets_open_public/tables/pets/policies/pets_policy.sql'
);
expect(existsSync(policyDeployPath)).toBe(false);

const tableDeployPath = join(
exportWorkspaceDir, 'packages', OPEN_EXTENSION_NAME, 'deploy', 'schemas/pets_open_public/tables/pets/table.sql'
);
expect(existsSync(tableDeployPath)).toBe(true);
});

it('should include security actions when excludeCategories is not passed', async () => {
const FULL_EXTENSION_NAME = 'pets-full';
const FULL_META_EXTENSION_NAME = 'pets-full-svc';

const schemaResult = await pg.query(
'SELECT schema_name FROM metaschema_public.schema WHERE database_id = $1',
[DATABASE_ID]
);
const schemaNames = schemaResult.rows.map((r: any) => r.schema_name);

scaffoldModule(FULL_EXTENSION_NAME, 'Exported pets database schema (full)');
scaffoldModule(FULL_META_EXTENSION_NAME, 'Exported pets service metadata (full)');

const project = new PgpmPackage(exportWorkspaceDir);

await exportMigrations({
project,
options: {
pg: dbConfig
},
dbInfo: {
dbname: dbConfig.database,
databaseName: 'pets',
database_ids: [DATABASE_ID]
},
author: 'test <test@test.local>',
outdir: join(exportWorkspaceDir, 'packages'),
schema_names: schemaNames,
extensionName: FULL_EXTENSION_NAME,
extensionDesc: 'Exported pets database schema (full)',
metaExtensionName: FULL_META_EXTENSION_NAME,
metaExtensionDesc: 'Exported pets service metadata (full)'
});

const planContent = readFileSync(
join(exportWorkspaceDir, 'packages', FULL_EXTENSION_NAME, 'pgpm.plan'),
'utf-8'
);
expect(planContent).toContain('schemas/pets_full_public/tables/pets/policies/pets_policy');
}, 180000);
});
});
22 changes: 19 additions & 3 deletions pgpm/export/src/export-graphql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@ export interface ExportGraphQLOptions {
username?: string;
serviceOutdir?: string;
skipSchemaRenaming?: boolean;
/**
* sql_actions categories to exclude from the export (e.g. ['security',
* 'permissions']). Rows whose category matches are omitted; rows with a
* null category are always kept.
*/
excludeCategories?: string[];
}

export const exportGraphQL = async ({
Expand All @@ -94,7 +100,8 @@ export const exportGraphQL = async ({
repoName,
username,
serviceOutdir,
skipSchemaRenaming = false
skipSchemaRenaming = false,
excludeCategories
}: ExportGraphQLOptions): Promise<void> => {
const normalizedOutdir = normalizeOutdir(outdir);
const svcOutdir = normalizeOutdir(serviceOutdir || outdir);
Expand Down Expand Up @@ -147,10 +154,19 @@ export const exportGraphQL = async ({
actionName: true,
actionId: true,
actorId: true,
payload: true
payload: true,
category: true
},
where: {
databaseId: { equalTo: databaseId }
databaseId: { equalTo: databaseId },
...(excludeCategories && excludeCategories.length > 0
? {
or: [
{ category: { isNull: true } },
{ category: { notIn: excludeCategories } }
]
}
: {})
},
orderBy: ['ID_ASC'],
first: PAGE_SIZE,
Expand Down
46 changes: 34 additions & 12 deletions pgpm/export/src/export-migrations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,12 @@ interface ExportMigrationsToDiskOptions {
* Useful for self-referential introspection where you want to apply policies to real schemas.
*/
skipSchemaRenaming?: boolean;
/**
* Action categories to exclude from the export (e.g. ['security', 'permissions']).
* Rows in db_migrate.sql_actions whose category matches are omitted from the
* generated pgpm plan and SQL files. Rows with a NULL category are always kept.
*/
excludeCategories?: string[];
}

interface ExportOptions {
Expand Down Expand Up @@ -75,6 +81,12 @@ interface ExportOptions {
* Useful for self-referential introspection where you want to apply policies to real schemas.
*/
skipSchemaRenaming?: boolean;
/**
* Action categories to exclude from the export (e.g. ['security', 'permissions']).
* Rows in db_migrate.sql_actions whose category matches are omitted from the
* generated pgpm plan and SQL files. Rows with a NULL category are always kept.
*/
excludeCategories?: string[];
}

const exportMigrationsToDisk = async ({
Expand All @@ -95,7 +107,8 @@ const exportMigrationsToDisk = async ({
repoName,
username,
serviceOutdir,
skipSchemaRenaming = false
skipSchemaRenaming = false,
excludeCategories
}: ExportMigrationsToDiskOptions): Promise<void> => {
const normalizedOutdir = normalizeOutdir(outdir);
// Use serviceOutdir for service module, defaulting to outdir if not provided
Expand Down Expand Up @@ -140,13 +153,22 @@ const exportMigrationsToDisk = async ({
name
});

// Filter sql_actions by database_id to avoid cross-database pollution
// Previously this query had no WHERE clause, which could export actions
// from unrelated databases in a persistent database environment
const results = await pgPool.query(
`select * from db_migrate.sql_actions where database_id = $1 order by id`,
[databaseId]
);
// Filter sql_actions by database_id to avoid cross-database pollution.
// Category exclusion is applied here explicitly rather than via the
// export.exclude_categories session GUC, which only affects the session
// it was set on and therefore cannot be relied upon across connections.
const results = excludeCategories && excludeCategories.length > 0
? await pgPool.query(
`select * from db_migrate.sql_actions
where database_id = $1
and (category is null or category != ALL($2::text[]))
order by id`,
[databaseId, excludeCategories]
)
: await pgPool.query(
`select * from db_migrate.sql_actions where database_id = $1 order by id`,
[databaseId]
);

const opts: SqlWriteOptions = {
name,
Expand Down Expand Up @@ -276,8 +298,6 @@ ${META_COMMON_FOOTER}

writePgpmPlan(metaPackage, opts);
writePgpmFiles(metaPackage, opts);

pgPool.end();
};

export const exportMigrations = async ({
Expand All @@ -296,7 +316,8 @@ export const exportMigrations = async ({
repoName,
username,
serviceOutdir,
skipSchemaRenaming
skipSchemaRenaming,
excludeCategories
}: ExportOptions): Promise<void> => {
for (let v = 0; v < dbInfo.database_ids.length; v++) {
const databaseId = dbInfo.database_ids[v];
Expand All @@ -318,7 +339,8 @@ export const exportMigrations = async ({
repoName,
username,
serviceOutdir,
skipSchemaRenaming
skipSchemaRenaming,
excludeCategories
});
}
};
6 changes: 6 additions & 0 deletions sdk/migrate-client/schemas/migrate.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -662,6 +662,7 @@ type SqlAction {
actionId: UUID
actionName: String
actorId: UUID
category: String
content: String
createdAt: Datetime
databaseId: UUID
Expand Down Expand Up @@ -716,6 +717,9 @@ input SqlActionFilter {
"""Checks for all expressions in this list."""
and: [SqlActionFilter!]

"""Filter by the object’s `category` field."""
category: StringFilter

"""Filter by the object’s `content` field."""
content: StringFilter

Expand Down Expand Up @@ -758,6 +762,8 @@ enum SqlActionOrderBy {
ACTION_NAME_DESC
ACTOR_ID_ASC
ACTOR_ID_DESC
CATEGORY_ASC
CATEGORY_DESC
CONTENT_ASC
CONTENT_DESC
CREATED_AT_ASC
Expand Down
Loading
Loading