Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions cmd/dump/dump_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,13 @@ func TestDumpCommand_Issue674ConstraintComment(t *testing.T) {
runExactMatchTest(t, "issue_674_constraint_comment")
}

func TestDumpCommand_Issue446DuplicateIndexConstraint(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
runExactMatchTest(t, "issue_446_duplicate_index_constraint")
}

func TestDumpCommand_Issue653InheritedConstraintIndexName(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
Expand Down
26 changes: 26 additions & 0 deletions internal/diff/constraint.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package diff

import (
"fmt"
"regexp"
"slices"
"strings"

Expand Down Expand Up @@ -162,6 +163,31 @@ func getInlineConstraintsForTable(table *ir.Table) []*ir.Constraint {
return inlineConstraints
}

// exclusionStorageParameters matches the WITH (...) storage parameters in the
// pg_get_constraintdef() text of an EXCLUDE constraint. An element of the
// constraint is "<expr> WITH <operator>", without a parenthesis after WITH.
var exclusionStorageParameters = regexp.MustCompile(` WITH \([^()]*\)`)

// indexConstraintKey identifies the index of a PRIMARY KEY, UNIQUE or EXCLUDE
// constraint by the fields that PostgreSQL compares to find duplicate index
// constraints in CREATE TABLE. It returns "" for other constraint types.
func indexConstraintKey(constraint *ir.Constraint) string {
switch constraint.Type {
case ir.ConstraintTypePrimaryKey, ir.ConstraintTypeUnique:
columns := make([]string, len(constraint.Columns))
for i, column := range constraint.Columns {
columns[i] = column.Name
}
return fmt.Sprintf("%q %t %q %t %t %t", columns, constraint.IsTemporal, constraint.IncludeColumns,
constraint.NullsNotDistinct, constraint.Deferrable, constraint.InitiallyDeferred)
case ir.ConstraintTypeExclusion:
// PostgreSQL does not compare the storage parameters of the indexes.
return exclusionStorageParameters.ReplaceAllString(constraint.ExclusionDefinition, "")
default:
return ""
}
}

// constraintsStructurallyEqual compares two constraints' definitions. A
// comment-only difference is applied with COMMENT ON CONSTRAINT instead of
// recreating the constraint (issue #674).
Expand Down
83 changes: 83 additions & 0 deletions internal/diff/constraint_duplicate_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package diff

import (
"strings"
"testing"

"github.com/pgplex/pgschema/ir"
"github.com/pgplex/pgschema/testutil"
"github.com/stretchr/testify/require"
)

// CREATE TABLE silently drops a key whose index equals an earlier one, so the
// generated SQL must add each such key after the CREATE TABLE (issue #446).
func TestDuplicateIndexConstraintRoundTrip(t *testing.T) {
db, _, _, _, _, _ := testutil.ConnectToPostgres(t, sharedTestPostgres)
defer db.Close()
majorVersion, err := testutil.GetMajorVersion(db)
require.NoError(t, err)

source := `
CREATE TABLE products (sku text PRIMARY KEY, name text, a int, b int,
CONSTRAINT products_name_key UNIQUE (name),
CONSTRAINT products_a_b_key UNIQUE (a, b),
CONSTRAINT products_b_a_key UNIQUE (b, a),
CONSTRAINT products_sku_include UNIQUE (sku) INCLUDE (name),
CONSTRAINT products_sku_deferrable UNIQUE (sku) DEFERRABLE,
CONSTRAINT products_sku_deferred UNIQUE (sku) DEFERRABLE INITIALLY DEFERRED,
CONSTRAINT products_a_excl EXCLUDE USING btree (a WITH =),
CONSTRAINT products_b_excl EXCLUDE USING btree (b WITH =) WITH (fillfactor=70));
ALTER TABLE products ADD CONSTRAINT products_sku_key UNIQUE (sku);
ALTER TABLE products ADD CONSTRAINT products_name_unique UNIQUE (name);
ALTER TABLE products ADD CONSTRAINT products_sku_deferrable2 UNIQUE (sku) DEFERRABLE;
ALTER TABLE products ADD CONSTRAINT products_a_excl2 EXCLUDE USING btree (a WITH =);
ALTER TABLE products ADD CONSTRAINT products_b_excl2 EXCLUDE USING btree (b WITH =);
COMMENT ON CONSTRAINT products_sku_key ON products IS 'target of order_lines_sku_fkey';
CREATE TABLE order_lines (id int PRIMARY KEY, sku text REFERENCES products (sku));
CREATE TABLE events (id int) PARTITION BY RANGE (id);
CREATE TABLE events_leaf PARTITION OF events (CONSTRAINT events_leaf_pkey PRIMARY KEY (id)) FOR VALUES FROM (0) TO (10);
ALTER TABLE events_leaf ADD CONSTRAINT events_leaf_id_key UNIQUE (id);
`
if majorVersion >= 15 {
source += `ALTER TABLE products ADD CONSTRAINT products_name_nulls_key UNIQUE NULLS NOT DISTINCT (name);`
}
original := testutil.ParseSQLToIR(t, sharedTestPostgres, source, "public")

for _, mode := range []string{"dump", "qualified dump", "initial migration"} {
t.Run(mode, func(t *testing.T) {
var statements []Diff
var err error
prefix := ""
switch mode {
case "dump":
statements, err = GenerateDump(original, "public", false)
require.NoError(t, err)
case "qualified dump":
statements, err = GenerateDump(original, "public", true)
require.NoError(t, err)
prefix = "public."
default:
statements = GenerateMigration(&ir.IR{}, original, "public")
}
added := []string{
"ALTER TABLE " + prefix + "products\nADD CONSTRAINT products_a_excl2 EXCLUDE USING btree (a WITH =);",
// PostgreSQL ignores the storage parameters when it compares the indexes.
"ALTER TABLE " + prefix + "products\nADD CONSTRAINT products_b_excl2 EXCLUDE USING btree (b WITH =);",
"ALTER TABLE " + prefix + "products\nADD CONSTRAINT products_name_unique UNIQUE (name);",
"ALTER TABLE " + prefix + "products\nADD CONSTRAINT products_sku_deferrable2 UNIQUE (sku) DEFERRABLE;",
"ALTER TABLE " + prefix + "products\nADD CONSTRAINT products_sku_key UNIQUE (sku);",
"ALTER TABLE " + prefix + "events_leaf\nADD CONSTRAINT events_leaf_id_key UNIQUE (id);",
}
sqlText := buildSQLFromSteps(statements)
for _, statement := range added {
require.Contains(t, sqlText, statement)
}
require.Equal(t, len(added), strings.Count(sqlText, "ADD CONSTRAINT"), sqlText)
comment := strings.Index(sqlText, "COMMENT ON CONSTRAINT products_sku_key")
require.Greater(t, comment, strings.Index(sqlText, added[4]))

replayed := testutil.ParseSQLToIR(t, sharedTestPostgres, sqlText, "public")
require.Empty(t, GenerateMigration(replayed, original, "public"))
})
}
}
6 changes: 4 additions & 2 deletions internal/diff/qualify_schema_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ func TestQualifySchema_TableAndColumnType(t *testing.T) {
}
empty := map[string]bool{}

def, _, _ := generateTableSQL(table, "public", false, empty, empty, empty, nil, nil)
statements, _, _ := generateTableSQL(table, "public", false, empty, empty, empty, nil, nil)
def := statements[0]
if !strings.Contains(def, "CREATE TABLE IF NOT EXISTS account (") {
t.Errorf("default should use the bare table name: %q", def)
}
Expand All @@ -71,7 +72,8 @@ func TestQualifySchema_TableAndColumnType(t *testing.T) {
t.Errorf("default should strip the target-schema prefix from the type ref: %q", def)
}

qualified, _, _ := generateTableSQL(table, "public", true, empty, empty, empty, nil, nil)
statements, _, _ = generateTableSQL(table, "public", true, empty, empty, empty, nil, nil)
qualified := statements[0]
if !strings.Contains(qualified, "CREATE TABLE IF NOT EXISTS public.account (") {
t.Errorf("forced qualification should qualify the table name: %q", qualified)
}
Expand Down
97 changes: 52 additions & 45 deletions internal/diff/table.go
Original file line number Diff line number Diff line change
Expand Up @@ -555,7 +555,7 @@ func generateCreateTablesSQL(
// Process tables in the provided order (already topologically sorted)
for _, table := range tables {
// Create the table, deferring FK constraints that reference not-yet-created tables
sql, inlined, tableDeferred := generateTableSQL(table, targetSchema, collector.qualifySchema, createdTables, existingTables, suppressedInlineFKs, allNewTables, partitionIndexes)
tableStatements, created, tableDeferred := generateTableSQL(table, targetSchema, collector.qualifySchema, createdTables, existingTables, suppressedInlineFKs, allNewTables, partitionIndexes)
deferredConstraints = append(deferredConstraints, tableDeferred...)

// Create context for this statement
Expand All @@ -570,7 +570,10 @@ func generateCreateTablesSQL(
// A child's local DEFAULT / NOT NULL on inherited columns belong to
// the same step, unless a parent still has to add the column
// (see inherit.go)
statements := []SQLStatement{{SQL: sql, CanRunInTransaction: true}}
var statements []SQLStatement
for _, sql := range tableStatements {
statements = append(statements, SQLStatement{SQL: sql, CanRunInTransaction: true})
}
for _, override := range inheritedColumnOverrideSQL(table, targetSchema, collector.qualifySchema, allNewTables, parentAddedColumns) {
statements = append(statements, SQLStatement{SQL: override, CanRunInTransaction: true})
}
Expand Down Expand Up @@ -612,9 +615,9 @@ func generateCreateTablesSQL(
}
}

// Add comments on the constraints created inline above. Deferred and
// postponed constraints get theirs where they are created (#674).
for _, constraint := range inlined {
// Add comments on the constraints created with the table above. Deferred
// and postponed constraints get theirs where they are created (#674).
for _, constraint := range created {
if constraint.Comment != "" {
generateConstraintComment(constraint, targetSchema, DiffOperationCreate, collector)
}
Expand Down Expand Up @@ -1045,9 +1048,10 @@ func generateDropTablesSQL(tables []*ir.Table, targetSchema string, collector *d
}

// generateTableSQL generates CREATE TABLE statement and returns any deferred FK constraints
// It returns the statement, the constraints created inline with it, and the
// foreign keys deferred until their referenced table exists.
func generateTableSQL(table *ir.Table, targetSchema string, qualifySchema bool, createdTables map[string]bool, existingTables map[string]bool, suppressedInlineFKs map[string]bool, allTables map[string]*ir.Table, partitionIndexes *partitionIndexCreationPlan) (string, []*ir.Constraint, []*deferredConstraint) {
// It returns the CREATE TABLE statement followed by the ALTER TABLE statements
// that add the keys it cannot inline, the constraints that these statements
// create, and the foreign keys deferred until their referenced table exists.
func generateTableSQL(table *ir.Table, targetSchema string, qualifySchema bool, createdTables map[string]bool, existingTables map[string]bool, suppressedInlineFKs map[string]bool, allTables map[string]*ir.Table, partitionIndexes *partitionIndexCreationPlan) ([]string, []*ir.Constraint, []*deferredConstraint) {
// Only include table name without schema if it's in the target schema (unless
// forced qualification is on).
tableName := ir.QualifyEntityNameWithQuotesMode(table.Schema, table.Name, targetSchema, qualifySchema)
Expand Down Expand Up @@ -1109,27 +1113,7 @@ func generateTableSQL(table *ir.Table, targetSchema string, qualifySchema bool,
}
}

var constraintParts []string
var inlined []*ir.Constraint
var deferred []*deferredConstraint
currentKey := fmt.Sprintf("%s.%s", table.Schema, table.Name)
for _, constraint := range getInlineConstraintsForTable(table) {
if suppressedInlineFKs[constraintPathKey(constraint)] || partitionIndexes.defersConstraint(table, constraint) {
continue
}
if shouldDeferConstraint(table, constraint, currentKey, createdTables, existingTables) ||
partitionIndexes.defersForeignKey(table, constraint) {
deferred = append(deferred, &deferredConstraint{
table: table,
constraint: constraint,
})
continue
}
if def := generateConstraintSQL(constraint, targetSchema, qualifySchema); def != "" {
constraintParts = append(constraintParts, fmt.Sprintf(" %s", def))
inlined = append(inlined, constraint)
}
}
constraintParts, added, created, deferred := newTableConstraintsSQL(table, tableName, targetSchema, qualifySchema, createdTables, existingTables, suppressedInlineFKs, partitionIndexes)

createPrefix := "CREATE TABLE IF NOT EXISTS"
if table.Unlogged {
Expand All @@ -1144,7 +1128,7 @@ func generateTableSQL(table *ir.Table, targetSchema string, qualifySchema bool,
} else {
sql = fmt.Sprintf("%s %s PARTITION OF %s %s%s;", createPrefix, tableName, parentName, table.PartitionBound, partitionClause)
}
return sql, inlined, deferred
return append([]string{sql}, added...), created, deferred
}

var parts []string
Expand Down Expand Up @@ -1182,8 +1166,30 @@ func generateTableSQL(table *ir.Table, targetSchema string, qualifySchema bool,
}

// Add constraints inline in the correct order (PRIMARY KEY, UNIQUE, FOREIGN KEY)
var inlined []*ir.Constraint
constraintParts, added, created, deferred := newTableConstraintsSQL(table, tableName, targetSchema, qualifySchema, createdTables, existingTables, suppressedInlineFKs, partitionIndexes)
columnParts = append(columnParts, constraintParts...)

// CHECK constraints the child declares as well as inherits (see inherit.go)
for _, def := range mergedInheritedCheckSQL(table, targetSchema, qualifySchema, allTables) {
columnParts = append(columnParts, fmt.Sprintf(" %s", def))
}

parts = append(parts, strings.Join(columnParts, ",\n"))

parts = append(parts, ")"+inheritsClauseSQL(table, targetSchema, qualifySchema)+partitionClause+";")

return append([]string{strings.Join(parts, "\n")}, added...), created, deferred
}

// newTableConstraintsSQL renders the constraints of a new table: the
// definitions inlined in its CREATE TABLE and the ALTER TABLE statements that
// must follow it. It also returns the constraints that these create and the
// foreign keys deferred until their referenced table exists.
func newTableConstraintsSQL(table *ir.Table, tableName string, targetSchema string, qualifySchema bool, createdTables map[string]bool, existingTables map[string]bool, suppressedInlineFKs map[string]bool, partitionIndexes *partitionIndexCreationPlan) ([]string, []string, []*ir.Constraint, []*deferredConstraint) {
var inline, added []string
var created []*ir.Constraint
var deferred []*deferredConstraint
inlinedIndexes := make(map[string]bool)
currentKey := fmt.Sprintf("%s.%s", table.Schema, table.Name)
for _, constraint := range getInlineConstraintsForTable(table) {
// FKs bound to a unique/PK constraint being replaced by this migration
Expand All @@ -1200,22 +1206,23 @@ func generateTableSQL(table *ir.Table, targetSchema string, qualifySchema bool,
continue
}
constraintDef := generateConstraintSQL(constraint, targetSchema, qualifySchema)
if constraintDef != "" {
columnParts = append(columnParts, fmt.Sprintf(" %s", constraintDef))
inlined = append(inlined, constraint)
if constraintDef == "" {
continue
}
created = append(created, constraint)
// CREATE TABLE silently drops an index constraint equal to an earlier
// one (transformIndexConstraints in parse_utilcmd.c); ALTER TABLE keeps
// it (issue #446).
if key := indexConstraintKey(constraint); key != "" {
if inlinedIndexes[key] {
added = append(added, fmt.Sprintf("ALTER TABLE %s\nADD %s;", tableName, constraintDef))
continue
}
inlinedIndexes[key] = true
}
inline = append(inline, fmt.Sprintf(" %s", constraintDef))
}

// CHECK constraints the child declares as well as inherits (see inherit.go)
for _, def := range mergedInheritedCheckSQL(table, targetSchema, qualifySchema, allTables) {
columnParts = append(columnParts, fmt.Sprintf(" %s", def))
}

parts = append(parts, strings.Join(columnParts, ",\n"))

parts = append(parts, ")"+inheritsClauseSQL(table, targetSchema, qualifySchema)+partitionClause+";")

return strings.Join(parts, "\n"), inlined, deferred
return inline, added, created, deferred
}

func shouldDeferConstraint(table *ir.Table, constraint *ir.Constraint, currentKey string, createdTables map[string]bool, existingTables map[string]bool) bool {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
CREATE TABLE IF NOT EXISTS products (
sku text,
name text,
CONSTRAINT products_pkey PRIMARY KEY (sku),
CONSTRAINT products_name_key UNIQUE (name)
);

ALTER TABLE products
ADD CONSTRAINT products_name_unique UNIQUE (name);

ALTER TABLE products
ADD CONSTRAINT products_sku_key UNIQUE (sku);

CREATE TABLE IF NOT EXISTS order_lines (
id integer,
sku text NOT NULL,
CONSTRAINT order_lines_pkey PRIMARY KEY (id),
CONSTRAINT order_lines_sku_fkey FOREIGN KEY (sku) REFERENCES products (sku)
);
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
-- CREATE TABLE drops a key whose index equals an earlier one, so the
-- duplicate keys are added with ALTER TABLE.
CREATE TABLE products (
sku text PRIMARY KEY,
name text,
CONSTRAINT products_name_key UNIQUE (name)
);
ALTER TABLE products ADD CONSTRAINT products_sku_key UNIQUE (sku);
ALTER TABLE products ADD CONSTRAINT products_name_unique UNIQUE (name);

CREATE TABLE order_lines (
id integer PRIMARY KEY,
sku text NOT NULL REFERENCES products (sku)
);
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"version": "1.0.0",
"pgschema_version": "1.14.1",
"created_at": "1970-01-01T00:00:00Z",
"source_fingerprint": {
"hash": "965b1131737c955e24c7f827c55bd78e4cb49a75adfd04229e0ba297376f5085"
},
"groups": [
{
"steps": [
{
"sql": "CREATE TABLE IF NOT EXISTS products (\n sku text,\n name text,\n CONSTRAINT products_pkey PRIMARY KEY (sku),\n CONSTRAINT products_name_key UNIQUE (name)\n);",
"type": "table",
"operation": "create",
"path": "public.products"
},
{
"sql": "ALTER TABLE products\nADD CONSTRAINT products_name_unique UNIQUE (name);",
"type": "table",
"operation": "create",
"path": "public.products"
},
{
"sql": "ALTER TABLE products\nADD CONSTRAINT products_sku_key UNIQUE (sku);",
"type": "table",
"operation": "create",
"path": "public.products"
},
{
"sql": "CREATE TABLE IF NOT EXISTS order_lines (\n id integer,\n sku text NOT NULL,\n CONSTRAINT order_lines_pkey PRIMARY KEY (id),\n CONSTRAINT order_lines_sku_fkey FOREIGN KEY (sku) REFERENCES products (sku)\n);",
"type": "table",
"operation": "create",
"path": "public.order_lines"
}
]
}
]
}
Loading
Loading