diff --git a/internal/diff/diff.go b/internal/diff/diff.go index 53432efd2..c4b48ddea 100644 --- a/internal/diff/diff.go +++ b/internal/diff/diff.go @@ -2011,13 +2011,13 @@ func (d *ddlDiff) collectMigrationSQL(targetSchema string, collector *diffCollec } // First: Drop operations (in reverse dependency order) - d.generateDropSQL(targetSchema, collector, preDroppedViews) + typeDropsAfterTables := d.generateDropSQL(targetSchema, collector, preDroppedViews) // Then: Create operations (in dependency order) d.generateCreateSQL(targetSchema, collector) // Finally: Modify operations - d.generateModifySQL(targetSchema, collector, preDroppedViews) + d.generateModifySQL(targetSchema, collector, preDroppedViews, typeDropsAfterTables) } // generatePreDropMaterializedViewsSQL drops materialized views that depend on @@ -3085,7 +3085,7 @@ func (d *ddlDiff) generateCreateSQL(targetSchema string, collector *diffCollecto // generateModifySQL generates ALTER statements // preDroppedViews contains views that were already dropped in the pre-drop phase -func (d *ddlDiff) generateModifySQL(targetSchema string, collector *diffCollector, preDroppedViews map[string]bool) { +func (d *ddlDiff) generateModifySQL(targetSchema string, collector *diffCollector, preDroppedViews map[string]bool, typeDropsAfterTables []droppedTypeOrFunction) { // Modify schemas // Note: Schema modification is out of scope for schema-level comparisons @@ -3105,6 +3105,10 @@ func (d *ddlDiff) generateModifySQL(targetSchema string, collector *diffCollecto // Columns inheritance descendants drop once every parent has (see inherit.go) generateInheritanceChildDropsSQL(d.inheritanceChildEffects, targetSchema, collector) + // Drop the types held back by generateDropSQL, now that the columns using + // them have been dropped or changed to another type (issue #710) + generateDropTypesAndFunctionsSQL(typeDropsAfterTables, nil, targetSchema, collector) + // Attach OWNED BY for explicitly created sequences whose owning column was // created by this migration, either with a new table (create phase) or by // ALTER TABLE ... ADD COLUMN just above. The sequence itself was created @@ -3207,7 +3211,11 @@ func (d *ddlDiff) generateModifySQL(targetSchema string, collector *diffCollecto // generateDropSQL generates DROP statements in reverse dependency order // preDroppedViews contains views that were already dropped in the pre-drop phase -func (d *ddlDiff) generateDropSQL(targetSchema string, collector *diffCollector, preDroppedViews map[string]bool) { +// generateDropSQL emits the drop phase. It returns the type drops (with the +// functions those types reference) that must wait for the modify phase, +// because a column of a modified table still uses the type until that table's +// ALTER TABLE has run (issue #710). +func (d *ddlDiff) generateDropSQL(targetSchema string, collector *diffCollector, preDroppedViews map[string]bool) []droppedTypeOrFunction { // REVOKE privileges BEFORE dropping objects (objects must exist for REVOKE to succeed) // Skip REVOKEs on relations already dropped in the pre-drop phase: REVOKE has @@ -3297,7 +3305,32 @@ func (d *ddlDiff) generateDropSQL(targetSchema string, collector *diffCollector, // Drop types, interleaved with the held-back functions in dependency // order: a range before its subtype_diff function, that function before // the subtype its signature takes, a domain before its CHECK function. - for _, node := range orderTypeAndFunctionDrops(droppedTypes, functionsAfterTypes) { + // A dropped type that a modified table's column still uses waits, along + // with whatever must be dropped after it, until that table has been + // altered (issue #710). The old definition of a replaced type cannot + // wait: its new definition is created in the create phase. + usedByColumns := typesUsedByAlteredColumns(d.droppedTypes, d.modifiedTables) + now, held := orderTypeAndFunctionDrops(droppedTypes, functionsAfterTypes, usedByColumns) + var afterTables []droppedTypeOrFunction + for _, node := range held { + if _, ok := replacedOld[node.typeObj]; ok && node.typeObj != nil { + now = append(now, node) + } else { + afterTables = append(afterTables, node) + } + } + generateDropTypesAndFunctionsSQL(now, replacedOld, targetSchema, collector) + + // Drop schemas + // Note: Schema deletion is out of scope for schema-level comparisons + + return afterTables +} + +// generateDropTypesAndFunctionsSQL emits the drops in the given order. The +// old definition of a replaced type is reported against its diff. +func generateDropTypesAndFunctionsSQL(nodes []droppedTypeOrFunction, replacedOld map[*ir.Type]*typeDiff, targetSchema string, collector *diffCollector) { + for _, node := range nodes { if node.function != nil { generateDropFunctionsSQL([]*ir.Function{node.function}, targetSchema, collector) continue @@ -3308,9 +3341,41 @@ func (d *ddlDiff) generateDropSQL(targetSchema string, collector *diffCollector, } generateDropTypeSQL(node.typeObj, source, targetSchema, collector) } +} - // Drop schemas - // Note: Schema deletion is out of scope for schema-level comparisons +// typesUsedByAlteredColumns returns the dropped types that a column of a +// modified table uses in the current state: a column the migration drops, or +// one it changes to another type. Such a type can only be dropped once the +// table has been altered. +func typesUsedByAlteredColumns(droppedTypes []*ir.Type, modifiedTables []*tableDiff) map[*ir.Type]bool { + if len(droppedTypes) == 0 || len(modifiedTables) == 0 { + return nil + } + byKey := make(map[string]*ir.Type, len(droppedTypes)) + for _, t := range droppedTypes { + byKey[typeGraphKey(t.Schema, t.Name)] = t + // A column typed as a range's multirange depends on the range. + if t.Kind == ir.TypeKindRange && t.MultirangeTypeName != "" { + if key := extractTypeName(t.MultirangeTypeName, t.Schema); key != "" { + byKey[key] = t + } + } + } + used := make(map[*ir.Type]bool) + mark := func(table *ir.Table, column *ir.Column) { + if t, ok := byKey[extractTypeName(column.DataType, table.Schema)]; ok { + used[t] = true + } + } + for _, td := range modifiedTables { + for _, column := range td.DroppedColumns { + mark(td.Table, column) + } + for _, columnDiff := range td.ModifiedColumns { + mark(td.Table, columnDiff.Old) + } + } + return used } // filterPreDroppedViews returns views that haven't been pre-dropped @@ -4756,7 +4821,10 @@ type droppedTypeOrFunction struct { // signature uses (that subtype_diff takes the subtype), and a caller before // its callee. Ties break with types first, then by name, so the output stays // deterministic. -func orderTypeAndFunctionDrops(types []*ir.Type, functions []*ir.Function) []droppedTypeOrFunction { +// +// The held types, and everything that has to be dropped after one of them, +// are returned separately (in the same order) for the caller to emit later. +func orderTypeAndFunctionDrops(types []*ir.Type, functions []*ir.Function, held map[*ir.Type]bool) (now, later []droppedTypeOrFunction) { typeMap, _, typeEdges := buildTypeGraph(types) typeKey := func(t *ir.Type) string { return "t\x00" + typeGraphKey(t.Schema, t.Name) } fnKey := func(f *ir.Function) string { return "f\x00" + f.Schema + "." + f.Name + "(" + f.GetArguments() + ")" } @@ -4817,6 +4885,26 @@ func orderTypeAndFunctionDrops(types []*ir.Type, functions []*ir.Function) []dro } } + // Holding a node holds everything that must be dropped after it. + heldKeys := make(map[string]bool) + var queue []string + for _, t := range types { + if held[t] { + heldKeys[typeKey(t)] = true + queue = append(queue, typeKey(t)) + } + } + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + for _, next := range after[current] { + if _, exists := nodes[next]; exists && !heldKeys[next] { + heldKeys[next] = true + queue = append(queue, next) + } + } + } + var ready []string for key := range nodes { if inDegree[key] == 0 { @@ -4824,9 +4912,8 @@ func orderTypeAndFunctionDrops(types []*ir.Type, functions []*ir.Function) []dro } } sort.Strings(ready) // "f\x00" sorts before "t\x00"; reversed below so types lead - var result []droppedTypeOrFunction done := make(map[string]bool, len(nodes)) - for len(result) < len(nodes) { + for len(done) < len(nodes) { if len(ready) == 0 { // A cycle cannot occur in PostgreSQL's catalog; keep going // deterministically rather than drop nodes on the floor. @@ -4845,7 +4932,11 @@ func orderTypeAndFunctionDrops(types []*ir.Type, functions []*ir.Function) []dro continue } done[current] = true - result = append(result, nodes[current]) + if heldKeys[current] { + later = append(later, nodes[current]) + } else { + now = append(now, nodes[current]) + } for _, next := range after[current] { inDegree[next]-- if inDegree[next] <= 0 && !done[next] { @@ -4854,7 +4945,7 @@ func orderTypeAndFunctionDrops(types []*ir.Type, functions []*ir.Function) []dro } } } - return result + return now, later } // typeReferencesNewFunction determines if a domain references any newly added diff --git a/internal/diff/type_test.go b/internal/diff/type_test.go index 2988e44ff..62d0fbafd 100644 --- a/internal/diff/type_test.go +++ b/internal/diff/type_test.go @@ -213,7 +213,8 @@ func TestOrderTypeAndFunctionDrops(t *testing.T) { spanSet := &ir.Type{Schema: "public", Name: "busy", Kind: ir.TypeKindDomain, BaseType: "public.positive_span_multirange"} var names []string - for _, node := range orderTypeAndFunctionDrops([]*ir.Type{positive, span, spanSet}, []*ir.Function{isPositive, positiveDiff}) { + nodes, _ := orderTypeAndFunctionDrops([]*ir.Type{positive, span, spanSet}, []*ir.Function{isPositive, positiveDiff}, nil) + for _, node := range nodes { if node.function != nil { names = append(names, node.function.Name) } else { @@ -232,7 +233,8 @@ func TestOrderTypeAndFunctionDrops(t *testing.T) { bodyFn := &ir.Function{Schema: "public", Name: "span_diff", TypeDependencies: []string{"public.z_status"}, Parameters: []*ir.Parameter{{Mode: "IN", DataType: "integer"}, {Mode: "IN", DataType: "integer"}}} intSpan := &ir.Type{Schema: "public", Name: "span", Kind: ir.TypeKindRange, Subtype: "integer", SubtypeDiff: "span_diff", MultirangeTypeName: "span_multirange"} names = nil - for _, node := range orderTypeAndFunctionDrops([]*ir.Type{status, intSpan}, []*ir.Function{bodyFn}) { + nodes, _ = orderTypeAndFunctionDrops([]*ir.Type{status, intSpan}, []*ir.Function{bodyFn}, nil) + for _, node := range nodes { if node.function != nil { names = append(names, node.function.Name) } else { @@ -251,7 +253,8 @@ func TestOrderTypeAndFunctionDrops(t *testing.T) { aSpan := &ir.Type{Schema: "public", Name: "a_span", Kind: ir.TypeKindRange, Subtype: "integer", SubtypeDiff: "a_diff", MultirangeTypeName: "a_span_multirange"} zSpan := &ir.Type{Schema: "public", Name: "z_span", Kind: ir.TypeKindRange, Subtype: "integer", SubtypeDiff: "z_diff", MultirangeTypeName: "z_span_multirange"} names = nil - for _, node := range orderTypeAndFunctionDrops([]*ir.Type{aSpan, zSpan}, []*ir.Function{aDiff, zDiff}) { + nodes, _ = orderTypeAndFunctionDrops([]*ir.Type{aSpan, zSpan}, []*ir.Function{aDiff, zDiff}, nil) + for _, node := range nodes { if node.function != nil { names = append(names, node.function.Name) } else { @@ -264,12 +267,32 @@ func TestOrderTypeAndFunctionDrops(t *testing.T) { // No held functions: plain reverse dependency order of the types. names = nil - for _, node := range orderTypeAndFunctionDrops([]*ir.Type{positive, span}, nil) { + nodes, _ = orderTypeAndFunctionDrops([]*ir.Type{positive, span}, nil, nil) + for _, node := range nodes { names = append(names, node.typeObj.Name) } if strings.Join(names, ",") != "positive_span,positive" { t.Fatalf("got %v", names) } + + // A type a modified table still uses is held back together with all + // that must be dropped after it; what depends on it still goes first. + names = nil + nodes, later := orderTypeAndFunctionDrops([]*ir.Type{positive, span, spanSet}, []*ir.Function{isPositive, positiveDiff}, map[*ir.Type]bool{span: true}) + for _, node := range nodes { + names = append(names, node.typeObj.Name) + } + names = append(names, "|") + for _, node := range later { + if node.function != nil { + names = append(names, node.function.Name) + } else { + names = append(names, node.typeObj.Name) + } + } + if strings.Join(names, ",") != "busy,|,positive_span,positive_diff,positive,is_positive" { + t.Fatalf("got %v", names) + } } func TestGenerateAlterCompositeStatements(t *testing.T) { diff --git a/testdata/diff/create_type/drop_type/diff.sql b/testdata/diff/create_type/drop_type/diff.sql index d7007085c..4a17856aa 100644 --- a/testdata/diff/create_type/drop_type/diff.sql +++ b/testdata/diff/create_type/drop_type/diff.sql @@ -1 +1,11 @@ +ALTER TABLE tickets DROP COLUMN priority; + +ALTER TABLE tickets DROP COLUMN busy; + +ALTER TABLE tickets ALTER COLUMN severity TYPE text USING severity::text; + +DROP TYPE IF EXISTS severity RESTRICT; + DROP TYPE IF EXISTS priority RESTRICT; + +DROP TYPE IF EXISTS period RESTRICT; diff --git a/testdata/diff/create_type/drop_type/new.sql b/testdata/diff/create_type/drop_type/new.sql index d18bc622e..014d9a128 100644 --- a/testdata/diff/create_type/drop_type/new.sql +++ b/testdata/diff/create_type/drop_type/new.sql @@ -2,4 +2,9 @@ CREATE TYPE public.status AS ENUM ( 'active', 'inactive', 'pending' -); \ No newline at end of file +); + +CREATE TABLE public.tickets ( + id integer PRIMARY KEY, + severity text +); diff --git a/testdata/diff/create_type/drop_type/old.sql b/testdata/diff/create_type/drop_type/old.sql index 5b20baf9b..64d8dfd8d 100644 --- a/testdata/diff/create_type/drop_type/old.sql +++ b/testdata/diff/create_type/drop_type/old.sql @@ -9,4 +9,23 @@ CREATE TYPE public.priority AS ENUM ( 'medium', 'high', 'urgent' -); \ No newline at end of file +); + +CREATE TYPE public.severity AS ENUM ( + 'minor', + 'major' +); + +CREATE TYPE public.period AS RANGE ( + subtype = date +); + +-- Columns using the dropped types go away, or change type, in the same +-- migration: they must be altered before DROP TYPE (issue #710). A multirange +-- column holds its range the same way. +CREATE TABLE public.tickets ( + id integer PRIMARY KEY, + priority public.priority, + severity public.severity, + busy public.period_multirange[] +); diff --git a/testdata/diff/create_type/drop_type/plan.json b/testdata/diff/create_type/drop_type/plan.json index 30508d6d4..702a11a2f 100644 --- a/testdata/diff/create_type/drop_type/plan.json +++ b/testdata/diff/create_type/drop_type/plan.json @@ -3,16 +3,46 @@ "pgschema_version": "1.14.1", "created_at": "1970-01-01T00:00:00Z", "source_fingerprint": { - "hash": "7739601f0084f192ccf2d01e2d6602820b2f4e39b5acdd7c4dd14c6d3b8a3da1" + "hash": "4e611f2c250cecd0eddab3611e85d8e7261c5c9d4ecae300961895c3e76e4fb8" }, "groups": [ { "steps": [ + { + "sql": "ALTER TABLE tickets DROP COLUMN priority;", + "type": "table.column", + "operation": "drop", + "path": "public.tickets.priority" + }, + { + "sql": "ALTER TABLE tickets DROP COLUMN busy;", + "type": "table.column", + "operation": "drop", + "path": "public.tickets.busy" + }, + { + "sql": "ALTER TABLE tickets ALTER COLUMN severity TYPE text USING severity::text;", + "type": "table.column", + "operation": "alter", + "path": "public.tickets.severity" + }, + { + "sql": "DROP TYPE IF EXISTS severity RESTRICT;", + "type": "type", + "operation": "drop", + "path": "public.severity" + }, { "sql": "DROP TYPE IF EXISTS priority RESTRICT;", "type": "type", "operation": "drop", "path": "public.priority" + }, + { + "sql": "DROP TYPE IF EXISTS period RESTRICT;", + "type": "type", + "operation": "drop", + "path": "public.period" } ] } diff --git a/testdata/diff/create_type/drop_type/plan.sql b/testdata/diff/create_type/drop_type/plan.sql index d7007085c..4a17856aa 100644 --- a/testdata/diff/create_type/drop_type/plan.sql +++ b/testdata/diff/create_type/drop_type/plan.sql @@ -1 +1,11 @@ +ALTER TABLE tickets DROP COLUMN priority; + +ALTER TABLE tickets DROP COLUMN busy; + +ALTER TABLE tickets ALTER COLUMN severity TYPE text USING severity::text; + +DROP TYPE IF EXISTS severity RESTRICT; + DROP TYPE IF EXISTS priority RESTRICT; + +DROP TYPE IF EXISTS period RESTRICT; diff --git a/testdata/diff/create_type/drop_type/plan.txt b/testdata/diff/create_type/drop_type/plan.txt index 9b8641456..7cc54d4df 100644 --- a/testdata/diff/create_type/drop_type/plan.txt +++ b/testdata/diff/create_type/drop_type/plan.txt @@ -1,12 +1,31 @@ -Plan: 1 to drop. +Plan: 1 to modify, 3 to drop. Summary by type: - types: 1 to drop + types: 3 to drop + tables: 1 to modify Types: + - period - priority + - severity + +Tables: + ~ tickets + - busy (column) + - priority (column) + ~ severity (column) DDL to be executed: -------------------------------------------------- +ALTER TABLE tickets DROP COLUMN priority; + +ALTER TABLE tickets DROP COLUMN busy; + +ALTER TABLE tickets ALTER COLUMN severity TYPE text USING severity::text; + +DROP TYPE IF EXISTS severity RESTRICT; + DROP TYPE IF EXISTS priority RESTRICT; + +DROP TYPE IF EXISTS period RESTRICT;