diff --git a/cmd/plan/external_db_integration_test.go b/cmd/plan/external_db_integration_test.go index dd92a343b..d2aef4dc5 100644 --- a/cmd/plan/external_db_integration_test.go +++ b/cmd/plan/external_db_integration_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "testing" + "github.com/pgplex/pgschema/cmd/util" "github.com/pgplex/pgschema/internal/postgres" "github.com/pgplex/pgschema/testutil" "github.com/stretchr/testify/assert" @@ -282,3 +283,82 @@ func TestExternalDatabase_SchemaIsolation(t *testing.T) { schema2 := extDB2.GetSchemaName() assert.NotEqual(t, schema1, schema2, "temporary schemas should have unique names") } + +// TestExternalDatabase_ExtensionOperatorInPublicPlansClean reproduces issue #697: +// a view in the public schema that uses an operator owned by an extension +// installed in public (hstore's ->) must plan as a no-op against the database +// it was dumped from when the desired state is built on an external plan +// database. +func TestExternalDatabase_ExtensionOperatorInPublicPlansClean(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + targetDB := testutil.SetupPostgres(t) + defer targetDB.Stop() + planDB := testutil.SetupPostgres(t) + defer planDB.Stop() + + targetConn, targetHost, targetPort, targetDatabase, targetUser, targetPassword := testutil.ConnectToPostgres(t, targetDB) + defer targetConn.Close() + // Operators pgschema does not manage, present on both databases: + // - public.+(integer, integer) duplicates pg_catalog's and is reachable + // only qualified: a fully shadowed name, kept as OPERATOR(public.+). + // - public.->(jsonb, text) duplicates pg_catalog's too, but hstore's + // ->(hstore, text) shares the name and is not shadowed, so the name + // is not exempted and the hstore views below must still plan clean. + // The operator functions live outside the managed schema so the plan + // does not see them as unmanaged objects to drop. + setupSQL := ` +CREATE EXTENSION hstore; +CREATE SCHEMA ops; +CREATE FUNCTION ops.int_add(integer, integer) RETURNS integer LANGUAGE sql IMMUTABLE AS 'SELECT $1 + $2'; +CREATE OPERATOR public.+ (LEFTARG = integer, RIGHTARG = integer, FUNCTION = ops.int_add); +CREATE FUNCTION ops.jsonb_arrow(jsonb, text) RETURNS jsonb LANGUAGE sql IMMUTABLE AS 'SELECT $1 -> $2'; +CREATE OPERATOR public.-> (LEFTARG = jsonb, RIGHTARG = text, FUNCTION = ops.jsonb_arrow); +` + // Every expression context that can carry an operator: column default and + // generated expression (deparsed under search_path = pg_catalog), CHECK + // constraint, index expression and predicate, routine parameter default, + // view and materialized view. The quoted alias must survive untouched. + schemaSQL := ` +CREATE TABLE t ( + id integer PRIMARY KEY, + h hstore, + d text DEFAULT ('a=>1'::hstore -> 'a'), + g text GENERATED ALWAYS AS (h -> 'a') STORED, + CONSTRAINT c CHECK ((h ? 'k') OR h IS NULL) +); +CREATE INDEX i ON t ((h -> 'b')) WHERE (h ? 'b'); +CREATE FUNCTION pick(x text DEFAULT ('a=>1'::hstore -> 'a')) RETURNS text LANGUAGE sql AS 'SELECT x'; +CREATE VIEW v AS SELECT id, h -> 'a' AS ha, h ? 'k' AS hk, 1 AS "OPERATOR(public.->)" FROM t; +CREATE MATERIALIZED VIEW mv AS SELECT id, h -> 'a' AS ha FROM t; +` + _, err := targetConn.Exec(setupSQL + schemaSQL) + require.NoError(t, err) + + planConn, planHost, planPort, planDatabase, planUser, planPassword := testutil.ConnectToPostgres(t, planDB) + defer planConn.Close() + _, err = planConn.Exec(setupSQL) + require.NoError(t, err) + + targetIR, err := util.GetIRFromDatabase(targetHost, targetPort, targetDatabase, targetUser, targetPassword, "disable", "public", "pgschema-test", nil, "") + require.NoError(t, err) + require.Equal(t, []string{"+"}, targetIR.Metadata.ShadowedOperators, "only a name whose every overload is shadowed is exempted") + + file := filepath.Join(t.TempDir(), "schema.sql") + require.NoError(t, os.WriteFile(file, []byte(schemaSQL), 0644)) + + config := &PlanConfig{ + Host: targetHost, Port: targetPort, DB: targetDatabase, User: targetUser, Password: targetPassword, + Schema: "public", File: file, ApplicationName: "pgschema-test", + PlanDBHost: planHost, PlanDBPort: planPort, PlanDBDatabase: planDatabase, PlanDBUser: planUser, PlanDBPassword: planPassword, + } + provider, err := CreateDesiredStateProvider(config) + require.NoError(t, err) + defer provider.Stop() + + p, err := GeneratePlan(config, provider) + require.NoError(t, err) + require.False(t, p.HasAnyChanges(), "view using an extension operator must not be re-planned, got:\n%s", p.HumanColored(false)) +} diff --git a/cmd/plan/plan.go b/cmd/plan/plan.go index dee14b67e..cbe19c63d 100644 --- a/cmd/plan/plan.go +++ b/cmd/plan/plan.go @@ -512,7 +512,11 @@ func normalizeSchemaNames(irData *ir.IR, fromSchema, toSchema string) { // contain "toSchema.func_name(" or "::toSchema.type" which are redundant same-schema // qualifiers. The initial normalizeIR (run by the inspector) couldn't strip these // because it ran with the temp schema name, not the target schema. See issue #283. - stripQualifiers := newSameSchemaQualifierStripper(toSchema) + shadowedOperators := make(map[string]bool, len(irData.Metadata.ShadowedOperators)) + for _, name := range irData.Metadata.ShadowedOperators { + shadowedOperators[name] = true + } + stripQualifiers := newSameSchemaQualifierStripper(toSchema, shadowedOperators) // Normalize schema names in Schemas map if schema, exists := irData.Schemas[fromSchema]; exists { @@ -676,6 +680,11 @@ func normalizeSchemaNames(irData *ir.IR, fromSchema, toSchema string) { } for _, param := range fn.Parameters { param.DataType = replaceString(param.DataType) + // Parameter defaults are deparsed like column defaults and + // compared verbatim; normalize them the same way (#697). + if param.DefaultValue != nil { + *param.DefaultValue = stripQualifiers(replaceString(*param.DefaultValue)) + } } // Normalize function dependencies for topological sorting for i := range fn.Dependencies { @@ -692,6 +701,9 @@ func normalizeSchemaNames(irData *ir.IR, fromSchema, toSchema string) { proc.Definition = replaceString(proc.Definition) for _, param := range proc.Parameters { param.DataType = replaceString(param.DataType) + if param.DefaultValue != nil { + *param.DefaultValue = stripQualifiers(replaceString(*param.DefaultValue)) + } } } @@ -841,44 +853,130 @@ func newSchemaStringReplacer(fromSchema, toSchema string) func(string) string { // with the target schema, expressions may contain "schema.func_name(" or "::schema.type" // where the qualifier matches the object's own schema. These are redundant and must be // stripped to match how the target database's inspector would produce them. See issue #283. -func newSameSchemaQualifierStripper(schema string) func(string) string { +// +// shadowedOperators names the operators that must keep their qualifier because +// an unqualified reference would resolve to a pg_catalog operator instead (see +// ir.Metadata.ShadowedOperators). +func newSameSchemaQualifierStripper(schema string, shadowedOperators map[string]bool) func(string) string { if schema == "" { return func(s string) string { return s } } - prefix := schema + "." - funcPattern := regexp.MustCompile(regexp.QuoteMeta(prefix) + `([a-zA-Z_][a-zA-Z0-9_$]*)\(`) - typePattern := regexp.MustCompile(`::` + regexp.QuoteMeta(prefix)) + // The deparsers render the schema bare or in quote_ident form ("My Schema"). + prefixes := []string{schema + "."} + if quoted := ir.QuoteIdentifier(schema); quoted != schema { + prefixes = append(prefixes, quoted+".") + } + quotedPrefixes := make([]string, len(prefixes)) + for i, p := range prefixes { + quotedPrefixes[i] = regexp.QuoteMeta(p) + } + prefixAlternatives := `(?:` + strings.Join(quotedPrefixes, "|") + `)` + // The object name after the qualifier is bare or quote_ident form too. + funcPattern := regexp.MustCompile(prefixAlternatives + `([a-zA-Z_][a-zA-Z0-9_$]*|"(?:[^"]|"")+")\(`) + typePattern := regexp.MustCompile(`::` + prefixAlternatives) + // OPERATOR(schema.op): ruleutils qualifies an operator whenever its schema + // is not on the deparsing session's search_path. The plan database is + // inspected with only the temporary schema on the path, so an operator in + // the target schema (e.g. hstore's -> installed in public) comes out as + // OPERATOR(public.->) while the target side prints the bare operator. + // Operator names use only the characters PostgreSQL allows (issue #697). + operatorPattern := regexp.MustCompile(`OPERATOR\(` + prefixAlternatives + `([` + regexp.QuoteMeta(ir.OperatorNameChars) + `]+)\)`) replaceQualifiers := func(s string) string { s = funcPattern.ReplaceAllString(s, `${1}(`) + s = operatorPattern.ReplaceAllStringFunc(s, func(match string) string { + // match is "OPERATOR(" + prefix + name + ")"; the name is the + // trailing run of operator characters. + nameStart := len(match) - 1 + for nameStart > 0 && strings.IndexByte(ir.OperatorNameChars, match[nameStart-1]) >= 0 { + nameStart-- + } + name := match[nameStart : len(match)-1] + if shadowedOperators[name] { + return match + } + return name + }) return typePattern.ReplaceAllString(s, "::") } + containsPrefix := func(s string) bool { + for _, p := range prefixes { + if strings.Contains(s, p) { + return true + } + } + return false + } + // quotedPrefixLen returns the length of the quote_ident-form schema prefix + // starting at s[i], or 0 when none starts there. + quotedPrefixLen := func(s string, i int) int { + for _, p := range prefixes[1:] { + if strings.HasPrefix(s[i:], p) { + return len(p) + } + } + return 0 + } + // endsWithPrefix reports whether s[:i] ends with a schema qualifier, so a + // quoted identifier starting at s[i] is the qualified object's own name. + endsWithPrefix := func(s string, i int) bool { + for _, p := range prefixes { + if strings.HasSuffix(s[:i], p) { + return true + } + } + return false + } + // closingQuote returns the index of the quote that closes the literal or + // identifier opened at s[i]; a doubled quote inside is an escape. + closingQuote := func(s string, i int) int { + quote := s[i] + for i++; i < len(s); i++ { + if s[i] != quote { + continue + } + if i+1 < len(s) && s[i+1] == quote { + i++ + continue + } + break + } + return min(i, len(s)-1) + } return func(s string) string { - if s == "" || !strings.Contains(s, prefix) { + if s == "" || !containsPrefix(s) { return s } + // Replace only outside single-quoted literals and double-quoted + // identifiers; a doubled quote inside either is an escape. Two quoted + // tokens are not opaque: the quoted schema itself ("My Schema".) and + // a quoted object name right after a qualifier (public."My Func"(). + // Both stay in the replaceable segment so the patterns can see the + // whole qualified reference. var result strings.Builder result.Grow(len(s)) segmentStart := 0 for i := 0; i < len(s); i++ { - if s[i] != '\'' { + quote := s[i] + if quote != '\'' && quote != '"' { continue } - - result.WriteString(replaceQualifiers(s[segmentStart:i])) - literalStart := i - for i++; i < len(s); i++ { - if s[i] != '\'' { + if quote == '"' { + if n := quotedPrefixLen(s, i); n > 0 { + i += n - 1 continue } - if i+1 < len(s) && s[i+1] == '\'' { - i++ + if endsWithPrefix(s, i) { + i = closingQuote(s, i) continue } - break } - result.WriteString(s[literalStart:min(i+1, len(s))]) - segmentStart = min(i+1, len(s)) + + result.WriteString(replaceQualifiers(s[segmentStart:i])) + literalStart := i + i = closingQuote(s, i) + result.WriteString(s[literalStart : i+1]) + segmentStart = i + 1 } result.WriteString(replaceQualifiers(s[segmentStart:])) return result.String() diff --git a/cmd/plan/plan_test.go b/cmd/plan/plan_test.go index 6569f3bea..d546170d0 100644 --- a/cmd/plan/plan_test.go +++ b/cmd/plan/plan_test.go @@ -239,6 +239,49 @@ func TestNormalizeSchemaNames_StripsSameSchemaQualifiersFromViewDefinitions(t *t } } +func TestNormalizeSchemaNames_StripsQuotedSchemaQualifiers(t *testing.T) { + // A target schema that needs quoting is rendered as "My Schema". by the + // deparsers; function, cast and operator qualifiers in that form are + // stripped too, while an identifier that merely contains the token stays. + const tempSchema = "pgschema_tmp_20260326_123456_abcd1234" + irData := &ir.IR{ + Metadata: ir.Metadata{ShadowedOperators: []string{"+"}}, + Schemas: map[string]*ir.Schema{ + tempSchema: { + Name: tempSchema, + Views: map[string]*ir.View{ + "v": { + Schema: tempSchema, + Name: "v", + Definition: " SELECT \"My Schema\".nlevel(path) AS depth,\n" + + " \"My Schema\".\"Fn Name\"(path) AS named,\n" + + " path::\"My Schema\".ltree AS p,\n" + + " path::\"My Schema\".\"Type Name\" AS typed,\n" + + " h OPERATOR(\"My Schema\".->) 'a'::text AS ha,\n" + + " id OPERATOR(\"My Schema\".+) 1 AS shadowed,\n" + + " 1 AS \"x \"\"My Schema\"\".y(\"\n" + + " FROM t", + }, + }, + }, + }, + } + + normalizeSchemaNames(irData, tempSchema, "My Schema") + + expected := " SELECT nlevel(path) AS depth,\n" + + " \"Fn Name\"(path) AS named,\n" + + " path::ltree AS p,\n" + + " path::\"Type Name\" AS typed,\n" + + " h -> 'a'::text AS ha,\n" + + " id OPERATOR(\"My Schema\".+) 1 AS shadowed,\n" + + " 1 AS \"x \"\"My Schema\"\".y(\"\n" + + " FROM t" + if got := irData.Schemas["My Schema"].Views["v"].Definition; got != expected { + t.Fatalf("expected quoted-schema qualifiers stripped:\n%s\ngot:\n%s", expected, got) + } +} + func TestNormalizeSchemaNames_PreservesInheritedPartitionDefault(t *testing.T) { const tempSchema = "pgschema_tmp_partition595" childDefault, parentDefault := "public.member_default()", "public.member_default()" @@ -324,3 +367,56 @@ func TestNormalizeSchemaNames_IndexAttachment(t *testing.T) { t.Fatalf("Incorrect partition-table metadata normalization: %+v", children) } } + +func TestNormalizeSchemaNames_StripsSameSchemaOperatorQualifiers(t *testing.T) { + // With an external plan database and target schema public, the plan side + // deparses extension operators living in public as OPERATOR(public.->), + // while the target side (inspected with public on its search_path) prints + // the bare operator. The same-schema qualifier must be stripped so the two + // definitions agree (issue #697). + const tempSchema = "pgschema_tmp_20260326_123456_abcd1234" + irData := &ir.IR{ + // A custom public.+(integer, integer) duplicates pg_catalog's; ruleutils + // always qualifies it, and so must the stripped definition. + Metadata: ir.Metadata{ShadowedOperators: []string{"+"}}, + Schemas: map[string]*ir.Schema{ + tempSchema: { + Name: tempSchema, + Views: map[string]*ir.View{ + "v": { + Schema: tempSchema, + Name: "v", + Definition: " SELECT id,\n" + + " h OPERATOR(public.->) 'a'::text AS ha,\n" + + " h OPERATOR(public.?) 'k'::text AS hk,\n" + + " t OPERATOR(public.@>) 'x'::public.ltree AS anc,\n" + + " id OPERATOR(public.+) 1 AS shadowed,\n" + + " 'OPERATOR(public.->)'::text AS literal,\n" + + " 1 AS \"OPERATOR(public.->)\",\n" + + " \"public.nlevel(\" AS \"it''s \"\"public.x(\"\"\",\n" + + " public.\"Fn Name\"(id) AS named, id::public.\"Type Name\" AS typed,\n" + + " j -> 'a'::text AS ja\n" + + " FROM t", + }, + }, + }, + }, + } + + normalizeSchemaNames(irData, tempSchema, "public") + + expected := " SELECT id,\n" + + " h -> 'a'::text AS ha,\n" + + " h ? 'k'::text AS hk,\n" + + " t @> 'x'::ltree AS anc,\n" + + " id OPERATOR(public.+) 1 AS shadowed,\n" + + " 'OPERATOR(public.->)'::text AS literal,\n" + + " 1 AS \"OPERATOR(public.->)\",\n" + + " \"public.nlevel(\" AS \"it''s \"\"public.x(\"\"\",\n" + + " \"Fn Name\"(id) AS named, id::\"Type Name\" AS typed,\n" + + " j -> 'a'::text AS ja\n" + + " FROM t" + if got := irData.Schemas["public"].Views["v"].Definition; got != expected { + t.Fatalf("expected operator qualifiers stripped:\n%s\ngot:\n%s", expected, got) + } +} diff --git a/ir/inspector.go b/ir/inspector.go index 9a300c35d..0c2ae8a32 100644 --- a/ir/inspector.go +++ b/ir/inspector.go @@ -125,6 +125,10 @@ func (i *Inspector) BuildIR(ctx context.Context, targetSchema string) (*IR, erro return nil, err } + if err := traced(ctx, "shadowed operators", func(ctx context.Context) error { return i.buildShadowedOperators(ctx, schema, targetSchema) }); err != nil { + return nil, fmt.Errorf("failed to query shadowed operators: %w", err) + } + if err := traced(ctx, "schemas", func(ctx context.Context) error { return i.buildSchemas(ctx, schema, targetSchema) }); err != nil { return nil, fmt.Errorf("failed to build schemas: %w", err) } @@ -230,6 +234,48 @@ func (i *Inspector) BuildIR(ctx context.Context, targetSchema string) (*IR, erro return schema, nil } +// buildShadowedOperators records the operator names in the inspected and +// managed schemas whose every overload is shadowed by a pg_catalog operator +// with the same name and operand types. pg_catalog is implicitly first on every +// search_path, so such an operator is reachable only schema-qualified: +// ruleutils always deparses it as OPERATOR(schema.op), and +// normalizeOperatorSyntax must leave that form alone instead of reducing it +// to the bare operator (issue #697). +// +// The decision is per name because the normalizer only sees the deparsed +// text, never the operand types. A name with both shadowed and unshadowed +// overloads (a custom public.->(jsonb, text) next to hstore's ->(hstore, text)) +// is therefore not exempted: the unshadowed, extension-provided overload is +// the common case, and exempting the name would make every view using it +// replan forever, which is the bug this normalization fixes. +func (i *Inspector) buildShadowedOperators(ctx context.Context, schema *IR, targetSchema string) error { + rows, err := i.db.QueryContext(ctx, ` +SELECT o.oprname +FROM pg_catalog.pg_operator o +JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace +WHERE n.nspname IN ($1, $2) +GROUP BY o.oprname +HAVING bool_and(EXISTS ( + SELECT 1 FROM pg_catalog.pg_operator c + WHERE c.oprnamespace = 'pg_catalog'::regnamespace + AND c.oprname = o.oprname + AND c.oprleft = o.oprleft + AND c.oprright = o.oprright)) +ORDER BY 1`, targetSchema, i.managedSchema) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return err + } + schema.Metadata.ShadowedOperators = append(schema.Metadata.ShadowedOperators, name) + } + return rows.Err() +} + // buildStep is one named introspection step of BuildIR. The name labels the // step's node in the query call tree. type buildStep struct { diff --git a/ir/ir.go b/ir/ir.go index 5243860ed..858cb2053 100644 --- a/ir/ir.go +++ b/ir/ir.go @@ -16,6 +16,14 @@ type IR struct { // Metadata contains information about the schema dump type Metadata struct { DatabaseVersion string `json:"database_version"` + // ShadowedOperators lists the operator names in the inspected and managed + // schemas whose every overload duplicates a pg_catalog operator's name and + // operand types. pg_catalog is searched first, so an unqualified reference + // to such a name resolves to the catalog operator; ruleutils always + // deparses these as OPERATOR(schema.op) and normalization keeps that form + // (issue #697). See Inspector.buildShadowedOperators for why this is + // decided per name rather than per overload. + ShadowedOperators []string `json:"shadowed_operators,omitempty"` } // Schema represents a single database schema (namespace) diff --git a/ir/normalize.go b/ir/normalize.go index c66cc48fd..ee29f8615 100644 --- a/ir/normalize.go +++ b/ir/normalize.go @@ -22,20 +22,26 @@ func normalizeIR(ir *IR) { return } + shadowed := make(map[string]bool, len(ir.Metadata.ShadowedOperators)) + for _, name := range ir.Metadata.ShadowedOperators { + shadowed[name] = true + } for _, schema := range ir.Schemas { - normalizeSchema(schema) + normalizeSchema(schema, shadowed) } } -// normalizeSchema normalizes all objects within a schema -func normalizeSchema(schema *Schema) { +// normalizeSchema normalizes all objects within a schema. shadowed holds the +// operator names that must keep their schema qualifier (see +// Metadata.ShadowedOperators). +func normalizeSchema(schema *Schema, shadowed map[string]bool) { if schema == nil { return } // Normalize tables for _, table := range schema.Tables { - normalizeTable(table) + normalizeTable(table, shadowed) } // Mark SERIAL columns once column defaults and sequences are normalized @@ -46,17 +52,17 @@ func normalizeSchema(schema *Schema) { // Normalize views for _, view := range schema.Views { - normalizeView(view) + normalizeView(view, shadowed) } // Normalize functions for _, function := range schema.Functions { - normalizeFunction(function) + normalizeFunction(function, shadowed) } // Normalize procedures for _, procedure := range schema.Procedures { - normalizeProcedure(procedure) + normalizeProcedure(procedure, shadowed) } // Normalize types (including domains) @@ -155,19 +161,19 @@ func SerialSequenceName(tableName, columnName string) string { } // normalizeTable normalizes table-related objects -func normalizeTable(table *Table) { +func normalizeTable(table *Table, shadowed map[string]bool) { if table == nil { return } // Normalize columns (pass table schema for context) for _, column := range table.Columns { - normalizeColumn(column, table.Schema) + normalizeColumn(column, table.Schema, shadowed) } // Compare parent expressions in the child's schema context, just like its // own defaults. The parent can live in a different (unmanaged) schema. for _, column := range table.PartitionParentColumns { - normalizeColumn(column, table.Schema) + normalizeColumn(column, table.Schema, shadowed) } // Normalize policies @@ -182,7 +188,7 @@ func normalizeTable(table *Table) { // Normalize indexes for _, index := range table.Indexes { - normalizeIndex(index) + normalizeIndex(index, shadowed) } // Normalize constraints @@ -193,7 +199,7 @@ func normalizeTable(table *Table) { // normalizeColumn normalizes column default values // tableSchema is used to strip same-schema qualifiers from function calls -func normalizeColumn(column *Column, tableSchema string) { +func normalizeColumn(column *Column, tableSchema string, shadowed map[string]bool) { if column == nil { return } @@ -202,7 +208,7 @@ func normalizeColumn(column *Column, tableSchema string) { // expression depending on the inspecting session's search_path; strip the // qualifier so current and desired state compare textually (issue #591). if column.GeneratedExpr != nil && tableSchema != "" { - stripped := StripSchemaPrefixFromBody(*column.GeneratedExpr, tableSchema) + stripped := normalizeDeparsedExpression(*column.GeneratedExpr, tableSchema, shadowed) column.GeneratedExpr = &stripped } @@ -210,16 +216,22 @@ func normalizeColumn(column *Column, tableSchema string) { return } - normalized := normalizeDefaultValue(*column.DefaultValue, tableSchema) + normalized := normalizeDefaultValue(*column.DefaultValue, tableSchema, shadowed) column.DefaultValue = &normalized } // normalizeDefaultValue normalizes default values for semantic comparison // tableSchema is used to strip same-schema qualifiers from function calls -func normalizeDefaultValue(value string, tableSchema string) string { +func normalizeDefaultValue(value string, tableSchema string, shadowed map[string]bool) string { // Remove unnecessary whitespace value = strings.TrimSpace(value) + // Defaults are deparsed with search_path = pg_catalog, so a same-schema + // operator appears as OPERATOR(schema.op); reduce it to the bare operator + // like every other expression context. Done before the nextval early + // return below so sequence-backed defaults are covered too. + value = normalizeOperatorSyntax(value, tableSchema, shadowed) + // Handle nextval sequence references - remove schema qualification if strings.Contains(value, "nextval(") { // Pattern: nextval('schema_name.seq_name'::regclass) -> nextval('seq_name'::regclass) @@ -235,13 +247,35 @@ func normalizeDefaultValue(value string, tableSchema string) string { // Normalize function calls - remove schema qualifiers for functions in the same schema // This matches PostgreSQL's pg_get_expr() behavior which strips same-schema qualifiers // Example: public.get_status() -> get_status() (when tableSchema is "public") + // public."MyFunc"() -> "MyFunc"() (quoted names too, like every other + // expression context, issue #697) // other_schema.get_status() -> other_schema.get_status() (preserved) if tableSchema != "" && strings.Contains(value, tableSchema+".") { - // Pattern: schema.function_name( - // Replace "tableSchema." with "" when followed by identifier and ( + // Pattern: schema.function_name( or schema."Function Name"( + // Replace "tableSchema." with "" when followed by an identifier and ( prefix := tableSchema + "." - pattern := regexp.MustCompile(regexp.QuoteMeta(prefix) + `([a-zA-Z_][a-zA-Z0-9_]*)\(`) - value = pattern.ReplaceAllString(value, `${1}(`) + pattern := regexp.MustCompile(regexp.QuoteMeta(prefix) + `([a-zA-Z_][a-zA-Z0-9_]*|"(?:[^"]|"")+")\(`) + segments := splitQuotedSegments(value) + for i, segment := range segments { + // A quoted function name is a segment of its own; join it with the + // preceding unquoted text so the pattern sees schema."Name"(. + if segment == "" || segment[0] == '\'' { + continue + } + if segment[0] == '"' { + if i == 0 || !strings.HasSuffix(segments[i-1], prefix) { + continue + } + segments[i-1] = strings.TrimSuffix(segments[i-1], prefix) + if i+1 < len(segments) && strings.HasPrefix(segments[i+1], "(") { + continue + } + segments[i-1] += prefix + continue + } + segments[i] = pattern.ReplaceAllString(segment, `${1}(`) + } + value = strings.Join(segments, "") } // Handle type casting - remove explicit type casts that are semantically equivalent @@ -374,14 +408,14 @@ func normalizePolicyExpression(expr string) string { // This happens when extension functions (e.g., ltree's nlevel()) or search_path differences // cause one side to produce "public.func()" and the other "func()". // Stripping same-schema qualifiers ensures the definitions compare as equal. (Issue #314) -func normalizeView(view *View) { +func normalizeView(view *View, shadowed map[string]bool) { if view == nil { return } // Strip same-schema qualifiers from view definition for consistent comparison. // This uses the same logic as function/procedure body normalization. - view.Definition = StripSchemaPrefixFromBody(view.Definition, view.Schema) + view.Definition = normalizeDeparsedExpression(view.Definition, view.Schema, shadowed) // Likewise for the column types: format_type() qualifies a same-schema // type whenever the schema is not on the search_path, and the types are @@ -397,12 +431,12 @@ func normalizeView(view *View) { // Normalize indexes on materialized views for _, index := range view.Indexes { - normalizeIndex(index) + normalizeIndex(index, shadowed) } } // normalizeFunction normalizes function signature and definition -func normalizeFunction(function *Function) { +func normalizeFunction(function *Function, shadowed map[string]bool) { if function == nil { return } @@ -433,7 +467,7 @@ func normalizeFunction(function *Function) { } // Normalize default values (pass function schema for context) if param.DefaultValue != nil { - normalized := normalizeDefaultValue(*param.DefaultValue, function.Schema) + normalized := normalizeDefaultValue(*param.DefaultValue, function.Schema, shadowed) param.DefaultValue = &normalized } } @@ -481,6 +515,116 @@ func StripSchemaPrefixFromBody(body, schema string) string { return stripSchemaPrefixOccurrences(body, schema+".") } +// normalizeDeparsedExpression normalizes an expression that came out of a +// PostgreSQL deparser (pg_get_viewdef, pg_get_expr, pg_get_indexdef): it strips +// same-schema qualifiers from identifiers and reduces same-schema operators +// written as OPERATOR(schema.op) to the bare operator. It is not used on raw +// function bodies, where OPERATOR() carries its own precedence and must be +// compared as written. +func normalizeDeparsedExpression(expr, schema string, shadowed map[string]bool) string { + return normalizeOperatorSyntax(StripSchemaPrefixFromBody(expr, schema), schema, shadowed) +} + +// OperatorNameChars are the characters PostgreSQL allows in an operator name. +const OperatorNameChars = "-+*/<>=~!@#%^&|`?" + +// normalizeOperatorSyntax rewrites OPERATOR(schema.op) to the bare op when +// schema is the object's own schema, unless op is a shadowed name. +// +// ruleutils emits OPERATOR(schema.op) whenever the operator's schema is not on +// the deparsing session's search_path. Column defaults, generated expressions +// and index expressions are deparsed with search_path = pg_catalog, so an +// extension operator installed in the managed schema (hstore's -> in public) +// comes out qualified there, while a view or CHECK constraint on the same +// database is deparsed with the schema on the path and prints the bare +// operator. Reducing the qualified form to the bare operator makes every +// context compare the same way on the current and desired side (issue #697). +// +// A shadowed operator (same name and operand types as a pg_catalog operator, +// see Metadata.ShadowedOperators) is the one case where the bare name would +// resolve differently, so its qualifier is kept. Operators in other schemas +// keep their qualifier. The schema is recognized in bare and quote_ident form +// ("My Schema".op); single-quoted literals and double-quoted identifiers are +// copied through untouched. +func normalizeOperatorSyntax(expr, schema string, shadowed map[string]bool) string { + const keyword = "operator" + if schema == "" || !strings.Contains(strings.ToLower(expr), keyword) { + return expr + } + bare := schema + "." + quoted := QuoteIdentifier(schema) + "." + + var out strings.Builder + out.Grow(len(expr)) + for i := 0; i < len(expr); { + c := expr[i] + if c == '\'' || c == '"' { + end := closingQuote(expr, i) + out.WriteString(expr[i:end]) + i = end + continue + } + // The keyword in any letter case, as a whole word, then optional + // whitespace and the opening parenthesis (same as followsOperatorKeyword). + if (c == 'O' || c == 'o') && i+len(keyword) <= len(expr) && strings.EqualFold(expr[i:i+len(keyword)], keyword) && + (i == 0 || !isIdentChar(expr[i-1])) && (i+len(keyword) == len(expr) || !isIdentChar(expr[i+len(keyword)])) { + open := i + len(keyword) + for open < len(expr) && (expr[open] == ' ' || expr[open] == '\t' || expr[open] == '\n' || expr[open] == '\r') { + open++ + } + if open < len(expr) && expr[open] == '(' { + rest := expr[open+1:] + prefixLen := 0 + if strings.HasPrefix(rest, bare) { + prefixLen = len(bare) + } else if strings.HasPrefix(rest, quoted) { + prefixLen = len(quoted) + } + if prefixLen > 0 { + nameEnd := prefixLen + for nameEnd < len(rest) && strings.IndexByte(OperatorNameChars, rest[nameEnd]) >= 0 { + nameEnd++ + } + if nameEnd > prefixLen && nameEnd < len(rest) && rest[nameEnd] == ')' { + end := open + 1 + nameEnd + 1 + if name := rest[prefixLen:nameEnd]; shadowed[name] { + out.WriteString(expr[i:end]) + } else { + out.WriteString(name) + } + i = end + continue + } + } + } + out.WriteString(expr[i:open]) + i = open + continue + } + out.WriteByte(c) + i++ + } + return out.String() +} + +// closingQuote returns the index just past the quoted literal or identifier +// that starts at expr[start], treating a doubled quote as an escape. An +// unterminated quote runs to the end of expr. +func closingQuote(expr string, start int) int { + q := expr[start] + for j := start + 1; j < len(expr); j++ { + if expr[j] != q { + continue + } + if j+1 < len(expr) && expr[j+1] == q { + j++ + continue + } + return j + 1 + } + return len(expr) +} + // stripSchemaPrefixOccurrences removes every occurrence of prefix ("schema." // in bare or quote_ident form) that starts an identifier reference outside // string literals, quoting the remaining identifier when it is a reserved word. @@ -539,8 +683,12 @@ func stripSchemaPrefixOccurrences(body, prefix string) string { continue } - // Only attempt replacement outside string literals - if !inString && i+prefixLen <= len(body) && body[i:i+prefixLen] == prefix { + // Only attempt replacement outside string literals. An operator's + // qualifier, OPERATOR(schema.op), is left to normalizeOperatorSyntax: + // dropping just the schema there would leave OPERATOR(op), which + // PostgreSQL never emits and which resolves differently for a + // shadowed operator. + if !inString && i+prefixLen <= len(body) && body[i:i+prefixLen] == prefix && !followsOperatorKeyword(body, i) { // Ensure this is a schema qualifier, not part of a longer identifier // (e.g., "not_public.users" should not match) if i == 0 || !isIdentChar(body[i-1]) { @@ -570,6 +718,24 @@ func stripSchemaPrefixOccurrences(body, prefix string) string { return result.String() } +// followsOperatorKeyword reports whether body[i] is the first character +// inside an OPERATOR( construct, i.e. preceded by the keyword OPERATOR in any +// letter case, optional whitespace, and the opening parenthesis. +func followsOperatorKeyword(body string, i int) bool { + const keyword = "operator" + j := i - 1 + if j < 0 || body[j] != '(' { + return false + } + for j--; j >= 0 && (body[j] == ' ' || body[j] == '\t' || body[j] == '\n' || body[j] == '\r'); j-- { + } + start := j + 1 - len(keyword) + if start < 0 || !strings.EqualFold(body[start:j+1], keyword) { + return false + } + return start == 0 || !isIdentChar(body[start-1]) +} + // StripSchemaQualifiers removes "schema." qualifiers from a string containing // SQL identifiers, such as a type or argument list, where the token directly // before an unquoted dot is the schema name in bare or quote_ident form. @@ -631,7 +797,7 @@ func isIdentChar(b byte) bool { } // normalizeProcedure normalizes procedure representation -func normalizeProcedure(procedure *Procedure) { +func normalizeProcedure(procedure *Procedure, shadowed map[string]bool) { if procedure == nil { return } @@ -649,7 +815,7 @@ func normalizeProcedure(procedure *Procedure) { } // Normalize default values (pass procedure schema for context) if param.DefaultValue != nil { - normalized := normalizeDefaultValue(*param.DefaultValue, procedure.Schema) + normalized := normalizeDefaultValue(*param.DefaultValue, procedure.Schema, shadowed) param.DefaultValue = &normalized } } @@ -1004,7 +1170,7 @@ func normalizeTriggerCondition(condition string) string { } // normalizeIndex normalizes index WHERE clauses and other properties -func normalizeIndex(index *Index) { +func normalizeIndex(index *Index, shadowed map[string]bool) { if index == nil { return } @@ -1013,14 +1179,14 @@ func normalizeIndex(index *Index) { if index.IsPartial && index.Where != "" { index.Where = normalizeIndexWhereClause(index.Where) if index.Schema != "" { - index.Where = StripSchemaPrefixFromBody(index.Where, index.Schema) + index.Where = normalizeDeparsedExpression(index.Where, index.Schema, shadowed) } } // Strip same-schema qualifiers from expression index column names if index.IsExpression && index.Schema != "" { for _, col := range index.Columns { - col.Name = StripSchemaPrefixFromBody(col.Name, index.Schema) + col.Name = normalizeDeparsedExpression(col.Name, index.Schema, shadowed) } } } diff --git a/ir/normalize_test.go b/ir/normalize_test.go index cfdd3f261..fb19549be 100644 --- a/ir/normalize_test.go +++ b/ir/normalize_test.go @@ -166,7 +166,7 @@ func TestNormalizeViewStripsSchemaPrefixFromDefinition(t *testing.T) { Name: "test_view", Definition: tt.definition, } - normalizeView(view) + normalizeView(view, nil) if view.Definition != tt.expected { t.Errorf("normalizeView() definition = %q, want %q", view.Definition, tt.expected) } @@ -782,3 +782,103 @@ func TestStripInheritedIdentityNested(t *testing.T) { } } } + +func TestNormalizeOperatorSyntax(t *testing.T) { + shadowed := map[string]bool{"+": true} + tests := []struct { + name string + input string + want string + }{ + { + name: "same-schema operator becomes bare", + input: "(h OPERATOR(public.->) 'a'::text) AND (h OPERATOR(public.?) 'k'::text)", + want: "(h -> 'a'::text) AND (h ? 'k'::text)", + }, + { + name: "shadowed same-schema operator keeps its qualifier", + input: "(a OPERATOR(public.+) 1) OR (h OPERATOR(public.->) 'a'::text)", + want: "(a OPERATOR(public.+) 1) OR (h -> 'a'::text)", + }, + { + name: "operator in another schema keeps its qualifier", + input: "(h OPERATOR(ext.->) 'a'::text)", + want: "(h OPERATOR(ext.->) 'a'::text)", + }, + { + name: "unterminated or non-operator OPERATOR( text is copied through", + input: "OPERATOR(public.foo) OPERATOR(public.-> xoperator(public.->) operator", + want: "OPERATOR(public.foo) OPERATOR(public.-> xoperator(public.->) operator", + }, + { + name: "keyword case and whitespace before the parenthesis", + input: "(h operator (public.->) 'a'::text) AND (h Operator\t(public.?) 'k'::text)", + want: "(h -> 'a'::text) AND (h ? 'k'::text)", + }, + { + name: "quoted text is untouched", + input: "'OPERATOR(public.->)'::text, \"OPERATOR(public.->)\", 'it''s OPERATOR(public.->)'", + want: "'OPERATOR(public.->)'::text, \"OPERATOR(public.->)\", 'it''s OPERATOR(public.->)'", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := normalizeOperatorSyntax(tt.input, "public", shadowed); got != tt.want { + t.Errorf("normalizeOperatorSyntax(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } + + // The schema is also recognized in quote_ident form, as the deparsers + // render a schema that needs quoting, while a quoted identifier that merely + // looks like the construct is copied through. + if got := normalizeOperatorSyntax(`(h OPERATOR("My Schema".->) 'a'::text) AS "OPERATOR(""My Schema"".->)"`, "My Schema", nil); got != `(h -> 'a'::text) AS "OPERATOR(""My Schema"".->)"` { + t.Errorf("normalizeOperatorSyntax with quoted schema = %q", got) + } + + // Deparsed expressions (generated expressions, index expressions, views) + // and column defaults, including nextval-backed ones, reduce the operator; + // a shadowed operator keeps its qualifier in both paths (issue #697). + if got := normalizeDeparsedExpression("(h OPERATOR(public.->) 'a'::text)", "public", shadowed); got != "(h -> 'a'::text)" { + t.Errorf("normalizeDeparsedExpression left operator syntax: %q", got) + } + if got := normalizeDefaultValue(`('"a"=>"1"'::hstore OPERATOR(public.->) 'a')`, "public", shadowed); got != `('"a"=>"1"'::hstore -> 'a')` { + t.Errorf("normalizeDefaultValue left operator syntax: %q", got) + } + if got := normalizeDefaultValue("ARRAY[nextval('s'::regclass)::integer] OPERATOR(public.->) ARRAY[1]", "public", shadowed); got != "ARRAY[nextval('s'::regclass)::integer] -> ARRAY[1]" { + t.Errorf("normalizeDefaultValue skipped the nextval default: %q", got) + } + if got := normalizeDefaultValue("(1 OPERATOR(public.+) 2)", "public", shadowed); got != "(1 OPERATOR(public.+) 2)" { + t.Errorf("normalizeDefaultValue rewrote a shadowed operator: %q", got) + } + + // Prefix stripping alone (used to compare raw function bodies) leaves + // OPERATOR(schema.op) exactly as written: OPERATOR() has its own + // precedence, and dropping only the schema would leave OPERATOR(op). + for input, want := range map[string]string{ + "SELECT 2 OPERATOR(public.*) 3 + public.f(4)": "SELECT 2 OPERATOR(public.*) 3 + f(4)", + "SELECT 2 operator (public.*) 3 + public.f(4)": "SELECT 2 operator (public.*) 3 + f(4)", + "SELECT 2 Operator\t(public.*) 3 + public.f(4)": "SELECT 2 Operator\t(public.*) 3 + f(4)", + "SELECT xoperator(public.f(4))": "SELECT xoperator(f(4))", + } { + if got := StripSchemaPrefixFromBody(input, "public"); got != want { + t.Errorf("StripSchemaPrefixFromBody(%q) = %q, want %q", input, got, want) + } + } +} + +func TestNormalizeDefaultValueStripsQuotedFunctionQualifier(t *testing.T) { + // Defaults strip the same-schema qualifier from quoted function names the + // way view, index and generated expressions already do (issue #697). + for input, want := range map[string]string{ + `public."MyFunc"()`: `"MyFunc"()`, + `public."My ""Func"""(1)`: `"My ""Func"""(1)`, + `other."MyFunc"()`: `other."MyFunc"()`, + `public.get_status()`: `get_status()`, + `'public."MyFunc"()'::text`: `'public."MyFunc"()'`, + } { + if got := normalizeDefaultValue(input, "public", nil); got != want { + t.Errorf("normalizeDefaultValue(%q) = %q, want %q", input, got, want) + } + } +} diff --git a/testdata/diff/dependency/issue_571_quoted_function_dependency/diff.sql b/testdata/diff/dependency/issue_571_quoted_function_dependency/diff.sql index fd1e9491c..74fcbc49d 100644 --- a/testdata/diff/dependency/issue_571_quoted_function_dependency/diff.sql +++ b/testdata/diff/dependency/issue_571_quoted_function_dependency/diff.sql @@ -30,7 +30,7 @@ $$; CREATE TABLE IF NOT EXISTS widgets ( id SERIAL, - code integer DEFAULT public."MyFunc"(), + code integer DEFAULT "MyFunc"(), CONSTRAINT widgets_pkey PRIMARY KEY (id), CONSTRAINT widgets_code_excl EXCLUDE USING btree ("MyFunc"(code) WITH =) ); diff --git a/testdata/diff/dependency/issue_571_quoted_function_dependency/plan.json b/testdata/diff/dependency/issue_571_quoted_function_dependency/plan.json index 7289f1e04..9caaccf16 100644 --- a/testdata/diff/dependency/issue_571_quoted_function_dependency/plan.json +++ b/testdata/diff/dependency/issue_571_quoted_function_dependency/plan.json @@ -27,7 +27,7 @@ "path": "public.AWrapper" }, { - "sql": "CREATE TABLE IF NOT EXISTS widgets (\n id SERIAL,\n code integer DEFAULT public.\"MyFunc\"(),\n CONSTRAINT widgets_pkey PRIMARY KEY (id),\n CONSTRAINT widgets_code_excl EXCLUDE USING btree (\"MyFunc\"(code) WITH =)\n);", + "sql": "CREATE TABLE IF NOT EXISTS widgets (\n id SERIAL,\n code integer DEFAULT \"MyFunc\"(),\n CONSTRAINT widgets_pkey PRIMARY KEY (id),\n CONSTRAINT widgets_code_excl EXCLUDE USING btree (\"MyFunc\"(code) WITH =)\n);", "type": "table", "operation": "create", "path": "public.widgets" diff --git a/testdata/diff/dependency/issue_571_quoted_function_dependency/plan.sql b/testdata/diff/dependency/issue_571_quoted_function_dependency/plan.sql index fd1e9491c..74fcbc49d 100644 --- a/testdata/diff/dependency/issue_571_quoted_function_dependency/plan.sql +++ b/testdata/diff/dependency/issue_571_quoted_function_dependency/plan.sql @@ -30,7 +30,7 @@ $$; CREATE TABLE IF NOT EXISTS widgets ( id SERIAL, - code integer DEFAULT public."MyFunc"(), + code integer DEFAULT "MyFunc"(), CONSTRAINT widgets_pkey PRIMARY KEY (id), CONSTRAINT widgets_code_excl EXCLUDE USING btree ("MyFunc"(code) WITH =) ); diff --git a/testdata/diff/dependency/issue_571_quoted_function_dependency/plan.txt b/testdata/diff/dependency/issue_571_quoted_function_dependency/plan.txt index 481f9a077..323914fc8 100644 --- a/testdata/diff/dependency/issue_571_quoted_function_dependency/plan.txt +++ b/testdata/diff/dependency/issue_571_quoted_function_dependency/plan.txt @@ -49,7 +49,7 @@ $$; CREATE TABLE IF NOT EXISTS widgets ( id SERIAL, - code integer DEFAULT public."MyFunc"(), + code integer DEFAULT "MyFunc"(), CONSTRAINT widgets_pkey PRIMARY KEY (id), CONSTRAINT widgets_code_excl EXCLUDE USING btree ("MyFunc"(code) WITH =) );