core: model the hidden columns of sqlite fts5 tables (#4579)

An fts5 virtual table offers columns beyond the ones it declares: one
named after the table itself, matched against the whole row and passed
to the auxiliary functions, rank, the current match's score, and rowid.
The analysis core knew none of them, so queries from #1797 like

    SELECT rowid, name FROM recipes_fts WHERE recipes_fts MATCH ?

failed with 'unknown column'.

Give sql_attribute a hidden flag and ast.ColumnDef an IsHidden marker.
The sqlite converter emits the three fts5 hidden columns; the core
analyzer resolves them by name while keeping them out of star
expansions, models and implicit INSERT targets. The legacy catalog
drops hidden columns, so the legacy path is unchanged.

The bm25 query in the virtual_table case now spells out its columns:
the core path does not yet expand stars in the query text, and that
gap is not this change's to close.


Claude-Session: https://claude.ai/code/session_01NrYAwWTgJMeea6n6wZ95oy

Co-authored-by: Claude <noreply@anthropic.com>

Author: kyleconroy

internal/core/analyzer/dml.go

@@ -108,7 +108,13 @@ func (a *analyzer) relationScope(relations, extra *ast.List, from ast.Node) (*sc
 func insertTargets(rel scopeRel, cols *ast.List) ([]core.ClassColumn, error) {
 	items := listItems(cols)
 	if len(items) == 0 {
-		return rel.cols, nil
+		out := make([]core.ClassColumn, 0, len(rel.cols))
+		for _, col := range rel.cols {
+			if !col.Hidden {
+				out = append(out, col)
+			}
+		}
+		return out, nil
 	}
 	out := make([]core.ClassColumn, 0, len(items))
 	for _, item := range items {

internal/core/analyzer/projection.go

@@ -90,6 +90,9 @@ func (a *analyzer) emitStar(fields []string) {
 		}
 		a.columns = slices.Grow(a.columns, len(rel.cols))
 		for _, c := range rel.cols {
+			if c.Hidden {
+				continue
+			}
 			col := core.Column{
 				Name:               c.Name,
 				TypeOID:            c.TypeOID,

internal/core/attribute.go

@@ -20,6 +20,10 @@ type AttributeSpec struct {
 	AutoIncrement bool
 	IsPrimaryKey  bool
 	IsUnique      bool
+	// Hidden columns resolve by name but stay out of a star expansion and
+	// the relation's model, like the column an sqlite fts5 table names
+	// after itself.
+	Hidden bool
 }
 
 func (c *Catalog) CreateAttributeSpec(s AttributeSpec) error {
@@ -36,6 +40,7 @@ func (c *Catalog) CreateAttributeSpec(s AttributeSpec) error {
 		AutoIncrement: boolToInt64(s.AutoIncrement),
 		IsPrimaryKey:  boolToInt64(s.IsPrimaryKey),
 		IsUnique:      boolToInt64(s.IsUnique),
+		Hidden:        boolToInt64(s.Hidden),
 	})
 	if err != nil {
 		return fmt.Errorf("create attribute %q on class %d: %w", s.Name, s.ClassOID, err)
@@ -213,6 +218,7 @@ type ClassColumn struct {
 	Name    string
 	TypeOID int64
 	NotNull bool
+	Hidden  bool
 }
 
 // ClassColumns returns a relation's columns in ordinal order.
@@ -228,6 +234,7 @@ func (c *Catalog) ClassColumns(classOID int64) ([]ClassColumn, error) {
 			Name:    r.Name,
 			TypeOID: r.TypeOid,
 			NotNull: r.NotNull != 0,
+			Hidden:  r.Hidden != 0,
 		})
 	}
 	return out, nil

internal/core/catalogdb/db.go

@@ -95,8 +95,8 @@ UPDATE sql_class SET name = sqlc.arg(new_name) WHERE oid = sqlc.arg(oid);
 INSERT INTO sql_attribute (
     class_oid, name, type_oid, not_null, has_default, num,
     decl_type, type_length, type_scale,
-    auto_increment, is_primary_key, is_unique
-) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
+    auto_increment, is_primary_key, is_unique, hidden
+) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
 
 -- name: SetAttributePrimaryKey :exec
 UPDATE sql_attribute SET is_primary_key = 1, not_null = 1
@@ -147,7 +147,7 @@ WHERE c.name = ?
 ORDER BY a.num;
 
 -- name: ClassAttributes :many
-SELECT oid, name, type_oid, not_null
+SELECT oid, name, type_oid, not_null, hidden
 FROM sql_attribute
 WHERE class_oid = ?
 ORDER BY num;
@@ -156,7 +156,7 @@ ORDER BY num;
 SELECT a.name AS column_name, t.name AS type_name, a.not_null
 FROM sql_attribute a
 JOIN sql_type t ON t.oid = a.type_oid
-WHERE a.class_oid = ?
+WHERE a.class_oid = ? AND a.hidden = 0
 ORDER BY a.num;
 
 -- name: LookupAttribute :one

internal/core/catalogdb/models.go

@@ -65,6 +65,9 @@ CREATE TABLE sql_class (
 --                    Set both for inline-column PK and for table-level PK.
 --   is_unique:       column has a UNIQUE constraint or a single-column UNIQUE
 --                    table constraint.
+--   hidden:          resolvable by name but absent from a star expansion and
+--                    from the relation's model, like the column an sqlite
+--                    fts5 table names after itself.
 CREATE TABLE sql_attribute (
     oid            INTEGER PRIMARY KEY AUTOINCREMENT,
     class_oid      INTEGER NOT NULL REFERENCES sql_class(oid),
@@ -79,6 +82,7 @@ CREATE TABLE sql_attribute (
     auto_increment INTEGER NOT NULL DEFAULT 0,
     is_primary_key INTEGER NOT NULL DEFAULT 0,
     is_unique      INTEGER NOT NULL DEFAULT 0,
+    hidden         INTEGER NOT NULL DEFAULT 0,
     UNIQUE(class_oid, name),
     UNIQUE(class_oid, num)
 );

internal/core/catalogdb/query.sql.go

@@ -163,6 +163,7 @@ func applyCreateTable(cat *core.Catalog, stmt *ast.CreateTableStmt) error {
 			NotNull:      col.IsNotNull || col.PrimaryKey,
 			IsPrimaryKey: col.PrimaryKey,
 			DeclType:     col.TypeName.Name,
+			Hidden:       col.IsHidden,
 		}); err != nil {
 			return fmt.Errorf("attr %s.%s: %w", stmt.Name.Name, col.Colname, err)
 		}

internal/core/catalogdef/query.sql

@@ -22,7 +22,7 @@ WHERE b MATCH ?;
 SELECT snippet(tbl_ft, 0, '<b>', '</b>', 'aa', ?) FROM tbl_ft;
 
 -- name: SelectBm25Func :many
-SELECT *, bm25(tbl_ft, 2.0) FROM tbl_ft
+SELECT b, c, bm25(tbl_ft, 2.0) FROM tbl_ft
 WHERE b MATCH ? ORDER BY bm25(tbl_ft);
 
 -- name: UpdateTblFt :exec