goldeneye: build SQLite from source and treat compile options as extensions

Which functions a SQLite has is decided when it is compiled, and the
official sqlite3 binary's option set matches neither the driver sqlc's
tests run against nor any other in particular. So instead of downloading
that binary, `install sqlite` downloads the pinned release's amalgamation,
checked against the SHA3-256 sqlite.org lists, and compiles the shell
from it with cc or $CC: once with the options sqlite.org's own configure
turns on by default, which gives functions.jsonl, and once per option in
the extension list, each of which gets a directory under extensions/
holding the functions its build adds over the default one — found the
way the PostgreSQL generator finds what CREATE EXTENSION adds, by
comparing the catalog with and without. GEOPOLY lives inside the RTREE
module, so its build carries both options. Each shell is checked against
pragma compile_options before it is read, so a stale build is caught.
The eight builds take about fifteen seconds unoptimised.

A schema says which options it needs the way SQLite itself does, with
CREATE VIRTUAL TABLE ... USING fts5. dialect.json gains a modules map
from a virtual table module to its extension, the seed resolves a module
or extension name through it, and both the legacy catalog and the core
load the extension when a virtual table is declared, so the fts5 case
keeps its typed bm25, highlight and snippet results. The legacy
catalog's createExtension also now records what it loaded, which it
never did.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RYkxvtwH6GgysuWYP87vyj

Author: claude

CLAUDE.md

@@ -153,7 +153,7 @@ is not available skip.
 ```bash
 cd internal/goldeneye
 go run ./cmd/goldeneye install clickhouse   # download the pinned clickhouse binary once
-go run ./cmd/goldeneye install sqlite       # download the pinned sqlite3 shell once
+go run ./cmd/goldeneye install sqlite       # build the pinned sqlite3 shells once; needs a C compiler
 POSTGRESQL_SERVER_URI="postgres://postgres:postgres@127.0.0.1:5432/postgres?sslmode=disable" go test ./...
 go run ./cmd/goldeneye generate postgresql  # rewrite the files after a change
 ```

internal/core/schema/schema.go

@@ -133,6 +133,13 @@ func applyCreateTable(cat *core.Catalog, stmt *ast.CreateTableStmt) error {
 	if stmt.Name == nil {
 		return fmt.Errorf("create table with nil name")
 	}
+	// A virtual table's module is the dialect's word for the extension it
+	// needs, the way CREATE EXTENSION is PostgreSQL's.
+	if stmt.Using != "" {
+		if err := cat.LoadExtension(stmt.Using); err != nil {
+			return err
+		}
+	}
 	nsOID, err := resolveOrCreateNamespace(cat, stmt.Name.Schema)
 	if err != nil {
 		return err

internal/core/seed/extension.go

@@ -9,17 +9,12 @@ import (
 	"github.com/sqlc-dev/sqlc/internal/core"
 )
 
-// applyExtension applies the named extension's directory to a catalog that
-// has already been seeded. Unlike the dialect's own seed, an extension lands
-// in a catalog full of types, so everything it names is resolved against what
-// is there before being created.
-func applyExtension(cat *core.Catalog, fsys fs.FS, name string) error {
-	dir := path.Join(ExtensionsDir, name)
-	if _, err := fs.Stat(fsys, dir); err != nil {
-		// An extension sqlc has no data for adds nothing, the way the legacy
-		// catalog has always treated one.
-		return nil
-	}
+// applyExtension applies the extension directory dir, relative to the
+// dialect, to a catalog that has already been seeded. Unlike the dialect's
+// own seed, an extension lands in a catalog full of types, so everything it
+// names is resolved against what is there before being created.
+func applyExtension(cat *core.Catalog, fsys fs.FS, dir string) error {
+	name := path.Base(dir)
 	sub, err := fs.Sub(fsys, dir)
 	if err != nil {
 		return fmt.Errorf("seed: extension %s: %w", name, err)

internal/core/seed/seed.go

@@ -18,7 +18,8 @@
 //
 // A dialect may also hold an extensions/ directory with one directory per
 // extension, each a smaller bundle of the same files, applied when a schema
-// says CREATE EXTENSION.
+// says CREATE EXTENSION — or, for a dialect whose settings map virtual table
+// modules to extensions, CREATE VIRTUAL TABLE ... USING.
 package seed
 
 import (
@@ -28,6 +29,7 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"path"
 	"slices"
 	"strings"
 
@@ -78,6 +80,15 @@ type Settings struct {
 	// same kind of value resolves. "*" makes every seeded type implicitly
 	// castable to every other, for dialects that compare across categories.
 	CastCategories string `json:"cast_categories,omitempty"`
+
+	// Modules names the extension a virtual table module belongs to, for a
+	// dialect whose schemas say CREATE VIRTUAL TABLE ... USING rather than
+	// CREATE EXTENSION: SQLite's fts5 module comes with the functions its
+	// enable_fts5 compile option adds.
+	Modules map[string]string `json:"modules,omitempty"`
+
+	// fsys is the dialect directory the settings were read from.
+	fsys fs.FS
 }
 
 // Type is a type the dialect defines. Aliases are spellings of the same type
@@ -155,21 +166,60 @@ func Dialect(fsys fs.FS, dir string) core.Option {
 		if err != nil {
 			return fmt.Errorf("seed: %s: %w", dir, err)
 		}
-		if err := apply(cat, sub); err != nil {
+		settings, err := loadSettings(sub)
+		if err != nil {
+			return err
+		}
+		if err := apply(cat, sub, settings); err != nil {
 			return err
 		}
 		cat.SetExtensionLoader(func(name string) error {
-			return applyExtension(cat, sub, name)
+			dir, ok := settings.extensionDir(name)
+			if !ok {
+				// An extension sqlc has no data for adds nothing, the way
+				// the legacy catalog has always treated one.
+				return nil
+			}
+			return applyExtension(cat, sub, dir)
 		})
 		return nil
 	})
 }
 
-func apply(cat *core.Catalog, fsys fs.FS) error {
-	settings, err := loadSettings(fsys)
+// ExtensionDir resolves what a schema named — an extension, or a virtual
+// table module the dialect's settings map to one — to the extension's
+// directory under dir, reporting whether the dialect has data for it.
+func ExtensionDir(fsys fs.FS, dir, name string) (string, bool) {
+	sub, err := fs.Sub(fsys, dir)
 	if err != nil {
-		return err
+		return "", false
 	}
+	settings, err := loadSettings(sub)
+	if err != nil {
+		return "", false
+	}
+	rel, ok := settings.extensionDir(name)
+	if !ok {
+		return "", false
+	}
+	return path.Join(dir, rel), true
+}
+
+// extensionDir is the directory of the extension a name refers to,
+// relative to the dialect, if the dialect ships one.
+func (s Settings) extensionDir(name string) (string, bool) {
+	if ext, ok := s.Modules[strings.ToLower(name)]; ok {
+		name = ext
+	}
+	dir := path.Join(ExtensionsDir, name)
+	if _, err := fs.Stat(s.fsys, dir); err != nil {
+		return "", false
+	}
+	return dir, true
+}
+
+func apply(cat *core.Catalog, fsys fs.FS, settings Settings) error {
+	var err error
 	b := &builder{
 		cat:        cat,
 		settings:   settings,
@@ -222,6 +272,7 @@ func loadSettings(fsys fs.FS) (Settings, error) {
 	if settings.Dialect == "" {
 		return Settings{}, fmt.Errorf("seed: %s: dialect has no name", SettingsFile)
 	}
+	settings.fsys = fsys
 	return settings, nil
 }
 

internal/engine/sqlite/catalog.go

@@ -9,6 +9,7 @@ func NewCatalog() *catalog.Catalog {
 		Schemas: []*catalog.Schema{
 			defaultSchema(def),
 		},
-		Extensions: map[string]struct{}{},
+		LoadExtension: loadExtension,
+		Extensions:    map[string]struct{}{},
 	}
 }

internal/engine/sqlite/dialect/dialect.json

@@ -11,5 +11,13 @@
   "comparison_categories": "BNSDU",
   "arithmetic": ["+", "-", "*", "/", "%"],
   "arithmetic_categories": "N",
-  "cast_categories": "*"
+  "cast_categories": "*",
+  "modules": {
+    "fts3": "enable_fts3",
+    "fts4": "enable_fts3",
+    "fts5": "enable_fts5",
+    "geopoly": "enable_geopoly",
+    "rtree": "enable_rtree",
+    "rtree_i32": "enable_rtree"
+  }
 }

internal/engine/sqlite/dialect/extensions/enable_fts3/functions.jsonl

@@ -0,0 +1,5 @@
+{"name":"matchinfo","args":[{"type":"any"}],"returns":"blob"}
+{"name":"matchinfo","args":[{"type":"any"},{"type":"text"}],"returns":"blob"}
+{"name":"offsets","args":[{"type":"any"}],"returns":"text"}
+{"name":"optimize","args":[{"type":"any"}],"returns":"text"}
+{"name":"snippet","args":[{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"any","mode":"v"}],"returns":"text"}

internal/engine/sqlite/dialect/extensions/enable_fts5/functions.jsonl

@@ -0,0 +1,7 @@
+{"name":"bm25","args":[{"type":"text","has_default":true},{"type":"real","mode":"v"}],"returns":"real"}
+{"name":"fts5_get_locale","args":[{"type":"any","has_default":true},{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"text","nullable":true}
+{"name":"fts5_insttoken","args":[{"type":"text"}],"returns":"text"}
+{"name":"fts5_locale","args":[{"type":"text"},{"type":"text"}],"returns":"text"}
+{"name":"fts5_source_id","returns":"text"}
+{"name":"highlight","args":[{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"}
+{"name":"snippet","args":[{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"any","mode":"v"}],"returns":"text"}

internal/engine/sqlite/dialect/extensions/enable_geopoly/functions.jsonl

@@ -0,0 +1,15 @@
+{"name":"geopoly_area","args":[{"type":"any"}],"returns":"real","nullable":true}
+{"name":"geopoly_bbox","args":[{"type":"any"}],"returns":"blob","nullable":true}
+{"name":"geopoly_blob","args":[{"type":"any"}],"returns":"blob","nullable":true}
+{"name":"geopoly_ccw","args":[{"type":"any"}],"returns":"blob","nullable":true}
+{"name":"geopoly_contains_point","args":[{"type":"any"},{"type":"real"},{"type":"real"}],"returns":"integer","nullable":true}
+{"name":"geopoly_group_bbox","kind":"a","args":[{"type":"any"}],"returns":"blob","nullable":true}
+{"name":"geopoly_json","args":[{"type":"any"}],"returns":"text","nullable":true}
+{"name":"geopoly_overlap","args":[{"type":"any"},{"type":"any"}],"returns":"integer","nullable":true}
+{"name":"geopoly_regular","args":[{"type":"real"},{"type":"real"},{"type":"real"},{"type":"integer"}],"returns":"blob"}
+{"name":"geopoly_svg","args":[{"type":"any","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text","nullable":true}
+{"name":"geopoly_within","args":[{"type":"any"},{"type":"any"}],"returns":"integer","nullable":true}
+{"name":"geopoly_xform","args":[{"type":"any"},{"type":"real"},{"type":"real"},{"type":"real"},{"type":"real"},{"type":"real"},{"type":"real"}],"returns":"blob","nullable":true}
+{"name":"rtreecheck","args":[{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"}
+{"name":"rtreedepth","args":[{"type":"blob"}],"returns":"integer"}
+{"name":"rtreenode","args":[{"type":"integer"},{"type":"blob"}],"returns":"text"}

internal/engine/sqlite/dialect/extensions/enable_offset_sql_func/functions.jsonl

@@ -0,0 +1 @@
+{"name":"sqlite_offset","args":[{"type":"any"}],"returns":"integer","nullable":true}