diff --git a/.github/workflows/gen.yml b/.github/workflows/gen.yml index c3b6bdd4ad..691c06d4da 100644 --- a/.github/workflows/gen.yml +++ b/.github/workflows/gen.yml @@ -24,6 +24,8 @@ jobs: check-latest: true - run: go run ./cmd/goldeneye install clickhouse working-directory: internal/goldeneye + - run: go run ./cmd/goldeneye install sqlite + working-directory: internal/goldeneye - run: go run ./cmd/goldeneye generate working-directory: internal/goldeneye env: diff --git a/CLAUDE.md b/CLAUDE.md index dfce6cfb08..facde359b9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -153,6 +153,7 @@ is not available skip. ```bash cd internal/goldeneye go run ./cmd/goldeneye install clickhouse # download the pinned clickhouse binary once +go run ./cmd/goldeneye install sqlite # build the pinned sqlite3 shells once; needs a C compiler POSTGRESQL_SERVER_URI="postgres://postgres:postgres@127.0.0.1:5432/postgres?sslmode=disable" go test ./... go run ./cmd/goldeneye generate postgresql # rewrite the files after a change ``` diff --git a/internal/core/schema/schema.go b/internal/core/schema/schema.go index 19dc3cc827..19cd4c7c61 100644 --- a/internal/core/schema/schema.go +++ b/internal/core/schema/schema.go @@ -133,6 +133,13 @@ func applyCreateTable(cat *core.Catalog, stmt *ast.CreateTableStmt) error { if stmt.Name == nil { return fmt.Errorf("create table with nil name") } + // A virtual table's module is the dialect's word for the extension it + // needs, the way CREATE EXTENSION is PostgreSQL's. + if stmt.Using != "" { + if err := cat.LoadExtension(stmt.Using); err != nil { + return err + } + } nsOID, err := resolveOrCreateNamespace(cat, stmt.Name.Schema) if err != nil { return err diff --git a/internal/core/seed/extension.go b/internal/core/seed/extension.go index f86fa4929e..ac7eacdde8 100644 --- a/internal/core/seed/extension.go +++ b/internal/core/seed/extension.go @@ -9,17 +9,12 @@ import ( "github.com/sqlc-dev/sqlc/internal/core" ) -// applyExtension applies the named extension's directory to a catalog that -// has already been seeded. Unlike the dialect's own seed, an extension lands -// in a catalog full of types, so everything it names is resolved against what -// is there before being created. -func applyExtension(cat *core.Catalog, fsys fs.FS, name string) error { - dir := path.Join(ExtensionsDir, name) - if _, err := fs.Stat(fsys, dir); err != nil { - // An extension sqlc has no data for adds nothing, the way the legacy - // catalog has always treated one. - return nil - } +// applyExtension applies the extension directory dir, relative to the +// dialect, to a catalog that has already been seeded. Unlike the dialect's +// own seed, an extension lands in a catalog full of types, so everything it +// names is resolved against what is there before being created. +func applyExtension(cat *core.Catalog, fsys fs.FS, dir string) error { + name := path.Base(dir) sub, err := fs.Sub(fsys, dir) if err != nil { return fmt.Errorf("seed: extension %s: %w", name, err) diff --git a/internal/core/seed/seed.go b/internal/core/seed/seed.go index cc0c5a7977..6440a88436 100644 --- a/internal/core/seed/seed.go +++ b/internal/core/seed/seed.go @@ -18,7 +18,8 @@ // // A dialect may also hold an extensions/ directory with one directory per // extension, each a smaller bundle of the same files, applied when a schema -// says CREATE EXTENSION. +// says CREATE EXTENSION — or, for a dialect whose settings map virtual table +// modules to extensions, CREATE VIRTUAL TABLE ... USING. package seed import ( @@ -28,6 +29,7 @@ import ( "fmt" "io" "io/fs" + "path" "slices" "strings" @@ -78,6 +80,15 @@ type Settings struct { // same kind of value resolves. "*" makes every seeded type implicitly // castable to every other, for dialects that compare across categories. CastCategories string `json:"cast_categories,omitempty"` + + // Modules names the extension a virtual table module belongs to, for a + // dialect whose schemas say CREATE VIRTUAL TABLE ... USING rather than + // CREATE EXTENSION: SQLite's fts5 module comes with the functions its + // enable_fts5 compile option adds. + Modules map[string]string `json:"modules,omitempty"` + + // fsys is the dialect directory the settings were read from. + fsys fs.FS } // Type is a type the dialect defines. Aliases are spellings of the same type @@ -155,21 +166,60 @@ func Dialect(fsys fs.FS, dir string) core.Option { if err != nil { return fmt.Errorf("seed: %s: %w", dir, err) } - if err := apply(cat, sub); err != nil { + settings, err := loadSettings(sub) + if err != nil { + return err + } + if err := apply(cat, sub, settings); err != nil { return err } cat.SetExtensionLoader(func(name string) error { - return applyExtension(cat, sub, name) + dir, ok := settings.extensionDir(name) + if !ok { + // An extension sqlc has no data for adds nothing, the way + // the legacy catalog has always treated one. + return nil + } + return applyExtension(cat, sub, dir) }) return nil }) } -func apply(cat *core.Catalog, fsys fs.FS) error { - settings, err := loadSettings(fsys) +// ExtensionDir resolves what a schema named — an extension, or a virtual +// table module the dialect's settings map to one — to the extension's +// directory under dir, reporting whether the dialect has data for it. +func ExtensionDir(fsys fs.FS, dir, name string) (string, bool) { + sub, err := fs.Sub(fsys, dir) if err != nil { - return err + return "", false } + settings, err := loadSettings(sub) + if err != nil { + return "", false + } + rel, ok := settings.extensionDir(name) + if !ok { + return "", false + } + return path.Join(dir, rel), true +} + +// extensionDir is the directory of the extension a name refers to, +// relative to the dialect, if the dialect ships one. +func (s Settings) extensionDir(name string) (string, bool) { + if ext, ok := s.Modules[strings.ToLower(name)]; ok { + name = ext + } + dir := path.Join(ExtensionsDir, name) + if _, err := fs.Stat(s.fsys, dir); err != nil { + return "", false + } + return dir, true +} + +func apply(cat *core.Catalog, fsys fs.FS, settings Settings) error { + var err error b := &builder{ cat: cat, settings: settings, @@ -222,6 +272,7 @@ func loadSettings(fsys fs.FS) (Settings, error) { if settings.Dialect == "" { return Settings{}, fmt.Errorf("seed: %s: dialect has no name", SettingsFile) } + settings.fsys = fsys return settings, nil } diff --git a/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go b/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go index 29f345be04..de2a1fa89c 100644 --- a/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go +++ b/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go @@ -47,9 +47,9 @@ const getGroupConcatInt = `-- name: GetGroupConcatInt :one SELECT group_concat(int_val) FROM test ` -func (q *Queries) GetGroupConcatInt(ctx context.Context) (string, error) { +func (q *Queries) GetGroupConcatInt(ctx context.Context) (sql.NullString, error) { row := q.db.QueryRowContext(ctx, getGroupConcatInt) - var group_concat string + var group_concat sql.NullString err := row.Scan(&group_concat) return group_concat, err } @@ -58,9 +58,9 @@ const getGroupConcatInt2 = `-- name: GetGroupConcatInt2 :one SELECT group_concat(1, ':') FROM test ` -func (q *Queries) GetGroupConcatInt2(ctx context.Context) (string, error) { +func (q *Queries) GetGroupConcatInt2(ctx context.Context) (sql.NullString, error) { row := q.db.QueryRowContext(ctx, getGroupConcatInt2) - var group_concat string + var group_concat sql.NullString err := row.Scan(&group_concat) return group_concat, err } @@ -69,9 +69,9 @@ const getGroupConcatText = `-- name: GetGroupConcatText :one SELECT group_concat(text_val) FROM test ` -func (q *Queries) GetGroupConcatText(ctx context.Context) (string, error) { +func (q *Queries) GetGroupConcatText(ctx context.Context) (sql.NullString, error) { row := q.db.QueryRowContext(ctx, getGroupConcatText) - var group_concat string + var group_concat sql.NullString err := row.Scan(&group_concat) return group_concat, err } @@ -80,9 +80,9 @@ const getGroupConcatText2 = `-- name: GetGroupConcatText2 :one SELECT group_concat(text_val, ':') FROM test ` -func (q *Queries) GetGroupConcatText2(ctx context.Context) (string, error) { +func (q *Queries) GetGroupConcatText2(ctx context.Context) (sql.NullString, error) { row := q.db.QueryRowContext(ctx, getGroupConcatText2) - var group_concat string + var group_concat sql.NullString err := row.Scan(&group_concat) return group_concat, err } diff --git a/internal/endtoend/testdata/builtins/sqlite/go/mathfunc.sql.go b/internal/endtoend/testdata/builtins/sqlite/go/mathfunc.sql.go index 30ae799ce5..152d80ad46 100644 --- a/internal/endtoend/testdata/builtins/sqlite/go/mathfunc.sql.go +++ b/internal/endtoend/testdata/builtins/sqlite/go/mathfunc.sql.go @@ -90,9 +90,9 @@ const getCeil = `-- name: GetCeil :one SELECT ceil(1.0) ` -func (q *Queries) GetCeil(ctx context.Context) (int64, error) { +func (q *Queries) GetCeil(ctx context.Context) (float64, error) { row := q.db.QueryRowContext(ctx, getCeil) - var ceil int64 + var ceil float64 err := row.Scan(&ceil) return ceil, err } @@ -101,9 +101,9 @@ const getCeilin = `-- name: GetCeilin :one SELECT ceiling(1.0) ` -func (q *Queries) GetCeilin(ctx context.Context) (int64, error) { +func (q *Queries) GetCeilin(ctx context.Context) (float64, error) { row := q.db.QueryRowContext(ctx, getCeilin) - var ceiling int64 + var ceiling float64 err := row.Scan(&ceiling) return ceiling, err } @@ -156,9 +156,9 @@ const getFloor = `-- name: GetFloor :one SELECT floor(1.0) ` -func (q *Queries) GetFloor(ctx context.Context) (int64, error) { +func (q *Queries) GetFloor(ctx context.Context) (float64, error) { row := q.db.QueryRowContext(ctx, getFloor) - var floor int64 + var floor float64 err := row.Scan(&floor) return floor, err } @@ -321,9 +321,9 @@ const getTrunc = `-- name: GetTrunc :one SELECT trunc(1.0) ` -func (q *Queries) GetTrunc(ctx context.Context) (int64, error) { +func (q *Queries) GetTrunc(ctx context.Context) (float64, error) { row := q.db.QueryRowContext(ctx, getTrunc) - var trunc int64 + var trunc float64 err := row.Scan(&trunc) return trunc, err } diff --git a/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go b/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go index 47989b287c..06ec0436ce 100644 --- a/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go +++ b/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go @@ -69,9 +69,9 @@ const getFormat = `-- name: GetFormat :one SELECT format('Hello %s', 'world') ` -func (q *Queries) GetFormat(ctx context.Context) (sql.NullString, error) { +func (q *Queries) GetFormat(ctx context.Context) (string, error) { row := q.db.QueryRowContext(ctx, getFormat) - var format sql.NullString + var format string err := row.Scan(&format) return format, err } @@ -124,9 +124,9 @@ const getInstr = `-- name: GetInstr :one SELECT instr('hello', 'l') ` -func (q *Queries) GetInstr(ctx context.Context) (sql.NullInt64, error) { +func (q *Queries) GetInstr(ctx context.Context) (int64, error) { row := q.db.QueryRowContext(ctx, getInstr) - var instr sql.NullInt64 + var instr int64 err := row.Scan(&instr) return instr, err } @@ -146,9 +146,9 @@ const getLength = `-- name: GetLength :one SELECT length('12345') ` -func (q *Queries) GetLength(ctx context.Context) (sql.NullInt64, error) { +func (q *Queries) GetLength(ctx context.Context) (int64, error) { row := q.db.QueryRowContext(ctx, getLength) - var length sql.NullInt64 + var length int64 err := row.Scan(&length) return length, err } @@ -267,9 +267,9 @@ const getPrintf = `-- name: GetPrintf :one SELECT printf('Hello %s', 'world') ` -func (q *Queries) GetPrintf(ctx context.Context) (sql.NullString, error) { +func (q *Queries) GetPrintf(ctx context.Context) (string, error) { row := q.db.QueryRowContext(ctx, getPrintf) - var printf sql.NullString + var printf string err := row.Scan(&printf) return printf, err } @@ -289,9 +289,9 @@ const getRandom = `-- name: GetRandom :one SELECT random() ` -func (q *Queries) GetRandom(ctx context.Context) (any, error) { +func (q *Queries) GetRandom(ctx context.Context) (int64, error) { row := q.db.QueryRowContext(ctx, getRandom) - var random any + var random int64 err := row.Scan(&random) return random, err } @@ -300,9 +300,9 @@ const getRandomBlob = `-- name: GetRandomBlob :one SELECT randomblob(16) ` -func (q *Queries) GetRandomBlob(ctx context.Context) (any, error) { +func (q *Queries) GetRandomBlob(ctx context.Context) ([]byte, error) { row := q.db.QueryRowContext(ctx, getRandomBlob) - var randomblob any + var randomblob []byte err := row.Scan(&randomblob) return randomblob, err } @@ -421,9 +421,9 @@ const getSoundex = `-- name: GetSoundex :one SELECT soundex('abc') ` -func (q *Queries) GetSoundex(ctx context.Context) (string, error) { +func (q *Queries) GetSoundex(ctx context.Context) (any, error) { row := q.db.QueryRowContext(ctx, getSoundex) - var soundex string + var soundex any err := row.Scan(&soundex) return soundex, err } diff --git a/internal/endtoend/testdata/table_function/sqlite/go/query.sql.go b/internal/endtoend/testdata/table_function/sqlite/go/query.sql.go index 2ee3387a50..345333f3f7 100644 --- a/internal/endtoend/testdata/table_function/sqlite/go/query.sql.go +++ b/internal/endtoend/testdata/table_function/sqlite/go/query.sql.go @@ -29,7 +29,7 @@ type GetTransactionParams struct { type GetTransactionRow struct { JsonExtract any - JsonGroupArray any + JsonGroupArray string } func (q *Queries) GetTransaction(ctx context.Context, arg GetTransactionParams) ([]GetTransactionRow, error) { diff --git a/internal/engine/sqlite/catalog.go b/internal/engine/sqlite/catalog.go index 1eee9def79..54d198ee9d 100644 --- a/internal/engine/sqlite/catalog.go +++ b/internal/engine/sqlite/catalog.go @@ -9,6 +9,7 @@ func NewCatalog() *catalog.Catalog { Schemas: []*catalog.Schema{ defaultSchema(def), }, - Extensions: map[string]struct{}{}, + LoadExtension: loadExtension, + Extensions: map[string]struct{}{}, } } diff --git a/internal/engine/sqlite/dialect/dialect.json b/internal/engine/sqlite/dialect/dialect.json index 06573b72b9..ba49e48300 100644 --- a/internal/engine/sqlite/dialect/dialect.json +++ b/internal/engine/sqlite/dialect/dialect.json @@ -11,5 +11,13 @@ "comparison_categories": "BNSDU", "arithmetic": ["+", "-", "*", "/", "%"], "arithmetic_categories": "N", - "cast_categories": "*" + "cast_categories": "*", + "modules": { + "fts3": "enable_fts3", + "fts4": "enable_fts3", + "fts5": "enable_fts5", + "geopoly": "enable_geopoly", + "rtree": "enable_rtree", + "rtree_i32": "enable_rtree" + } } diff --git a/internal/engine/sqlite/dialect/extensions/enable_fts3/functions.jsonl b/internal/engine/sqlite/dialect/extensions/enable_fts3/functions.jsonl new file mode 100644 index 0000000000..5f04bc6133 --- /dev/null +++ b/internal/engine/sqlite/dialect/extensions/enable_fts3/functions.jsonl @@ -0,0 +1,5 @@ +{"name":"matchinfo","args":[{"type":"any"}],"returns":"blob"} +{"name":"matchinfo","args":[{"type":"any"},{"type":"text"}],"returns":"blob"} +{"name":"offsets","args":[{"type":"any"}],"returns":"text"} +{"name":"optimize","args":[{"type":"any"}],"returns":"text"} +{"name":"snippet","args":[{"type":"any","has_default":true},{"type":"any","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"integer","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} diff --git a/internal/engine/sqlite/dialect/extensions/enable_fts5/functions.jsonl b/internal/engine/sqlite/dialect/extensions/enable_fts5/functions.jsonl new file mode 100644 index 0000000000..aac7a8163e --- /dev/null +++ b/internal/engine/sqlite/dialect/extensions/enable_fts5/functions.jsonl @@ -0,0 +1,7 @@ +{"name":"bm25","args":[{"type":"any","has_default":true},{"type":"real","mode":"v"}],"returns":"real"} +{"name":"fts5_get_locale","args":[{"type":"any","has_default":true},{"type":"integer","has_default":true},{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"fts5_insttoken","args":[{"type":"any"}],"returns":"any"} +{"name":"fts5_locale","args":[{"type":"text"},{"type":"text"}],"returns":"text"} +{"name":"fts5_source_id","returns":"text"} +{"name":"highlight","args":[{"type":"any","has_default":true},{"type":"integer","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"snippet","args":[{"type":"any","has_default":true},{"type":"any","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"integer","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} diff --git a/internal/engine/sqlite/dialect/extensions/enable_geopoly/functions.jsonl b/internal/engine/sqlite/dialect/extensions/enable_geopoly/functions.jsonl new file mode 100644 index 0000000000..a37aa982b1 --- /dev/null +++ b/internal/engine/sqlite/dialect/extensions/enable_geopoly/functions.jsonl @@ -0,0 +1,15 @@ +{"name":"geopoly_area","args":[{"type":"any"}],"returns":"real","nullable":true} +{"name":"geopoly_bbox","args":[{"type":"any"}],"returns":"blob","nullable":true} +{"name":"geopoly_blob","args":[{"type":"any"}],"returns":"blob","nullable":true} +{"name":"geopoly_ccw","args":[{"type":"any"}],"returns":"blob","nullable":true} +{"name":"geopoly_contains_point","args":[{"type":"any"},{"type":"real"},{"type":"real"}],"returns":"integer","nullable":true} +{"name":"geopoly_group_bbox","kind":"a","args":[{"type":"any"}],"returns":"blob","nullable":true} +{"name":"geopoly_json","args":[{"type":"any"}],"returns":"text","nullable":true} +{"name":"geopoly_overlap","args":[{"type":"any"},{"type":"any"}],"returns":"integer","nullable":true} +{"name":"geopoly_regular","args":[{"type":"real"},{"type":"real"},{"type":"real"},{"type":"integer"}],"returns":"blob"} +{"name":"geopoly_svg","args":[{"type":"text","mode":"v"}],"returns":"text","nullable":true} +{"name":"geopoly_within","args":[{"type":"any"},{"type":"any"}],"returns":"integer","nullable":true} +{"name":"geopoly_xform","args":[{"type":"any"},{"type":"real"},{"type":"real"},{"type":"real"},{"type":"real"},{"type":"real"},{"type":"real"}],"returns":"blob","nullable":true} +{"name":"rtreecheck","args":[{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"rtreedepth","args":[{"type":"blob"}],"returns":"integer"} +{"name":"rtreenode","args":[{"type":"integer"},{"type":"blob"}],"returns":"text"} diff --git a/internal/engine/sqlite/dialect/extensions/enable_offset_sql_func/functions.jsonl b/internal/engine/sqlite/dialect/extensions/enable_offset_sql_func/functions.jsonl new file mode 100644 index 0000000000..7e8563d626 --- /dev/null +++ b/internal/engine/sqlite/dialect/extensions/enable_offset_sql_func/functions.jsonl @@ -0,0 +1 @@ +{"name":"sqlite_offset","args":[{"type":"any"}],"returns":"integer","nullable":true} diff --git a/internal/engine/sqlite/dialect/extensions/enable_percentile/functions.jsonl b/internal/engine/sqlite/dialect/extensions/enable_percentile/functions.jsonl new file mode 100644 index 0000000000..d872692b01 --- /dev/null +++ b/internal/engine/sqlite/dialect/extensions/enable_percentile/functions.jsonl @@ -0,0 +1,4 @@ +{"name":"median","kind":"a","args":[{"type":"real"}],"returns":"real","nullable":true} +{"name":"percentile","kind":"a","args":[{"type":"real"},{"type":"real"}],"returns":"real","nullable":true} +{"name":"percentile_cont","kind":"a","args":[{"type":"real"},{"type":"real"}],"returns":"real","nullable":true} +{"name":"percentile_disc","kind":"a","args":[{"type":"real"},{"type":"real"}],"returns":"real","nullable":true} diff --git a/internal/engine/sqlite/dialect/extensions/enable_rtree/functions.jsonl b/internal/engine/sqlite/dialect/extensions/enable_rtree/functions.jsonl new file mode 100644 index 0000000000..a154d3bfc0 --- /dev/null +++ b/internal/engine/sqlite/dialect/extensions/enable_rtree/functions.jsonl @@ -0,0 +1,3 @@ +{"name":"rtreecheck","args":[{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"rtreedepth","args":[{"type":"blob"}],"returns":"integer"} +{"name":"rtreenode","args":[{"type":"integer"},{"type":"blob"}],"returns":"text"} diff --git a/internal/engine/sqlite/dialect/extensions/soundex/functions.jsonl b/internal/engine/sqlite/dialect/extensions/soundex/functions.jsonl new file mode 100644 index 0000000000..2644ef5d0b --- /dev/null +++ b/internal/engine/sqlite/dialect/extensions/soundex/functions.jsonl @@ -0,0 +1 @@ +{"name":"soundex","args":[{"type":"text"}],"returns":"text"} diff --git a/internal/engine/sqlite/dialect/functions.jsonl b/internal/engine/sqlite/dialect/functions.jsonl index 44a60e4dac..da25d4d581 100644 --- a/internal/engine/sqlite/dialect/functions.jsonl +++ b/internal/engine/sqlite/dialect/functions.jsonl @@ -1,92 +1,158 @@ -{"name":"AVG","args":[{"type":"any"}],"returns":"real","nullable":true} -{"name":"COUNT","returns":"integer"} -{"name":"COUNT","args":[{"type":"any"}],"returns":"integer"} -{"name":"GROUP_CONCAT","args":[{"type":"any"}],"returns":"text"} -{"name":"GROUP_CONCAT","args":[{"type":"any"},{"type":"text"}],"returns":"text"} -{"name":"MAX","args":[{"type":"any"}],"returns":"any","nullable":true} -{"name":"MIN","args":[{"type":"any"}],"returns":"any","nullable":true} -{"name":"SUM","args":[{"type":"any"}],"returns":"real","nullable":true} -{"name":"TOTAL","args":[{"type":"any"}],"returns":"real"} -{"name":"ACOS","args":[{"type":"any"}],"returns":"real"} -{"name":"ACOSH","args":[{"type":"any"}],"returns":"real"} -{"name":"ASIN","args":[{"type":"any"}],"returns":"real"} -{"name":"ASINH","args":[{"type":"any"}],"returns":"real"} -{"name":"ATAN","args":[{"type":"any"}],"returns":"real"} -{"name":"ATAN2","args":[{"type":"any"},{"type":"any"}],"returns":"real"} -{"name":"ATANH","args":[{"type":"any"}],"returns":"real"} -{"name":"CEIL","args":[{"type":"any"}],"returns":"integer"} -{"name":"CEILING","args":[{"type":"any"}],"returns":"integer"} -{"name":"COS","args":[{"type":"any"}],"returns":"real"} -{"name":"COSH","args":[{"type":"any"}],"returns":"real"} -{"name":"DEGREES","args":[{"type":"any"}],"returns":"real"} -{"name":"EXP","args":[{"type":"any"}],"returns":"real"} -{"name":"FLOOR","args":[{"type":"any"}],"returns":"integer"} -{"name":"LN","args":[{"type":"any"}],"returns":"real"} -{"name":"LOG","args":[{"type":"any"}],"returns":"real"} -{"name":"LOG10","args":[{"type":"any"}],"returns":"real"} -{"name":"LOG","args":[{"type":"any"},{"type":"any"}],"returns":"real"} -{"name":"LOG2","args":[{"type":"any"}],"returns":"real"} -{"name":"MOD","args":[{"type":"any"},{"type":"any"}],"returns":"real"} -{"name":"PI","returns":"real"} -{"name":"POW","args":[{"type":"any"},{"type":"any"}],"returns":"real"} -{"name":"POWER","args":[{"type":"any"},{"type":"any"}],"returns":"real"} -{"name":"RADIANS","args":[{"type":"any"}],"returns":"real"} -{"name":"SIN","args":[{"type":"any"}],"returns":"real"} -{"name":"SINH","args":[{"type":"any"}],"returns":"real"} -{"name":"SQRT","args":[{"type":"any"}],"returns":"real"} -{"name":"TAN","args":[{"type":"any"}],"returns":"real"} -{"name":"TANH","args":[{"type":"any"}],"returns":"real"} -{"name":"TRUNC","args":[{"type":"any"}],"returns":"integer"} -{"name":"ABS","args":[{"type":"any"}],"returns":"real"} -{"name":"CHANGES","returns":"integer"} -{"name":"CHAR","args":[{"type":"int"},{"type":"int","mode":"v"}],"returns":"text"} -{"name":"COALESCE","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} -{"name":"FORMAT","args":[{"type":"text"},{"type":"any","mode":"v"}],"returns":"text","nullable":true} -{"name":"GLOB","args":[{"type":"text"},{"type":"text"}],"returns":"integer"} -{"name":"HEX","args":[{"type":"any"}],"returns":"text"} -{"name":"IFNULL","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} -{"name":"IIF","args":[{"type":"any"},{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} -{"name":"INSTR","args":[{"type":"text"},{"type":"text"}],"returns":"integer","nullable":true} -{"name":"LAST_INSERT_ROWID","returns":"integer"} -{"name":"LENGTH","args":[{"type":"any"}],"returns":"integer","nullable":true} -{"name":"LIKE","args":[{"type":"text"},{"type":"text"}],"returns":"integer"} -{"name":"LIKE","args":[{"type":"text"},{"type":"text"},{"type":"text"}],"returns":"integer"} -{"name":"LIKELIHOOD","args":[{"type":"any"},{"type":"real"}],"returns":"any","nullable":true} -{"name":"LIKELY","args":[{"type":"any"}],"returns":"any","nullable":true} -{"name":"LOWER","args":[{"type":"text"}],"returns":"text"} -{"name":"LTRIM","args":[{"type":"text"}],"returns":"text"} -{"name":"LTRIM","args":[{"type":"text"},{"type":"text"}],"returns":"text"} -{"name":"MAX","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} -{"name":"MIN","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} -{"name":"NULLIF","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} -{"name":"PRINTF","args":[{"type":"text"},{"type":"any","mode":"v"}],"returns":"text","nullable":true} -{"name":"QUOTE","args":[{"type":"any"}],"returns":"text"} -{"name":"RAMDOM","returns":"integer"} -{"name":"RAMDOMBLOB","args":[{"type":"integer"}],"returns":"blob"} -{"name":"REPLACE","args":[{"type":"text"},{"type":"text"},{"type":"text"}],"returns":"text"} -{"name":"ROUND","args":[{"type":"real"}],"returns":"real"} -{"name":"ROUND","args":[{"type":"real"},{"type":"real"}],"returns":"real"} -{"name":"RTRIM","args":[{"type":"text"}],"returns":"text"} -{"name":"RTRIM","args":[{"type":"text"},{"type":"text"}],"returns":"text"} -{"name":"SIGN","args":[{"type":"any"}],"returns":"integer","nullable":true} -{"name":"SOUNDEX","args":[{"type":"text"}],"returns":"text"} -{"name":"SQLITE_COMPILEOPTION_GET","args":[{"type":"integer"}],"returns":"text","nullable":true} -{"name":"SQLITE_COMPILEOPTION_USED","args":[{"type":"text"}],"returns":"integer"} -{"name":"SQLITE_OFFSET","args":[{"type":"any"}],"returns":"integer","nullable":true} -{"name":"SQLITE_SOURCE_ID","returns":"text"} -{"name":"SQLITE_VERSION","returns":"text"} -{"name":"SUBSTR","args":[{"type":"any"},{"type":"integer"}],"returns":"text"} -{"name":"SUBSTR","args":[{"type":"any"},{"type":"integer"},{"type":"integer"}],"returns":"text"} -{"name":"SUBSTRING","args":[{"type":"any"},{"type":"integer"}],"returns":"text"} -{"name":"SUBSTRING","args":[{"type":"any"},{"type":"integer"},{"type":"integer"}],"returns":"text"} -{"name":"TOTAL_CHANGES","returns":"integer"} -{"name":"TRIM","args":[{"type":"text"}],"returns":"text"} -{"name":"TRIM","args":[{"type":"text"},{"type":"text"}],"returns":"text"} -{"name":"TYPEOF","args":[{"type":"any"}],"returns":"text"} -{"name":"UNICODE","args":[{"type":"any"}],"returns":"integer"} -{"name":"UNLIKELY","args":[{"type":"any"}],"returns":"any","nullable":true} -{"name":"UPPER","args":[{"type":"text"}],"returns":"text"} -{"name":"ZEROBLOB","args":[{"type":"integer"}],"returns":"blob"} -{"name":"HIGHLIGHT","args":[{"type":"text"},{"type":"integer"},{"type":"text"},{"type":"text"}],"returns":"text"} -{"name":"SNIPPET","args":[{"type":"text"},{"type":"integer"},{"type":"text"},{"type":"text"},{"type":"text"},{"type":"integer"}],"returns":"text"} -{"name":"bm25","args":[{"type":"text"},{"type":"real","mode":"v"}],"returns":"real"} +{"name":"-\u003e","args":[{"type":"any"},{"type":"any"}],"returns":"text","nullable":true} +{"name":"-\u003e\u003e","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} +{"name":"abs","args":[{"type":"any"}],"returns":"real"} +{"name":"acos","args":[{"type":"real"}],"returns":"real"} +{"name":"acosh","args":[{"type":"real"}],"returns":"real"} +{"name":"asin","args":[{"type":"real"}],"returns":"real"} +{"name":"asinh","args":[{"type":"real"}],"returns":"real"} +{"name":"atan","args":[{"type":"real"}],"returns":"real"} +{"name":"atan2","args":[{"type":"real"},{"type":"real"}],"returns":"real"} +{"name":"atanh","args":[{"type":"real"}],"returns":"real"} +{"name":"avg","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} +{"name":"ceil","args":[{"type":"any"}],"returns":"real"} +{"name":"ceiling","args":[{"type":"any"}],"returns":"real"} +{"name":"changes","returns":"integer"} +{"name":"char","args":[{"type":"integer","mode":"v"}],"returns":"text"} +{"name":"coalesce","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any"} +{"name":"concat","args":[{"type":"any"},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"concat_ws","args":[{"type":"text"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"cos","args":[{"type":"real"}],"returns":"real"} +{"name":"cosh","args":[{"type":"real"}],"returns":"real"} +{"name":"count","kind":"a","returns":"integer"} +{"name":"count","kind":"a","args":[{"type":"any"}],"returns":"integer"} +{"name":"cume_dist","kind":"w","returns":"real"} +{"name":"current_date","returns":"text"} +{"name":"current_time","returns":"text"} +{"name":"current_timestamp","returns":"text"} +{"name":"date","args":[{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"datetime","args":[{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"degrees","args":[{"type":"real"}],"returns":"real"} +{"name":"dense_rank","kind":"w","returns":"integer"} +{"name":"exp","args":[{"type":"real"}],"returns":"real"} +{"name":"first_value","kind":"w","args":[{"type":"any"}],"returns":"any","nullable":true} +{"name":"floor","args":[{"type":"any"}],"returns":"real"} +{"name":"format","args":[{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"glob","args":[{"type":"text"},{"type":"text"}],"returns":"integer"} +{"name":"group_concat","kind":"a","args":[{"type":"text"}],"returns":"text","nullable":true} +{"name":"group_concat","kind":"a","args":[{"type":"text"},{"type":"text"}],"returns":"text","nullable":true} +{"name":"hex","args":[{"type":"blob"}],"returns":"text"} +{"name":"if","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any"} +{"name":"ifnull","args":[{"type":"any"},{"type":"any"}],"returns":"any"} +{"name":"iif","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any"} +{"name":"instr","args":[{"type":"any"},{"type":"any"}],"returns":"integer"} +{"name":"json","args":[{"type":"any"}],"returns":"text"} +{"name":"json_array","args":[{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_array_insert","args":[{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_array_length","args":[{"type":"any"}],"returns":"integer","nullable":true} +{"name":"json_array_length","args":[{"type":"any"},{"type":"text"}],"returns":"integer","nullable":true} +{"name":"json_error_position","args":[{"type":"text"}],"returns":"integer"} +{"name":"json_extract","args":[{"type":"text","mode":"v"}],"returns":"any","nullable":true} +{"name":"json_group_array","kind":"a","args":[{"type":"any"}],"returns":"text"} +{"name":"json_group_object","kind":"a","args":[{"type":"text"},{"type":"any"}],"returns":"text"} +{"name":"json_insert","args":[{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_object","args":[{"type":"text","mode":"v"}],"returns":"text"} +{"name":"json_patch","args":[{"type":"any"},{"type":"any"}],"returns":"text"} +{"name":"json_pretty","args":[{"type":"any"}],"returns":"text"} +{"name":"json_pretty","args":[{"type":"any"},{"type":"text"}],"returns":"text"} +{"name":"json_quote","args":[{"type":"any"}],"returns":"text"} +{"name":"json_remove","args":[{"type":"text","mode":"v"}],"returns":"text"} +{"name":"json_replace","args":[{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_set","args":[{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_type","args":[{"type":"any"}],"returns":"text","nullable":true} +{"name":"json_type","args":[{"type":"any"},{"type":"text"}],"returns":"text","nullable":true} +{"name":"json_valid","args":[{"type":"any"}],"returns":"integer"} +{"name":"json_valid","args":[{"type":"any"},{"type":"integer"}],"returns":"integer"} +{"name":"jsonb","args":[{"type":"any"}],"returns":"blob"} +{"name":"jsonb_array","args":[{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"jsonb_array_insert","args":[{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"jsonb_extract","args":[{"type":"text","mode":"v"}],"returns":"any","nullable":true} +{"name":"jsonb_group_array","kind":"a","args":[{"type":"any"}],"returns":"blob"} +{"name":"jsonb_group_object","kind":"a","args":[{"type":"text"},{"type":"any"}],"returns":"blob"} +{"name":"jsonb_insert","args":[{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"jsonb_object","args":[{"type":"text","mode":"v"}],"returns":"blob"} +{"name":"jsonb_patch","args":[{"type":"any"},{"type":"any"}],"returns":"blob"} +{"name":"jsonb_remove","args":[{"type":"text","mode":"v"}],"returns":"blob"} +{"name":"jsonb_replace","args":[{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"jsonb_set","args":[{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"julianday","args":[{"type":"any","mode":"v"}],"returns":"real","nullable":true} +{"name":"lag","kind":"w","args":[{"type":"any"}],"returns":"any","nullable":true} +{"name":"lag","kind":"w","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} +{"name":"lag","kind":"w","args":[{"type":"any"},{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} +{"name":"last_insert_rowid","returns":"integer"} +{"name":"last_value","kind":"w","args":[{"type":"any"}],"returns":"any","nullable":true} +{"name":"lead","kind":"w","args":[{"type":"any"}],"returns":"any","nullable":true} +{"name":"lead","kind":"w","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} +{"name":"lead","kind":"w","args":[{"type":"any"},{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} +{"name":"length","args":[{"type":"text"}],"returns":"integer"} +{"name":"like","args":[{"type":"text"},{"type":"text"}],"returns":"integer"} +{"name":"like","args":[{"type":"text"},{"type":"text"},{"type":"text"}],"returns":"integer"} +{"name":"likelihood","args":[{"type":"any"},{"type":"any"}],"returns":"any"} +{"name":"likely","args":[{"type":"any"}],"returns":"any"} +{"name":"ln","args":[{"type":"real"}],"returns":"real"} +{"name":"log","args":[{"type":"real"}],"returns":"real"} +{"name":"log","args":[{"type":"real"},{"type":"real"}],"returns":"real"} +{"name":"log10","args":[{"type":"real"}],"returns":"real"} +{"name":"log2","args":[{"type":"real"}],"returns":"real"} +{"name":"lower","args":[{"type":"text"}],"returns":"text"} +{"name":"ltrim","args":[{"type":"text"}],"returns":"text"} +{"name":"ltrim","args":[{"type":"text"},{"type":"text"}],"returns":"text"} +{"name":"max","kind":"a","args":[{"type":"any"}],"returns":"any","nullable":true} +{"name":"max","args":[{"type":"any"},{"type":"any","mode":"v"}],"returns":"any"} +{"name":"min","kind":"a","args":[{"type":"any"}],"returns":"any","nullable":true} +{"name":"min","args":[{"type":"any"},{"type":"any","mode":"v"}],"returns":"any"} +{"name":"mod","args":[{"type":"real"},{"type":"real"}],"returns":"real"} +{"name":"nth_value","kind":"w","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} +{"name":"ntile","kind":"w","args":[{"type":"integer"}],"returns":"integer"} +{"name":"nullif","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} +{"name":"octet_length","args":[{"type":"any"}],"returns":"integer"} +{"name":"percent_rank","kind":"w","returns":"real"} +{"name":"pi","returns":"real"} +{"name":"pow","args":[{"type":"real"},{"type":"real"}],"returns":"real"} +{"name":"power","args":[{"type":"real"},{"type":"real"}],"returns":"real"} +{"name":"printf","args":[{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"quote","args":[{"type":"any"}],"returns":"text"} +{"name":"radians","args":[{"type":"real"}],"returns":"real"} +{"name":"random","returns":"integer"} +{"name":"randomblob","args":[{"type":"integer"}],"returns":"blob"} +{"name":"rank","kind":"w","returns":"integer"} +{"name":"replace","args":[{"type":"text"},{"type":"text"},{"type":"text"}],"returns":"text"} +{"name":"round","args":[{"type":"real"}],"returns":"real"} +{"name":"round","args":[{"type":"real"},{"type":"integer"}],"returns":"real"} +{"name":"row_number","kind":"w","returns":"integer"} +{"name":"rtrim","args":[{"type":"text"}],"returns":"text"} +{"name":"rtrim","args":[{"type":"text"},{"type":"text"}],"returns":"text"} +{"name":"sign","args":[{"type":"real"}],"returns":"integer","nullable":true} +{"name":"sin","args":[{"type":"real"}],"returns":"real"} +{"name":"sinh","args":[{"type":"real"}],"returns":"real"} +{"name":"sqlite_compileoption_get","args":[{"type":"integer"}],"returns":"text","nullable":true} +{"name":"sqlite_compileoption_used","args":[{"type":"text"}],"returns":"integer"} +{"name":"sqlite_source_id","returns":"text"} +{"name":"sqlite_version","returns":"text"} +{"name":"sqrt","args":[{"type":"real"}],"returns":"real"} +{"name":"strftime","args":[{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"string_agg","kind":"a","args":[{"type":"text"},{"type":"text"}],"returns":"text","nullable":true} +{"name":"substr","args":[{"type":"any"},{"type":"integer"}],"returns":"text"} +{"name":"substr","args":[{"type":"any"},{"type":"integer"},{"type":"integer"}],"returns":"text"} +{"name":"substring","args":[{"type":"any"},{"type":"integer"}],"returns":"text"} +{"name":"substring","args":[{"type":"any"},{"type":"integer"},{"type":"integer"}],"returns":"text"} +{"name":"subtype","args":[{"type":"any"}],"returns":"integer"} +{"name":"sum","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} +{"name":"tan","args":[{"type":"real"}],"returns":"real"} +{"name":"tanh","args":[{"type":"real"}],"returns":"real"} +{"name":"time","args":[{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"timediff","args":[{"type":"any"},{"type":"any"}],"returns":"text","nullable":true} +{"name":"total","kind":"a","args":[{"type":"any"}],"returns":"real"} +{"name":"total_changes","returns":"integer"} +{"name":"trim","args":[{"type":"text"}],"returns":"text"} +{"name":"trim","args":[{"type":"text"},{"type":"text"}],"returns":"text"} +{"name":"trunc","args":[{"type":"any"}],"returns":"real"} +{"name":"typeof","args":[{"type":"any"}],"returns":"text"} +{"name":"unhex","args":[{"type":"text"}],"returns":"blob","nullable":true} +{"name":"unhex","args":[{"type":"text"},{"type":"text"}],"returns":"blob","nullable":true} +{"name":"unicode","args":[{"type":"text"}],"returns":"integer"} +{"name":"unistr","args":[{"type":"text"}],"returns":"text"} +{"name":"unistr_quote","args":[{"type":"any"}],"returns":"text"} +{"name":"unixepoch","args":[{"type":"any","mode":"v"}],"returns":"real","nullable":true} +{"name":"unlikely","args":[{"type":"any"}],"returns":"any"} +{"name":"upper","args":[{"type":"text"}],"returns":"text"} +{"name":"zeroblob","args":[{"type":"integer"}],"returns":"blob"} diff --git a/internal/engine/sqlite/extension.go b/internal/engine/sqlite/extension.go new file mode 100644 index 0000000000..44265489a9 --- /dev/null +++ b/internal/engine/sqlite/extension.go @@ -0,0 +1,26 @@ +package sqlite + +import ( + "github.com/sqlc-dev/sqlc/internal/core/seed" + "github.com/sqlc-dev/sqlc/internal/sql/catalog" +) + +// loadExtension returns the functions a compile option adds, read from the +// option's directory under the dialect's extensions/. The name is either the +// directory's — enable_fts5 — or a virtual table module the dialect maps to +// one, since a schema that says CREATE VIRTUAL TABLE ... USING fts5 has said +// which build of SQLite it runs on. An option the dialect has no data for +// adds nothing. +func loadExtension(name string) *catalog.Schema { + dir, ok := seed.ExtensionDir(dialectFS, "dialect", name) + if !ok { + return nil + } + funcs, err := seed.Functions(dialectFS, dir) + if err != nil { + // The list is embedded in the binary: a failure here means sqlc was + // built from a broken tree, which no caller can do anything about. + panic(err) + } + return &catalog.Schema{Name: "main", Funcs: funcs} +} diff --git a/internal/engine/sqlite/stdlib.go b/internal/engine/sqlite/stdlib.go index 67e1f12dae..210aa29961 100644 --- a/internal/engine/sqlite/stdlib.go +++ b/internal/engine/sqlite/stdlib.go @@ -5,13 +5,9 @@ import ( ) // defaultSchema is SQLite's standard library, read from the dialect -// directory's functions.jsonl. -// -// The functions are drawn from: -// -// https://www.sqlite.org/lang_aggfunc.html -// https://www.sqlite.org/lang_mathfunc.html -// https://www.sqlite.org/lang_corefunc.html +// directory's functions.jsonl, which internal/goldeneye generates from a +// default build of SQLite. The functions further compile options add live +// under the dialect's extensions/, one directory per option. func defaultSchema(name string) *catalog.Schema { return &catalog.Schema{Name: name, Funcs: stdlib()} } diff --git a/internal/goldeneye/README.md b/internal/goldeneye/README.md index 3bd0a35787..f06dfba04e 100644 --- a/internal/goldeneye/README.md +++ b/internal/goldeneye/README.md @@ -15,6 +15,7 @@ reads the files: the files are the contract. Run it from this directory: ```bash go run ./cmd/goldeneye install clickhouse # download the pinned clickhouse binary once +go run ./cmd/goldeneye install sqlite # build the pinned sqlite3 shells once; needs a C compiler go run ./cmd/goldeneye check # check every engine whose database is available go run ./cmd/goldeneye check postgresql # check one engine go run ./cmd/goldeneye generate [engine] # rewrite the generated files from the database @@ -54,14 +55,41 @@ the hand-written files alone, and the checks do not look at them. `clickhouse/install.go`, and a download that does not match is discarded. ClickHouse describes its functions no further than their names, so `functions.jsonl` is hand-written. +- **`sqlite`** needs no server either: `functions.jsonl` comes from + `pragma_function_list` of a `sqlite3` shell run against an in-memory + database. Which functions a SQLite has is decided when it is compiled, so + `install` downloads the pinned release's amalgamation, checked against the + SHA3-256 the download page lists, and compiles the shell from it with the + compiler `CC` names, or `cc` — once with the options sqlite.org's own + configure turns on by default, which gives `functions.jsonl`, and once + more per option in `sqlite/install.go`'s extension list, each of which + gets a directory under `extensions/` holding the functions its build adds + over the default one, the way each PostgreSQL contrib extension holds what + `CREATE EXTENSION` adds; a schema that says `CREATE VIRTUAL TABLE ... USING + fts5` loads the option's directory, through the `modules` map in the + hand-written `dialect.json`. SQLite describes its functions as far as their + names, their kinds and the number of arguments each overload takes, and no + further — it types values, not functions — so what each returns and what + its arguments hold is read from the amalgamation: every function is + registered with the C functions that implement it, and those set their + result through `sqlite3_result_*` and read their arguments through + `sqlite3_value_*`. A function a shell reports that the source does not + register fails the run rather than being guessed at. Whether an aggregate + returns NULL over no rows is found by running it over none; the scalar + functions that return NULL for arguments that are not are a short list in + `sqlite/signatures.go`, since a SQLite function returns NULL as often by + setting no result as by saying so. The pinned release is the one the main module's + driver embeds. SQLite has no catalog of types or operators, so + `types.jsonl` and `operators.jsonl` are hand-written. ## Layout - `dialect/` — the record types the files are made of, mirrored from `internal/core/seed`, and the helpers that write a generated set of files into an engine directory or diff it against what is committed. -- `postgresql/`, `duckdb/`, `clickhouse/` — one package per engine, each - exposing `Locate`, `Version` and `Generate`, and a test that runs the check. +- `postgresql/`, `duckdb/`, `clickhouse/`, `sqlite/` — one package per + engine, each exposing `Locate`, `Version` and `Generate`, and a test that + runs the check. - `cmd/goldeneye/` — the command. The analysis checks — verifying the `analyze_*` cases under diff --git a/internal/goldeneye/cmd/goldeneye/main.go b/internal/goldeneye/cmd/goldeneye/main.go index c3af78df38..ba6ff1c229 100644 --- a/internal/goldeneye/cmd/goldeneye/main.go +++ b/internal/goldeneye/cmd/goldeneye/main.go @@ -5,6 +5,7 @@ // Usage, from internal/goldeneye: // // go run ./cmd/goldeneye install clickhouse # download the pinned clickhouse binary +// go run ./cmd/goldeneye install sqlite # build the pinned sqlite3 shells from source // go run ./cmd/goldeneye generate [engine] # rewrite the generated files from the database // go run ./cmd/goldeneye check [engine] # compare the committed files with the database // @@ -26,6 +27,7 @@ import ( "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" "github.com/sqlc-dev/sqlc/internal/goldeneye/duckdb" "github.com/sqlc-dev/sqlc/internal/goldeneye/postgresql" + "github.com/sqlc-dev/sqlc/internal/goldeneye/sqlite" ) func main() { @@ -36,14 +38,15 @@ func main() { } const usage = `usage: - goldeneye install clickhouse [-version V] - download the pinned clickhouse binary into the user cache directory + goldeneye install clickhouse|sqlite [-version V] + put the pinned release of an engine into the user cache directory: clickhouse is + downloaded, sqlite is built from the downloaded amalgamation with cc or $CC goldeneye generate [engine] rewrite the generated dialect files from the database, for every available engine or one goldeneye check [engine] compare the committed dialect files with the database, for every available engine or one -engines: clickhouse, duckdb, postgresql` +engines: clickhouse, duckdb, postgresql, sqlite` // engine is one database goldeneye knows how to read a dialect from. type engine struct { @@ -61,6 +64,19 @@ var engines = []engine{ {clickhouse.Engine, clickhouse.Locate, clickhouse.Version, clickhouse.Generate}, {duckdb.Engine, duckdb.Locate, duckdb.Version, duckdb.Generate}, {postgresql.Engine, postgresql.Locate, postgresql.Version, postgresql.Generate}, + {sqlite.Engine, sqlite.Locate, sqlite.Version, sqlite.Generate}, +} + +// installer puts the binary an engine is read through in place, for the +// engines that need no server and whose release is pinned in their package. +type installer struct { + defaultVersion string + install func(ctx context.Context, version, goos, goarch string, progress io.Writer) (string, error) +} + +var installers = map[string]installer{ + clickhouse.Engine: {clickhouse.DefaultVersion, clickhouse.Install}, + sqlite.Engine: {sqlite.DefaultVersion, sqlite.Install}, } func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { @@ -84,16 +100,20 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { } func install(ctx context.Context, args []string, stdout, stderr io.Writer) error { - if len(args) == 0 || args[0] != clickhouse.Engine { - return errors.New("install takes the engine to install: clickhouse") + if len(args) == 0 { + return errors.New("install takes the engine to install: clickhouse or sqlite") + } + inst, ok := installers[args[0]] + if !ok { + return fmt.Errorf("install takes the engine to install, clickhouse or sqlite, not %q", args[0]) } - fs := flag.NewFlagSet("install", flag.ContinueOnError) + fs := flag.NewFlagSet("install "+args[0], flag.ContinueOnError) fs.SetOutput(stderr) - version := fs.String("version", clickhouse.DefaultVersion, "ClickHouse release to install") + version := fs.String("version", inst.defaultVersion, args[0]+" release to install") if err := fs.Parse(args[1:]); err != nil { return err } - path, err := clickhouse.Install(ctx, *version, runtime.GOOS, runtime.GOARCH, stderr) + path, err := inst.install(ctx, *version, runtime.GOOS, runtime.GOARCH, stderr) if err != nil { return err } diff --git a/internal/goldeneye/sqlite/install.go b/internal/goldeneye/sqlite/install.go new file mode 100644 index 0000000000..0a0ed1d674 --- /dev/null +++ b/internal/goldeneye/sqlite/install.go @@ -0,0 +1,318 @@ +package sqlite + +import ( + "archive/zip" + "context" + "crypto/sha3" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" +) + +// DefaultVersion is the SQLite release the dialect is generated from. It is +// the release the ncruces/go-sqlite3 driver in the main module embeds, so +// the functions the dialect knows are the ones the tests run against. +// Bumping it is a deliberate change: releases add functions and overloads, +// so regenerate and review the dialect after changing it, and add the new +// release's amalgamation to the table below. +const DefaultVersion = "3.53.4" + +// defaultOptions are the compile options the base dialect is built with: +// the ones sqlite.org's own configure turns on by default, and so the ones +// a stock sqlite3 has. JSON is part of the library unless omitted, so only +// the math functions need naming. +var defaultOptions = []string{"SQLITE_ENABLE_MATH_FUNCTIONS"} + +// extensions are the compile options that each become a directory under +// the dialect's extensions/, holding the functions a build with the option +// adds over the default build — the way each PostgreSQL contrib extension +// holds what CREATE EXTENSION adds. Each is named after its option as +// pragma compile_options spells it, in lower case, and lists the options +// its build needs: GEOPOLY lives inside the RTREE module, so enabling it +// alone adds nothing. Options that add virtual tables but no functions, +// such as SESSION and DBSTAT, are not listed, since the dialect has nothing +// to say about them, and ENABLE_FTS4 is the same module as ENABLE_FTS3. +var extensions = []build{ + {"soundex", []string{"SQLITE_SOUNDEX"}}, + {"enable_fts3", []string{"SQLITE_ENABLE_FTS3"}}, + {"enable_fts5", []string{"SQLITE_ENABLE_FTS5"}}, + {"enable_geopoly", []string{"SQLITE_ENABLE_RTREE", "SQLITE_ENABLE_GEOPOLY"}}, + {"enable_offset_sql_func", []string{"SQLITE_ENABLE_OFFSET_SQL_FUNC"}}, + {"enable_percentile", []string{"SQLITE_ENABLE_PERCENTILE"}}, + {"enable_rtree", []string{"SQLITE_ENABLE_RTREE"}}, +} + +// asset is one downloadable amalgamation of SQLite: the zip published on +// sqlite.org holding sqlite3.c, sqlite3.h and the shell's shell.c. +type asset struct { + Version string + // Path is the download's address under https://sqlite.org/, the + // release year included, as the download page's index lists it. + Path string + // SHA3 is the SHA3-256 of the zip, as the download page lists it. + SHA3 string +} + +// assets lists every amalgamation Install knows how to fetch, with the +// SHA3-256 of the download. A version that is not in this table cannot be +// installed: verifying the download is the point of the table. The +// checksums are the ones in the index at the foot of +// https://sqlite.org/download.html. +var assets = []asset{ + {"3.53.4", "2026/sqlite-amalgamation-3530400.zip", "628a44cfe82c66aed1ccbbe85a562d2e33ebe64b3288981ed76285612227934e"}, +} + +// sources are the files taken out of the amalgamation. +var sources = []string{"sqlite3.c", "sqlite3.h", "shell.c"} + +func releaseAsset(version string) (asset, error) { + for _, a := range assets { + if a.Version == version { + return a, nil + } + } + return asset{}, fmt.Errorf("SQLite %s is not in the asset table; add its amalgamation and checksum to install.go", version) +} + +func (a asset) url() string { + return "https://sqlite.org/" + a.Path +} + +// cacheDir is where Install puts a version: the sources under src/, and one +// shell per build under default/ and under each option's extension name. +func cacheDir(version string) (string, error) { + dir, err := os.UserCacheDir() + if err != nil { + return "", err + } + return filepath.Join(dir, "sqlc-sqlite", version), nil +} + +// build is one compiled shell: its name, which is the directory it is +// under in the version's cache directory and, for an extension, under the +// dialect's extensions/, and the options it is built with beyond the +// default ones. +type build struct { + name string + options []string +} + +// builds lists the default build first, then one per extension. +func builds() []build { + return append([]build{{"default", nil}}, extensions...) +} + +// flags are every option a build is compiled with. +func (b build) flags() []string { + return append(append([]string{}, defaultOptions...), b.options...) +} + +func (b build) binary(dir string) string { + return filepath.Join(dir, b.name, "sqlite3") +} + +// Locate finds the directory of shells Install made for DefaultVersion. +func Locate() (string, error) { + dir, err := cacheDir(DefaultVersion) + if err != nil { + return "", err + } + for _, b := range builds() { + if _, err := os.Stat(b.binary(dir)); err != nil { + return "", fmt.Errorf("sqlite %s is not built with %s: run `go run ./cmd/goldeneye install sqlite` in internal/goldeneye", DefaultVersion, strings.Join(b.flags(), " ")) + } + } + return dir, nil +} + +// Install downloads the amalgamation for a version into the cache and +// compiles the shell from it once per build, returning the directory the +// shells are under. Builds already there are kept, so adding an option +// compiles only its shell. The download is checked against the table's +// SHA3-256 before it is unpacked. Compiling takes the compiler CC names, or +// cc, and a few seconds per build without optimisation, which a shell that +// only reads catalogs does not need. +// +// goos and goarch are what every installer is handed; the sources build the +// same everywhere a C compiler is, but no default compiler or link line is +// known for Windows. +func Install(ctx context.Context, version, goos, goarch string, progress io.Writer) (string, error) { + if goos == "windows" { + return "", errors.New("building SQLite from source is not supported on Windows") + } + dir, err := cacheDir(version) + if err != nil { + return "", err + } + var missing []build + for _, b := range builds() { + if _, err := os.Stat(b.binary(dir)); err != nil { + missing = append(missing, b) + } + } + if len(missing) == 0 { + return dir, nil + } + if err := download(ctx, version, dir, progress); err != nil { + return "", err + } + if err := compile(ctx, dir, missing, progress); err != nil { + return "", err + } + return dir, nil +} + +// download fetches the amalgamation and unpacks the sources into src/, +// unless they are already there. +func download(ctx context.Context, version, dir string, progress io.Writer) error { + src := filepath.Join(dir, "src") + have := true + for _, name := range sources { + if _, err := os.Stat(filepath.Join(src, name)); err != nil { + have = false + } + } + if have { + return nil + } + a, err := releaseAsset(version) + if err != nil { + return err + } + if err := os.MkdirAll(src, 0o755); err != nil { + return err + } + + url := a.url() + fmt.Fprintf(progress, "downloading %s\n", url) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("downloading %s: %s", url, resp.Status) + } + + // The zip has to land on disk before it can be read — its directory is + // at the end — so download it whole and hash every byte on the way. + archive, err := os.CreateTemp(dir, "sqlite-amalgamation-*.partial") + if err != nil { + return err + } + defer os.Remove(archive.Name()) + sum := sha3.New256() + if _, err := io.Copy(io.MultiWriter(archive, sum), resp.Body); err != nil { + archive.Close() + return err + } + if err := archive.Close(); err != nil { + return err + } + if got := hex.EncodeToString(sum.Sum(nil)); got != a.SHA3 { + return fmt.Errorf("downloading %s: SHA3-256 mismatch: got %s, want %s", url, got, a.SHA3) + } + + zr, err := zip.OpenReader(archive.Name()) + if err != nil { + return err + } + defer zr.Close() + for _, name := range sources { + if err := extract(zr, name, filepath.Join(src, name)); err != nil { + return fmt.Errorf("downloading %s: %w", url, err) + } + } + return nil +} + +// extract copies the named file out of the zip. The amalgamation's files +// sit in one directory named after the release, so the name is matched on +// its base. +func extract(zr *zip.ReadCloser, name, dest string) error { + for _, f := range zr.File { + if filepath.Base(f.Name) != name || f.FileInfo().IsDir() { + continue + } + rc, err := f.Open() + if err != nil { + return err + } + defer rc.Close() + tmp, err := os.CreateTemp(filepath.Dir(dest), name+"-*.partial") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := io.Copy(tmp, rc); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), dest) + } + return fmt.Errorf("zip does not contain %s", name) +} + +// compile builds the shells, as many at a time as there are CPUs. +func compile(ctx context.Context, dir string, todo []build, progress io.Writer) error { + cc := os.Getenv("CC") + if cc == "" { + cc = "cc" + } + if _, err := exec.LookPath(cc); err != nil { + return fmt.Errorf("no C compiler found: put cc on PATH or set CC to one") + } + var wg sync.WaitGroup + slots := make(chan struct{}, max(1, runtime.NumCPU())) + errs := make([]error, len(todo)) + for i, b := range todo { + wg.Add(1) + go func() { + defer wg.Done() + slots <- struct{}{} + defer func() { <-slots }() + errs[i] = b.compile(ctx, cc, dir, progress) + }() + } + wg.Wait() + return errors.Join(errs...) +} + +// compile builds one shell from the sources in src/, writing it beside its +// destination and renaming so a failed build never masquerades as one. +func (b build) compile(ctx context.Context, cc, dir string, progress io.Writer) error { + dest := b.binary(dir) + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return err + } + tmp := dest + ".partial" + defer os.Remove(tmp) + args := []string{"-O0"} + for _, opt := range b.flags() { + args = append(args, "-D"+opt) + } + src := filepath.Join(dir, "src") + args = append(args, filepath.Join(src, "shell.c"), filepath.Join(src, "sqlite3.c"), "-o", tmp, "-lm", "-ldl", "-lpthread") + fmt.Fprintf(progress, "building %s: %s %s\n", b.name, cc, strings.Join(args, " ")) + cmd := exec.CommandContext(ctx, cc, args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("building %s: %w\n%s", b.name, err, strings.TrimSpace(string(out))) + } + return os.Rename(tmp, dest) +} diff --git a/internal/goldeneye/sqlite/signatures.go b/internal/goldeneye/sqlite/signatures.go new file mode 100644 index 0000000000..121bf36507 --- /dev/null +++ b/internal/goldeneye/sqlite/signatures.go @@ -0,0 +1,124 @@ +package sqlite + +import "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" + +// signature is what a function returns and what its arguments hold, read +// from the amalgamation by source.signature. Args types the leading +// arguments in order; a position it does not cover is "any", which the +// analyzer resolves to the argument's own type. Variadic types the +// arguments an overload of variable arity repeats, "any" when empty. How +// many arguments an overload takes, and how many of them it requires, +// comes from the shell. Nullable is decided afterwards: for an aggregate by +// running it over no rows, for a scalar by the nullable list below. +type signature struct { + Args []string + Variadic string + Returns string + Nullable bool +} + +// args builds an overload's parameters from the shell's argument count. A +// fixed count is that many parameters; a negative count is the required +// leading parameters, any further ones the signature types marked as +// having a default, and a variadic tail. +func (s signature) args(narg int) []dialect.Arg { + if narg >= 0 { + args := make([]dialect.Arg, narg) + for i := range args { + args[i] = dialect.Arg{Type: s.argType(i)} + } + return args + } + required := minArgs(narg) + n := max(required, len(s.Args)) + args := make([]dialect.Arg, 0, n+1) + for i := 0; i < n; i++ { + args = append(args, dialect.Arg{Type: s.argType(i), HasDefault: i >= required}) + } + tail := s.Variadic + if tail == "" { + tail = "any" + } + return append(args, dialect.Arg{Type: tail, Mode: "v"}) +} + +func (s signature) argType(i int) string { + if i < len(s.Args) { + return s.Args[i] + } + return "any" +} + +// inlineReturns is what the functions the VDBE implements in bytecode +// return, by the INLINEFUNC_* constant their registration carries. All but +// one hand back one of their arguments, which is what the default of "any" +// says; sqlite_offset is a byte offset. +var inlineReturns = map[string]string{ + "INLINEFUNC_sqlite_offset": "integer", +} + +// omitted are functions the dialect leaves out: ones that exist for their +// side effect and return nothing a query can use, and ones an extension +// uses to pass pointers to itself. +var omitted = map[string]bool{ + // Loads a shared library and returns NULL. + "load_extension": true, + // Writes to the error log and returns NULL. + "sqlite_log": true, + // FTS3's and FTS5's ways of passing pointers to their virtual tables, + // not functions a query calls. + "fts3_tokenizer": true, + "fts5": true, + // A debugging aid of GEOPOLY's. + "geopoly_debug": true, +} + +// nullable are the scalar functions that return NULL for arguments that +// are not: a lookup that finds nothing, an input that does not parse, a +// value with no sign. The source cannot say this — a SQLite function +// returns NULL as often by setting no result as by calling +// sqlite3_result_null — so it is listed by the documentation of each. +// Aggregates are not listed: whether one returns NULL over no rows is +// found by running it over none. +var nullable = map[string]bool{ + // Core functions. + "nullif": true, + "sign": true, + "sqlite_compileoption_get": true, + "sqlite_offset": true, + "unhex": true, + // Date and time functions, for a time value they cannot parse. + "date": true, + "datetime": true, + "julianday": true, + "strftime": true, + "time": true, + "timediff": true, + "unixepoch": true, + // JSON functions, for a path that leads nowhere. + "->": true, + "->>": true, + "json_array_length": true, + "json_extract": true, + "json_type": true, + "jsonb_extract": true, + // Window functions, for a row outside the frame. + "first_value": true, + "lag": true, + "last_value": true, + "lead": true, + "nth_value": true, + // FTS5, for a table with no locale. + "fts5_get_locale": true, + // GEOPOLY functions, for an argument that is not a polygon. + "geopoly_area": true, + "geopoly_bbox": true, + "geopoly_blob": true, + "geopoly_ccw": true, + "geopoly_contains_point": true, + "geopoly_json": true, + "geopoly_overlap": true, + "geopoly_svg": true, + "geopoly_within": true, + "geopoly_xform": true, +} diff --git a/internal/goldeneye/sqlite/source.go b/internal/goldeneye/sqlite/source.go new file mode 100644 index 0000000000..1599e839f9 --- /dev/null +++ b/internal/goldeneye/sqlite/source.go @@ -0,0 +1,517 @@ +package sqlite + +import ( + "fmt" + "os" + "regexp" + "sort" + "strings" +) + +// source is what the amalgamation says about how SQLite's functions answer. +// A function is registered in one of a few shapes — the FuncDef macros of +// the built-in tables, a sqlite3_create_function call, or a struct table an +// extension walks — each naming the C functions that implement it. Those +// implementations set their result through sqlite3_result_* and read their +// arguments through sqlite3_value_*, which is as close as SQLite comes to +// declaring a signature. +type source struct { + funcs map[string]cfunc + aliases map[string]string + regs map[string]*registration +} + +// cfunc is one C function definition: its parameter list and its body. +type cfunc struct { + params string + body string +} + +// registration is the union of everything the source registers under one +// SQL function name. +type registration struct { + // scalar implements a scalar overload: its results and its arguments. + scalar []string + // step and final implement an aggregate or window overload: step reads + // the arguments, final and value set the result. + step []string + final []string + // inline names the INLINEFUNC_* constant of a function the VDBE + // implements in bytecode, which has no C body to read. + inline string + // json records what a JSON function's registration says it returns, + // "text" or "blob", since the json_ and jsonb_ forms share an + // implementation; jsonAlways says the registration promises JSON text + // whatever the implementation might otherwise return, as -> does. + json string + jsonAlways bool + // table marks a registration found only in a struct table, which is + // consulted when nothing more direct registered the name. + table bool +} + +// The FuncDef macros, with the positions of the C functions in each. +var macroEntry = regexp.MustCompile(`\b(FUNCTION2|FUNCTION|VFUNCTION|SFUNCTION|MFUNCTION|JFUNCTION|INLINE_FUNC|DFUNCTION|PURE_DATE|STR_FUNCTION|LIKEFUNC|WAGGREGATE|WINDOWFUNCX|WINDOWFUNCALL|WINDOWFUNCNOOP)\(`) + +var ( + createFunction = regexp.MustCompile(`\bsqlite3_create_(window_)?function\(`) + defineAlias = regexp.MustCompile(`(?m)^#define\s+(\w+)\s+(\w+)\s*$`) + definition = regexp.MustCompile(`(?m)^(?:static\s+|SQLITE_PRIVATE\s+)?(?:const\s+)?(?:unsigned\s+)?[A-Za-z_]\w*(?:\s*\*+\s*|\s+)([A-Za-z_]\w*)\(`) + initializer = regexp.MustCompile(`\{[^{}]*"([a-z_0-9>-]+)"[^{}]*\}`) + identifier = regexp.MustCompile(`\b([A-Za-z_]\w*)\b`) + resultCall = regexp.MustCompile(`\bsqlite3_result_(\w+)\(`) + valueCall = regexp.MustCompile(`\bsqlite3_value_(\w+)\(\s*(?:argv|apVal|apArg)\[(\w+)\]`) + callee = regexp.MustCompile(`\b([A-Za-z_]\w*)\s*\(`) +) + +// readSource reads the amalgamation. +func readSource(path string) (*source, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + text := string(data) + s := &source{ + funcs: map[string]cfunc{}, + aliases: map[string]string{}, + regs: map[string]*registration{}, + } + for _, m := range defineAlias.FindAllStringSubmatch(text, -1) { + s.aliases[m[1]] = m[2] + } + s.readDefinitions(text) + s.readMacros(text) + s.readCreateFunctions(text) + s.readTables(text) + return s, nil +} + +// readDefinitions indexes every function defined at the left margin. +func (s *source) readDefinitions(text string) { + for _, m := range definition.FindAllStringSubmatchIndex(text, -1) { + name := text[m[2]:m[3]] + params, end, ok := balanced(text, m[1]-1, '(', ')') + if !ok { + continue + } + i := end + for i < len(text) && (text[i] == ' ' || text[i] == '\n' || text[i] == '\r' || text[i] == '\t') { + i++ + } + if i >= len(text) || text[i] != '{' { + continue + } + body, _, ok := balanced(text, i, '{', '}') + if !ok { + continue + } + if _, dup := s.funcs[name]; !dup { + s.funcs[name] = cfunc{params: params, body: body} + } + } +} + +// balanced returns the text between the bracket at open and its match, +// skipping string and character literals and comments, and the index after +// the closing bracket. +func balanced(text string, open int, lb, rb byte) (string, int, bool) { + depth := 0 + for i := open; i < len(text); i++ { + switch c := text[i]; { + case c == '"' || c == '\'': + for i++; i < len(text) && text[i] != c; i++ { + if text[i] == '\\' { + i++ + } + } + case c == '/' && i+1 < len(text) && text[i+1] == '*': + end := strings.Index(text[i+2:], "*/") + if end < 0 { + return "", 0, false + } + i += end + 3 + case c == '/' && i+1 < len(text) && text[i+1] == '/': + end := strings.IndexByte(text[i:], '\n') + if end < 0 { + return "", 0, false + } + i += end + case c == lb: + depth++ + case c == rb: + depth-- + if depth == 0 { + return text[open+1 : i], i + 1, true + } + } + } + return "", 0, false +} + +// arguments splits a bracketed argument list at the commas of its own +// level, and reports the index after the closing bracket. +func arguments(text string, open int) ([]string, int, bool) { + inner, end, ok := balanced(text, open, '(', ')') + if !ok { + return nil, 0, false + } + var args []string + depth, start := 0, 0 + for i := 0; i < len(inner); i++ { + switch inner[i] { + case '(', '[', '{': + depth++ + case ')', ']', '}': + depth-- + case '"': + for i++; i < len(inner) && inner[i] != '"'; i++ { + if inner[i] == '\\' { + i++ + } + } + case ',': + if depth == 0 { + args = append(args, strings.TrimSpace(inner[start:i])) + start = i + 1 + } + } + } + args = append(args, strings.TrimSpace(inner[start:])) + return args, end, true +} + +func (s *source) reg(name string) *registration { + name = strings.ToLower(name) + r, ok := s.regs[name] + if !ok { + r = ®istration{} + s.regs[name] = r + } + return r +} + +// readMacros reads the FuncDef tables. The macro definitions themselves +// match too, and are told apart by their formal parameter names. +func (s *source) readMacros(text string) { + for _, m := range macroEntry.FindAllStringSubmatchIndex(text, -1) { + macro := text[m[2]:m[3]] + args, _, ok := arguments(text, m[1]-1) + if !ok || len(args) < 2 || args[0] == "zName" || args[0] == "name" { + continue + } + name := args[0] + switch macro { + case "FUNCTION", "FUNCTION2", "VFUNCTION", "SFUNCTION", "DFUNCTION", "PURE_DATE", "STR_FUNCTION": + if len(args) > 4 { + r := s.reg(name) + r.scalar = append(r.scalar, args[4]) + } + case "MFUNCTION": + if len(args) > 3 { + r := s.reg(name) + r.scalar = append(r.scalar, args[3]) + } + case "JFUNCTION": + // JFUNCTION(zName, nArg, bUseCache, bWS, bRS, bJsonB, iArg, xFunc) + if len(args) > 7 { + r := s.reg(name) + r.scalar = append(r.scalar, args[7]) + r.json = "text" + if args[5] == "1" { + r.json = "blob" + } + r.jsonAlways = strings.Contains(args[6], "JSON_JSON") + } + case "INLINE_FUNC": + if len(args) > 2 { + s.reg(name).inline = args[2] + } + case "LIKEFUNC": + r := s.reg(name) + r.scalar = append(r.scalar, "likeFunc") + case "WAGGREGATE": + // WAGGREGATE(zName, nArg, arg, nc, xStep, xFinal, xValue, xInverse, f) + // The JSON aggregates use it too, told by their implementation, + // with JSON_BLOB as the user data of the jsonb_ forms. + if len(args) > 6 { + r := s.reg(name) + r.step = append(r.step, args[4]) + r.final = append(r.final, args[5], args[6]) + if strings.HasPrefix(args[4], "json") { + r.json = "text" + if strings.Contains(args[2], "JSON_BLOB") { + r.json = "blob" + } + } + } + case "WINDOWFUNCX", "WINDOWFUNCALL": + r := s.reg(name) + r.step = append(r.step, name+"StepFunc") + r.final = append(r.final, name+"ValueFunc") + case "WINDOWFUNCNOOP": + s.reg(name).inline = "bytecode" + } + } +} + +// readCreateFunctions reads the functions extensions register by calling +// sqlite3_create_function or sqlite3_create_window_function with a literal +// name. +func (s *source) readCreateFunctions(text string) { + for _, m := range createFunction.FindAllStringSubmatchIndex(text, -1) { + window := m[2] != -1 + args, _, ok := arguments(text, m[1]-1) + if !ok || len(args) < 8 || !strings.HasPrefix(args[1], `"`) { + continue + } + r := s.reg(strings.Trim(args[1], `"`)) + if window { + // (db, zName, nArg, eTextRep, pApp, xStep, xFinal, xValue, xInverse, xDestroy) + r.step = given(r.step, args[5]) + r.final = given(r.final, args[6], args[7]) + } else { + // (db, zName, nArg, eTextRep, pApp, xFunc, xStep, xFinal) + r.scalar = given(r.scalar, args[5]) + r.step = given(r.step, args[6]) + r.final = given(r.final, args[7]) + } + } +} + +// given appends the implementations a registration names, leaving out the +// methods it passes as 0 or NULL. +func given(impls []string, names ...string) []string { + for _, name := range names { + if name != "0" && name != "NULL" { + impls = append(impls, name) + } + } + return impls +} + +// readTables reads the struct tables extensions walk to register their +// functions — geopoly's aFunc, FTS5's aBuiltin, FTS3's aOverload — each row +// an initializer holding the function's name and the C functions that +// implement it. Any C function in the row counts as an implementation; +// which reads arguments and which sets the result comes out in the +// reading. A row is consulted only for a name nothing else registered. +func (s *source) readTables(text string) { + for _, m := range initializer.FindAllStringSubmatch(text, -1) { + var impls []string + for _, id := range identifier.FindAllStringSubmatch(m[0], -1) { + if _, ok := s.funcs[s.resolve(id[1])]; ok { + impls = append(impls, id[1]) + } + } + if len(impls) == 0 { + continue + } + name := strings.ToLower(m[1]) + r, ok := s.regs[name] + if ok && !r.table { + continue + } + r = s.reg(name) + r.table = true + r.scalar = append(r.scalar, impls...) + } +} + +// resolve follows #define aliases from a C function name to the one that +// is defined. +func (s *source) resolve(name string) string { + for i := 0; i < 8; i++ { + alias, ok := s.aliases[name] + if !ok { + return name + } + name = alias + } + return name +} + +// noop reports a C function that does nothing: the stand-in for a function +// the VDBE implements in bytecode, whose result is one of its arguments. +func noop(name string) bool { + return strings.HasPrefix(name, "noop") +} + +// The kinds a result or argument call names, by the sqlite3_result_* or +// sqlite3_value_* suffix with any 64 dropped. Suffixes not listed — error, +// subtype, type, bytes, dup — say nothing about a type. +var ( + resultKinds = map[string]string{ + "int": "integer", "double": "real", + "text": "text", "text16": "text", "text16le": "text", "text16be": "text", + "blob": "blob", "zeroblob": "blob", + "value": "any", "pointer": "any", + } + valueKinds = map[string]string{ + "int": "integer", "double": "real", + "text": "text", "text16": "text", "text16le": "text", "text16be": "text", + "blob": "blob", + } +) + +func kindOf(kinds map[string]string, suffix string) (string, bool) { + k, ok := kinds[strings.TrimSuffix(suffix, "64")] + return k, ok +} + +// results collects the kinds a C function sets its result to, following +// the helpers it calls a few levels down, since many functions hand their +// result to one. +func (s *source) results(name string, depth int, seen map[string]bool) map[string]bool { + kinds := map[string]bool{} + name = s.resolve(name) + if noop(name) { + kinds["any"] = true + return kinds + } + fn, ok := s.funcs[name] + if !ok || seen[name] || depth > 3 { + return kinds + } + seen[name] = true + for _, m := range resultCall.FindAllStringSubmatch(fn.body, -1) { + if k, ok := kindOf(resultKinds, m[1]); ok { + kinds[k] = true + } + } + for _, m := range callee.FindAllStringSubmatch(fn.body, -1) { + c := m[1] + if c == name || strings.HasPrefix(c, "sqlite3_") || strings.HasPrefix(c, "sqlite3Vdbe") { + continue + } + if _, ok := s.funcs[c]; ok { + for k := range s.results(c, depth+1, seen) { + kinds[k] = true + } + } + } + return kinds +} + +// args collects the kinds a C function reads each of its arguments as, by +// position, and the kind it reads a run of arguments as under a loop +// index. An implementation called through the FTS5 extension API is handed +// the table as an implicit first argument, so its positions shift by one. +func (s *source) args(name string, positions map[int]map[string]bool, variadic map[string]bool) { + fn, ok := s.funcs[s.resolve(name)] + if !ok { + return + } + shift := 0 + if strings.Contains(fn.params, "Fts5ExtensionApi") { + shift = 1 + if positions[0] == nil { + positions[0] = map[string]bool{} + } + positions[0]["any"] = true + } + for _, m := range valueCall.FindAllStringSubmatch(fn.body, -1) { + k, ok := kindOf(valueKinds, m[1]) + if !ok { + continue + } + var pos int + if _, err := fmt.Sscanf(m[2], "%d", &pos); err != nil { + variadic[k] = true + continue + } + pos += shift + if positions[pos] == nil { + positions[pos] = map[string]bool{} + } + positions[pos][k] = true + } +} + +// single reduces the kinds seen at one position to a type: the one kind +// seen, or "any" for a mixture or nothing. +func single(kinds map[string]bool) string { + if len(kinds) == 1 { + for k := range kinds { + return k + } + } + return "any" +} + +// signature derives what the source says a SQL function returns and takes. +// A result of one kind is that type. A function that returns one of its +// arguments, or a mixture of kinds, takes the type of its first argument, +// which the seed spells "any" — except that integer and real together widen +// to real, as SQLite's own arithmetic does, and text and blob together to +// text, since a function that returns either is handing back the bytes it +// was given, and the legacy compiler cannot follow "any" to an argument. +func (s *source) signature(name string) (signature, error) { + r, ok := s.regs[strings.ToLower(name)] + if !ok { + return signature{}, fmt.Errorf("the amalgamation registers no function named %s", name) + } + var sig signature + switch { + case r.inline != "": + sig.Returns = inlineReturns[r.inline] + if sig.Returns == "" { + sig.Returns = "any" + } + default: + kinds := map[string]bool{} + for _, fn := range append(append([]string{}, r.scalar...), r.final...) { + for k := range s.results(fn, 0, map[string]bool{}) { + kinds[k] = true + } + } + jsonOnly := r.json != "" && len(kinds) <= 2 && !kinds["integer"] && !kinds["real"] && !kinds["any"] + switch { + case len(kinds) == 0: + return signature{}, fmt.Errorf("cannot tell what %s returns: no sqlite3_result call in %s", name, strings.Join(append(append([]string{}, r.scalar...), r.final...), ", ")) + case r.jsonAlways: + sig.Returns = "text" + case kinds["any"]: + sig.Returns = "any" + case len(kinds) == 1: + for k := range kinds { + sig.Returns = k + } + case jsonOnly: + // The shared implementation writes JSON text or JSONB; the + // registration says which this form gets. + sig.Returns = r.json + case len(kinds) == 2 && kinds["integer"] && kinds["real"]: + sig.Returns = "real" + case len(kinds) == 2 && kinds["text"] && kinds["blob"]: + sig.Returns = "text" + default: + sig.Returns = "any" + } + } + + positions := map[int]map[string]bool{} + variadic := map[string]bool{} + for _, fn := range append(append([]string{}, r.scalar...), r.step...) { + s.args(fn, positions, variadic) + } + if len(positions) > 0 { + var order []int + for pos := range positions { + order = append(order, pos) + } + sort.Ints(order) + sig.Args = make([]string, order[len(order)-1]+1) + for i := range sig.Args { + sig.Args[i] = single(positions[i]) + } + // Trailing positions nothing typed say nothing, unless a variadic + // tail follows them, when they keep it from starting early. + for len(variadic) == 0 && len(sig.Args) > 0 && sig.Args[len(sig.Args)-1] == "any" { + sig.Args = sig.Args[:len(sig.Args)-1] + } + } + if len(variadic) == 1 { + sig.Variadic = single(variadic) + } + return sig, nil +} diff --git a/internal/goldeneye/sqlite/sqlite.go b/internal/goldeneye/sqlite/sqlite.go new file mode 100644 index 0000000000..d8aa1b62ba --- /dev/null +++ b/internal/goldeneye/sqlite/sqlite.go @@ -0,0 +1,401 @@ +// Package sqlite generates the SQLite dialect seed under +// internal/engine/sqlite/dialect from sqlite3 shells built from the +// amalgamation sqlite.org publishes, run against in-memory databases that +// need no server, and from the amalgamation itself. +// +// SQLite describes its functions as far as their names, their kinds and the +// number of arguments each takes — pragma_function_list — and no further: +// it types values rather than columns or functions, so nothing in the +// database says what a function returns or what it expects. The source +// does, in its way. Every function is registered with the C functions that +// implement it, and those set their result through sqlite3_result_* and +// read their arguments through sqlite3_value_*, which is as close as SQLite +// comes to declaring a signature. So functions.jsonl is built from both: +// the shell says which functions exist, how many arguments each overload +// takes and whether it aggregates, and the amalgamation says what each +// returns and what its arguments hold. A function the shell reports that +// the source does not register, or whose implementation sets no result, +// fails generation rather than being guessed at. What neither can say — +// which scalar functions return NULL for arguments that are not — is a +// short list in signatures.go; for aggregates it is found by running each +// over no rows. +// +// Which functions a SQLite has is decided when it is compiled, so the +// dialect treats compile options the way the PostgreSQL dialect treats +// contrib extensions. functions.jsonl is what a build with the default +// options has, and each further option gets a directory under extensions/ +// holding the functions a build with that option adds — found the way +// CREATE EXTENSION's additions are, by comparing the catalog with and +// without. SQLite has no catalog of types or operators, so types.jsonl and +// operators.jsonl are hand-written. +// +// The shells are built once per pinned version by Install. +package sqlite + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "os/exec" + "path" + "path/filepath" + "slices" + "sort" + "strconv" + "strings" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" +) + +// Engine is the name of the engine directory the dialect lives under. +const Engine = "sqlite" + +// functionList is every function the connection knows, one row per +// overload: name, whether the library builds it in or the shell or an +// extension registered it, 's', 'a' or 'w' for how it is called, and how +// many arguments it takes. Fixed arities come before the variable one so +// that an exact match is found first by anything reading the file in +// order. +const functionList = ` +SELECT name, builtin, type, narg +FROM pragma_function_list +ORDER BY name, narg < 0, narg` + +type functionRow struct { + Name string `json:"name"` + Builtin int `json:"builtin"` + Type string `json:"type"` + NArg int `json:"narg"` +} + +// key tells one overload from another. +func (r functionRow) key() string { + return r.Name + "/" + strconv.Itoa(r.NArg) +} + +// Version reports the release the default shell is. +func Version(ctx context.Context, dir string) (string, error) { + out, err := exec.CommandContext(ctx, builds()[0].binary(dir), "--version").Output() + if err != nil { + return "", fmt.Errorf("sqlite3 --version: %w", err) + } + return "SQLite " + strings.TrimSpace(string(out)), nil +} + +// query runs a SQL statement against an in-memory database and decodes the +// shell's JSON output into rows. +func query(ctx context.Context, binary, sql string, rows any) error { + cmd := exec.CommandContext(ctx, binary, "-json", "-bail", ":memory:", sql) + out, err := cmd.Output() + if err != nil { + var exit *exec.ExitError + if errors.As(err, &exit) { + return fmt.Errorf("sqlite3: %s: %s", err, bytes.TrimSpace(exit.Stderr)) + } + return fmt.Errorf("sqlite3: %w", err) + } + // An empty result set prints nothing rather than []. + if len(bytes.TrimSpace(out)) == 0 { + return nil + } + return json.Unmarshal(out, rows) +} + +// minArgs decodes a negative argument count from pragma_function_list, +// which prints the count SQLite keeps for the function as it is: -1 for any +// number of arguments, and the two values reserved for built-ins, -3 for +// one or more and -4 for two or more — see matchQuality in the SQLite +// source. No function is counted as -2. +func minArgs(narg int) int { + if narg < -2 { + return -2 - narg + } + return 0 +} + +// functionKind maps the type the shell reports onto the seed's letters. +// The shell says 's' for a scalar function, 'a' for an aggregate and 'w' +// for one with a window implementation — but every one of SQLite's +// aggregates has one, so 'w' covers avg and count as well as row_number. +// The two are told apart by calling the function without an OVER clause, +// which only a window function refuses. +func functionKind(ctx context.Context, binary string, row functionRow) (string, error) { + switch row.Type { + case "s": + return "", nil + case "a": + return "a", nil + case "w": + window, err := isWindowFunction(ctx, binary, row) + if err != nil { + return "", err + } + if window { + return "w", nil + } + return "a", nil + } + return "", fmt.Errorf("sqlite: function %s has unknown type %q", row.Name, row.Type) +} + +// isWindowFunction reports whether the shell refuses to call a function +// without an OVER clause. Any other complaint — an aggregate objecting to +// the NULLs it is handed — means the call was accepted as an aggregate. +func isWindowFunction(ctx context.Context, binary string, row functionRow) (bool, error) { + n := row.NArg + if n < 0 { + n = minArgs(n) + } + args := strings.TrimSuffix(strings.Repeat("NULL, ", n), ", ") + cmd := exec.CommandContext(ctx, binary, ":memory:", fmt.Sprintf("SELECT %s(%s)", row.Name, args)) + var stderr bytes.Buffer + cmd.Stdout = io.Discard + cmd.Stderr = &stderr + err := cmd.Run() + if err == nil { + return false, nil + } + if strings.Contains(stderr.String(), "misuse of window function") { + return true, nil + } + var exit *exec.ExitError + if errors.As(err, &exit) { + return false, nil + } + return false, fmt.Errorf("sqlite3: %w", err) +} + +// shell is one built sqlite3 and the functions it reports. +type shell struct { + build build + binary string + rows []functionRow +} + +// readShell lists a build's functions, after checking that the shell was +// built with the options the build says — a cached shell from before the +// option lists changed would otherwise describe the wrong dialect. +func readShell(ctx context.Context, dir string, b build) (*shell, error) { + s := &shell{build: b, binary: b.binary(dir)} + var used []struct { + Option string `json:"option"` + Used int `json:"used"` + } + known := map[string]bool{} + for _, b := range builds() { + for _, opt := range b.flags() { + known[opt] = true + } + } + var clauses []string + for _, opt := range slices.Sorted(maps.Keys(known)) { + clauses = append(clauses, fmt.Sprintf("SELECT '%s' AS option, sqlite_compileoption_used('%s') AS used", opt, opt)) + } + if err := query(ctx, s.binary, strings.Join(clauses, " UNION ALL "), &used); err != nil { + return nil, err + } + for _, u := range used { + want := 0 + if slices.Contains(b.flags(), u.Option) { + want = 1 + } + if u.Used != want { + return nil, fmt.Errorf("sqlite: the %s shell was not built with the options it should have been (%s is %d): remove %s and run `go run ./cmd/goldeneye install sqlite` again", b.name, u.Option, u.Used, dir) + } + } + if err := query(ctx, s.binary, functionList, &s.rows); err != nil { + return nil, err + } + return s, nil +} + +// generator accumulates the functions of every build, reading their +// signatures from the amalgamation, and remembers which names were +// reported so that the lists in signatures.go can be checked against the +// shells at the end. +type generator struct { + ctx context.Context + src *source + reported map[string]bool +} + +// overload is one row of a shell's list with what was found out about it. +type overload struct { + row functionRow + kind string + sig signature +} + +// functions turns rows into records, one per overload. Every row has to be +// a function the amalgamation registers unless omitted. In the default +// build only what the library builds in counts: the rest is what the shell +// registers on top — edit, sha3, the extensions it bundles — which is not +// the dialect's business, and which a comparison with the default build has +// already removed from an option's rows. +func (g *generator) functions(s *shell, base bool) ([]dialect.Function, error) { + var overloads []overload + seen := map[string]bool{} + var missing []string + for _, row := range s.rows { + g.reported[row.Name] = true + if omitted[row.Name] || seen[row.key()] || base && row.Builtin == 0 { + continue + } + seen[row.key()] = true + sig, err := g.src.signature(row.Name) + if err != nil { + if !seen[row.Name] { + seen[row.Name] = true + missing = append(missing, err.Error()) + } + continue + } + kind, err := functionKind(g.ctx, s.binary, row) + if err != nil { + return nil, err + } + overloads = append(overloads, overload{row: row, kind: kind, sig: sig}) + } + if len(missing) > 0 { + return nil, fmt.Errorf("sqlite: %s build: %s", s.build.name, strings.Join(missing, "; ")) + } + empty, err := overNoRows(g.ctx, s.binary, overloads) + if err != nil { + return nil, err + } + funcs := make([]dialect.Function, 0, len(overloads)) + for _, o := range overloads { + isNullable := nullable[o.row.Name] + if o.kind == "a" { + isNullable = empty[o.row.key()] == "null" + } + funcs = append(funcs, dialect.Function{ + Name: o.row.Name, + Kind: o.kind, + Args: o.sig.args(o.row.NArg), + Returns: o.sig.Returns, + Nullable: isNullable, + }) + } + return funcs, nil +} + +// overNoRows runs every aggregate over no rows and reports the type of +// what each returns, "null" for the ones — avg, max, group_concat — that +// return NULL when there is nothing to aggregate, and not count or total. +// The probes go through one shell process, each statement labelled with +// its overload so that the answers can be told apart. +func overNoRows(ctx context.Context, binary string, overloads []overload) (map[string]string, error) { + var script strings.Builder + for _, o := range overloads { + if o.kind != "a" { + continue + } + n := o.row.NArg + if n < 0 { + n = minArgs(n) + } + args := strings.TrimSuffix(strings.Repeat("x, ", n), ", ") + fmt.Fprintf(&script, "SELECT '%s', typeof(%s(%s)) FROM (SELECT NULL AS x) WHERE 0;\n", o.row.key(), o.row.Name, args) + } + results := map[string]string{} + if script.Len() == 0 { + return results, nil + } + cmd := exec.CommandContext(ctx, binary, "-list", ":memory:", script.String()) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + var exit *exec.ExitError + if !errors.As(err, &exit) { + return nil, fmt.Errorf("sqlite3: %w", err) + } + } + for _, line := range strings.Split(strings.TrimSpace(stdout.String()), "\n") { + key, typ, ok := strings.Cut(line, "|") + if ok { + results[key] = typ + } + } + for _, o := range overloads { + if o.kind == "a" { + if _, ok := results[o.row.key()]; !ok { + return nil, fmt.Errorf("sqlite: cannot run %s over no rows: %s", o.row.Name, strings.TrimSpace(stderr.String())) + } + } + } + return results, nil +} + +// added returns the rows of an option's shell that the default shell does +// not have: what the option adds. +func added(opt, base *shell) *shell { + have := map[string]bool{} + for _, row := range base.rows { + have[row.key()] = true + } + diff := &shell{build: opt.build, binary: opt.binary} + for _, row := range opt.rows { + if !have[row.key()] { + diff.rows = append(diff.rows, row) + } + } + return diff +} + +// Generate reads the dialect from the shells under dir and the +// amalgamation they were built from. +func Generate(ctx context.Context, dir string) (dialect.Files, error) { + src, err := readSource(filepath.Join(dir, "src", "sqlite3.c")) + if err != nil { + return nil, err + } + g := &generator{ctx: ctx, src: src, reported: map[string]bool{}} + all := builds() + base, err := readShell(ctx, dir, all[0]) + if err != nil { + return nil, err + } + funcs, err := g.functions(base, true) + if err != nil { + return nil, err + } + files := dialect.Files{} + if files[dialect.FunctionsFile], err = dialect.JSONL(funcs); err != nil { + return nil, err + } + for _, b := range all[1:] { + s, err := readShell(ctx, dir, b) + if err != nil { + return nil, err + } + funcs, err := g.functions(added(s, base), false) + if err != nil { + return nil, err + } + if len(funcs) == 0 { + return nil, fmt.Errorf("sqlite: %s adds no functions over the default build; drop it from the extensions in install.go", strings.Join(b.options, " ")) + } + if files[path.Join(dialect.ExtensionsDir, b.name, dialect.FunctionsFile)], err = dialect.JSONL(funcs); err != nil { + return nil, err + } + } + var stale []string + for _, list := range []map[string]bool{omitted, nullable} { + for name := range list { + if !g.reported[name] { + stale = append(stale, name) + } + } + } + if len(stale) > 0 { + sort.Strings(stale) + return nil, fmt.Errorf("sqlite: signatures.go lists function(s) no shell reports: %s", strings.Join(stale, ", ")) + } + return files, nil +} diff --git a/internal/goldeneye/sqlite/sqlite_test.go b/internal/goldeneye/sqlite/sqlite_test.go new file mode 100644 index 0000000000..028f015fbf --- /dev/null +++ b/internal/goldeneye/sqlite/sqlite_test.go @@ -0,0 +1,37 @@ +package sqlite + +import ( + "context" + "testing" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" +) + +// TestDialect verifies the committed SQLite dialect against what the pinned +// sqlite3 shell reports. It skips unless the shell is installed. +func TestDialect(t *testing.T) { + binary, err := Locate() + if err != nil { + t.Skip(err) + } + ctx := context.Background() + version, err := Version(ctx, binary) + if err != nil { + t.Fatal(err) + } + files, err := Generate(ctx, binary) + if err != nil { + t.Fatal(err) + } + dir, err := dialect.Dir(Engine) + if err != nil { + t.Fatal(err) + } + report, err := dialect.Check(dir, files) + if err != nil { + t.Fatal(err) + } + if report != "" { + t.Errorf("%s does not match what %s reports:\n%s", dir, version, report) + } +} diff --git a/internal/sql/catalog/extension.go b/internal/sql/catalog/extension.go index fdb717f2d2..268bf01927 100644 --- a/internal/sql/catalog/extension.go +++ b/internal/sql/catalog/extension.go @@ -9,16 +9,28 @@ func (c *Catalog) createExtension(stmt *ast.CreateExtensionStmt) error { return nil } // TODO: Implement IF NOT EXISTS - if _, exists := c.Extensions[*stmt.Extname]; exists { + return c.loadExtension(*stmt.Extname) +} + +// loadExtension adds what the engine knows of the named extension — or of +// the virtual table module that stands for one — to the default schema, +// once. An engine without extension data, or without data for this one, +// adds nothing. +func (c *Catalog) loadExtension(name string) error { + if _, exists := c.Extensions[name]; exists { return nil } if c.LoadExtension == nil { return nil } - ext := c.LoadExtension(*stmt.Extname) + ext := c.LoadExtension(name) if ext == nil { return nil } + if c.Extensions == nil { + c.Extensions = map[string]struct{}{} + } + c.Extensions[name] = struct{}{} s, err := c.getSchema(c.DefaultSchema) if err != nil { return err diff --git a/internal/sql/catalog/table.go b/internal/sql/catalog/table.go index ec2a122a96..a91e399a7e 100644 --- a/internal/sql/catalog/table.go +++ b/internal/sql/catalog/table.go @@ -248,6 +248,13 @@ func (c *Catalog) alterTableSetSchema(stmt *ast.AlterTableSetSchemaStmt) error { } func (c *Catalog) createTable(stmt *ast.CreateTableStmt) error { + // A virtual table's module is the engine's word for the extension it + // needs, the way CREATE EXTENSION is PostgreSQL's. + if stmt.Using != "" { + if err := c.loadExtension(stmt.Using); err != nil { + return err + } + } ns := stmt.Name.Schema if ns == "" { ns = c.DefaultSchema