Milestone 7: advance the pin to DuckDB v2.0.0

.github/workflows/difftest.yml

@@ -2,9 +2,9 @@ name: difftest
 
 # The milestone-6 cron lane: differential fuzzing of darkwing against the
 # pinned DuckDB CLI, plus a native fuzz run of the public Parse API. The
-# nightly artifact only matches the pin until upstream moves it, so the
-# lane verifies the version first and skips (green) once it drifts —
-# advancing the pin is its own workflow (cmd/regenerate), not this one.
+# pin is a tagged release, so the CLI comes from its immutable release
+# asset (duckdbsrc.ReleaseURL); the version check stays as a guard against
+# a pin advanced without updating that URL.
 
 on:
   schedule:
@@ -23,13 +23,13 @@ jobs:
       - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
-      - name: Download the nightly DuckDB CLI
+      - name: Download the pinned DuckDB CLI
         run: |
-          curl -sSL -o duckdb.zip https://artifacts.duckdb.org/latest/duckdb-binaries-linux-amd64.zip
-          unzip -o -q duckdb.zip duckdb_cli-linux-amd64.zip
+          url=$(sed -n 's/.*ReleaseURL = "\(.*\)"/\1/p' internal/duckdbsrc/duckdbsrc.go)
+          curl -fsSL -o duckdb_cli-linux-amd64.zip "$url"
           unzip -o -q duckdb_cli-linux-amd64.zip
           ./duckdb --version
-      - name: Check the nightly against the pin
+      - name: Check the CLI against the pin
         id: pin
         run: |
           pin=$(sed -n 's/.*PinnedVersion = "\(.*\)"/\1/p' internal/duckdbsrc/duckdbsrc.go)
@@ -38,7 +38,7 @@ jobs:
             echo "matches=true" >> "$GITHUB_OUTPUT"
           else
             echo "matches=false" >> "$GITHUB_OUTPUT"
-            echo "nightly is $version, pin is $pin - skipping until the pin advances"
+            echo "CLI is $version, pin is $pin - skipping until ReleaseURL matches the pin"
           fi
       - name: Differential fuzzing vs the pinned CLI
         if: steps.pin.outputs.matches == 'true'

.gitignore

@@ -1,4 +1,4 @@
-duckdb
+/duckdb
 *.db
 *.db.wal
 *.zip

CLAUDE.md

@@ -7,7 +7,7 @@ in `internal/grammar/README.md`.
 
 ## Layout
 
-- `token/` — token kinds, `Token{Kind, Text, Span}`, the five keyword
+- `token/` — token kinds, `Token{Kind, Text, Span}`, the six keyword
   categories (built from the vendored `.list` files).
 - `lexer/` — port of `base_tokenizer.cpp` + `parser_tokenizer.cpp`.
 - `internal/grammar/` — vendored `.gram`/`.list` files (never edit them by
@@ -46,10 +46,14 @@ in `internal/grammar/README.md`.
 
 - **Effective behavior over textbook behavior.** darkwing replicates what the
   pinned DuckDB build actually does, including acknowledged quirks: negative
-  lookahead (`!`) is parsed but ignored by the matcher; `/` binds only the
-  immediately preceding element (upstream grammars parenthesize sequences in
-  alternatives); `*`/`+`/`?` wrap only the last element of the current list.
-  Do not "fix" these — they are pinned by tests.
+  lookahead (`!`) is parsed but ignored by the matcher. Do not "fix" these —
+  they are pinned by tests. (Since v2.0.0 the grammar parser uses ordinary
+  PEG precedence — postfix, then sequence, then `/` — so the earlier
+  "`/` binds one element" quirk is gone.)
+- **The oracle is the library parser.** The pinned CLI is run with
+  `SET active_grammar_extensions=[]` (its `shell` extension re-adds bare
+  expression statements) and `DUCKDB_AGENT_MODE=0`; see
+  `internal/duckdbsrc`.
 - **Vendored files are verbatim.** `internal/grammar/duckdb/**` is copied
   unmodified from upstream. Changing grammar behavior means advancing the pin,
   never editing the files.
@@ -109,7 +113,7 @@ DARKWING_DUCKDB=/path/to/duckdb-cli go run ./cmd/serialize-diff -corpus
 ```
 
 Regenerating the corpus (needs a DuckDB checkout at the pinned commit and
-the matching nightly CLI; see `internal/grammar/README.md` for the pin):
+the matching release CLI; see `internal/grammar/README.md` for the pin):
 
 ```
 DARKWING_DUCKDB=/path/to/duckdb-cli \

ast/ddl.go

@@ -169,6 +169,8 @@ type CreateTableInfo struct {
 	Query *SelectStatement `json:"query,omitempty"`
 	// QueryAliases are the CTAS column aliases (`CREATE TABLE t (a, b) AS`).
 	QueryAliases []string `json:"query_aliases,omitempty"`
+	// Options is the WITH (...) option list.
+	Options []RelOption `json:"options,omitempty"`
 }
 
 func (i *CreateTableInfo) Children() []Node {
@@ -179,25 +181,44 @@ func (i *CreateTableInfo) Children() []Node {
 	for _, c := range i.Constraints {
 		nodes = add(nodes, c)
 	}
-	return add(nodes, i.Query)
+	return relOptionExprs(add(nodes, i.Query), i.Options)
+}
+
+// RelOption is one WITH (name = value) entry (upstream's RelOptionList,
+// a case-insensitive option map). A bare name carries a NULL constant.
+type RelOption struct {
+	Name  string `json:"name"`
+	Value Expr   `json:"value,omitempty"`
+}
+
+func relOptionExprs(nodes []Node, opts []RelOption) []Node {
+	for _, o := range opts {
+		nodes = add(nodes, o.Value)
+	}
+	return nodes
 }
 
 // CreateViewInfo is CREATE VIEW. Port of CreateViewInfo.
 type CreateViewInfo struct {
 	CreateInfoBase
+	// Secure is CREATE SECURE VIEW (upstream's ViewSecurityType).
+	Secure    bool             `json:"secure,omitempty"`
 	Recursive bool             `json:"recursive,omitempty"`
 	Aliases   []string         `json:"aliases,omitempty"`
+	Options   []RelOption      `json:"options,omitempty"`
 	Query     *SelectStatement `json:"query"`
 }
 
-func (i *CreateViewInfo) Children() []Node { return add(nil, i.Query) }
+func (i *CreateViewInfo) Children() []Node { return add(relOptionExprs(nil, i.Options), i.Query) }
 
 // CreateSchemaInfo is CREATE SCHEMA. Port of CreateSchemaInfo.
 type CreateSchemaInfo struct {
 	CreateInfoBase
+	// Options is the WITH (...) option list.
+	Options []RelOption `json:"options,omitempty"`
 }
 
-func (i *CreateSchemaInfo) Children() []Node { return nil }
+func (i *CreateSchemaInfo) Children() []Node { return relOptionExprs(nil, i.Options) }
 
 // IndexElement is one indexed expression with its ordering.
 type IndexElement struct {
@@ -215,6 +236,7 @@ type CreateIndexInfo struct {
 	// verbatim here).
 	IndexType string         `json:"index_type,omitempty"`
 	Elements  []IndexElement `json:"elements,omitempty"`
+	Options   []RelOption    `json:"options,omitempty"`
 	Where     Expr           `json:"where,omitempty"`
 }
 
@@ -223,7 +245,7 @@ func (i *CreateIndexInfo) Children() []Node {
 	for j := range i.Elements {
 		nodes = add(nodes, i.Elements[j].Expr)
 	}
-	return add(nodes, i.Where)
+	return relOptionExprs(add(nodes, i.Where), i.Options)
 }
 
 // CreateSequenceInfo is CREATE SEQUENCE. Port of CreateSequenceInfo.
@@ -251,7 +273,8 @@ func (i *CreateSequenceInfo) Children() []Node {
 // CreateTypeInfo is CREATE TYPE. Port of CreateTypeInfo.
 type CreateTypeInfo struct {
 	CreateInfoBase
-	// Type is the aliased type (`CREATE TYPE t AS <type>`).
+	// Type is the aliased type (`CREATE TYPE t AS <type>`), or for
+	// `AS ENUM (...)[]` the list/array wrapping of the enum.
 	Type *TypeExpression `json:"type,omitempty"`
 	// EnumValues is `AS ENUM ('a', 'b')`.
 	EnumValues []string `json:"enum_values,omitempty"`
@@ -370,9 +393,10 @@ func (a *DropColumnInfo) Children() []Node { return nil }
 // AlterColumnTypeInfo is ALTER TABLE ALTER COLUMN ... [SET DATA] TYPE.
 type AlterColumnTypeInfo struct {
 	alterInfoBase
-	Column []string        `json:"column"`
-	Type   *TypeExpression `json:"type,omitempty"`
-	Using  Expr            `json:"using,omitempty"`
+	Column    []string        `json:"column"`
+	Type      *TypeExpression `json:"type,omitempty"`
+	Collation string          `json:"collation,omitempty"`
+	Using     Expr            `json:"using,omitempty"`
 }
 
 func (a *AlterColumnTypeInfo) Children() []Node {
@@ -449,8 +473,22 @@ func (a *SetDatabaseAliasInfo) Children() []Node { return nil }
 type GenericAlterInfo struct {
 	alterInfoBase
 	// Kind names the option (e.g. "SET_PARTITIONED_BY").
-	Kind        string `json:"kind"`
-	Expressions []Expr `json:"expressions,omitempty"`
+	Kind        string      `json:"kind"`
+	Expressions []Expr      `json:"expressions,omitempty"`
+	Options     []RelOption `json:"options,omitempty"`
+}
+
+func (a *GenericAlterInfo) Children() []Node {
+	return relOptionExprs(exprs(a.Expressions), a.Options)
+}
+
+// DropConstraintInfo is ALTER TABLE DROP CONSTRAINT (parsed upstream,
+// then rejected as not implemented after parsing).
+type DropConstraintInfo struct {
+	alterInfoBase
+	IfExists bool   `json:"if_exists,omitempty"`
+	Name     string `json:"name"`
+	Cascade  bool   `json:"cascade,omitempty"`
 }
 
-func (a *GenericAlterInfo) Children() []Node { return exprs(a.Expressions) }
+func (a *DropConstraintInfo) Children() []Node { return nil }

ast/expr.go

@@ -62,10 +62,14 @@ type ColumnRefExpression struct {
 
 func (e *ColumnRefExpression) Children() []Node { return nil }
 
-// ConstantExpression is a literal value. Port of ConstantExpression.
+// ConstantExpression is a literal atom. Port of ConstantExpression: since
+// DuckDB v2.0 it carries the Literal as written; Value is the literal's
+// value (Literal::ToValue), derived at construction for the
+// transformers' convenience.
 type ConstantExpression struct {
 	exprBase
-	Value Value `json:"value"`
+	Literal Literal `json:"literal"`
+	Value   Value   `json:"value"`
 }
 
 func (e *ConstantExpression) Children() []Node { return nil }
@@ -144,15 +148,19 @@ type FunctionArgument struct {
 // FunctionExpression.
 type FunctionExpression struct {
 	exprBase
-	Catalog      string             `json:"catalog,omitempty"`
-	Schema       string             `json:"schema,omitempty"`
-	FunctionName string             `json:"function_name"`
-	Arguments    []FunctionArgument `json:"arguments"`
-	Distinct     bool               `json:"distinct,omitempty"`
-	IsOperator   bool               `json:"is_operator,omitempty"`
-	ExportState  bool               `json:"export_state,omitempty"`
-	Filter       Expr               `json:"filter,omitempty"`
-	OrderBys     []OrderByNode      `json:"order_bys,omitempty"`
+	Catalog string `json:"catalog,omitempty"`
+	Schema  string `json:"schema,omitempty"`
+	// Qualification is the full qualification path when it is deeper
+	// than catalog.schema (a nested schema chain); Catalog and Schema
+	// then hold QualifiedName::Catalog()/Schema() of it.
+	Qualification []string           `json:"qualification,omitempty"`
+	FunctionName  string             `json:"function_name"`
+	Arguments     []FunctionArgument `json:"arguments"`
+	Distinct      bool               `json:"distinct,omitempty"`
+	IsOperator    bool               `json:"is_operator,omitempty"`
+	ExportState   bool               `json:"export_state,omitempty"`
+	Filter        Expr               `json:"filter,omitempty"`
+	OrderBys      []OrderByNode      `json:"order_bys,omitempty"`
 }
 
 func (e *FunctionExpression) Children() []Node {
@@ -377,9 +385,12 @@ func (e *DefaultExpression) Children() []Node { return nil }
 // alias). Port of upstream's TYPE expression class.
 type TypeExpression struct {
 	exprBase
-	Catalog  string `json:"catalog,omitempty"`
-	Schema   string `json:"schema,omitempty"`
-	TypeName string `json:"type_name"`
+	Catalog string `json:"catalog,omitempty"`
+	Schema  string `json:"schema,omitempty"`
+	// Qualification is the full qualification path when it is deeper
+	// than catalog.schema (see FunctionExpression.Qualification).
+	Qualification []string `json:"qualification,omitempty"`
+	TypeName      string   `json:"type_name"`
 	// Args holds type modifiers (constants) and element types (nested
 	// TypeExpressions; a struct field's name is the child's Alias).
 	Args []Expr `json:"children,omitempty"`
@@ -419,20 +430,23 @@ const (
 // aggregates). Port of WindowExpression.
 type WindowExpression struct {
 	exprBase
-	Type         ExpressionType     `json:"type"`
-	Catalog      string             `json:"catalog,omitempty"`
-	Schema       string             `json:"schema,omitempty"`
-	FunctionName string             `json:"function_name"`
-	Arguments    []FunctionArgument `json:"arguments"`
-	Partitions   []Expr             `json:"partitions,omitempty"`
-	Orders       []OrderByNode      `json:"orders,omitempty"`
-	FrameStart   WindowBoundary     `json:"start"`
-	FrameEnd     WindowBoundary     `json:"end"`
-	StartExpr    Expr               `json:"start_expr,omitempty"`
-	EndExpr      Expr               `json:"end_expr,omitempty"`
-	OffsetExpr   Expr               `json:"offset_expr,omitempty"`
-	DefaultExpr  Expr               `json:"default_expr,omitempty"`
-	IgnoreNulls  bool               `json:"ignore_nulls,omitempty"`
+	Type    ExpressionType `json:"type"`
+	Catalog string         `json:"catalog,omitempty"`
+	Schema  string         `json:"schema,omitempty"`
+	// Qualification is the full qualification path when it is deeper
+	// than catalog.schema (see FunctionExpression.Qualification).
+	Qualification []string           `json:"qualification,omitempty"`
+	FunctionName  string             `json:"function_name"`
+	Arguments     []FunctionArgument `json:"arguments"`
+	Partitions    []Expr             `json:"partitions,omitempty"`
+	Orders        []OrderByNode      `json:"orders,omitempty"`
+	FrameStart    WindowBoundary     `json:"start"`
+	FrameEnd      WindowBoundary     `json:"end"`
+	StartExpr     Expr               `json:"start_expr,omitempty"`
+	EndExpr       Expr               `json:"end_expr,omitempty"`
+	OffsetExpr    Expr               `json:"offset_expr,omitempty"`
+	DefaultExpr   Expr               `json:"default_expr,omitempty"`
+	IgnoreNulls   bool               `json:"ignore_nulls,omitempty"`
 	// HasIgnoreNulls records that IGNORE or RESPECT NULLS was written.
 	HasIgnoreNulls bool                `json:"has_ignore_nulls,omitempty"`
 	Distinct       bool                `json:"distinct,omitempty"`