Prepare Yardstick for DuckDB 2.0

.github/workflows/MainDistributionPipeline.yml

@@ -1,6 +1,6 @@
 #
 # Build and test the extension using the DuckDB reusable CI workflow.
-# Require the stable release and the upcoming release branch to build and test.
+# Require the legacy release and the pinned DuckDB 2.0 release to build and test.
 # DuckDB main is checked separately by the scheduled canary workflow.
 #
 name: Main Extension Distribution Pipeline
@@ -26,13 +26,16 @@ jobs:
       extra_toolchains: rust
       exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;linux_amd64_gcc4;windows_amd64_rtools;windows_amd64_mingw"
 
-  duckdb-next-build:
-    name: Build DuckDB v2.0-cyanoptera release branch extension binaries
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
+  duckdb-release-build:
+    name: Build DuckDB v2.0.0 extension binaries
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@fe955ed3a046b0324cac2c691e73d8d776a7d5cc
     with:
-      duckdb_version: v2.0-cyanoptera
-      ci_tools_version: main
+      duckdb_version: v2.0.0
+      duckdb_commit: d54cbbe3df17e8e5ce68312f483b4155836112aa
+      ci_tools_version: fe955ed3a046b0324cac2c691e73d8d776a7d5cc
       test_config: '{"test_env_variables":{"YARDSTICK_NATIVE_PEG":"1"}}'
       extension_name: yardstick
       extra_toolchains: rust
       exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;linux_amd64_gcc4;windows_amd64_rtools;windows_amd64_mingw"
+      opt_in_archs: osx_amd64
+      reduced_ci_mode: disabled

CMakeLists.txt

@@ -30,6 +30,7 @@ function(yardstick_detect_apis)
     unset(YARDSTICK_NEW_EXPR_API CACHE)
     unset(YARDSTICK_RESULT_ACCESSORS CACHE)
     unset(YARDSTICK_GRAMMAR_EXTENSION CACHE)
+    unset(YARDSTICK_PARSER_INSTANCE_API CACHE)
     check_cxx_source_compiles("
         #include <duckdb/parser/parser_extension.hpp>
         #include <type_traits>
@@ -45,6 +46,13 @@ function(yardstick_detect_apis)
         #include <duckdb/main/query_result.hpp>
         void probe(duckdb::QueryResult &result) { result.GetTypes(); }
     " YARDSTICK_RESULT_ACCESSORS)
+    check_cxx_source_compiles("
+        #include <duckdb/parser/parser.hpp>
+        void probe() {
+            duckdb::Parser parser(duckdb::ParserOptions::Builtin());
+            parser.ParseExpressionList(\"1\");
+        }
+    " YARDSTICK_PARSER_INSTANCE_API)
     check_cxx_source_compiles("
         #include <duckdb/parser/grammar_extension.hpp>
         #include <duckdb/parser/peg/compiled_grammar.hpp>

README.md

@@ -194,7 +194,7 @@ This is a staged frontend migration: forms the native grammar cannot parse retai
 
 For the combined-grammar regression suite, configure with `-DYARDSTICK_BUILD_GRAMMAR_TEST_EXTENSION=ON`, build `yardstick_test_grammar_loadable_extension`, and run the SQL tests with `YARDSTICK_NATIVE_PEG=1` and `YARDSTICK_GRAMMAR_TEST_EXTENSION` set to the fixture extension's absolute path. The fixture is excluded from normal builds.
 
-Supported build targets are DuckDB **v1.5.5** and the upcoming **v2.0-cyanoptera release branch**. The latter is a moving branch, not a released version. CI requires both targets to build and pass tests; DuckDB `main` runs separately as a scheduled, advisory canary.
+Supported build targets are DuckDB **v2.0.0** (pinned to `d54cbbe3df17e8e5ce68312f483b4155836112aa`) and legacy **v1.5.5**. The submodule and release CI use that exact 2.0 engine commit. CI requires both targets to build and pass tests; DuckDB `main` runs separately as a scheduled, advisory canary.
 
 Prerequisites:
 - CMake 3.12+

src/aggregate_decorations.cpp

@@ -1,4 +1,5 @@
 #include "aggregate_decorations.hpp"
+#include "yardstick_parser_compat.hpp"
 
 #if YARDSTICK_GRAMMAR_EXTENSION
 #include "duckdb/catalog/catalog.hpp"
@@ -32,7 +33,7 @@ namespace {
 using Names = std::unordered_set<string>;
 
 unique_ptr<ParsedExpression> ParseOne(const string &sql, const ParserOptions &options) {
-    auto expressions = Parser::ParseExpressionList(sql, options);
+    auto expressions = YardstickParseExpressionList(sql, options);
     if (expressions.size() != 1) {
         throw ParserException("Expected one expression in decorated AGGREGATE");
     }

src/aggregate_state.cpp

@@ -1,4 +1,5 @@
 #include "aggregate_state.hpp"
+#include "yardstick_parser_compat.hpp"
 
 #if YARDSTICK_GRAMMAR_EXTENSION
 #include "duckdb/common/exception.hpp"
@@ -102,7 +103,7 @@ class StateFormulaBinder : public ConstantBinder {
 };
 
 unique_ptr<ParsedExpression> ParseFormula(const string &formula) {
-	auto expressions = Parser::ParseExpressionList(formula);
+	auto expressions = YardstickParseExpressionList(formula);
 	if (expressions.size() != 1 || !expressions[0]->GetAlias().empty()) {
 		throw BinderException("yardstick_state requires exactly one scalar finalization expression");
 	}

src/frontend_peg.cpp

@@ -1,9 +1,11 @@
 #include "frontend_peg.hpp"
+#include "yardstick_parser_compat.hpp"
 
 #if YARDSTICK_GRAMMAR_EXTENSION
 #include "native_statement_traversal.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/database.hpp"
+#include "duckdb/main/settings.hpp"
 #include "duckdb/parser/expression/function_expression.hpp"
 #include "duckdb/parser/expression/columnref_expression.hpp"
 #include "duckdb/parser/expression/constant_expression.hpp"
@@ -296,7 +298,7 @@ void ReadModifier(PEGTransformer &transformer, ParseResult &result, vector<Nativ
     }
 }
 
-unique_ptr<TransformResultValue> TransformAt(PEGTransformer &transformer, ParseResult &result) {
+arena_ptr<TransformResultValue> TransformAt(PEGTransformer &transformer, ParseResult &result) {
     CaptureSpan(result, result);
     // BaseExpression inserts the left operand into this marker. Keep its
     // identity so aggregate traversal associates suffixes through AST parents.
@@ -311,10 +313,10 @@ unique_ptr<TransformResultValue> TransformAt(PEGTransformer &transformer, ParseR
         }
         active_capture->clauses.push_back(std::move(clause));
     }
-    return make_uniq<TypedTransformResult<unique_ptr<ParsedExpression>>>(std::move(expression));
+    return transformer.MakeResult<unique_ptr<ParsedExpression>>(std::move(expression));
 }
 
-unique_ptr<TransformResultValue> TransformMeasure(PEGTransformer &transformer, ParseResult &result) {
+arena_ptr<TransformResultValue> TransformMeasure(PEGTransformer &transformer, ParseResult &result) {
     auto &list = result.Cast<ListParseResult>();
     vector<unique_ptr<ParsedExpression>> children;
     children.push_back(transformer.Transform<unique_ptr<ParsedExpression>>(list.GetChild(0)));
@@ -333,14 +335,14 @@ unique_ptr<TransformResultValue> TransformMeasure(PEGTransformer &transformer, P
                                                expression.get()});
         }
     }
-    return make_uniq<TypedTransformResult<unique_ptr<ParsedExpression>>>(std::move(expression));
+    return transformer.MakeResult<unique_ptr<ParsedExpression>>(std::move(expression));
 }
 
-unique_ptr<TransformProcess> StartAt(PEGTransformer &transformer, ParseResult &result) {
-    return make_uniq<FinalizeTransformProcess>(transformer, result, TransformAt);
+arena_ptr<TransformProcess> StartAt(PEGTransformer &transformer, ParseResult &result) {
+    return transformer.MakeProcess<FinalizeTransformProcess>(transformer, result, TransformAt);
 }
 
-unique_ptr<TransformResultValue> TransformCurrentDimension(PEGTransformer &transformer, ParseResult &result,
+arena_ptr<TransformResultValue> TransformCurrentDimension(PEGTransformer &transformer, ParseResult &result,
                                                          idx_t dimension_index) {
     auto &list = result.Cast<ListParseResult>();
     vector<unique_ptr<ParsedExpression>> children;
@@ -350,25 +352,25 @@ unique_ptr<TransformResultValue> TransformCurrentDimension(PEGTransformer &trans
                                                       result.offset.GetIndex() + result.length.GetIndex()});
     }
     unique_ptr<ParsedExpression> expression = make_uniq<FunctionExpression>("current", std::move(children));
-    return make_uniq<TypedTransformResult<unique_ptr<ParsedExpression>>>(std::move(expression));
+    return transformer.MakeResult<unique_ptr<ParsedExpression>>(std::move(expression));
 }
 
-unique_ptr<TransformProcess> StartCurrentReference(PEGTransformer &transformer, ParseResult &result) {
-    return make_uniq<FinalizeTransformProcess>(transformer, result,
+arena_ptr<TransformProcess> StartCurrentReference(PEGTransformer &transformer, ParseResult &result) {
+    return transformer.MakeProcess<FinalizeTransformProcess>(transformer, result,
         [](PEGTransformer &transformer, ParseResult &result) {
             return TransformCurrentDimension(transformer, result, 1);
         });
 }
 
-unique_ptr<TransformProcess> StartCurrentCall(PEGTransformer &transformer, ParseResult &result) {
-    return make_uniq<FinalizeTransformProcess>(transformer, result,
+arena_ptr<TransformProcess> StartCurrentCall(PEGTransformer &transformer, ParseResult &result) {
+    return transformer.MakeProcess<FinalizeTransformProcess>(transformer, result,
         [](PEGTransformer &transformer, ParseResult &result) {
             return TransformCurrentDimension(transformer, result, 2);
         });
 }
 
-unique_ptr<TransformProcess> StartMeasure(PEGTransformer &transformer, ParseResult &result) {
-    return make_uniq<FinalizeTransformProcess>(transformer, result, TransformMeasure);
+arena_ptr<TransformProcess> StartMeasure(PEGTransformer &transformer, ParseResult &result) {
+    return transformer.MakeProcess<FinalizeTransformProcess>(transformer, result, TransformMeasure);
 }
 
 bool ExpandDeclarationReferences(unique_ptr<ParsedExpression> &expression,
@@ -496,6 +498,24 @@ NativeYardstickParseScope::~NativeYardstickParseScope() {
     active_parse_scope = previous;
 }
 
+ParserOptions YardstickContextParserOptions(ClientContext &context) {
+#if YARDSTICK_PARSER_INSTANCE_API
+    // GetParserOptions is private in 2.0. Snapshot the same public settings for
+    // deferred binding; NativeYardstickParseScope disables override recursion.
+    auto options = ParserOptions::Builtin();
+    options.identifier_case_mode = Settings::Get<PreserveIdentifierCaseSetting>(context);
+    options.integer_division = Settings::Get<IntegerDivisionSetting>(context);
+    options.regex_match_operator_semantics = Settings::Get<RegexMatchOperatorSemanticsSetting>(context);
+    options.max_expression_depth = Settings::Get<MaxExpressionDepthSetting>(context);
+    options.parser_override_setting = Settings::Get<AllowParserOverrideExtensionSetting>(context);
+    options.compiled_grammar = CompiledGrammar::Get(context);
+    options.context = context;
+    return options;
+#else
+    return context.GetParserOptions();
+#endif
+}
+
 const ParserOptions *CurrentNativeYardstickParserOptions() {
     return active_parse_scope ? &active_parse_scope->ParserConfig() : nullptr;
 }
@@ -821,10 +841,20 @@ YardstickCreateViewInfo *FindNativeYardstickMeasures(const char *sql_p) {
 namespace {
 vector<reference<ParseResult>> QuerySyntaxChildren(ParseResult &result) {
     switch (result.type) {
-    case ParseResultType::LIST:
-        return result.Cast<ListParseResult>().GetChildren();
-    case ParseResultType::REPEAT:
-        return result.Cast<RepeatParseResult>().GetChildren();
+    case ParseResultType::LIST: {
+        vector<reference<ParseResult>> children;
+        for (auto &child : result.Cast<ListParseResult>().GetChildren()) {
+            children.push_back(child);
+        }
+        return children;
+    }
+    case ParseResultType::REPEAT: {
+        vector<reference<ParseResult>> children;
+        for (auto &child : result.Cast<RepeatParseResult>().GetChildren()) {
+            children.push_back(child);
+        }
+        return children;
+    }
     case ParseResultType::CHOICE:
         return {result.Cast<ChoiceParseResult>().GetResult()};
     case ParseResultType::OPTIONAL:
@@ -861,11 +891,11 @@ bool CollectNativeQueryScopes(ParseResult &root, PEGTransformer &transformer,
         auto work = std::move(pending.back());
         pending.pop_back();
         auto &node = *work.node;
-        if (node.name == "YardstickMeasureAlias") {
+        if (node.Name() == "YardstickMeasureAlias") {
             return false; // Declaration registration owns this statement.
         }
         auto children = QuerySyntaxChildren(node);
-        if (node.type == ParseResultType::LIST && node.name == "SimpleSelect") {
+        if (node.type == ParseResultType::LIST && node.Name() == "SimpleSelect") {
             auto location = node.GetLocation();
             if (!location.IsValid() || location.Start() == location.End()) {
                 return false;
@@ -884,15 +914,15 @@ bool CollectNativeQueryScopes(ParseResult &root, PEGTransformer &transformer,
         if (node.type == ParseResultType::LIST && !children.empty() &&
             children[0].get().type == ParseResultType::OPTIONAL) {
             auto &optional_with = children[0].get().Cast<OptionalParseResult>();
-            if (optional_with.HasResult() && optional_with.GetResult().name == "WithClause") {
+            if (optional_with.HasResult() && optional_with.GetResult().Name() == "WithClause") {
                 auto &with = optional_with.GetResult().Cast<ListParseResult>();
                 bool recursive = with.Child<OptionalParseResult>(1).HasResult();
                 vector<ParseResult *> declarations;
                 vector<ParseResult *> search {&with.GetChild(2)};
                 while (!search.empty()) {
                     auto *candidate = search.back();
                     search.pop_back();
-                    if (candidate->type == ParseResultType::LIST && candidate->name == "WithStatement") {
+                    if (candidate->type == ParseResultType::LIST && candidate->Name() == "WithStatement") {
                         declarations.push_back(candidate);
                         continue;
                     }
@@ -922,7 +952,7 @@ bool CollectNativeQueryScopes(ParseResult &root, PEGTransformer &transformer,
                 first_child = 1;
             }
         }
-        if (node.type == ParseResultType::LIST && node.name == "SelectStatementInternal") {
+        if (node.type == ParseResultType::LIST && node.Name() == "SelectStatementInternal") {
             work.main_query = node.Cast<ListParseResult>().GetChild(1).GetLocation();
             work.modifier_end = node.GetLocation().End();
         }
@@ -954,11 +984,14 @@ YardstickQueryScopeList *FindNativeYardstickQueryScopes(const char *sql_p) {
         while (iterator.HasMoreStatements()) {
             vector<MatcherSuggestion> suggestions;
             ParseResultAllocator results;
-            ParserPackratCache packrat;
             idx_t max_token = iterator.Position();
             ArenaAllocator process_allocator(Allocator::DefaultAllocator());
+            // Match processes rewind their arena. Cached parse results must
+            // outlive those frames, as in DuckDB's own top-level matcher.
+            ArenaAllocator packrat_allocator(Allocator::DefaultAllocator());
+            ParserPackratCache packrat(packrat_allocator);
             MatchContext context(suggestions, results, process_allocator, max_token, MatchMode::BUILD_PARSE_RESULT,
-                                 options.identifier_case_mode, options.heap_based_parser, &packrat);
+                                 options.identifier_case_mode, &packrat);
             MatchState state(iterator, context);
             auto match = grammar.TopLevelStatementMatcher().MatchParseResult(state);
             if (!match.IsSuccess() || !match.HasParseResult() || state.token_iterator.Position() <= iterator.Position()) {

src/include/frontend_peg.hpp

@@ -12,6 +12,7 @@ class Parser;
 
 #if YARDSTICK_GRAMMAR_EXTENSION
 shared_ptr<ParserExtensionInfo> RegisterYardstickGrammar(DatabaseInstance &db);
+ParserOptions YardstickContextParserOptions(ClientContext &context);
 
 // Keep native parsing scoped to the override's database, active grammar, and
 // parser settings. Nested overrides restore the previous scope on return.

src/include/yardstick_compat.hpp.in

@@ -5,3 +5,4 @@
 #cmakedefine01 YARDSTICK_NEW_EXPR_API
 #cmakedefine01 YARDSTICK_RESULT_ACCESSORS
 #cmakedefine01 YARDSTICK_GRAMMAR_EXTENSION
+#cmakedefine01 YARDSTICK_PARSER_INSTANCE_API

src/include/yardstick_parser_compat.hpp

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "yardstick_compat.hpp"
+#include "duckdb/parser/parser.hpp"
+
+namespace duckdb {
+
+inline ParserOptions YardstickBuiltinParserOptions() {
+#if YARDSTICK_PARSER_INSTANCE_API
+    return ParserOptions::Builtin();
+#else
+    return ParserOptions();
+#endif
+}
+
+inline vector<unique_ptr<ParsedExpression>> YardstickParseExpressionList(
+    const string &sql, const ParserOptions &options = YardstickBuiltinParserOptions()) {
+#if YARDSTICK_PARSER_INSTANCE_API
+    Parser parser(options);
+    return parser.ParseExpressionList(sql);
+#else
+    return Parser::ParseExpressionList(sql, options);
+#endif
+}
+
+} // namespace duckdb

src/measure_windows.cpp

@@ -1,4 +1,5 @@
 #include "measure_windows.hpp"
+#include "yardstick_parser_compat.hpp"
 
 #if YARDSTICK_GRAMMAR_EXTENSION
 #include "frontend_peg.hpp"
@@ -46,7 +47,7 @@ string Quote(const string &text) {
 }
 
 unique_ptr<ParsedExpression> Expression(const string &sql, const ParserOptions &options) {
-    auto expressions = Parser::ParseExpressionList(sql, options);
+    auto expressions = YardstickParseExpressionList(sql, options);
     if (expressions.size() != 1) {
         throw ParserException("Expected one expression while rewriting a measure window");
     }

src/yardstick_extension.cpp

@@ -1,6 +1,10 @@
 #define DUCKDB_EXTENSION_MAIN
 
 #include "yardstick_extension.hpp"
+#include "yardstick_parser_compat.hpp"
+#if YARDSTICK_PARSER_INSTANCE_API
+#include "duckdb/main/settings.hpp"
+#endif
 #include "yardstick_parser_extension.hpp"
 #include "frontend_peg.hpp"
 #include "aggregate_state.hpp"
@@ -120,7 +124,7 @@ static ParserOptions YardstickParserOptions() {
         return *options;
     }
 #endif
-    return ParserOptions();
+    return YardstickBuiltinParserOptions();
 }
 
 static std::string RewritePercentileWithinGroup(const std::string &sql) {
@@ -168,7 +172,7 @@ static unique_ptr<FunctionData> YardstickQueryBind(ClientContext &context,
 #if YARDSTICK_GRAMMAR_EXTENSION
     NativeYardstickBindScope bind_scope(context);
     auto &info = input.info->Cast<DeferredMeasureFunctionInfo>();
-    NativeYardstickParseScope native_scope(info.parser_info.get(), context.GetParserOptions());
+    NativeYardstickParseScope native_scope(info.parser_info.get(), YardstickContextParserOptions(context));
 #endif
     auto data = make_uniq<YardstickQueryData>();
     data->original_sql = input.inputs[0].GetValue<string>();
@@ -204,6 +208,13 @@ static unique_ptr<FunctionData> YardstickQueryBind(ClientContext &context,
     auto &execution_config = ClientConfig::GetConfig(*con.context);
     execution_config.active_grammar_extensions = source_config.active_grammar_extensions;
     execution_config.cached_grammar = source_config.cached_grammar;
+#if YARDSTICK_PARSER_INSTANCE_API
+    // This is a binder setting, not a ParserOptions field. The private
+    // execution connection must allow the same ORDER BY syntax as its caller.
+    Settings::Set<OrderByNonIntegerLiteralSetting>(
+        *con.context, SetScope::SESSION,
+        Value::BOOLEAN(Settings::Get<OrderByNonIntegerLiteralSetting>(context)));
+#endif
 #endif
     auto query_result = con.Query(data->rewritten_sql);
 
@@ -2283,7 +2294,7 @@ static BoundStatement BindDeferredMeasureStatement(ClientContext &context, Binde
 static unique_ptr<TableRef> DeferredMeasureSelectBindReplace(ClientContext &context, TableFunctionBindInput &input) {
     auto &info = input.info->Cast<DeferredMeasureFunctionInfo>();
     NativeYardstickBindScope bind_scope(context);
-    NativeYardstickParseScope parse_scope(info.parser_info.get(), context.GetParserOptions());
+    NativeYardstickParseScope parse_scope(info.parser_info.get(), YardstickContextParserOptions(context));
     auto state = context.registered_state->GetOrCreate<DeferredMeasureCatalogState>("yardstick_deferred_catalog");
     DeferredTemporaryMeasureScope temporary_scope(*state);
     string sql = input.inputs[0].GetValue<string>();