Update vendored DuckDB sources to 0b272c23b19

CMakeLists.txt

@@ -142,6 +142,7 @@ set(DUCKDB_SRC_FILES
   src/duckdb/extension/json/json_scan.cpp
   src/duckdb/extension/json/json_serializer.cpp
   src/duckdb/extension/json/serialize_json.cpp
+  src/duckdb/extension/loader/static_extension_autoregister.cpp
   src/duckdb/extension/parquet/column_reader.cpp
   src/duckdb/extension/parquet/column_writer.cpp
   src/duckdb/extension/parquet/parquet_column_schema.cpp
@@ -151,6 +152,7 @@ set(DUCKDB_SRC_FILES
   src/duckdb/extension/parquet/parquet_file_metadata_cache.cpp
   src/duckdb/extension/parquet/parquet_float16.cpp
   src/duckdb/extension/parquet/parquet_geometry.cpp
+  src/duckdb/extension/parquet/parquet_int96.cpp
   src/duckdb/extension/parquet/parquet_metadata.cpp
   src/duckdb/extension/parquet/parquet_multi_file_info.cpp
   src/duckdb/extension/parquet/parquet_prefetch_cost_model.cpp
@@ -159,6 +161,8 @@ set(DUCKDB_SRC_FILES
   src/duckdb/extension/parquet/parquet_statistics.cpp
   src/duckdb/extension/parquet/parquet_timestamp.cpp
   src/duckdb/extension/parquet/parquet_writer.cpp
+  src/duckdb/extension/parquet/read_single_parquet_file.cpp
+  src/duckdb/extension/parquet/reader/variant/parquet_variant_shredding.cpp
   src/duckdb/extension/parquet/serialize_parquet.cpp
   src/duckdb/extension/parquet/zstd_file_system.cpp
   src/duckdb/generated_extension_loader_package_build.cpp
@@ -222,9 +226,14 @@ set(DUCKDB_SRC_FILES
   src/duckdb/src/main/extension/extension_alias.cpp
   src/duckdb/src/main/extension/extension_helper.cpp
   src/duckdb/src/main/extension/extension_install.cpp
+  src/duckdb/src/main/extension/extension_install_dynamic.cpp
   src/duckdb/src/main/extension/extension_load.cpp
+  src/duckdb/src/main/extension/extension_load_dynamic.cpp
   src/duckdb/src/main/extension/extension_loader.cpp
   src/duckdb/src/main/extension/extension_repository_manager.cpp
+  src/duckdb/src/main/extension/external_extension_provider.cpp
+  src/duckdb/src/main/extension/linked_extension_registry.cpp
+  src/duckdb/src/main/http/http_client_httplib.cpp
   src/duckdb/src/optimizer/matcher/expression_matcher.cpp
   src/duckdb/third_party/brotli/common/constants.cpp
   src/duckdb/third_party/brotli/common/context.cpp
@@ -371,7 +380,6 @@ set(DUCKDB_SRC_FILES
   src/duckdb/ub_extension_json_json_functions.cpp
   src/duckdb/ub_extension_parquet_decoder.cpp
   src/duckdb/ub_extension_parquet_reader.cpp
-  src/duckdb/ub_extension_parquet_reader_variant.cpp
   src/duckdb/ub_extension_parquet_writer.cpp
   src/duckdb/ub_extension_parquet_writer_variant.cpp
   src/duckdb/ub_src_catalog.cpp
@@ -443,7 +451,6 @@ set(DUCKDB_SRC_FILES
   src/duckdb/ub_src_main.cpp
   src/duckdb/ub_src_main_buffered_data.cpp
   src/duckdb/ub_src_main_capi_v2.cpp
-  src/duckdb/ub_src_main_chunk_scan_state.cpp
   src/duckdb/ub_src_main_http.cpp
   src/duckdb/ub_src_main_profiler.cpp
   src/duckdb/ub_src_main_relation.cpp
@@ -479,6 +486,7 @@ set(DUCKDB_SRC_FILES
   src/duckdb/ub_src_planner_expression_binder.cpp
   src/duckdb/ub_src_planner_filter.cpp
   src/duckdb/ub_src_planner_operator.cpp
+  src/duckdb/ub_src_planner_sql_export.cpp
   src/duckdb/ub_src_planner_subquery.cpp
   src/duckdb/ub_src_storage.cpp
   src/duckdb/ub_src_storage_buffer.cpp

src/duckdb/extension/core_functions/aggregate/algebraic/avg.cpp

@@ -48,7 +48,7 @@ struct KahanAvgState {
 	void Combine(const KahanAvgState &other) {
 		this->count += other.count;
 		KahanAddInternal(other.value, this->value, this->err);
-		KahanAddInternal(other.err, this->value, this->err);
+		KahanAddInternal(-other.err, this->value, this->err);
 	}
 };
 
@@ -143,6 +143,27 @@ struct DiscreteAverageOperation : public BaseSumOperation<AverageSetOperation, A
 	}
 };
 
+//! The average of TIME values is within a day - reject imported states that average outside of it
+void ValidateTimeAverageStates(const_data_ptr_t states, idx_t count, idx_t stride) {
+	for (idx_t i = 0; i < count; i++) {
+		const auto state = Load<AvgState<hugeint_t>>(states + i * stride);
+		if (state.count == 0) {
+			continue;
+		}
+		const auto max_value = hugeint_t(0, state.count) * hugeint_t(Interval::MICROS_PER_DAY);
+		if (state.value < 0 || state.value > max_value) {
+			throw InvalidInputException("Invalid avg state - the average of %s over %llu values is not a valid time",
+			                            state.value.ToString(), state.count);
+		}
+	}
+}
+
+AggregateStateLayout TimeAverageStateLayout(AggregateLayoutInput &input) {
+	auto layout = AggregateFunction::StructStateLayout<AvgState<hugeint_t>>(input);
+	layout.validate_state = ValidateTimeAverageStates;
+	return layout;
+}
+
 struct HugeintAverageOperation : public BaseSumOperation<AverageSetOperation, HugeintAdd> {
 	template <class STATE, class OP>
 	static void RepeatedCombine(const STATE &source, STATE &target, AggregateInputData &input, idx_t count) {
@@ -182,7 +203,7 @@ struct KahanAverageOperation : public BaseSumOperation<AverageSetOperation, Kaha
 		if (state.count == 0) {
 			finalize_data.ReturnNull();
 		} else {
-			target = (state.value / state.count) + (state.err / state.count);
+			target = (state.value / state.count) - (state.err / state.count);
 		}
 	}
 };
@@ -218,8 +239,9 @@ struct IntervalAverageOperation : public BaseSumOperation<AverageSetOperation, I
 			target.days += months_remainder / count;
 			days_remainder += months_remainder % count;
 
-			days_remainder *= Interval::MICROS_PER_DAY;
-			micros_remainder += days_remainder / count;
+			// the remainder in micros can exceed int64 for large counts, but divided by the count it is below two days
+			const auto shifted_days = hugeint_t(days_remainder) * hugeint_t(Interval::MICROS_PER_DAY);
+			micros_remainder += Hugeint::Cast<int64_t>(shifted_days / hugeint_t(count));
 			target.micros += micros_remainder;
 		}
 	}
@@ -299,7 +321,7 @@ AggregateFunction GetAverageAggregate(PhysicalType type) {
 unique_ptr<FunctionData> BindDecimalAvg(BindAggregateFunctionInput &input) {
 	auto &function = input.GetBoundFunction();
 	auto &arguments = input.GetArguments();
-	auto decimal_type = arguments[0]->GetReturnType();
+	auto decimal_type = BindDecimalArgument(input.GetClientContext(), arguments[0], "avg");
 	function.ReplaceImplementation(GetAverageAggregate(decimal_type.InternalType()));
 	function.SetName("avg");
 	function.GetArguments()[0] = decimal_type;
@@ -343,12 +365,14 @@ AggregateFunctionSet AvgFun::GetFunctions() {
 	auto time_avg = AggregateFunction::UnaryAggregate<AvgState<hugeint_t>, int64_t, int64_t, DiscreteAverageOperation>(
 	    LogicalType::TIME, LogicalType::TIME);
 	time_avg.GetSignature().GetParameter(0).SetName("x");
+	time_avg.SetStructStateExport(TimeAverageStateLayout);
 	avg.AddFunction(time_avg);
 
 	auto time_tz_avg =
 	    AggregateFunction::UnaryAggregate<AvgState<hugeint_t>, dtime_tz_t, dtime_tz_t, TimeTZAverageOperation>(
 	        LogicalType::TIME_TZ, LogicalType::TIME_TZ);
 	time_tz_avg.GetSignature().GetParameter(0).SetName("x");
+	time_tz_avg.SetStructStateExport(TimeAverageStateLayout);
 	avg.AddFunction(time_tz_avg);
 
 	return avg;

src/duckdb/extension/core_functions/aggregate/distributive/approx_count.cpp

@@ -1,5 +1,8 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/types/hyperloglog.hpp"
+#include "duckdb/common/vector/flat_vector.hpp"
+#include "duckdb/common/vector/vector_iterator.hpp"
+#include "duckdb/common/vector/vector_writer.hpp"
 #include "core_functions/aggregate/distributive_functions.hpp"
 
 namespace duckdb {
@@ -9,8 +12,10 @@ namespace duckdb {
 // Otmar Ertl, arXiv:1702.01284
 namespace {
 
+using ApproxCountDistinctHLL = HyperLogLogP<10>;
+
 struct ApproxDistinctCountState {
-	HyperLogLogP<10> hll;
+	ApproxCountDistinctHLL hll;
 };
 
 struct ApproxCountDistinctFunction {
@@ -54,6 +59,91 @@ void ApproxCountDistinctUpdateFunction(Vector inputs[], AggregateInputData &, id
 	}
 }
 
+//===--------------------------------------------------------------------===//
+// State Export
+//===--------------------------------------------------------------------===//
+//! Exported state: STRUCT(precision UTINYINT, registers UTINYINT[]) - the sketch itself. A HyperLogLog is a fixed
+//! array of 2^precision one-byte registers, and merging two sketches takes the per-register maximum, so an exported
+//! state combines (combine_aggr) and finalizes exactly as the in-memory one does. The static field layouts describe
+//! scalars and linked lists, not a fixed array, hence the explicit callbacks.
+LogicalType ApproxCountDistinctExportType() {
+	child_list_t<LogicalType> children;
+	children.emplace_back("precision", LogicalType::UTINYINT);
+	children.emplace_back("registers", LogicalType::LIST(LogicalType::UTINYINT));
+	return LogicalType::STRUCT(std::move(children));
+}
+
+//! The shape of the exported state: STRUCT(precision, registers UTINYINT[])
+using APPROX_COUNT_DISTINCT_EXPORT_TYPE = VectorStructType<uint8_t, VectorListType<uint8_t>>;
+
+AggregateStateLayout ApproxCountDistinctGetStateType(AggregateLayoutInput &) {
+	AggregateStateLayout layout;
+	layout.type = ApproxCountDistinctExportType();
+	layout.total_state_size = AlignValue<idx_t>(sizeof(ApproxDistinctCountState));
+	return layout;
+}
+
+void ApproxCountDistinctExportState(Vector &state_vector, AggregateFinalizeInputData &, Vector &result, idx_t count,
+                                    idx_t offset) {
+	auto states = state_vector.Values<ApproxDistinctCountState *>();
+	auto writer = FlatVector::Writer<APPROX_COUNT_DISTINCT_EXPORT_TYPE>(result, count, offset);
+	for (idx_t i = 0; i < count; i++) {
+		// an empty sketch (all registers zero) is exported as-is rather than as NULL: finalize does not import NULL
+		// states, and approx_count_distinct of no values is 0, not NULL
+		auto &hll = states[i].GetValue()->hll;
+		writer.WriteValue([&](auto &precision_writer, auto &registers_writer) {
+			precision_writer.WriteValue(UnsafeNumericCast<uint8_t>(ApproxCountDistinctHLL::PRECISION));
+			idx_t register_idx = 0;
+			for (auto &register_writer : registers_writer.WriteList(ApproxCountDistinctHLL::M)) {
+				register_writer.WriteValue(hll.GetRegister(register_idx++));
+			}
+		});
+	}
+}
+
+void ApproxCountDistinctImportState(AggregateImportInputData &input) {
+	const auto &layout = input.layout;
+	const auto &input_vec = input.input_vec;
+	const auto count = input_vec.size();
+	const auto dest_buffer = input.dest_buffer;
+	auto entries = input_vec.Values<APPROX_COUNT_DISTINCT_EXPORT_TYPE>();
+	for (idx_t i = 0; i < count; i++) {
+		auto &state = *reinterpret_cast<ApproxDistinctCountState *>(dest_buffer + i * layout.total_state_size);
+		state.hll = ApproxCountDistinctHLL();
+		const auto entry = entries[i];
+		if (!entry.IsValid()) {
+			// NULL input (e.g. to_aggregate_state(NULL, ...)) - leave the sketch empty
+			continue;
+		}
+		const auto precision_entry = entry.template GetChildValue<0>();
+		const auto register_list = entry.template GetChildValue<1>();
+		if (!precision_entry.IsValid() || !register_list.IsValid()) {
+			throw InvalidInputException("Invalid approx_count_distinct state - the state fields cannot be NULL");
+		}
+		if (precision_entry.GetValue() != ApproxCountDistinctHLL::PRECISION) {
+			throw InvalidInputException("Invalid approx_count_distinct state - expected precision %d, got %d",
+			                            ApproxCountDistinctHLL::PRECISION, precision_entry.GetValue());
+		}
+		if (register_list.GetListLength() != ApproxCountDistinctHLL::M) {
+			throw InvalidInputException("Invalid approx_count_distinct state - expected %llu registers, got %llu",
+			                            ApproxCountDistinctHLL::M, register_list.GetListLength());
+		}
+		idx_t register_idx = 0;
+		for (const auto register_entry : register_list.GetChildValues()) {
+			if (!register_entry.IsValid()) {
+				throw InvalidInputException("Invalid approx_count_distinct state - the registers cannot be NULL");
+			}
+			const auto value = register_entry.GetValue();
+			// a register holds the position of the first set bit of a hash, which is at most Q + 1
+			if (value > ApproxCountDistinctHLL::Q + 1) {
+				throw InvalidInputException("Invalid approx_count_distinct state - register value %d exceeds %d", value,
+				                            ApproxCountDistinctHLL::Q + 1);
+			}
+			state.hll.SetRegister(register_idx++, value);
+		}
+	}
+}
+
 AggregateFunction GetApproxCountDistinctFunction(const LogicalType &input_type) {
 	auto fun = AggregateFunction(
 	    {}, LogicalTypeId::BIGINT, AggregateFunction::StateSize<ApproxDistinctCountState>,
@@ -63,6 +153,8 @@ AggregateFunction GetApproxCountDistinctFunction(const LogicalType &input_type)
 	    AggregateFunction::StateFinalize<ApproxDistinctCountState, int64_t, ApproxCountDistinctFunction>, nullptr);
 	fun.GetSignature().AddParameter("any", input_type);
 	fun.SetNullHandling(FunctionNullHandling::SPECIAL_HANDLING);
+	fun.SetStateExportCallbacks(ApproxCountDistinctGetStateType, ApproxCountDistinctExportState,
+	                            ApproxCountDistinctImportState);
 	return fun;
 }
 

src/duckdb/extension/core_functions/aggregate/distributive/arg_min_max.cpp

@@ -29,19 +29,19 @@ struct ArgMinMaxValueAssign {
 template <>
 struct ArgMinMaxValueAssign<string_t> {
 	//! The size of the arena allocation for a non-inlined string value
-	uint32_t alloc_size;
+	idx_t alloc_size;
 
 	void Assign(string_t &target, string_t new_value, AggregateInputData &aggregate_input_data) {
 		if (new_value.IsInlined()) {
 			target = new_value;
 			alloc_size = 0;
 		} else {
-			auto len = UnsafeNumericCast<uint32_t>(new_value.GetSize());
+			auto len = UnsafeNumericCast<idx_t>(new_value.GetSize());
 			char *ptr;
 			if (alloc_size >= len) {
 				ptr = target.GetDataWriteable();
 			} else {
-				alloc_size = UnsafeNumericCast<uint32_t>(NextPowerOfTwo(len));
+				alloc_size = UnsafeNumericCast<idx_t>(NextPowerOfTwo(len));
 				ptr = char_ptr_cast(aggregate_input_data.allocator.Allocate(alloc_size));
 			}
 			memcpy(ptr, new_value.GetData(), len);
@@ -210,9 +210,7 @@ struct ArgMinMaxBase {
 		auto &context = input.GetClientContext();
 		auto &function = input.GetBoundFunction();
 		auto &arguments = input.GetArguments();
-		if (arguments[1]->GetReturnType().InternalType() == PhysicalType::VARCHAR) {
-			ExpressionBinder::PushCollation(context, arguments[1], arguments[1]->GetReturnType());
-		}
+		ExpressionBinder::PushCollation(context, arguments[1], arguments[1]->GetReturnType());
 		function.GetArguments()[0] = arguments[0]->GetReturnType();
 		function.GetArguments()[1] = arguments[1]->GetReturnType();
 		function.SetReturnType(arguments[0]->GetReturnType());
@@ -379,9 +377,7 @@ struct VectorArgMinMaxBase : ArgMinMaxBase<COMPARATOR> {
 		auto &context = input.GetClientContext();
 		auto &function = input.GetBoundFunction();
 		auto &arguments = input.GetArguments();
-		if (arguments[1]->GetReturnType().InternalType() == PhysicalType::VARCHAR) {
-			ExpressionBinder::PushCollation(context, arguments[1], arguments[1]->GetReturnType());
-		}
+		ExpressionBinder::PushCollation(context, arguments[1], arguments[1]->GetReturnType());
 		function.GetArguments()[0] = arguments[0]->GetReturnType();
 		function.GetArguments()[1] = arguments[1]->GetReturnType();
 		function.SetReturnType(arguments[0]->GetReturnType());
@@ -886,13 +882,15 @@ void SpecializeArgMinMaxNullNFunction(PhysicalType val_type, PhysicalType arg_ty
 
 template <ArgMinMaxNullHandling NULL_HANDLING, bool NULLS_LAST, class COMPARATOR>
 unique_ptr<FunctionData> ArgMinMaxNBind(BindAggregateFunctionInput &input) {
+	auto &context = input.GetClientContext();
 	auto &function = input.GetBoundFunction();
 	auto &arguments = input.GetArguments();
 	for (auto &arg : arguments) {
 		if (arg->GetReturnType().id() == LogicalTypeId::UNKNOWN) {
 			throw ParameterNotResolvedException();
 		}
 	}
+	ExpressionBinder::PushCollation(context, arguments[1], arguments[1]->GetReturnType());
 
 	const auto val_type = arguments[0]->GetReturnType().InternalType();
 	const auto arg_type = arguments[1]->GetReturnType().InternalType();
@@ -918,7 +916,7 @@ void AddArgMinMaxNFunction(AggregateFunctionSet &set) {
 	    .AddParameter("val", LogicalTypeId::ANY)
 	    .AddParameter("N", LogicalType::BIGINT);
 
-	return set.AddFunction(function);
+	set.AddFunction(function);
 }
 
 } // namespace

src/duckdb/extension/core_functions/aggregate/distributive/bitagg.cpp

@@ -24,7 +24,7 @@ static AggregateFunction NameArgParameter(AggregateFunction fun) {
 }
 
 template <class OP>
-AggregateFunction GetBitfieldUnaryAggregate(LogicalType type) {
+AggregateFunction GetBitfieldUnaryAggregate(const LogicalType &type) {
 	switch (type.id()) {
 	case LogicalTypeId::TINYINT:
 		return NameArgParameter(AggregateFunction::UnaryAggregate<BitState<uint8_t>, int8_t, int8_t, OP>(type, type));
@@ -111,7 +111,9 @@ struct BitwiseOperation {
 };
 
 template <class OP>
-struct NumericBitwiseOperation : public BitwiseOperation, public ClusteredStateCopy {
+struct NumericBitwiseOperation // NOLINT(bugprone-crtp-constructor-accessibility)
+    : public BitwiseOperation,
+      public ClusteredStateCopy {
 	template <class INPUT_TYPE, class STATE>
 	static void UpdateClusteredLocal(STATE &local, const INPUT_TYPE &input) {
 		if (!local.is_set) {

src/duckdb/extension/core_functions/aggregate/distributive/bitstring_agg.cpp

@@ -123,15 +123,12 @@ struct BitStringAggOperation {
 		if (min > max) {
 			throw InvalidInputException("Invalid explicit bitstring range: Minimum (%d) > maximum (%d)", min, max);
 		}
-		INPUT_TYPE result;
-		if (!TrySubtractOperator::Operation(max, min, result)) {
+		// the difference of two (at most 64-bit) values does not always fit in their own type, but it fits in a hugeint
+		const auto range = Hugeint::Convert(max) - Hugeint::Convert(min);
+		if (range >= Hugeint::Convert(NumericLimits<idx_t>::Maximum())) {
 			return NumericLimits<idx_t>::Maximum();
 		}
-		auto val = NumericCast<idx_t>(result);
-		if (val == NumericLimits<idx_t>::Maximum()) {
-			return val;
-		}
-		return val + 1;
+		return Hugeint::Cast<idx_t>(range) + 1;
 	}
 
 	template <class INPUT_TYPE, class STATE>