diff --git a/datafusion/functions/src/core/struct.rs b/datafusion/functions/src/core/struct.rs index 164b9d2032f4..d689f3aac493 100644 --- a/datafusion/functions/src/core/struct.rs +++ b/datafusion/functions/src/core/struct.rs @@ -15,11 +15,13 @@ // specific language governing permissions and limitations // under the License. +use super::getfield::GetFieldFunc; use arrow::array::StructArray; use arrow::datatypes::{DataType, Field, FieldRef}; -use datafusion_common::{Result, exec_err, internal_err}; +use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; use datafusion_expr::{ - ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, + StructFieldMapping, }; use datafusion_expr::{ScalarUDFImpl, Signature, Volatility}; use datafusion_macros::user_doc; @@ -150,4 +152,16 @@ impl ScalarUDFImpl for StructFunc { fn documentation(&self) -> Option<&Documentation> { self.doc() } + + fn struct_field_mapping( + &self, + literal_args: &[Option], + ) -> Option { + Some(StructFieldMapping { + field_accessor: Arc::new(ScalarUDF::from(GetFieldFunc::new())), + fields: (0..literal_args.len()) + .map(|i| (vec![ScalarValue::Utf8(Some(format!("c{i}")))], i)) + .collect(), + }) + } } diff --git a/datafusion/physical-expr/src/utils/guarantee.rs b/datafusion/physical-expr/src/utils/guarantee.rs index 8b870c573d39..7ec3f6042c65 100644 --- a/datafusion/physical-expr/src/utils/guarantee.rs +++ b/datafusion/physical-expr/src/utils/guarantee.rs @@ -20,8 +20,9 @@ use crate::utils::split_disjunction; use crate::{PhysicalExpr, split_conjunction}; +use arrow::array::{Array, RecordBatch}; use datafusion_common::{Column, HashMap, ScalarValue}; -use datafusion_expr::Operator; +use datafusion_expr::{Operator, Volatility}; use std::collections::HashSet; use std::fmt::{self, Display, Formatter}; use std::sync::Arc; @@ -135,6 +136,16 @@ impl LiteralGuarantee { inlist.guarantee, inlist.list.iter().map(|lit| lit.value()), ) + } else if let Some(projected) = project_struct_in_list(inlist) { + projected + .into_iter() + .fold(builder, |builder, (col, values)| { + builder.aggregate_multi_conjunct( + col, + Guarantee::In, + &values, + ) + }) } else { builder } @@ -310,11 +321,11 @@ impl<'a> GuaranteeBuilder<'a> { /// * `AND (a != 1 OR a != 2 OR a != 3)`: a is not in (1, 2, or 3) /// * `AND (a NOT IN (1,2,3))`: a is not in (1, 2, or 3) #[allow(clippy::allow_attributes, clippy::mutable_key_type)] // ScalarValue has interior mutability but is intentionally used as hash key - fn aggregate_multi_conjunct( + fn aggregate_multi_conjunct<'b>( mut self, col: &'a crate::expressions::Column, guarantee: Guarantee, - new_values: impl IntoIterator, + new_values: impl IntoIterator, ) -> Self { let key = (col, guarantee); if let Some(index) = self.map.get(&key) { @@ -377,6 +388,85 @@ impl<'a> GuaranteeBuilder<'a> { } } +/// Project necessary per-column guarantees; the original predicate retains tuple correlation. +fn project_struct_in_list( + inlist: &crate::expressions::InListExpr, +) -> Option)>> { + if inlist.negated() || inlist.is_empty() { + return None; + } + let expr = inlist.expr().downcast_ref::()?; + let literal_args = expr + .args() + .iter() + .map(|arg| { + arg.downcast_ref::() + .map(|lit| lit.value().clone()) + }) + .collect::>(); + let mapping = expr.fun().struct_field_mapping(&literal_args)?; + if mapping.field_accessor.signature().volatility != Volatility::Immutable { + return None; + } + let tuples = inlist + .list() + .iter() + .map(|value| { + let literal = value.downcast_ref::()?; + let ScalarValue::Struct(array) = literal.value() else { + return None; + }; + (array.len() == 1).then_some(literal.value()) + }) + .collect::>>()?; + // Null tuples cannot make a positive IN predicate true. + let tuples = ScalarValue::iter_to_array( + tuples.into_iter().filter(|tuple| !tuple.is_null()).cloned(), + ) + .ok()?; + let batch = RecordBatch::try_from_iter([("tuple", tuples)]).ok()?; + + let mut projected = Vec::new(); + for (accessor_args, source_index) in mapping.fields { + let column = expr + .args() + .get(source_index)? + .downcast_ref::()?; + let mut args: Vec> = + vec![Arc::new(crate::expressions::Column::new("tuple", 0))]; + args.extend(accessor_args.into_iter().map(crate::expressions::lit)); + let accessor = crate::ScalarFunctionExpr::try_new( + Arc::clone(&mapping.field_accessor), + args, + batch.schema_ref(), + Arc::new(expr.config_options().clone()), + ) + .ok()?; + let array = accessor + .evaluate(&batch) + .ok()? + .into_array_of_size(batch.num_rows()) + .ok()?; + let mut values = Vec::new(); + for index in 0..array.len() { + let value = ScalarValue::try_from_array(array.as_ref(), index).ok()?; + let value = match value { + ScalarValue::Dictionary(_, value) => *value, + value => value, + }; + if value.is_null() { + values.clear(); + break; + } + values.push(value); + } + if !values.is_empty() { + projected.push((column, values)); + } + } + Some(projected) +} + /// Represents a single `col [not]in literal` expression struct ColOpLit<'a> { col: &'a crate::expressions::Column, @@ -442,7 +532,6 @@ impl<'a> ColInList<'a> { /// /// Returns None otherwise fn try_new(inlist: &'a crate::expressions::InListExpr) -> Option { - // Only support single-column inlist currently, multi-column inlist is not supported let col = inlist.expr().downcast_ref::()?; let literals = inlist @@ -842,6 +931,98 @@ mod test { ); } + #[test] + fn test_struct_inlist_guarantees() { + use crate::expressions::InListExpr; + use arrow::array::{ArrayRef, Int32Array, StringArray, StructArray}; + use arrow::buffer::NullBuffer; + + let make_expr = |strings: ArrayRef, nulls: Option, negated| { + let schema = Schema::new(vec![ + Field::new("a", strings.data_type().clone(), true), + Field::new("b", DataType::Int32, true), + ]); + let expr = logical2physical( + &datafusion_functions::core::r#struct().call(vec![col("a"), col("b")]), + &schema, + ); + let DataType::Struct(fields) = expr.data_type(&schema).unwrap() else { + unreachable!() + }; + let values = Arc::new(StructArray::new( + fields, + vec![strings, Arc::new(Int32Array::from(vec![1, 2, 3]))], + nulls, + )); + Arc::new( + InListExpr::try_new_from_array(expr, values, negated, &schema).unwrap(), + ) as Arc + }; + let strings: ArrayRef = Arc::new(StringArray::from(vec!["foo", "foo", "bar"])); + assert_eq!( + LiteralGuarantee::analyze(&make_expr(Arc::clone(&strings), None, false)), + vec![ + in_guarantee("a", ["foo", "bar"]), + in_guarantee("b", [1, 2, 3]) + ] + ); + assert!( + LiteralGuarantee::analyze(&make_expr(Arc::clone(&strings), None, true)) + .is_empty() + ); + assert_eq!( + LiteralGuarantee::analyze(&make_expr( + strings, + Some(vec![false, true, true].into()), + false + )), + vec![in_guarantee("a", ["foo", "bar"]), in_guarantee("b", [2, 3])] + ); + + let strings = StringArray::from(vec![Some("foo"), None, Some("bar")]); + let dictionary = + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)); + for data_type in [DataType::Utf8, dictionary] { + let strings = arrow::compute::cast(&strings, &data_type).unwrap(); + assert_eq!( + LiteralGuarantee::analyze(&make_expr(strings, None, false)), + vec![in_guarantee("b", [1, 2, 3])] + ); + } + let strings = arrow::compute::cast( + &StringArray::from(vec!["foo", "foo", "bar"]), + &DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + ) + .unwrap(); + assert_eq!( + LiteralGuarantee::analyze(&make_expr(strings, None, false)), + vec![ + in_guarantee("a", ["foo", "bar"]), + in_guarantee("b", [1, 2, 3]) + ] + ); + + // Named fields map to nonconsecutive arguments, in a different column order. + let tuple = RecordBatch::try_from_iter([ + ("right", Arc::new(Int32Array::from(vec![1])) as ArrayRef), + ("left", Arc::new(StringArray::from(vec!["foo"])) as ArrayRef), + ]) + .unwrap(); + let expr = datafusion_functions::core::named_struct().call(vec![ + lit("right"), + col("b"), + lit("left"), + col("a"), + ]); + test_analyze( + expr.in_list( + vec![lit(ScalarValue::Struct(Arc::new(StructArray::from(tuple))))], + false, + ), + vec![in_guarantee("a", ["foo"]), in_guarantee("b", [1])], + ); + } + #[test] fn test_inlist_conjunction() { // b IN (1, 2, 3) AND b IN (2, 3, 4) diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt index 8dd763205f6c..caaa2856519b 100644 --- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt +++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt @@ -346,24 +346,33 @@ COPY ( ) TO 'test_files/scratch/push_down_filter_parquet/join_build.parquet' STORED AS PARQUET; +# Three row groups: exact matches, Bloom-absent values within the min/max ranges, +# and crossed pairs that survive per-column pruning but fail the tuple filter. statement ok COPY ( SELECT * FROM (VALUES ('aa', 'ba', 1.0), ('ab', 'bb', 2.0), - ('ac', 'bc', 3.0), - ('ad', 'bd', 4.0) + ('aa', 'baz', 3.0), + ('ab', 'baz', 4.0), + ('aa', 'bb', 5.0), + ('ab', 'ba', 6.0) ) AS v(a, b, e) ) TO 'test_files/scratch/push_down_filter_parquet/join_probe.parquet' -STORED AS PARQUET; +STORED AS PARQUET +OPTIONS ( + 'format.max_row_group_size' '2', + 'format.bloom_filter_on_write' 'true', + 'format.bloom_filter_fpp' '0.000001' +); statement ok -CREATE EXTERNAL TABLE join_build (a VARCHAR, b VARCHAR, c DOUBLE) +CREATE EXTERNAL TABLE join_build (a VARCHAR NOT NULL, b VARCHAR NOT NULL, c DOUBLE) STORED AS PARQUET LOCATION 'test_files/scratch/push_down_filter_parquet/join_build.parquet'; statement ok -CREATE EXTERNAL TABLE join_probe (a VARCHAR, b VARCHAR, e DOUBLE) +CREATE EXTERNAL TABLE join_probe (a VARCHAR NOT NULL, b VARCHAR NOT NULL, e DOUBLE) STORED AS PARQUET LOCATION 'test_files/scratch/push_down_filter_parquet/join_probe.parquet'; @@ -389,7 +398,7 @@ FROM join_probe p INNER JOIN join_build AS build Plan with Metrics 01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)], projection=[a@3, b@4, c@2, e@5], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)] 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.88% (196/986)] -03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.37% (228/1.02 K)] +03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[a in (aa, ab), b in (ba, bb)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=3 total → 3 matched, row_groups_pruned_bloom_filter=3 total → 2 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=4, predicate_cache_records=4, scan_efficiency_ratio=23.61% (606/2.57 K)] statement ok reset datafusion.explain.analyze_categories; @@ -874,7 +883,7 @@ ON lj_build.a = lj_probe.a AND lj_build.b = lj_probe.b; Plan with Metrics 01)HashJoinExec: mode=CollectLeft, join_type=Left, on=[(a@0, a@0), (b@1, b@1)], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=2, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)] 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.88% (196/986)] -03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.37% (228/1.02 K)] +03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[a in (aa, ab), b in (ba, bb)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.37% (228/1.02 K)] # LEFT SEMI JOIN: only matching build rows are returned; probe scan still # receives the dynamic filter. @@ -890,7 +899,7 @@ WHERE EXISTS ( Plan with Metrics 01)HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(a@0, a@0), (b@1, b@1)], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=2, input_rows=4, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)] 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.88% (196/986)] -03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=15.11% (154/1.02 K)] +03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[a in (aa, ab), b in (ba, bb)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=15.11% (154/1.02 K)] statement ok reset datafusion.explain.analyze_categories;