From acab062dd31cd6070dcb6f75107f7a9fe84adee3 Mon Sep 17 00:00:00 2001 From: Gang Wu <4684607+wgtmac@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:58:21 +0800 Subject: [PATCH] [C++][Parquet] Record Thrift parser optimization experiments --- cpp/src/parquet/CMakeLists.txt | 1 + cpp/src/parquet/THRIFT_CUSTOM_PARSER.md | 215 ++ cpp/src/parquet/column_reader.cc | 14 +- cpp/src/parquet/direct_metadata.h | 161 ++ .../experiments/thrift_parser_benchmark.cc | 230 +++ cpp/src/parquet/file_reader.cc | 5 +- cpp/src/parquet/metadata.cc | 740 ++++++- cpp/src/parquet/metadata.h | 78 +- cpp/src/parquet/metadata_test.cc | 65 + cpp/src/parquet/properties.h | 55 + cpp/src/parquet/schema.cc | 68 + cpp/src/parquet/schema_internal.h | 10 + cpp/src/parquet/thrift_compact_reader.cc | 1733 +++++++++++++++++ cpp/src/parquet/thrift_compact_reader.h | 318 +++ cpp/src/parquet/thrift_internal.h | 133 +- 15 files changed, 3752 insertions(+), 74 deletions(-) create mode 100644 cpp/src/parquet/THRIFT_CUSTOM_PARSER.md create mode 100644 cpp/src/parquet/direct_metadata.h create mode 100644 cpp/src/parquet/experiments/thrift_parser_benchmark.cc create mode 100644 cpp/src/parquet/thrift_compact_reader.cc create mode 100644 cpp/src/parquet/thrift_compact_reader.h diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index 6affbde88b48..467d901a7669 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -180,6 +180,7 @@ set(PARQUET_SRCS level_comparison.cc level_conversion.cc metadata.cc + thrift_compact_reader.cc xxhasher.cc page_index.cc "${PARQUET_THRIFT_SOURCE_DIR}/parquet_types.cpp" diff --git a/cpp/src/parquet/THRIFT_CUSTOM_PARSER.md b/cpp/src/parquet/THRIFT_CUSTOM_PARSER.md new file mode 100644 index 000000000000..c79d8def15db --- /dev/null +++ b/cpp/src/parquet/THRIFT_CUSTOM_PARSER.md @@ -0,0 +1,215 @@ + + +# Parquet Thrift parser optimization experiments + +**Exploratory snapshot — DO NOT REVIEW OR MERGE.** This branch preserves code, +measurements and unsuccessful ideas for future reference. It is not a complete +feature implementation or a production-ready replacement for the current reader. +The limitations below are part of the experiment record. + +## Motivation and reading paths + +The starting point is the Apache Arrow article +[Parquet Metadata in Rust](https://arrow.apache.org/blog/2025/10/23/rust-parquet-metadata/). +It reports improvements of up to 9x for particular pathological workloads. Those +results motivated this exploration; they are not a directly reproduced C++ target. +There is no same-machine, same-input Rust comparison in this record. + +The experiment separates three kinds of work: + +1. **Generated footer API:** libthrift parsing plus the existing `FileMetaData` + initialization. Set `PARQUET_USE_GENERATED_THRIFT=1` for this baseline. +2. **Direct footer API:** a custom compact-protocol cursor fills a borrowed internal + representation, then builds a schema tree for the existing metadata accessors. + This snapshot selects that path by default for eligible plaintext footers. +3. **Summary API:** `FastFileMetaData` discovers footer counts and `created_by`, + traversing past row groups without retaining column metadata. It cannot supply + column statistics, offsets or a `SchemaDescriptor`. + +The summary API does less work. Its timings must not be presented as an equivalent +full-metadata speedup. Reading data pages is outside these footer benchmarks. + +## Implemented directions + +| Direction | Code in this snapshot | Intended benefit / qualification | +| --- | --- | --- | +| Custom compact reader | `thrift_compact_reader.{h,cc}` | Direct field decoding and unknown-field skipping without generated protocol glue. | +| Borrowed footer fields | `Direct*` structures and retained `Buffer` ownership | Schema names, paths, statistics binary values, key/value pairs and selected footer strings refer to footer storage. The raw-pointer entry point makes a bulk owning copy. | +| Direct schema construction | `schema::UnflattenDirect` | Builds the schema tree without a generated `SchemaElement` vector. Logical annotations still use some generated structures. | +| On-demand compatibility objects | `metadata.cc` | Generated metadata remains available for serialization and fallback; some statistics and public string accessors materialize owning objects on demand. | +| Selective statistics | `ReaderProperties` and the compact reader | Can omit column/encoding/size statistics or retain them for selected leaf columns. This is statistics selection, not full column or row-group projection. | +| Encoding-statistics mask | `thrift_encoding_stats_as_mask` | Retains an encoding-presence bitset instead of the original list. Page types and counts are not preserved by this experimental policy. | +| Summary-only footer API | `FastFileMetaData` | Avoids schema-tree and column-object construction while retaining summary counts. | +| Varint common path | `CompactInput::ReadVarint` | Specialized one-byte case; the measurements below do not isolate its individual contribution. | +| Page header, page index and bloom header decoding | `ThriftDeserializer` compact-reader dispatch | These objects use custom readers but still populate generated structs. They are not measured by the footer-only table. | + +Page headers still use the existing `Peek`/retry/`Advance` loop. This is not a +resumable incremental decoder. A page-statistics skip option was started, but its +end-to-end forwarding is incomplete and must not be described as fully implemented. +Encrypted footer and encrypted-column compatibility paths also remain unfinished. + +## Reproducible measurement + +Validation and measurements below were rerun on **October 23, 2026**, using: + +- Source baseline: `ca5564e007a209e251b9d4648af4bc6a53c68c4f` plus this experiment. +- Apple M5 Pro, 48 GiB memory; Apple Clang 21.0.0; CMake Release build. +- Existing CMake configuration with tests enabled and encryption not required. +- Seven samples per mode, preceded by one warm-up per mode; mode order rotates + each round. Results are medians, with construction and destruction timed separately. +- The same retained in-memory footer buffer is passed to each mode. File I/O, + synthetic generation and initial buffer creation are excluded from the timer. +- Counts are checked against the summary API in every mode. This is a timing and + basic consistency check, not an exhaustive semantic-equivalence test. + +The standalone driver is +[`experiments/thrift_parser_benchmark.cc`](experiments/thrift_parser_benchmark.cc). +It is intentionally outside the default build and uses only synthetic values when +generating a footer. It prints aggregate sizes, counts and times, never source +paths, schema names, statistics values or key-value metadata. + +From the repository root, after configuring `cpp/build-release`: + +```sh +(cd cpp/build-release && make -j8) +c++ -std=c++20 -O3 -Icpp/src -Icpp/build-release/src \ + cpp/src/parquet/experiments/thrift_parser_benchmark.cc \ + -Lcpp/build-release/release \ + -Wl,-rpath,"$PWD/cpp/build-release/release" \ + -lparquet -larrow -o /tmp/thrift_parser_benchmark +/tmp/thrift_parser_benchmark --synthetic 100000 20 7 +``` + +This is a Unix/macOS-oriented experiment driver, not a portable benchmark target. +Libraries using nonstandard runtime dependencies may need additional linker flags. +The generator creates metadata only: 100,000 optional String columns, 20 one-row +row groups, synthetic min/max values and encoding-statistics entries. It does not +write a complete Parquet data file. Its serialized footer is **170,453,566 bytes**. +This fixture is newly reproducible here; it differs from earlier local experiments. + +Workload A has these aggregate properties only: + +| Property | Value | +| --- | ---: | +| File bytes | 536,809,024 | +| Footer bytes | 154,661,319 | +| Leaf columns | 83,925 | +| Row groups | 8 | + +Its source file, identifiers, schema and data are not included. Public reproduction +uses the synthetic generator; Workload A is a supplemental local observation. + +### Footer construction time + +| Mode | Synthetic footer (ms) | Workload A (ms) | +| --- | ---: | ---: | +| Generated footer API | 459.995 | 260.994 | +| Direct footer API, full statistics | 285.512 | 163.663 | +| Direct API, encoding-presence mask | 284.650 | 164.140 | +| Direct API, all three statistics groups omitted | 272.452 | 157.567 | +| Direct API, statistics retained only for leaf column 0 | 278.880 | 165.778 | +| Summary-only API | 187.604 | 102.205 | + +Full direct construction is **1.61x** faster on the synthetic footer and **1.59x** +on Workload A compared with generated construction. Statistics-skipping policies +change what is retained and show workload-dependent results. In particular, the +encoding mask and single-column statistics policy do not consistently improve open +time across both inputs. + +Summary construction is **2.45x** and **2.55x** faster than generated construction, +respectively, while omitting column metadata and schema-tree construction. These +figures do not demonstrate a 9x C++ improvement or explain the Rust/C++ difference. +The seven-sample results do not establish confidence intervals or an optimization +contribution breakdown. + +### Metadata destruction time + +| Mode | Synthetic footer (ms) | Workload A (ms) | +| --- | ---: | ---: | +| Generated footer API | 54.317 | 40.373 | +| Direct footer API, full statistics | 52.368 | 31.219 | +| Encoding-presence mask | 33.958 | 27.157 | +| All three statistics groups omitted | 34.416 | 21.094 | +| Statistics retained only for leaf column 0 | 35.366 | 21.930 | +| Summary-only API | 0.001 | 0.001 | + +Keeping destruction separate avoids hiding the allocation/lifetime costs of the +retained metadata. No peak-RSS or allocation-count study was completed. + +## Earlier exploration and uncompleted ideas + +Earlier exploratory runs used different fixtures and timing boundaries. For +example, an earlier 100,000-String-column, 20-row-group footer was 185,780,176 bytes; +it is not byte-for-byte identical to the new synthetic fixture. Those runs should +not be combined with the table above into a causal optimization progression. + +A **summary schema-count reuse** API was tried and removed. Recorded times were +245.44 vs 246.28 ms for an earlier wide synthetic workload and 100.57 vs 101.45 ms +for Workload A (without vs with reuse). These small differences did not justify +keeping that API. This was not an implementation of full `SchemaDescriptor` reuse. + +The following ideas were discussed but have no completed implementation and +controlled A/B measurement in this snapshot. They must not be labelled as tested +and rejected for lack of benefit: + +- Full schema reuse, full row-group/column projection and a resumable input decoder. +- PMR/arena allocation, specialized flat-schema parsing and SIMD varint decoding. +- mmap footer access and metadata caching. +- Borrowed page-index/header/bloom APIs and removal of all generated compatibility + structures. The bloom-filter payload itself is not a Thrift object. + +## Validation and remaining work + +The following were run against this snapshot on October 23, 2026: + +| Check | Result | +| --- | --- | +| `make -j8` in the configured Release build | Passed | +| `parquet-internals-test` | 354 passed | +| `parquet-reader-test` | 154 passed, 4 skipped | +| `parquet-arrow-reader-writer-test` | 847 passed, 7 skipped; 1 disabled | +| `parquet-schema-test` | 43 passed | +| `parquet-file-deserialize-test` | 33 passed | + +The added summary tests check counts, retained-buffer lifetime and rejection of +invalid/truncated buffers. Test runs use the repository's standard Arrow and +Parquet test-data environment variables. The local configuration does not validate +every encryption/compression path. + +Known gaps before any production proposal include: + +- Custom-cursor string/container limits, malformed-input handling and resource + limits need a dedicated audit; fallback is not a substitute for this audit. +- Equality in some direct metadata proxies compares only a subset of fields. + Existing API semantic equivalence has not been established. +- Borrowed storage, mutable caches, serialization and concurrent accessor use need + further lifetime and thread-safety review. +- Statistics policies and environment overrides are not uniformly applied across + every fallback/header path. An encoding-presence mask is not the complete + encoding-statistics contract. +- `FastFileMetaData` traverses omitted wire fields; it is neither constant-time + footer discovery nor a full metadata validator. `Skip(Binary)` still creates + temporary strings in this snapshot. +- Public API/ABI compatibility, Meson integration, portability, fuzzing and + representative end-to-end performance have not been completed. + +This branch intentionally retains the experimental paths and their limitations +for later investigation. Passing the listed tests is not a claim that this is a +complete or mergeable feature. diff --git a/cpp/src/parquet/column_reader.cc b/cpp/src/parquet/column_reader.cc index ee2e30fb62c5..47a430cfd1c5 100644 --- a/cpp/src/parquet/column_reader.cc +++ b/cpp/src/parquet/column_reader.cc @@ -384,8 +384,10 @@ bool SerializedPageReader::ShouldSkipPage(EncodedStatistics* data_page_statistic if (page_type == PageType::DATA_PAGE) { const format::DataPageHeader& header = current_page_header_.data_page_header; CheckNumValuesInHeader(header.num_values); - *data_page_statistics = ExtractStatsFromHeader(header, stats_min_max_field_); seen_num_values_ += header.num_values; + if (!properties_.thrift_skip_page_statistics() || data_page_filter_) { + *data_page_statistics = ExtractStatsFromHeader(header, stats_min_max_field_); + } if (data_page_filter_) { const EncodedStatistics* filter_statistics = data_page_statistics->is_set() ? data_page_statistics : nullptr; @@ -405,8 +407,10 @@ bool SerializedPageReader::ShouldSkipPage(EncodedStatistics* data_page_statistic header.repetition_levels_byte_length < 0) { throw ParquetException("Invalid page header (negative levels byte length)"); } - *data_page_statistics = ExtractStatsFromHeader(header, stats_min_max_field_); seen_num_values_ += header.num_values; + if (!properties_.thrift_skip_page_statistics() || data_page_filter_) { + *data_page_statistics = ExtractStatsFromHeader(header, stats_min_max_field_); + } if (data_page_filter_) { const EncodedStatistics* filter_statistics = data_page_statistics->is_set() ? data_page_statistics : nullptr; @@ -429,7 +433,11 @@ bool SerializedPageReader::ShouldSkipPage(EncodedStatistics* data_page_statistic } std::shared_ptr SerializedPageReader::NextPage() { - ThriftDeserializer deserializer(properties_); + // Page statistics can be skipped explicitly when no data-page filter is installed. + ReaderProperties page_properties = properties_; + page_properties.set_thrift_skip_page_statistics( + properties_.thrift_skip_page_statistics() && !data_page_filter_); + ThriftDeserializer deserializer(page_properties); // Loop here because there may be unhandled page types that we skip until // finding a page that we do know what to do with diff --git a/cpp/src/parquet/direct_metadata.h b/cpp/src/parquet/direct_metadata.h new file mode 100644 index 000000000000..1a2c6f8df579 --- /dev/null +++ b/cpp/src/parquet/direct_metadata.h @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include + +#include "generated/parquet_types.h" + +namespace parquet::internal { + +struct DirectStatistics { + std::string_view max, min, max_value, min_value; + int64_t null_count = 0, distinct_count = 0, nan_count = 0; + bool has_max = false, has_min = false, has_max_value = false, has_min_value = false; + bool has_null_count = false, has_distinct_count = false, has_nan_count = false; + bool is_max_value_exact = false, is_min_value_exact = false; + bool has_max_value_exact = false, has_min_value_exact = false; +}; + +struct DirectKeyValue { + std::string_view key; + std::string_view value; + bool has_value = false; +}; + +// Compact representation of SchemaElement used while opening a footer. The +// generated Thrift object has storage for optional annotations. This representation +// is consumed directly by UnflattenDirect; generated schema elements are only +// materialized by the serialization/compatibility path. +struct DirectSchemaElement { + format::Type::type type = static_cast(0); + int32_t type_length = 0; + format::FieldRepetitionType::type repetition_type = + format::FieldRepetitionType::REQUIRED; + std::string_view name; + int32_t num_children = 0; + format::ConvertedType::type converted_type = format::ConvertedType::UTF8; + int32_t scale = 0; + int32_t precision = 0; + int32_t field_id = 0; + format::LogicalType logical_type; + bool has_type = false; + bool has_type_length = false; + bool has_repetition_type = false; + bool has_num_children = false; + bool has_converted_type = false; + bool has_scale = false; + bool has_precision = false; + bool has_field_id = false; + bool has_logical_type = false; +}; + +struct DirectSizeStatistics { + int64_t unencoded_byte_array_data_bytes = 0; + std::vector repetition_level_histogram, definition_level_histogram; + bool has_unencoded_byte_array_data_bytes = false; + bool has_repetition_level_histogram = false, has_definition_level_histogram = false; +}; + +struct DirectColumnMetadata { + format::Type::type type = static_cast(0); + std::vector encodings; + std::vector path_in_schema; + format::CompressionCodec::type codec = format::CompressionCodec::UNCOMPRESSED; + int64_t num_values = 0; + int64_t total_uncompressed_size = 0; + int64_t total_compressed_size = 0; + std::vector key_value_metadata; + int64_t data_page_offset = 0; + int64_t index_page_offset = 0; + int64_t dictionary_page_offset = 0; + bool has_index_page_offset = false; + bool has_dictionary_page_offset = false; + DirectStatistics statistics; + bool has_statistics = false; + std::vector encoding_stats; + uint64_t encoding_stats_mask = 0; + bool has_encoding_stats_mask = false; + int64_t bloom_filter_offset = 0; + int32_t bloom_filter_length = 0; + bool has_bloom_filter_offset = false; + bool has_bloom_filter_length = false; + DirectSizeStatistics size_statistics; + bool has_size_statistics = false; + format::GeospatialStatistics geospatial_statistics; + bool has_geospatial_statistics = false; + mutable std::unique_ptr materialized; +}; + +struct DirectColumnChunk { + std::string_view file_path; + bool has_file_path = false; + int64_t file_offset = 0; + bool has_file_offset = false; + DirectColumnMetadata metadata; + bool has_metadata = false; + int64_t offset_index_offset = 0; + int32_t offset_index_length = 0; + int64_t column_index_offset = 0; + int32_t column_index_length = 0; + bool has_offset_index_offset = false; + bool has_offset_index_length = false; + bool has_column_index_offset = false; + bool has_column_index_length = false; + format::ColumnCryptoMetaData crypto_metadata; + bool has_crypto_metadata = false; + std::string_view encrypted_column_metadata; + bool has_encrypted_column_metadata = false; +}; + +struct DirectRowGroup { + std::vector columns; + int64_t total_byte_size = 0; + int64_t num_rows = 0; + std::vector sorting_columns; + int64_t file_offset = 0; + int64_t total_compressed_size = 0; + int16_t ordinal = 0; + bool has_sorting_columns = false; + bool has_file_offset = false; + bool has_total_compressed_size = false; + bool has_ordinal = false; +}; + +struct DirectFileMetaData { + int32_t version = 0; + std::vector schema; + int64_t num_rows = 0; + std::vector row_groups; + std::vector key_value_metadata; + std::string_view created_by; + std::vector column_orders; + format::EncryptionAlgorithm encryption_algorithm; + std::string_view footer_signing_key_metadata; + bool has_key_value_metadata = false; + bool has_created_by = false; + bool has_column_orders = false; + bool has_encryption_algorithm = false; + bool has_footer_signing_key_metadata = false; + // Keeps borrowed schema/string views valid for the lifetime of direct metadata. + std::shared_ptr owned_buffer; +}; + +struct FastMetadataSummary { + int32_t version = 0; + int64_t num_rows = 0; + int32_t num_schema_elements = 0; + int32_t num_columns = 0; + int32_t num_row_groups = 0; + std::string_view created_by; + bool has_created_by = false; +}; + +} // namespace parquet::internal diff --git a/cpp/src/parquet/experiments/thrift_parser_benchmark.cc b/cpp/src/parquet/experiments/thrift_parser_benchmark.cc new file mode 100644 index 000000000000..45ec09b10503 --- /dev/null +++ b/cpp/src/parquet/experiments/thrift_parser_benchmark.cc @@ -0,0 +1,230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Standalone Unix experiment driver, intentionally outside the default build. +// It creates synthetic metadata or reads only a local file's plaintext footer. +// Output contains aggregate sizes, counts and timings, never file paths, schema +// names, statistics values, key-value metadata or serialized footer contents. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/buffer.h" +#include "parquet/exception.h" +#include "parquet/metadata.h" +#include "parquet/properties.h" +#include "parquet/schema.h" +#include "parquet/statistics.h" + +namespace { +using Clock = std::chrono::steady_clock; + +std::shared_ptr SyntheticFooter(int columns, int row_groups) { + parquet::schema::NodeVector fields; + fields.reserve(columns); + for (int i = 0; i < columns; ++i) { + fields.push_back(parquet::schema::PrimitiveNode::Make( + "column_" + std::to_string(i), parquet::Repetition::OPTIONAL, + parquet::LogicalType::String(), parquet::Type::BYTE_ARRAY)); + } + parquet::SchemaDescriptor schema; + schema.Init( + parquet::schema::GroupNode::Make("schema", parquet::Repetition::REQUIRED, fields)); + auto properties = parquet::WriterProperties::Builder().build(); + auto builder = parquet::FileMetaDataBuilder::Make(&schema, properties); + parquet::EncodedStatistics statistics; + statistics.set_min("alpha").set_max("omega").set_null_count(0).set_is_signed(false); + const std::map dictionary_encodings{ + {parquet::Encoding::PLAIN, 1}}; + const std::map data_encodings{ + {parquet::Encoding::RLE_DICTIONARY, 1}}; + for (int r = 0; r < row_groups; ++r) { + auto row_group = builder->AppendRowGroup(); + for (int c = 0; c < columns; ++c) { + auto chunk = row_group->NextColumnChunk(); + chunk->SetStatistics(statistics); + const int64_t offset = 4 + (static_cast(r) * columns + c) * 128; + chunk->Finish(1, offset, 0, offset + 32, 128, 128, true, false, + dictionary_encodings, data_encodings); + } + row_group->set_num_rows(1); + row_group->Finish(static_cast(columns) * 128); + } + auto metadata = builder->Finish(); + return arrow::Buffer::FromString(metadata->SerializeToString()); +} + +std::shared_ptr ReadFooter(const std::string& path) { + std::ifstream input(path, std::ios::binary | std::ios::ate); + const auto file_size = static_cast(input.tellg()); + if (!input || file_size < 12) throw std::runtime_error("Cannot read input footer"); + std::array trailer{}; + input.seekg(file_size - 8); + input.read(reinterpret_cast(trailer.data()), trailer.size()); + if (!input || + std::string(reinterpret_cast(trailer.data() + 4), 4) != "PAR1") { + throw std::runtime_error("Expected a plaintext Parquet footer"); + } + const uint32_t length = static_cast(trailer[0]) | + (static_cast(trailer[1]) << 8) | + (static_cast(trailer[2]) << 16) | + (static_cast(trailer[3]) << 24); + if (length > file_size - 12) throw std::runtime_error("Invalid footer length"); + std::string bytes(length, '\0'); + input.seekg(file_size - 8 - length); + input.read(bytes.data(), length); + if (!input) throw std::runtime_error("Incomplete footer read"); + std::cout << "file_bytes=" << file_size << '\n'; + return arrow::Buffer::FromString(std::move(bytes)); +} + +enum Mode { + Generated, + Direct, + Mask, + SkipStatistics, + FirstColumnStatistics, + Summary, + Count +}; +const std::array kNames{"generated", "direct", + "encoding_mask", "skip_statistics", + "column_0_statistics", "summary_only"}; + +struct Sample { + double open_ms; + double destroy_ms; +}; + +template +Sample Measure(const std::shared_ptr& metadata, Clock::time_point start, + int expected_columns, int expected_row_groups, Clock::time_point* opened) { + *opened = Clock::now(); + if (metadata->num_columns() != expected_columns || + metadata->num_row_groups() != expected_row_groups) { + throw std::runtime_error("Metadata counts differ between parser modes"); + } + return {std::chrono::duration(*opened - start).count(), 0}; +} + +Sample Run(Mode mode, const std::shared_ptr& footer, + int expected_columns, int expected_row_groups) { + if (mode == Generated) { + setenv("PARQUET_USE_GENERATED_THRIFT", "1", 1); + } else { + unsetenv("PARQUET_USE_GENERATED_THRIFT"); + } + parquet::ReaderProperties properties; + if (mode == Mask) properties.set_thrift_encoding_stats_as_mask(true); + if (mode == SkipStatistics) { + properties.set_thrift_skip_column_statistics(true); + properties.set_thrift_skip_encoding_statistics(true); + properties.set_thrift_skip_size_statistics(true); + } + if (mode == FirstColumnStatistics) properties.set_thrift_statistics_columns({0}); + const auto start = Clock::now(); + Clock::time_point opened; + Sample sample; + if (mode == Summary) { + auto metadata = parquet::FastFileMetaData::Make(footer, footer->size(), properties); + sample = Measure(metadata, start, expected_columns, expected_row_groups, &opened); + metadata.reset(); + } else { + auto metadata = parquet::FileMetaData::Make(footer, footer->size(), properties); + sample = Measure(metadata, start, expected_columns, expected_row_groups, &opened); + metadata.reset(); + } + sample.destroy_ms = + std::chrono::duration(Clock::now() - opened).count(); + return sample; +} + +double Median(std::vector values) { + std::sort(values.begin(), values.end()); + const auto middle = values.size() / 2; + return values.size() % 2 ? values[middle] : (values[middle - 1] + values[middle]) / 2; +} +} // namespace + +int main(int argc, char** argv) { + try { + if (argc < 4) { + std::cerr + << "Usage: thrift_parser_benchmark --synthetic COLUMNS ROW_GROUPS [ROUNDS]\n" + " or: thrift_parser_benchmark --file LOCAL_FILE ROUNDS\n"; + return 2; + } + unsetenv("PARQUET_USE_GENERATED_THRIFT"); + unsetenv("PARQUET_SKIP_COLUMN_STATS"); + unsetenv("PARQUET_SKIP_ENCODING_STATS"); + unsetenv("PARQUET_SKIP_SIZE_STATS"); + const std::string kind(argv[1]); + int rounds = 5; + std::shared_ptr footer; + if (kind == "--synthetic") { + const int columns = std::stoi(argv[2]); + const int row_groups = std::stoi(argv[3]); + if (columns < 1 || row_groups < 1) throw std::runtime_error("Invalid dimensions"); + rounds = argc > 4 ? std::stoi(argv[4]) : 5; + footer = SyntheticFooter(columns, row_groups); + } else if (kind == "--file") { + rounds = std::stoi(argv[3]); + footer = ReadFooter(argv[2]); + } else { + throw std::runtime_error("Unknown experiment input mode"); + } + if (rounds < 1) throw std::runtime_error("Invalid round count"); + auto summary = parquet::FastFileMetaData::Make(footer, footer->size()); + const int columns = summary->num_columns(); + const int row_groups = summary->num_row_groups(); + std::cout << "footer_bytes=" << footer->size() << ",columns=" << columns + << ",row_groups=" << row_groups << ",rounds=" << rounds << '\n'; + std::array, Count> open_times, destroy_times; + for (int mode = 0; mode < Count; ++mode) { + Run(static_cast(mode), footer, columns, row_groups); // Untimed warm-up. + } + for (int round = 0; round < rounds; ++round) { + for (int i = 0; i < Count; ++i) { + const int mode = (i + round) % Count; // Rotate to reduce order bias. + const auto sample = Run(static_cast(mode), footer, columns, row_groups); + open_times[mode].push_back(sample.open_ms); + destroy_times[mode].push_back(sample.destroy_ms); + } + } + std::cout << "mode,median_open_ms,median_destroy_ms\n" + << std::fixed << std::setprecision(3); + for (int mode = 0; mode < Count; ++mode) { + std::cout << kNames[mode] << ',' << Median(open_times[mode]) << ',' + << Median(destroy_times[mode]) << '\n'; + } + unsetenv("PARQUET_USE_GENERATED_THRIFT"); + } catch (...) { + // Do not echo library errors: they may contain paths or file metadata. + std::cerr << "Experiment failed; inspect the local input and build configuration.\n"; + return 1; + } +} diff --git a/cpp/src/parquet/file_reader.cc b/cpp/src/parquet/file_reader.cc index 9214f7b4abb7..74ce11270241 100644 --- a/cpp/src/parquet/file_reader.cc +++ b/cpp/src/parquet/file_reader.cc @@ -648,8 +648,9 @@ int64_t SerializedFile::ParseUnencryptedFileMetadata( } int64_t read_metadata_len = metadata_len; // The encrypted read path falls through to here, so pass in the decryptor - file_metadata_ = FileMetaData::Make(metadata_buffer->data(), read_metadata_len, - properties_, std::move(file_decryptor)); + file_metadata_ = FileMetaData::Make(std::const_pointer_cast(metadata_buffer), + read_metadata_len, properties_, + std::move(file_decryptor)); return file_metadata_->size(); } diff --git a/cpp/src/parquet/metadata.cc b/cpp/src/parquet/metadata.cc index 61a111fc0c34..d6739e41dadc 100644 --- a/cpp/src/parquet/metadata.cc +++ b/cpp/src/parquet/metadata.cc @@ -30,6 +30,7 @@ #include #include "arrow/io/memory.h" +#include "arrow/buffer.h" #include "arrow/util/key_value_metadata.h" #include "arrow/util/logging_internal.h" #include "arrow/util/pcg_random.h" @@ -46,6 +47,37 @@ using ::arrow::util::SecureString; namespace parquet { +namespace { +std::vector MaterializeSchema( + const std::vector& direct) { + std::vector result; + result.reserve(direct.size()); + for (const auto& src : direct) { + format::SchemaElement dst; + if (src.has_type) dst.__set_type(src.type); + if (src.has_type_length) dst.__set_type_length(src.type_length); + if (src.has_repetition_type) dst.__set_repetition_type(src.repetition_type); + dst.name = src.name; + if (src.has_num_children) dst.__set_num_children(src.num_children); + if (src.has_converted_type) dst.__set_converted_type(src.converted_type); + if (src.has_scale) dst.__set_scale(src.scale); + if (src.has_precision) dst.__set_precision(src.precision); + if (src.has_field_id) dst.__set_field_id(src.field_id); + if (src.has_logical_type) dst.__set_logicalType(src.logical_type); + result.emplace_back(std::move(dst)); + } + return result; +} + +std::vector MaterializePath( + const std::vector& direct_path) { + std::vector result; + result.reserve(direct_path.size()); + for (const auto value : direct_path) result.emplace_back(value); + return result; +} +} // namespace + const ApplicationVersion& ApplicationVersion::PARQUET_251_FIXED_VERSION() { static ApplicationVersion version("parquet-mr", 1, 8, 0); return version; @@ -317,19 +349,74 @@ class ColumnChunkMetaData::ColumnChunkMetaDataImpl { InitKeyValueMetadata(); } + explicit ColumnChunkMetaDataImpl( + const internal::DirectColumnChunk* column, const ColumnDescriptor* descr, + int16_t row_group_ordinal, int16_t column_ordinal, + const ReaderProperties& properties, const ApplicationVersion* writer_version, + const std::shared_ptr& file_decryptor) + : column_(nullptr), + column_metadata_(nullptr), + direct_column_(column), + direct_metadata_(&column->metadata), + descr_(descr), + properties_(properties), + writer_version_(writer_version) { + for (const auto& encoding : direct_metadata_->encodings) { + encodings_.push_back(LoadEnumSafe(&encoding)); + } + if (!direct_metadata_->has_encoding_stats_mask) { + for (const auto& encoding_stats : direct_metadata_->encoding_stats) { + encoding_stats_.push_back({LoadEnumSafe(&encoding_stats.page_type), + LoadEnumSafe(&encoding_stats.encoding), + encoding_stats.count}); + } + } + if (direct_metadata_->has_size_statistics) { + auto& metadata = MaterializeColumnMetadata(); + size_statistics_ = + std::make_shared(FromThrift(metadata.size_statistics)); + size_statistics_->Validate(descr_); + } + possible_stats_ = nullptr; + possible_encoded_stats_ = nullptr; + possible_geo_stats_ = nullptr; + } + bool Equals(const ColumnChunkMetaDataImpl& other) const { + if (direct_metadata_ != nullptr || other.direct_metadata_ != nullptr) { + return file_offset() == other.file_offset() && num_values() == other.num_values() && + total_compressed_size() == other.total_compressed_size(); + } return *column_metadata_ == *other.column_metadata_; } // column chunk - inline int64_t file_offset() const { return column_->file_offset; } - inline const std::string& file_path() const { return column_->file_path; } + inline int64_t file_offset() const { + return direct_column_ != nullptr ? direct_column_->file_offset : column_->file_offset; + } + inline const std::string& file_path() const { + if (direct_column_ != nullptr) { + file_path_materialized_ = std::string(direct_column_->file_path); + return file_path_materialized_; + } + return column_->file_path; + } - inline Type::type type() const { return LoadEnumSafe(&column_metadata_->type); } + inline Type::type type() const { + return direct_metadata_ != nullptr ? LoadEnumSafe(&direct_metadata_->type) + : LoadEnumSafe(&column_metadata_->type); + } - inline int64_t num_values() const { return column_metadata_->num_values; } + inline int64_t num_values() const { + return direct_metadata_ != nullptr ? direct_metadata_->num_values + : column_metadata_->num_values; + } std::shared_ptr path_in_schema() { + if (direct_metadata_ != nullptr) { + return std::make_shared( + MaterializePath(direct_metadata_->path_in_schema)); + } return std::make_shared(column_metadata_->path_in_schema); } @@ -338,14 +425,21 @@ class ColumnChunkMetaData::ColumnChunkMetaDataImpl { // 2) Statistics must not be corrupted inline bool is_stats_set() const { DCHECK(writer_version_ != nullptr); - if (!column_metadata_->__isset.statistics) { + if (direct_metadata_ != nullptr ? !direct_metadata_->has_statistics + : !column_metadata_->__isset.statistics) { return false; } { const std::lock_guard guard(stats_mutex_); if (possible_encoded_stats_ == nullptr) { - possible_encoded_stats_ = std::make_shared( - FromThrift(column_metadata_->statistics, GetStatisticsMinMaxField(*descr_))); + if (direct_metadata_ != nullptr) { + auto& metadata = MaterializeColumnMetadata(); + possible_encoded_stats_ = std::make_shared( + FromThrift(metadata.statistics, GetStatisticsMinMaxField(*descr_))); + } else { + possible_encoded_stats_ = std::make_shared(FromThrift( + column_metadata_->statistics, GetStatisticsMinMaxField(*descr_))); + } } } return writer_version_->HasCorrectStatistics(type(), *possible_encoded_stats_, @@ -355,8 +449,14 @@ class ColumnChunkMetaData::ColumnChunkMetaDataImpl { inline bool is_geo_stats_set() const { const std::lock_guard guard(stats_mutex_); if (possible_geo_stats_ == nullptr && - column_metadata_->__isset.geospatial_statistics) { - possible_geo_stats_ = MakeColumnGeometryStats(*column_metadata_, descr_); + (direct_metadata_ != nullptr ? direct_metadata_->has_geospatial_statistics + : column_metadata_->__isset.geospatial_statistics)) { + if (direct_metadata_ != nullptr) { + auto& metadata = MaterializeColumnMetadata(); + possible_geo_stats_ = MakeColumnGeometryStats(metadata, descr_); + } else { + possible_geo_stats_ = MakeColumnGeometryStats(*column_metadata_, descr_); + } } return possible_geo_stats_ != nullptr && possible_geo_stats_->is_valid(); @@ -370,8 +470,13 @@ class ColumnChunkMetaData::ColumnChunkMetaDataImpl { if (is_stats_set()) { const std::lock_guard guard(stats_mutex_); if (possible_stats_ == nullptr) { - possible_stats_ = - MakeColumnStats(*column_metadata_, descr_, properties_.memory_pool()); + if (direct_metadata_ != nullptr) { + auto& metadata = MaterializeColumnMetadata(); + possible_stats_ = MakeColumnStats(metadata, descr_, properties_.memory_pool()); + } else { + possible_stats_ = + MakeColumnStats(*column_metadata_, descr_, properties_.memory_pool()); + } } return possible_stats_; } @@ -387,14 +492,26 @@ class ColumnChunkMetaData::ColumnChunkMetaDataImpl { } inline Compression::type compression() const { - return LoadEnumSafe(&column_metadata_->codec); + return direct_metadata_ != nullptr ? FromThriftUnsafe(direct_metadata_->codec) + : LoadEnumSafe(&column_metadata_->codec); } const std::vector& encodings() const { return encodings_; } const std::vector& encoding_stats() const { return encoding_stats_; } + uint64_t encoding_stats_mask() const { + return direct_metadata_ != nullptr && direct_metadata_->has_encoding_stats_mask + ? direct_metadata_->encoding_stats_mask + : 0; + } + inline std::optional bloom_filter_offset() const { + if (direct_metadata_ != nullptr) { + return direct_metadata_->has_bloom_filter_offset + ? std::optional(direct_metadata_->bloom_filter_offset) + : std::nullopt; + } if (column_metadata_->__isset.bloom_filter_offset) { return column_metadata_->bloom_filter_offset; } @@ -402,6 +519,11 @@ class ColumnChunkMetaData::ColumnChunkMetaDataImpl { } inline std::optional bloom_filter_length() const { + if (direct_metadata_ != nullptr) { + return direct_metadata_->has_bloom_filter_length + ? std::optional(direct_metadata_->bloom_filter_length) + : std::nullopt; + } if (column_metadata_->__isset.bloom_filter_length) { return column_metadata_->bloom_filter_length; } @@ -409,30 +531,48 @@ class ColumnChunkMetaData::ColumnChunkMetaDataImpl { } inline bool has_dictionary_page() const { - return column_metadata_->__isset.dictionary_page_offset; + return direct_metadata_ != nullptr ? direct_metadata_->has_dictionary_page_offset + : column_metadata_->__isset.dictionary_page_offset; } inline int64_t dictionary_page_offset() const { - return column_metadata_->dictionary_page_offset; + return direct_metadata_ != nullptr ? direct_metadata_->dictionary_page_offset + : column_metadata_->dictionary_page_offset; } - inline int64_t data_page_offset() const { return column_metadata_->data_page_offset; } + inline int64_t data_page_offset() const { + return direct_metadata_ != nullptr ? direct_metadata_->data_page_offset + : column_metadata_->data_page_offset; + } inline bool has_index_page() const { - return column_metadata_->__isset.index_page_offset; + return direct_metadata_ != nullptr ? direct_metadata_->has_index_page_offset + : column_metadata_->__isset.index_page_offset; } - inline int64_t index_page_offset() const { return column_metadata_->index_page_offset; } + inline int64_t index_page_offset() const { + return direct_metadata_ != nullptr ? direct_metadata_->index_page_offset + : column_metadata_->index_page_offset; + } inline int64_t total_compressed_size() const { - return column_metadata_->total_compressed_size; + return direct_metadata_ != nullptr ? direct_metadata_->total_compressed_size + : column_metadata_->total_compressed_size; } inline int64_t total_uncompressed_size() const { - return column_metadata_->total_uncompressed_size; + return direct_metadata_ != nullptr ? direct_metadata_->total_uncompressed_size + : column_metadata_->total_uncompressed_size; } inline std::unique_ptr crypto_metadata() const { + if (direct_column_ != nullptr) { + if (direct_column_->has_crypto_metadata) { + return ColumnCryptoMetaData::Make( + reinterpret_cast(&direct_column_->crypto_metadata)); + } + return nullptr; + } if (column_->__isset.crypto_metadata) { return ColumnCryptoMetaData::Make( reinterpret_cast(&column_->crypto_metadata)); @@ -442,6 +582,14 @@ class ColumnChunkMetaData::ColumnChunkMetaDataImpl { } std::optional GetColumnIndexLocation() const { + if (direct_column_ != nullptr) { + if (direct_column_->has_column_index_offset && + direct_column_->has_column_index_length) { + return IndexLocation{direct_column_->column_index_offset, + direct_column_->column_index_length}; + } + return std::nullopt; + } if (column_->__isset.column_index_offset && column_->__isset.column_index_length) { return IndexLocation{column_->column_index_offset, column_->column_index_length}; } @@ -449,6 +597,14 @@ class ColumnChunkMetaData::ColumnChunkMetaDataImpl { } std::optional GetOffsetIndexLocation() const { + if (direct_column_ != nullptr) { + if (direct_column_->has_offset_index_offset && + direct_column_->has_offset_index_length) { + return IndexLocation{direct_column_->offset_index_offset, + direct_column_->offset_index_length}; + } + return std::nullopt; + } if (column_->__isset.offset_index_offset && column_->__isset.offset_index_length) { return IndexLocation{column_->offset_index_offset, column_->offset_index_length}; } @@ -456,12 +612,105 @@ class ColumnChunkMetaData::ColumnChunkMetaDataImpl { } const std::shared_ptr& key_value_metadata() const { + if (direct_metadata_ != nullptr && !key_value_metadata_initialized_) { + const_cast(this)->InitKeyValueMetadata(); + } return key_value_metadata_; } private: + format::ColumnMetaData& MaterializeColumnMetadata() const { + if (direct_metadata_->materialized == nullptr) { + direct_metadata_->materialized = std::make_unique(); + } + auto& metadata = *direct_metadata_->materialized; + metadata.__set_type(direct_metadata_->type); + metadata.__set_encodings(direct_metadata_->encodings); + metadata.__set_path_in_schema(MaterializePath(direct_metadata_->path_in_schema)); + metadata.__set_codec(direct_metadata_->codec); + metadata.__set_num_values(direct_metadata_->num_values); + metadata.__set_total_uncompressed_size(direct_metadata_->total_uncompressed_size); + metadata.__set_total_compressed_size(direct_metadata_->total_compressed_size); + if (!direct_metadata_->key_value_metadata.empty()) { + std::vector key_values; + key_values.reserve(direct_metadata_->key_value_metadata.size()); + for (const auto& kv : direct_metadata_->key_value_metadata) { + format::KeyValue value; + value.__set_key(std::string(kv.key)); + if (kv.has_value) value.__set_value(std::string(kv.value)); + key_values.emplace_back(std::move(value)); + } + metadata.__set_key_value_metadata(std::move(key_values)); + } + metadata.__set_data_page_offset(direct_metadata_->data_page_offset); + if (direct_metadata_->has_index_page_offset) { + metadata.__set_index_page_offset(direct_metadata_->index_page_offset); + } + if (direct_metadata_->has_dictionary_page_offset) { + metadata.__set_dictionary_page_offset(direct_metadata_->dictionary_page_offset); + } + if (direct_metadata_->has_statistics) { + format::Statistics stats; + const auto& direct = direct_metadata_->statistics; + if (direct.has_max) stats.__set_max(std::string(direct.max)); + if (direct.has_min) stats.__set_min(std::string(direct.min)); + if (direct.has_null_count) stats.__set_null_count(direct.null_count); + if (direct.has_distinct_count) stats.__set_distinct_count(direct.distinct_count); + if (direct.has_max_value) stats.__set_max_value(std::string(direct.max_value)); + if (direct.has_min_value) stats.__set_min_value(std::string(direct.min_value)); + if (direct.has_max_value_exact) + stats.__set_is_max_value_exact(direct.is_max_value_exact); + if (direct.has_min_value_exact) + stats.__set_is_min_value_exact(direct.is_min_value_exact); + if (direct.has_nan_count) stats.__set_nan_count(direct.nan_count); + metadata.__set_statistics(stats); + } + if (!direct_metadata_->encoding_stats.empty()) + metadata.__set_encoding_stats(direct_metadata_->encoding_stats); + if (direct_metadata_->has_bloom_filter_offset) + metadata.__set_bloom_filter_offset(direct_metadata_->bloom_filter_offset); + if (direct_metadata_->has_bloom_filter_length) + metadata.__set_bloom_filter_length(direct_metadata_->bloom_filter_length); + if (direct_metadata_->has_size_statistics) { + format::SizeStatistics stats; + const auto& direct = direct_metadata_->size_statistics; + if (direct.has_unencoded_byte_array_data_bytes) + stats.__set_unencoded_byte_array_data_bytes( + direct.unencoded_byte_array_data_bytes); + if (direct.has_repetition_level_histogram) + stats.__set_repetition_level_histogram(direct.repetition_level_histogram); + if (direct.has_definition_level_histogram) + stats.__set_definition_level_histogram(direct.definition_level_histogram); + metadata.__set_size_statistics(stats); + } + if (direct_metadata_->has_geospatial_statistics) + metadata.__set_geospatial_statistics(direct_metadata_->geospatial_statistics); + return metadata; + } + void InitKeyValueMetadata() { - key_value_metadata_ = FromThriftKeyValueMetadata(*column_metadata_); + if (direct_metadata_ != nullptr) { + key_value_metadata_initialized_ = true; + if (direct_metadata_->key_value_metadata.empty()) { + key_value_metadata_ = nullptr; + } else { + std::vector keys, values; + keys.reserve(direct_metadata_->key_value_metadata.size()); + values.reserve(direct_metadata_->key_value_metadata.size()); + for (const auto& kv : direct_metadata_->key_value_metadata) { + keys.emplace_back(kv.key); + if (kv.has_value) { + values.emplace_back(kv.value); + } else { + values.emplace_back(); + } + } + key_value_metadata_ = + std::make_shared(std::move(keys), std::move(values)); + } + } else { + key_value_metadata_ = FromThriftKeyValueMetadata(*column_metadata_); + } } mutable std::mutex stats_mutex_; @@ -472,12 +721,16 @@ class ColumnChunkMetaData::ColumnChunkMetaDataImpl { std::vector encoding_stats_; const format::ColumnChunk* column_; const format::ColumnMetaData* column_metadata_; + const internal::DirectColumnChunk* direct_column_ = nullptr; + const internal::DirectColumnMetadata* direct_metadata_ = nullptr; format::ColumnMetaData decrypted_metadata_; const ColumnDescriptor* descr_; const ReaderProperties properties_; const ApplicationVersion* writer_version_; std::shared_ptr key_value_metadata_; std::shared_ptr size_statistics_; + mutable std::string file_path_materialized_; + mutable bool key_value_metadata_initialized_ = false; }; std::unique_ptr ColumnChunkMetaData::Make( @@ -490,6 +743,16 @@ std::unique_ptr ColumnChunkMetaData::Make( properties, writer_version, std::move(file_decryptor))); } +std::unique_ptr ColumnChunkMetaData::MakeDirect( + const internal::DirectColumnChunk* metadata, const ColumnDescriptor* descr, + const ReaderProperties& properties, const ApplicationVersion* writer_version, + int16_t row_group_ordinal, int16_t column_ordinal, + std::shared_ptr file_decryptor) { + return std::unique_ptr( + new ColumnChunkMetaData(metadata, descr, row_group_ordinal, column_ordinal, + properties, writer_version, std::move(file_decryptor))); +} + ColumnChunkMetaData::ColumnChunkMetaData( const void* metadata, const ColumnDescriptor* descr, int16_t row_group_ordinal, int16_t column_ordinal, const ReaderProperties& properties, @@ -500,6 +763,15 @@ ColumnChunkMetaData::ColumnChunkMetaData( row_group_ordinal, column_ordinal, properties, writer_version, std::move(file_decryptor))} {} +ColumnChunkMetaData::ColumnChunkMetaData( + const internal::DirectColumnChunk* metadata, const ColumnDescriptor* descr, + int16_t row_group_ordinal, int16_t column_ordinal, const ReaderProperties& properties, + const ApplicationVersion* writer_version, + std::shared_ptr file_decryptor) + : impl_{new ColumnChunkMetaDataImpl(metadata, descr, row_group_ordinal, + column_ordinal, properties, writer_version, + std::move(file_decryptor))} {} + ColumnChunkMetaData::~ColumnChunkMetaData() = default; // column chunk @@ -577,6 +849,10 @@ const std::vector& ColumnChunkMetaData::encoding_stats() cons return impl_->encoding_stats(); } +uint64_t ColumnChunkMetaData::encoding_stats_mask() const { + return impl_->encoding_stats_mask(); +} + int64_t ColumnChunkMetaData::total_uncompressed_size() const { return impl_->total_uncompressed_size(); } @@ -626,28 +902,70 @@ class RowGroupMetaData::RowGroupMetaDataImpl { } } + explicit RowGroupMetaDataImpl(const internal::DirectRowGroup* row_group, + const SchemaDescriptor* schema, + const ReaderProperties& properties, + const ApplicationVersion* writer_version, + std::shared_ptr file_decryptor) + : row_group_(nullptr), + direct_row_group_(row_group), + schema_(schema), + properties_(properties), + writer_version_(writer_version), + file_decryptor_(std::move(file_decryptor)) { + if (ARROW_PREDICT_FALSE(row_group->columns.size() > + static_cast(std::numeric_limits::max()))) { + throw ParquetException("Row group had too many columns: ", + row_group->columns.size()); + } + } + bool Equals(const RowGroupMetaDataImpl& other) const { + if (direct_row_group_ != nullptr || other.direct_row_group_ != nullptr) { + return num_rows() == other.num_rows() && num_columns() == other.num_columns(); + } return *row_group_ == *other.row_group_; } - inline int num_columns() const { return static_cast(row_group_->columns.size()); } + inline int num_columns() const { + return static_cast(direct_row_group_ != nullptr + ? direct_row_group_->columns.size() + : row_group_->columns.size()); + } - inline int64_t num_rows() const { return row_group_->num_rows; } + inline int64_t num_rows() const { + return direct_row_group_ != nullptr ? direct_row_group_->num_rows + : row_group_->num_rows; + } - inline int64_t total_byte_size() const { return row_group_->total_byte_size; } + inline int64_t total_byte_size() const { + return direct_row_group_ != nullptr ? direct_row_group_->total_byte_size + : row_group_->total_byte_size; + } inline int64_t total_compressed_size() const { - return row_group_->total_compressed_size; + return direct_row_group_ != nullptr ? direct_row_group_->total_compressed_size + : row_group_->total_compressed_size; } - inline int64_t file_offset() const { return row_group_->file_offset; } + inline int64_t file_offset() const { + return direct_row_group_ != nullptr ? direct_row_group_->file_offset + : row_group_->file_offset; + } inline const SchemaDescriptor* schema() const { return schema_; } std::unique_ptr ColumnChunk(int i) { if (i >= 0 && i < num_columns()) { int16_t row_group_ordinal = - row_group_->__isset.ordinal ? row_group_->ordinal : static_cast(-1); + direct_row_group_ != nullptr + ? (direct_row_group_->has_ordinal ? direct_row_group_->ordinal : -1) + : (row_group_->__isset.ordinal ? row_group_->ordinal : -1); + if (direct_row_group_ != nullptr) { + return ColumnChunkMetaData::MakeDirect( + &direct_row_group_->columns[i], schema_->Column(i), properties_, + writer_version_, row_group_ordinal, i, file_decryptor_); + } return ColumnChunkMetaData::Make(&row_group_->columns[i], schema_->Column(i), properties_, writer_version_, row_group_ordinal, i, file_decryptor_); @@ -658,6 +976,14 @@ class RowGroupMetaData::RowGroupMetaDataImpl { std::vector sorting_columns() const { std::vector sorting_columns; + if (direct_row_group_ != nullptr) { + if (!direct_row_group_->has_sorting_columns) return {}; + std::vector result; + result.reserve(direct_row_group_->sorting_columns.size()); + for (const auto& sorting : direct_row_group_->sorting_columns) + result.push_back(FromThrift(sorting)); + return result; + } if (!row_group_->__isset.sorting_columns) { return sorting_columns; } @@ -670,6 +996,7 @@ class RowGroupMetaData::RowGroupMetaDataImpl { private: const format::RowGroup* row_group_; + const internal::DirectRowGroup* direct_row_group_ = nullptr; const SchemaDescriptor* schema_; const ReaderProperties properties_; const ApplicationVersion* writer_version_; @@ -684,6 +1011,14 @@ std::unique_ptr RowGroupMetaData::Make( metadata, schema, properties, writer_version, std::move(file_decryptor))); } +std::unique_ptr RowGroupMetaData::MakeDirect( + const internal::DirectRowGroup* metadata, const SchemaDescriptor* schema, + const ReaderProperties& properties, const ApplicationVersion* writer_version, + std::shared_ptr file_decryptor) { + return std::unique_ptr(new RowGroupMetaData( + metadata, schema, properties, writer_version, std::move(file_decryptor))); +} + RowGroupMetaData::RowGroupMetaData(const void* metadata, const SchemaDescriptor* schema, const ReaderProperties& properties, const ApplicationVersion* writer_version, @@ -692,6 +1027,14 @@ RowGroupMetaData::RowGroupMetaData(const void* metadata, const SchemaDescriptor* schema, properties, writer_version, std::move(file_decryptor))} {} +RowGroupMetaData::RowGroupMetaData(const internal::DirectRowGroup* metadata, + const SchemaDescriptor* schema, + const ReaderProperties& properties, + const ApplicationVersion* writer_version, + std::shared_ptr file_decryptor) + : impl_{new RowGroupMetaDataImpl(metadata, schema, properties, writer_version, + std::move(file_decryptor))} {} + RowGroupMetaData::~RowGroupMetaData() = default; bool RowGroupMetaData::Equals(const RowGroupMetaData& other) const { @@ -773,6 +1116,52 @@ static void Scrub(format::FileMetaData* md) { Scrub(&md->footer_signing_key_metadata); } +class FastFileMetaData::Impl { + public: + Impl(std::shared_ptr buffer, int64_t length, + const ReaderProperties& properties) + : buffer_(std::move(buffer)), properties_(properties) { + if (buffer_ == nullptr || length < 0 || length > buffer_->size()) { + throw ParquetException("Invalid fast metadata buffer"); + } + internal::CompactInput input(buffer_->data(), length); + internal::ReadFastFileMetaData(input, &metadata_); + size_ = input.position(); + } + + int64_t size() const { return size_; } + int64_t num_rows() const { return metadata_.num_rows; } + int num_row_groups() const { return metadata_.num_row_groups; } + int num_schema_elements() const { return metadata_.num_schema_elements; } + int num_columns() const { return metadata_.num_columns; } + std::string_view created_by() const { return metadata_.created_by; } + + private: + std::shared_ptr buffer_; + ReaderProperties properties_; + internal::FastMetadataSummary metadata_; + int64_t size_ = 0; +}; + +FastFileMetaData::FastFileMetaData( + std::shared_ptr serialized_metadata, int64_t metadata_len, + const ReaderProperties& properties) + : impl_(new Impl(std::move(serialized_metadata), metadata_len, properties)) {} + +std::shared_ptr FastFileMetaData::Make( + std::shared_ptr serialized_metadata, int64_t metadata_len, + const ReaderProperties& properties) { + return std::shared_ptr( + new FastFileMetaData(std::move(serialized_metadata), metadata_len, properties)); +} + +int64_t FastFileMetaData::size() const { return impl_->size(); } +int64_t FastFileMetaData::num_rows() const { return impl_->num_rows(); } +int FastFileMetaData::num_row_groups() const { return impl_->num_row_groups(); } +int FastFileMetaData::num_schema_elements() const { return impl_->num_schema_elements(); } +int FastFileMetaData::num_columns() const { return impl_->num_columns(); } +std::string_view FastFileMetaData::created_by() const { return impl_->created_by(); } + // file metadata class FileMetaData::FileMetaDataImpl { public: @@ -783,19 +1172,58 @@ class FileMetaData::FileMetaDataImpl { FileMetaDataImpl(const void* metadata, int64_t metadata_len, ReaderProperties properties, - std::shared_ptr file_decryptor = nullptr) - : properties_(std::move(properties)), file_decryptor_(std::move(file_decryptor)) { - metadata_ = std::make_unique(); - + std::shared_ptr file_decryptor = nullptr, + std::shared_ptr metadata_buffer = nullptr) + : properties_(std::move(properties)), + file_decryptor_(std::move(file_decryptor)), + metadata_buffer_(std::move(metadata_buffer)) { auto footer_decryptor = file_decryptor_ != nullptr ? file_decryptor_->GetFooterDecryptor() : nullptr; ThriftDeserializer deserializer(properties_); - metadata_len_ = deserializer.DeserializeMessage( - reinterpret_cast(metadata), metadata_len, metadata_.get(), - footer_decryptor.get()); + const char* generated = std::getenv("PARQUET_USE_GENERATED_THRIFT"); + if ((generated == nullptr || generated[0] != '1') && footer_decryptor == nullptr) { + try { + direct_metadata_ = std::make_unique(); + const uint8_t* parse_data = reinterpret_cast(metadata); + if (footer_decryptor == nullptr && metadata_buffer_ == nullptr) { + // Keep one bulk copy of the footer so direct string views remain valid; + // this replaces thousands of independent string allocations. + direct_metadata_->owned_buffer = std::make_shared( + reinterpret_cast(metadata), static_cast(metadata_len)); + parse_data = reinterpret_cast(direct_metadata_->owned_buffer->data()); + } else if (footer_decryptor == nullptr) { + parse_data = metadata_buffer_->data(); + } + metadata_len_ = deserializer.DeserializeMessage( + parse_data, metadata_len, direct_metadata_.get(), footer_decryptor.get()); + if (direct_metadata_ != nullptr) { + for (const auto& row_group : direct_metadata_->row_groups) { + for (const auto& column : row_group.columns) { + if (column.has_encrypted_column_metadata) { + direct_metadata_.reset(); + break; + } + } + if (direct_metadata_ == nullptr) break; + } + } + } catch (const ParquetException&) { + direct_metadata_.reset(); + } + } + if (direct_metadata_ == nullptr) { + metadata_ = std::make_unique(); + metadata_len_ = deserializer.DeserializeMessage( + reinterpret_cast(metadata), metadata_len, metadata_.get(), + footer_decryptor.get()); + } - if (metadata_->__isset.created_by) { + if (direct_metadata_ != nullptr) { + writer_version_ = direct_metadata_->has_created_by + ? ApplicationVersion(std::string(direct_metadata_->created_by)) + : ApplicationVersion("unknown 0.0.0"); + } else if (metadata_->__isset.created_by) { writer_version_ = ApplicationVersion(metadata_->created_by); } else { writer_version_ = ApplicationVersion("unknown 0.0.0"); @@ -803,7 +1231,7 @@ class FileMetaData::FileMetaDataImpl { InitSchema(); InitColumnOrders(); - InitKeyValueMetadata(); + if (direct_metadata_ == nullptr) InitKeyValueMetadata(); } bool VerifySignature(const void* signature) { @@ -811,6 +1239,7 @@ class FileMetaData::FileMetaDataImpl { if (file_decryptor_ == nullptr) { throw ParquetException("Decryption not set properly. cannot verify signature"); } + EnsureFormatMetadata(); // serialize the footer ThriftSerializer serializer; auto serialized_data_span = serializer.SerializeToBuffer(metadata_.get()); @@ -840,23 +1269,43 @@ class FileMetaData::FileMetaDataImpl { inline int64_t size() const { return metadata_len_; } inline int num_columns() const { return schema_.num_columns(); } - inline int64_t num_rows() const { return metadata_->num_rows; } + inline int64_t num_rows() const { + return direct_metadata_ != nullptr ? direct_metadata_->num_rows : metadata_->num_rows; + } inline int num_row_groups() const { - return static_cast(metadata_->row_groups.size()); + return static_cast(direct_metadata_ != nullptr + ? direct_metadata_->row_groups.size() + : metadata_->row_groups.size()); + } + inline int32_t version() const { + return direct_metadata_ != nullptr ? direct_metadata_->version : metadata_->version; + } + inline const std::string& created_by() const { + if (direct_metadata_ != nullptr) { + created_by_materialized_ = std::string(direct_metadata_->created_by); + return created_by_materialized_; + } + return metadata_->created_by; } - inline int32_t version() const { return metadata_->version; } - inline const std::string& created_by() const { return metadata_->created_by; } inline int num_schema_elements() const { - return static_cast(metadata_->schema.size()); + return static_cast(direct_metadata_ != nullptr ? direct_metadata_->schema.size() + : metadata_->schema.size()); } inline bool is_encryption_algorithm_set() const { - return metadata_->__isset.encryption_algorithm; + return direct_metadata_ != nullptr ? direct_metadata_->has_encryption_algorithm + : metadata_->__isset.encryption_algorithm; } inline EncryptionAlgorithm encryption_algorithm() { - return FromThrift(metadata_->encryption_algorithm); + return FromThrift(direct_metadata_ != nullptr ? direct_metadata_->encryption_algorithm + : metadata_->encryption_algorithm); } inline const std::string& footer_signing_key_metadata() { + if (direct_metadata_ != nullptr) { + footer_signing_key_materialized_ = + std::string(direct_metadata_->footer_signing_key_metadata); + return footer_signing_key_materialized_; + } return metadata_->footer_signing_key_metadata; } @@ -864,6 +1313,7 @@ class FileMetaData::FileMetaDataImpl { void WriteTo(::arrow::io::OutputStream* dst, const std::shared_ptr& encryptor) const { + const_cast(this)->EnsureFormatMetadata(); ThriftSerializer serializer; // Only in encrypted files with plaintext footers the // encryption_algorithm is set in footer @@ -897,21 +1347,38 @@ class FileMetaData::FileMetaDataImpl { << " row groups, requested metadata for row group: " << i; throw ParquetException(ss.str()); } + if (direct_metadata_ != nullptr) { + return RowGroupMetaData::MakeDirect(&direct_metadata_->row_groups[i], &schema_, + properties_, &writer_version_, file_decryptor_); + } return RowGroupMetaData::Make(&metadata_->row_groups[i], &schema_, properties_, &writer_version_, file_decryptor_); } bool Equals(const FileMetaDataImpl& other) const { + if (direct_metadata_ != nullptr || other.direct_metadata_ != nullptr) { + const_cast(this)->EnsureFormatMetadata(); + const_cast(other).EnsureFormatMetadata(); + } return *metadata_ == *other.metadata_; } const SchemaDescriptor* schema() const { return &schema_; } const std::shared_ptr& key_value_metadata() const { + if (direct_metadata_ != nullptr && !key_value_metadata_initialized_) { + const_cast(this)->InitKeyValueMetadata(); + } return key_value_metadata_; } void set_file_path(const std::string& path) { + if (direct_metadata_ != nullptr) { + for (auto& row_group : direct_metadata_->row_groups) { + for (auto& chunk : row_group.columns) chunk.file_path = path; + } + if (metadata_ == nullptr) return; + } for (format::RowGroup& row_group : metadata_->row_groups) { for (format::ColumnChunk& chunk : row_group.columns) { chunk.__set_file_path(path); @@ -920,6 +1387,7 @@ class FileMetaData::FileMetaDataImpl { } format::RowGroup& row_group(int i) { + EnsureFormatMetadata(); if (!(i >= 0 && i < num_row_groups())) { std::stringstream ss; ss << "The file only has " << num_row_groups() @@ -930,13 +1398,16 @@ class FileMetaData::FileMetaDataImpl { } void AppendRowGroups(FileMetaDataImpl* other) { + EnsureFormatMetadata(); + other->EnsureFormatMetadata(); std::ostringstream diff_output; if (!schema()->Equals(*other->schema(), &diff_output)) { auto msg = "AppendRowGroups requires equal schemas.\n" + diff_output.str(); throw ParquetException(msg); } - // ARROW-13654: `other` may point to self, be careful not to enter an infinite loop + // ARROW-13654: `other` may point to self, be careful not to enter an infinite + // loop const int n = other->num_row_groups(); // ARROW-16613: do not use reserve() as that may suppress overallocation // and incur O(n²) behavior on repeated calls to AppendRowGroups(). @@ -951,6 +1422,7 @@ class FileMetaData::FileMetaDataImpl { } std::shared_ptr Subset(const std::vector& row_groups) { + EnsureFormatMetadata(); for (int i : row_groups) { if (i < num_row_groups()) continue; @@ -991,6 +1463,7 @@ class FileMetaData::FileMetaDataImpl { } std::string SerializeUnencrypted(bool scrub, bool debug) const { + const_cast(this)->EnsureFormatMetadata(); auto md = *metadata_; if (scrub) Scrub(&md); if (debug) { @@ -1015,16 +1488,150 @@ class FileMetaData::FileMetaDataImpl { friend FileMetaDataBuilder; int64_t metadata_len_ = 0; std::unique_ptr metadata_; + std::unique_ptr direct_metadata_; SchemaDescriptor schema_; ApplicationVersion writer_version_; std::shared_ptr key_value_metadata_; const ReaderProperties properties_; std::shared_ptr file_decryptor_; + std::shared_ptr metadata_buffer_; + mutable std::string created_by_materialized_; + mutable std::string footer_signing_key_materialized_; + mutable bool key_value_metadata_initialized_ = false; + + void EnsureFormatMetadata() { + if (metadata_ != nullptr || direct_metadata_ == nullptr) return; + metadata_ = std::make_unique(); + metadata_->version = direct_metadata_->version; + metadata_->schema = MaterializeSchema(direct_metadata_->schema); + metadata_->num_rows = direct_metadata_->num_rows; + metadata_->key_value_metadata.reserve(direct_metadata_->key_value_metadata.size()); + for (const auto& kv : direct_metadata_->key_value_metadata) { + format::KeyValue value; + value.__set_key(std::string(kv.key)); + if (kv.has_value) value.__set_value(std::string(kv.value)); + metadata_->key_value_metadata.emplace_back(std::move(value)); + } + metadata_->created_by = std::string(direct_metadata_->created_by); + metadata_->column_orders = direct_metadata_->column_orders; + metadata_->encryption_algorithm = direct_metadata_->encryption_algorithm; + metadata_->footer_signing_key_metadata = + std::string(direct_metadata_->footer_signing_key_metadata); + metadata_->__isset.key_value_metadata = direct_metadata_->has_key_value_metadata; + metadata_->__isset.created_by = direct_metadata_->has_created_by; + metadata_->__isset.column_orders = direct_metadata_->has_column_orders; + metadata_->__isset.encryption_algorithm = direct_metadata_->has_encryption_algorithm; + metadata_->__isset.footer_signing_key_metadata = + direct_metadata_->has_footer_signing_key_metadata; + metadata_->row_groups.reserve(direct_metadata_->row_groups.size()); + for (const auto& direct_rg : direct_metadata_->row_groups) { + auto& rg = metadata_->row_groups.emplace_back(); + rg.__set_total_byte_size(direct_rg.total_byte_size); + rg.__set_num_rows(direct_rg.num_rows); + if (direct_rg.has_sorting_columns) + rg.__set_sorting_columns(direct_rg.sorting_columns); + if (direct_rg.has_file_offset) rg.__set_file_offset(direct_rg.file_offset); + if (direct_rg.has_total_compressed_size) + rg.__set_total_compressed_size(direct_rg.total_compressed_size); + if (direct_rg.has_ordinal) rg.__set_ordinal(direct_rg.ordinal); + rg.columns.reserve(direct_rg.columns.size()); + for (const auto& direct_col : direct_rg.columns) { + auto& col = rg.columns.emplace_back(); + if (direct_col.has_file_path) + col.__set_file_path(std::string(direct_col.file_path)); + if (direct_col.has_file_offset) col.__set_file_offset(direct_col.file_offset); + const auto& dm = direct_col.metadata; + auto& m = col.meta_data; + m.__set_type(dm.type); + m.__set_encodings(dm.encodings); + m.__set_path_in_schema(MaterializePath(dm.path_in_schema)); + m.__set_codec(dm.codec); + m.__set_num_values(dm.num_values); + m.__set_total_uncompressed_size(dm.total_uncompressed_size); + m.__set_total_compressed_size(dm.total_compressed_size); + if (!dm.key_value_metadata.empty()) { + std::vector key_values; + key_values.reserve(dm.key_value_metadata.size()); + for (const auto& kv : dm.key_value_metadata) { + format::KeyValue value; + value.__set_key(std::string(kv.key)); + if (kv.has_value) value.__set_value(std::string(kv.value)); + key_values.emplace_back(std::move(value)); + } + m.__set_key_value_metadata(std::move(key_values)); + } + m.__set_data_page_offset(dm.data_page_offset); + if (dm.has_index_page_offset) m.__set_index_page_offset(dm.index_page_offset); + if (dm.has_dictionary_page_offset) + m.__set_dictionary_page_offset(dm.dictionary_page_offset); + if (dm.has_statistics) { + format::Statistics stats; + if (dm.statistics.has_max) stats.__set_max(std::string(dm.statistics.max)); + if (dm.statistics.has_min) stats.__set_min(std::string(dm.statistics.min)); + if (dm.statistics.has_null_count) + stats.__set_null_count(dm.statistics.null_count); + if (dm.statistics.has_distinct_count) + stats.__set_distinct_count(dm.statistics.distinct_count); + if (dm.statistics.has_max_value) + stats.__set_max_value(std::string(dm.statistics.max_value)); + if (dm.statistics.has_min_value) + stats.__set_min_value(std::string(dm.statistics.min_value)); + if (dm.statistics.has_max_value_exact) + stats.__set_is_max_value_exact(dm.statistics.is_max_value_exact); + if (dm.statistics.has_min_value_exact) + stats.__set_is_min_value_exact(dm.statistics.is_min_value_exact); + if (dm.statistics.has_nan_count) stats.__set_nan_count(dm.statistics.nan_count); + m.__set_statistics(stats); + } + if (!dm.encoding_stats.empty()) m.__set_encoding_stats(dm.encoding_stats); + if (dm.has_bloom_filter_offset) + m.__set_bloom_filter_offset(dm.bloom_filter_offset); + if (dm.has_bloom_filter_length) + m.__set_bloom_filter_length(dm.bloom_filter_length); + if (dm.has_size_statistics) { + format::SizeStatistics stats; + if (dm.size_statistics.has_unencoded_byte_array_data_bytes) + stats.__set_unencoded_byte_array_data_bytes( + dm.size_statistics.unencoded_byte_array_data_bytes); + if (dm.size_statistics.has_repetition_level_histogram) + stats.__set_repetition_level_histogram( + dm.size_statistics.repetition_level_histogram); + if (dm.size_statistics.has_definition_level_histogram) + stats.__set_definition_level_histogram( + dm.size_statistics.definition_level_histogram); + m.__set_size_statistics(stats); + } + if (dm.has_geospatial_statistics) + m.__set_geospatial_statistics(dm.geospatial_statistics); + col.__isset.meta_data = direct_col.has_metadata; + if (direct_col.has_offset_index_offset) + col.__set_offset_index_offset(direct_col.offset_index_offset); + if (direct_col.has_offset_index_length) + col.__set_offset_index_length(direct_col.offset_index_length); + if (direct_col.has_column_index_offset) + col.__set_column_index_offset(direct_col.column_index_offset); + if (direct_col.has_column_index_length) + col.__set_column_index_length(direct_col.column_index_length); + if (direct_col.has_crypto_metadata) + col.__set_crypto_metadata(direct_col.crypto_metadata); + if (direct_col.has_encrypted_column_metadata) + col.__set_encrypted_column_metadata( + std::string(direct_col.encrypted_column_metadata)); + } + } + } void InitSchema() { - if (metadata_->schema.empty()) { - throw ParquetException("Empty file schema (no root)"); + if (direct_metadata_ != nullptr) { + if (direct_metadata_->schema.empty()) { + throw ParquetException("Empty file schema (no root)"); + } + schema_.Init(schema::UnflattenDirect( + std::span(direct_metadata_->schema), + properties_.schema_depth_limit())); + return; } + if (metadata_->schema.empty()) throw ParquetException("Empty file schema (no root)"); schema_.Init(schema::Unflatten(metadata_->schema, /*max_depth=*/properties_.schema_depth_limit())); } @@ -1032,9 +1639,15 @@ class FileMetaData::FileMetaDataImpl { void InitColumnOrders() { // update ColumnOrder std::vector column_orders; - if (metadata_->__isset.column_orders) { - column_orders.reserve(metadata_->column_orders.size()); - for (auto& column_order : metadata_->column_orders) { + const bool has_orders = direct_metadata_ != nullptr + ? direct_metadata_->has_column_orders + : metadata_->__isset.column_orders; + const auto& thrift_orders = direct_metadata_ != nullptr + ? direct_metadata_->column_orders + : metadata_->column_orders; + if (has_orders) { + column_orders.reserve(thrift_orders.size()); + for (auto& column_order : thrift_orders) { if (column_order.__isset.TYPE_ORDER) { column_orders.push_back(ColumnOrder::type_defined_); } else { @@ -1049,7 +1662,22 @@ class FileMetaData::FileMetaDataImpl { } void InitKeyValueMetadata() { - key_value_metadata_ = FromThriftKeyValueMetadata(*metadata_); + if (direct_metadata_ == nullptr) { + key_value_metadata_ = FromThriftKeyValueMetadata(*metadata_); + } else if (direct_metadata_->key_value_metadata.empty()) { + key_value_metadata_ = nullptr; + } else { + std::vector keys, values; + keys.reserve(direct_metadata_->key_value_metadata.size()); + values.reserve(direct_metadata_->key_value_metadata.size()); + for (const auto& kv : direct_metadata_->key_value_metadata) { + keys.emplace_back(kv.key); + values.emplace_back(kv.value); + } + key_value_metadata_ = + std::make_shared(std::move(keys), std::move(values)); + } + key_value_metadata_initialized_ = true; } }; @@ -1061,6 +1689,15 @@ std::shared_ptr FileMetaData::Make( new FileMetaData(metadata, metadata_len, properties, std::move(file_decryptor))); } +std::shared_ptr FileMetaData::Make( + std::shared_ptr metadata, int64_t metadata_len, + const ReaderProperties& properties, std::shared_ptr file_decryptor) { + if (metadata == nullptr) throw ParquetException("Null metadata buffer"); + return std::shared_ptr(new FileMetaData( + metadata->data(), metadata_len, properties, std::move(file_decryptor), + std::move(metadata))); +} + // (deprecated) std::shared_ptr FileMetaData::Make( const void* metadata, uint32_t* metadata_len, const ReaderProperties& properties, @@ -1073,9 +1710,10 @@ std::shared_ptr FileMetaData::Make( FileMetaData::FileMetaData(const void* metadata, int64_t metadata_len, const ReaderProperties& properties, - std::shared_ptr file_decryptor) + std::shared_ptr file_decryptor, + std::shared_ptr metadata_buffer) : impl_(new FileMetaDataImpl(metadata, metadata_len, properties, - std::move(file_decryptor))) {} + std::move(file_decryptor), std::move(metadata_buffer))) {} FileMetaData::FileMetaData(ReaderProperties properties) : impl_(new FileMetaDataImpl(std::move(properties))) {} diff --git a/cpp/src/parquet/metadata.h b/cpp/src/parquet/metadata.h index a79f790dfba1..63ce28a9cf59 100644 --- a/cpp/src/parquet/metadata.h +++ b/cpp/src/parquet/metadata.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "parquet/encryption/type_fwd.h" @@ -30,8 +31,18 @@ #include "parquet/platform.h" #include "parquet/properties.h" #include "parquet/type_fwd.h" + +namespace arrow { +class Buffer; +} + namespace parquet { +namespace internal { +struct DirectColumnChunk; +struct DirectRowGroup; +} // namespace internal + using KeyValueMetadata = ::arrow::KeyValueMetadata; class PARQUET_EXPORT ApplicationVersion { @@ -113,6 +124,13 @@ class PARQUET_EXPORT ColumnChunkMetaData { int16_t column_ordinal = -1, std::shared_ptr file_decryptor = NULLPTR); + static std::unique_ptr MakeDirect( + const internal::DirectColumnChunk* metadata, const ColumnDescriptor* descr, + const ReaderProperties& properties = default_reader_properties(), + const ApplicationVersion* writer_version = NULLPTR, int16_t row_group_ordinal = -1, + int16_t column_ordinal = -1, + std::shared_ptr file_decryptor = NULLPTR); + ~ColumnChunkMetaData(); bool Equals(const ColumnChunkMetaData& other) const; @@ -146,6 +164,9 @@ class PARQUET_EXPORT ColumnChunkMetaData { const std::vector& encodings() const; const std::vector& encoding_stats() const; + /// Return the compact encoding-statistics mask when enabled by ReaderProperties. + /// Each set bit corresponds to a format::Encoding value. + uint64_t encoding_stats_mask() const; std::optional bloom_filter_offset() const; std::optional bloom_filter_length() const; bool has_dictionary_page() const; @@ -166,6 +187,12 @@ class PARQUET_EXPORT ColumnChunkMetaData { int16_t column_ordinal, const ReaderProperties& properties, const ApplicationVersion* writer_version = NULLPTR, std::shared_ptr file_decryptor = NULLPTR); + explicit ColumnChunkMetaData( + const internal::DirectColumnChunk* metadata, const ColumnDescriptor* descr, + int16_t row_group_ordinal, int16_t column_ordinal, + const ReaderProperties& properties, + const ApplicationVersion* writer_version = NULLPTR, + std::shared_ptr file_decryptor = NULLPTR); // PIMPL Idiom class ColumnChunkMetaDataImpl; std::unique_ptr impl_; @@ -181,6 +208,12 @@ class PARQUET_EXPORT RowGroupMetaData { const ApplicationVersion* writer_version = NULLPTR, std::shared_ptr file_decryptor = NULLPTR); + static std::unique_ptr MakeDirect( + const internal::DirectRowGroup* metadata, const SchemaDescriptor* schema, + const ReaderProperties& properties = default_reader_properties(), + const ApplicationVersion* writer_version = NULLPTR, + std::shared_ptr file_decryptor = NULLPTR); + ~RowGroupMetaData(); bool Equals(const RowGroupMetaData& other) const; @@ -231,6 +264,11 @@ class PARQUET_EXPORT RowGroupMetaData { const ReaderProperties& properties, const ApplicationVersion* writer_version = NULLPTR, std::shared_ptr file_decryptor = NULLPTR); + explicit RowGroupMetaData( + const internal::DirectRowGroup* metadata, const SchemaDescriptor* schema, + const ReaderProperties& properties, + const ApplicationVersion* writer_version = NULLPTR, + std::shared_ptr file_decryptor = NULLPTR); // PIMPL Idiom class RowGroupMetaDataImpl; std::unique_ptr impl_; @@ -238,6 +276,36 @@ class PARQUET_EXPORT RowGroupMetaData { class FileMetaDataBuilder; +/// A read-only metadata view for latency-sensitive footer inspection. +/// +/// Unlike FileMetaData, this API does not build a SchemaDescriptor or generated +/// Thrift compatibility objects. It only discovers footer-level counts and keeps +/// returned string views valid through the retained serialized footer buffer. +/// +/// This experimental summary API scans past row groups without decoding their +/// column metadata. It is not a replacement for FileMetaData or a full metadata +/// validator. Input must be a plaintext compact-Thrift footer; encrypted footers +/// and incremental input are not supported by this entry point. +class PARQUET_EXPORT FastFileMetaData { + public: + static std::shared_ptr Make( + std::shared_ptr serialized_metadata, int64_t metadata_len, + const ReaderProperties& properties = default_reader_properties()); + + int64_t size() const; + int64_t num_rows() const; + int num_row_groups() const; + int num_schema_elements() const; + int num_columns() const; + std::string_view created_by() const; + + private: + FastFileMetaData(std::shared_ptr serialized_metadata, + int64_t metadata_len, const ReaderProperties& properties); + class Impl; + std::unique_ptr impl_; +}; + /// \brief FileMetaData is a proxy around format::FileMetaData. class PARQUET_EXPORT FileMetaData { public: @@ -256,6 +324,13 @@ class PARQUET_EXPORT FileMetaData { const ReaderProperties& properties = default_reader_properties(), std::shared_ptr file_decryptor = NULLPTR); + // Internal reader path retaining the footer buffer so direct metadata can + // borrow strings without copying the entire footer. + static std::shared_ptr Make( + std::shared_ptr serialized_metadata, int64_t metadata_len, + const ReaderProperties& properties = default_reader_properties(), + std::shared_ptr file_decryptor = NULLPTR); + ~FileMetaData(); bool Equals(const FileMetaData& other) const; @@ -389,7 +464,8 @@ class PARQUET_EXPORT FileMetaData { explicit FileMetaData(ReaderProperties properties); FileMetaData(const void* serialized_metadata, int64_t metadata_len, const ReaderProperties& properties, - std::shared_ptr file_decryptor = NULLPTR); + std::shared_ptr file_decryptor = NULLPTR, + std::shared_ptr metadata_buffer = nullptr); void set_file_decryptor(std::shared_ptr file_decryptor); const std::shared_ptr& file_decryptor() const; diff --git a/cpp/src/parquet/metadata_test.cc b/cpp/src/parquet/metadata_test.cc index ee2f0619ebf2..ed4da8aaa716 100644 --- a/cpp/src/parquet/metadata_test.cc +++ b/cpp/src/parquet/metadata_test.cc @@ -22,6 +22,7 @@ #include +#include "arrow/buffer.h" #include "arrow/util/key_value_metadata.h" #include "parquet/exception.h" #include "parquet/file_reader.h" @@ -348,6 +349,70 @@ void AssertColumnChunkMinMax(const FileMetaData& metadata, } // namespace +TEST(Metadata, FastMetadataSummaryRetainsFooter) { + auto thrift = SingleInt32MetadataWithStats(); + thrift.__set_created_by("synthetic-parser-experiment"); + auto serialized = SerializeMetadata(thrift); + auto expected = FileMetaData::Make(serialized.data(), serialized.size()); + std::shared_ptr buffer = + ::arrow::Buffer::FromString(std::move(serialized)); + auto summary = FastFileMetaData::Make(buffer, buffer->size()); + EXPECT_EQ(expected->size(), summary->size()); + EXPECT_EQ(expected->num_rows(), summary->num_rows()); + EXPECT_EQ(expected->num_row_groups(), summary->num_row_groups()); + EXPECT_EQ(expected->num_schema_elements(), summary->num_schema_elements()); + EXPECT_EQ(expected->num_columns(), summary->num_columns()); + buffer.reset(); + EXPECT_EQ("synthetic-parser-experiment", summary->created_by()); +} + +TEST(Metadata, FastMetadataSummaryRejectsInvalidBuffers) { + auto buffer = ::arrow::Buffer::FromString( + SerializeMetadata(SingleInt32MetadataWithStats())); + EXPECT_THROW(FastFileMetaData::Make(nullptr, 0), ParquetException); + EXPECT_THROW(FastFileMetaData::Make(buffer, -1), ParquetException); + EXPECT_THROW(FastFileMetaData::Make(buffer, buffer->size() + 1), ParquetException); + EXPECT_THROW(FastFileMetaData::Make(buffer, buffer->size() - 1), ParquetException); +} + +TEST(Metadata, CustomDeserializerStatisticsPolicies) { + const auto serialized = SerializeMetadata(SingleInt32MetadataWithStats()); + ReaderProperties properties; + properties.set_thrift_skip_column_statistics(true); + properties.set_thrift_skip_encoding_statistics(true); + properties.set_thrift_skip_size_statistics(true); + + format::FileMetaData parsed; + ThriftDeserializer deserializer(properties); + ASSERT_GT( + deserializer.DeserializeMessage(reinterpret_cast(serialized.data()), + serialized.size(), &parsed), + 0); + ASSERT_FALSE(parsed.row_groups.empty()); + ASSERT_FALSE(parsed.row_groups[0].columns.empty()); + const auto& column = parsed.row_groups[0].columns[0].meta_data; + EXPECT_FALSE(column.__isset.statistics); + EXPECT_FALSE(column.__isset.encoding_stats); + EXPECT_FALSE(column.__isset.size_statistics); +} + +TEST(Metadata, CustomDeserializerEncodingStatisticsMask) { + auto thrift = SingleInt32MetadataWithStats(); + format::PageEncodingStats encoding_stats; + encoding_stats.__set_page_type(format::PageType::DATA_PAGE); + encoding_stats.__set_encoding(format::Encoding::PLAIN); + encoding_stats.__set_count(3); + thrift.row_groups[0].columns[0].meta_data.__set_encoding_stats({encoding_stats}); + const auto serialized = SerializeMetadata(thrift); + ReaderProperties properties; + properties.set_thrift_encoding_stats_as_mask(true); + auto metadata = FileMetaData::Make(serialized.data(), serialized.size(), properties); + auto column = metadata->RowGroup(0)->ColumnChunk(0); + EXPECT_EQ(uint64_t{1} << static_cast(format::Encoding::PLAIN), + column->encoding_stats_mask()); + EXPECT_TRUE(column->encoding_stats().empty()); +} + TEST(Metadata, UnknownCompressionCodecThrows) { format::FileMetaData thrift_metadata = SingleInt32MetadataWithStats(); auto metadata = ParseMetadata(SerializeMetadata(thrift_metadata)); diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h index 89f2be42bffd..3c3c82a549ba 100644 --- a/cpp/src/parquet/properties.h +++ b/cpp/src/parquet/properties.h @@ -17,12 +17,14 @@ #pragma once +#include #include #include #include #include #include #include +#include #include "arrow/buffer.h" #include "arrow/io/caching.h" @@ -156,6 +158,52 @@ class PARQUET_EXPORT ReaderProperties { void set_footer_read_size(size_t size) { footer_read_size_ = size; } size_t footer_read_size() const { return footer_read_size_; } + /// Skip decoding column-level statistics in the custom metadata parser. + /// The fields remain available as absent statistics to callers. + bool thrift_skip_column_statistics() const { return thrift_skip_column_statistics_; } + void set_thrift_skip_column_statistics(bool value) { + thrift_skip_column_statistics_ = value; + } + + /// Skip decoding page encoding statistics in the custom metadata parser. + bool thrift_skip_encoding_statistics() const { + return thrift_skip_encoding_statistics_; + } + void set_thrift_skip_encoding_statistics(bool value) { + thrift_skip_encoding_statistics_ = value; + } + + /// Skip decoding size statistics in the custom metadata parser. + bool thrift_skip_size_statistics() const { return thrift_skip_size_statistics_; } + void set_thrift_skip_size_statistics(bool value) { + thrift_skip_size_statistics_ = value; + } + + /// Decode statistics only for the listed flattened leaf-column indices. + /// Calling this enables a selective metadata policy; an empty list skips all + /// three statistics groups. + const std::vector& thrift_statistics_columns() const { + return thrift_statistics_columns_; + } + bool thrift_statistics_columns_set() const { return thrift_statistics_columns_set_; } + void set_thrift_statistics_columns(std::vector columns) { + std::sort(columns.begin(), columns.end()); + columns.erase(std::unique(columns.begin(), columns.end()), columns.end()); + thrift_statistics_columns_ = std::move(columns); + thrift_statistics_columns_set_ = true; + } + void clear_thrift_statistics_columns() { + thrift_statistics_columns_.clear(); + thrift_statistics_columns_set_ = false; + } + + bool thrift_skip_page_statistics() const { return thrift_skip_page_statistics_; } + void set_thrift_skip_page_statistics(bool value) { + thrift_skip_page_statistics_ = value; + } + bool thrift_encoding_stats_as_mask() const { return thrift_encoding_stats_as_mask_; } + void set_thrift_encoding_stats_as_mask(bool value) { thrift_encoding_stats_as_mask_ = value; } + private: MemoryPool* pool_; int64_t buffer_size_ = kDefaultBufferSize; @@ -166,6 +214,13 @@ class PARQUET_EXPORT ReaderProperties { bool page_checksum_verification_ = false; // Used with a RecordReader. bool read_dense_for_nullable_ = false; + bool thrift_skip_column_statistics_ = false; + bool thrift_skip_encoding_statistics_ = false; + bool thrift_skip_size_statistics_ = false; + bool thrift_statistics_columns_set_ = false; + std::vector thrift_statistics_columns_; + bool thrift_skip_page_statistics_ = false; + bool thrift_encoding_stats_as_mask_ = false; size_t footer_read_size_ = kDefaultFooterReadSize; std::shared_ptr file_decryption_properties_; }; diff --git a/cpp/src/parquet/schema.cc b/cpp/src/parquet/schema.cc index 882c615ed070..1c61da5b8953 100644 --- a/cpp/src/parquet/schema.cc +++ b/cpp/src/parquet/schema.cc @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -28,6 +29,8 @@ #include "arrow/util/logging.h" #include "parquet/exception.h" #include "parquet/schema_internal.h" + +#include "parquet/direct_metadata.h" #include "parquet/thrift_internal.h" #include "parquet/windows_fixup.h" // for OPTIONAL @@ -618,6 +621,71 @@ std::unique_ptr Unflatten(std::span elements, return root; } +NodePtr UnflattenDirect( + std::span elements, int max_depth) { + if (elements.empty()) throw ParquetException("Empty Parquet schema (no root)"); + size_t pos = 0; + size_t num_reserved = 0; + auto converted_type = [](format::ConvertedType::type value) { + // The generated Thrift enum omits the internal NONE sentinel used by the + // parquet C++ schema API, so all real values are shifted by one. + return static_cast(static_cast(value) + 1); + }; + std::function next = [&](int depth) -> NodePtr { + if (pos == elements.size()) + throw ParquetException("Malformed Parquet schema: not enough elements"); + const auto& element = elements[pos++]; + const auto repetition = static_cast(element.repetition_type); + const int field_id = element.has_field_id ? element.field_id : -1; + const bool leaf = element.has_type && + (!element.has_num_children || element.num_children == 0); + if (leaf) { + if (!element.has_type) + throw ParquetException("Malformed Parquet schema: leaf has no type"); + if (element.has_logical_type) { + auto logical = LogicalType::FromThrift(element.logical_type); + if (logical && !logical->is_applicable(static_cast(element.type), + element.has_type_length ? element.type_length : 0)) { + logical = UndefinedLogicalType::Make(); + } + return PrimitiveNode::Make(std::string(element.name), repetition, std::move(logical), + static_cast(element.type), + element.has_type_length ? element.type_length : 0, + field_id); + } + return PrimitiveNode::Make( + std::string(element.name), repetition, static_cast(element.type), + element.has_converted_type ? converted_type(element.converted_type) + : ConvertedType::NONE, + element.has_type_length ? element.type_length : 0, + element.has_precision ? element.precision : -1, element.has_scale ? element.scale : -1, + field_id); + } + if (depth >= max_depth) + throw ParquetException("Parquet schema too deeply nested, consider increasing schema depth limit"); + if (element.num_children < 0) + throw ParquetException("Malformed Parquet schema: negative number of children"); + num_reserved += static_cast(element.num_children); + if (num_reserved > elements.size()) + throw ParquetException("Malformed Parquet schema: not enough elements"); + NodeVector fields; + fields.reserve(static_cast(element.num_children)); + for (int i = 0; i < element.num_children; ++i) fields.emplace_back(next(depth + 1)); + if (element.has_logical_type) { + return GroupNode::Make(std::string(element.name), repetition, fields, + LogicalType::FromThrift(element.logical_type), field_id); + } + return GroupNode::Make(std::string(element.name), repetition, fields, + element.has_converted_type ? converted_type(element.converted_type) + : ConvertedType::NONE, + field_id); + }; + auto root = next(1); + if (pos != elements.size()) + throw ParquetException("Malformed Parquet schema: too many elements"); + return root; +} + std::shared_ptr SchemaFromThrift(std::span schema, int max_depth) { std::unique_ptr root = Unflatten(schema, max_depth); diff --git a/cpp/src/parquet/schema_internal.h b/cpp/src/parquet/schema_internal.h index 56b6dc1bf247..2ae3005c2984 100644 --- a/cpp/src/parquet/schema_internal.h +++ b/cpp/src/parquet/schema_internal.h @@ -29,6 +29,10 @@ namespace parquet { +namespace internal { +struct DirectSchemaElement; +} + namespace format { class SchemaElement; } @@ -46,6 +50,12 @@ PARQUET_EXPORT std::unique_ptr Unflatten(std::span schema, int max_depth); +// Build a schema tree directly from the compact parser's lightweight schema +// representation, avoiding a generated SchemaElement vector. +PARQUET_EXPORT +NodePtr UnflattenDirect( + std::span schema, int max_depth); + // ---------------------------------------------------------------------- // Conversion to Parquet Thrift metadata diff --git a/cpp/src/parquet/thrift_compact_reader.cc b/cpp/src/parquet/thrift_compact_reader.cc new file mode 100644 index 000000000000..7755117da20b --- /dev/null +++ b/cpp/src/parquet/thrift_compact_reader.cc @@ -0,0 +1,1733 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// SPDX-License-Identifier: Apache-2.0 + +#include "parquet/thrift_compact_reader.h" + +#include + +namespace parquet { +namespace internal { + +using namespace format; + +namespace { + +void ReadDirectStatistics(CompactInput& input, DirectStatistics* out) { + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::Binary); + out->max = input.ReadBinaryView(); + out->has_max = true; + break; + case 2: + Expect(f.type, CompactType::Binary); + out->min = input.ReadBinaryView(); + out->has_min = true; + break; + case 3: + Expect(f.type, CompactType::I64); + out->null_count = input.ReadI64(); + out->has_null_count = true; + break; + case 4: + Expect(f.type, CompactType::I64); + out->distinct_count = input.ReadI64(); + out->has_distinct_count = true; + break; + case 5: + Expect(f.type, CompactType::Binary); + out->max_value = input.ReadBinaryView(); + out->has_max_value = true; + break; + case 6: + Expect(f.type, CompactType::Binary); + out->min_value = input.ReadBinaryView(); + out->has_min_value = true; + break; + case 7: + Expect(f.type, CompactType::BoolTrue); + out->is_max_value_exact = f.bool_value; + out->has_max_value_exact = true; + break; + case 8: + Expect(f.type, CompactType::BoolTrue); + out->is_min_value_exact = f.bool_value; + out->has_min_value_exact = true; + break; + case 9: + Expect(f.type, CompactType::I64); + out->nan_count = input.ReadI64(); + out->has_nan_count = true; + break; + default: + input.Skip(f.type); + } + }); +} + +void ReadDirectSizeStatistics(CompactInput& input, DirectSizeStatistics* out) { + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::I64); + out->unencoded_byte_array_data_bytes = input.ReadI64(); + out->has_unencoded_byte_array_data_bytes = true; + break; + case 2: + Expect(f.type, CompactType::List); + out->repetition_level_histogram = input.ReadList( + CompactType::I64, [&](CompactType) { return input.ReadI64(); }); + out->has_repetition_level_histogram = true; + break; + case 3: + Expect(f.type, CompactType::List); + out->definition_level_histogram = input.ReadList( + CompactType::I64, [&](CompactType) { return input.ReadI64(); }); + out->has_definition_level_histogram = true; + break; + default: + input.Skip(f.type); + } + }); +} + +template +T ReadStructValue(CompactInput& input, F&& f) { + T value; + f(input, &value); + return value; +} + +template +std::vector ReadStructList(CompactInput& input, F&& f) { + return input.ReadList(CompactType::Struct, + [&](CompactType) { return ReadStructValue(input, f); }); +} + +template +void ReadOptionalStruct(CompactInput& input, const CompactField& field, F&& f, T* out) { + Expect(field.type, CompactType::Struct); + f(input, out); +} + +} // namespace + +void ReadSizeStatistics(CompactInput& input, format::SizeStatistics* out) { + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::I64); + out->__set_unencoded_byte_array_data_bytes(input.ReadI64()); + break; + case 2: { + Expect(f.type, CompactType::List); + out->repetition_level_histogram = input.ReadList( + CompactType::I64, [&](CompactType) { return input.ReadI64(); }); + out->__isset.repetition_level_histogram = true; + break; + } + case 3: { + Expect(f.type, CompactType::List); + out->definition_level_histogram = input.ReadList( + CompactType::I64, [&](CompactType) { return input.ReadI64(); }); + out->__isset.definition_level_histogram = true; + break; + } + default: + input.Skip(f.type); + } + }); +} + +void ReadBoundingBox(CompactInput& input, format::BoundingBox* out) { + bool xmin = false, xmax = false, ymin = false, ymax = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::Double); + out->__set_xmin(input.ReadDouble()); + xmin = true; + break; + case 2: + Expect(f.type, CompactType::Double); + out->__set_xmax(input.ReadDouble()); + xmax = true; + break; + case 3: + Expect(f.type, CompactType::Double); + out->__set_ymin(input.ReadDouble()); + ymin = true; + break; + case 4: + Expect(f.type, CompactType::Double); + out->__set_ymax(input.ReadDouble()); + ymax = true; + break; + case 5: + Expect(f.type, CompactType::Double); + out->__set_zmin(input.ReadDouble()); + break; + case 6: + Expect(f.type, CompactType::Double); + out->__set_zmax(input.ReadDouble()); + break; + case 7: + Expect(f.type, CompactType::Double); + out->__set_mmin(input.ReadDouble()); + break; + case 8: + Expect(f.type, CompactType::Double); + out->__set_mmax(input.ReadDouble()); + break; + default: + input.Skip(f.type); + } + }); + if (!xmin || !xmax || !ymin || !ymax) + throw ParquetException("Required BoundingBox field missing"); +} + +void ReadGeospatialStatistics(CompactInput& input, format::GeospatialStatistics* out) { + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + ReadOptionalStruct(input, f, ReadBoundingBox, &out->bbox); + out->__isset.bbox = true; + break; + case 2: + Expect(f.type, CompactType::List); + out->geospatial_types = input.ReadList( + CompactType::I32, [&](CompactType) { return input.ReadI32(); }); + out->__isset.geospatial_types = true; + break; + default: + input.Skip(f.type); + } + }); +} + +void ReadStatistics(CompactInput& input, format::Statistics* out) { + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::Binary); + out->max = input.ReadBinary(); + out->__isset.max = true; + break; + case 2: + Expect(f.type, CompactType::Binary); + out->min = input.ReadBinary(); + out->__isset.min = true; + break; + case 3: + Expect(f.type, CompactType::I64); + out->__set_null_count(input.ReadI64()); + break; + case 4: + Expect(f.type, CompactType::I64); + out->__set_distinct_count(input.ReadI64()); + break; + case 5: + Expect(f.type, CompactType::Binary); + out->max_value = input.ReadBinary(); + out->__isset.max_value = true; + break; + case 6: + Expect(f.type, CompactType::Binary); + out->min_value = input.ReadBinary(); + out->__isset.min_value = true; + break; + case 7: + Expect(f.type, CompactType::BoolTrue); + out->__set_is_max_value_exact(f.bool_value); + break; + case 8: + Expect(f.type, CompactType::BoolTrue); + out->__set_is_min_value_exact(f.bool_value); + break; + case 9: + Expect(f.type, CompactType::I64); + out->__set_nan_count(input.ReadI64()); + break; + default: + input.Skip(f.type); + } + }); +} + +void ReadTimeUnit(CompactInput& input, format::TimeUnit* out) { + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_MILLIS({}); + break; + case 2: + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_MICROS({}); + break; + case 3: + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_NANOS({}); + break; + default: + input.Skip(f.type); + } + }); +} + +void ReadLogicalType(CompactInput& input, format::LogicalType* out) { + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_STRING({}); + break; + case 2: + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_MAP({}); + break; + case 3: + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_LIST({}); + break; + case 4: + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_ENUM({}); + break; + case 5: { + format::DecimalType value; + bool scale = false, precision = false; + input.ReadStruct([&](const CompactField& nested) { + if (nested.id == 1) { + Expect(nested.type, CompactType::I32); + value.__set_scale(input.ReadI32()); + scale = true; + } else if (nested.id == 2) { + Expect(nested.type, CompactType::I32); + value.__set_precision(input.ReadI32()); + precision = true; + } else + input.Skip(nested.type); + }); + if (!scale || !precision) + throw ParquetException("Required DecimalType field missing"); + out->__set_DECIMAL(value); + break; + } + case 6: + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_DATE({}); + break; + case 7: { + Expect(f.type, CompactType::Struct); + format::TimeType value; + bool utc = false, unit = false; + input.ReadStruct([&](const CompactField& n) { + if (n.id == 1) { + Expect(n.type, CompactType::BoolTrue); + value.__set_isAdjustedToUTC(n.bool_value); + utc = true; + } else if (n.id == 2) { + Expect(n.type, CompactType::Struct); + ReadTimeUnit(input, &value.unit); + value.__set_unit(value.unit); + unit = true; + } else + input.Skip(n.type); + }); + if (!utc || !unit) throw ParquetException("Required TimeType field missing"); + out->__set_TIME(value); + break; + } + case 8: { + Expect(f.type, CompactType::Struct); + format::TimestampType value; + bool utc = false, unit = false; + input.ReadStruct([&](const CompactField& n) { + if (n.id == 1) { + Expect(n.type, CompactType::BoolTrue); + value.__set_isAdjustedToUTC(n.bool_value); + utc = true; + } else if (n.id == 2) { + Expect(n.type, CompactType::Struct); + ReadTimeUnit(input, &value.unit); + value.__set_unit(value.unit); + unit = true; + } else + input.Skip(n.type); + }); + if (!utc || !unit) throw ParquetException("Required TimestampType field missing"); + out->__set_TIMESTAMP(value); + break; + } + case 10: { + Expect(f.type, CompactType::Struct); + format::IntType value; + bool width = false, sign = false; + input.ReadStruct([&](const CompactField& n) { + if (n.id == 1) { + Expect(n.type, CompactType::Byte); + value.__set_bitWidth(input.ReadI8()); + width = true; + } else if (n.id == 2) { + Expect(n.type, CompactType::BoolTrue); + value.__set_isSigned(n.bool_value); + sign = true; + } else + input.Skip(n.type); + }); + if (!width || !sign) throw ParquetException("Required IntType field missing"); + out->__set_INTEGER(value); + break; + } + case 11: + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_UNKNOWN({}); + break; + case 12: + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_JSON({}); + break; + case 13: + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_BSON({}); + break; + case 14: + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_UUID({}); + break; + case 15: + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_FLOAT16({}); + break; + case 16: { + format::VariantType value; + input.ReadStruct([&](const CompactField& n) { + if (n.id == 1) { + Expect(n.type, CompactType::Byte); + value.__set_specification_version(input.ReadI8()); + } else + input.Skip(n.type); + }); + out->__set_VARIANT(value); + break; + } + case 17: { + format::GeometryType value; + input.ReadStruct([&](const CompactField& n) { + if (n.id == 1) { + Expect(n.type, CompactType::Binary); + value.crs = input.ReadBinary(); + value.__isset.crs = true; + } else + input.Skip(n.type); + }); + out->__set_GEOMETRY(value); + break; + } + case 18: { + format::GeographyType value; + input.ReadStruct([&](const CompactField& n) { + if (n.id == 1) { + Expect(n.type, CompactType::Binary); + value.crs = input.ReadBinary(); + value.__isset.crs = true; + } else if (n.id == 2) { + Expect(n.type, CompactType::I32); + value.__set_algorithm(ReadEnum(input)); + } else + input.Skip(n.type); + }); + out->__set_GEOGRAPHY(value); + break; + } + case 19: + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_FILE({}); + break; + default: + input.Skip(f.type); + } + }); +} + +void ReadSchemaElement(CompactInput& input, format::SchemaElement* out) { + bool name = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::I32); + out->__set_type(ReadEnum(input)); + break; + case 2: + Expect(f.type, CompactType::I32); + out->__set_type_length(input.ReadI32()); + break; + case 3: + Expect(f.type, CompactType::I32); + out->__set_repetition_type(ReadEnum(input)); + break; + case 4: + Expect(f.type, CompactType::Binary); + out->name = input.ReadBinary(); + name = true; + break; + case 5: + Expect(f.type, CompactType::I32); + out->__set_num_children(input.ReadI32()); + break; + case 6: + Expect(f.type, CompactType::I32); + out->__set_converted_type(ReadEnum(input)); + break; + case 7: + Expect(f.type, CompactType::I32); + out->__set_scale(input.ReadI32()); + break; + case 8: + Expect(f.type, CompactType::I32); + out->__set_precision(input.ReadI32()); + break; + case 9: + Expect(f.type, CompactType::I32); + out->__set_field_id(input.ReadI32()); + break; + case 10: + Expect(f.type, CompactType::Struct); + ReadLogicalType(input, &out->logicalType); + out->__isset.logicalType = true; + break; + default: + input.Skip(f.type); + } + }); + if (!name) throw ParquetException("Required SchemaElement name missing"); +} + +void ReadDirectSchemaElement(CompactInput& input, DirectSchemaElement* out) { + bool name = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::I32); + out->type = ReadEnum(input); + out->has_type = true; + break; + case 2: + Expect(f.type, CompactType::I32); + out->type_length = input.ReadI32(); + out->has_type_length = true; + break; + case 3: + Expect(f.type, CompactType::I32); + out->repetition_type = ReadEnum(input); + out->has_repetition_type = true; + break; + case 4: + Expect(f.type, CompactType::Binary); + out->name = input.ReadBinaryView(); + name = true; + break; + case 5: + Expect(f.type, CompactType::I32); + out->num_children = input.ReadI32(); + out->has_num_children = true; + break; + case 6: + Expect(f.type, CompactType::I32); + out->converted_type = ReadEnum(input); + out->has_converted_type = true; + break; + case 7: + Expect(f.type, CompactType::I32); + out->scale = input.ReadI32(); + out->has_scale = true; + break; + case 8: + Expect(f.type, CompactType::I32); + out->precision = input.ReadI32(); + out->has_precision = true; + break; + case 9: + Expect(f.type, CompactType::I32); + out->field_id = input.ReadI32(); + out->has_field_id = true; + break; + case 10: + Expect(f.type, CompactType::Struct); + ReadLogicalType(input, &out->logical_type); + out->has_logical_type = true; + break; + default: + input.Skip(f.type); + } + }); + if (!name) throw ParquetException("Required SchemaElement name missing"); +} + +void ReadKeyValue(CompactInput& input, format::KeyValue* out) { + bool key = false; + input.ReadStruct([&](const CompactField& f) { + if (f.id == 1) { + Expect(f.type, CompactType::Binary); + out->key = input.ReadBinary(); + key = true; + } else if (f.id == 2) { + Expect(f.type, CompactType::Binary); + out->value = input.ReadBinary(); + out->__isset.value = true; + } else + input.Skip(f.type); + }); + if (!key) throw ParquetException("Required KeyValue key missing"); +} + +void ReadDirectKeyValue(CompactInput& input, DirectKeyValue* out) { + bool key = false; + input.ReadStruct([&](const CompactField& f) { + if (f.id == 1) { + Expect(f.type, CompactType::Binary); + out->key = input.ReadBinaryView(); + key = true; + } else if (f.id == 2) { + Expect(f.type, CompactType::Binary); + out->value = input.ReadBinaryView(); + out->has_value = true; + } else { + input.Skip(f.type); + } + }); + if (!key) throw ParquetException("Required KeyValue key missing"); +} + +void ReadSortingColumn(CompactInput& input, format::SortingColumn* out) { + bool a = false, b = false, c = false; + input.ReadStruct([&](const CompactField& f) { + if (f.id == 1) { + Expect(f.type, CompactType::I32); + out->__set_column_idx(input.ReadI32()); + a = true; + } else if (f.id == 2) { + Expect(f.type, CompactType::BoolTrue); + out->__set_descending(f.bool_value); + b = true; + } else if (f.id == 3) { + Expect(f.type, CompactType::BoolTrue); + out->__set_nulls_first(f.bool_value); + c = true; + } else + input.Skip(f.type); + }); + if (!a || !b || !c) throw ParquetException("Required SortingColumn field missing"); +} + +void ReadPageEncodingStats(CompactInput& input, format::PageEncodingStats* out) { + bool a = false, b = false, c = false; + input.ReadStruct([&](const CompactField& f) { + if (f.id == 1) { + Expect(f.type, CompactType::I32); + out->__set_page_type(ReadEnum(input)); + a = true; + } else if (f.id == 2) { + Expect(f.type, CompactType::I32); + out->__set_encoding(ReadEnum(input)); + b = true; + } else if (f.id == 3) { + Expect(f.type, CompactType::I32); + out->__set_count(input.ReadI32()); + c = true; + } else + input.Skip(f.type); + }); + if (!a || !b || !c) throw ParquetException("Required PageEncodingStats field missing"); +} + +void ReadColumnMetaData(CompactInput& input, format::ColumnMetaData* out) { + bool type = false, enc = false, path = false, codec = false, values = false, us = false, + cs = false, offset = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::I32); + out->__set_type(ReadEnum(input)); + type = true; + break; + case 2: + out->encodings = input.ReadList( + CompactType::I32, + [&](CompactType) { return ReadEnum(input); }); + enc = true; + break; + case 3: + out->path_in_schema = input.ReadList( + CompactType::Binary, [&](CompactType) { return input.ReadBinary(); }); + path = true; + break; + case 4: + Expect(f.type, CompactType::I32); + out->__set_codec(ReadEnum(input)); + codec = true; + break; + case 5: + Expect(f.type, CompactType::I64); + out->__set_num_values(input.ReadI64()); + values = true; + break; + case 6: + Expect(f.type, CompactType::I64); + out->__set_total_uncompressed_size(input.ReadI64()); + us = true; + break; + case 7: + Expect(f.type, CompactType::I64); + out->__set_total_compressed_size(input.ReadI64()); + cs = true; + break; + case 8: + out->key_value_metadata = ReadStructList(input, ReadKeyValue); + out->__isset.key_value_metadata = true; + break; + case 9: + Expect(f.type, CompactType::I64); + out->__set_data_page_offset(input.ReadI64()); + offset = true; + break; + case 10: + Expect(f.type, CompactType::I64); + out->__set_index_page_offset(input.ReadI64()); + break; + case 11: + Expect(f.type, CompactType::I64); + out->__set_dictionary_page_offset(input.ReadI64()); + break; + case 12: + Expect(f.type, CompactType::Struct); + if (input.skip_column_stats() || input.skip_for_column()) { + input.Skip(f.type); + } else { + ReadStatistics(input, &out->statistics); + out->__isset.statistics = true; + } + break; + case 13: + if (input.skip_encoding_stats() || input.skip_for_column()) { + Expect(f.type, CompactType::List); + input.Skip(f.type); + } else { + out->encoding_stats = + ReadStructList(input, ReadPageEncodingStats); + out->__isset.encoding_stats = true; + } + break; + case 14: + Expect(f.type, CompactType::I64); + out->__set_bloom_filter_offset(input.ReadI64()); + break; + case 15: + Expect(f.type, CompactType::I32); + out->__set_bloom_filter_length(input.ReadI32()); + break; + case 16: + Expect(f.type, CompactType::Struct); + if (input.skip_size_stats() || input.skip_for_column()) { + input.Skip(f.type); + } else { + ReadSizeStatistics(input, &out->size_statistics); + out->__isset.size_statistics = true; + } + break; + case 17: + Expect(f.type, CompactType::Struct); + ReadGeospatialStatistics(input, &out->geospatial_statistics); + out->__isset.geospatial_statistics = true; + break; + default: + input.Skip(f.type); + } + }); + if (!type || !enc || !path || !codec || !values || !us || !cs || !offset) { + throw ParquetException("Required ColumnMetaData field missing"); + } +} + +void ReadColumnCryptoMetaData(CompactInput& input, format::ColumnCryptoMetaData* out) { + bool seen = false; + input.ReadStruct([&](const CompactField& f) { + if (f.id == 1) { + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_ENCRYPTION_WITH_FOOTER_KEY({}); + seen = true; + } else if (f.id == 2) { + format::EncryptionWithColumnKey value; + bool path = false; + Expect(f.type, CompactType::Struct); + input.ReadStruct([&](const CompactField& n) { + if (n.id == 1) { + value.path_in_schema = input.ReadList( + CompactType::Binary, [&](CompactType) { return input.ReadBinary(); }); + path = true; + } else if (n.id == 2) { + Expect(n.type, CompactType::Binary); + value.key_metadata = input.ReadBinary(); + value.__isset.key_metadata = true; + } else + input.Skip(n.type); + }); + if (!path) throw ParquetException("Required encryption path missing"); + out->__set_ENCRYPTION_WITH_COLUMN_KEY(value); + seen = true; + } else + input.Skip(f.type); + }); + if (!seen) throw ParquetException("Empty ColumnCryptoMetaData union"); +} + +void ReadColumnChunk(CompactInput& input, format::ColumnChunk* out) { + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::Binary); + out->file_path = input.ReadBinary(); + out->__isset.file_path = true; + break; + case 2: + Expect(f.type, CompactType::I64); + out->__set_file_offset(input.ReadI64()); + break; + case 3: + Expect(f.type, CompactType::Struct); + ReadColumnMetaData(input, &out->meta_data); + out->__isset.meta_data = true; + break; + case 4: + Expect(f.type, CompactType::I64); + out->__set_offset_index_offset(input.ReadI64()); + break; + case 5: + Expect(f.type, CompactType::I32); + out->__set_offset_index_length(input.ReadI32()); + break; + case 6: + Expect(f.type, CompactType::I64); + out->__set_column_index_offset(input.ReadI64()); + break; + case 7: + Expect(f.type, CompactType::I32); + out->__set_column_index_length(input.ReadI32()); + break; + case 8: + Expect(f.type, CompactType::Struct); + ReadColumnCryptoMetaData(input, &out->crypto_metadata); + out->__isset.crypto_metadata = true; + break; + case 9: + Expect(f.type, CompactType::Binary); + out->encrypted_column_metadata = input.ReadBinary(); + out->__isset.encrypted_column_metadata = true; + break; + default: + input.Skip(f.type); + } + }); +} + +void ReadRowGroup(CompactInput& input, format::RowGroup* out) { + bool columns = false, bytes = false, rows = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::List); + { + const uint64_t size = input.ReadListSize(CompactType::Struct); + out->columns.clear(); + out->columns.reserve(static_cast(size)); + for (uint64_t i = 0; i < size; ++i) { + input.SetColumnIndex(static_cast(i)); + format::ColumnChunk column; + ReadColumnChunk(input, &column); + out->columns.emplace_back(std::move(column)); + } + input.SetColumnIndex(-1); + } + columns = true; + break; + case 2: + Expect(f.type, CompactType::I64); + out->__set_total_byte_size(input.ReadI64()); + bytes = true; + break; + case 3: + Expect(f.type, CompactType::I64); + out->__set_num_rows(input.ReadI64()); + rows = true; + break; + case 4: + out->sorting_columns = + ReadStructList(input, ReadSortingColumn); + out->__isset.sorting_columns = true; + break; + case 5: + Expect(f.type, CompactType::I64); + out->__set_file_offset(input.ReadI64()); + break; + case 6: + Expect(f.type, CompactType::I64); + out->__set_total_compressed_size(input.ReadI64()); + break; + case 7: + Expect(f.type, CompactType::I16); + out->__set_ordinal(input.ReadI16()); + break; + default: + input.Skip(f.type); + } + }); + if (!columns || !bytes || !rows) + throw ParquetException("Required RowGroup field missing"); +} + +void ReadColumnOrder(CompactInput& input, format::ColumnOrder* out) { + bool seen = false; + input.ReadStruct([&](const CompactField& f) { + if (f.id == 1) { + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_TYPE_ORDER({}); + seen = true; + } else if (f.id == 2) { + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_IEEE_754_TOTAL_ORDER({}); + seen = true; + } else if (f.id == 3) { + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_INT96_TIMESTAMP_ORDER({}); + seen = true; + } else + input.Skip(f.type); + }); + if (!seen) throw ParquetException("Empty ColumnOrder union"); +} + +void ReadEncryptionAlgorithm(CompactInput& input, format::EncryptionAlgorithm* out) { + bool seen = false; + input.ReadStruct([&](const CompactField& f) { + if (f.id == 1 || f.id == 2) { + Expect(f.type, CompactType::Struct); + std::string prefix, unique; + bool supply = false; + input.ReadStruct([&](const CompactField& n) { + if (n.id == 1) { + Expect(n.type, CompactType::Binary); + prefix = input.ReadBinary(); + } else if (n.id == 2) { + Expect(n.type, CompactType::Binary); + unique = input.ReadBinary(); + } else if (n.id == 3) { + Expect(n.type, CompactType::BoolTrue); + supply = n.bool_value; + } else + input.Skip(n.type); + }); + if (f.id == 1) { + format::AesGcmV1 value; + value.aad_prefix = std::move(prefix); + value.aad_file_unique = std::move(unique); + value.supply_aad_prefix = supply; + value.__isset.aad_prefix = !value.aad_prefix.empty(); + value.__isset.aad_file_unique = !value.aad_file_unique.empty(); + value.__set_supply_aad_prefix(supply); + out->__set_AES_GCM_V1(value); + } else { + format::AesGcmCtrV1 value; + value.aad_prefix = std::move(prefix); + value.aad_file_unique = std::move(unique); + value.supply_aad_prefix = supply; + value.__isset.aad_prefix = !value.aad_prefix.empty(); + value.__isset.aad_file_unique = !value.aad_file_unique.empty(); + value.__set_supply_aad_prefix(supply); + out->__set_AES_GCM_CTR_V1(value); + } + seen = true; + } else + input.Skip(f.type); + }); + if (!seen) throw ParquetException("Empty EncryptionAlgorithm union"); +} + +void ReadFileMetaData(CompactInput& input, format::FileMetaData* out) { + bool version = false, schema = false, rows = false, groups = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::I32); + out->__set_version(input.ReadI32()); + version = true; + break; + case 2: + out->schema = ReadStructList(input, ReadSchemaElement); + schema = true; + break; + case 3: + Expect(f.type, CompactType::I64); + out->__set_num_rows(input.ReadI64()); + rows = true; + break; + case 4: + out->row_groups = ReadStructList(input, ReadRowGroup); + groups = true; + break; + case 5: + out->key_value_metadata = ReadStructList(input, ReadKeyValue); + out->__isset.key_value_metadata = true; + break; + case 6: + Expect(f.type, CompactType::Binary); + out->created_by = input.ReadBinary(); + out->__isset.created_by = true; + break; + case 7: + out->column_orders = ReadStructList(input, ReadColumnOrder); + out->__isset.column_orders = true; + break; + case 8: + Expect(f.type, CompactType::Struct); + ReadEncryptionAlgorithm(input, &out->encryption_algorithm); + out->__set_encryption_algorithm(out->encryption_algorithm); + break; + case 9: + Expect(f.type, CompactType::Binary); + out->footer_signing_key_metadata = input.ReadBinary(); + out->__isset.footer_signing_key_metadata = true; + break; + default: + input.Skip(f.type); + } + }); + if (!version || !schema || !rows || !groups) + throw ParquetException("Required FileMetaData field missing"); +} + +void ReadPageLocation(CompactInput& input, format::PageLocation* out) { + bool offset = false, size = false, row = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::I64); + out->__set_offset(input.ReadI64()); + offset = true; + break; + case 2: + Expect(f.type, CompactType::I32); + out->__set_compressed_page_size(input.ReadI32()); + size = true; + break; + case 3: + Expect(f.type, CompactType::I64); + out->__set_first_row_index(input.ReadI64()); + row = true; + break; + default: + input.Skip(f.type); + } + }); + if (!offset || !size || !row) + throw ParquetException("Required PageLocation field missing"); +} + +void ReadOffsetIndex(CompactInput& input, format::OffsetIndex* out) { + bool pages = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::List); + out->page_locations = + ReadStructList(input, ReadPageLocation); + pages = true; + break; + case 2: + Expect(f.type, CompactType::List); + out->unencoded_byte_array_data_bytes = input.ReadList( + CompactType::I64, [&](CompactType) { return input.ReadI64(); }); + out->__isset.unencoded_byte_array_data_bytes = true; + break; + default: + input.Skip(f.type); + } + }); + if (!pages) throw ParquetException("Required OffsetIndex page_locations missing"); +} + +void ReadColumnIndex(CompactInput& input, format::ColumnIndex* out) { + bool null_pages = false, min_values = false, max_values = false, order = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::List); + out->null_pages = input.ReadList( + CompactType::BoolTrue, [&](CompactType) { return input.ReadBoolElement(); }); + null_pages = true; + break; + case 2: + Expect(f.type, CompactType::List); + out->min_values = input.ReadList( + CompactType::Binary, [&](CompactType) { return input.ReadBinary(); }); + min_values = true; + break; + case 3: + Expect(f.type, CompactType::List); + out->max_values = input.ReadList( + CompactType::Binary, [&](CompactType) { return input.ReadBinary(); }); + max_values = true; + break; + case 4: + Expect(f.type, CompactType::I32); + out->__set_boundary_order(ReadEnum(input)); + order = true; + break; + case 5: + Expect(f.type, CompactType::List); + out->__set_null_counts(input.ReadList( + CompactType::I64, [&](CompactType) { return input.ReadI64(); })); + break; + case 6: + Expect(f.type, CompactType::List); + out->__set_repetition_level_histograms(input.ReadList( + CompactType::I64, [&](CompactType) { return input.ReadI64(); })); + break; + case 7: + Expect(f.type, CompactType::List); + out->__set_definition_level_histograms(input.ReadList( + CompactType::I64, [&](CompactType) { return input.ReadI64(); })); + break; + case 8: + Expect(f.type, CompactType::List); + out->__set_nan_counts(input.ReadList( + CompactType::I64, [&](CompactType) { return input.ReadI64(); })); + break; + default: + input.Skip(f.type); + } + }); + if (!null_pages || !min_values || !max_values || !order) { + throw ParquetException("Required ColumnIndex field missing"); + } +} + +void ReadDataPageHeader(CompactInput& input, format::DataPageHeader* out) { + bool values = false, encoding = false, definition = false, repetition = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::I32); + out->__set_num_values(input.ReadI32()); + values = true; + break; + case 2: + Expect(f.type, CompactType::I32); + out->__set_encoding(ReadEnum(input)); + encoding = true; + break; + case 3: + Expect(f.type, CompactType::I32); + out->__set_definition_level_encoding(ReadEnum(input)); + definition = true; + break; + case 4: + Expect(f.type, CompactType::I32); + out->__set_repetition_level_encoding(ReadEnum(input)); + repetition = true; + break; + case 5: + Expect(f.type, CompactType::Struct); + if (input.skip_page_stats()) { + input.Skip(f.type); + } else { + ReadStatistics(input, &out->statistics); + out->__isset.statistics = true; + } + break; + default: + input.Skip(f.type); + } + }); + if (!values || !encoding || !definition || !repetition) { + throw ParquetException("Required DataPageHeader field missing"); + } +} + +void ReadDataPageHeaderV2(CompactInput& input, format::DataPageHeaderV2* out) { + bool values = false, nulls = false, rows = false, encoding = false, definition = false, + repetition = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::I32); + out->__set_num_values(input.ReadI32()); + values = true; + break; + case 2: + Expect(f.type, CompactType::I32); + out->__set_num_nulls(input.ReadI32()); + nulls = true; + break; + case 3: + Expect(f.type, CompactType::I32); + out->__set_num_rows(input.ReadI32()); + rows = true; + break; + case 4: + Expect(f.type, CompactType::I32); + out->__set_encoding(ReadEnum(input)); + encoding = true; + break; + case 5: + Expect(f.type, CompactType::I32); + out->__set_definition_levels_byte_length(input.ReadI32()); + definition = true; + break; + case 6: + Expect(f.type, CompactType::I32); + out->__set_repetition_levels_byte_length(input.ReadI32()); + repetition = true; + break; + case 7: + Expect(f.type, CompactType::BoolTrue); + out->__set_is_compressed(f.bool_value); + break; + case 8: + Expect(f.type, CompactType::Struct); + if (input.skip_page_stats()) { + input.Skip(f.type); + } else { + ReadStatistics(input, &out->statistics); + out->__isset.statistics = true; + } + break; + default: + input.Skip(f.type); + } + }); + if (!values || !nulls || !rows || !encoding || !definition || !repetition) { + throw ParquetException("Required DataPageHeaderV2 field missing"); + } +} + +void ReadDictionaryPageHeader(CompactInput& input, format::DictionaryPageHeader* out) { + bool values = false, encoding = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::I32); + out->__set_num_values(input.ReadI32()); + values = true; + break; + case 2: + Expect(f.type, CompactType::I32); + out->__set_encoding(ReadEnum(input)); + encoding = true; + break; + case 3: + Expect(f.type, CompactType::BoolTrue); + out->__set_is_sorted(f.bool_value); + break; + default: + input.Skip(f.type); + } + }); + if (!values || !encoding) + throw ParquetException("Required DictionaryPageHeader field missing"); +} + +void ReadPageHeader(CompactInput& input, format::PageHeader* out) { + bool type = false, uncompressed = false, compressed = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::I32); + out->__set_type(ReadEnum(input)); + type = true; + break; + case 2: + Expect(f.type, CompactType::I32); + out->__set_uncompressed_page_size(input.ReadI32()); + uncompressed = true; + break; + case 3: + Expect(f.type, CompactType::I32); + out->__set_compressed_page_size(input.ReadI32()); + compressed = true; + break; + case 4: + Expect(f.type, CompactType::I32); + out->__set_crc(input.ReadI32()); + break; + case 5: + Expect(f.type, CompactType::Struct); + ReadDataPageHeader(input, &out->data_page_header); + out->__isset.data_page_header = true; + break; + case 6: + Expect(f.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->__set_index_page_header({}); + break; + case 7: + Expect(f.type, CompactType::Struct); + ReadDictionaryPageHeader(input, &out->dictionary_page_header); + out->__isset.dictionary_page_header = true; + break; + case 8: + Expect(f.type, CompactType::Struct); + ReadDataPageHeaderV2(input, &out->data_page_header_v2); + out->__isset.data_page_header_v2 = true; + break; + default: + input.Skip(f.type); + } + }); + if (!type || !uncompressed || !compressed) + throw ParquetException("Required PageHeader field missing"); +} + +void ReadBloomFilterHeader(CompactInput& input, format::BloomFilterHeader* out) { + bool bytes = false, algorithm = false, hash = false, compression = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::I32); + out->__set_numBytes(input.ReadI32()); + bytes = true; + break; + case 2: + Expect(f.type, CompactType::Struct); + input.ReadStruct([&](const CompactField& n) { + if (n.id == 1) { + Expect(n.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->algorithm.__set_BLOCK({}); + } else + input.Skip(n.type); + }); + algorithm = true; + break; + case 3: + Expect(f.type, CompactType::Struct); + input.ReadStruct([&](const CompactField& n) { + if (n.id == 1) { + Expect(n.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->hash.__set_XXHASH({}); + } else + input.Skip(n.type); + }); + hash = true; + break; + case 4: + Expect(f.type, CompactType::Struct); + input.ReadStruct([&](const CompactField& n) { + if (n.id == 1) { + Expect(n.type, CompactType::Struct); + input.ReadEmptyStruct(); + out->compression.__set_UNCOMPRESSED({}); + } else + input.Skip(n.type); + }); + compression = true; + break; + default: + input.Skip(f.type); + } + }); + if (!bytes || !algorithm || !hash || !compression) { + throw ParquetException("Required BloomFilterHeader field missing"); + } +} + +namespace { + +void ReadDirectColumnMetadata(CompactInput& input, DirectColumnMetadata* out) { + bool type = false, encodings = false, path = false, codec = false, values = false, + uncompressed = false, compressed = false, data_offset = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::I32); + out->type = ReadEnum(input); + type = true; + break; + case 2: + Expect(f.type, CompactType::List); + out->encodings = input.ReadList( + CompactType::I32, + [&](CompactType) { return ReadEnum(input); }); + encodings = true; + break; + case 3: + Expect(f.type, CompactType::List); + out->path_in_schema = input.ReadList( + CompactType::Binary, [&](CompactType) { return input.ReadBinaryView(); }); + path = true; + break; + case 4: + Expect(f.type, CompactType::I32); + out->codec = ReadEnum(input); + codec = true; + break; + case 5: + Expect(f.type, CompactType::I64); + out->num_values = input.ReadI64(); + values = true; + break; + case 6: + Expect(f.type, CompactType::I64); + out->total_uncompressed_size = input.ReadI64(); + uncompressed = true; + break; + case 7: + Expect(f.type, CompactType::I64); + out->total_compressed_size = input.ReadI64(); + compressed = true; + break; + case 8: + Expect(f.type, CompactType::List); + out->key_value_metadata = + ReadStructList(input, ReadDirectKeyValue); + break; + case 9: + Expect(f.type, CompactType::I64); + out->data_page_offset = input.ReadI64(); + data_offset = true; + break; + case 10: + Expect(f.type, CompactType::I64); + out->index_page_offset = input.ReadI64(); + out->has_index_page_offset = true; + break; + case 11: + Expect(f.type, CompactType::I64); + out->dictionary_page_offset = input.ReadI64(); + out->has_dictionary_page_offset = true; + break; + case 12: + Expect(f.type, CompactType::Struct); + if (input.skip_column_stats() || input.skip_for_column()) + input.Skip(f.type); + else { + ReadDirectStatistics(input, &out->statistics); + out->has_statistics = true; + } + break; + case 13: + Expect(f.type, CompactType::List); + if (input.skip_encoding_stats() || input.skip_for_column()) + input.Skip(f.type); + else if (input.encoding_stats_as_mask()) { + const uint64_t size = input.ReadListSize(CompactType::Struct); + for (uint64_t i = 0; i < size; ++i) { + format::PageEncodingStats value; + ReadPageEncodingStats(input, &value); + const auto encoding = static_cast(value.encoding); + if (encoding < 64) out->encoding_stats_mask |= uint64_t{1} << encoding; + } + out->has_encoding_stats_mask = true; + } else + out->encoding_stats = + ReadStructList(input, ReadPageEncodingStats); + break; + case 14: + Expect(f.type, CompactType::I64); + out->bloom_filter_offset = input.ReadI64(); + out->has_bloom_filter_offset = true; + break; + case 15: + Expect(f.type, CompactType::I32); + out->bloom_filter_length = input.ReadI32(); + out->has_bloom_filter_length = true; + break; + case 16: + Expect(f.type, CompactType::Struct); + if (input.skip_size_stats() || input.skip_for_column()) + input.Skip(f.type); + else { + ReadDirectSizeStatistics(input, &out->size_statistics); + out->has_size_statistics = true; + } + break; + case 17: + Expect(f.type, CompactType::Struct); + ReadGeospatialStatistics(input, &out->geospatial_statistics); + out->has_geospatial_statistics = true; + break; + default: + input.Skip(f.type); + } + }); + if (!type || !encodings || !path || !codec || !values || !uncompressed || !compressed || + !data_offset) { + throw ParquetException("Required direct ColumnMetaData field missing"); + } +} + +void ReadDirectColumnChunk(CompactInput& input, DirectColumnChunk* out) { + bool offset = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::Binary); + out->file_path = input.ReadBinaryView(); + out->has_file_path = true; + break; + case 2: + Expect(f.type, CompactType::I64); + out->file_offset = input.ReadI64(); + out->has_file_offset = true; + offset = true; + break; + case 3: + Expect(f.type, CompactType::Struct); + ReadDirectColumnMetadata(input, &out->metadata); + out->has_metadata = true; + break; + case 4: + Expect(f.type, CompactType::I64); + out->offset_index_offset = input.ReadI64(); + out->has_offset_index_offset = true; + break; + case 5: + Expect(f.type, CompactType::I32); + out->offset_index_length = input.ReadI32(); + out->has_offset_index_length = true; + break; + case 6: + Expect(f.type, CompactType::I64); + out->column_index_offset = input.ReadI64(); + out->has_column_index_offset = true; + break; + case 7: + Expect(f.type, CompactType::I32); + out->column_index_length = input.ReadI32(); + out->has_column_index_length = true; + break; + case 8: + Expect(f.type, CompactType::Struct); + ReadColumnCryptoMetaData(input, &out->crypto_metadata); + out->has_crypto_metadata = true; + break; + case 9: + Expect(f.type, CompactType::Binary); + out->encrypted_column_metadata = input.ReadBinaryView(); + out->has_encrypted_column_metadata = true; + break; + default: + input.Skip(f.type); + } + }); + if (!offset) throw ParquetException("Required direct ColumnChunk file_offset missing"); +} + +void ReadDirectRowGroup(CompactInput& input, DirectRowGroup* out) { + bool columns = false, bytes = false, rows = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: { + Expect(f.type, CompactType::List); + const uint64_t size = input.ReadListSize(CompactType::Struct); + out->columns.clear(); + out->columns.reserve(static_cast(size)); + for (uint64_t i = 0; i < size; ++i) { + input.SetColumnIndex(static_cast(i)); + DirectColumnChunk value; + ReadDirectColumnChunk(input, &value); + out->columns.emplace_back(std::move(value)); + } + input.SetColumnIndex(-1); + columns = true; + break; + } + case 2: + Expect(f.type, CompactType::I64); + out->total_byte_size = input.ReadI64(); + bytes = true; + break; + case 3: + Expect(f.type, CompactType::I64); + out->num_rows = input.ReadI64(); + rows = true; + break; + case 4: + Expect(f.type, CompactType::List); + out->sorting_columns = + ReadStructList(input, ReadSortingColumn); + out->has_sorting_columns = true; + break; + case 5: + Expect(f.type, CompactType::I64); + out->file_offset = input.ReadI64(); + out->has_file_offset = true; + break; + case 6: + Expect(f.type, CompactType::I64); + out->total_compressed_size = input.ReadI64(); + out->has_total_compressed_size = true; + break; + case 7: + Expect(f.type, CompactType::I16); + out->ordinal = input.ReadI16(); + out->has_ordinal = true; + break; + default: + input.Skip(f.type); + } + }); + if (!columns || !bytes || !rows) + throw ParquetException("Required direct RowGroup field missing"); +} + +} // namespace + +void ReadDirectFileMetaData(CompactInput& input, DirectFileMetaData* out) { + bool version = false, schema = false, rows = false, groups = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::I32); + out->version = input.ReadI32(); + version = true; + break; + case 2: + Expect(f.type, CompactType::List); + out->schema = + input.ReadList(CompactType::Struct, [&](CompactType) { + DirectSchemaElement value; + ReadDirectSchemaElement(input, &value); + return value; + }); + schema = true; + break; + case 3: + Expect(f.type, CompactType::I64); + out->num_rows = input.ReadI64(); + rows = true; + break; + case 4: + Expect(f.type, CompactType::List); + out->row_groups = + input.ReadList(CompactType::Struct, [&](CompactType) { + DirectRowGroup value; + ReadDirectRowGroup(input, &value); + return value; + }); + groups = true; + break; + case 5: + Expect(f.type, CompactType::List); + out->key_value_metadata = + ReadStructList(input, ReadDirectKeyValue); + out->has_key_value_metadata = true; + break; + case 6: + Expect(f.type, CompactType::Binary); + out->created_by = input.ReadBinaryView(); + out->has_created_by = true; + break; + case 7: + Expect(f.type, CompactType::List); + out->column_orders = ReadStructList(input, ReadColumnOrder); + out->has_column_orders = true; + break; + case 8: + Expect(f.type, CompactType::Struct); + ReadEncryptionAlgorithm(input, &out->encryption_algorithm); + out->has_encryption_algorithm = true; + break; + case 9: + Expect(f.type, CompactType::Binary); + out->footer_signing_key_metadata = input.ReadBinaryView(); + out->has_footer_signing_key_metadata = true; + break; + default: + input.Skip(f.type); + } + }); + if (!version || !schema || !rows || !groups) + throw ParquetException("Required direct FileMetaData field missing"); +} + +void ReadFastFileMetaData(CompactInput& input, FastMetadataSummary* out) { + bool version = false, schema = false, rows = false, groups = false; + input.ReadStruct([&](const CompactField& f) { + switch (f.id) { + case 1: + Expect(f.type, CompactType::I32); + out->version = input.ReadI32(); + version = true; + break; + case 2: { + Expect(f.type, CompactType::List); + const auto size = input.ReadListSize(CompactType::Struct); + out->num_schema_elements = static_cast(size); + for (uint64_t i = 0; i < size; ++i) { + bool has_type = false; + input.ReadStruct([&](const CompactField& schema_field) { + if (schema_field.id == 1) { + Expect(schema_field.type, CompactType::I32); + ReadEnum(input); + has_type = true; + } else { + input.Skip(schema_field.type); + } + }); + if (has_type) ++out->num_columns; + } + schema = true; + break; + } + case 3: + Expect(f.type, CompactType::I64); + out->num_rows = input.ReadI64(); + rows = true; + break; + case 4: { + Expect(f.type, CompactType::List); + const auto size = input.ReadListSize(CompactType::Struct); + out->num_row_groups = static_cast(size); + for (uint64_t i = 0; i < size; ++i) input.Skip(CompactType::Struct); + groups = true; + break; + } + case 6: + Expect(f.type, CompactType::Binary); + out->created_by = input.ReadBinaryView(); + out->has_created_by = true; + break; + default: + input.Skip(f.type); + } + }); + if (!version || !schema || !rows || !groups) + throw ParquetException("Required fast FileMetaData field missing"); +} + +} // namespace internal +} // namespace parquet diff --git a/cpp/src/parquet/thrift_compact_reader.h b/cpp/src/parquet/thrift_compact_reader.h new file mode 100644 index 000000000000..483a808651d8 --- /dev/null +++ b/cpp/src/parquet/thrift_compact_reader.h @@ -0,0 +1,318 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "generated/parquet_types.h" +#include "parquet/direct_metadata.h" +#include "parquet/exception.h" + +namespace parquet { +namespace internal { + +enum class CompactType : uint8_t { + Stop = 0, + BoolTrue = 1, + BoolFalse = 2, + Byte = 3, + I16 = 4, + I32 = 5, + I64 = 6, + Double = 7, + Binary = 8, + List = 9, + Set = 10, + Map = 11, + Struct = 12, + Uuid = 13, +}; + +struct CompactField { + CompactType type; + int16_t id; + bool bool_value; +}; + +class CompactInput { + public: + CompactInput(const uint8_t* data, int64_t length, bool skip_column_stats = false, + bool skip_encoding_stats = false, bool skip_size_stats = false, + const std::vector* statistics_columns = nullptr, + bool skip_page_stats = false, bool encoding_stats_as_mask = false) + : begin_(data), + current_(data), + end_(data == nullptr ? data : data + length), + skip_column_stats_(skip_column_stats), + skip_encoding_stats_(skip_encoding_stats), + skip_size_stats_(skip_size_stats), + statistics_columns_(statistics_columns), + skip_page_stats_(skip_page_stats), + encoding_stats_as_mask_(encoding_stats_as_mask) { + if (length < 0 || (length != 0 && data == nullptr)) { + throw ParquetException("Invalid Thrift compact buffer"); + } + } + + int64_t position() const { return current_ - begin_; } + bool skip_column_stats() const { return skip_column_stats_; } + bool skip_encoding_stats() const { return skip_encoding_stats_; } + bool skip_size_stats() const { return skip_size_stats_; } + bool skip_page_stats() const { return skip_page_stats_; } + bool encoding_stats_as_mask() const { return encoding_stats_as_mask_; } + void SetColumnIndex(int32_t index) { column_index_ = index; } + bool skip_for_column() const { + return statistics_columns_ != nullptr && + !std::binary_search(statistics_columns_->begin(), statistics_columns_->end(), + column_index_); + } + + uint64_t ReadListSize(CompactType expected) { + const uint8_t header = ReadByte(); + const auto actual = static_cast(header & 0x0f); + if (actual != expected && + !(expected == CompactType::BoolTrue && actual == CompactType::BoolFalse)) { + throw ParquetException("Unexpected Thrift compact list element type"); + } + uint64_t size = header >> 4; + if (size == 15) size = ReadVarint(); + if (size > static_cast(std::numeric_limits::max())) { + throw ParquetException("Thrift compact list is too large"); + } + return size; + } + + uint8_t ReadByte() { + if (current_ == end_) throw ParquetException("Truncated Thrift compact message"); + return *current_++; + } + + uint64_t ReadVarint() { + // Most compact-protocol integers fit in one byte. Keep the common path + // branch-light and avoid repeated ReadByte() calls; the slow path retains + // the same truncation and overflow checks for malformed input. + if (current_ == end_) throw ParquetException("Truncated Thrift compact message"); + uint8_t byte = *current_++; + if ((byte & 0x80) == 0) return byte; + uint64_t value = byte & 0x7f; + for (int shift = 7; shift < 64; shift += 7) { + if (current_ == end_) throw ParquetException("Truncated Thrift compact message"); + byte = *current_++; + if (shift == 63 && (byte & 0x7e) != 0) { + throw ParquetException("Thrift compact varint overflow"); + } + value |= static_cast(byte & 0x7f) << shift; + if ((byte & 0x80) == 0) return value; + } + throw ParquetException("Invalid Thrift compact varint"); + } + + int64_t ReadZigZag() { + const uint64_t value = ReadVarint(); + return static_cast((value >> 1) ^ + static_cast(-static_cast(value & 1))); + } + + int8_t ReadI8() { return static_cast(ReadByte()); } + bool ReadBoolElement() { + const uint8_t value = ReadByte(); + if (value == 1) return true; + if (value == 0 || value == 2) return false; + throw ParquetException("Invalid Thrift compact boolean"); + } + int16_t ReadI16() { return CheckedCast(ReadZigZag()); } + int32_t ReadI32() { return CheckedCast(ReadZigZag()); } + int64_t ReadI64() { return ReadZigZag(); } + + double ReadDouble() { + uint64_t bits = 0; + for (int i = 0; i < 8; ++i) bits |= static_cast(ReadByte()) << (8 * i); + double value; + std::memcpy(&value, &bits, sizeof(value)); + return value; + } + + std::string ReadBinary() { + const uint64_t size = ReadVarint(); + if (size > static_cast(end_ - current_)) { + throw ParquetException("Truncated Thrift compact binary"); + } + std::string value(reinterpret_cast(current_), static_cast(size)); + current_ += size; + return value; + } + + std::string_view ReadBinaryView() { + const uint64_t size = ReadVarint(); + if (size > static_cast(end_ - current_)) { + throw ParquetException("Truncated Thrift compact binary"); + } + const char* data = reinterpret_cast(current_); + current_ += size; + return std::string_view(data, static_cast(size)); + } + + CompactField ReadField(int16_t last_field_id) { + const uint8_t header = ReadByte(); + const auto type = static_cast(header & 0x0f); + if (type == CompactType::Stop) return {type, 0, false}; + const uint8_t delta = header >> 4; + const int16_t id = + delta == 0 ? ReadI16() : CheckedCast(last_field_id + delta); + return {type, id, type == CompactType::BoolTrue}; + } + + template + void ReadStruct(Fn&& fn) { + int16_t last_field_id = 0; + while (true) { + const CompactField field = ReadField(last_field_id); + if (field.type == CompactType::Stop) return; + fn(field); + last_field_id = field.id; + } + } + + template + std::vector ReadList(CompactType expected, Fn&& fn) { + const uint64_t size = ReadListSize(expected); + std::vector result; + result.reserve(static_cast(size)); + for (uint64_t i = 0; i < size; ++i) result.emplace_back(fn(expected)); + return result; + } + + void Skip(CompactType type, int depth = 64) { + if (depth == 0) throw ParquetException("Thrift compact nesting is too deep"); + switch (type) { + case CompactType::BoolTrue: + case CompactType::BoolFalse: + return; + case CompactType::Byte: + ReadByte(); + return; + case CompactType::I16: + case CompactType::I32: + case CompactType::I64: + ReadVarint(); + return; + case CompactType::Double: + for (int i = 0; i < 8; ++i) ReadByte(); + return; + case CompactType::Binary: + ReadBinary(); + return; + case CompactType::Struct: + ReadStruct([&](const CompactField& field) { Skip(field.type, depth - 1); }); + return; + case CompactType::List: + case CompactType::Set: { + const uint8_t header = ReadByte(); + const auto element_type = static_cast(header & 0x0f); + uint64_t size = header >> 4; + if (size == 15) size = ReadVarint(); + if (size > static_cast(std::numeric_limits::max())) { + throw ParquetException("Thrift compact list is too large"); + } + for (uint64_t i = 0; i < size; ++i) Skip(element_type, depth - 1); + return; + } + case CompactType::Map: { + const uint64_t size = ReadVarint(); + if (size == 0) return; + if (size > static_cast(std::numeric_limits::max())) { + throw ParquetException("Thrift compact map is too large"); + } + const uint8_t types = ReadByte(); + const auto key_type = static_cast(types >> 4); + const auto value_type = static_cast(types & 0x0f); + for (uint64_t i = 0; i < size; ++i) { + Skip(key_type, depth - 1); + Skip(value_type, depth - 1); + } + return; + } + case CompactType::Uuid: + for (int i = 0; i < 16; ++i) ReadByte(); + return; + default: + throw ParquetException("Unknown Thrift compact type"); + } + } + + void ReadEmptyStruct() { + if (ReadByte() != 0) throw ParquetException("Expected an empty Thrift struct"); + } + + private: + template + static T CheckedCast(int64_t value) { + if (value < std::numeric_limits::min() || value > std::numeric_limits::max()) { + throw ParquetException("Thrift compact integer overflow"); + } + return static_cast(value); + } + + const uint8_t* begin_; + const uint8_t* current_; + const uint8_t* end_; + bool skip_column_stats_ = false; + bool skip_encoding_stats_ = false; + bool skip_size_stats_ = false; + const std::vector* statistics_columns_ = nullptr; + int32_t column_index_ = -1; + bool skip_page_stats_ = false; + bool encoding_stats_as_mask_ = false; +}; + +inline void Expect(CompactType actual, CompactType expected) { + if (actual != expected && + !(expected == CompactType::BoolTrue && + (actual == CompactType::BoolFalse || actual == CompactType::BoolTrue))) { + throw ParquetException("Unexpected Thrift compact field type"); + } +} + +template +Enum ReadEnum(CompactInput& input) { + return static_cast(input.ReadI32()); +} + +void ReadSizeStatistics(CompactInput&, format::SizeStatistics*); +void ReadBoundingBox(CompactInput&, format::BoundingBox*); +void ReadGeospatialStatistics(CompactInput&, format::GeospatialStatistics*); +void ReadStatistics(CompactInput&, format::Statistics*); +void ReadLogicalType(CompactInput&, format::LogicalType*); +void ReadSchemaElement(CompactInput&, format::SchemaElement*); +void ReadKeyValue(CompactInput&, format::KeyValue*); +void ReadSortingColumn(CompactInput&, format::SortingColumn*); +void ReadPageEncodingStats(CompactInput&, format::PageEncodingStats*); +void ReadColumnMetaData(CompactInput&, format::ColumnMetaData*); +void ReadColumnCryptoMetaData(CompactInput&, format::ColumnCryptoMetaData*); +void ReadColumnChunk(CompactInput&, format::ColumnChunk*); +void ReadRowGroup(CompactInput&, format::RowGroup*); +void ReadColumnOrder(CompactInput&, format::ColumnOrder*); +void ReadEncryptionAlgorithm(CompactInput&, format::EncryptionAlgorithm*); +void ReadFileMetaData(CompactInput&, format::FileMetaData*); +void ReadPageLocation(CompactInput&, format::PageLocation*); +void ReadOffsetIndex(CompactInput&, format::OffsetIndex*); +void ReadColumnIndex(CompactInput&, format::ColumnIndex*); +void ReadDataPageHeader(CompactInput&, format::DataPageHeader*); +void ReadDataPageHeaderV2(CompactInput&, format::DataPageHeaderV2*); +void ReadDictionaryPageHeader(CompactInput&, format::DictionaryPageHeader*); +void ReadPageHeader(CompactInput&, format::PageHeader*); +void ReadBloomFilterHeader(CompactInput&, format::BloomFilterHeader*); +void ReadDirectFileMetaData(CompactInput&, DirectFileMetaData*); +void ReadFastFileMetaData(CompactInput&, FastMetadataSummary*); + +} // namespace internal +} // namespace parquet diff --git a/cpp/src/parquet/thrift_internal.h b/cpp/src/parquet/thrift_internal.h index 5a0fc7d70e69..91d5108da5c1 100644 --- a/cpp/src/parquet/thrift_internal.h +++ b/cpp/src/parquet/thrift_internal.h @@ -20,6 +20,7 @@ #include "parquet/windows_compatibility.h" #include +#include #include #include #include @@ -48,6 +49,7 @@ #include "parquet/schema.h" #include "parquet/size_statistics.h" #include "parquet/statistics.h" +#include "parquet/thrift_compact_reader.h" #include "parquet/types.h" #include "generated/parquet_types.h" // IWYU pragma: export @@ -580,11 +582,32 @@ class ThriftDeserializer { public: explicit ThriftDeserializer(const ReaderProperties& properties) : ThriftDeserializer(properties.thrift_string_size_limit(), - properties.thrift_container_size_limit()) {} - - ThriftDeserializer(int32_t string_size_limit, int32_t container_size_limit) + properties.thrift_container_size_limit(), + properties.thrift_skip_column_statistics(), + properties.thrift_skip_encoding_statistics(), + properties.thrift_skip_size_statistics(), + properties.thrift_statistics_columns_set(), + properties.thrift_statistics_columns(), + properties.thrift_skip_page_statistics(), + properties.thrift_encoding_stats_as_mask()) {} + + ThriftDeserializer(int32_t string_size_limit, int32_t container_size_limit, + bool skip_column_statistics = false, + bool skip_encoding_statistics = false, + bool skip_size_statistics = false, + bool statistics_columns_set = false, + std::vector statistics_columns = {}, + bool skip_page_statistics = false, + bool encoding_stats_as_mask = false) : string_size_limit_(string_size_limit), - container_size_limit_(container_size_limit) {} + container_size_limit_(container_size_limit), + skip_column_statistics_(skip_column_statistics), + skip_encoding_statistics_(skip_encoding_statistics), + skip_size_statistics_(skip_size_statistics), + statistics_columns_set_(statistics_columns_set), + statistics_columns_(std::move(statistics_columns)), + skip_page_statistics_(skip_page_statistics), + encoding_stats_as_mask_(encoding_stats_as_mask) {} // Deserialize a thrift message from buf/len. buf/len must at least contain // all the bytes needed to store the thrift message. @@ -643,24 +666,100 @@ class ThriftDeserializer { template int64_t DeserializeUnencryptedMessage(const uint8_t* buf, int64_t len, T* deserialized_msg) { - // Deserialize msg bytes into c++ thrift msg using memory transport. - auto tmem_transport = CreateReadOnlyMemoryBuffer(const_cast(buf), len); - auto tproto = apache::thrift::protocol::TCompactProtocolT( - tmem_transport, string_size_limit_, container_size_limit_); - try { - deserialized_msg - ->template read>( - &tproto); - } catch (std::exception& e) { - std::stringstream ss; - ss << "Couldn't deserialize thrift: " << e.what() << "\n"; - throw ParquetException(ss.str()); + // File metadata is parsed by the allocation-light compact reader. The + // generated thrift reader remains a compatibility fallback for malformed + // or newer metadata that the hand-written reader does not understand. + const char* generated_reader = std::getenv("PARQUET_USE_GENERATED_THRIFT"); + if constexpr (std::is_same_v) { + if (generated_reader == nullptr || generated_reader[0] != '1') { + try { + const bool skip_column_stats = + skip_column_statistics_ || + std::getenv("PARQUET_SKIP_COLUMN_STATS") != nullptr; + const bool skip_encoding_stats = + skip_encoding_statistics_ || + std::getenv("PARQUET_SKIP_ENCODING_STATS") != nullptr; + const bool skip_size_stats = + skip_size_statistics_ || std::getenv("PARQUET_SKIP_SIZE_STATS") != nullptr; + internal::CompactInput reader( + buf, len, skip_column_stats, skip_encoding_stats, skip_size_stats, + statistics_columns_set_ ? &statistics_columns_ : nullptr, + skip_page_statistics_, encoding_stats_as_mask_); + internal::ReadFileMetaData(reader, deserialized_msg); + return reader.position(); + } catch (const ParquetException&) { + // Preserve libthrift's established diagnostics for malformed input. + *deserialized_msg = T{}; + } + } + } + if constexpr (std::is_same_v || + std::is_same_v) { + try { + internal::CompactInput reader(buf, len); + if constexpr (std::is_same_v) { + internal::ReadColumnIndex(reader, deserialized_msg); + } else { + internal::ReadOffsetIndex(reader, deserialized_msg); + } + return reader.position(); + } catch (const ParquetException&) { + *deserialized_msg = T{}; + } + } + if constexpr (std::is_same_v) { + try { + internal::CompactInput reader(buf, len); + internal::ReadPageHeader(reader, deserialized_msg); + return reader.position(); + } catch (const ParquetException&) { + *deserialized_msg = T{}; + } + } + if constexpr (std::is_same_v) { + try { + internal::CompactInput reader(buf, len); + internal::ReadBloomFilterHeader(reader, deserialized_msg); + return reader.position(); + } catch (const ParquetException&) { + *deserialized_msg = T{}; + } + } + if constexpr (std::is_same_v) { + internal::CompactInput reader( + buf, len, skip_column_statistics_, skip_encoding_statistics_, + skip_size_statistics_, statistics_columns_set_ ? &statistics_columns_ : nullptr, + skip_page_statistics_, encoding_stats_as_mask_); + internal::ReadDirectFileMetaData(reader, deserialized_msg); + return reader.position(); + } + if constexpr (!std::is_same_v) { + // Deserialize msg bytes into c++ thrift msg using memory transport. + auto tmem_transport = CreateReadOnlyMemoryBuffer(const_cast(buf), len); + auto tproto = apache::thrift::protocol::TCompactProtocolT( + tmem_transport, string_size_limit_, container_size_limit_); + try { + deserialized_msg + ->template read>( + &tproto); + } catch (std::exception& e) { + std::stringstream ss; + ss << "Couldn't deserialize thrift: " << e.what() << "\n"; + throw ParquetException(ss.str()); + } + return len - static_cast(tmem_transport->available_read()); } - return len - static_cast(tmem_transport->available_read()); } const int32_t string_size_limit_; const int32_t container_size_limit_; + const bool skip_column_statistics_; + const bool skip_encoding_statistics_; + const bool skip_size_statistics_; + const bool statistics_columns_set_; + const std::vector statistics_columns_; + const bool skip_page_statistics_; + const bool encoding_stats_as_mask_; }; /// Utility class to serialize thrift objects to a binary format. This object